diff --git a/.dockerignore b/.dockerignore index 271d27a7a..eca6c8fe8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,6 +15,10 @@ build/ # at runtime — never baked into the image. Mirrored in .gitignore. secrets.env secrets.env.* +secrets.env~ +.secrets.env.swp +.secrets.env.swo +**/#secrets.env# !secrets.env.example /data/ /logs/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3784e65ae..787bd9dea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: name: Python syntax (compileall) runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -32,7 +32,7 @@ jobs: name: JS syntax (node --check) runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 @@ -54,7 +54,7 @@ jobs: # ROADMAP "fresh install smoke tests" item; make this required once green. continue-on-error: true steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml index 2551ee4f7..f1c4b5bfd 100644 --- a/.github/workflows/container-scan.yml +++ b/.github/workflows/container-scan.yml @@ -37,7 +37,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false diff --git a/.github/workflows/container-trivy.yml b/.github/workflows/container-trivy.yml index 999e8d96d..2a482f067 100644 --- a/.github/workflows/container-trivy.yml +++ b/.github/workflows/container-trivy.yml @@ -52,7 +52,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -93,7 +93,7 @@ jobs: security-events: write # upload SARIF to the Security tab steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index c6f3cf4ad..0a587de19 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -36,7 +36,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -55,7 +55,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5e822ab07..d52c0c4e8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -45,7 +45,7 @@ jobs: arch: arm64 runner: ubuntu-24.04-arm steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Buildx @@ -86,7 +86,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Read APP_VERSION + short sha diff --git a/.github/workflows/issue-description-check.yml b/.github/workflows/issue-description-check.yml index 3d0cf094e..52e9dddae 100644 --- a/.github/workflows/issue-description-check.yml +++ b/.github/workflows/issue-description-check.yml @@ -14,7 +14,7 @@ jobs: # Skip bots (Dependabot, release-drafter, etc.) if: ${{ github.event.issue.user.type != 'Bot' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: sparse-checkout: .github/scripts persist-credentials: false diff --git a/.github/workflows/pr-description-check.yml b/.github/workflows/pr-description-check.yml index c8fbe4b0f..53f0b5f50 100644 --- a/.github/workflows/pr-description-check.yml +++ b/.github/workflows/pr-description-check.yml @@ -23,7 +23,7 @@ jobs: # Skip bots: they open PRs programmatically and have their own process. if: github.event.pull_request.user.type != 'Bot' steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.base_ref }} sparse-checkout: .github/scripts diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index c270ef73b..02512204a 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -35,7 +35,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # Full history so a secret committed in an earlier commit (and later # deleted) is still caught -- deletion does not remove it from Git. diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml index f8b6fc804..ee345333b 100644 --- a/.github/workflows/workflow-security.yml +++ b/.github/workflows/workflow-security.yml @@ -36,7 +36,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -61,7 +61,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 174a4f2f6..efb38ed24 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,7 @@ Manual development uses Python 3.11+: python3 -m venv venv source venv/bin/activate pip install -r requirements.txt -python -m uvicorn app:app --host 0.0.0.0 --port 7000 +python -m uvicorn app:app --host 127.0.0.1 --port 7000 ``` Windows is not actively tested. Docker on Linux or a Linux/macOS manual install is the safer path for now. diff --git a/Odysseus.spec b/Odysseus.spec new file mode 100644 index 000000000..547460c69 --- /dev/null +++ b/Odysseus.spec @@ -0,0 +1,45 @@ +# -*- mode: python ; coding: utf-8 -*- + + +a = Analysis( + ['launcher.py'], + pathex=[], + binaries=[], + datas=[('static', 'static'), ('scripts', 'scripts'), ('mcp_servers', 'mcp_servers'), ('services/hwfit/data', 'services/hwfit/data'), ('config', 'config'), ('.env.example', '.env.example')], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, + optimize=0, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name='Odysseus', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=False, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, + icon=['static\\icon.ico'], +) +coll = COLLECT( + exe, + a.binaries, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='Odysseus', +) diff --git a/app.py b/app.py index 8d84a1940..57d091efd 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,7 @@ # app.py — slim orchestrator import mimetypes import os +import sys def register_static_mime_types() -> None: @@ -38,7 +39,7 @@ load_dotenv(encoding="utf-8-sig") import asyncio import logging import secrets -from datetime import datetime +from datetime import datetime, timezone from typing import Dict from contextlib import asynccontextmanager @@ -113,12 +114,13 @@ app = FastAPI( ) # ========= CORS ========= +CORS_ALLOW_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"] allowed_origins = os.getenv("ALLOWED_ORIGINS", "http://localhost,http://127.0.0.1").split(",") app.add_middleware( CORSMiddleware, allow_origins=allowed_origins, allow_credentials=True, - allow_methods=["GET", "POST", "PUT", "DELETE"], + allow_methods=CORS_ALLOW_METHODS, allow_headers=[ "Accept", "Authorization", @@ -316,7 +318,7 @@ if AUTH_ENABLED: # (no admin cookie available in that context). Restricted to # loopback clients + matching token to keep it locked down. try: - from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN as _ITT + from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN as _ITT, INTERNAL_TOOL_USER _hdr = request.headers.get(INTERNAL_TOOL_HEADER) if _hdr and secrets.compare_digest(_hdr, _ITT) and _is_trusted_loopback(request): # Impersonation: when the agent's loopback call sets @@ -328,11 +330,11 @@ if AUTH_ENABLED: if _impersonate and _impersonate in getattr(_auth_mgr, "users", {}): request.state.current_user = _impersonate else: - request.state.current_user = "internal-tool" + request.state.current_user = INTERNAL_TOOL_USER request.state.api_token = False return await call_next(request) - except Exception: - pass + except Exception as _e: + logger.warning("Internal tool auth header check failed", exc_info=_e) # Allow DIRECT localhost requests (internal service calls from # heartbeats etc.). Tunnel/proxy-forwarded requests are excluded by # _is_trusted_loopback so LOCALHOST_BYPASS can't be abused over a @@ -385,11 +387,10 @@ if AUTH_ENABLED: _db.close() try: await _asyncio.to_thread(_do) - except Exception: - pass + except Exception as _e: + logger.debug("Failed to update token last_used_at", exc_info=_e) _asyncio.create_task(_touch_last_used(matched_id)) # Keep bearer-token callers out of normal cookie/user - # routes. API-aware routes can read api_token_owner. request.state.current_user = "api" request.state.api_token = True request.state.api_token_id = matched_id @@ -438,7 +439,7 @@ class _RevalidatingStatic(StaticFiles): return resp -app.mount("/static", _RevalidatingStatic(directory="static"), name="static") +app.mount("/static", _RevalidatingStatic(directory=STATIC_DIR), name="static") # ========= GENERATED IMAGES ========= @app.get("/api/generated-image/{filename}") @@ -464,8 +465,8 @@ async def serve_generated_image(filename: str, request: Request): _db.close() except HTTPException: raise - except Exception: - pass + except Exception as _e: + logger.warning("Image ownership verification failed for %r", filename, exc_info=_e) ext = filename.rsplit('.', 1)[-1].lower() mime = { "png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", @@ -528,6 +529,7 @@ memory_vector = components.get("memory_vector") upload_handler = components["upload_handler"] app.state.upload_handler = upload_handler personal_docs_mgr = components["personal_docs_manager"] +app.state.personal_docs_manager = personal_docs_mgr api_key_manager = components["api_key_manager"] preset_manager = components["preset_manager"] chat_processor = components["chat_processor"] @@ -861,7 +863,7 @@ async def get_version(): @app.get("/api/health") async def health_check() -> Dict[str, str]: - return {"status": "healthy", "timestamp": datetime.utcnow().isoformat()} + return {"status": "healthy", "timestamp": datetime.now(timezone.utc).isoformat()} @app.get("/api/ready") async def readiness_check() -> JSONResponse: @@ -1171,3 +1173,12 @@ async def _shutdown_event(): except Exception as e: logger.warning(f"MCP shutdown error: {e}") logger.info("Application shutdown complete") + + +if __name__ == "__main__": + import uvicorn + + bind_host = os.getenv("APP_BIND", "127.0.0.1") + bind_port = int(os.getenv("APP_PORT", "7000")) + + uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") diff --git a/build-windows-portable.ps1 b/build-windows-portable.ps1 new file mode 100644 index 000000000..52f71a191 --- /dev/null +++ b/build-windows-portable.ps1 @@ -0,0 +1,72 @@ +#Requires -Version 5.1 +<# + Build a portable Windows distribution for Odysseus. + + Output layout: + dist\Odysseus\Odysseus.exe + dist\Odysseus\static\... + dist\Odysseus\scripts\... + dist\Odysseus\mcp_servers\... + dist\Odysseus\services\hwfit\data\... + + The app then keeps using its normal filesystem layout when frozen. + + Usage: + powershell -ExecutionPolicy Bypass -File .\build-windows-portable.ps1 +#> + +$ErrorActionPreference = "Stop" +Set-Location -Path $PSScriptRoot + +function Write-Step($msg) { Write-Host ""; Write-Host ("==> " + $msg) -ForegroundColor Cyan } +function Fail($msg) { + Write-Host "" + Write-Host ("ERROR: " + $msg) -ForegroundColor Red + exit 1 +} + +Write-Step "Checking for Python" +$pyExe = $null +if (Test-Path ".\.venv\Scripts\python.exe") { + $pyExe = (Resolve-Path ".\.venv\Scripts\python.exe").Path +} else { + foreach ($c in @("py", "python")) { + $cmd = Get-Command $c -ErrorAction SilentlyContinue + if ($cmd) { $pyExe = $cmd.Source; break } + } + if ($pyExe -like "*WindowsApps*python.exe") { + $pyCmd = Get-Command py -ErrorAction SilentlyContinue + if ($pyCmd) { + $pyExe = $pyCmd.Source + } + } +} +if (-not $pyExe) { + Fail "Python not found on PATH. Install Python 3.11+ first." +} +Write-Host ("Using Python: " + $pyExe) + +Write-Step "Installing build dependencies" +& $pyExe -m pip install --upgrade pip --quiet +& $pyExe -m pip install -r requirements.txt pyinstaller pystray Pillow +if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed." } + +Write-Step "Building portable exe bundle" +Remove-Item -Recurse -Force build, dist -ErrorAction SilentlyContinue + +$dataArgs = @( + "--add-data", "static;static", + "--add-data", "scripts;scripts", + "--add-data", "mcp_servers;mcp_servers", + "--add-data", "services/hwfit/data;services/hwfit/data", + "--add-data", "config;config", + "--add-data", ".env.example;.env.example" +) + +& $pyExe -m PyInstaller --noconfirm --clean --onedir --noconsole --icon=static/icon.ico --name Odysseus @dataArgs launcher.py +if ($LASTEXITCODE -ne 0) { Fail "PyInstaller build failed." } + +Write-Host "" +Write-Host "Build complete." -ForegroundColor Green +Write-Host "Portable app folder: $PSScriptRoot\dist\Odysseus" -ForegroundColor Green +Write-Host "Distribute the whole folder (or zip it) so static assets and scripts stay with the exe." -ForegroundColor Green \ No newline at end of file diff --git a/companion/routes.py b/companion/routes.py index 9c8464f0f..0191640ef 100644 --- a/companion/routes.py +++ b/companion/routes.py @@ -5,8 +5,9 @@ offers and pair to it, without duplicating any LLM logic. Auth is enforced globally by AuthMiddleware (app.py), so reaching a handler here means the caller is authenticated by either a cookie session or a Bearer `ody_` -API token. The read endpoints (ping/info/models) accept either; the pairing -endpoints are admin-cookie only. +API token. Ping/info accept either credential type, models requires a chat- +scoped API token for bearer callers, and the pairing endpoints are admin-cookie +only. Pairing CSRF posture: minting happens ONLY on POST. The session cookie is SameSite=Lax (routes/auth_routes.py), which a browser does not send on a @@ -18,7 +19,7 @@ on a GET would be unsafe (Lax cookies ride top-level GET navigations), so GET import html -from fastapi import APIRouter, Request +from fastapi import APIRouter, HTTPException, Request from fastapi.responses import HTMLResponse from core.middleware import require_admin @@ -52,6 +53,18 @@ def owner_can_see(row_owner, owner) -> bool: return row_owner is None or row_owner == owner +def require_models_scope(request: Request) -> None: + """Require the companion chat scope for bearer-token model inventory.""" + if not getattr(request.state, "api_token", False): + return + scopes = getattr(request.state, "api_token_scopes", None) or [] + if isinstance(scopes, str): + scopes = [scope.strip() for scope in scopes.split(",")] + scope_set = {str(scope).strip() for scope in scopes if str(scope).strip()} + if _pairing.COMPANION_SCOPE not in scope_set: + raise HTTPException(403, "API token requires chat scope") + + def mint_pairing_token(owner: str, invalidate=None) -> tuple[str, str]: """Mint a pairing token AND invalidate the auth middleware's in-memory token cache, so the new token is accepted on the very next request without a server @@ -103,6 +116,7 @@ def setup_companion_routes() -> APIRouter: rows -- the same rule as owner_filter. Read-only; never returns api_key material. """ + require_models_scope(request) import json as _json from core.database import SessionLocal, ModelEndpoint diff --git a/core/auth.py b/core/auth.py index 7f085c065..3bdf0f390 100644 --- a/core/auth.py +++ b/core/auth.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) from core.atomic_io import atomic_write_json as _atomic_write_json # noqa: E402 +from core.middleware import INTERNAL_TOOL_USER # noqa: E402 DEFAULT_PRIVILEGES = { "can_use_agent": True, @@ -47,7 +48,7 @@ ADMIN_PRIVILEGES["allowed_models_restricted"] = False # backwards for this sentinel. ADMIN_PRIVILEGES["block_all_models"] = False -from src.constants import AUTH_FILE +from src.constants import AUTH_FILE, PASSWORD_MIN_LENGTH DEFAULT_AUTH_PATH = AUTH_FILE TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days @@ -65,7 +66,7 @@ TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days # of those names would be denied an assistant and inconsistently owner-scoped. # Refuse to create or rename into any of them so the sentinels can't be # impersonated. (Keep this in sync with that synthetic-owner set.) -RESERVED_USERNAMES = frozenset({"internal-tool", "api", "demo", "system"}) +RESERVED_USERNAMES = frozenset({INTERNAL_TOOL_USER, "api", "demo", "system"}) def normalize_known_username(users: Dict[str, Any], username: str | None) -> Optional[str]: @@ -243,6 +244,15 @@ class AuthManager: def is_configured(self) -> bool: return len(self.users) > 0 + def policy(self) -> dict: + """Return public auth policy constants for the frontend.""" + return { + "password_min_length": PASSWORD_MIN_LENGTH, + "reserved_usernames": sorted(RESERVED_USERNAMES), + "signup_enabled": self.signup_enabled, + "session_days": TOKEN_TTL // 86400, + } + # ------------------------------------------------------------------ # Account management # ------------------------------------------------------------------ @@ -573,16 +583,20 @@ class AuthManager: return None return self.create_session_trusted(username) - def create_session_trusted(self, username: str) -> str: + def create_session_trusted(self, username: str) -> Optional[str]: """Issue a session token for an already-verified user. Call only after verify_password (and TOTP if enabled) have passed.""" username = username.strip().lower() token = secrets.token_hex(32) - with self._sessions_lock: - self._sessions[token] = { - "username": username, - "expiry": time.time() + TOKEN_TTL, - } + with self._config_lock: + if username not in self.users: + logger.warning("Refused to issue session for missing user '%s'", username) + return None + with self._sessions_lock: + self._sessions[token] = { + "username": username, + "expiry": time.time() + TOKEN_TTL, + } self._save_sessions() return token diff --git a/core/database.py b/core/database.py index e4acc8d54..0f1089b39 100644 --- a/core/database.py +++ b/core/database.py @@ -2,12 +2,15 @@ import os import logging import sqlite3 from datetime import datetime, timezone +from pathlib import Path from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text from sqlalchemy.engine import Engine from sqlalchemy.types import TypeDecorator from sqlalchemy.ext.declarative import declarative_base, declared_attr from sqlalchemy.orm import relationship, sessionmaker, backref +from src.runtime_paths import get_app_root + logger = logging.getLogger(__name__) # Create base class for declarative models @@ -29,9 +32,26 @@ class TimestampMixin: def updated_at(cls): return Column(DateTime, default=utcnow_naive, onupdate=utcnow_naive, nullable=False) -# Get database URL from environment, default to SQLite in DATA_DIR +# Ensure the writable data directory exists before SQLite connects. from src.constants import DATA_DIR, AUTH_FILE, MEMORY_FILE, USER_PREFS_FILE, SETTINGS_FILE -DATABASE_URL = os.getenv("DATABASE_URL", f"sqlite:///{DATA_DIR}/app.db") +Path(DATA_DIR).mkdir(parents=True, exist_ok=True) + + +def _default_database_url() -> str: + return f"sqlite:///{Path(DATA_DIR) / 'app.db'}" + + +def _normalize_sqlite_url(url: str) -> str: + if not url.startswith("sqlite:///"): + return url + db_path = url.replace("sqlite:///", "", 1) + if db_path == ":memory:" or os.path.isabs(db_path): + return url + return f"sqlite:///{(Path(get_app_root()) / db_path).resolve().as_posix()}" + + +# Get database URL from environment, default to SQLite in DATA_DIR +DATABASE_URL = _normalize_sqlite_url(os.getenv("DATABASE_URL", _default_database_url())) # Create engine engine = create_engine( @@ -324,6 +344,13 @@ class EmailAccount(TimestampMixin, Base): smtp_password = Column(String, default="") from_address = Column(String, default="") + display_name = Column(String, nullable=True) # "Hriday Ranka" — used in From: header + + # OAuth2 (Google / Google Workspace). Tokens stored encrypted via secret_storage. + oauth_provider = Column(String, nullable=True) # "google" or None + oauth_access_token = Column(String, nullable=True) # encrypted + oauth_refresh_token = Column(String, nullable=True) # encrypted + oauth_token_expiry = Column(String, nullable=True) # unix timestamp string __table_args__ = ( Index('ix_email_accounts_owner_default', 'owner', 'is_default'), @@ -1427,6 +1454,25 @@ def _migrate_add_task_automation_columns(): except Exception as e: logging.getLogger(__name__).warning(f"task automation migration: {e}") +def _migrate_add_email_oauth_columns(): + """Add Google OAuth and display_name columns to email_accounts if missing.""" + try: + with engine.connect() as conn: + cols = [r[1] for r in conn.execute(text("PRAGMA table_info(email_accounts)"))] + for col, typedef in [ + ("oauth_provider", "TEXT"), + ("oauth_access_token", "TEXT"), + ("oauth_refresh_token", "TEXT"), + ("oauth_token_expiry", "TEXT"), + ("display_name", "TEXT"), + ]: + if col not in cols: + conn.execute(text(f"ALTER TABLE email_accounts ADD COLUMN {col} {typedef}")) + conn.commit() + except Exception as e: + logging.getLogger(__name__).warning(f"email oauth columns migration: {e}") + + def _migrate_add_oauth_config(): """Add oauth_config column to mcp_servers table if missing.""" try: @@ -1771,6 +1817,7 @@ def init_db(): _migrate_add_tidy_verdict() _migrate_add_doc_source_email_cols() _migrate_add_oauth_config() + _migrate_add_email_oauth_columns() _migrate_add_task_automation_columns() _migrate_add_disabled_tools() _migrate_add_mcp_oauth_tokens_column() diff --git a/core/middleware.py b/core/middleware.py index 550ee3bd7..fe1f30110 100644 --- a/core/middleware.py +++ b/core/middleware.py @@ -15,6 +15,8 @@ from starlette.responses import Response # same value from this module. Never persisted or exposed externally. INTERNAL_TOOL_TOKEN = os.environ.get("ODYSSEUS_INTERNAL_TOKEN") or secrets.token_hex(32) INTERNAL_TOOL_HEADER = "X-Odysseus-Internal-Token" +# Pseudo-username on in-process tool-loopback requests; require_admin trusts it and it is reserved. +INTERNAL_TOOL_USER = "internal-tool" def is_cors_preflight(method: str, headers) -> bool: @@ -39,7 +41,7 @@ def require_admin(request: Request): hdr = request.headers.get(INTERNAL_TOOL_HEADER) if hdr and secrets.compare_digest(hdr, INTERNAL_TOOL_TOKEN): return - if getattr(request.state, "current_user", None) == "internal-tool": + if getattr(request.state, "current_user", None) == INTERNAL_TOOL_USER: return except Exception: pass @@ -65,10 +67,9 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware): response = await call_next(request) path = request.url.path - # Tool render endpoints are served inside iframes — allow framing by self + # Tool render endpoints is_tool_render = path.startswith("/api/tools/") and path.endswith("/render") - # PDF previews are embedded by the in-app document library. Keep the - # exception route-scoped so normal app pages remain unframeable. + # Document library PDF preview endpoint is_document_pdf_preview = path.startswith("/api/document/") and path.endswith("/render-pdf") # Visual report pages are self-contained HTML — need inline scripts + external images is_report = path.startswith("/api/research/report/") @@ -95,9 +96,7 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware): "frame-ancestors 'none'" ) elif is_tool_render: - # Tool iframe content: skip all framing headers — the iframe's - # sandbox="allow-scripts" attribute provides isolation. - # Don't overwrite the route's own restrictive CSP either. + # Skip framing headers for tools. pass elif is_document_pdf_preview: response.headers["X-Frame-Options"] = "SAMEORIGIN" diff --git a/docker-compose.gpu-amd.yml b/docker-compose.gpu-amd.yml index c823e0698..82e22e440 100644 --- a/docker-compose.gpu-amd.yml +++ b/docker-compose.gpu-amd.yml @@ -60,6 +60,13 @@ services: - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1} - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost} - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760} + - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600} + - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760} + - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} + - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker-compose.gpu-nvidia.yml b/docker-compose.gpu-nvidia.yml index 7766dd0ed..1b551c669 100644 --- a/docker-compose.gpu-nvidia.yml +++ b/docker-compose.gpu-nvidia.yml @@ -59,6 +59,13 @@ services: - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1} - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost} - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760} + - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600} + - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760} + - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} + - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker-compose.yml b/docker-compose.yml index dd708303f..77840e22b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,6 +58,13 @@ services: - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1} - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost} - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760} + - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600} + - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760} + - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} + - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7d796d9ff..fc0e87a08 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -13,6 +13,8 @@ set -e PUID="${PUID:-1000}" PGID="${PGID:-1000}" +GOSU_BIN="$(command -v gosu)" +PYTHON_BIN="$(command -v python)" # Reuse an existing matching group/user if the host's UID/GID already # corresponds to one in /etc/passwd (e.g. when the image is rebuilt @@ -24,51 +26,78 @@ if ! getent passwd "$PUID" >/dev/null 2>&1; then useradd -u "$PUID" -g "$PGID" -M -s /bin/sh -d /app odysseus fi -# Docker-socket group plumbing. When /var/run/docker.sock is bind- -# mounted (cookbook uses `docker exec` to reach sibling containers -# like ollama-rocm), the socket is owned by root:docker on the host. -# We need the in-container odysseus user to be in the matching group -# so `gosu PUID:PGID` doesn't strip it. compose's `group_add` only -# applies to the initial root process — gosu drop resets supplementary -# groups — so detect the socket's GID here and add the user via the -# username form `gosu odysseus` below. +ODY_USER="$(getent passwd "$PUID" | cut -d: -f1)" +[ -z "$ODY_USER" ] && ODY_USER=odysseus + +# Docker-socket group plumbing. When /var/run/docker.sock is bind-mounted +# (Cookbook uses docker exec to reach sibling containers), the socket is +# owned by root:. Add the app user to that group and later +# call gosu by username so supplementary groups are retained. DOCKER_SOCK="${DOCKER_SOCK:-/var/run/docker.sock}" if [ -S "$DOCKER_SOCK" ]; then SOCK_GID="$(stat -c '%g' "$DOCKER_SOCK" 2>/dev/null || echo '')" if [ -n "$SOCK_GID" ] && [ "$SOCK_GID" != "0" ]; then - # Create the group locally if missing, then add odysseus to it. if ! getent group "$SOCK_GID" >/dev/null 2>&1; then groupadd -g "$SOCK_GID" docker_host || true fi SOCK_GROUP="$(getent group "$SOCK_GID" | cut -d: -f1)" if [ -n "$SOCK_GROUP" ]; then - ODY_USER="$(getent passwd "$PUID" | cut -d: -f1)" - [ -z "$ODY_USER" ] && ODY_USER=odysseus usermod -aG "$SOCK_GROUP" "$ODY_USER" 2>/dev/null || true fi fi fi -# Repair ownership on every writable path the app touches at runtime. -# -# Bind-mounted dirs (/app/data, /app/logs) are the obvious ones, but -# the app ALSO writes inside the image's own source tree at runtime: -# - services/cache/{search,content}/* (search cache LRU) -# - services/search_analytics.json -# - services/search_engine_error.log -# - services/tts cache, etc. -# These dirs were created as root during `docker build`, so dropping -# to PUID:PGID would otherwise crash on the first import that tries -# to mkdir them. Chown the whole /app tree — fast (<1s on this size) -# and idempotent via the `-not -uid` filter so we only touch files -# that need fixing. -for dir in /app /app/data /app/logs; do +mount_root_for() { + awk -v target="$1" '$5 == target { print $4; exit }' /proc/self/mountinfo 2>/dev/null || true +} + +is_broad_mount_root() { + case "$1" in + /|/home|/srv|/var|/usr|/opt|/tmp|/mnt|/media) + return 0 + ;; + esac + return 1 +} + +repair_tree_ownership() { + dir="$1" if [ -d "$dir" ]; then - # `find ... -not -uid` keeps this O(touched-files), not - # O(everything), so terabyte-sized maildirs don't slow startup. - find "$dir" -not -uid "$PUID" -print0 2>/dev/null \ + find "$dir" -xdev -not -uid "$PUID" -print0 2>/dev/null \ | xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true fi +} + +repair_app_tree_ownership() { + if [ -d /app ]; then + find /app -xdev \ + \( -path /app/data -o -path /app/logs -o -path /app/.ssh -o -path /app/.cache -o -path /app/.local \) -prune \ + -o -not -uid "$PUID" -print0 2>/dev/null \ + | xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true + fi +} + +repair_bind_mount_ownership() { + dir="$1" + if [ ! -d "$dir" ]; then + return + fi + + mount_root="$(mount_root_for "$dir")" + if is_broad_mount_root "$mount_root"; then + echo "Skipping recursive ownership repair for $dir because it maps to broad host path $mount_root" >&2 + chown "$PUID:$PGID" "$dir" 2>/dev/null || true + return + fi + + repair_tree_ownership "$dir" +} + +# Repair image-owned writable paths without walking into bind-mounted host +# trees, then repair the app-owned mount roots separately. +repair_app_tree_ownership +for dir in /app/data /app/logs /app/.ssh /app/.cache/huggingface /app/.local; do + repair_bind_mount_ownership "$dir" done # Cookbook installs vllm/etc. via `pip install --user`, which pulls @@ -95,6 +124,7 @@ for cu in \ break fi done + # Disable the FlashInfer JIT sampler unconditionally — it is sampler-only # and has no impact on the attention path, but requires nvcc + matching # CUDA headers at startup. Without this, vLLM crashes with "Could not find @@ -108,13 +138,9 @@ export PATH="/app/.local/bin:$PATH" # Run first-time setup as the app user so data/ files get the right ownership. # setup.py is idempotent — skips auth.json / .env if they already exist. # || true so a setup failure never prevents the container from starting. -# Use the username form (no :GID) so supplementary groups from /etc/group -# (including the docker-socket group set above) flow through to the child. -ODY_USER="$(getent passwd "$PUID" | cut -d: -f1)" -[ -z "$ODY_USER" ] && ODY_USER="$PUID:$PGID" -gosu "$ODY_USER" python /app/setup.py || true +"$GOSU_BIN" "$ODY_USER" "$PYTHON_BIN" /app/setup.py || true # Drop root and run the actual app. `gosu` is preferred over `su` / # `sudo` because it cleans up the process tree (no extra shell layer) # so signals (SIGTERM from `docker stop`) reach uvicorn directly. -exec gosu "$ODY_USER" "$@" +exec "$GOSU_BIN" "$ODY_USER" "$@" diff --git a/docs/chat.gif b/docs/chat.gif deleted file mode 100644 index 90ca0eaac..000000000 Binary files a/docs/chat.gif and /dev/null differ diff --git a/docs/compare.gif b/docs/compare.gif deleted file mode 100644 index 7b939aa01..000000000 Binary files a/docs/compare.gif and /dev/null differ diff --git a/docs/document.gif b/docs/document.gif deleted file mode 100644 index b2a89e435..000000000 Binary files a/docs/document.gif and /dev/null differ diff --git a/docs/notes.gif b/docs/notes.gif deleted file mode 100644 index 891ec2e1b..000000000 Binary files a/docs/notes.gif and /dev/null differ diff --git a/docs/research.gif b/docs/research.gif deleted file mode 100644 index b817eeb1a..000000000 Binary files a/docs/research.gif and /dev/null differ diff --git a/docs/security-ci.md b/docs/security-ci.md index c25838f72..f21643de5 100644 --- a/docs/security-ci.md +++ b/docs/security-ci.md @@ -1,14 +1,16 @@ # Security CI guide -This project runs a set of automated security checks on every pull request and -on every push to `main`. This page explains what each one does, whether it can +This project runs a set of automated security checks on pull requests and +selected branch pushes. This page explains what each one does, whether it can block a merge, and the few one-time settings you should turn on to get the full benefit. ## What runs, and why -Each check lives in its own file under `.github/workflows/`. They run -automatically; you do not start them. +Most checks live in files under `.github/workflows/`. CodeQL is configured +through GitHub's code scanning default setup, so it appears as a dynamic GitHub +workflow instead of a checked-in workflow file. They run automatically; you do +not start them. | Check | What it protects against | Blocks a merge? | |---|---|---| @@ -88,11 +90,14 @@ let the workflows run on one pull request first, then add them here. 2. Turn on **Dependency graph** (usually on by default for public repos) -- this powers Dependency review and Dependabot. 3. Turn on **Dependabot alerts** and **Dependabot security updates**. -4. Under **Code scanning**, you have two ways to scan the app code with CodeQL: - - The included `codeql.yml` workflow already scans `main` and runs weekly. - - To also scan **pull requests** (recommended, since most contributions come - from forks), click **Set up -> Default** under Code scanning. GitHub then - runs CodeQL on pull requests for you, with no token limitations. +4. Under **Code scanning**, use **Set up -> Default** for CodeQL. GitHub then + runs CodeQL as a dynamic workflow without the fork-token limitations that + affect checked-in advanced workflows. + + Do not also add a checked-in CodeQL workflow while default setup is enabled: + GitHub rejects advanced CodeQL uploads when default setup is active. If the + project later needs an advanced CodeQL workflow, disable default setup first + and keep only one CodeQL publishing path active. ## Keeping it current diff --git a/launch-windows.ps1 b/launch-windows.ps1 index 16938c195..263d95127 100644 --- a/launch-windows.ps1 +++ b/launch-windows.ps1 @@ -105,6 +105,14 @@ if (-not $pyExe) { } } +if ($pyExe -like "*WindowsApps*python.exe") { + $pyCmd = Get-Command py -ErrorAction SilentlyContinue + if ($pyCmd) { + $pyExe = $pyCmd.Source + $pyArgs = @("-3.11") + } +} + if (-not $pyExe) { Fail "Couldn't find Python 3.11+ for Windows setup. Install Python 3.11+ (or open the Python launcher with 'py -3.11') from https://www.python.org/downloads/, then re-run this script." } diff --git a/launcher.py b/launcher.py new file mode 100644 index 000000000..ba158444f --- /dev/null +++ b/launcher.py @@ -0,0 +1,142 @@ +# launcher.py +"""Dedicated entrypoint for the standalone Windows portable launcher. + +Handles: +- Immediate GUI splash screen creation using tkinter. +- Suppressing console stream crashes in windowed GUI mode via NullWriter. +- Spawning system tray icon via pystray and Pillow (lazy-loaded). +- Auto-opening default browser pointing to the running backend. +- Launching the FastAPI server (importing and running app.py). +""" +import os +import sys +import threading +import time +import webbrowser + +# Define a dummy NullWriter to suppress standard stream crashes (isatty etc.) in GUI mode +class NullWriter: + def write(self, text): + pass + def flush(self): + pass + def isatty(self): + return False + +if sys.stdout is None: + sys.stdout = NullWriter() +if sys.stderr is None: + sys.stderr = NullWriter() + + +splash_root = None + +# If running from a frozen PyInstaller bundle, launch the splash screen IMMEDIATELY +if getattr(sys, 'frozen', False): + import tkinter as tk + + def show_splash_instantly(): + global splash_root + try: + splash_root = tk.Tk() + splash_root.title("Odysseus") + splash_root.overrideredirect(True) + splash_root.configure(bg="#1a1c23") + + # Accented borders + splash_root.config(highlightbackground="#e06c75", highlightcolor="#e06c75", highlightthickness=1) + + w, h = 360, 160 + ws = splash_root.winfo_screenwidth() + hs = splash_root.winfo_screenheight() + x = (ws - w) // 2 + y = (hs - h) // 2 + splash_root.geometry(f"{w}x{h}+{x}+{y}") + + tk.Label(splash_root, text="⛵ Odysseus", font=("Segoe UI", 22, "bold"), bg="#1a1c23", fg="#e06c75").pack(pady=(22, 2)) + tk.Label(splash_root, text="Launching background services...", font=("Segoe UI", 10), bg="#1a1c23", fg="#d1d4e0").pack(pady=2) + tk.Label(splash_root, text="Please wait, this will take a few seconds.", font=("Segoe UI", 8, "italic"), bg="#1a1c23", fg="#5c6370").pack(pady=(12, 0)) + + splash_root.attributes("-topmost", True) + splash_root.mainloop() + except Exception: + pass + + # Launch the GUI splash screen immediately on a background thread + threading.Thread(target=show_splash_instantly, daemon=True).start() + + +def create_tray_image(): + # Generate a beautiful 64x64 icon matching Odysseus brand red accent (#e06c75) + from PIL import Image, ImageDraw + image = Image.new('RGBA', (64, 64), (0, 0, 0, 0)) + dc = ImageDraw.Draw(image) + accent_red = (224, 108, 117, 255) + light_red = (224, 108, 117, 150) + + # Draw premium sailing boat + dc.polygon([(32, 10), (32, 45), (12, 45)], fill=accent_red) + dc.polygon([(32, 18), (32, 45), (48, 45)], fill=light_red) + dc.polygon([(8, 48), (56, 48), (44, 56), (20, 56)], fill=accent_red) + return image + + +def on_open_browser(icon, item, url): + webbrowser.open(url) + + +def on_exit(icon, item): + icon.stop() + os._exit(0) + + +def setup_system_tray(url): + try: + import pystray + icon_img = create_tray_image() + menu = ( + pystray.MenuItem('Open Odysseus', lambda icon, item: on_open_browser(icon, item, url), default=True), + pystray.MenuItem('Exit', on_exit) + ) + tray_icon = pystray.Icon( + "Odysseus", + icon_img, + "Odysseus", + menu + ) + tray_icon.run() + except Exception: + pass + + +def open_browser(url): + # Allow uvicorn and app lifecycles to complete warmups + time.sleep(3.5) + + # Safely close the splash screen + try: + global splash_root + if splash_root: + splash_root.after(0, splash_root.destroy) + except Exception: + pass + + webbrowser.open(url) + + +if __name__ == "__main__": + import uvicorn + # Import the FastAPI app from app.py + from app import app + + bind_host = os.getenv("APP_BIND", "127.0.0.1") + bind_port = int(os.getenv("APP_PORT", "7000")) + url = f"http://{bind_host}:{bind_port}" + + if getattr(sys, 'frozen', False): + # Start browser manager thread + threading.Thread(target=open_browser, args=(url,), daemon=True).start() + # Start system tray manager thread + threading.Thread(target=setup_system_tray, args=(url,), daemon=True).start() + + uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") diff --git a/mcp_servers/email_server.py b/mcp_servers/email_server.py index 305225db0..2611491ae 100644 --- a/mcp_servers/email_server.py +++ b/mcp_servers/email_server.py @@ -23,6 +23,7 @@ import os.path from pathlib import Path from datetime import datetime, timedelta import uuid +from contextvars import ContextVar from mcp.server import Server from mcp.server.stdio import stdio_server @@ -55,6 +56,8 @@ def _uid_fetch_rows(data) -> list: # flat keys when no DB row matches (legacy single-account behaviour). _ACCOUNT_CACHE: dict = {} # key = normalized account selector -> config dict +_MCP_OWNER_ARG = "_odysseus_owner" +_CURRENT_OWNER: ContextVar[str | None] = ContextVar("email_mcp_owner", default=None) def _clean_header_value(value) -> str: @@ -68,6 +71,45 @@ def _db_path() -> Path: return Path(APP_DB) +def _current_owner() -> str: + owner = _CURRENT_OWNER.get() + return str(owner or "").strip() + + +def _account_visible_to_owner(row: dict, owner: str) -> bool: + row_owner = str(row.get("owner") or "").strip() + if row_owner == owner: + return True + if row_owner: + return False + # Legacy ownerless accounts are only visible to a scoped caller when the + # mailbox itself matches the owner, mirroring the HTTP email route fallback. + owner_l = owner.lower() + return owner_l in { + str(row.get("imap_user") or "").strip().lower(), + str(row.get("from_address") or "").strip().lower(), + } + + +def _filter_accounts_for_owner(rows: list[dict]) -> list[dict]: + owner = _current_owner() + if owner: + return [r for r in rows if _account_visible_to_owner(r, owner)] + + owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()} + if len(owners) > 1: + return [] + return rows + + +def _mcp_owner_required(rows: list[dict] | None = None) -> bool: + if _current_owner(): + return False + rows = rows if rows is not None else _read_accounts_from_db() + owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()} + return len(owners) > 1 + + def _load_email_writing_style() -> str: """Return the existing Settings > Email > Writing Style value.""" try: @@ -121,9 +163,8 @@ def _default_document_owner() -> str | None: return None -def _list_accounts_raw() -> list: - """Return list of dicts from the email_accounts table. Empty list if table - missing or empty. Never raises.""" +def _read_accounts_from_db() -> list: + """Return all enabled email account rows. Empty list if missing. Never raises.""" path = _db_path() if not path.exists(): return [] @@ -131,9 +172,10 @@ def _list_accounts_raw() -> list: conn = sqlite3.connect(str(path)) conn.row_factory = sqlite3.Row columns = {r[1] for r in conn.execute("PRAGMA table_info(email_accounts)").fetchall()} + owner_select = "owner" if "owner" in columns else "NULL AS owner" smtp_security_select = "smtp_security" if "smtp_security" in columns else "'' AS smtp_security" rows = conn.execute(f""" - SELECT id, name, is_default, enabled, + SELECT id, {owner_select}, name, is_default, enabled, imap_host, imap_port, imap_user, imap_password, imap_starttls, smtp_host, smtp_port, {smtp_security_select}, smtp_user, smtp_password, from_address FROM email_accounts WHERE enabled = 1 @@ -147,11 +189,15 @@ def _list_accounts_raw() -> list: return [] -def _resolve_account(selector: str | None) -> dict | None: +def _list_accounts_raw() -> list: + """Return owner-visible email account rows for the active MCP call.""" + return _filter_accounts_for_owner(_read_accounts_from_db()) + + +def _resolve_account_from_rows(rows: list[dict], selector: str | None) -> dict | None: """Given a selector (None = default, or a name/user/id string), return the matching row or None. Matching is case-insensitive substring on name + imap_user + from_address, plus exact id match.""" - rows = _list_accounts_raw() if not rows: return None if not selector: @@ -186,6 +232,10 @@ def _resolve_account(selector: str | None) -> dict | None: return None +def _resolve_account(selector: str | None) -> dict | None: + return _resolve_account_from_rows(_list_accounts_raw(), selector) + + def _load_config(account: str | None = None) -> dict: """Return the full config dict for the requested account (or default). @@ -194,7 +244,7 @@ def _load_config(account: str | None = None) -> dict: 2. env vars + settings.json flat keys (legacy) 3. hardcoded fallbacks (localhost:31143 etc.) """ - cache_key = (account or "").strip().lower() or "__default__" + cache_key = (_current_owner(), (account or "").strip().lower() or "__default__") if cache_key in _ACCOUNT_CACHE: return _ACCOUNT_CACHE[cache_key] @@ -223,8 +273,11 @@ def _load_config(account: str | None = None) -> dict: "account_name": None, } - rows = _list_accounts_raw() - row = _resolve_account(account) + raw_rows = _read_accounts_from_db() + rows = _filter_accounts_for_owner(raw_rows) + row = _resolve_account_from_rows(rows, account) + if _current_owner() and raw_rows and not rows: + raise ValueError("No email account is configured for the authenticated owner") if account and rows and not row: available = ", ".join( f"{r.get('name') or r.get('imap_user')} <{r.get('imap_user') or r.get('from_address') or '?'}>" @@ -953,7 +1006,7 @@ def _stash_agent_draft(*, to, subject, body, in_reply_to=None, references=None, now, account or None, "agent_draft", - "", + _current_owner(), )) conn.commit() conn.close() @@ -1139,7 +1192,7 @@ def _create_email_draft_document( doc_id = str(uuid.uuid4()) ver_id = str(uuid.uuid4()) doc_title = (title or subject or "Email draft").strip() or "Email draft" - doc_owner = _default_document_owner() + doc_owner = _current_owner() or _default_document_owner() db = SessionLocal() try: @@ -1925,10 +1978,22 @@ async def list_tools() -> list[Tool]: @server.call_tool() async def call_tool(name: str, arguments: dict) -> list[TextContent]: + arguments = dict(arguments) if isinstance(arguments, dict) else {} + owner = str(arguments.pop(_MCP_OWNER_ARG, "") or "").strip() + owner_token = _CURRENT_OWNER.set(owner or None) try: + all_db_accounts = _read_accounts_from_db() + if _mcp_owner_required(all_db_accounts): + return [TextContent( + type="text", + text="Error: email MCP requires an authenticated owner when multiple email account owners are configured.", + )] + if name == "list_email_accounts": - rows = _list_accounts_raw() + rows = _filter_accounts_for_owner(all_db_accounts) if not rows: + if all_db_accounts and owner: + return [TextContent(type="text", text="No email accounts configured for this owner.")] return [TextContent(type="text", text="No email accounts configured. Legacy single-account mode active.")] lines = [f"Found {len(rows)} email account(s):\n"] for r in rows: @@ -2108,6 +2173,16 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: bcc=arguments.get("bcc"), account=acct, ) + if "error" in result: + return [TextContent(type="text", text=f"Error: {result['error']}")] + if result.get("pending"): + return [TextContent( + type="text", + text=( + f"Draft staged for approval (pending id: {result.get('pending_id')}). " + "Nothing has been sent yet. Review and approve it in Odysseus before delivery." + ), + )] acct_note = f" (from {result['account']})" if result.get("account") else "" return [TextContent(type="text", text=f"Sent email to {result['to']} with subject '{result['subject']}'{acct_note}.")] @@ -2283,6 +2358,8 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: except Exception as e: return [TextContent(type="text", text=f"Error: {e}")] + finally: + _CURRENT_OWNER.reset(owner_token) # ── Main ── diff --git a/mcp_servers/memory_server.py b/mcp_servers/memory_server.py index 63c8a2bd8..fafbcfc2b 100644 --- a/mcp_servers/memory_server.py +++ b/mcp_servers/memory_server.py @@ -6,6 +6,7 @@ Imports MemoryManager and MemoryVectorStore from the Odysseus codebase. """ import asyncio +import os import sys import time from pathlib import Path @@ -23,6 +24,55 @@ _memory_manager = None _memory_vector = None _initialized = False +_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_MEMORY_OWNER", "ODYSSEUS_MEMORY_OWNER") +_OWNER_SCOPE_ERROR = ( + "Error: Memory MCP owner is not configured for an owner-scoped memory store. " + "Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool." +) + + +def _configured_owner() -> str | None: + for key in _OWNER_ENV_KEYS: + owner = os.environ.get(key, "").strip() + if owner: + return owner + return None + + +def _entry_owner(entry: dict) -> str | None: + owner = entry.get("owner") + if owner is None: + return None + owner_text = str(owner).strip() + return owner_text or None + + +def _owner_scoped_store(entries: list[dict]) -> bool: + return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict)) + + +def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]: + """Return configured owner, all entries, visible entries, and optional error.""" + entries = _memory_manager.load_all() + owner = _configured_owner() + if owner is None and _owner_scoped_store(entries): + return None, entries, [], _OWNER_SCOPE_ERROR + if owner is None: + visible = [ + entry for entry in entries + if isinstance(entry, dict) and _entry_owner(entry) is None + ] + else: + visible = [ + entry for entry in entries + if isinstance(entry, dict) and _entry_owner(entry) == owner + ] + return owner, entries, visible, None + + +def _text_result(text: str) -> list[TextContent]: + return [TextContent(type="text", text=text)] + def _ensure_init(): """Lazy-init memory managers on first use.""" @@ -75,24 +125,26 @@ async def list_tools() -> list[Tool]: @server.call_tool() async def call_tool(name: str, arguments: dict) -> list[TextContent]: if name != "manage_memory": - return [TextContent(type="text", text=f"Unknown tool: {name}")] + return _text_result(f"Unknown tool: {name}") _ensure_init() if not _memory_manager: - return [TextContent(type="text", text="Error: Memory manager not available")] + return _text_result("Error: Memory manager not available") action = arguments.get("action", "") if action == "list": category_filter = arguments.get("category", "") - memories = _memory_manager.load() + _owner, _all_memories, memories, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) if category_filter: memories = [m for m in memories if m.get("category", "").lower() == category_filter.lower()] if not memories: msg = "No memories found" if category_filter: msg += f" in category '{category_filter}'" - return [TextContent(type="text", text=msg + ".")] + return _text_result(msg + ".") lines = [f"Found {len(memories)} memory entries:\n"] for m in memories: @@ -102,15 +154,17 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: if len(text) > 150: text = text[:150] + "..." lines.append(f"- [{cat}] `{mid}` — {text}") - return [TextContent(type="text", text="\n".join(lines))] + return _text_result("\n".join(lines)) elif action == "add": text = arguments.get("text", "") category = arguments.get("category", "fact") if not text: - return [TextContent(type="text", text="Error: Memory text cannot be empty")] - entry = _memory_manager.add_entry(text, source="ai_agent", category=category) - memories = _memory_manager.load_all() + return _text_result("Error: Memory text cannot be empty") + owner, memories, _visible, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) + entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner) memories.append(entry) _memory_manager.save(memories) if _memory_vector and _memory_vector.healthy: @@ -118,25 +172,28 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: _memory_vector.add(entry["id"], text) except Exception: pass - return [TextContent(type="text", text=f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")] + return _text_result(f"Memory added: [{category}] {text} (id: {entry['id'][:8]})") elif action == "edit": memory_id = arguments.get("memory_id", "") new_text = arguments.get("text", "") if not memory_id or not new_text: - return [TextContent(type="text", text="Error: edit needs memory_id and text")] - memories = _memory_manager.load_all() - found = False + return _text_result("Error: edit needs memory_id and text") + _owner, memories, visible, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) full_id = None - for m in memories: + for m in visible: if m.get("id", "").startswith(memory_id): - m["text"] = new_text - m["timestamp"] = int(time.time()) - found = True full_id = m["id"] break - if not found: - return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")] + if not full_id: + return _text_result(f"Error: Memory '{memory_id}' not found") + for m in memories: + if m.get("id") == full_id: + m["text"] = new_text + m["timestamp"] = int(time.time()) + break _memory_manager.save(memories) if _memory_vector and _memory_vector.healthy and full_id: try: @@ -144,24 +201,26 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: _memory_vector.add(full_id, new_text) except Exception: pass - return [TextContent(type="text", text=f"Memory updated: {new_text}")] + return _text_result(f"Memory updated: {new_text}") elif action == "delete": memory_id = arguments.get("memory_id", "") if not memory_id: - return [TextContent(type="text", text="Error: delete needs memory_id")] - memories = _memory_manager.load_all() + return _text_result("Error: delete needs memory_id") + _owner, memories, visible, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) full_id = None deleted_text = "" deleted_category = "" - for m in memories: + for m in visible: if m.get("id", "").startswith(memory_id): full_id = m["id"] deleted_text = m.get("text", "") deleted_category = m.get("category", "") break if not full_id: - return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")] + return _text_result(f"Error: Memory '{memory_id}' not found") memories = [m for m in memories if m.get("id") != full_id] _memory_manager.save(memories) if _memory_vector and _memory_vector.healthy and full_id: @@ -171,30 +230,32 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: pass cat = f"[{deleted_category}] " if deleted_category else "" snippet = deleted_text if len(deleted_text) <= 120 else deleted_text[:117] + "..." - return [TextContent(type="text", text=f"Memory deleted: {cat}{snippet} (id: {memory_id})")] + return _text_result(f"Memory deleted: {cat}{snippet} (id: {memory_id})") elif action == "search": query = arguments.get("text", "") if not query: - return [TextContent(type="text", text="Error: search needs text (query)")] - memories = _memory_manager.load() + return _text_result("Error: search needs text (query)") + _owner, _all_memories, memories, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) if hasattr(_memory_manager, 'get_relevant_memories'): results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20) else: query_lower = query.lower() results = [m for m in memories if query_lower in m.get("text", "").lower()][:20] if not results: - return [TextContent(type="text", text=f"No memories found matching '{query}'.")] + return _text_result(f"No memories found matching '{query}'.") lines = [f"Found {len(results)} matching memories:\n"] for m in results: cat = m.get("category", "fact") mid = m.get("id", "?")[:8] text = m.get("text", "") lines.append(f"- [{cat}] `{mid}` — {text}") - return [TextContent(type="text", text="\n".join(lines))] + return _text_result("\n".join(lines)) else: - return [TextContent(type="text", text=f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")] + return _text_result(f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search") async def run(): diff --git a/package-lock.json b/package-lock.json index 39e4c9964..eac6229e7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,93 +4,19 @@ "requires": true, "packages": { "": { - "dependencies": { - "@anthropic-ai/sdk": "^0.104.1" - }, "devDependencies": { - "@antithesishq/bombadil": "^0.5.0" - } - }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.104.1", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.104.1.tgz", - "integrity": "sha512-gGACa/+IaiXzRRmF96aOhamoBgapKRBiFWbmmTFP8aMkpaEcuStF+Q61bjo4vPxBM7gqWJNZqsngslRdnLHv0Q==", - "license": "MIT", - "dependencies": { - "json-schema-to-ts": "^3.1.1", - "standardwebhooks": "^1.0.0" - }, - "bin": { - "anthropic-ai-sdk": "bin/cli" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } + "@antithesishq/bombadil": "^0.6.1" } }, "node_modules/@antithesishq/bombadil": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.5.0.tgz", - "integrity": "sha512-s0zImmr0iyvSP6QcVLvf40CUiZYIdWBAxiq20uhzujwvfitYa3PGJN652k/pLtVccHM/JrGQxZdvLnihZpltHA==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.6.1.tgz", + "integrity": "sha512-d1iufG3MI7gSMSiSmMeNdcMW+qR0yQXL2zdkVynC3n3DYgFJYlYXKUQzygmqU12m4RWlR5iOdQU1hsx5UT6+IA==", "dev": true, "license": "MIT", "bin": { "bombadil": "bin/bombadil.js" } - }, - "node_modules/@babel/runtime": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", - "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@stablelib/base64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", - "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", - "license": "MIT" - }, - "node_modules/fast-sha256": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", - "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", - "license": "Unlicense" - }, - "node_modules/json-schema-to-ts": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", - "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.18.3", - "ts-algebra": "^2.0.0" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/standardwebhooks": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", - "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", - "license": "MIT", - "dependencies": { - "@stablelib/base64": "^1.0.0", - "fast-sha256": "^1.3.0" - } - }, - "node_modules/ts-algebra": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", - "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", - "license": "MIT" } } } diff --git a/package.json b/package.json index 71b622722..0f1ef7fad 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,6 @@ "url": "https://github.com/pewdiepie-archdaemon/odysseus.git" }, "devDependencies": { - "@antithesishq/bombadil": "^0.5.0" - }, - "dependencies": { - "@anthropic-ai/sdk": "^0.104.1" + "@antithesishq/bombadil": "^0.6.1" } } diff --git a/routes/api_token_routes.py b/routes/api_token_routes.py index 954e1e802..cbc828731 100644 --- a/routes/api_token_routes.py +++ b/routes/api_token_routes.py @@ -160,6 +160,8 @@ def setup_api_token_routes() -> APIRouter: payload = await request.json() except Exception: payload = {} + if not isinstance(payload, dict): + payload = {} with get_db_session() as db: token = db.query(ApiToken).filter(ApiToken.id == token_id).first() if not token: diff --git a/routes/assistant_routes.py b/routes/assistant_routes.py index 17c50163d..0b609e37f 100644 --- a/routes/assistant_routes.py +++ b/routes/assistant_routes.py @@ -16,6 +16,7 @@ from pydantic import BaseModel from core.database import SessionLocal, CrewMember, ScheduledTask from src.auth_helpers import get_current_user +from core.auth import RESERVED_USERNAMES from src.task_scheduler import compute_next_run @@ -89,11 +90,11 @@ def setup_assistant_routes(task_scheduler) -> APIRouter: # check-in tasks seeded. Hitting any /assistant route under one of these # used to seed a full CrewMember + Morning/Midday/Evening tasks under that # owner, which then double-fired alongside the real user's check-ins. - _SYNTHETIC_OWNERS = frozenset({"internal-tool", "api", "demo", "system", ""}) + # RESERVED_USERNAMES covers the same set; the `not owner` guard handles "". async def _get_or_create(owner: str) -> CrewMember: """Return the per-owner assistant CrewMember, creating it on demand.""" - if not owner or owner in _SYNTHETIC_OWNERS: + if not owner or owner in RESERVED_USERNAMES: raise HTTPException(status_code=400, detail=f"Cannot seed assistant for {owner!r}") db = SessionLocal() try: diff --git a/routes/auth_routes.py b/routes/auth_routes.py index 6173b0c14..5c7a4e04a 100644 --- a/routes/auth_routes.py +++ b/routes/auth_routes.py @@ -12,8 +12,8 @@ import re from pathlib import Path from core.atomic_io import atomic_write_json, atomic_write_text -from core.auth import AuthManager, SetAdminResult -from src.constants import DEEP_RESEARCH_DIR, MEMORY_FILE, SKILLS_DIR +from core.auth import AuthManager, RESERVED_USERNAMES, SetAdminResult, TOKEN_TTL +from src.constants import DEEP_RESEARCH_DIR, MEMORY_FILE, PASSWORD_MIN_LENGTH, SKILLS_DIR from src.rate_limiter import RateLimiter from src.settings_scrub import scrub_settings from src.settings import ( @@ -102,8 +102,12 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: raise HTTPException(429, "Too many requests — try again later") if auth_manager.is_configured: raise HTTPException(400, "Already configured") - if len(body.password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") + if len(body.password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") + if len(body.username.strip()) < 1: + raise HTTPException(400, "Username is required") + if body.username.lower() in RESERVED_USERNAMES: + raise HTTPException(403, "Username is reserved") ok = await asyncio.to_thread(auth_manager.setup, body.username, body.password) if not ok: raise HTTPException(500, "Setup failed") @@ -118,10 +122,12 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: raise HTTPException(400, "Run setup first") if not auth_manager.signup_enabled: raise HTTPException(403, "Registration is disabled. Ask an admin for an account.") - if len(body.password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") + if len(body.password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") if len(body.username.strip()) < 1: raise HTTPException(400, "Username is required") + if body.username.lower() in RESERVED_USERNAMES: + raise HTTPException(403, "Username is reserved") ok = await asyncio.to_thread(auth_manager.create_user, body.username, body.password, is_admin=False) if not ok: raise HTTPException(409, "Username already taken") @@ -144,6 +150,8 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: raise HTTPException(401, "Invalid 2FA code") # All checks passed — create session (password already verified above) token = await asyncio.to_thread(auth_manager.create_session_trusted, username) + if not token: + raise HTTPException(401, "Invalid credentials") cookie_kwargs = dict( key=SESSION_COOKIE, value=token, @@ -153,7 +161,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: path="/", ) if body.remember: - cookie_kwargs["max_age"] = 60 * 60 * 24 * 7 # 7 days + cookie_kwargs["max_age"] = TOKEN_TTL response.set_cookie(**cookie_kwargs) return {"ok": True, "username": username} @@ -182,13 +190,18 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: pass return result + @router.get("/policy") + async def auth_policy(): + """Return public auth policy constants for the frontend.""" + return auth_manager.policy() + @router.post("/change-password") async def change_password(body: ChangePasswordRequest, request: Request): user = _get_current_user(request) if not user: raise HTTPException(401, "Not authenticated") - if len(body.new_password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") + if len(body.new_password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") current_token = request.cookies.get(SESSION_COOKIE) ok = await asyncio.to_thread(auth_manager.change_password, user, body.current_password, body.new_password) if not ok: @@ -268,8 +281,12 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: user = _get_current_user(request) if not user or not auth_manager.is_admin(user): raise HTTPException(403, "Admin only") - if len(body.password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") + if len(body.password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") + if len(body.username.strip()) < 1: + raise HTTPException(400, "Username is required") + if body.username.lower() in RESERVED_USERNAMES: + raise HTTPException(403, "Username is reserved") ok = auth_manager.create_user(body.username, body.password, body.is_admin) if not ok: raise HTTPException(409, "Username already taken") @@ -432,6 +449,23 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: except Exception as e: logger.warning("Failed to rename upload owner references %s -> %s: %s", old_username, new_username, e) + # direct personal RAG uploads live in per-owner directories and the + # vector metadata also carries the username used for owner-filtered + # search. Keep both in sync with the auth rename. + try: + from routes.personal_routes import rename_personal_upload_owner + personal_docs_manager = getattr(request.app.state, "personal_docs_manager", None) + if personal_docs_manager is not None: + rag_manager = getattr(personal_docs_manager, "rag_manager", None) + rename_personal_upload_owner( + old_username, + new_username, + personal_docs_manager=personal_docs_manager, + rag_manager=rag_manager, + ) + except Exception as e: + logger.warning("Failed to rename personal RAG upload owner references %s -> %s: %s", old_username, new_username, e) + # skills: SKILL.md frontmatter carries owner: ; the usage # sidecar (_usage.json) keys entries as owner::skill-name. Both must # be updated or the renamed user's Skills panel goes empty. diff --git a/routes/chat_helpers.py b/routes/chat_helpers.py index f030d6c91..06c92ac6b 100644 --- a/routes/chat_helpers.py +++ b/routes/chat_helpers.py @@ -14,7 +14,7 @@ from core.database import Session as DBSession, ModelEndpoint from src.llm_core import normalize_model_id from src.endpoint_resolver import normalize_base from src.context_compactor import maybe_compact, trim_for_context -from src.auth_helpers import get_current_user +from src.auth_helpers import effective_user from src.prompt_security import untrusted_context_message from routes.prefs_routes import _load_for_user as load_prefs_for_user @@ -48,6 +48,22 @@ def _is_casual_low_signal(text: str) -> bool: return len(tail_words) <= 2 +# Strong references to in-flight fire-and-forget tasks scheduled from this +# module. asyncio only keeps weak references to tasks created via +# create_task, so without this the GC can collect a task mid-execution and +# the background work (extraction, auto-naming) silently never runs. +# Mirrors WebhookManager._spawn_tracked from src/webhook_manager.py. +_BG_TASKS: set[asyncio.Task] = set() + + +def _spawn_bg(coro) -> asyncio.Task: + """Schedule a background task and hold a strong reference until it finishes.""" + task = asyncio.create_task(coro) + _BG_TASKS.add(task) + task.add_done_callback(_BG_TASKS.discard) + return task + + # ── Data containers ────────────────────────────────────────────────────── # @dataclass @@ -103,7 +119,7 @@ def _enforce_chat_privileges(request, sess) -> None: which means unrestricted allowed_models / zero cap -> no-op for them. """ try: - user = get_current_user(request) + user = effective_user(request) except Exception: user = None if not user: @@ -184,17 +200,9 @@ async def auto_name_session(session_manager, sess): return owner = getattr(sess, "owner", None) - t_url, t_model, t_headers = resolve_task_endpoint(owner=owner) - if not t_model: - # If no task/utility model is configured at all, fall back to - # the session's own model so auto-naming still works even on - # minimal setups. - from src.endpoint_resolver import resolve_endpoint - _fallback = resolve_endpoint("default", owner=owner) - if _fallback and _fallback[1]: - t_url, t_model, t_headers = _fallback - else: - t_url, t_model, t_headers = sess.endpoint_url, sess.model, sess.headers + t_url, t_model, t_headers = resolve_task_endpoint( + sess.endpoint_url, sess.model, sess.headers, owner=owner + ) if not t_model: logger.debug("[auto-name] No model provided, skipping") return @@ -371,11 +379,11 @@ def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, inco def fire_message_event(request, webhook_manager, session_id: str, sess, message: str, compare_mode: bool = False): """Fire webhook and event_bus events for a new user message.""" if webhook_manager and not compare_mode: - asyncio.create_task(webhook_manager.fire("chat.message", { + webhook_manager.fire_and_forget("chat.message", { "session_id": session_id, "model": sess.model, "message": message[:2000], - })) + }) from src.event_bus import fire_event - user = get_current_user(request) + user = effective_user(request) fire_event("message_sent", user) @@ -601,8 +609,9 @@ async def build_chat_context( if not incognito: fire_message_event(request, webhook_manager, session_id, sess, message, compare_mode) - # Resolve user prefs - user = get_current_user(request) + # Resolve owner-scoped prefs/context. Browser requests keep the cookie user; + # bearer-token chat requests use the token owner instead of the "api" sentinel. + user = effective_user(request) uprefs = load_prefs_for_user(user) casual_low_signal = _is_casual_low_signal(message) @@ -1141,7 +1150,7 @@ def run_post_response_tasks( ))) if _extraction_jobs: - asyncio.create_task(_run_extraction_jobs_sequentially(session_id, _extraction_jobs)) + _spawn_bg(_run_extraction_jobs_sequentially(session_id, _extraction_jobs)) # Token accumulation if last_metrics: @@ -1149,11 +1158,11 @@ def run_post_response_tasks( # Webhook if webhook_manager and not compare_mode: - asyncio.create_task(webhook_manager.fire("chat.completed", { + webhook_manager.fire_and_forget("chat.completed", { "session_id": session_id, "model": sess.model, "user_message": message, "response": full_response[:2000], - })) + }) # Auto-name if needs_auto_name(sess.name): - asyncio.create_task(auto_name_session(session_manager, sess)) + _spawn_bg(auto_name_session(session_manager, sess)) diff --git a/routes/chat_routes.py b/routes/chat_routes.py index 0967667f1..b4a6ed837 100644 --- a/routes/chat_routes.py +++ b/routes/chat_routes.py @@ -23,7 +23,7 @@ from src.endpoint_resolver import normalize_base as _normalize_base, build_chat_ from src.session_search import search_session_messages from src.prompt_security import untrusted_context_message from core.exceptions import SessionNotFoundError -from src.auth_helpers import get_current_user +from src.auth_helpers import effective_user, get_current_user from routes.session_routes import _verify_session_owner from routes.document_helpers import _owner_session_filter from core.database import SessionLocal, get_session_mode, set_session_mode @@ -126,7 +126,8 @@ def _clear_orphaned_session_endpoint(sess, owner: str | None = None) -> bool: sess.model = "" sess.headers = {} return True - except Exception: + except Exception as e: + logger.warning("Failed to clear orphaned session endpoint", exc_info=e) db.rollback() return False finally: @@ -144,7 +145,8 @@ def _endpoint_cache_contains_model(endpoint, model: str) -> bool: return True try: models = json.loads(raw) if isinstance(raw, str) else raw - except Exception: + except Exception as e: + logger.warning("Failed to parse cached models list, treating as containing model", exc_info=e) return True if not isinstance(models, list) or not models: return True @@ -236,7 +238,8 @@ def _recover_empty_session_model(sess, session_id: str, owner: str | None = None is_chatgpt_subscription = False try: cached = json.loads(ep.cached_models) if isinstance(ep.cached_models, str) else (ep.cached_models or []) - except Exception: + except Exception as e: + logger.warning("Failed to parse cached_models for endpoint %r", getattr(ep, "id", "?"), exc_info=e) cached = [] if not cached: visible = [] @@ -360,7 +363,7 @@ def setup_chat_routes( sess = session_manager.get_session(session) except KeyError: raise HTTPException(404, f"Session '{session}' not found") - owner = get_current_user(request) + owner = effective_user(request) if _clear_orphaned_session_endpoint(sess, owner=owner): raise HTTPException(400, "Selected model endpoint was removed. Pick another model in Settings.") @@ -600,7 +603,7 @@ def setup_chat_routes( # but BEFORE loading. Prevents cross-user session hijack. _verify_session_owner(request, session) sess = session_manager.get_session(session) - owner = get_current_user(request) + owner = effective_user(request) if _clear_orphaned_session_endpoint(sess, owner=owner): raise HTTPException(400, "Selected model endpoint was removed. Pick another model in Settings.") # Issue #587: picker shows a model from the endpoint cache but @@ -631,7 +634,7 @@ def setup_chat_routes( _enforce_chat_privileges(request, sess) # Ensure session has auth headers - resolve_session_auth(sess, session, owner=get_current_user(request)) + resolve_session_auth(sess, session, owner=effective_user(request)) # Check for research_pending BEFORE mode persist overwrites it do_research = str(use_research).lower() == "true" @@ -646,8 +649,8 @@ def setup_chat_routes( elif attachments: try: att_ids = [str(x) for x in json.loads(attachments)] - except Exception: - pass + except Exception as e: + logger.warning("Failed to parse attachments JSON, ignoring attachments", exc_info=e) no_memory = str(form_data.get("no_memory", "")).lower() == "true" pre_context_tool_policy = build_effective_tool_policy( @@ -1491,7 +1494,7 @@ def setup_chat_routes( if not q or not q.strip(): return [] - _user = get_current_user(request) + _user = effective_user(request) return [ result.to_dict() for result in search_session_messages( diff --git a/routes/codex_routes.py b/routes/codex_routes.py index 579f47ddb..22fc7feeb 100644 --- a/routes/codex_routes.py +++ b/routes/codex_routes.py @@ -46,8 +46,12 @@ def _ssh_prefix_for_task(task: dict) -> tuple[str, str]: shell metacharacters in ``remoteHost`` is rejected with 400 rather than injected. """ - host = validate_remote_host((task.get("remoteHost") or "").strip() or None) or "" - ssh_port = validate_ssh_port((task.get("sshPort") or "").strip() or None) or "" + raw_host = task.get("remoteHost") + raw_port = task.get("sshPort") + host_value = str(raw_host).strip() if raw_host is not None else None + port_value = str(raw_port).strip() if raw_port is not None else None + host = validate_remote_host(host_value or None) or "" + ssh_port = validate_ssh_port(port_value or None) or "" port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else "" return host, port_flag @@ -306,7 +310,10 @@ def setup_codex_routes( @router.post("/emails/draft-document") async def codex_email_draft_document(request: Request, body: dict[str, Any] = Body(default_factory=dict)): - owner = _scope_owner_all(request, {"email:draft", "documents:write"}) + owner = _scope_owner(request, EMAIL_DRAFT_SCOPES) + docs_owner = _scope_owner_all(request, DOCS_WRITE_SCOPES) + if docs_owner != owner: + raise HTTPException(403, "API token owner mismatch") if documents_create_endpoint is None: raise HTTPException(503, "Documents integration is not available") from routes.document_routes import DocumentCreate @@ -790,7 +797,7 @@ def setup_codex_routes( norm = dict(body or {}) sess = (norm.get("tmux_session") or norm.get("session_id") or "").strip() model = (norm.get("model") or norm.get("repo_id") or "").strip() - host = (norm.get("host") or norm.get("remote_host") or "").strip() + host = validate_remote_host((norm.get("host") or norm.get("remote_host") or "").strip() or None) or "" port = norm.get("port") or 8000 import re as _re if not sess or not _re.fullmatch(r"[a-zA-Z0-9_-]+", sess): diff --git a/routes/cookbook_helpers.py b/routes/cookbook_helpers.py index bf1124933..51f019edb 100644 --- a/routes/cookbook_helpers.py +++ b/routes/cookbook_helpers.py @@ -505,6 +505,8 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache: " if u.startswith('KB'): return int(n * 1024)", " return int(n)", "def scan_ollama():", + " if any(m.get('is_ollama') for m in models): return", + " if os.name == 'nt' and not os.environ.get('ODYSSEUS_ALLOW_OLLAMA_CLI_SCAN'): return", " if not shutil.which('ollama'): return", " try:", " p = subprocess.run(['ollama', 'list'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=6)", @@ -535,8 +537,8 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache: " models.append({'repo_id':name,'size_bytes':size_bytes,'nb_files':1,'has_incomplete':False,'path':'ollama','backend':'ollama','is_ollama':True})", " return", "for _hf_cache in hf_cache_paths(): scan_hf(_hf_cache)", - "scan_ollama()", "scan_ollama_api()", + "scan_ollama()", ] for model_dir in model_dirs or []: lines.append(f"scan_dir(os.path.expanduser({model_dir!r}))") diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index 0d8257574..3308d26ae 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -1384,6 +1384,11 @@ def setup_cookbook_routes() -> APIRouter: # LOCAL execution on a native-Windows host never uses tmux (detached # process path below), regardless of the UI-supplied platform. local_windows = IS_WINDOWS and not remote + if is_windows and remote and "diffusion_server.py" in req.cmd: + raise HTTPException( + 400, + "Remote Windows Diffusers serving is not supported yet; use local Windows or a Linux remote server.", + ) if not is_windows and not local_windows and not await _binary_available("tmux", remote, req.ssh_port): return { diff --git a/routes/document_helpers.py b/routes/document_helpers.py index 57acc50e7..0de4cc2a3 100644 --- a/routes/document_helpers.py +++ b/routes/document_helpers.py @@ -102,8 +102,11 @@ def _owner_session_filter(q, user): The owner backfill runs in init_db before the app serves requests, so by the time this filter is live there are no NULL-owner rows to leak; - we therefore match the owner strictly.""" - if user is None: + we therefore match the owner strictly for authenticated callers.""" + if not user: + from src.auth_helpers import _auth_disabled + if user == "" or _auth_disabled(): + return q return q.filter(False) return q.filter(Document.owner == user) diff --git a/routes/document_routes.py b/routes/document_routes.py index e4598d925..d35d2a79e 100644 --- a/routes/document_routes.py +++ b/routes/document_routes.py @@ -503,7 +503,8 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: user = get_current_user(request) try: data = await request.json() - except Exception: + except Exception as e: + logger.warning("Failed to parse export request body, defaulting to empty", exc_info=e) data = {} ids = data.get("ids") or [] if not ids: @@ -645,8 +646,8 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: try: from src.agent_tools.document_tools import clear_active_document clear_active_document(doc_id) - except Exception: - pass + except Exception as e: + logger.warning("Failed to clear active document %r on detach", doc_id, exc_info=e) db.commit() db.refresh(doc) return _doc_to_dict(doc) @@ -1331,6 +1332,12 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: if not pdf_path: raise HTTPException(404, f"Source PDF {upload_id} not found") + # Fail fast with a clear 503 if the optional PyMuPDF dependency + # is missing — fill_fields/stamp_annotations will otherwise + # raise RuntimeError deep inside and bubble out as a 500. + # Mirrors the convention in _load_pdf_viewer_fitz above. + _load_pdf_viewer_fitz() + values = parse_markdown_to_values(doc.current_content or "") out_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name _to_unlink.append(out_path) diff --git a/routes/email_helpers.py b/routes/email_helpers.py index a054f4df8..513ec1f0a 100644 --- a/routes/email_helpers.py +++ b/routes/email_helpers.py @@ -13,6 +13,8 @@ and `email_pollers.py` (the background loops): """ import os +import base64 +import time import imaplib import smtplib import email as email_mod @@ -38,6 +40,106 @@ from src.secret_storage import decrypt as _decrypt logger = logging.getLogger(__name__) +def _xoauth2_raw(user: str, access_token: str) -> str: + """The SASL XOAUTH2 initial-response string (unencoded). + + Both smtplib.SMTP.auth() and imaplib.IMAP4.authenticate() base64-encode + the value their callback returns, so callers pass this raw form — never + pre-encoded — to avoid double base64. + """ + return f"user={user}\x01auth=Bearer {access_token}\x01\x01" + + +def _xoauth2_bytes(user: str, access_token: str) -> bytes: + """Raw XOAUTH2 bytes for imaplib's authenticate() callback.""" + return _xoauth2_raw(user, access_token).encode() + + +def make_oauth_state(account_id: str, owner: str) -> str: + """Return an HMAC-signed, base64-encoded OAuth state token. + + Encodes account_id + owner + a random nonce, signed with the app secret + so the callback can validate that the flow was initiated by an + authenticated, owning user (CSRF / state-forgery protection). + """ + import hmac as _hmac, hashlib as _hl, secrets as _sec + from src.secret_storage import _load_or_create_key + nonce = _sec.token_hex(16) + payload = json.dumps({"a": account_id, "o": owner, "n": nonce}, separators=(",", ":")) + sig = _hmac.new(_load_or_create_key(), payload.encode(), _hl.sha256).hexdigest() + return base64.urlsafe_b64encode(f"{payload}|{sig}".encode()).decode() + + +def verify_oauth_state(state: str) -> dict | None: + """Verify an OAuth state token's HMAC signature. + + Returns the decoded payload dict ({"a", "o", "n"}) on success, or None if + the token is malformed, tampered, or signed with a different key. + """ + import hmac as _hmac, hashlib as _hl + from src.secret_storage import _load_or_create_key + try: + decoded = base64.urlsafe_b64decode(state.encode()).decode() + payload, sig = decoded.rsplit("|", 1) + expected = _hmac.new(_load_or_create_key(), payload.encode(), _hl.sha256).hexdigest() + if not _hmac.compare_digest(sig, expected): + return None + return json.loads(payload) + except Exception: + return None + + +def _refresh_google_token(account_id: str) -> str | None: + """Exchange the stored refresh token for a new access token and persist it.""" + import httpx + from core.database import SessionLocal as _SL, EmailAccount as _EA + from src.secret_storage import encrypt as _enc, decrypt as _dec + client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "") + client_secret = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET", "") + if not client_id or not client_secret: + return None + db = _SL() + try: + row = db.get(_EA, account_id) + if not row or not row.oauth_refresh_token: + return None + refresh_token = _dec(row.oauth_refresh_token or "") + if not refresh_token: + return None + resp = httpx.post("https://oauth2.googleapis.com/token", data={ + "client_id": client_id, + "client_secret": client_secret, + "refresh_token": refresh_token, + "grant_type": "refresh_token", + }, timeout=10) + resp.raise_for_status() + data = resp.json() + access_token = data["access_token"] + row.oauth_access_token = _enc(access_token) + row.oauth_token_expiry = str(int(time.time()) + data.get("expires_in", 3600)) + db.commit() + return access_token + except Exception: + logger.warning(f"Google token refresh failed for account {account_id}") + return None + finally: + db.close() + + +def _get_valid_google_token(account_id: str, cfg: dict) -> str | None: + """Return a valid Google access token, refreshing if expired or missing.""" + from src.secret_storage import decrypt as _dec + access_token = _dec(cfg.get("oauth_access_token") or "") + expiry_str = cfg.get("oauth_token_expiry") or "" + if access_token and expiry_str: + try: + if int(expiry_str) - 60 > time.time(): + return access_token + except (ValueError, TypeError): + pass + return _refresh_google_token(account_id) + + def _smtp_security_mode(cfg: dict) -> str: raw = str(cfg.get("smtp_security") or "").strip().lower() if raw in {"ssl", "starttls", "none"}: @@ -54,20 +156,29 @@ def _send_smtp_message(cfg: dict, from_addr: str, recipients: list[str], message port = int(cfg.get("smtp_port") or 465) user = cfg.get("smtp_user") or "" password = cfg.get("smtp_password") or "" + + def _auth_smtp(smtp): + if cfg.get("oauth_provider") == "google": + token = _get_valid_google_token(cfg.get("account_id"), cfg) + if not token: + raise RuntimeError("Google OAuth token unavailable — reconnect the account") + smtp.ehlo() + smtp.auth("XOAUTH2", lambda challenge=None: _xoauth2_raw(user, token), initial_response_ok=True) + elif user and password: + smtp.login(user, password) + security = _smtp_security_mode(cfg) if security == "ssl": with smtplib.SMTP_SSL(host, port, timeout=timeout) as smtp: - if user and password: - smtp.login(user, password) + _auth_smtp(smtp) smtp.sendmail(from_addr, recipients, message) return with smtplib.SMTP(host, port, timeout=timeout) as smtp: if security == "starttls": smtp.starttls() - if user and password: - smtp.login(user, password) + _auth_smtp(smtp) smtp.sendmail(from_addr, recipients, message) @@ -701,10 +812,16 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict: "imap_password": _decrypt(row.imap_password or ""), "imap_starttls": bool(row.imap_starttls), "from_address": row.from_address or row.imap_user or "", + "oauth_provider": row.oauth_provider or "", + "oauth_access_token": row.oauth_access_token or "", + "oauth_refresh_token": row.oauth_refresh_token or "", + "oauth_token_expiry": row.oauth_token_expiry or "", + "display_name": row.display_name or "", } - if not (cfg["smtp_host"] and cfg["smtp_user"] and cfg["smtp_password"]): + is_oauth = bool(cfg.get("oauth_provider")) + if not is_oauth and not (cfg["smtp_host"] and cfg["smtp_user"] and cfg["smtp_password"]): logger.warning(f"SMTP not configured for account {row.name!r}") - if not (cfg["imap_host"] and cfg["imap_user"] and cfg["imap_password"]): + if not is_oauth and not (cfg["imap_host"] and cfg["imap_user"] and cfg["imap_password"]): logger.warning(f"IMAP not configured for account {row.name!r}") return cfg finally: @@ -825,12 +942,19 @@ def _imap_connect(account_id: str | None = None, owner: str = "", timeout=timeout, ) try: - conn.login(cfg["imap_user"], cfg["imap_password"]) + if cfg.get("oauth_provider") == "google": + token = _get_valid_google_token(cfg.get("account_id"), cfg) + if not token: + raise RuntimeError("Google OAuth token unavailable — reconnect the account in Settings → Integrations") + conn.authenticate("XOAUTH2", lambda x: _xoauth2_bytes(cfg["imap_user"], token)) + else: + conn.login(cfg["imap_user"], cfg["imap_password"]) except Exception: # A failed AUTHENTICATE (e.g. an Office 365 app password on an - # MFA-enabled tenant, #3174) otherwise orphans the already-connected - # socket; close it before propagating so a misconfigured account - # can't leak one descriptor per retry / background poller pass. + # MFA-enabled tenant, #3174, or an expired/revoked OAuth token) + # otherwise orphans the already-connected socket; close it before + # propagating so a misconfigured account can't leak one descriptor + # per retry / background poller pass. try: conn.shutdown() except Exception: diff --git a/routes/email_routes.py b/routes/email_routes.py index 81d2b7330..ace319626 100644 --- a/routes/email_routes.py +++ b/routes/email_routes.py @@ -13,7 +13,9 @@ handlers need. The split is mechanical — no behavior change. """ import asyncio +import os import sqlite3 as _sql3 +import time import email as email_mod import email.header import email.utils @@ -43,6 +45,7 @@ from routes.email_helpers import ( _load_settings, _save_settings, _get_email_config, _send_smtp_message, _smtp_security_mode, _IMAP_TIMEOUT_SECONDS, _open_imap_connection, + make_oauth_state, verify_oauth_state, _imap_connect, _imap, _decode_header, _detect_sent_folder, _detect_drafts_folder, _extract_attachment_text, _list_attachments_from_msg, _has_visible_attachments, _is_likely_signature_image_attachment, _extract_attachment_to_disk, _extract_html, _extract_text, @@ -77,15 +80,16 @@ def _email_tag_owner_aliases(account_id: str | None, owner: str = "") -> list[st cfg.get("smtp_user") or "", cfg.get("from_address") or "", ]) - except Exception: + except Exception as _e: + logger.warning("Failed to resolve email account alias", exc_info=_e) resolved_account_id = None row = db.get(_EA, resolved_account_id) if resolved_account_id else None if row: aliases.extend([row.owner or "", row.imap_user or "", row.from_address or ""]) finally: db.close() - except Exception: - pass + except Exception as _e: + logger.warning("Failed to load email aliases", exc_info=_e) out = [] for a in aliases: a = (a or "").strip() @@ -301,7 +305,9 @@ def _group_uid_fetch_records(msg_data) -> list: def _smtp_ready(cfg: dict) -> bool: - return bool(cfg.get("smtp_host") and cfg.get("smtp_user") and cfg.get("smtp_password")) + if not cfg.get("smtp_host") or not cfg.get("smtp_user"): + return False + return bool(cfg.get("smtp_password") or cfg.get("oauth_provider")) def _resolve_send_config(account_id: str | None = None, owner: str = "") -> dict: @@ -2165,7 +2171,7 @@ def setup_email_routes(): to = _normalize_addr_field(to or "") cc = _normalize_addr_field(cc or "") bcc = _normalize_addr_field(bcc or "") - outer["From"] = cfg["from_address"] + outer["From"] = email.utils.formataddr((cfg.get("display_name") or "", cfg["from_address"])) outer["To"] = to if cc: outer["Cc"] = cc @@ -2309,12 +2315,10 @@ def setup_email_routes(): try: conn = sqlite3.connect(SCHEDULED_DB) conn.row_factory = sqlite3.Row - # The MCP server can't easily set owner, so it stores '' — fall - # back to those rows in addition to the caller's owner. rows = conn.execute( """SELECT id, to_addr, subject, body, created_at, account_id FROM scheduled_emails - WHERE status = 'agent_draft' AND (owner = ? OR owner = '') + WHERE status = 'agent_draft' AND owner = ? ORDER BY created_at DESC""", (owner or "",), ).fetchall() @@ -2335,7 +2339,7 @@ def setup_email_routes(): cur = conn.execute( """UPDATE scheduled_emails SET status = 'pending', send_at = ? - WHERE id = ? AND status = 'agent_draft' AND (owner = ? OR owner = '')""", + WHERE id = ? AND status = 'agent_draft' AND owner = ?""", (datetime.utcnow().isoformat(), sid, owner or ""), ) conn.commit() @@ -2356,7 +2360,7 @@ def setup_email_routes(): conn = sqlite3.connect(SCHEDULED_DB) cur = conn.execute( """UPDATE scheduled_emails SET status = 'cancelled' - WHERE id = ? AND status = 'agent_draft' AND (owner = ? OR owner = '')""", + WHERE id = ? AND status = 'agent_draft' AND owner = ?""", (sid, owner or ""), ) conn.commit() @@ -2429,6 +2433,7 @@ def setup_email_routes(): try: cfg = _resolve_send_config(req.account_id, owner=owner) except Exception as e: + logger.warning(f"No SMTP-capable account resolved: {e}") return {"success": False, "error": str(e) or "No SMTP-capable email account configured"} # Use 'mixed' if we have attachments, 'alternative' otherwise @@ -2444,7 +2449,7 @@ def setup_email_routes(): req.to = _normalize_addr_field(req.to or "") req.cc = _normalize_addr_field(req.cc or "") req.bcc = _normalize_addr_field(req.bcc or "") - outer["From"] = cfg["from_address"] + outer["From"] = email.utils.formataddr((cfg.get("display_name") or "", cfg["from_address"])) outer["To"] = req.to if req.cc: outer["Cc"] = req.cc @@ -2495,6 +2500,10 @@ def setup_email_routes(): _account_id = cfg.get("account_id") or req.account_id # capture for the IMAP append in the closure _in_reply_to = (req.in_reply_to or "").strip() + _oauth_provider = cfg.get("oauth_provider") or "" + _oauth_access_token = cfg.get("oauth_access_token") or "" + _oauth_refresh_token = cfg.get("oauth_refresh_token") or "" + _oauth_token_expiry = cfg.get("oauth_token_expiry") or "" def _deliver(): try: @@ -2505,6 +2514,11 @@ def setup_email_routes(): "smtp_security": _smtp_security, "smtp_user": _smtp_user, "smtp_password": _smtp_pw, + "account_id": _account_id, + "oauth_provider": _oauth_provider, + "oauth_access_token": _oauth_access_token, + "oauth_refresh_token": _oauth_refresh_token, + "oauth_token_expiry": _oauth_token_expiry, }, _from, _recipients, @@ -2617,7 +2631,7 @@ def setup_email_routes(): msg.attach(MIMEText(_draft_html, "html", "utf-8")) else: msg = MIMEText(req.body, "plain", "utf-8") - msg["From"] = cfg["from_address"] + msg["From"] = email.utils.formataddr((cfg.get("display_name") or "", cfg["from_address"])) msg["To"] = req.to if req.cc: msg["Cc"] = req.cc @@ -3269,6 +3283,8 @@ def setup_email_routes(): "from_address": r.from_address or "", "has_imap_password": bool(r.imap_password), "has_smtp_password": bool(r.smtp_password), + "oauth_provider": r.oauth_provider or "", + "display_name": r.display_name or "", }) return {"accounts": out} finally: @@ -3301,6 +3317,7 @@ def setup_email_routes(): smtp_user=(data.get("smtp_user") or "").strip(), smtp_password=_enc(data.get("smtp_password") or ""), from_address=(data.get("from_address") or "").strip(), + display_name=(data.get("display_name") or "").strip(), # SECURITY: stamp the creator so all subsequent reads / mutations # can filter by user. Without this every new account leaks to # every other user. @@ -3335,7 +3352,7 @@ def setup_email_routes(): if not row: return {"ok": False, "error": "Account not found"} # Simple fields - for key in ("name", "imap_host", "imap_user", "smtp_host", "smtp_user", "from_address"): + for key in ("name", "imap_host", "imap_user", "smtp_host", "smtp_user", "from_address", "display_name"): if key in data: setattr(row, key, (data[key] or "").strip()) for key in ("imap_port", "smtp_port"): @@ -3524,4 +3541,123 @@ def setup_email_routes(): finally: db.close() + # ── Google OAuth2 routes ── + + @router.get("/oauth/google/authorize") + async def google_oauth_authorize(account_id: str = Query(...), request: Request = None, owner: str = Depends(require_user)): + import urllib.parse + _assert_owns_account(account_id, owner) + client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "") + if not client_id: + raise HTTPException(400, "GOOGLE_OAUTH_CLIENT_ID not set — add it to .env") + redirect_uri = ( + os.environ.get("GOOGLE_OAUTH_REDIRECT_URI") + or f"http://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback" + ) + state = make_oauth_state(account_id, owner) + params = urllib.parse.urlencode({ + "client_id": client_id, + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": "https://mail.google.com/ email", + "access_type": "offline", + "prompt": "consent", + "state": state, + }) + from fastapi.responses import RedirectResponse as _RR + return _RR(f"https://accounts.google.com/o/oauth2/v2/auth?{params}") + + @router.get("/oauth/google/callback") + async def google_oauth_callback( + code: str = Query(None), + state: str = Query(None), + error: str = Query(None), + request: Request = None, + ): + import urllib.parse + from fastapi.responses import RedirectResponse as _RR + if error: + return _RR("/?section=integrations&email_oauth_error=google_error") + if not code or not state: + return _RR("/?section=integrations&email_oauth_error=missing_code") + state_data = verify_oauth_state(state) + if not state_data: + return _RR("/?section=integrations&email_oauth_error=invalid_state") + account_id = state_data.get("a", "") + owner = state_data.get("o", "") + client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "") + client_secret = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET", "") + redirect_uri = ( + os.environ.get("GOOGLE_OAUTH_REDIRECT_URI") + or f"http://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback" + ) + import httpx as _httpx + try: + resp = _httpx.post("https://oauth2.googleapis.com/token", data={ + "code": code, + "client_id": client_id, + "client_secret": client_secret, + "redirect_uri": redirect_uri, + "grant_type": "authorization_code", + }, timeout=10) + resp.raise_for_status() + data = resp.json() + except Exception: + logger.warning("Google token exchange failed") + return _RR("/?section=integrations&email_oauth_error=token_exchange_failed") + access_token = data.get("access_token", "") + refresh_token = data.get("refresh_token", "") + expiry = str(int(time.time()) + data.get("expires_in", 3600)) + # Fetch the email address from userinfo so we can auto-fill imap_user. + email_addr = "" + display_name = "" + try: + ui = _httpx.get("https://www.googleapis.com/oauth2/v1/userinfo", + headers={"Authorization": f"Bearer {access_token}"}, timeout=10) + if ui.is_success: + ui_data = ui.json() + email_addr = ui_data.get("email", "") + display_name = ui_data.get("name", "") + except Exception: + pass + from core.database import SessionLocal, EmailAccount + from src.secret_storage import encrypt as _enc + db = SessionLocal() + try: + row = db.query(EmailAccount).filter(EmailAccount.id == account_id).first() + if not row: + return _RR("/?section=integrations&email_oauth_error=account_not_found") + # SECURITY: verify the account belongs to the initiating user. + if owner and row.owner and row.owner != owner: + logger.warning("OAuth callback owner mismatch — rejecting token write") + return _RR("/?section=integrations&email_oauth_error=ownership_error") + row.oauth_provider = "google" + row.oauth_access_token = _enc(access_token) + if refresh_token: + row.oauth_refresh_token = _enc(refresh_token) + row.oauth_token_expiry = expiry + # Auto-fill Google IMAP/SMTP settings if not already configured. + if not row.imap_host: + row.imap_host = "imap.gmail.com" + row.imap_port = 993 + row.imap_starttls = False + if not row.smtp_host: + row.smtp_host = "smtp.gmail.com" + row.smtp_port = 587 + if email_addr: + if not row.imap_user: + row.imap_user = email_addr + if not row.smtp_user: + row.smtp_user = email_addr + if not row.from_address: + row.from_address = email_addr + if not row.name or row.name == row.id: + row.name = email_addr + if display_name and not row.display_name: + row.display_name = display_name + db.commit() + finally: + db.close() + return _RR("/?section=integrations&email_oauth_success=1") + return router diff --git a/routes/embedding_routes.py b/routes/embedding_routes.py index a237e0b4c..62a459ae4 100644 --- a/routes/embedding_routes.py +++ b/routes/embedding_routes.py @@ -9,6 +9,7 @@ from pathlib import Path from fastapi import APIRouter, HTTPException, Form, Depends from core.constants import EMBEDDING_ENDPOINT_FILE, FASTEMBED_CACHE_DIR from core.middleware import require_admin +from src.runtime_paths import get_app_root logger = logging.getLogger(__name__) diff --git a/routes/gallery_routes.py b/routes/gallery_routes.py index 239281ac1..38bb51cdd 100644 --- a/routes/gallery_routes.py +++ b/routes/gallery_routes.py @@ -67,14 +67,6 @@ def _gallery_image_path(filename: str) -> Path: raise HTTPException(400, "Unsafe gallery filename") if safe_name != original: raise HTTPException(400, "Unsafe gallery filename") - if not path.exists(): - cwd_root = (Path.cwd() / "data" / "generated_images").resolve() - cwd_path = (cwd_root / safe_name).resolve() - try: - if os.path.commonpath([str(cwd_root), str(cwd_path)]) == str(cwd_root) and cwd_path.exists(): - return cwd_path - except Exception: - pass return path @@ -232,8 +224,6 @@ def setup_gallery_routes() -> APIRouter: @router.post("/api/gallery/{image_id}/replace") async def gallery_replace(request: Request, image_id: str): """Replace an existing gallery image file with a new one.""" - from pathlib import Path - user = get_current_user(request) db = SessionLocal() try: @@ -249,9 +239,8 @@ def setup_gallery_routes() -> APIRouter: raise HTTPException(400, "No image provided") content = await read_upload_limited(file, GALLERY_UPLOAD_MAX_BYTES, "Gallery replacement") - img_dir = Path(GENERATED_IMAGES_DIR) - img_dir.mkdir(parents=True, exist_ok=True) - img_path = img_dir / _sanitize_gallery_filename(img.filename) + GALLERY_IMAGE_DIR.mkdir(parents=True, exist_ok=True) + img_path = _gallery_image_path(img.filename) img_path.write_bytes(content) # Refresh dimensions in case the editor resized the canvas. diff --git a/routes/memory_routes.py b/routes/memory_routes.py index e788f82d2..d290046ec 100644 --- a/routes/memory_routes.py +++ b/routes/memory_routes.py @@ -273,65 +273,30 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM async def api_audit_memories(request: Request, session: str = Form(None)): """Deduplicate and consolidate memories via LLM. - Uses the default model from settings, or falls back to a session's model. + Uses task/utility/default settings through the shared resolver, with + the active session as fallback when no task or utility model is set. Returns before and after memory counts. """ - from routes.model_routes import _load_settings, _normalize_base, build_chat_url - from core.database import ModelEndpoint - import json as _json - - endpoint_url = model = None - headers = {} - - # Try utility model from settings first — memory audit is a background - # task and should prefer the lighter utility model over the main chat model. - from src.task_endpoint import resolve_task_endpoint user = _owner(request) - t_url, t_model, t_headers = resolve_task_endpoint(owner=user) - if t_url and t_model: - endpoint_url, model, headers = t_url, t_model, t_headers - else: - # Fall back to default model if no task/utility model configured - settings = _load_settings() - ep_id = settings.get("default_endpoint_id", "") - default_model = settings.get("default_model", "") - if ep_id: - db = SessionLocal() - try: - ep = db.query(ModelEndpoint).filter( - ModelEndpoint.id == ep_id, ModelEndpoint.is_enabled == True - ).first() - if ep: - base = _normalize_base(ep.base_url) - endpoint_url = build_chat_url(base) - model = default_model - if not model and ep.models: - try: - models = _json.loads(ep.models) if isinstance(ep.models, str) else ep.models - if models: - model = models[0] - except Exception: - pass - if ep.api_key: - headers = {"Authorization": f"Bearer {ep.api_key}"} - finally: - db.close() + fallback_url = fallback_model = None + fallback_headers = None + if session: + try: + sess = session_manager.get_session(session) + _assert_session_owner(sess, user) + fallback_url = sess.endpoint_url + fallback_model = sess.model + fallback_headers = sess.headers + except KeyError: + pass - # Fall back to session model if no default configured - if not endpoint_url and session: - try: - sess = session_manager.get_session(session) - _assert_session_owner(sess, _owner(request)) - endpoint_url = sess.endpoint_url - model = sess.model - headers = sess.headers - except KeyError: - pass + endpoint_url, model, headers = resolve_task_endpoint( + fallback_url, fallback_model, fallback_headers, owner=user + ) if not endpoint_url or not model: raise HTTPException(400, "No default model configured — set one in Settings") - user = _owner(request) result = await audit_memories( memory_manager, memory_vector, @@ -369,18 +334,28 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM model = None headers = {} + user = _owner(request) + if session: try: sess = session_manager.get_session(session) - _assert_session_owner(sess, _owner(request)) - endpoint_url, model, headers = resolve_task_endpoint( - sess.endpoint_url, sess.model, sess.headers, owner=_owner(request) - ) + _assert_session_owner(sess, user) except KeyError: - logger.warning("Session %s not found, falling back to utility endpoint", session) - endpoint_url, model, headers = resolve_endpoint("utility", owner=_owner(request)) + sess = None + except HTTPException as exc: + if exc.status_code != 404: + raise + sess = None + + if sess is None: + logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session) + endpoint_url, model, headers = resolve_endpoint("utility", owner=user) + else: + endpoint_url, model, headers = resolve_task_endpoint( + sess.endpoint_url, sess.model, sess.headers, owner=user + ) else: - endpoint_url, model, headers = resolve_task_endpoint(owner=_owner(request)) + endpoint_url, model, headers = resolve_task_endpoint(owner=user) if not endpoint_url or not model: raise HTTPException(400, "No LLM model configured. Set a default model in Settings.") diff --git a/routes/model_routes.py b/routes/model_routes.py index 69528f6dc..69e882d60 100644 --- a/routes/model_routes.py +++ b/routes/model_routes.py @@ -5,6 +5,7 @@ import re import uuid import json import hashlib +import ipaddress import socket import time as _time import logging @@ -26,7 +27,7 @@ from src.endpoint_resolver import ( build_models_url, build_headers, ) -from src.auth_helpers import _auth_disabled, owner_filter +from src.auth_helpers import _auth_disabled, effective_user, owner_filter logger = logging.getLogger(__name__) @@ -565,6 +566,8 @@ def _safe_build_models_url(base_url: str) -> str: """Build a /models URL without letting optional provider imports break probes.""" try: return build_models_url(base_url) + except ValueError: + raise except Exception as exc: logger.debug("Model URL detection failed for %s: %s", base_url, exc) return f"{(base_url or '').rstrip('/')}/models" @@ -636,7 +639,7 @@ def _probe_single_model(base: str, api_key: str, model_id: str, timeout: int = 1 try: t0 = _time.time() - r = httpx.post(target_url, headers=h, json=payload, timeout=timeout) + r = httpx.post(target_url, headers=h, json=payload, timeout=timeout, verify=llm_verify()) latency = round((_time.time() - t0) * 1000) if r.is_success: return {"status": "ok", "latency_ms": latency} @@ -662,13 +665,20 @@ def _probe_single_model(base: str, api_key: str, model_id: str, timeout: int = 1 # Hostnames / IP prefixes that indicate a local endpoint _LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"} -_PRIVATE_PREFIXES = ("10.", "172.16.", "172.17.", "172.18.", "172.19.", - "172.20.", "172.21.", "172.22.", "172.23.", "172.24.", - "172.25.", "172.26.", "172.27.", "172.28.", "172.29.", - "172.30.", "172.31.", "192.168.") +_PRIVATE_NETWORKS = ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), +) +_TAILSCALE_CGNAT = ipaddress.ip_network("100.64.0.0/10") -_TAILSCALE_RE = re.compile(r"^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\.") +def _local_ip_literal(host: str) -> bool: + try: + ip = ipaddress.ip_address(host) + except ValueError: + return False + return any(ip in network for network in _PRIVATE_NETWORKS) or ip in _TAILSCALE_CGNAT def _classify_endpoint(base_url: str, endpoint_kind: str = "auto") -> str: @@ -682,9 +692,7 @@ def _classify_endpoint(base_url: str, endpoint_kind: str = "auto") -> str: return "api" try: host = urlparse(base_url).hostname or "" - if host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES): - return "local" - if _TAILSCALE_RE.match(host): + if host in _LOCAL_HOSTS or _local_ip_literal(host): return "local" except Exception: pass @@ -1278,13 +1286,16 @@ def setup_model_routes(model_discovery): # Require auth; "" is the unconfigured single-user mode, treated as # "see everything" by _fetch_models. try: - from src.auth_helpers import get_current_user as _gcu - owner = _gcu(request) or "" - except Exception: - owner = "" - # Reject anonymous in configured deployments — no leaking the model - # list to unauthenticated callers. - try: + if getattr(request.state, "api_token", False): + scopes = set(getattr(request.state, "api_token_scopes", []) or []) + if "chat" not in scopes: + raise HTTPException(403, "API token is not scoped for chat") + if not getattr(request.state, "api_token_owner", None): + raise HTTPException(403, "API token has no owner") + owner = effective_user(request) or "" + + # Reject anonymous in configured deployments — no leaking the model + # list to unauthenticated callers. auth_mgr = getattr(request.app.state, "auth_manager", None) if not owner and not _auth_disabled() and auth_mgr is not None and getattr(auth_mgr, "is_configured", False): raise HTTPException(401, "Not authenticated") diff --git a/routes/note_routes.py b/routes/note_routes.py index 0d06d9484..c4674e489 100644 --- a/routes/note_routes.py +++ b/routes/note_routes.py @@ -10,7 +10,8 @@ from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel from core.database import SessionLocal, Note -from src.auth_helpers import get_current_user +from core.middleware import INTERNAL_TOOL_USER +from src.auth_helpers import require_user from src.constants import DATA_DIR from sqlalchemy.orm.attributes import flag_modified @@ -570,10 +571,19 @@ def setup_note_routes(task_scheduler=None): router = APIRouter(prefix="/api/notes", tags=["notes"]) def _owner(request: Request) -> Optional[str]: - return get_current_user(request) + # require_user, not bare get_current_user: a request that reaches + # these owner-scoped routes with NO identity (auth-middleware + # regression, SSRF from a sibling service) must fail closed (401) + # when auth is configured — not be treated as the single-user mode + # and handed blanket access to every account's notes. The documented + # anonymous modes (AUTH_ENABLED=false, LOCALHOST_BYPASS on loopback, + # unconfigured first-run) still resolve to None, the single-user + # path. fire_reminder below already gated this way; the CRUD routes + # did not. + return require_user(request) or None def _is_admin_or_single_user(request: Request, user: str | None) -> bool: - if user == "internal-tool": + if user == INTERNAL_TOOL_USER: return True if not user: # require_user() already admitted this request, which only happens @@ -805,8 +815,7 @@ def setup_note_routes(task_scheduler=None): Returns {synthesis, email_sent}. """ # Gate against anonymous callers — LLM synthesis can burn tokens. - from src.auth_helpers import require_user as _ru - user = _ru(request) + user = require_user(request) body = await request.json() note_id = str(body.get("note_id") or "").strip() if not note_id: diff --git a/routes/personal_routes.py b/routes/personal_routes.py index a078e580c..a42615be7 100644 --- a/routes/personal_routes.py +++ b/routes/personal_routes.py @@ -2,8 +2,9 @@ """Routes for personal documents management.""" import os import logging +import shutil import uuid -from typing import List, Tuple +from typing import Any, Dict, List, Tuple from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Depends from src.request_models import DirectoryRequest from core.constants import BASE_DIR, PERSONAL_DIR, PERSONAL_UPLOADS_DIR @@ -18,14 +19,15 @@ UPLOADS_DIR = PERSONAL_UPLOADS_DIR logger = logging.getLogger(__name__) -def _personal_upload_dir_for_owner(owner: str | None) -> str: +def _personal_upload_dir_for_owner(owner: str | None, *, create: bool = True) -> str: """Return the per-owner upload directory used for direct RAG uploads.""" owner_segment = secure_filename((owner or "local").strip())[:80] or "local" upload_dir = os.path.abspath(os.path.join(UPLOADS_DIR, owner_segment)) base_abs = os.path.abspath(UPLOADS_DIR) if os.path.commonpath([upload_dir, base_abs]) != base_abs: raise ValueError("Unsafe upload owner path") - os.makedirs(upload_dir, exist_ok=True) + if create: + os.makedirs(upload_dir, exist_ok=True) return upload_dir @@ -44,6 +46,87 @@ def _unique_personal_upload_path(upload_dir: str, original_name: str | None) -> raise ValueError("Unsafe upload filename") return file_path, filename, safe_name + +def _unique_existing_target(path: str) -> str: + """Return a non-existing sibling path for rename collision handling.""" + if not os.path.exists(path): + return path + stem, ext = os.path.splitext(path) + while True: + candidate = f"{stem}-{uuid.uuid4().hex[:10]}{ext}" + if not os.path.exists(candidate): + return candidate + + +def _remove_empty_tree(path: str) -> None: + """Best-effort removal of empty directories under ``path``.""" + if not os.path.isdir(path): + return + for root, dirs, _files in os.walk(path, topdown=False): + for dirname in dirs: + candidate = os.path.join(root, dirname) + try: + os.rmdir(candidate) + except OSError: + pass + try: + os.rmdir(path) + except OSError: + pass + + +def rename_personal_upload_owner( + old_owner: str, + new_owner: str, + *, + personal_docs_manager: Any = None, + rag_manager: Any = None, +) -> Dict[str, Any]: + """Move direct personal uploads and rewrite RAG owner metadata on user rename.""" + old_dir = _personal_upload_dir_for_owner(old_owner, create=False) + new_dir = _personal_upload_dir_for_owner(new_owner, create=False) + path_map: Dict[str, str] = {} + moved_files = 0 + + if os.path.isdir(old_dir) and old_dir != new_dir: + os.makedirs(new_dir, exist_ok=True) + for root, _dirs, files in os.walk(old_dir): + rel_root = os.path.relpath(root, old_dir) + target_root = new_dir if rel_root == "." else os.path.join(new_dir, rel_root) + os.makedirs(target_root, exist_ok=True) + for filename in files: + source = os.path.abspath(os.path.join(root, filename)) + target = _unique_existing_target(os.path.abspath(os.path.join(target_root, filename))) + shutil.move(source, target) + path_map[source] = target + moved_files += 1 + _remove_empty_tree(old_dir) + + if personal_docs_manager is not None: + rename_directory = getattr(personal_docs_manager, "rename_directory", None) + if callable(rename_directory): + rename_directory(old_dir, new_dir, path_map=path_map) + + rag_result = None + if rag_manager is not None: + rename_owner = getattr(rag_manager, "rename_owner", None) + if callable(rename_owner): + rag_result = rename_owner( + old_owner, + new_owner, + path_map=path_map, + path_prefixes=[(old_dir, new_dir)], + ) + + return { + "old_dir": old_dir, + "new_dir": new_dir, + "moved_files": moved_files, + "path_map": path_map, + "rag_result": rag_result, + } + + def setup_personal_routes(personal_docs_manager, rag_manager, rag_available): """ Setup personal documents related routes. @@ -275,11 +358,13 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available): except Exception as e: logger.warning(f"RAG removal failed for {filepath}: {e}") - # Delete file from disk if it's in uploads dir + # Delete file from disk if it's in the caller's own uploads dir. + # Scope to the per-owner subdir, not the shared uploads root, so one + # admin can't delete another user's personal files by path. deleted_from_disk = False try: - abs_target = os.path.abspath(filepath) - base_abs = os.path.abspath(UPLOADS_DIR) + abs_target = os.path.realpath(filepath) + base_abs = os.path.realpath(_personal_upload_dir_for_owner(owner, create=False)) in_uploads = ( abs_target == base_abs or os.path.commonpath([abs_target, base_abs]) == base_abs diff --git a/routes/research_routes.py b/routes/research_routes.py index 1ef36bd75..889298f7d 100644 --- a/routes/research_routes.py +++ b/routes/research_routes.py @@ -12,8 +12,10 @@ from typing import Optional from fastapi import APIRouter, HTTPException, Query, Request from fastapi.responses import HTMLResponse, StreamingResponse from pydantic import BaseModel, Field +from core.middleware import INTERNAL_TOOL_USER from src.endpoint_resolver import resolve_endpoint from src.auth_helpers import _auth_disabled, get_current_user +from core.auth import RESERVED_USERNAMES from src.constants import DEEP_RESEARCH_DIR _SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$") @@ -385,9 +387,9 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: """Launch a research job from the dedicated panel.""" from src.auth_helpers import require_privilege user = require_privilege(request, "can_use_research") - if user == "internal-tool": + if user == INTERNAL_TOOL_USER: tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip() - if tool_owner and tool_owner not in {"internal-tool", "api", "demo", "system"}: + if tool_owner and tool_owner not in RESERVED_USERNAMES: auth_mgr = getattr(request.app.state, "auth_manager", None) if auth_mgr is not None and getattr(auth_mgr, "is_configured", False): try: diff --git a/routes/session_routes.py b/routes/session_routes.py index 1fb2a487a..19b897f29 100644 --- a/routes/session_routes.py +++ b/routes/session_routes.py @@ -11,7 +11,7 @@ from core.session_manager import SessionManager from core.models import ChatMessage from src.request_models import SessionResponse from core.database import Session as DbSession, SessionLocal, Document, GalleryImage, utcnow_naive -from src.auth_helpers import get_current_user, effective_user, _auth_disabled, owner_filter +from src.auth_helpers import effective_user, _auth_disabled, owner_filter from src.session_actions import is_session_recently_active @@ -328,7 +328,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ endpoint_id: str = Form(""), ): skip_val = str(skip_validation).lower() == "true" - user = get_current_user(request) + user = effective_user(request) endpoint_api_key = "" endpoint_base_url = "" _reject_raw_endpoint_url_for_non_admin(request, user, endpoint_id, endpoint_url) @@ -477,7 +477,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ db.close() # Switch model/endpoint mid-session if model is not None and endpoint_url is not None: - user = get_current_user(request) + user = effective_user(request) _reject_raw_endpoint_url_for_non_admin(request, user, endpoint_id, endpoint_url) endpoint_api_key = "" endpoint_base_url = "" @@ -1004,6 +1004,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ """ from src.llm_core import llm_call user = effective_user(request) + single_user_mode = not user and _auth_disabled() user_sessions = session_manager.get_sessions_for_user(user) # Delete empty and throwaway sessions before sorting @@ -1022,7 +1023,12 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ } _THROWAWAY_MAX_MESSAGES = 4 # only delete if <= this many messages try: - rows = db.query(DbSession).filter(DbSession.archived == False, DbSession.owner == user).limit(2000).all() + rows_q = db.query(DbSession).filter(DbSession.archived == False) + if user: + rows_q = rows_q.filter(DbSession.owner == user) + elif not single_user_mode: + rows_q = rows_q.filter(DbSession.owner == user) + rows = rows_q.limit(2000).all() folder_map = {r.id: r.folder for r in rows} # Precompute per-session message counts in TWO aggregate queries # instead of 1–3 queries PER session — with many chats the per-row @@ -1242,7 +1248,12 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ db = SessionLocal() try: for sid, folder_name in assignments.items(): - db_session = db.query(DbSession).filter(DbSession.id == sid, DbSession.owner == user).first() + db_session_q = db.query(DbSession).filter(DbSession.id == sid) + if user: + db_session_q = db_session_q.filter(DbSession.owner == user) + elif not single_user_mode: + db_session_q = db_session_q.filter(DbSession.owner == user) + db_session = db_session_q.first() if db_session: db_session.folder = folder_name db_session.updated_at = datetime.utcnow() diff --git a/routes/shell_routes.py b/routes/shell_routes.py index 245181832..52b39e3bb 100644 --- a/routes/shell_routes.py +++ b/routes/shell_routes.py @@ -15,6 +15,7 @@ from collections import namedtuple from pathlib import Path from typing import Dict, Any from core.platform_compat import IS_APPLE_SILICON, which_tool +from core.middleware import INTERNAL_TOOL_USER from src.optional_deps import prepare_optional_dependency_import # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist @@ -55,7 +56,7 @@ def _require_admin(request: Request): # In-process tool loopback. The AuthMiddleware already validated the # internal token + loopback client before setting this marker, so # honour it here as admin-equivalent. - if user == "internal-tool": + if user == INTERNAL_TOOL_USER: return if not user or user == "api": raise HTTPException(403, "Admin only") diff --git a/routes/task_routes.py b/routes/task_routes.py index 5698353bf..d38040fde 100644 --- a/routes/task_routes.py +++ b/routes/task_routes.py @@ -11,6 +11,7 @@ from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel from core.database import SessionLocal, ScheduledTask, TaskRun +from core.middleware import INTERNAL_TOOL_USER from core.constants import internal_api_base from src.auth_helpers import get_current_user from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR @@ -427,7 +428,7 @@ def setup_task_routes(task_scheduler) -> APIRouter: # In-process tool-loopback marker — AuthMiddleware validated # the internal token + loopback client before stamping this, # so treat as admin-equivalent. - if user == "internal-tool": + if user == INTERNAL_TOOL_USER: return True try: from core.auth import AuthManager diff --git a/routes/upload_routes.py b/routes/upload_routes.py index 489e4923a..1e197dd49 100644 --- a/routes/upload_routes.py +++ b/routes/upload_routes.py @@ -7,7 +7,7 @@ from fastapi import APIRouter, Request, File, UploadFile, HTTPException from typing import List import logging from core.middleware import require_admin -from src.auth_helpers import get_current_user +from src.auth_helpers import effective_user from src.upload_handler import count_recent_uploads logger = logging.getLogger(__name__) @@ -78,7 +78,7 @@ def setup_upload_routes(upload_handler): for u in files: try: - meta = upload_handler.save_upload(u, client_ip, owner=get_current_user(request)) + meta = upload_handler.save_upload(u, client_ip, owner=effective_user(request)) out.append({ "id": meta["id"], "name": meta["name"], @@ -138,7 +138,7 @@ def setup_upload_routes(upload_handler): original_name = info.get("name", file_id) auth_mgr = getattr(request.app.state, "auth_manager", None) auth_configured = bool(auth_mgr and auth_mgr.is_configured) - current_user = get_current_user(request) + current_user = effective_user(request) file_owner = info.get("owner") if info else None if auth_configured: if not current_user: @@ -204,7 +204,7 @@ def setup_upload_routes(upload_handler): info = _load_upload_info(file_id) auth_mgr = getattr(request.app.state, "auth_manager", None) auth_configured = bool(auth_mgr and auth_mgr.is_configured) - current_user = get_current_user(request) + current_user = effective_user(request) file_owner = info.get("owner") if info else None if auth_configured: if not current_user: @@ -247,7 +247,7 @@ def setup_upload_routes(upload_handler): raise HTTPException(404, "File not found") auth_mgr = getattr(request.app.state, "auth_manager", None) auth_configured = bool(auth_mgr and auth_mgr.is_configured) - current_user = get_current_user(request) + current_user = effective_user(request) file_owner = info.get("owner") if auth_configured: if not current_user: diff --git a/routes/webhook_routes.py b/routes/webhook_routes.py index 77902c24b..c9cf856ca 100644 --- a/routes/webhook_routes.py +++ b/routes/webhook_routes.py @@ -1,6 +1,5 @@ """Webhook, API Token, and sync chat routes.""" -import asyncio import uuid import logging from typing import Optional @@ -385,10 +384,10 @@ def setup_webhook_routes( sess.add_message(ChatMessage("assistant", reply)) session_manager.save_sessions() - asyncio.create_task(webhook_manager.fire("chat.completed", { + webhook_manager.fire_and_forget("chat.completed", { "session_id": session_id, "model": sess.model, "user_message": message[:2000], "response": reply[:2000], - })) + }) return {"response": reply, "session_id": session_id, "model": sess.model} diff --git a/scripts/odysseus-calendar b/scripts/odysseus-calendar index 562551040..5a5f345bc 100755 --- a/scripts/odysseus-calendar +++ b/scripts/odysseus-calendar @@ -103,9 +103,13 @@ def cmd_list(args) -> None: end = _parse_dt(args.end) if args.end else (start + timedelta(days=30)) db = SessionLocal() try: + # Overlap semantics, matching the web route (routes/calendar_routes.py) + # and the recurring-expansion contract: an event is in the window when + # it starts before the window end AND ends after the window start. This + # includes multi-day / in-progress events that began before `start`. q = db.query(CalendarEvent).filter( - CalendarEvent.dtstart >= start, CalendarEvent.dtstart < end, + CalendarEvent.dtend > start, ) if args.calendar: cal = db.query(CalendarCal).filter(CalendarCal.name == args.calendar).first() diff --git a/services/hwfit/fit.py b/services/hwfit/fit.py index 7a3d4c4f2..a5a49a7ff 100644 --- a/services/hwfit/fit.py +++ b/services/hwfit/fit.py @@ -19,6 +19,10 @@ GPU_BANDWIDTH = { "6950 xt": 576, "6900 xt": 512, "6800 xt": 512, "6800": 512, "6700 xt": 384, "6600 xt": 256, "6600": 224, "mi300x": 5300, "mi300": 5300, "mi250x": 3277, "mi250": 3277, "mi210": 1638, "mi100": 1229, "9070 xt": 624, "9070": 488, "9060 xt": 322, "9060": 322, + # NVIDIA GB10 Grace-Blackwell superchip (DGX Spark). Unified LPDDR5X memory, + # not Apple Silicon, so it lives in the generic GPU table — the Apple-only + # lookup never matches it (its name carries no "apple"). + "gb10": 273, } # Pre-sort keys by length descending for correct substring matching @@ -126,6 +130,44 @@ def _lookup_bandwidth(system): return None +def _canonical_cpu_backend(system): + """Return the canonical CPU backend for cpu_only speed estimation. + + Normalizes CPU-architecture aliases separately from the GPU backend, and + overrides GPU-only backends (CUDA/ROCm/Metal) so they do not inherit a + discrete-GPU fallback constant when the model is actually running on CPU. + """ + backend = (system.get("backend") or "").lower().strip() + cpu_arch = (system.get("cpu_arch") or "").lower().strip() + cpu_name = (system.get("cpu_name") or "").lower() + gpu_name = (system.get("gpu_name") or "").lower() + + # Already-canonical CPU backends + if backend in ("cpu_x86", "cpu_arm"): + return backend + + # Raw CPU-architecture aliases. Treat plain "arm" as 32-bit ARM, not the + # ARM64-class CPU fallback used for Apple Silicon/aarch64 machines. + if backend in ("x86_64", "amd64", "i386", "i686"): + return "cpu_x86" + if backend in ("arm64", "aarch64"): + return "cpu_arm" + + # Prefer an explicit CPU architecture field when present + if cpu_arch: + if cpu_arch in ("x86_64", "amd64", "x86", "i386", "i686"): + return "cpu_x86" + if cpu_arch in ("arm64", "aarch64"): + return "cpu_arm" + + # Apple Silicon enters ranking as backend="metal"; its CPU path is ARM. + if backend in ("metal", "mps", "apple") or "apple" in cpu_name or "apple" in gpu_name: + return "cpu_arm" + + # Conservative default for CUDA/ROCm/discrete GPU backends and unknowns. + return "cpu_x86" + + def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0): """Estimate tok/s. Uses active params for MoE (only active experts run per token). @@ -143,6 +185,11 @@ def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0): bw = _lookup_bandwidth(system) backend = system.get("backend", "cpu_x86") + # CPU-only inference must never inherit a GPU backend's fallback constant, + # even if the detected system happens to report a CUDA/Metal/ROCm backend. + if run_mode == "cpu_only": + backend = _canonical_cpu_backend(system) + if bw and run_mode in ("gpu", "cpu_offload"): bpp = QUANT_BYTES_PER_PARAM.get(quant, 0.5) model_gb = pb * bpp diff --git a/services/hwfit/hardware.py b/services/hwfit/hardware.py index 0473475ed..ddb53bb90 100644 --- a/services/hwfit/hardware.py +++ b/services/hwfit/hardware.py @@ -330,7 +330,7 @@ def _detect_apple_silicon(): # Only Apple Silicon (arm64) has a Metal GPU worth serving LLMs on; Intel # Macs fall through to the CPU path. - if "arm" not in arch and "aarch64" not in arch: + if _canonical_cpu_arch(arch) != "arm64": return None # Chip name, e.g. "Apple M4 Max" — carries the Pro/Max/Ultra variant that @@ -513,6 +513,25 @@ def _get_cpu_count(): return os.cpu_count() or 1 +def _canonical_cpu_arch(value): + arch = str(value or "").lower().strip().replace("-", "_") + if arch in ("x86_64", "amd64", "x64"): + return "x86_64" + if arch in ("i386", "i686", "x86"): + return "x86" + if arch in ("arm64", "aarch64"): + return "arm64" + if arch == "arm" or arch.startswith("armv"): + return "arm" + return arch + + +def _get_cpu_arch(): + if _remote_host: + return _canonical_cpu_arch(_run(["uname", "-m"]) or "") + return _canonical_cpu_arch(platform.machine()) + + def _powershell_exe(): """Pick the best PowerShell executable for LOCAL execution: prefer pwsh (PowerShell 7+), fall back to Windows PowerShell 5.1. Returns an absolute @@ -538,6 +557,7 @@ def _detect_windows(): $r.cpu_name = $cpu.Name $r.cpu_cores = (Get-CimInstance Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum $r.arch = $cpu.AddressWidth + $r.cpu_arch = if ($env:PROCESSOR_ARCHITEW6432) { $env:PROCESSOR_ARCHITEW6432 } else { $env:PROCESSOR_ARCHITECTURE } # GPU detection via nvidia-smi (fastest) or WMI fallback try { $nv = nvidia-smi --query-gpu=memory.total,name --format=csv,noheader,nounits 2>$null @@ -609,6 +629,7 @@ def _detect_windows(): "available_ram_gb": d.get("avail_gb", 0), "cpu_cores": _as_int(d.get("cpu_cores"), 1), "cpu_name": _cpu_name, + "cpu_arch": _canonical_cpu_arch(d.get("cpu_arch")), "has_gpu": bool(d.get("gpu_name")), "gpu_name": d.get("gpu_name"), "gpu_vram_gb": d.get("gpu_vram_gb"), @@ -804,6 +825,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False): available_ram = round(_get_available_ram_gb(), 1) cpu_cores = _get_cpu_count() cpu_name = _get_cpu_name() + cpu_arch = _get_cpu_arch() gpu_info = _detect_apple_silicon() or _detect_nvidia() or _detect_amd() @@ -813,6 +835,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False): "available_ram_gb": available_ram, "cpu_cores": cpu_cores, "cpu_name": cpu_name, + "cpu_arch": cpu_arch, "has_gpu": True, "gpu_name": gpu_info["gpu_name"], "gpu_vram_gb": gpu_info["gpu_vram_gb"], @@ -827,17 +850,13 @@ def detect_system(host="", ssh_port="", platform="", fresh=False): "unified_memory": gpu_info.get("unified_memory", False), } else: - if _remote_host: - arch_out = _run(["uname", "-m"]) or "" - else: - import platform as _platform - arch_out = _platform.machine().lower() - backend = "cpu_arm" if "aarch64" in arch_out or "arm" in arch_out else "cpu_x86" + backend = "cpu_arm" if cpu_arch == "arm64" else "cpu_x86" result = { "total_ram_gb": total_ram, "available_ram_gb": available_ram, "cpu_cores": cpu_cores, "cpu_name": cpu_name, + "cpu_arch": cpu_arch, "has_gpu": False, "gpu_name": None, "gpu_vram_gb": None, diff --git a/services/search/content.py b/services/search/content.py index ac9b4a99c..49d050a4f 100644 --- a/services/search/content.py +++ b/services/search/content.py @@ -15,6 +15,8 @@ from urllib.parse import urljoin, urlparse import httpx from bs4 import BeautifulSoup +from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT + from .analytics import RateLimitError, error_logger from .cache import ( CONTENT_CACHE_DIR, @@ -89,18 +91,128 @@ def _public_http_url(url: str) -> bool: return False -def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5) -> httpx.Response: +class BodyTooLargeError(Exception): + """The server declared a body larger than the hard fetch ceiling.""" + + def __init__(self, url: str, declared_bytes: int): + self.url = url + self.declared_bytes = declared_bytes + super().__init__( + f"response body is {declared_bytes:,} bytes, over the " + f"{WEB_FETCH_HARD_MAX_BYTES:,}-byte hard cap" + ) + + +class _CappedFetch: + """Result of a size-capped streaming GET. + + Carries just what fetch_webpage_content needs from an httpx.Response, + plus the cap bookkeeping: the (possibly truncated) body, whether the + cap cut it short, and the size the server declared via Content-Length + (wire bytes; None when absent). + """ + + __slots__ = ("status_code", "headers", "content", "truncated", + "declared_bytes", "encoding", "url") + + def __init__(self, status_code, headers, content, truncated, + declared_bytes, encoding, url): + self.status_code = status_code + self.headers = headers + self.content = content + self.truncated = truncated + self.declared_bytes = declared_bytes + self.encoding = encoding + self.url = url + + @property + def text(self) -> str: + return self.content.decode(self.encoding or "utf-8", errors="replace") + + def raise_for_status(self): + if self.status_code >= 400: + request = httpx.Request("GET", self.url) + raise httpx.HTTPStatusError( + f"HTTP {self.status_code} for {self.url}", + request=request, + response=httpx.Response(self.status_code, request=request), + ) + + +def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5, + max_bytes: int = None) -> "_CappedFetch": + """Capped streaming GET with SSRF-guarded manual redirects. + + The body is streamed and buffering stops at ``max_bytes`` (default: the + soft cap), so an oversized resource cannot be pulled into memory or the + content cache in full. When Content-Length already declares a body over + the hard ceiling, the fetch is refused before any body bytes are read. + """ + cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES) current = url for _ in range(max_redirects + 1): if not _public_http_url(current): raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current)) - response = httpx.get(current, headers=headers, timeout=timeout, follow_redirects=False) - if response.status_code not in (301, 302, 303, 307, 308): - return response - location = response.headers.get("location") - if not location: - return response - current = urljoin(str(response.url), location) + # Force identity transfer-encoding. With gzip/deflate the wire bytes + # (and Content-Length) can be a small fraction of the decoded body, so + # a tiny compressed response could pass the hard-cap preflight and then + # expand past the ceiling in a single decoded chunk before the streamed + # cap below can slice it. Identity makes Content-Length the true body + # size and keeps each streamed chunk bounded by the network read. + req_headers = dict(headers or {}) + req_headers["Accept-Encoding"] = "identity" + with httpx.stream("GET", current, headers=req_headers, timeout=timeout, + follow_redirects=False) as response: + if response.status_code in (301, 302, 303, 307, 308): + location = response.headers.get("location") + if not location: + return _CappedFetch(response.status_code, response.headers, b"", + False, None, response.encoding, str(response.url)) + current = urljoin(str(response.url), location) + continue + + # A server can ignore the identity request and still return a + # compressed body; httpx.iter_bytes would then decode it, and a tiny + # gzip can balloon into one decoded chunk far past the cap before we + # slice. Refuse a compressed Content-Encoding so the streamed cap + # stays a real memory bound (Content-Length is the compressed wire + # length here, so the preflight and size metadata are unreliable too). + enc = (response.headers.get("content-encoding") or "").strip().lower() + if enc and enc != "identity": + raise httpx.RequestError( + f"Refusing compressed response (Content-Encoding: {enc}) after " + "requesting identity: cannot bound decoded body size", + request=httpx.Request("GET", current), + ) + + declared = None + raw_len = response.headers.get("content-length") + if raw_len and raw_len.isdigit(): + declared = int(raw_len) + # Refuse before buffering anything when the server already tells + # us the body exceeds the absolute ceiling (Content-Length is wire + # bytes; the decompressed body can only be larger). + if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES: + raise BodyTooLargeError(current, declared) + + chunks = [] + read = 0 + truncated = False + # We requested identity above, so iter_bytes yields the raw body in + # network-read-sized chunks (no decompression expansion); the cap + # therefore bounds what we actually buffer. + for chunk in response.iter_bytes(): + read += len(chunk) + if read > cap: + keep = cap - (read - len(chunk)) + if keep > 0: + chunks.append(chunk[:keep]) + truncated = True + break + chunks.append(chunk) + return _CappedFetch(response.status_code, response.headers, + b"".join(chunks), truncated, declared, + response.encoding, str(response.url)) raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current)) # PDF extraction (optional dependency) @@ -222,9 +334,19 @@ def _empty_result(url: str, error: str = "") -> dict: # ---------------------------------------------------------------------- # Main content fetcher # ---------------------------------------------------------------------- -def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> dict: - """Fetch and extract meaningful content from a webpage with caching.""" - cache_key = generate_cache_key(url) +def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0, + max_bytes: int = None) -> dict: + """Fetch and extract meaningful content from a webpage with caching. + + ``max_bytes`` raises the download budget per call (clamped to the hard + cap); the default is the soft cap. When the body is cut short the result + carries ``truncated``/``fetched_bytes``/``total_bytes`` so callers can + tell the model the content is partial (#3812). + """ + effective_cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES) + # The cap is part of the cache identity: a truncated soft-cap fetch must + # not be served to a later full-budget request for the same URL. + cache_key = generate_cache_key(f"{url}#cap={effective_cap}") cache_file = CONTENT_CACHE_DIR / f"{cache_key}.cache" # Check cache @@ -247,18 +369,24 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> # Fetch try: headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "User-Agent": WEB_FETCH_USER_AGENT, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", - "Accept-Encoding": "gzip, deflate", + # identity so the streamed size cap in _get_public_url stays honest + # (a compressed body can decode to far more than Content-Length). + "Accept-Encoding": "identity", "Connection": "keep-alive", } - response = _get_public_url(url, headers=headers, timeout=timeout) + response = _get_public_url(url, headers=headers, timeout=timeout, + max_bytes=effective_cap) if response.status_code == 429: raise RateLimitError(f"Rate limit hit for {url} (attempt {retry_attempt})") response.raise_for_status() + except BodyTooLargeError as e: + error_logger.warning(f"Refused oversized body for {url}: {e}") + return _empty_result(url, f"TooLarge: {e}") except httpx.HTTPStatusError as e: error_logger.warning(f"HTTP {e.response.status_code} fetching {url}: {e}") return _empty_result(url, f"HTTP {e.response.status_code}: {e}") @@ -269,9 +397,27 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> error_logger.error(str(e)) return _empty_result(url, str(e)) + # Size bookkeeping shared by every content branch below. getattr keeps + # plain httpx.Response stand-ins (tests) working without the cap fields. + _size_fields = { + "truncated": getattr(response, "truncated", False), + "fetched_bytes": len(response.content), + "total_bytes": getattr(response, "declared_bytes", None), + } + # PDF handling content_type = response.headers.get("Content-Type", "").lower() if "application/pdf" in content_type or url.lower().endswith(".pdf"): + if _size_fields["truncated"]: + # A PDF cut mid-stream is not parseable; unlike text there is no + # useful partial result, so report the budget problem instead. + _declared = _size_fields["total_bytes"] + return _empty_result( + url, + f"TooLarge: PDF exceeds the {effective_cap:,}-byte fetch budget" + + (f" (size {_declared:,} bytes)" if _declared else "") + + "; retry with a larger budget if it fits under the hard cap", + ) if pdf_extract_text is None: logger.error("pdfminer.six is not installed; cannot extract PDF text.") pdf_text = "" @@ -295,6 +441,7 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> "js_message": "", "success": bool(pdf_text), "error": "" if pdf_text else "Failed to extract PDF text", + **_size_fields, } _cache_result(cache_file, cache_key, result, url) return result @@ -329,6 +476,7 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> "js_message": "", "success": bool(text_body), "error": "" if text_body else "Empty response body", + **_size_fields, } _cache_result(cache_file, cache_key, result, url) return result @@ -391,6 +539,7 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> "js_message": js_message, "success": True, "error": "", + **_size_fields, } _cache_result(cache_file, cache_key, result, url) return result diff --git a/services/search/providers.py b/services/search/providers.py index b913e1c6f..d0ca1b0de 100644 --- a/services/search/providers.py +++ b/services/search/providers.py @@ -9,14 +9,12 @@ from urllib.parse import urljoin, urlparse, parse_qs import httpx from bs4 import BeautifulSoup -from src.constants import SEARXNG_INSTANCE +from src.constants import SEARXNG_INSTANCE, REQUEST_TIMEOUT, WEB_FETCH_USER_AGENT from .analytics import RateLimitError, error_logger from .query import build_enhanced_query logger = logging.getLogger(__name__) -REQUEST_TIMEOUT = 20 - # Provider registry — maps setting value to (label, needs_key, needs_url) PROVIDER_INFO = { "searxng": ("SearXNG", False, True), @@ -140,7 +138,7 @@ def searxng_search_api(query: str, count: Optional[int] = None, categories: str count = count if count is not None else _get_result_count() instance = _get_search_instance() api_key = "" - headers = {"User-Agent": "Mozilla/5.0"} + headers = {"User-Agent": WEB_FETCH_USER_AGENT} if api_key: headers["Authorization"] = f"Bearer {api_key}" # News/fresh queries do badly in the 'general' category — it favours @@ -252,7 +250,7 @@ def searxng_search(query, max_results=10): """Search using SearXNG instance - parsing HTML.""" instance = _get_search_instance() api_key = "" - req_headers = {"User-Agent": "Mozilla/5.0"} + req_headers = {"User-Agent": WEB_FETCH_USER_AGENT} if api_key: req_headers["Authorization"] = f"Bearer {api_key}" try: @@ -391,7 +389,7 @@ def duckduckgo_search(query: str, count: Optional[int] = None, time_filter: Opti response = httpx.get( "https://html.duckduckgo.com/html/", params={"q": query, "kp": _safesearch_for("duckduckgo_html")}, - headers={"User-Agent": "Mozilla/5.0"}, + headers={"User-Agent": WEB_FETCH_USER_AGENT}, timeout=REQUEST_TIMEOUT, ) response.raise_for_status() diff --git a/setup.py b/setup.py index 81fcc87ab..a9c565282 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,9 @@ sys.path.insert(0, BASE_DIR) from src.constants import ( DATA_DIR, AUTH_FILE, UPLOAD_DIR, PERSONAL_DIR, PERSONAL_UPLOADS_DIR, TTS_CACHE_DIR, GENERATED_IMAGES_DIR, DEEP_RESEARCH_DIR, CHROMA_DIR, - RAG_DIR, MEMORY_VECTORS_DIR, + RAG_DIR, MEMORY_VECTORS_DIR, PASSWORD_MIN_LENGTH, ) +from core.auth import RESERVED_USERNAMES DIRS = [ DATA_DIR, @@ -59,15 +60,23 @@ def _prompt_admin_credentials(): print(" (Press Enter to accept defaults)") print() - username = input(" Username [admin]: ").strip().lower() - if not username: - username = "admin" + while True: + username = input(" Username [admin]: ").strip().lower() + if not username: + username = "admin" + if username in RESERVED_USERNAMES: + print(f" '{username}' is a reserved username. Choose another.") + continue + break while True: password = getpass.getpass(" Password: ") if not password: print(" Password cannot be empty.") continue + if len(password) < PASSWORD_MIN_LENGTH: + print(f" Password must be at least {PASSWORD_MIN_LENGTH} characters.") + continue confirm = getpass.getpass(" Confirm password: ") if password != confirm: print(" Passwords don't match. Try again.") @@ -93,8 +102,13 @@ def create_default_admin(): password = os.getenv("ODYSSEUS_ADMIN_PASSWORD", "").strip() if username and password: - # Both provided via env — use them directly - pass + # Both provided via env — validate before using + if username in RESERVED_USERNAMES: + print(f" [error] ODYSSEUS_ADMIN_USER '{username}' is a reserved username") + return "failed" + if len(password) < PASSWORD_MIN_LENGTH: + print(f" [error] ODYSSEUS_ADMIN_PASSWORD must be at least {PASSWORD_MIN_LENGTH} characters") + return "failed" elif sys.stdin.isatty() and not os.getenv("ODYSSEUS_SKIP_ADMIN_PROMPT"): # Interactive terminal — ask the user username, password = _prompt_admin_credentials() diff --git a/specs/architecture-runtime-inventory.md b/specs/architecture-runtime-inventory.md new file mode 100644 index 000000000..1030b2bd0 --- /dev/null +++ b/specs/architecture-runtime-inventory.md @@ -0,0 +1,412 @@ +# Architecture Runtime Inventory + +> **Purpose**: Phase 0 planning baseline for codebase readability improvements (#4071). +> **Parent issue**: [#4082](https://github.com/pewdiepie-archdaemon/odysseus/issues/4082) +> **Last updated**: dev@b58af42 | 2026-06-16 +> **Status**: Draft — to be reviewed before follow-up slices open. +> **Snapshot basis**: Importer / file / import-line counts are refreshed to `dev@b58af42` (2026-06-16) and are recomputable via the commands in §3.4. **Line counts** in §2.1 / §2.2 are a snapshot from an earlier baseline and drift as `dev` moves — recompute any of them with `wc -l `. This inventory tracks structure and risk, not live metrics. + +This document maps the current runtime module structure, identifies high-risk boundaries, and recommends safe first refactor slices. It does **not** move files, change imports, or alter runtime behavior. + +--- + +## 1. Current Structure Overview + +### 1.1 Top-Level Layout + +``` +odysseus/ +├── app.py # FastAPI app entrypoint (1,145 lines) +├── conf/ # Configuration (config.py, settings.py, settings_scrub.py) +├── src/ # 95 flat .py files + 2 subdirectories +│ ├── agent_tools/ # Tool helpers: document, filesystem, subprocess, web +│ └── search/ # Search subsystem +├── routes/ # 54 flat .py files — HTTP route handlers +├── core/ # 10 files — database models, auth, middleware, session +├── mcp_servers/ # 5 files — MCP server implementations +├── scripts/ # CLI tools and one-shot scripts +├── static/ # Frontend HTML/CSS/JS +├── tests/ # 583 test files (~54,800 lines) +└── services/ # (exists as needed) +``` + +### 1.2 Directory Flatness Metric + +| Directory | Flat `.py` Files | Subdirectories | Concern | +|-----------|-----------------|----------------|---------| +| `src/` | **95** | 2 (`agent_tools/`, `search/`) | No domain grouping; 95 files in one directory | +| `routes/` | **54** | 0 | All route handlers in one flat directory | +| `core/` | 10 | 0 | Manageable, but `database.py` is oversized | + +--- + +## 2. Largest Runtime Modules + +### 2.1 Python Backend + +| Rank | File | Lines | Classes | Functions | Risk | +|------|------|-------|---------|-----------|------| +| 1 | `src/tool_implementations.py` | **4,032** | 0 | ~48 | **HIGH** | +| 2 | `routes/email_routes.py` | **3,245** | — | — | **MEDIUM** | +| 3 | `routes/cookbook_routes.py` | **2,969** | — | — | **MEDIUM** | +| 4 | `src/agent_loop.py` | **2,961** | 0 | ~24 | **HIGH** | +| 5 | `src/task_scheduler.py` | **2,330** | — | 5 | MEDIUM | +| 6 | `routes/model_routes.py` | **2,266** | — | — | MEDIUM | +| 7 | `core/database.py` | **2,265** | 28 | ~59 helpers | **HIGH** | +| 8 | `src/builtin_actions.py` | **2,262** | 2 | ~24 | MEDIUM | +| 9 | `src/llm_core.py` | **2,164** | — | — | MEDIUM | +| 10 | `mcp_servers/email_server.py` | 2,197 | — | — | LOW (separate process) | +| 11 | `src/visual_report.py` | 1,918 | — | — | LOW | +| 12 | `routes/gallery_routes.py` | 1,896 | — | — | LOW | +| 13 | `src/ai_interaction.py` | 1,846 | — | — | MEDIUM | +| 14 | `routes/document_routes.py` | 1,717 | — | — | LOW | +| 15 | `routes/skills_routes.py` | 1,648 | — | — | LOW | + +**Heuristic**: Files > 2,000 lines with 20+ public symbols and many importers are the highest-risk splits. Files 1,000–2,000 lines are medium-risk if tightly coupled. + +### 2.2 Frontend + +| File | Lines | Concern | +|------|-------|---------| +| `static/style.css` | **36,653** | Entire app CSS in one file (tracked separately in #2617) | +| `static/js/document.js` | **9,776** | Single JS file for document functionality | +| `static/js/slashCommands.js` | 6,498 | | +| `static/js/settings.js` | 5,266 | | +| `static/js/emailLibrary.js` | 5,217 | | +| `static/js/notes.js` | 5,124 | | +| `static/js/chat.js` | 4,985 | | +| `static/app.js` | 4,090 | | + +**Note**: Frontend modularization is tracked separately in #2617 (CSS) and is not the focus of this Phase 0 inventory. Frontend is listed here for completeness but follow-up slices should target Python backend boundaries first. + +--- + +## 3. Import Dependency Graph + +### 3.1 Who Depends on `core/database.py` + +**102 files** import from `core.database` — this is the most depended-upon module: + +- All route handlers (`routes/*.py`) +- Most `src/*.py` files +- `core/session_manager.py`, `core/auth.py` +- Multiple test files + +**Implication**: Any split of `core/database.py` is the highest-risk refactor. It should be tackled **last**, never first. + +### 3.2 Who Depends on `src/tool_implementations.py` + +**17 files** import from `src.tool_implementations`: +- `src/agent_loop.py`, `src/builtin_actions.py`, `src/tool_index.py` +- `src/task_scheduler.py`, `src/tool_policy.py` +- Various tests + +### 3.3 Who Depends on `src/agent_loop.py` + +**22 files** import from `src.agent_loop`: + +- `src/tool_policy.py`, `src/teacher_escalation.py`, `src/bg_monitor.py` +- `src/task_scheduler.py` +- Multiple test files + +### 3.4 Cross-Layer Import Violations + +**`src/` importing from `routes/`** (backwards dependency — domain logic depending on HTTP layer): + +``` +src/tool_implementations.py ──→ routes/calendar_routes.py +src/tool_implementations.py ──→ routes/cookbook_helpers.py +src/tool_implementations.py ──→ routes/email_helpers.py +src/tool_implementations.py ──→ routes/email_pollers.py +src/tool_implementations.py ──→ routes/email_routes.py +src/tool_implementations.py ──→ routes/model_routes.py +src/tool_implementations.py ──→ routes/note_routes.py +src/tool_implementations.py ──→ routes/prefs_routes.py +``` + +> These are **runtime imports** (inside function bodies, not at module top), which mitigates circular import risk but indicates fuzzy layer boundaries. Function-level inline imports from the HTTP layer into business logic are a code smell. + +**Import counts (top-level)**: +| Direction | Count | Notes | +|-----------|-------|-------| +| `routes/` → `src/` | **374** | Expected: HTTP handlers call domain logic | +| `routes/` → `core/` | **126** | Expected: handlers access DB models | +| `src/` → `routes/` | **31** | **Unexpected**: domain logic reaching into HTTP layer (direct grep of import lines referencing `routes/`) | +| `src/` → `core/` | **106** | Acceptable but could be reduced with a data-access layer | + +> **How the metrics in this document are computed** — recompute against current `dev` before treating any count as authoritative (the tree drifts; these numbers are a snapshot, not a live value): +> - `src/` flat `.py` files: `find src -maxdepth 1 -name '*.py' | wc -l` +> - `tests/` test files: `find tests -name 'test_*.py' | wc -l` +> - `core.database` importers: `grep -rlE '(from|import) +core\.database' --include='*.py' . | grep -v core/database.py | wc -l` +> - `src.agent_loop` importers: `grep -rlE '(from|import) +src\.agent_loop' --include='*.py' . | grep -v src/agent_loop.py | wc -l` +> - Cross-layer import lines: `grep -rhE '(from|import) +' --include='*.py' / | wc -l` (e.g. `(from|import) +routes` over `src/`) + +--- + +## 4. Route Ownership Map + +Routes can be grouped into logical feature domains. Current flat structure obscures these boundaries: + +| Domain | Route Files | Total Lines | Review Complexity | +|--------|-------------|-------------|-------------------| +| **Email** | `email_routes.py`, `email_helpers.py`, `email_pollers.py` | 5,936 | HIGH — most complex domain | +| **Chat / Agent** | `chat_routes.py`, `chat_helpers.py`, `shell_routes.py`, `codex_routes.py`, `skills_routes.py` | 6,365 | HIGH — core interaction surface | +| **Cookbook** | `cookbook_routes.py`, `cookbook_helpers.py`, `cookbook_output.py` | 4,110 | MEDIUM | +| **Model / LLM** | `model_routes.py`, `assistant_routes.py`, `copilot_routes.py` | 2,764 | MEDIUM | +| **Calendar / Contacts** | `calendar_routes.py`, `contacts_routes.py` | 2,336 | MEDIUM | +| **Documents** | `document_routes.py`, `document_helpers.py` | 1,954 | LOW | +| **Auth** | `auth_routes.py`, `api_token_routes.py`, `device_flow.py` | 1,171 | LOW | +| **Tasks** | `task_routes.py` (standalone) | 1,157 | LOW | +| **Session** | `session_routes.py` (standalone) | 1,287 | LOW | +| **Gallery** | `gallery_routes.py`, `gallery_helpers.py` | 1,896 | LOW | +| **Memory** | `memory_routes.py` | — | LOW | +| **Research** | `research_routes.py` | — | LOW | +| **MCP** | `mcp_routes.py` | — | LOW | +| **Notes** | `note_routes.py` | — | LOW | +| **Other** | `prefs_routes.py`, `upload_routes.py`, `vault_routes.py`, `webhook_routes.py`, `workspace_routes.py`, `search_routes.py`, `history_routes.py`, `hwfit_routes.py`, `preset_routes.py`, `signature_routes.py`, `backup_routes.py`, `cleanup_routes.py`, `diagnostics_routes.py`, `embedding_routes.py`, `emoji_routes.py`, `font_routes.py`, `stt_routes.py`, `tts_routes.py`, `compare_routes.py`, `personal_routes.py`, `editor_draft_routes.py`, `admin_wipe_routes.py`, `chatgpt_subscription_routes.py` | 2,000+ | LOW individual, HIGH cumulative | + +--- + +## 5. Tool Registry & Implementation Boundaries + +### 5.1 Current Tool Architecture + +| Component | File | Lines | Role | +|-----------|------|-------|------| +| Tool schemas | `src/tool_schemas.py` | 1,392 | JSON Schema tool definitions (Duck-TypedDict) | +| Tool index | `src/tool_index.py` | 542 | RAG-based tool retrieval from ChromaDB | +| Tool implementations | `src/tool_implementations.py` | 4,032 | 33 `do_*` functions — all tool execution logic | +| Tool security | `src/tool_security.py` | — | Owner-scoped tool blocking | +| Tool policy | `src/tool_policy.py` | — | Guide-only directive, plan-mode disabled tools | +| Tool utils | `src/tool_utils.py` | — | Shared tool helpers | + +### 5.2 Tool Implementation Categories + +The 33 `do_*` functions in `tool_implementations.py` fall into natural domain groups — the basis for slice 1's split in §6.2: + +| Category | `do_*` functions | Count | +|----------|------------------|-------| +| **System / config** | `do_manage_skills`, `do_manage_tasks`, `do_manage_endpoints`, `do_manage_mcp`, `do_manage_webhooks`, `do_manage_tokens`, `do_manage_settings`, `do_api_call`, `do_app_api` | 9 | +| **Cookbook / model serving** | `do_download_model`, `do_serve_model`, `do_list_served_models`, `do_stop_served_model`, `do_tail_serve_output`, `do_list_downloads`, `do_cancel_download`, `do_search_hf_models`, `do_adopt_served_model`, `do_list_cookbook_servers`, `do_list_serve_presets`, `do_serve_preset`, `do_list_cached_models` | 13 | +| **Notes** | `do_manage_notes` | 1 | +| **Calendar** | `do_manage_calendar` | 1 | +| **Search** | `do_search_chats` | 1 | +| **Research** | `do_manage_research`, `do_trigger_research` | 2 | +| **Contacts** | `do_resolve_contact`, `do_manage_contact` | 2 | +| **Vault** | `do_vault_search`, `do_vault_get`, `do_vault_unlock` | 3 | +| **Image** | `do_edit_image` | 1 | +| | **Total** | **33** | + +> Low-level tools (filesystem, subprocess, web fetch, document parsing) live in `src/agent_tools/`, **not** in `tool_implementations.py` — out of scope for this split. + +--- + +## 6. Risk Assessment & Candidate Slice Ranking + +> **Candidate proposals, not a committed plan.** The rankings, package shapes (e.g. `src/pkg/`, `src/domain/`, `src/infra/`, `src/api/`), split ordering, and route-grouping strategy below are **options for maintainer discussion**. Per #4082/#4071, slice ownership and order are settled by maintainers before any follow-up PR. §1–§3 above are the factual current-state inventory. + +### 6.1 Risk Scale + +| Level | Criteria | +|-------|----------| +| **LOW** | File has ≤3 importers AND ≤500 lines, OR is a pure refactor with clear boundaries | +| **MEDIUM** | File has 4–15 importers OR 500–1,500 lines | +| **HIGH** | File has 16+ importers OR >2,000 lines, OR has cross-layer import violations | + +### 6.2 Ranked Split Candidates + +| Priority | Target | Risk | Rationale | +|----------|--------|------|-----------| +| **1** | `src/tool_implementations.py` → `src/tools/*.py` | **MEDIUM** | 4,032 lines → ~10 files by tool category. Already has natural boundaries. 17 importers, tracked in #3629. Use `__init__.py` shim to keep existing imports working. | +| **2** | `routes/` → domain subdirectories (one domain per PR) | **MEDIUM** | 54 flat files. Done **one domain at a time** (e.g. a standalone PR for the email domain, then chat, …), not a broad reorganization — route modules carry helper imports, registration assumptions, and test import paths. | +| **3** | `src/agent_loop.py` → `src/agent/loop.py` + submodules | **MEDIUM-HIGH** | 2,961 lines, 24 functions. Can extract prompt building, classification, verification, and runaway detection. Tracked in #3266. | +| **4** | `src/` → `src/pkg/`, `src/domain/`, `src/infra/`, `src/api/` | **MEDIUM** | Structural reorganization. Split flat `src/` into layered packages. Must come after routes and tools are stable. | +| **5** | `routes/email_*.py` consolidation | **LOW** | Already grouped by filename prefix. Low-risk cleanup within the email domain. | +| **6** | `core/database.py` → `src/infra/database/models/*.py` | **HIGH** | 28 classes, 102 importers. Highest-risk split. Must be **last** in any sequence. Requires careful import shim strategy. | +| **7** | Frontend CSS modularization | **MEDIUM** | 36,653 lines. Tracked in #2617. Separate timeline from backend work. | +| **8** | Frontend JS modularization | **MEDIUM** | 9,776 lines in `document.js`. Introduce ES modules at minimum. | + +### 6.3 Candidate First 3 Behavior-Preserving Slices + +**Slice 1: Split `tool_implementations.py`** (Lowest-risk high-impact) + +- Create `src/tools/` package with one file per tool category +- Add `src/tools/__init__.py` re-exporting all symbols with current names +- Update 17 importers to use new paths (can be deferred via shim) +- Validation: `python -m pytest tests/ -x -q` + manual smoke test of tool execution +- Reference: #3629 + +**Slice 2: Group `routes/` by domain** (one domain per PR, not a broad sweep) + +Route modules carry helper imports, router registration assumptions, and test import paths, so this must be done **one domain at a time** rather than as a single reorganization PR. Example sequence (each its own PR): + +- PR 2a: move the **email** domain (`email_routes.py`, `email_helpers.py`, `email_pollers.py`) → `routes/email/` + shim +- PR 2b: move the **chat/agent** domain → `routes/chat/` + shim +- PR 2c: move the **cookbook** domain → `routes/cookbook/` + shim +- …and so on per domain from §4 + +Each PR: add `__init__.py` re-exporting old names, update `app.py` router imports, validation `python app.py` starts clean. **No behavior change** — pure file reorganization. + +**Slice 3: Extract `agent_loop.py` submodules** (Improve reviewability) + +- Move prompt assembly → `src/agent/prompt.py` +- Move request classification → `src/agent/classifier.py` +- Move sub-agent verification → `src/agent/verifier.py` +- Move runaway detection → `src/agent/runaway.py` +- Move context management → `src/agent/context.py` +- Keep `src/agent/loop.py` as the main orchestration module +- Validation: `python -m pytest tests/test_agent_loop.py tests/test_loop_breaker_runaway.py -v` + +--- + +## 7. Safety Guardrails for Follow-Up Work + +Per maintainer guidance in #4082 and #4071: + +- [ ] **One domain/slice per PR** — never mix multiple reorganizations +- [ ] **No behavior changes** mixed with file moves — pure reorganization only +- [ ] **Keep compatibility shims** — `__init__.py` re-exports for all existing import paths +- [ ] **Add or identify focused tests** before risky splits +- [ ] **Do not start with `core/database.py`** or broad route movement unless this inventory shows a safe boundary +- [ ] **Prefer small, reviewable slices** over large restructures +- [ ] **No packaging/runtime/tooling migration** mixed into file moves +- [ ] **No frontend framework migration** inside this stabilization lane +- [ ] **Validate with `python -m compileall`** — every PR must pass CI checks +- [ ] **Validate with `pytest`** — run the full test suite before opening each PR + +--- + +## 8. Validation Commands + +Each follow-up PR should be verifiable with these commands before submission: + +```bash +# Syntax check — must pass with zero errors +python -m compileall src/ routes/ core/ conf/ + +# Full test suite — must match baseline pass rate +python -m pytest tests/ -x -q + +# Import shim verification — existing import paths must still work +python -c "from src.tool_implementations import do_search_chats; print('OK')" + +# App startup smoke test (if backend touched) +timeout 5 python app.py 2>&1 | head -5 || true +``` + +--- + +## 9. Open Questions + +1. Is `#2538` (specs ground truth) the canonical behavior map baseline, and should this inventory be kept in sync with those specs once merged? +2. Should route grouping follow the domain map proposed here, or is there a different taxonomy preferred by maintainers? +3. For the `tool_implementations.py` split (#3629), is the tool categorization in §5.2 acceptable, or should it follow a different grouping? +4. Should compatibility shims (`__init__.py`) be temporary (removed in a follow-up wave) or permanent? +5. Should an ADR (Architecture Decision Record) document be started to track decisions made during this process? + +--- + +## 10. Future Direction (NOT current state) + +The following are **future refactor targets** (candidate directions **pending maintainer agreement**, not committed), recorded here so this inventory does not imply they exist today. None of them are present in the current `dev` tree: + +- `main.py` — proposed rename of the `app.py` entrypoint. Today the app boots via `app.py`. +- `src/agent/` — proposed package to hold `agent_loop.py` submodules (prompt/classifier/verifier/runaway/context). Today `agent_loop.py` is a single flat file in `src/`. +- `src/infra/`, `src/domain/`, `src/pkg/`, `src/api/` — proposed layered reorganization of the flat `src/` directory (slice 4 in §6). + +These become real only when the corresponding slices land. + +--- + +## Appendix A: File Listing + +### `src/` (95 files — 61 shown; run `ls src/*.py` for the full list) + +``` +agent_loop.py tool_implementations.py tool_schemas.py +tool_index.py tool_security.py tool_policy.py +tool_utils.py builtin_actions.py task_scheduler.py +llm_core.py model_context.py model_discovery.py +session_search.py context_budget.py context_compactor.py +ai_interaction.py action_intents.py agent_runs.py +app_helpers.py app_initializer.py config.py +database.py memory.py memory_provider.py +secret_storage.py prompt_security.py url_security.py +url_safety.py rate_limiter.py cleanup_service.py +readiness.py service_health.py exceptions.py +request_models.py assistant_log.py bg_monitor.py +builtin_mcp.py chat_helpers.py chroma_client.py +document_processor.py embedding_lanes.py deep_research.py +research_handler.py research_utils.py personal_docs.py +rag_manager.py rag_singleton.py topic_analyzer.py +visual_report.py youtube_handler.py pdf_forms.py +pdf_form_doc.py pdf_runtime.py caldav_writeback.py +email_thread_parser.py text_helpers.py user_time.py +teacher_escalation.py cookbook_serve_lifecycle.py +chatgpt_subscription.py mcp_manager.py +``` + +### `routes/` (54 files) + +``` +__init__.py _validators.py +auth_routes.py api_token_routes.py device_flow.py +chat_routes.py chat_helpers.py shell_routes.py +codex_routes.py skills_routes.py +email_routes.py email_helpers.py email_pollers.py +cookbook_routes.py cookbook_helpers.py cookbook_output.py +model_routes.py assistant_routes.py copilot_routes.py +calendar_routes.py contacts_routes.py +document_routes.py document_helpers.py +gallery_routes.py gallery_helpers.py +task_routes.py session_routes.py +note_routes.py memory_routes.py research_routes.py +mcp_routes.py search_routes.py history_routes.py +webhook_routes.py workspace_routes.py upload_routes.py +vault_routes.py prefs_routes.py preset_routes.py +signature_routes.py personal_routes.py hwfit_routes.py +backup_routes.py cleanup_routes.py diagnostics_routes.py +embedding_routes.py emoji_routes.py font_routes.py +stt_routes.py tts_routes.py compare_routes.py +editor_draft_routes.py chatgpt_subscription_routes.py admin_wipe_routes.py +``` + +### `core/` (10 files) + +``` +__init__.py constants.py database.py models.py +auth.py middleware.py session_manager.py exceptions.py +atomic_io.py platform_compat.py +``` + +--- + +## Appendix B: Key Import Relationships + +``` +core/database.py ←── 102 importers (routes/*, src/*, core/*, tests/*) + ↑ + ├── routes/auth_routes.py + ├── routes/email_routes.py + ├── src/builtin_actions.py + ├── src/task_scheduler.py + ├── src/tool_implementations.py (inline) + └── ...97 more + +src/tool_implementations.py ←── 17 importers + ↑ + ├── src/agent_loop.py + ├── src/builtin_actions.py + ├── src/tool_index.py + ├── src/task_scheduler.py + ├── src/tool_policy.py + └── ...12 more (mostly tests) + +src/agent_loop.py ←── 22 importers + ↑ + ├── src/tool_policy.py + ├── src/teacher_escalation.py + ├── src/bg_monitor.py + ├── src/task_scheduler.py + └── 18 more (incl. tests) +``` diff --git a/src/agent_loop.py b/src/agent_loop.py index 40e4232bb..27a448da4 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -267,6 +267,10 @@ _DOMAIN_RULES = { - Use `resolve_contact` to look up a contact's email or phone number by name. Searches the CardDAV address book and sent email history. - Use `manage_contact` to list, add, update, or delete contacts in the address book. - Do NOT use `manage_memory` for contact lookups — contact details live in the address book, not memory.""", + "integrations": """\ +## Integration/API rules +- To query or control a configured service integration (Home Assistant, Miniflux, Gitea, Linkding, Jellyfin, or any other registered service), use `api_call` with the integration name, HTTP method, path, and optional JSON body. +- Do not use shell, curl, or `app_api` to reach a user's connected integration when `api_call` is available.""", } _DOMAIN_TOOL_MAP = { @@ -277,9 +281,10 @@ _DOMAIN_TOOL_MAP = { "notes_calendar_tasks": {"manage_notes", "manage_calendar", "manage_tasks"}, "ui": {"ui_control"}, "sessions": {"create_session", "list_sessions", "manage_session", "send_to_session", "search_chats"}, - "files": {"bash", "python", "read_file", "write_file", "edit_file", "grep", "glob", "ls", "get_workspace"}, + "files": {"bash", "python", "read_file", "write_file", "edit_file", "grep", "glob", "ls", "get_workspace", "manage_bg_jobs"}, "settings": {"manage_settings", "manage_endpoints", "manage_mcp", "manage_webhooks", "manage_tokens", "app_api"}, "contacts": {"resolve_contact", "manage_contact"}, + "integrations": {"api_call"}, } def _domain_rules_for_tools(tool_names: set) -> list[str]: @@ -524,7 +529,7 @@ def get_builtin_overrides() -> dict: ov = get_setting("builtin_tool_overrides", {}) return ov if isinstance(ov, dict) else {} except Exception as e: - logger.warning('Failed to load builtin tool overrides: %s', e) + logger.warning("Failed to load builtin tool overrides, using defaults", exc_info=e) return {} @@ -909,10 +914,25 @@ def _classify_agent_request(messages: List[Dict], last_user: str) -> Dict[str, o domains.add("sessions") if has(r"\b(file|folder|directory|repo|git|grep|find in files|read file|edit file|shell|terminal|bash|python)\b"): domains.add("files") + # Managing detached bash jobs: "kill the background job", "stop the job", + # "kill that job", "check the job output", "is the bg job done". + if (has(r"\b(background|bg)\s+(jobs?|task)\b") + or has(r"\b(kill|stop|cancel|terminate|check|tail|show|list)\b.{0,16}\bjobs?\b") + or has(r"\bjobs?\b.{0,16}\b(output|status|done|finished|running)\b")): + domains.add("files") if has(r"\b(endpoint|api token|mcp|webhook|preference|configure|config|setting)\b"): domains.add("settings") if has(r"\b(contact|contacts|phone|phone number|address book|vcard)\b"): domains.add("contacts") + # API-integration intent — calling a configured service via the api_call + # tool. Without this the #3794 repro ("Use the api_call tool to call Home + # Assistant GET /api/states") matched no domain, classified as low-signal, + # and the tool never reached the schema filter. Detect it explicitly so the + # "integrations" domain seeds api_call deterministically (see + # _DOMAIN_TOOL_MAP), independent of embedding retrieval. + if has(r"\bapi[ _]call\b", r"\bintegrations?\b", + r"\b(?:home ?assistant|miniflux|gitea|linkding|jellyfin)\b"): + domains.add("integrations") low_signal = not continuation and not domains return { @@ -941,8 +961,11 @@ def _recent_context_for_retrieval(messages: List[Dict], max_user: int = 3, max_c if isinstance(content, list): content = " ".join(b.get("text", "") for b in content if isinstance(b, dict)) content = (content or "").strip() - # Skip injected tool-result envelopes — role=user but not human intent. - if not content or content.startswith("[Tool execution results]"): + # Skip injected envelopes — role=user but not human intent. Tool results + # are now wrapped via untrusted_context_message (metadata.trusted=False); + # keep the legacy "[Tool execution results]" prefix for older histories. + meta = msg.get("metadata") or {} + if not content or meta.get("trusted") is False or content.startswith("[Tool execution results]"): continue collected.append(content) if len(collected) >= max_user: @@ -1030,8 +1053,8 @@ def _build_system_prompt( try: from src.user_time import current_datetime_context_message _datetime_message = current_datetime_context_message() - except Exception: - pass + except Exception as e: + logger.warning("Failed to build datetime context message", exc_info=e) # Document context is kept as a SEPARATE message (not merged into the tool # prompt) so the context trimmer doesn't destroy it when truncating the @@ -1074,8 +1097,8 @@ def _build_system_prompt( try: from src.pdf_form_doc import find_source_upload_id _is_form_backed = bool(find_source_upload_id(active_document.current_content or "")) - except Exception: - pass + except Exception as e: + logger.warning("Failed to detect if document is form-backed, assuming plain", exc_info=e) if _is_form_backed: doc_ctx = ( @@ -1664,8 +1687,14 @@ def _append_tool_results( if round_reasoning: msg["reasoning_content"] = round_reasoning messages.append(msg) + # Tool output (shell/python stdout, file reads, fetched pages, email + # bodies, MCP results) is sourced from outside the server. Wrap it as + # untrusted data so prompt-injection inside a tool result is treated as + # data, not instructions — same hardening as skills (#788) and the + # web/RAG context. THREAT_MODEL.md lists tool output as a surface that + # must go through untrusted_context_message. messages.append( - {"role": "user", "content": f"[Tool execution results]\n\n{tool_output_text}"} + untrusted_context_message("tool execution results", tool_output_text) ) diff --git a/src/agent_tools/__init__.py b/src/agent_tools/__init__.py index 52fe4a99c..ba39b4f53 100644 --- a/src/agent_tools/__init__.py +++ b/src/agent_tools/__init__.py @@ -22,6 +22,9 @@ from .subprocess_tools import BashTool, PythonTool from .web_tools import WebSearchTool, WebFetchTool from .filesystem_tools import ReadFileTool, WriteFileTool, EditFileTool, LsTool, GlobTool, GrepTool, GetWorkspaceTool from .document_tools import CreateDocumentTool, UpdateDocumentTool, EditDocumentTool, SuggestDocumentTool, ManageDocumentTool +from .model_interaction_tools import ChatWithModelTool, AskTeacherTool, ListModelsTool +from .bg_job_tools import ManageBgJobsTool +from .session_tools import CreateSessionTool, ListSessionsTool, SendToSessionTool, ManageSessionTool TOOL_HANDLERS = { "bash": BashTool().execute, @@ -40,6 +43,14 @@ TOOL_HANDLERS = { "suggest_document": SuggestDocumentTool().execute, "manage_documents": ManageDocumentTool().execute, "get_workspace": GetWorkspaceTool().execute, + "chat_with_model": ChatWithModelTool().execute, + "ask_teacher": AskTeacherTool().execute, + "list_models": ListModelsTool().execute, + "manage_bg_jobs": ManageBgJobsTool().execute, + "create_session": CreateSessionTool().execute, + "list_sessions": ListSessionsTool().execute, + "send_to_session": SendToSessionTool().execute, + "manage_session": ManageSessionTool().execute, } # --------------------------------------------------------------------------- @@ -52,7 +63,7 @@ PYTHON_TIMEOUT = 30 # Tool types that trigger execution TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_file", "edit_file", - "grep", "glob", "ls", "get_workspace", + "grep", "glob", "ls", "get_workspace", "manage_bg_jobs", "create_document", "update_document", "edit_document", "search_chats", "chat_with_model", "create_session", "list_sessions", diff --git a/src/agent_tools/bg_job_tools.py b/src/agent_tools/bg_job_tools.py new file mode 100644 index 000000000..a29e813cc --- /dev/null +++ b/src/agent_tools/bg_job_tools.py @@ -0,0 +1,98 @@ +"""Agent tool to inspect and control detached background `bash` jobs. + +`bash` blocks prefixed with a `#!bg` marker run detached via `src.bg_jobs`; the +agent is auto-re-invoked with the output when they finish. This tool covers the +gaps in that flow: list the jobs in the current chat, read a still-running job's +output on demand, and kill a runaway job instead of waiting out its max-runtime. + +Registry tool (`TOOL_HANDLERS["manage_bg_jobs"]`). Jobs are scoped to the chat +that launched them, so every action requires the caller's `session_id` and a job +from another session is treated as not found. +""" + +import json +import time +from typing import Any, Dict, List + +_LIST_ACTIONS = {"list", "ls", "jobs"} +_OUTPUT_ACTIONS = {"output", "get", "read", "tail", "status", "show"} +_KILL_ACTIONS = {"kill", "stop", "cancel", "terminate"} + + +def _age(rec: Dict[str, Any]) -> str: + start = rec.get("started_at") + if not start: + return "?" + secs = int(time.time() - start) + if secs < 60: + return f"{secs}s" + if secs < 3600: + return f"{secs // 60}m" + return f"{secs // 3600}h{(secs % 3600) // 60}m" + + +def _status_label(rec: Dict[str, Any]) -> str: + status = rec.get("status", "?") + if rec.get("killed"): + return "killed" + if rec.get("timed_out"): + return "timed out" + if rec.get("died"): + return "died" + if status in ("done", "failed"): + return f"{status} (exit {rec.get('exit_code')})" + return status + + +def _row(rec: Dict[str, Any]) -> str: + cmd = (rec.get("command") or "").strip().splitlines()[0][:80] + return f"[{rec.get('id')}] {_status_label(rec)} | {_age(rec)} | {cmd}" + + +class ManageBgJobsTool: + async def execute(self, content: str, ctx: dict) -> dict: + from src import bg_jobs + + session_id = ctx.get("session_id") + raw = (content or "").strip() + try: + args = json.loads(raw) if raw else {} + except (ValueError, TypeError): + args = {} + if not isinstance(args, dict): + args = {} + action = str(args.get("action", "list")).strip().lower() + job_id = str(args.get("job_id") or args.get("id") or "").strip() + + if not session_id: + return {"error": "manage_bg_jobs: no active chat session; background jobs are scoped to a chat.", "exit_code": 1} + + if action in _LIST_ACTIONS: + jobs: List[Dict[str, Any]] = bg_jobs.list_for_session(session_id) + if not jobs: + return {"output": "No background jobs in this chat.", "exit_code": 0} + jobs.sort(key=lambda r: r.get("started_at") or 0, reverse=True) + lines = "\n".join(_row(r) for r in jobs) + return {"output": f"{len(jobs)} background job(s):\n{lines}", "exit_code": 0} + + if action in _OUTPUT_ACTIONS or action in _KILL_ACTIONS: + if not job_id: + return {"error": f"manage_bg_jobs: action '{action}' requires a job_id (see action='list').", "exit_code": 1} + rec = bg_jobs.get(job_id) + # Scope: only the chat that launched a job may see or control it. + if rec is None or rec.get("session_id") != session_id: + return {"error": f"manage_bg_jobs: no background job '{job_id}' in this chat.", "exit_code": 1} + + if action in _KILL_ACTIONS: + if rec.get("status") != "running": + return {"output": f"Job `{job_id}` already {_status_label(rec)}; nothing to kill.", "exit_code": 0} + killed = bg_jobs.kill(job_id) + return {"output": f"Killed background job `{job_id}` ({(killed or {}).get('command', '').splitlines()[0][:80]}).", "exit_code": 0} + + out = rec.get("output") or "(no output yet)" + return { + "output": f"Job `{job_id}` [{_status_label(rec)}, {_age(rec)}]\nCommand: {rec.get('command')}\n\nOutput:\n{out}", + "exit_code": 0, + } + + return {"error": f"manage_bg_jobs: unknown action '{action}'. Use list, output, or kill.", "exit_code": 1} diff --git a/src/agent_tools/filesystem_tools.py b/src/agent_tools/filesystem_tools.py index 7ba22161c..a55944ae5 100644 --- a/src/agent_tools/filesystem_tools.py +++ b/src/agent_tools/filesystem_tools.py @@ -1,6 +1,7 @@ import asyncio import json import os +import re import difflib import fnmatch import shutil @@ -16,6 +17,31 @@ _CODENAV_SKIP_DIRS = frozenset({ _CODENAV_MAX_HITS = 200 _CODENAV_MAX_LINE = 400 + +def _glob_to_regex(pat: str) -> "re.Pattern": + """Translate a forward-slash glob (**, *, ?) into a compiled regex. + `**/` matches zero or more complete directories. + `*` matches within a single path segment (does not cross /). + """ + i, n, out = 0, len(pat), [] + while i < n: + if pat[i : i + 3] == "**/": + out.append("(?:[^/]+/)*") + i += 3 + elif pat[i : i + 2] == "**": + out.append(".*") + i += 2 + elif pat[i] == "*": + out.append("[^/]*") + i += 1 + elif pat[i] == "?": + out.append("[^/]") + i += 1 + else: + out.append(re.escape(pat[i])) + i += 1 + return re.compile("".join(out)) + def _unified_diff(old: str, new: str, path: str) -> Optional[Dict[str, Any]]: if old == new: return None @@ -259,23 +285,38 @@ class GlobTool: return {"error": f"glob: {e}", "exit_code": 1} def _glob(): - from pathlib import Path - base = Path(root) - if not base.is_dir(): + base = os.path.abspath(root) + if not os.path.isdir(base): return None, f"glob: {root}: not a directory" + norm_pat = pattern.replace("\\", "/") + # Fast path: literal pattern (no wildcards) → direct path lookup. + if not any(c in norm_pat for c in "*?["): + cand = os.path.normpath(os.path.join(base, norm_pat)) + if os.path.exists(cand): + return [cand], None + # Literal not at exact path — fall through to walk so + # e.g. "foo.py" still matches at any depth (like rglob). + # Compile glob to regex: * stays within one segment, **/ spans dirs. + regex = _glob_to_regex(norm_pat) matched = [] + cap = _CODENAV_MAX_HITS * 5 try: - for p in base.rglob(pattern): - if set(p.relative_to(base).parts) & _CODENAV_SKIP_DIRS: - continue - try: - mtime = p.stat().st_mtime - except OSError: - mtime = 0 - matched.append((mtime, str(p))) - if len(matched) > _CODENAV_MAX_HITS * 5: + for dp, dns, fns in os.walk(base): + # Prune skipped dirs before descending (unlike rglob which + # descends first then filters — fatal on large node_modules). + dns[:] = [d for d in dns if d not in _CODENAV_SKIP_DIRS] + for name in fns + dns: + full = os.path.join(dp, name) + rel = os.path.relpath(full, base).replace(os.sep, "/") + if regex.fullmatch(rel) or regex.fullmatch(name): + try: + mtime = os.stat(full).st_mtime + except OSError: + mtime = 0 + matched.append((mtime, full)) + if len(matched) > cap: break - except (OSError, ValueError) as _e: + except OSError as _e: return None, f"glob: {_e}" matched.sort(key=lambda t: t[0], reverse=True) return [pth for _, pth in matched[:_CODENAV_MAX_HITS]], None diff --git a/src/agent_tools/model_interaction_tools.py b/src/agent_tools/model_interaction_tools.py new file mode 100644 index 000000000..6cbabe919 --- /dev/null +++ b/src/agent_tools/model_interaction_tools.py @@ -0,0 +1,208 @@ +"""model_interaction_tools.py - agent tools for talking to other models. + +Owns the model-interaction tool implementations (chat_with_model, ask_teacher, +list_models) and their handler classes, registered in ``TOOL_HANDLERS``. Part +of the tool -> registry migration (#3629): the implementations were moved here +out of ``src.ai_interaction`` so dispatch flows through the registry instead of +the elif chain / dispatch_ai_tool in tool_execution.py. + +Shared helpers that still live in ``src.ai_interaction`` and are used by tools +not yet migrated (``_resolve_model``, ``AI_CHAT_TIMEOUT``) are imported lazily +inside the functions to avoid an import cycle at module load. +""" +import logging +from typing import Dict, Optional + +logger = logging.getLogger(__name__) + + +_TEACHER_SYSTEM_PROMPT = ( + "You are a senior AI mentor. A less capable model is stuck on a problem and asking for help. " + "Provide clear, actionable guidance:\n" + "1. Brief analysis of the problem\n" + "2. Recommended approach (step by step)\n" + "3. Key things to watch out for\n\n" + "Be concise and practical. No preamble." +) + + +async def chat_with_model(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: + """Send a message to a specific model and return its response. + + Content format: + Line 1: model_name (or model_name@endpoint_name) + Line 2+: the message to send + """ + from src.ai_interaction import _resolve_model, AI_CHAT_TIMEOUT + from src.llm_core import llm_call_async + + lines = content.strip().split("\n", 1) + if not lines or not lines[0].strip(): + return {"error": "First line must be the model name"} + + model_spec = lines[0].strip() + message = lines[1].strip() if len(lines) > 1 else "" + if not message: + return {"error": "No message provided (line 2+ is the message)"} + + try: + url, model, headers = _resolve_model(model_spec, owner=owner) + except ValueError as e: + return {"error": str(e)} + + try: + response = await llm_call_async( + url, model, + [{"role": "user", "content": message}], + headers=headers, + timeout=AI_CHAT_TIMEOUT, + ) + # Truncate very long responses + if len(response) > 10000: + response = response[:10000] + "\n... (truncated)" + return {"model": model, "response": response} + except Exception as e: + logger.error(f"chat_with_model failed: {e}") + return {"error": f"Failed to get response from {model_spec}: {e}"} + + +async def ask_teacher(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: + """Ask a more capable model for help. + + Content format: + Line 1: model_name (or 'auto') + Line 2+: the problem description + """ + from src.ai_interaction import _resolve_model, AI_CHAT_TIMEOUT + from src.llm_core import llm_call_async + from src.settings import get_setting + + lines = content.strip().split("\n", 1) + model_spec = lines[0].strip() if lines else "auto" + problem = lines[1].strip() if len(lines) > 1 else "" + + if not problem: + return {"error": "No problem description provided"} + + if model_spec.lower() in ("auto", ""): + model_spec = get_setting("teacher_model", "") + if not model_spec: + return {"error": "No teacher model configured. Specify a model name or set teacher_model in settings."} + + try: + url, model, headers = _resolve_model(model_spec, owner=owner) + except ValueError as e: + return {"error": str(e)} + + try: + response = await llm_call_async( + url, model, + [ + {"role": "system", "content": _TEACHER_SYSTEM_PROMPT}, + {"role": "user", "content": f"Problem:\n{problem}"}, + ], + headers=headers, + timeout=AI_CHAT_TIMEOUT, + ) + if len(response) > 8000: + response = response[:8000] + "\n... (truncated)" + return {"model": model, "response": response, "teacher": True} + except Exception as e: + logger.error(f"ask_teacher failed: {e}") + return {"error": f"Teacher call failed ({model_spec}): {e}"} + + +async def list_models(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: + """List all available models across configured endpoints. + + Content = optional filter keyword. + """ + import json + import httpx + from src.database import SessionLocal, ModelEndpoint + from src.llm_core import _detect_provider, ANTHROPIC_MODELS + from src.auth_helpers import owner_filter + from src.endpoint_resolver import resolve_endpoint_runtime, build_headers, build_models_url + + keyword = content.strip().lower() if content.strip() else None + + db = SessionLocal() + try: + query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) + if owner: + query = owner_filter(query, ModelEndpoint, owner) + endpoints = query.all() + if not endpoints: + return {"results": "No enabled model endpoints configured."} + + result_lines = [] + total_models = 0 + + for ep in endpoints: + try: + base, api_key = resolve_endpoint_runtime(ep, owner=owner) + except Exception: + continue + provider = _detect_provider(base) + headers = build_headers(api_key, base) + + model_ids = [] + if provider == "anthropic": + model_ids = list(ANTHROPIC_MODELS) + else: + try: + models_url = build_models_url(base) + if models_url: + r = httpx.get(models_url, headers=headers, timeout=5) + r.raise_for_status() + data = r.json() + model_ids = [m.get("id") for m in (data.get("data") or []) if m.get("id")] + if not model_ids: + model_ids = [ + m.get("name") or m.get("model") + for m in (data.get("models") or []) + if m.get("name") or m.get("model") + ] + else: + model_ids = json.loads(ep.cached_models or "[]") + except Exception: + model_ids = ["(endpoint offline)"] + + if keyword: + model_ids = [m for m in model_ids if keyword in m.lower() or keyword in (ep.name or "").lower()] + + if model_ids: + result_lines.append(f"\n**{ep.name or base}** ({provider}):") + for mid in model_ids: + result_lines.append(f" - `{mid}`") + total_models += 1 + + if not result_lines: + return {"results": "No models found" + (f" matching '{keyword}'" if keyword else "") + "."} + + header = f"Available models ({total_models} total):" + return {"results": header + "\n".join(result_lines)} + except Exception as e: + logger.error(f"list_models failed: {e}") + return {"error": str(e)} + finally: + db.close() + + +# --------------------------------------------------------------------------- +# Handler classes registered in TOOL_HANDLERS +# --------------------------------------------------------------------------- + +class ChatWithModelTool: + async def execute(self, content: str, ctx: dict) -> Dict: + return await chat_with_model(content, ctx.get("session_id"), owner=ctx.get("owner")) + + +class AskTeacherTool: + async def execute(self, content: str, ctx: dict) -> Dict: + return await ask_teacher(content, ctx.get("session_id"), owner=ctx.get("owner")) + + +class ListModelsTool: + async def execute(self, content: str, ctx: dict) -> Dict: + return await list_models(content, ctx.get("session_id"), owner=ctx.get("owner")) diff --git a/src/agent_tools/session_tools.py b/src/agent_tools/session_tools.py new file mode 100644 index 000000000..797235c5d --- /dev/null +++ b/src/agent_tools/session_tools.py @@ -0,0 +1,464 @@ +"""session_tools.py - agent tools for AI-to-AI session management. + +Owns create_session, list_sessions, send_to_session and manage_session, moved +out of src.ai_interaction as part of the tool -> registry migration (#3629), and +their handler classes registered in TOOL_HANDLERS. + +The session manager is a runtime-set singleton in src.ai_interaction, so each +function fetches it via get_session_manager() (imported here); _resolve_model and +AI_CHAT_TIMEOUT are reused from there too. +""" +import json +import logging +import uuid +from typing import Dict, Optional + +from src.ai_interaction import get_session_manager, _resolve_model, AI_CHAT_TIMEOUT + +logger = logging.getLogger(__name__) + + +async def create_session(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: + """Create a new chat session. + + Content format: + Line 1: session name + Line 2: model_name (or model_name@endpoint_name) + """ + _session_manager = get_session_manager() + if not _session_manager: + return {"error": "Session manager not available"} + + lines = content.strip().split("\n") + if len(lines) < 2: + return {"error": "Need 2 lines: session name, then model spec"} + + name = lines[0].strip() + model_spec = lines[1].strip() + + if not name: + return {"error": "Session name cannot be empty"} + + try: + url, model, headers = _resolve_model(model_spec, owner=owner) + except ValueError as e: + return {"error": str(e)} + + sid = str(uuid.uuid4())[:8] + try: + _session_manager.create_session( + session_id=sid, + name=name, + endpoint_url=url, + model=model, + rag=False, + owner=owner, + ) + # Store headers on session for future calls + sess = _session_manager.get_session(sid) + if sess and headers: + sess.headers = headers + try: + from src.event_bus import fire_event + fire_event("session_created", owner) + except Exception: + logger.debug("session_created event dispatch failed", exc_info=True) + + return {"session_id": sid, "name": name, "model": model, "endpoint_url": url} + except Exception as e: + logger.error(f"create_session failed: {e}") + return {"error": f"Failed to create session: {e}"} + +async def list_sessions(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: + """List sessions sorted by most-recently-active first. + + Output includes a relative "last active" timestamp per row so the + agent can answer "open my last chat" without guessing from titles. + The most-recent session is always first in the list. + + Content = optional filter keyword (matches session name). + """ + _session_manager = get_session_manager() + if not _session_manager: + return {"error": "Session manager not available"} + + keyword = content.strip().lower() if content.strip() else None + + try: + from core.database import SessionLocal, Session as DbSession + from datetime import datetime, timezone + + # Pull every session's last_accessed from the DB so we can sort + # by recency. In-memory sessions hold name + model + msg_count; + # the DB row holds the timestamps. + db = SessionLocal() + try: + db_rows = {r.id: r for r in db.query(DbSession).all()} + finally: + db.close() + + # SECURITY: scope to the caller's sessions. Passing None returned + # every user's sessions, which the agent tool then exposed via the + # "list my chats" reply. + sessions = _session_manager.get_sessions_for_user(owner) + rows = [] + for sid, sess in sessions.items(): + if keyword and keyword not in (sess.name or "").lower(): + continue + db_row = db_rows.get(sid) + # Prefer last_accessed; fall back to updated_at, then created_at. + ts = None + if db_row: + ts = getattr(db_row, 'last_accessed', None) or getattr(db_row, 'updated_at', None) or getattr(db_row, 'created_at', None) + rows.append((ts, sid, sess)) + + # Sort by timestamp DESC; rows without a timestamp sink to the bottom. + rows.sort(key=lambda r: r[0] or datetime.min, reverse=True) + + def _rel(ts): + if not ts: + return 'never' + now = datetime.utcnow() + try: + if ts.tzinfo is not None: + now = datetime.now(timezone.utc) + diff = (now - ts).total_seconds() + except Exception: + return 'unknown' + if diff < 60: return 'just now' + if diff < 3600: return f'{int(diff / 60)}m ago' + if diff < 86400: return f'{int(diff / 3600)}h ago' + if diff < 86400 * 7: return f'{int(diff / 86400)}d ago' + return ts.strftime('%Y-%m-%d') + + lines = [] + for i, (ts, sid, sess) in enumerate(rows): + if i >= 50: + lines.append(f"... and {len(rows) - 50} more (showing first 50)") + break + safe_name = (sess.name or "Untitled").replace("[", "\\[").replace("]", "\\]") + msg_count = getattr(sess, "message_count", 0) or 0 + model = getattr(sess, "model", "unknown") + marker = " ← most recent" if i == 0 else "" + lines.append(f"- **[{safe_name}](#session-{sid})** (id: `{sid}`, model: {model}, {msg_count} msgs, last active {_rel(ts)}){marker}") + + if not lines: + return {"results": "No sessions found" + (f" matching '{keyword}'" if keyword else "") + "."} + + return { + "results": ( + f"Found {len(rows)} session(s), sorted most-recent first:\n" + + "\n".join(lines) + + "\n\nAssistant: when replying to the user, preserve the chat-title markdown links exactly as shown, e.g. `[Chat](#session-id)`. Do not rewrite this as a plain, non-clickable table." + ) + } + except Exception as e: + logger.error(f"list_sessions failed: {e}") + return {"error": str(e)} + +async def send_to_session(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: + """Send a message to an existing session and get a response. + + Content format: + Line 1: session_id + Line 2+: message + """ + _session_manager = get_session_manager() + from src.llm_core import llm_call_async + from core.models import ChatMessage + + if not _session_manager: + return {"error": "Session manager not available"} + + lines = content.strip().split("\n", 1) + if len(lines) < 2: + return {"error": "Need 2 lines: session_id, then message"} + + target_sid = lines[0].strip() + message = lines[1].strip() + + sess = _session_manager.get_session(target_sid) + if not sess: + return {"error": f"Session '{target_sid}' not found"} + + # Owner-scope: reject access to another user's session + if owner and getattr(sess, "owner", None) and sess.owner != owner: + return {"error": f"Session '{target_sid}' not found"} + + if not message: + return {"error": "No message provided"} + + try: + # Build context from session history + context = sess.get_context_messages() + context.append({"role": "user", "content": message}) + + response = await llm_call_async( + sess.endpoint_url, sess.model, context, + headers=sess.headers, + timeout=AI_CHAT_TIMEOUT, + ) + + # Save both messages to session + sess.add_message(ChatMessage("user", message)) + sess.add_message(ChatMessage("assistant", response)) + + # Truncate for tool output + if len(response) > 10000: + response = response[:10000] + "\n... (truncated)" + + return { + "session_id": target_sid, + "session_name": sess.name, + "response": response, + } + except Exception as e: + logger.error(f"send_to_session failed: {e}") + return {"error": f"Failed to send to session: {e}"} + +async def manage_session(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: + """Manage sessions: rename, archive, delete, important, truncate, fork. + + Content format: + Line 1: action (rename|archive|unarchive|delete|important|unimportant|truncate|fork) + Line 2: target session_id (or "current" to use the active session) + Line 3+: action-specific params (e.g. new name for rename, keep_count for truncate) + """ + _session_manager = get_session_manager() + if not _session_manager: + return {"error": "Session manager not available"} + + from src.database import SessionLocal, Session as DbSession + + # Accept BOTH the structured JSON args the tool schema advertises + # ({action, session_id, value}) AND the legacy line-based format + # (line1=action, line2=session_id, line3=value). Native function-calling + # models send JSON; fenced-block callers send lines. Previously only the + # line format was parsed, so a model that followed the schema (JSON) got + # "Need at least 2 lines" / "Rename needs line 3" and couldn't drive it. + _raw = (content or "").strip() + action = "" + target_sid = "" + value = None # the action param: new name (rename) / keep_count (truncate, fork) + _list_filter = "" + _parsed = None + if _raw.startswith("{"): + try: + _parsed = json.loads(_raw) + except Exception: + _parsed = None + if isinstance(_parsed, dict): + action = str(_parsed.get("action") or "").strip().lower() + target_sid = str(_parsed.get("session_id") or _parsed.get("session") or _parsed.get("id") or "").strip() + _v = _parsed.get("value") + if _v is None: + _v = (_parsed.get("name") or _parsed.get("new_name") + or _parsed.get("title") or _parsed.get("keep_count")) + value = None if _v is None else str(_v).strip() + _list_filter = str(_parsed.get("filter") or "").strip() + else: + lines = _raw.split("\n") + if not lines or not lines[0].strip(): + return {"error": "Missing action (rename|archive|delete|important|truncate|fork|list|switch)"} + action = lines[0].strip().lower() + target_sid = lines[1].strip() if len(lines) >= 2 else "" + value = lines[2].strip() if len(lines) >= 3 else None + _list_filter = "\n".join(lines[1:]).strip() + + if not action: + return {"error": "Missing action (rename|archive|delete|important|truncate|fork|list|switch)"} + + # `list` alias - dispatch to list_sessions so the agent's natural + # first guess (every other manage_* tool has a `list` action) works. + if action == "list": + return await list_sessions(_list_filter, session_id, owner=owner) + + if not target_sid: + return {"error": "Need a session_id (or 'current' for the active chat)"} + + # Allow "current" to refer to the active session + if target_sid.lower() == "current" and session_id: + target_sid = session_id + + # `switch` / `open` / `select` / `view` - the agent reaches for + # these when the user asks to "open" or "switch to" a session. + # There's no server-side way to make the browser navigate, so we + # just return a clickable anchor link the user can click. The + # frontend's chat-history click delegate routes `#session-` + # to selectSession(). The agent's reply naturally embeds this + # result so the user sees a single clickable line. + def _session_query(db): + query = db.query(DbSession).filter(DbSession.id == target_sid) + if owner is not None: + query = query.filter(DbSession.owner == owner) + return query + + if action in ("switch", "open", "select", "view"): + db = SessionLocal() + try: + db_sess = _session_query(db).first() + if not db_sess: + return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} + name = db_sess.name or target_sid + finally: + db.close() + return { + "action": action, + "session_id": target_sid, + "name": name, + "results": f"[{name}](#session-{target_sid}) - click to open.", + } + + db = SessionLocal() + try: + if action == "rename": + if not value: + return {"error": "rename needs a new name (the `value` arg, or line 3 in the legacy format)"} + new_name = value + db_sess = _session_query(db).first() + if not db_sess: + return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} + db_sess.name = new_name + db.commit() + _session_manager.update_session_name(target_sid, new_name) + return {"action": "rename", "session_id": target_sid, "name": new_name, + "results": f"Session renamed to '{new_name}'"} + + elif action == "archive": + db_sess = _session_query(db).first() + if not db_sess: + return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} + db_sess.archived = True + db.commit() + return {"action": "archive", "session_id": target_sid, + "results": f"Session '{db_sess.name}' archived"} + + elif action == "unarchive": + db_sess = _session_query(db).first() + if not db_sess: + return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} + db_sess.archived = False + db.commit() + return {"action": "unarchive", "session_id": target_sid, + "results": f"Session '{db_sess.name}' unarchived"} + + elif action == "delete": + if target_sid == session_id: + return {"error": "Cannot delete the current session while chatting in it. Delete other sessions first."} + db_sess = _session_query(db).first() + if not db_sess: + return {"error": f"Session '{target_sid}' not found. Refusing to delete an unknown chat id; use the exact id from list_sessions."} + if db_sess and db_sess.is_important: + return {"error": f"Session '{db_sess.name}' is starred/favorited. Unstar it first before deleting."} + try: + ok = _session_manager.delete_session(target_sid) + if not ok: + return {"error": f"Session '{target_sid}' was not deleted because it no longer exists."} + return {"action": "delete", "session_id": target_sid, + "results": f"Session '{db_sess.name or target_sid}' deleted"} + except Exception as e: + return {"error": f"Failed to delete session: {e}"} + + elif action in ("important", "unimportant"): + is_important = action == "important" + db_sess = _session_query(db).first() + if not db_sess: + return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} + # Prevent AI from unstarring sessions - only the user can do that manually + if not is_important and db_sess.is_important: + return {"error": f"Session '{db_sess.name}' is starred by the user. Only the user can unstar sessions manually."} + db_sess.is_important = is_important + db.commit() + status = "marked as important" if is_important else "unmarked as important" + return {"action": action, "session_id": target_sid, + "results": f"Session '{db_sess.name}' {status}"} + + elif action == "truncate": + db_sess = _session_query(db).first() + if not db_sess: + return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} + keep_count = 10 + if value: + try: + keep_count = int(value) + except ValueError: + pass + success = _session_manager.truncate_messages(target_sid, keep_count) + if success: + return {"action": "truncate", "session_id": target_sid, + "results": f"Session truncated to last {keep_count} messages"} + return {"error": f"Failed to truncate session '{target_sid}'"} + + elif action == "fork": + db_sess = _session_query(db).first() + if not db_sess: + return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} + keep_count = 0 # 0 = all messages + if value: + try: + keep_count = int(value) + except ValueError: + pass + + source = _session_manager.get_session(target_sid) + if not source: + return {"error": f"Session '{target_sid}' not found"} + + new_sid = str(uuid.uuid4())[:8] + _session_manager.create_session( + session_id=new_sid, + name=f"Fork: {source.name}", + endpoint_url=source.endpoint_url, + model=source.model, + rag=False, + owner=owner, + ) + # Copy messages + history = source.get_context_messages() + if keep_count > 0: + history = history[:keep_count] + from core.models import ChatMessage as InMemoryMsg + new_sess = _session_manager.get_session(new_sid) + for msg in history: + new_sess.add_message(InMemoryMsg(msg["role"], msg["content"])) + try: + from src.event_bus import fire_event + fire_event("session_created", owner) + except Exception: + logger.debug("session_created event dispatch failed", exc_info=True) + + return {"action": "fork", "session_id": new_sid, + "source_session": target_sid, "messages_copied": len(history), + "results": f"Forked session '{source.name}' -> new session {new_sid} ({len(history)} messages)"} + + else: + return {"error": f"Unknown action '{action}'. Use: list, switch, rename, archive, unarchive, delete, important, unimportant, truncate, fork"} + except Exception as e: + logger.error(f"manage_session failed: {e}") + return {"error": str(e)} + finally: + db.close() + + +# --------------------------------------------------------------------------- +# Handler classes registered in TOOL_HANDLERS +# --------------------------------------------------------------------------- + +class CreateSessionTool: + async def execute(self, content: str, ctx: dict) -> Dict: + return await create_session(content, ctx.get("session_id"), owner=ctx.get("owner")) + + +class ListSessionsTool: + async def execute(self, content: str, ctx: dict) -> Dict: + return await list_sessions(content, ctx.get("session_id"), owner=ctx.get("owner")) + + +class SendToSessionTool: + async def execute(self, content: str, ctx: dict) -> Dict: + return await send_to_session(content, ctx.get("session_id"), owner=ctx.get("owner")) + + +class ManageSessionTool: + async def execute(self, content: str, ctx: dict) -> Dict: + return await manage_session(content, ctx.get("session_id"), owner=ctx.get("owner")) diff --git a/src/agent_tools/web_tools.py b/src/agent_tools/web_tools.py index f1410e18e..19ece8cd8 100644 --- a/src/agent_tools/web_tools.py +++ b/src/agent_tools/web_tools.py @@ -79,13 +79,23 @@ class WebSearchTool: class WebFetchTool: async def execute(self, content: str, ctx: dict) -> dict: from src.search.content import fetch_webpage_content + from src.constants import WEB_FETCH_HARD_MAX_BYTES raw = content.strip() url = "" + max_bytes = None if raw.startswith("{"): try: parsed = json.loads(raw) if isinstance(parsed, dict): url = str(parsed.get("url") or "").strip() + # Download-budget override (#3812): "full": true raises the + # budget to the hard cap; an explicit max_bytes is clamped + # to the hard cap downstream. Default stays the soft cap. + if parsed.get("full") is True: + max_bytes = WEB_FETCH_HARD_MAX_BYTES + mb = parsed.get("max_bytes") + if isinstance(mb, int) and mb > 0: + max_bytes = mb except json.JSONDecodeError: url = "" if not url: @@ -100,7 +110,7 @@ class WebFetchTool: loop = asyncio.get_running_loop() try: result = await asyncio.wait_for( - loop.run_in_executor(None, lambda: fetch_webpage_content(url, timeout=10)), + loop.run_in_executor(None, lambda: fetch_webpage_content(url, timeout=10, max_bytes=max_bytes)), timeout=30, ) except asyncio.TimeoutError: @@ -116,8 +126,28 @@ class WebFetchTool: return {"error": f"web_fetch: {url}: {err}", "exit_code": 1} return {"error": f"web_fetch: {url}: no readable text content (not HTML, or the page needs JS/login)", "exit_code": 1} + # Tell the model when the download budget cut the body short and how + # to get the rest, instead of silently presenting a partial page as + # the whole thing. + size_note = "" + if result.get("truncated"): + fetched = result.get("fetched_bytes") or 0 + total = result.get("total_bytes") + total_txt = f" of {total:,} bytes" if total else "" + size_note = ( + f"[partial content: download stopped at {fetched:,} bytes{total_txt}. " + f'Re-call with {{"url": "{url}", "full": true}} to fetch up to ' + f"{WEB_FETCH_HARD_MAX_BYTES:,} bytes.]\n\n" + ) + + # The notice must lead the output so the MAX_OUTPUT_CHARS trim below can + # never drop it. The title is untrusted, uncapped page content, so a + # giant title ahead of the notice could push it out of range; keep the + # notice first and cap the title as a second guard. + if len(title) > 300: + title = title[:300] + "..." header = (f"# {title}\n" if title else "") + f"Source: {url}\n\n" - output = header + text + output = size_note + header + text if len(output) > MAX_OUTPUT_CHARS: output = output[:MAX_OUTPUT_CHARS] + "\n\n[...truncated]" return {"output": output, "exit_code": 0} diff --git a/src/ai_interaction.py b/src/ai_interaction.py index 45ff2e472..6655beaf4 100644 --- a/src/ai_interaction.py +++ b/src/ai_interaction.py @@ -1,8 +1,14 @@ """ ai_interaction.py -AI-to-AI interaction tools: chat_with_model, create_session, list_sessions, -send_to_session, pipeline. +AI-to-AI interaction tools: pipeline and manage_memory, plus shared model +resolution (_resolve_model), the session-manager singleton, and dispatch_ai_tool. + +As part of the tool -> registry migration (#3629), chat_with_model, ask_teacher +and list_models moved to src/agent_tools/model_interaction_tools.py, and +create_session, list_sessions, send_to_session and manage_session moved to +src/agent_tools/session_tools.py. Those modules reuse get_session_manager / +_resolve_model / AI_CHAT_TIMEOUT from here. These are agent tools — the LLM writes fenced code blocks and they execute through the standard agent_tools.py pipeline. @@ -159,440 +165,6 @@ def _resolve_model(spec: str, owner: Optional[str] = None) -> Tuple[str, str, Di # Tool implementations # --------------------------------------------------------------------------- -async def do_chat_with_model(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: - """Send a message to a specific model and return its response. - - Content format: - Line 1: model_name (or model_name@endpoint_name) - Line 2+: the message to send - """ - from src.llm_core import llm_call_async - - lines = content.strip().split("\n", 1) - if not lines or not lines[0].strip(): - return {"error": "First line must be the model name"} - - model_spec = lines[0].strip() - message = lines[1].strip() if len(lines) > 1 else "" - if not message: - return {"error": "No message provided (line 2+ is the message)"} - - try: - url, model, headers = _resolve_model(model_spec, owner=owner) - except ValueError as e: - return {"error": str(e)} - - try: - response = await llm_call_async( - url, model, - [{"role": "user", "content": message}], - headers=headers, - timeout=AI_CHAT_TIMEOUT, - ) - # Truncate very long responses - if len(response) > 10000: - response = response[:10000] + "\n... (truncated)" - return {"model": model, "response": response} - except Exception as e: - logger.error(f"chat_with_model failed: {e}") - return {"error": f"Failed to get response from {model_spec}: {e}"} - - -_TEACHER_SYSTEM_PROMPT = ( - "You are a senior AI mentor. A less capable model is stuck on a problem and asking for help. " - "Provide clear, actionable guidance:\n" - "1. Brief analysis of the problem\n" - "2. Recommended approach (step by step)\n" - "3. Key things to watch out for\n\n" - "Be concise and practical. No preamble." -) - - -async def do_ask_teacher(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: - """Ask a more capable model for help. - - Content format: - Line 1: model_name (or 'auto') - Line 2+: the problem description - """ - from src.llm_core import llm_call_async - from src.settings import get_setting - - lines = content.strip().split("\n", 1) - model_spec = lines[0].strip() if lines else "auto" - problem = lines[1].strip() if len(lines) > 1 else "" - - if not problem: - return {"error": "No problem description provided"} - - if model_spec.lower() in ("auto", ""): - model_spec = get_setting("teacher_model", "") - if not model_spec: - return {"error": "No teacher model configured. Specify a model name or set teacher_model in settings."} - - try: - url, model, headers = _resolve_model(model_spec, owner=owner) - except ValueError as e: - return {"error": str(e)} - - try: - response = await llm_call_async( - url, model, - [ - {"role": "system", "content": _TEACHER_SYSTEM_PROMPT}, - {"role": "user", "content": f"Problem:\n{problem}"}, - ], - headers=headers, - timeout=AI_CHAT_TIMEOUT, - ) - if len(response) > 8000: - response = response[:8000] + "\n... (truncated)" - return {"model": model, "response": response, "teacher": True} - except Exception as e: - logger.error(f"ask_teacher failed: {e}") - return {"error": f"Teacher call failed ({model_spec}): {e}"} - - -async def do_second_opinion(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: - """Get a second opinion from another model, then have the original model - evaluate the feedback and produce a unified version. - - Content format: - Line 1: model_name (or model_name@endpoint_name) - Line 2+ (optional): specific question or focus area - - Flow: - 1. Pull recent conversation context - 2. Send to reviewer model → get honest feedback - 3. Send feedback back to the session's own model → evaluate & unify - 4. Return both the review and the unified response - """ - from src.llm_core import llm_call_async - - lines = content.strip().split("\n", 1) - if not lines or not lines[0].strip(): - return {"error": "First line must be the model name"} - - model_spec = lines[0].strip() - focus = lines[1].strip() if len(lines) > 1 else "" - - try: - reviewer_url, reviewer_model, reviewer_headers = _resolve_model(model_spec, owner=owner) - except ValueError as e: - return {"error": str(e)} - - # Pull recent conversation context from current session - context_text = "" - sess = None - if session_id and _session_manager: - sess = _session_manager.get_session(session_id) - if sess: - messages = sess.get_context_messages() - recent = messages[-15:] if len(messages) > 15 else messages - parts = [] - for m in recent: - role = m.get("role", "unknown").upper() - text = m.get("content", "") - if isinstance(text, list): - text = " ".join( - p.get("text", "") for p in text if isinstance(p, dict) - ) - if text: - parts.append(f"[{role}]: {text[:2000]}") - context_text = "\n\n".join(parts) - - if not context_text: - return {"error": "No conversation context found to review"} - - # ── Step 1: Get the reviewer's feedback ── - reviewer_system = ( - "You are giving a second opinion on a conversation between a user and an AI assistant. " - "Your job is to be genuinely helpful and honest — not a yes-man, but not a contrarian either.\n\n" - "Guidelines:\n" - "- If the plan/idea is solid, say so clearly. Don't manufacture problems that aren't there.\n" - "- If you spot a real flaw, blind spot, or simpler approach — call it out directly.\n" - "- Be practical. Don't over-engineer or over-analyze. Real-world tradeoffs matter.\n" - "- If there's a meaningfully better way to do something, suggest it concretely.\n" - "- Give credit where it's due — highlight what's working well.\n" - "- Keep it concise and actionable. No fluff.\n" - "- You're a second pair of eyes, not a professor grading a paper." - ) - - reviewer_message = f"Here's the conversation so far:\n\n{context_text}" - if focus: - reviewer_message += f"\n\n---\nSpecifically, I want your take on: {focus}" - else: - reviewer_message += "\n\n---\nGive me your honest second opinion on what's being discussed." - - try: - review = await llm_call_async( - reviewer_url, reviewer_model, - [ - {"role": "system", "content": reviewer_system}, - {"role": "user", "content": reviewer_message}, - ], - headers=reviewer_headers, - timeout=AI_CHAT_TIMEOUT, - ) - if len(review) > 8000: - review = review[:8000] + "\n... (truncated)" - except Exception as e: - logger.error(f"second_opinion reviewer call failed: {e}") - return {"error": f"Failed to get second opinion from {model_spec}: {e}"} - - # ── Step 2: Send review back to session's own model for evaluation ── - unified = "" - original_model = "unknown" - if sess: - original_url = sess.endpoint_url - original_model = sess.model - original_headers = getattr(sess, "headers", None) or {} - - unify_system = ( - "Another AI model just reviewed the conversation you've been having with the user. " - "Read their feedback carefully, then respond with:\n\n" - "1. **What you agree with** — acknowledge valid points honestly.\n" - "2. **What you disagree with** — explain why, briefly.\n" - "3. **Unified version** — produce an updated/refined version of whatever was being discussed, " - "incorporating the feedback you found valid. Don't accept every note blindly — " - "use your judgment on what actually improves things vs what's unnecessary.\n\n" - "Be concise and practical. The user wants a better result, not a meta-discussion." - ) - - unify_message = ( - f"Here's the conversation context:\n\n{context_text}\n\n" - f"---\n\n" - f"**Review from {reviewer_model}:**\n\n{review}\n\n" - f"---\n\n" - f"Evaluate this feedback and produce a unified improved version." - ) - - try: - unified = await llm_call_async( - original_url, original_model, - [ - {"role": "system", "content": unify_system}, - {"role": "user", "content": unify_message}, - ], - headers=original_headers, - timeout=AI_CHAT_TIMEOUT, - ) - if len(unified) > 10000: - unified = unified[:10000] + "\n... (truncated)" - except Exception as e: - logger.error(f"second_opinion unify call failed: {e}") - unified = f"(Failed to get unified response: {e})" - - # Build combined result - combined = ( - f"## Second Opinion from {reviewer_model}\n\n{review}" - f"\n\n---\n\n" - f"## {original_model}'s Response\n\n{unified}" - ) - - return { - "model": reviewer_model, - "response": combined, - "instruction": "Present these results to the user exactly as they are. Do NOT call second_opinion again. The user can continue the conversation from here.", - } - - -async def do_create_session(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: - """Create a new chat session. - - Content format: - Line 1: session name - Line 2: model_name (or model_name@endpoint_name) - """ - if not _session_manager: - return {"error": "Session manager not available"} - - lines = content.strip().split("\n") - if len(lines) < 2: - return {"error": "Need 2 lines: session name, then model spec"} - - name = lines[0].strip() - model_spec = lines[1].strip() - - if not name: - return {"error": "Session name cannot be empty"} - - try: - url, model, headers = _resolve_model(model_spec, owner=owner) - except ValueError as e: - return {"error": str(e)} - - sid = str(uuid.uuid4())[:8] - try: - _session_manager.create_session( - session_id=sid, - name=name, - endpoint_url=url, - model=model, - rag=False, - owner=owner, - ) - # Store headers on session for future calls - sess = _session_manager.get_session(sid) - if sess and headers: - sess.headers = headers - try: - from src.event_bus import fire_event - fire_event("session_created", owner) - except Exception: - logger.debug("session_created event dispatch failed", exc_info=True) - - return {"session_id": sid, "name": name, "model": model, "endpoint_url": url} - except Exception as e: - logger.error(f"create_session failed: {e}") - return {"error": f"Failed to create session: {e}"} - - -async def do_list_sessions(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: - """List sessions sorted by most-recently-active first. - - Output includes a relative "last active" timestamp per row so the - agent can answer "open my last chat" without guessing from titles. - The most-recent session is always first in the list. - - Content = optional filter keyword (matches session name). - """ - if not _session_manager: - return {"error": "Session manager not available"} - - keyword = content.strip().lower() if content.strip() else None - - try: - from core.database import SessionLocal, Session as DbSession - from datetime import datetime, timezone - - # Pull every session's last_accessed from the DB so we can sort - # by recency. In-memory sessions hold name + model + msg_count; - # the DB row holds the timestamps. - db = SessionLocal() - try: - db_rows = {r.id: r for r in db.query(DbSession).all()} - finally: - db.close() - - # SECURITY: scope to the caller's sessions. Passing None returned - # every user's sessions, which the agent tool then exposed via the - # "list my chats" reply. - sessions = _session_manager.get_sessions_for_user(owner) - rows = [] - for sid, sess in sessions.items(): - if keyword and keyword not in (sess.name or "").lower(): - continue - db_row = db_rows.get(sid) - # Prefer last_accessed; fall back to updated_at, then created_at. - ts = None - if db_row: - ts = getattr(db_row, 'last_accessed', None) or getattr(db_row, 'updated_at', None) or getattr(db_row, 'created_at', None) - rows.append((ts, sid, sess)) - - # Sort by timestamp DESC; rows without a timestamp sink to the bottom. - rows.sort(key=lambda r: r[0] or datetime.min, reverse=True) - - def _rel(ts): - if not ts: - return 'never' - now = datetime.utcnow() - try: - if ts.tzinfo is not None: - now = datetime.now(timezone.utc) - diff = (now - ts).total_seconds() - except Exception: - return 'unknown' - if diff < 60: return 'just now' - if diff < 3600: return f'{int(diff / 60)}m ago' - if diff < 86400: return f'{int(diff / 3600)}h ago' - if diff < 86400 * 7: return f'{int(diff / 86400)}d ago' - return ts.strftime('%Y-%m-%d') - - lines = [] - for i, (ts, sid, sess) in enumerate(rows): - if i >= 50: - lines.append(f"... and {len(rows) - 50} more (showing first 50)") - break - safe_name = (sess.name or "Untitled").replace("[", "\\[").replace("]", "\\]") - msg_count = getattr(sess, "message_count", 0) or 0 - model = getattr(sess, "model", "unknown") - marker = " ← most recent" if i == 0 else "" - lines.append(f"- **[{safe_name}](#session-{sid})** (id: `{sid}`, model: {model}, {msg_count} msgs, last active {_rel(ts)}){marker}") - - if not lines: - return {"results": "No sessions found" + (f" matching '{keyword}'" if keyword else "") + "."} - - return { - "results": ( - f"Found {len(rows)} session(s), sorted most-recent first:\n" - + "\n".join(lines) - + "\n\nAssistant: when replying to the user, preserve the chat-title markdown links exactly as shown, e.g. `[Chat](#session-id)`. Do not rewrite this as a plain, non-clickable table." - ) - } - except Exception as e: - logger.error(f"list_sessions failed: {e}") - return {"error": str(e)} - - -async def do_send_to_session(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: - """Send a message to an existing session and get a response. - - Content format: - Line 1: session_id - Line 2+: message - """ - from src.llm_core import llm_call_async - from core.models import ChatMessage - - if not _session_manager: - return {"error": "Session manager not available"} - - lines = content.strip().split("\n", 1) - if len(lines) < 2: - return {"error": "Need 2 lines: session_id, then message"} - - target_sid = lines[0].strip() - message = lines[1].strip() - - sess = _session_manager.get_session(target_sid) - if not sess: - return {"error": f"Session '{target_sid}' not found"} - - # Owner-scope: reject access to another user's session - if owner and getattr(sess, "owner", None) and sess.owner != owner: - return {"error": f"Session '{target_sid}' not found"} - - if not message: - return {"error": "No message provided"} - - try: - # Build context from session history - context = sess.get_context_messages() - context.append({"role": "user", "content": message}) - - response = await llm_call_async( - sess.endpoint_url, sess.model, context, - headers=sess.headers, - timeout=AI_CHAT_TIMEOUT, - ) - - # Save both messages to session - sess.add_message(ChatMessage("user", message)) - sess.add_message(ChatMessage("assistant", response)) - - # Truncate for tool output - if len(response) > 10000: - response = response[:10000] + "\n... (truncated)" - - return { - "session_id": target_sid, - "session_name": sess.name, - "response": response, - } - except Exception as e: - logger.error(f"send_to_session failed: {e}") - return {"error": f"Failed to send to session: {e}"} async def stream_ai_tool(tool: str, content: str, session_id: Optional[str] = None, owner: Optional[str] = None): @@ -715,229 +287,6 @@ async def do_pipeline(content: str, session_id: Optional[str] = None, owner: Opt # Session management tool # --------------------------------------------------------------------------- -async def do_manage_session(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: - """Manage sessions: rename, archive, delete, important, truncate, fork. - - Content format: - Line 1: action (rename|archive|unarchive|delete|important|unimportant|truncate|fork) - Line 2: target session_id (or "current" to use the active session) - Line 3+: action-specific params (e.g. new name for rename, keep_count for truncate) - """ - if not _session_manager: - return {"error": "Session manager not available"} - - from src.database import SessionLocal, Session as DbSession - - # Accept BOTH the structured JSON args the tool schema advertises - # ({action, session_id, value}) AND the legacy line-based format - # (line1=action, line2=session_id, line3=value). Native function-calling - # models send JSON; fenced-block callers send lines. Previously only the - # line format was parsed, so a model that followed the schema (JSON) got - # "Need at least 2 lines" / "Rename needs line 3" and couldn't drive it. - _raw = (content or "").strip() - action = "" - target_sid = "" - value = None # the action param: new name (rename) / keep_count (truncate, fork) - _list_filter = "" - _parsed = None - if _raw.startswith("{"): - try: - _parsed = json.loads(_raw) - except Exception: - _parsed = None - if isinstance(_parsed, dict): - action = str(_parsed.get("action") or "").strip().lower() - target_sid = str(_parsed.get("session_id") or _parsed.get("session") or _parsed.get("id") or "").strip() - _v = _parsed.get("value") - if _v is None: - _v = (_parsed.get("name") or _parsed.get("new_name") - or _parsed.get("title") or _parsed.get("keep_count")) - value = None if _v is None else str(_v).strip() - _list_filter = str(_parsed.get("filter") or "").strip() - else: - lines = _raw.split("\n") - if not lines or not lines[0].strip(): - return {"error": "Missing action (rename|archive|delete|important|truncate|fork|list|switch)"} - action = lines[0].strip().lower() - target_sid = lines[1].strip() if len(lines) >= 2 else "" - value = lines[2].strip() if len(lines) >= 3 else None - _list_filter = "\n".join(lines[1:]).strip() - - if not action: - return {"error": "Missing action (rename|archive|delete|important|truncate|fork|list|switch)"} - - # `list` alias — dispatch to do_list_sessions so the agent's natural - # first guess (every other manage_* tool has a `list` action) works. - if action == "list": - return await do_list_sessions(_list_filter, session_id, owner=owner) - - if not target_sid: - return {"error": "Need a session_id (or 'current' for the active chat)"} - - # Allow "current" to refer to the active session - if target_sid.lower() == "current" and session_id: - target_sid = session_id - - # `switch` / `open` / `select` / `view` — the agent reaches for - # these when the user asks to "open" or "switch to" a session. - # There's no server-side way to make the browser navigate, so we - # just return a clickable anchor link the user can click. The - # frontend's chat-history click delegate routes `#session-` - # to selectSession(). The agent's reply naturally embeds this - # result so the user sees a single clickable line. - def _session_query(db): - query = db.query(DbSession).filter(DbSession.id == target_sid) - if owner is not None: - query = query.filter(DbSession.owner == owner) - return query - - if action in ("switch", "open", "select", "view"): - db = SessionLocal() - try: - db_sess = _session_query(db).first() - if not db_sess: - return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} - name = db_sess.name or target_sid - finally: - db.close() - return { - "action": action, - "session_id": target_sid, - "name": name, - "results": f"[{name}](#session-{target_sid}) — click to open.", - } - - db = SessionLocal() - try: - if action == "rename": - if not value: - return {"error": "rename needs a new name (the `value` arg, or line 3 in the legacy format)"} - new_name = value - db_sess = _session_query(db).first() - if not db_sess: - return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} - db_sess.name = new_name - db.commit() - _session_manager.update_session_name(target_sid, new_name) - return {"action": "rename", "session_id": target_sid, "name": new_name, - "results": f"Session renamed to '{new_name}'"} - - elif action == "archive": - db_sess = _session_query(db).first() - if not db_sess: - return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} - db_sess.archived = True - db.commit() - return {"action": "archive", "session_id": target_sid, - "results": f"Session '{db_sess.name}' archived"} - - elif action == "unarchive": - db_sess = _session_query(db).first() - if not db_sess: - return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} - db_sess.archived = False - db.commit() - return {"action": "unarchive", "session_id": target_sid, - "results": f"Session '{db_sess.name}' unarchived"} - - elif action == "delete": - if target_sid == session_id: - return {"error": "Cannot delete the current session while chatting in it. Delete other sessions first."} - db_sess = _session_query(db).first() - if not db_sess: - return {"error": f"Session '{target_sid}' not found. Refusing to delete an unknown chat id; use the exact id from list_sessions."} - if db_sess and db_sess.is_important: - return {"error": f"Session '{db_sess.name}' is starred/favorited. Unstar it first before deleting."} - try: - ok = _session_manager.delete_session(target_sid) - if not ok: - return {"error": f"Session '{target_sid}' was not deleted because it no longer exists."} - return {"action": "delete", "session_id": target_sid, - "results": f"Session '{db_sess.name or target_sid}' deleted"} - except Exception as e: - return {"error": f"Failed to delete session: {e}"} - - elif action in ("important", "unimportant"): - is_important = action == "important" - db_sess = _session_query(db).first() - if not db_sess: - return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} - # Prevent AI from unstarring sessions — only the user can do that manually - if not is_important and db_sess.is_important: - return {"error": f"Session '{db_sess.name}' is starred by the user. Only the user can unstar sessions manually."} - db_sess.is_important = is_important - db.commit() - status = "marked as important" if is_important else "unmarked as important" - return {"action": action, "session_id": target_sid, - "results": f"Session '{db_sess.name}' {status}"} - - elif action == "truncate": - db_sess = _session_query(db).first() - if not db_sess: - return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} - keep_count = 10 - if value: - try: - keep_count = int(value) - except ValueError: - pass - success = _session_manager.truncate_messages(target_sid, keep_count) - if success: - return {"action": "truncate", "session_id": target_sid, - "results": f"Session truncated to last {keep_count} messages"} - return {"error": f"Failed to truncate session '{target_sid}'"} - - elif action == "fork": - db_sess = _session_query(db).first() - if not db_sess: - return {"error": f"Session '{target_sid}' not found. Use list_sessions and pass the exact id it returned."} - keep_count = 0 # 0 = all messages - if value: - try: - keep_count = int(value) - except ValueError: - pass - - source = _session_manager.get_session(target_sid) - if not source: - return {"error": f"Session '{target_sid}' not found"} - - new_sid = str(uuid.uuid4())[:8] - _session_manager.create_session( - session_id=new_sid, - name=f"Fork: {source.name}", - endpoint_url=source.endpoint_url, - model=source.model, - rag=False, - owner=owner, - ) - # Copy messages - history = source.get_context_messages() - if keep_count > 0: - history = history[:keep_count] - from core.models import ChatMessage as InMemoryMsg - new_sess = _session_manager.get_session(new_sid) - for msg in history: - new_sess.add_message(InMemoryMsg(msg["role"], msg["content"])) - try: - from src.event_bus import fire_event - fire_event("session_created", owner) - except Exception: - logger.debug("session_created event dispatch failed", exc_info=True) - - return {"action": "fork", "session_id": new_sid, - "source_session": target_sid, "messages_copied": len(history), - "results": f"Forked session '{source.name}' -> new session {new_sid} ({len(history)} messages)"} - - else: - return {"error": f"Unknown action '{action}'. Use: list, switch, rename, archive, unarchive, delete, important, unimportant, truncate, fork"} - except Exception as e: - logger.error(f"manage_session failed: {e}") - return {"error": str(e)} - finally: - db.close() - - # --------------------------------------------------------------------------- # Memory management tool # --------------------------------------------------------------------------- @@ -1104,83 +453,6 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner return {"error": f"Unknown action '{action}'. Use: list, add, edit, delete, search"} -# --------------------------------------------------------------------------- -# List models tool -# --------------------------------------------------------------------------- - -async def do_list_models(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: - """List all available models across configured endpoints. - - Content = optional filter keyword. - """ - import httpx - from src.database import SessionLocal, ModelEndpoint - from src.llm_core import _detect_provider, ANTHROPIC_MODELS - from src.auth_helpers import owner_filter - - keyword = content.strip().lower() if content.strip() else None - - db = SessionLocal() - try: - query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) - if owner: - query = owner_filter(query, ModelEndpoint, owner) - endpoints = query.all() - if not endpoints: - return {"results": "No enabled model endpoints configured."} - - result_lines = [] - total_models = 0 - - for ep in endpoints: - try: - base, api_key = resolve_endpoint_runtime(ep, owner=owner) - except Exception: - continue - provider = _detect_provider(base) - headers = build_headers(api_key, base) - - model_ids = [] - if provider == "anthropic": - model_ids = list(ANTHROPIC_MODELS) - else: - try: - models_url = build_models_url(base) - if models_url: - r = httpx.get(models_url, headers=headers, timeout=5) - r.raise_for_status() - data = r.json() - model_ids = [m.get("id") for m in (data.get("data") or []) if m.get("id")] - if not model_ids: - model_ids = [ - m.get("name") or m.get("model") - for m in (data.get("models") or []) - if m.get("name") or m.get("model") - ] - else: - model_ids = json.loads(ep.cached_models or "[]") - except Exception: - model_ids = ["(endpoint offline)"] - - if keyword: - model_ids = [m for m in model_ids if keyword in m.lower() or keyword in (ep.name or "").lower()] - - if model_ids: - result_lines.append(f"\n**{ep.name or base}** ({provider}):") - for mid in model_ids: - result_lines.append(f" - `{mid}`") - total_models += 1 - - if not result_lines: - return {"results": "No models found" + (f" matching '{keyword}'" if keyword else "") + "."} - - header = f"Available models ({total_models} total):" - return {"results": header + "\n".join(result_lines)} - except Exception as e: - logger.error(f"list_models failed: {e}") - return {"error": str(e)} - finally: - db.close() # --------------------------------------------------------------------------- @@ -1613,7 +885,9 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne """ import base64 import httpx + import os from pathlib import Path + from src.url_safety import check_outbound_url lines = content.strip().split("\n") prompt = lines[0].strip() if lines else "" @@ -1779,8 +1053,15 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne elif img.get("url"): # Download external URL and save locally (DALL-E returns temp URLs) + result_url = img["url"] + ok, reason = check_outbound_url( + result_url, + block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + return {"error": f"Image API returned unsafe image URL: {reason}"} try: - dl_resp = httpx.get(img["url"], timeout=60) + dl_resp = httpx.get(result_url, timeout=60) if dl_resp.status_code == 200: img_dir = Path(GENERATED_IMAGES_DIR) img_dir.mkdir(parents=True, exist_ok=True) @@ -1790,10 +1071,10 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne image_url = f"/api/generated-image/{filename}" image_id = _save_to_gallery(filename) else: - image_url = img["url"] # fallback to external URL + image_url = result_url # fallback to external URL except Exception as _dl_e: logger.warning(f"Failed to download DALL-E image: {_dl_e}") - image_url = img["url"] # fallback to external URL + image_url = result_url # fallback to external URL else: return {"error": "Image API returned unexpected format (no b64_json or url)"} @@ -1822,55 +1103,20 @@ async def dispatch_ai_tool( ) -> Tuple[str, Dict]: """Dispatch an AI interaction tool. Returns (description, result_dict).""" - if tool == "chat_with_model": - model_spec = content.split("\n")[0].strip()[:60] - desc = f"chat_with_model: {model_spec}" - result = await do_chat_with_model(content, session_id, owner=owner) - - elif tool == "create_session": - name = content.split("\n")[0].strip()[:60] - desc = f"create_session: {name}" - result = await do_create_session(content, session_id, owner=owner) - - elif tool == "list_sessions": - keyword = content.strip()[:40] - desc = f"list_sessions{': ' + keyword if keyword else ''}" - result = await do_list_sessions(content, session_id, owner=owner) - - elif tool == "send_to_session": - sid = content.split("\n")[0].strip()[:20] - desc = f"send_to_session: {sid}" - result = await do_send_to_session(content, session_id, owner=owner) - - elif tool == "pipeline": + if tool == "pipeline": desc = "pipeline: running steps" result = await do_pipeline(content, session_id, owner=owner) - elif tool == "manage_session": - action = content.split("\n")[0].strip()[:40] - desc = f"manage_session: {action}" - result = await do_manage_session(content, session_id, owner=owner) - elif tool == "manage_memory": action = content.split("\n")[0].strip()[:40] desc = f"manage_memory: {action}" result = await do_manage_memory(content, session_id, owner=owner) - elif tool == "list_models": - keyword = content.strip()[:40] - desc = f"list_models{': ' + keyword if keyword else ''}" - result = await do_list_models(content, session_id, owner=owner) - elif tool == "ui_control": action = content.split("\n")[0].strip()[:60] desc = f"ui_control: {action}" result = await do_ui_control(content, session_id, owner=owner) - elif tool == "ask_teacher": - problem = content.split("\n", 1)[-1].strip()[:60] - desc = f"ask_teacher: {problem}" - result = await do_ask_teacher(content, session_id, owner=owner) - else: desc = f"unknown ai tool: {tool}" result = {"error": f"Unknown AI interaction tool: {tool}"} diff --git a/src/bg_jobs.py b/src/bg_jobs.py index 8e452106b..f864f8ef1 100644 --- a/src/bg_jobs.py +++ b/src/bg_jobs.py @@ -263,10 +263,32 @@ def list_for_session(session_id: str) -> List[Dict[str, Any]]: return [r for r in refresh().values() if r.get("session_id") == session_id] +def kill(job_id: str) -> Optional[Dict[str, Any]]: + """Terminate a running job's process tree and mark it killed. Returns the + updated record, or None if the id is unknown. Idempotent: a job that already + finished is returned unchanged. Sets followed_up so the monitor does not also + fire an auto-continue for a job the agent deliberately stopped.""" + jobs = _load() + rec = jobs.get(job_id) + if rec is None: + return None + if rec.get("status") == "running": + _kill(rec.get("pid")) + rec["status"] = "failed" + rec["exit_code"] = -1 + rec["ended_at"] = time.time() + rec["killed"] = True + rec["followed_up"] = True + _save(jobs) + return rec + + def result_text(rec: Dict[str, Any]) -> str: """Human/agent-readable summary of a finished job, for the follow-up.""" out = _read_output(rec) - if rec.get("timed_out"): + if rec.get("killed"): + head = "Background job was killed." + elif rec.get("timed_out"): head = f"Background job timed out after {rec.get('max_runtime_s')}s." elif rec.get("died"): head = "Background job process died unexpectedly (no exit code)." diff --git a/src/builtin_mcp.py b/src/builtin_mcp.py index 0154d2fb9..93ef0ee61 100644 --- a/src/builtin_mcp.py +++ b/src/builtin_mcp.py @@ -14,6 +14,7 @@ import subprocess import sys from core.platform_compat import IS_WINDOWS, which_tool +from src.runtime_paths import get_app_root logger = logging.getLogger(__name__) @@ -81,7 +82,7 @@ _BUILTIN_NPX_SERVERS = { "name": "Built-in: Browser", "command": "npx", "args": ["-y", "@playwright/mcp@latest", "--headless", "--caps", "vision"], - }, + } } # Global flag to disable MCP if there are compatibility issues @@ -94,7 +95,7 @@ async def register_builtin_servers(mcp_manager): logger.info("Built-in MCP servers disabled via ODYSSEUS_DISABLE_MCP") return - base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + base_dir = get_app_root() python = sys.executable async def _connect_python_server(server_id: str, script_path: str, name: str): diff --git a/src/config.py b/src/config.py index 8b9bd5148..d5cfa21a7 100644 --- a/src/config.py +++ b/src/config.py @@ -5,6 +5,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict from pydantic import Field, field_validator from src.constants import DATA_DIR as _DATA_DIR_CONST +from src.runtime_paths import get_app_root # Cross-platform OS flag, exposed here so callers can `from src.config import # IS_WINDOWS`. Defined locally (a trivial `os.name == "nt"`) rather than imported @@ -19,7 +20,7 @@ IS_WINDOWS = os.name == "nt" class DataConfig(BaseSettings): """Configuration for data storage and file handling.""" # Base directory - base_dir: Path = Field(default=Path(__file__).parent.parent, description="Base directory for the application") + base_dir: Path = Field(default=Path(get_app_root()), description="Base directory for the application") # Data paths data_dir: Path = Field(default=Path(_DATA_DIR_CONST), description="Main data directory") @@ -138,7 +139,7 @@ class AppConfig(BaseSettings): if isinstance(v, dict) and "base_dir" in v: base_dir = v["base_dir"] else: - base_dir = Path(__file__).parent.parent + base_dir = Path(get_app_root()) # Convert string paths to Path objects relative to base_dir data_dir = Path(_DATA_DIR_CONST) diff --git a/src/constants.py b/src/constants.py index 16aecf19a..eceeb6eb0 100644 --- a/src/constants.py +++ b/src/constants.py @@ -2,12 +2,14 @@ """Application-wide constants and configuration values.""" import os +from src.runtime_paths import get_app_root, get_default_data_dir + APP_VERSION = "1.0.1" # Base paths -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/" +BASE_DIR = os.path.join(get_app_root(), "") STATIC_DIR = os.path.join(BASE_DIR, "static") -DATA_DIR = os.getenv("ODYSSEUS_DATA_DIR", os.path.join(BASE_DIR, "data")) +DATA_DIR = os.getenv("ODYSSEUS_DATA_DIR", get_default_data_dir()) # Data file paths # Single source of truth: every persisted file/dir lives under DATA_DIR, which @@ -55,7 +57,13 @@ MEMORY_VECTORS_DIR = os.path.join(DATA_DIR, "memory_vectors") # Paths with an intentional dedicated env override, defaulting under DATA_DIR. MAIL_ATTACHMENTS_DIR = os.getenv("ODYSSEUS_MAIL_ATTACHMENTS_DIR", os.path.join(DATA_DIR, "mail-attachments")) -FASTEMBED_CACHE_DIR = os.getenv("FASTEMBED_CACHE_PATH", os.path.join(DATA_DIR, "fastembed_cache")) +# `or` (not os.getenv's default arg) so a PRESENT-but-EMPTY value falls back to +# the default. docker-compose.yml injects `FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-}`, +# which sets the var to "" when the host hasn't defined it. os.getenv(name, default) +# only returns the default when the var is ABSENT, so the empty string would win → +# os.makedirs("") raises [Errno 2] No such file or directory: '' → FastEmbed fails to +# init and all vector features (RAG, semantic memory, tool index) silently degrade. +FASTEMBED_CACHE_DIR = os.getenv("FASTEMBED_CACHE_PATH") or os.path.join(DATA_DIR, "fastembed_cache") # Agent tool output limits (single source of truth — imported by tool_execution.py, # tool_implementations.py, agent_tools.py, and any other module that needs them) @@ -63,11 +71,26 @@ MAX_OUTPUT_CHARS = 10_000 # cap for bash/python/web_search/web_fetch outpu MAX_READ_CHARS = 20_000 # cap for read_file / document preview MAX_DIFF_LINES = 400 # cap for edit_file unified-diff display +# web_fetch response-size policy (#3812). MAX_OUTPUT_CHARS above only trims +# what the agent SEES; these caps bound what the server downloads, parses, +# and writes to the content cache. The soft cap is the default download +# budget; the agent can raise it per call (full/max_bytes) but never past +# the hard cap, so a model can't decide to pull a multi-GB file. +WEB_FETCH_SOFT_MAX_BYTES = 2_000_000 # default download budget (2 MB) +WEB_FETCH_HARD_MAX_BYTES = 20_000_000 # absolute ceiling, even with override (20 MB) + # API Configuration MAX_CONTEXT_MESSAGES = 90 REQUEST_TIMEOUT = 20 OPENAI_COMPAT_PATH = "/v1/chat/completions" +# Outbound UA for web_fetch / web_search scraping; common desktop UA so pages serve normal HTML. +WEB_FETCH_USER_AGENT = os.environ.get( + "WEB_FETCH_USER_AGENT", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", +) + # Environment variables with defaults DEFAULT_HOST = os.getenv("LLM_HOST", "localhost") LLM_HOSTS = [h.strip() for h in os.getenv("LLM_HOSTS", "").split(",") if h.strip()] @@ -79,6 +102,9 @@ SEARXNG_INSTANCE = os.getenv("SEARXNG_INSTANCE", "http://localhost:8080") CLEANUP_ENABLED = os.getenv("CLEANUP_ENABLED", "True").lower() == "true" CLEANUP_INTERVAL_HOURS = int(os.getenv("CLEANUP_INTERVAL_HOURS", "24")) +# Auth policy +PASSWORD_MIN_LENGTH = 8 + # Default parameters DEFAULT_TEMPERATURE = 1.0 DEFAULT_MAX_TOKENS = 0 diff --git a/src/cookbook_serve_lifecycle.py b/src/cookbook_serve_lifecycle.py index fcdacbe7a..f2700cf7d 100644 --- a/src/cookbook_serve_lifecycle.py +++ b/src/cookbook_serve_lifecycle.py @@ -161,11 +161,13 @@ async def _tick() -> None: # Re-read state once before writing so we capture any updates from # concurrent UI syncs. stopped_any = False + successfully_stopped_sids = set() for sid, host, port in to_stop: ok = await _stop_serve(sid, host, port) logger.info(f"cookbook_serve_lifecycle: stop {sid} (host={host or 'local'}): {'ok' if ok else 'failed'}") if ok: stopped_any = True + successfully_stopped_sids.add(sid) # Drop the auto-registered endpoint so the model picker and # the chat router don't keep pointing at a dead server. for t in tasks: @@ -188,12 +190,11 @@ async def _tick() -> None: except Exception: fresh = state fresh_tasks = tasks - stopped_sids = {sid for sid, _, _ in to_stop} for ft in fresh_tasks: if not isinstance(ft, dict): continue ft_sid = ft.get("sessionId") or ft.get("id") - if ft_sid in stopped_sids: + if ft_sid in successfully_stopped_sids: ft["status"] = "stopped" ft["_scheduledStopAtMs"] = None ft["_lastStatusFlipAt"] = now_ms diff --git a/src/embeddings.py b/src/embeddings.py index 85a55c386..746044c47 100644 --- a/src/embeddings.py +++ b/src/embeddings.py @@ -31,6 +31,8 @@ import numpy as np import httpx from typing import List, Optional +from src.runtime_paths import get_app_root + logger = logging.getLogger(__name__) _DEFAULT_MODEL = "all-minilm:l6-v2" diff --git a/src/endpoint_resolver.py b/src/endpoint_resolver.py index ac5a6b7ad..57361c673 100644 --- a/src/endpoint_resolver.py +++ b/src/endpoint_resolver.py @@ -161,6 +161,32 @@ def normalize_base(url: str) -> str: return url +def _validated_endpoint_base(url: str) -> str: + """Return a base URL that is safe for endpoint path appends.""" + base = (url or "").strip().rstrip("/") + if "?" in base or "#" in base: + raise ValueError("Endpoint base URL must not include query or fragment") + return urlunparse(urlparse(base)._replace(query="", fragment="")).rstrip("/") + + +def _prepare_endpoint_base(base: str) -> str: + base = _validated_endpoint_base(normalize_base(base)) + return _validated_endpoint_base(normalize_base(resolve_url(base))) + + +def _append_endpoint_path(base: str, suffix: str) -> str: + parsed = urlparse(base) + current = (parsed.path or "").rstrip("/") + extra = "/" + suffix.lstrip("/") + path = f"{current}{extra}" if current else extra + return urlunparse(parsed._replace(path=path, query="", fragment="")) + + +def _pathless_host(base: str, host: str) -> bool: + parsed = urlparse(base) + return (parsed.hostname or "").lower() == host and not (parsed.path or "").strip("/") + + def _anthropic_api_root(base: str) -> str: """Return Anthropic's API root, preserving /v1 for OpenAI-compatible APIs elsewhere.""" base = (base or "").strip().rstrip("/") @@ -171,15 +197,17 @@ def _anthropic_api_root(base: str) -> str: def build_chat_url(base: str) -> str: """Return the correct chat endpoint URL for a given base.""" - base = resolve_url(base) + base = _prepare_endpoint_base(base) provider = _detect_provider(base) if provider == "anthropic": - return _anthropic_api_root(base) + "/v1/messages" + return _append_endpoint_path(_anthropic_api_root(base), "/v1/messages") if provider == "ollama": - return _ollama_api_root(base) + "/chat" + return _append_endpoint_path(_ollama_api_root(base), "/chat") if provider == "chatgpt-subscription": - return base.rstrip("/") + "/responses" - return base + "/chat/completions" + return _append_endpoint_path(base, "/responses") + if _pathless_host(base, "api.openai.com"): + base = _append_endpoint_path(base, "/v1") + return _append_endpoint_path(base, "/chat/completions") def build_models_url(base: str) -> Optional[str]: @@ -193,12 +221,12 @@ def build_models_url(base: str) -> Optional[str]: untouched (so custom prefixes like ``/openai`` or ``/api/openai/v1`` keep their semantics). """ - base = normalize_base(resolve_url(base)) + base = _prepare_endpoint_base(base) provider = _detect_provider(base) if provider == "anthropic": - return _anthropic_api_root(base) + "/v1/models" + return _append_endpoint_path(_anthropic_api_root(base), "/v1/models") if provider == "ollama": - return _ollama_api_root(base) + "/tags" + return _append_endpoint_path(_ollama_api_root(base), "/tags") if provider == "chatgpt-subscription": return None # Generic OpenAI-compatible fallback: local model servers with no explicit @@ -208,10 +236,10 @@ def build_models_url(base: str) -> Optional[str]: parsed = urlparse(base) host = (parsed.hostname or "").lower() is_local = host in {"localhost", "127.0.0.1", "::1", "host.docker.internal"} - uses_v1_models_by_default = is_local or host in {"api.deepseek.com"} + uses_v1_models_by_default = is_local or host in {"api.deepseek.com", "api.openai.com"} if not parsed.path and uses_v1_models_by_default: - base = base + "/v1" - return base + "/models" + base = _append_endpoint_path(base, "/v1") + return _append_endpoint_path(base, "/models") def build_headers(api_key: Optional[str], base: str) -> Dict[str, str]: diff --git a/src/integrations.py b/src/integrations.py index 54357511f..3b2b88859 100644 --- a/src/integrations.py +++ b/src/integrations.py @@ -4,6 +4,7 @@ import uuid import logging import re from typing import Dict, List, Optional, Any +from urllib.parse import urljoin, urlparse, urlunparse import httpx from fastapi import HTTPException @@ -202,6 +203,22 @@ def mask_integration_secret(integration: Dict[str, Any]) -> Dict[str, Any]: return safe +def _normalize_integration_base_url(base_url: Any) -> str: + if not isinstance(base_url, str) or not base_url.strip(): + raise ValueError("Integration base URL is required") + cleaned = base_url.strip().rstrip("/") + if "?" in cleaned or "#" in cleaned: + raise ValueError("Integration base URL must not include query or fragment") + parsed = urlparse(cleaned) + if parsed.scheme.lower() not in ("http", "https") or not parsed.hostname: + raise ValueError("Integration base URL must be an HTTP(S) URL") + return urlunparse(parsed._replace(scheme=parsed.scheme.lower(), query="", fragment="")).rstrip("/") + + +def _join_integration_url(base_url: str, path: str) -> str: + return urljoin(base_url.rstrip("/") + "/", path.lstrip("/")) + + def load_integrations() -> List[Dict[str, Any]]: """Load all integrations from disk with secrets decrypted for runtime use.""" if not os.path.exists(DATA_FILE): @@ -261,8 +278,10 @@ def add_integration(data: Dict[str, Any]) -> Dict[str, Any]: if not isinstance(integration.get("name"), str) or not integration["name"].strip(): raise HTTPException(400, "Integration name is required") - if not isinstance(integration.get("base_url"), str) or not integration["base_url"].strip(): - raise HTTPException(400, "Integration base URL is required") + try: + integration["base_url"] = _normalize_integration_base_url(integration.get("base_url")) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc integrations = load_integrations() integrations.append(integration) @@ -272,10 +291,14 @@ def add_integration(data: Dict[str, Any]) -> Dict[str, Any]: def update_integration(integration_id: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Update fields on an existing integration. Returns updated integration or None.""" + data = dict(data) if "name" in data and (not isinstance(data["name"], str) or not data["name"].strip()): raise HTTPException(400, "Integration name is required") - if "base_url" in data and (not isinstance(data["base_url"], str) or not data["base_url"].strip()): - raise HTTPException(400, "Integration base URL is required") + if "base_url" in data: + try: + data["base_url"] = _normalize_integration_base_url(data["base_url"]) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc integrations = load_integrations() for item in integrations: @@ -341,9 +364,10 @@ async def execute_api_call( if not integration.get("enabled", True): return {"error": f"Integration '{integration.get('name')}' is disabled", "exit_code": 1} - base_url = integration.get("base_url", "").rstrip("/") - if not base_url: - return {"error": "Integration has no base_url configured", "exit_code": 1} + try: + base_url = _normalize_integration_base_url(integration.get("base_url", "")) + except ValueError as exc: + return {"error": str(exc), "exit_code": 1} # Strip common API path suffixes users might accidentally include # (e.g. "http://host/v1/" → "http://host"). The integration's preset @@ -366,7 +390,10 @@ async def execute_api_call( if re.search(r"^https?://", path) or "://" in path: return {"error": "Path must not contain a protocol scheme", "exit_code": 1} - url = base_url + path + if "#" in path: + return {"error": "Path must not contain a fragment", "exit_code": 1} + + url = _join_integration_url(base_url, path) method = method.upper() # Build headers diff --git a/src/llm_core.py b/src/llm_core.py index ba567ed81..9981e41e2 100644 --- a/src/llm_core.py +++ b/src/llm_core.py @@ -87,7 +87,7 @@ _host_health_lock = threading.Lock() _model_activity: Dict[str, float] = {} _HARMONY_MARKER_RE = re.compile( - r"<\|channel\|>(analysis|final)" + r"<\|channel\|>(analysis|commentary|final)" r"|<\|start\|>(?:assistant|system|user|tool)?" r"|<\|message\|>" r"|<\|end\|>" @@ -96,6 +96,7 @@ _HARMONY_MARKER_RE = re.compile( ) _HARMONY_MARKERS = ( "<|channel|>analysis", + "<|channel|>commentary", "<|channel|>final", "<|start|>assistant", "<|start|>system", @@ -145,7 +146,10 @@ class _HarmonyStreamRouter: out.append((text, False)) return if self._in_message: - out.append((text, self._channel == "analysis")) + # analysis + commentary (tool-call preambles / function-arg bodies) + # are internal, not user-facing — route them to thinking so they + # don't leak into the visible answer; only `final` is visible. + out.append((text, self._channel in ("analysis", "commentary"))) def _handle_marker(self, match: re.Match[str]) -> None: marker = match.group(0) @@ -283,7 +287,8 @@ def _is_ollama_native_url(url: str) -> bool: """Return True for native Ollama API URLs, including Ollama Cloud.""" try: parsed = urlparse(url or "") - except Exception: + except Exception as e: + logger.warning("Failed to parse URL for Ollama detection", exc_info=e) return False host = parsed.hostname or "" path = (parsed.path or "").rstrip("/") @@ -1345,8 +1350,8 @@ def list_model_ids( r = httpx.get(root + "/api/tags", timeout=timeout) r.raise_for_status() return [m.get("name") or m.get("model") for m in (r.json().get("models") or []) if m.get("name") or m.get("model")] - except Exception: - pass + except Exception as e: + logger.warning("Failed to fetch model list from configured endpoint", exc_info=e) return [] def normalize_model_id( diff --git a/src/mcp_manager.py b/src/mcp_manager.py index 29fdedebf..8f4322375 100644 --- a/src/mcp_manager.py +++ b/src/mcp_manager.py @@ -11,6 +11,8 @@ import os import re from typing import Any, Dict, List, Optional, Set, Tuple +from src.runtime_paths import get_app_root + logger = logging.getLogger(__name__) def _format_mcp_connection_error(name: str, command: str = "", args: Optional[List[str]] = None, error: Exception = None) -> str: @@ -508,7 +510,7 @@ class McpManager: return False script_rel, name = _BUILTIN_SERVERS[server_id] - base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + base_dir = get_app_root() script_path = os.path.join(base_dir, script_rel) # Clean up old connection diff --git a/src/model_context.py b/src/model_context.py index d87168cca..72526e744 100644 --- a/src/model_context.py +++ b/src/model_context.py @@ -17,10 +17,11 @@ import httpx logger = logging.getLogger(__name__) _LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1", "host.docker.internal"} -_PRIVATE_PREFIXES = ("10.", "172.16.", "172.17.", "172.18.", "172.19.", - "172.20.", "172.21.", "172.22.", "172.23.", "172.24.", - "172.25.", "172.26.", "172.27.", "172.28.", "172.29.", - "172.30.", "172.31.", "192.168.") +_PRIVATE_NETWORKS = ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), +) # Tailscale uses the CGNAT range 100.64.0.0/10, NOT all of 100.0.0.0/8. # A bare "100." prefix would classify public addresses (e.g. AWS ranges @@ -36,6 +37,14 @@ def _in_tailscale_range(host: str) -> bool: return False +def _is_private_ip_literal(host: str) -> bool: + try: + ip = ipaddress.ip_address(host) + except ValueError: + return False + return any(ip in network for network in _PRIVATE_NETWORKS) + + def _normalize_base_for_compare(url: str) -> str: url = (url or "").strip().rstrip("/") for suffix in ("/chat/completions", "/models", "/completions", "/v1/messages"): @@ -87,7 +96,7 @@ def is_local_endpoint(url: str) -> bool: return True try: host = urlparse(url).hostname or "" - return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES) or _in_tailscale_range(host) + return host in _LOCAL_HOSTS or _is_private_ip_literal(host) or _in_tailscale_range(host) except Exception: return False diff --git a/src/personal_docs.py b/src/personal_docs.py index 92ba1bc66..7ffb5cfb9 100644 --- a/src/personal_docs.py +++ b/src/personal_docs.py @@ -322,6 +322,47 @@ class PersonalDocsManager: else: logger.info(f"Directory not in index: {directory}") + def rename_directory(self, old_directory: str, new_directory: str, *, path_map: Dict[str, str] = None): + """Rewrite tracked directory and excluded-file paths after an owner rename.""" + old_directory = os.path.abspath(old_directory) + new_directory = os.path.abspath(new_directory) + path_map = {os.path.abspath(k): os.path.abspath(v) for k, v in (path_map or {}).items()} + + def rewrite(path: str) -> str: + abs_path = os.path.abspath(path) + mapped = path_map.get(abs_path) + if mapped: + return mapped + if abs_path == old_directory: + return new_directory + if abs_path.startswith(old_directory + os.sep): + return new_directory + abs_path[len(old_directory):] + return abs_path + + changed_dirs = False + rewritten_dirs = [] + for directory in self.indexed_directories: + rewritten = rewrite(directory) + changed_dirs = changed_dirs or rewritten != os.path.abspath(directory) + if rewritten not in rewritten_dirs: + rewritten_dirs.append(rewritten) + if changed_dirs: + self.indexed_directories = rewritten_dirs + self.save_directories() + + changed_excluded = False + rewritten_excluded = set() + for path in self.excluded_files: + rewritten = rewrite(path) + changed_excluded = changed_excluded or rewritten != os.path.abspath(path) + rewritten_excluded.add(rewritten) + if changed_excluded: + self.excluded_files = rewritten_excluded + self._save_excluded() + + if changed_dirs or changed_excluded: + self.refresh_index() + def get_indexed_directories(self): """Get the list of all indexed directories.""" return self.indexed_directories.copy() diff --git a/src/rag_singleton.py b/src/rag_singleton.py index 7bc5d74b4..9fa728293 100644 --- a/src/rag_singleton.py +++ b/src/rag_singleton.py @@ -7,6 +7,7 @@ import time from pathlib import Path from src.constants import RAG_DIR +from src.runtime_paths import get_app_root logger = logging.getLogger(__name__) diff --git a/src/rag_vector.py b/src/rag_vector.py index fc66c82e1..9a4c67cfa 100644 --- a/src/rag_vector.py +++ b/src/rag_vector.py @@ -50,6 +50,23 @@ def _generate_doc_id(text: str, owner: str = "") -> str: return f"doc_{hashlib.sha256(key.encode('utf-8')).hexdigest()[:16]}" +def _rewrite_owner_path(value: str, path_map: Dict[str, str], path_prefixes: List[tuple]) -> str: + if not isinstance(value, str) or not value: + return value + abs_value = os.path.abspath(value) + mapped = path_map.get(abs_value) + if mapped: + return mapped + for old_prefix, new_prefix in path_prefixes: + old_abs = os.path.abspath(old_prefix) + new_abs = os.path.abspath(new_prefix) + if abs_value == old_abs: + return new_abs + if abs_value.startswith(old_abs + os.sep): + return new_abs + abs_value[len(old_abs):] + return value + + class VectorRAG: """RAG system using ChromaDB vector storage with hybrid search.""" @@ -250,6 +267,75 @@ class VectorRAG: "failed_count": len(docs) - len(valid), } + def rename_owner( + self, + old_owner: str, + new_owner: str, + *, + path_map: Optional[Dict[str, str]] = None, + path_prefixes: Optional[List[tuple]] = None, + ) -> Dict[str, Any]: + """Rewrite existing RAG metadata after an auth username rename.""" + if not self.healthy: + return {"success": False, "updated_count": 0, "message": "Collection not initialized"} + + old_owner = (old_owner or "").strip().lower() + new_owner = (new_owner or "").strip().lower() + if not old_owner or not new_owner or old_owner == new_owner: + return {"success": True, "updated_count": 0, "message": "No owner rename needed"} + + path_map = {os.path.abspath(k): os.path.abspath(v) for k, v in (path_map or {}).items()} + path_prefixes = path_prefixes or [] + updated_ids = set() + failed_count = 0 + + for lane_name, collection in self._collections_for_delete(): + try: + results = collection.get( + where={"owner": old_owner}, + include=["metadatas"], + ) + except Exception as e: + logger.warning("rename_owner metadata scan failed in %s lane: %s", lane_name, e) + failed_count += 1 + continue + + ids = results.get("ids") or [] + metadatas = results.get("metadatas") or [] + if not ids: + continue + + new_metas = [] + selected_ids = [] + for doc_id, meta in zip(ids, metadatas): + if not isinstance(meta, dict): + continue + next_meta = dict(meta) + if str(next_meta.get("owner", "")).strip().lower() == old_owner: + next_meta["owner"] = new_owner + for key in ("source", "directory"): + next_meta[key] = _rewrite_owner_path(next_meta.get(key), path_map, path_prefixes) + selected_ids.append(doc_id) + new_metas.append(next_meta) + + if not selected_ids: + continue + + try: + collection.update(ids=selected_ids, metadatas=new_metas) + updated_ids.update(selected_ids) + except Exception as e: + logger.warning("rename_owner metadata update failed in %s lane: %s", lane_name, e) + failed_count += len(selected_ids) + + success = failed_count == 0 + return { + "success": success, + "updated_count": len(updated_ids), + "failed_count": failed_count, + "message": f"Updated {len(updated_ids)} RAG chunk(s)", + } + # ------------------------------------------------------------------ # Search — hybrid: vector similarity + keyword overlap # ------------------------------------------------------------------ diff --git a/src/runtime_paths.py b/src/runtime_paths.py new file mode 100644 index 000000000..9a8ffe7f9 --- /dev/null +++ b/src/runtime_paths.py @@ -0,0 +1,30 @@ +"""Helpers for resolving runtime paths in source and frozen builds.""" + +import os +import sys + + +def get_app_root() -> str: + """Return the app root directory. + + In normal source runs, this is the repository root. In a frozen Windows + build, it is the bundle content root (PyInstaller's internal directory) + so bundled runtime folders like `static/`, `scripts/`, and `data/` stay + together with the executable payload. + """ + if getattr(sys, "frozen", False): + return getattr(sys, "_MEIPASS", os.path.dirname(os.path.abspath(sys.executable))) + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def get_default_data_dir() -> str: + """Return the default path to the data directory. + + In normal runs, this is a 'data' subdirectory under the app root. + In frozen builds, it is a persistent user directory (~/.odysseus/data) + to prevent SQLite databases and other persistent files from being + written to the ephemeral, temporary extraction bundle directory. + """ + if getattr(sys, "frozen", False): + return os.path.join(os.path.expanduser("~"), ".odysseus", "data") + return os.path.join(get_app_root(), "data") \ No newline at end of file diff --git a/src/task_scheduler.py b/src/task_scheduler.py index b9ff51b6b..e5389df99 100644 --- a/src/task_scheduler.py +++ b/src/task_scheduler.py @@ -9,6 +9,8 @@ import uuid from datetime import datetime, timedelta, timezone from typing import Any, Awaitable, Callable, Dict, Tuple +from core.auth import RESERVED_USERNAMES + logger = logging.getLogger(__name__) @@ -17,6 +19,34 @@ def _utcnow() -> datetime: return datetime.now(timezone.utc).replace(tzinfo=None) +# Shell/file tools a scheduled task's agent should be offered by default, +# mirroring the chat agent (where these are on unless a privilege or global +# setting turns them off). The RAG tool selector + ASSISTANT_ALWAYS_AVAILABLE +# never include bash/python, so on a host with an empty/degraded tool-embedding +# index a task could not run shell or Python even for an admin owner. Offering +# them here is safe: stream_agent_loop's blocked_tools_for_owner() still strips +# this whole group for non-admin multi-user owners, and only admits it for +# admins and single-user (AUTH_ENABLED=false) deployments. +TASK_DEFAULT_SHELL_TOOLS = frozenset({ + "bash", "python", "read_file", "write_file", "edit_file", + "grep", "glob", "ls", "get_workspace", +}) + + +def compose_task_relevant_tools(rag_tools, assistant_always, disabled_tools): + """Compose the relevant-tools set offered to a scheduled task's agent. + + Unions the RAG-retrieved tools, the assistant's always-available set, and + the default shell/file group, then removes anything the task's crew + explicitly disabled via its `enabled_tools` allowlist. Per-owner admin + gating is applied later by stream_agent_loop (blocked_tools_for_owner). + """ + tools = set(rag_tools) | set(assistant_always) | set(TASK_DEFAULT_SHELL_TOOLS) + if disabled_tools: + tools -= set(disabled_tools) + return tools + + # ── Shared TTL cache (singleflight) ──────────────────────────────────────── # Multiple scheduled tasks firing in the same minute often need the same # external data (Miniflux unreads, MCP tool snapshots, etc.). This cache @@ -236,6 +266,29 @@ def _digest_windows(now): ] +def _checkin_calendar_events(db, owner, start, end): + """Calendar events in [start, end] for ONE owner, for the check-in digest. + + Ownership lives on CalendarCal.owner; events inherit it via calendar_id. + The digest query had no owner scope, so it pulled EVERY user's events into + one user's check-in (a cross-tenant leak of summaries/locations). Scope it + by joining CalendarCal, mirroring routes/calendar_routes.list_events. + """ + from core.database import CalendarEvent as _CE, CalendarCal as _CC + return ( + db.query(_CE) + .join(_CC, _CE.calendar_id == _CC.id) + .filter( + _CC.owner == owner, + _CE.dtstart >= start, + _CE.dtstart <= end, + _CE.status != "cancelled", + ) + .order_by(_CE.dtstart) + .all() + ) + + class TaskScheduler: def __init__(self, session_manager): self._session_manager = session_manager @@ -1135,11 +1188,7 @@ class TaskScheduler: # Strip timezone for naive DB comparison _s = start.replace(tzinfo=None) if start.tzinfo else start _e = end.replace(tzinfo=None) if end.tzinfo else end - evs = _db.query(_CE).filter( - _CE.dtstart >= _s, - _CE.dtstart <= _e, - _CE.status != "cancelled", - ).order_by(_CE.dtstart).all() + evs = _checkin_calendar_events(_db, task.owner, _s, _e) if not evs: continue # Group by importance for richer output @@ -1378,17 +1427,30 @@ class TaskScheduler: time_str = _utcnow().strftime("%A, %B %d %Y, %H:%M UTC") system_prompt = f"Current time: {time_str}\n\n{system_prompt}" - # Compute tool filter from CrewMember.enabled_tools if set - disabled_tools = None + # Compute the disabled-tools set: the crew's enabled_tools allowlist + # (inverted) plus the operator's global disabled_tools setting. The + # global list must be merged here — chat does the same merge before + # entering the agent loop (routes/chat_routes.py) — otherwise an admin + # or AUTH_ENABLED=false scheduled task would still see and call shell/ + # file tools after the operator disabled them globally, because the + # prompt/schema/execution gates only enforce what is passed in. + disabled_tools: set[str] = set() if crew and crew.enabled_tools: try: enabled = json.loads(crew.enabled_tools) if isinstance(enabled, list) and enabled: from src.tool_index import BUILTIN_TOOL_DESCRIPTIONS all_tools = set(BUILTIN_TOOL_DESCRIPTIONS.keys()) - disabled_tools = all_tools - set(enabled) + disabled_tools |= all_tools - set(enabled) except Exception: pass + try: + from src.settings import get_setting + _global_disabled = get_setting("disabled_tools", []) + if isinstance(_global_disabled, list): + disabled_tools.update(_global_disabled) + except Exception: + pass # RAG-select relevant tools for this prompt + always-available assistant tools. # Without this, all 40+ tools get sent and models hit their tool limit. @@ -1398,10 +1460,10 @@ class TaskScheduler: tool_idx = get_tool_index() if tool_idx: rag_tools = tool_idx.get_tools_for_query(task.prompt or "", k=8) - relevant_tools = (rag_tools | ASSISTANT_ALWAYS_AVAILABLE) - if disabled_tools: - relevant_tools -= disabled_tools - logger.info(f"[assistant] RAG selected {len(rag_tools)} tools + {len(ASSISTANT_ALWAYS_AVAILABLE)} always-available = {len(relevant_tools)} total for '{task.name}'") + relevant_tools = compose_task_relevant_tools( + rag_tools, ASSISTANT_ALWAYS_AVAILABLE, disabled_tools + ) + logger.info(f"[assistant] RAG selected {len(rag_tools)} tools + {len(ASSISTANT_ALWAYS_AVAILABLE)} always-available + shell/file defaults = {len(relevant_tools)} total for '{task.name}'") except Exception as e: logger.warning(f"[assistant] RAG tool selection failed, using all: {e}") @@ -1409,7 +1471,7 @@ class TaskScheduler: try: result = await self._run_agent_loop( endpoint_url, model, task, session_id, - system_prompt=system_prompt, disabled_tools=disabled_tools, + system_prompt=system_prompt, disabled_tools=disabled_tools or None, relevant_tools=relevant_tools, ) except Exception as e: @@ -2221,7 +2283,7 @@ class TaskScheduler: # check-ins seeded, which then double-fire alongside the human user's # check-ins. This was the root cause of the duplicate 'Morning check-in' # rows we had to manually clean up. - if not owner or owner in {"internal-tool", "api", "demo", "system"}: + if not owner or owner in RESERVED_USERNAMES: logger.info(f"ensure_assistant_defaults: skip synthetic owner {owner!r}") return from core.database import SessionLocal, CrewMember, ScheduledTask diff --git a/src/tool_execution.py b/src/tool_execution.py index 612364b66..3b4ba5eab 100644 --- a/src/tool_execution.py +++ b/src/tool_execution.py @@ -323,6 +323,24 @@ _MCP_TOOL_MAP = { "web_fetch": ("web_fetch", "web_fetch"), "generate_image": ("image_gen", "generate_image"), } +_EMAIL_MCP_OWNER_ARG = "_odysseus_owner" + + +def _parse_qualified_mcp_args(tool: str, content: str) -> tuple[Dict, Optional[str]]: + raw = (content or "").strip() + if not raw: + return {}, None + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, TypeError): + if tool.startswith("mcp__email__"): + return {}, "Email MCP tool arguments must be a JSON object." + return {}, None + if not isinstance(parsed, dict): + if tool.startswith("mcp__email__"): + return {}, "Email MCP tool arguments must be a JSON object." + return {}, None + return parsed, None def _parse_generate_image(content: str) -> Dict: @@ -453,6 +471,8 @@ async def _direct_fallback( tool: str, content: str, progress_cb: Optional[Callable[[Dict], Awaitable[None]]] = None, + session_id: Optional[str] = None, + owner: Optional[str] = None, ) -> Optional[Dict]: _subproc_env = { **os.environ, @@ -466,6 +486,8 @@ async def _direct_fallback( ctx = { "progress_cb": progress_cb, "subproc_env": _subproc_env, + "session_id": session_id, + "owner": owner, } from src.agent_tools import TOOL_HANDLERS @@ -713,10 +735,13 @@ async def _execute_tool_block_impl( desc = f"bash (background): {short}" result = { "output": ( - f"Started background job `{rec['id']}`. It is running detached — " + f"Started background job `{rec['id']}`. It is running detached; " f"do NOT wait for it or poll it. You will be automatically re-invoked " f"with its full output when it finishes. Continue with other work, or " - f"end your turn now and resume when the result arrives." + f"end your turn now and resume when the result arrives. If the user " + f"later asks to check progress or stop it, call the manage_bg_jobs " + f"tool yourself (output or kill); do not tell them to run a tool " + f"command, and do not surface raw tool syntax in your reply." ), "exit_code": 0, "bg_job_id": rec["id"], @@ -737,6 +762,11 @@ async def _execute_tool_block_impl( desc = f"{tool}: {first_line}" result = await _direct_fallback(tool, content, progress_cb=progress_cb) \ or {"error": f"{tool}: execution failed", "exit_code": 1} + elif tool == "manage_bg_jobs": + # Inspect/kill detached `bash` jobs; needs session_id to scope to chat. + desc = f"manage_bg_jobs: {content.split(chr(10))[0][:80]}" + result = await _direct_fallback(tool, content, session_id=session_id, owner=owner) \ + or {"error": "manage_bg_jobs: execution failed", "exit_code": 1} elif tool in ("create_document", "update_document", "edit_document", "suggest_document", "manage_documents"): desc = f"{tool}: {content.split(chr(10))[0][:80]}" @@ -748,10 +778,24 @@ async def _execute_tool_block_impl( query = content.split("\n")[0].strip() desc = f"search_chats: {query[:80]}" result = await do_search_chats(query, owner=owner) - elif tool in ("chat_with_model", "create_session", "list_sessions", - "send_to_session", "pipeline", - "manage_session", "manage_memory", "list_models", - "ui_control", "ask_teacher"): + elif tool in ("chat_with_model", "ask_teacher", "list_models"): + # Migrated to the agent_tools registry (#3629): dispatched through + # TOOL_HANDLERS with the owner/session ctx these tools need, instead + # of the legacy dispatch_ai_tool elif. The impls live in + # src/agent_tools/model_interaction_tools.py. + first_line = content.split(chr(10))[0].strip()[:60] + desc = f"{tool}: {first_line}" if first_line else tool + result = await _document_tool_dispatch(tool, content, session_id, owner) \ + or {"error": f"{tool}: execution failed", "exit_code": 1} + elif tool in ("create_session", "list_sessions", "send_to_session", "manage_session"): + # Migrated to the agent_tools registry (#3629): dispatched through + # TOOL_HANDLERS with the owner/session ctx these tools need. The impls + # live in src/agent_tools/session_tools.py. + first_line = content.split(chr(10))[0].strip()[:60] + desc = f"{tool}: {first_line}" if first_line else tool + result = await _document_tool_dispatch(tool, content, session_id, owner) \ + or {"error": f"{tool}: execution failed", "exit_code": 1} + elif tool in ("pipeline", "manage_memory", "ui_control"): from src.ai_interaction import dispatch_ai_tool desc, result = await dispatch_ai_tool(tool, content, session_id, owner=owner) elif tool == "manage_tasks": @@ -858,12 +902,15 @@ async def _execute_tool_block_impl( # MCP tool dispatch mcp = get_mcp_manager() if mcp: - try: - args = json.loads(content) if content.strip().startswith("{") else {} - except (json.JSONDecodeError, TypeError): - args = {} desc = f"mcp: {tool}" - result = await mcp.call_tool(tool, args) + args, parse_error = _parse_qualified_mcp_args(tool, content) + if parse_error: + result = {"error": parse_error, "exit_code": 1} + else: + if tool.startswith("mcp__email__") and owner: + args = dict(args) + args[_EMAIL_MCP_OWNER_ARG] = owner + result = await mcp.call_tool(tool, args) else: desc = f"mcp: {tool}" result = {"error": "MCP manager not available", "exit_code": 1} diff --git a/src/tool_implementations.py b/src/tool_implementations.py index f1ac33007..3ce3ee613 100644 --- a/src/tool_implementations.py +++ b/src/tool_implementations.py @@ -12,12 +12,24 @@ import os import re from typing import Any, Dict, List, Optional +from fastapi import HTTPException from src.constants import MAX_READ_CHARS, DEEP_RESEARCH_DIR, VAULT_FILE from src.tool_utils import get_mcp_manager from core.constants import internal_api_base +from routes._validators import validate_remote_host, validate_ssh_port logger = logging.getLogger(__name__) + +def _string_arg(value: Any) -> str: + return "" if value is None else str(value).strip() + + +def _validate_cookbook_ssh_target(remote_host: Any, ssh_port: Any = "") -> tuple[str, str]: + remote = validate_remote_host(_string_arg(remote_host) or None) or "" + sport = validate_ssh_port(_string_arg(ssh_port) or None) or "" + return remote, sport + # --------------------------------------------------------------------------- # Active email state # --------------------------------------------------------------------------- @@ -645,6 +657,137 @@ async def do_manage_endpoints(content: str, owner: Optional[str] = None) -> Dict # MCP server management tool # --------------------------------------------------------------------------- +# Parallel to routes/cookbook_helpers._validate_serve_cmd but deliberately the +# opposite policy: that gate guards an admin-only serve command and allows +# interpreters (python3/etc) because model-serving needs them, whereas this is +# the model/prompt-injection-reachable manage_mcp path, so interpreters and +# runners are denied here. +# +# Commands that can execute arbitrary code regardless of their arguments. These +# are NEVER accepted on the manage_mcp agent path, even if an operator lists one +# in ODYSSEUS_MCP_ALLOWED_COMMANDS -- a stdio server that genuinely needs an +# interpreter or package runner must be registered via the trusted admin route. +_MCP_DENIED_COMMANDS = frozenset({ + "sh", "bash", "zsh", "fish", "dash", "ksh", "csh", "tcsh", "ash", "busybox", + "cmd", "command.com", "powershell", "pwsh", + "python", "pypy", "node", "nodejs", "deno", "bun", "ruby", "jruby", + "perl", "raku", "php", "lua", "luajit", "tclsh", "wish", "expect", "rscript", + "groovy", "scala", "elixir", "erl", "iex", "java", "javac", "jshell", "jbang", + "kotlin", "kotlinc", "dotnet", "mono", "swift", "osascript", "tsx", "ts-node", + "npx", "bunx", "uvx", "pipx", "npm", "pnpm", "yarn", "pip", "uv", + "gem", "cargo", "go", "bundle", "poetry", "conda", "mamba", "brew", + "apt", "apt-get", "yum", "dnf", "pacman", "apk", + "env", "xargs", "nohup", "setsid", "nice", "ionice", "time", "timeout", + "watch", "stdbuf", "unbuffer", "script", "ssh", "scp", "sshpass", "sudo", + "doas", "su", "make", "cmake", "docker", "podman", "kubectl", "find", + "awk", "gawk", "sed", "vi", "vim", "nvim", "emacs", "ed", "tee", "eval", +}) + +# Argv flags that make even an allowlisted binary execute inline code. Matched +# by prefix so glued forms (-cimport os, --eval=...) are caught, not just the +# exact-token form. +_MCP_CODE_EXEC_SHORT_FLAGS = ("-c", "-e", "-m") +_MCP_CODE_EXEC_LONG_FLAGS = ("--eval", "--exec", "--print", "--module", "--command", "--require") + +_MCP_URL_SCHEMES = ("http://", "https://", "ftp://", "ftps://", "file://", "data:", "jar:", "blob:") + +# Shell metacharacters refused in command/args. Args are passed as an argv list +# (no shell), but refusing these keeps the surface narrow and obvious. +_MCP_SHELL_METACHARS = set(";|&$`><\n\r") + +# Env vars that let a child process load attacker-supplied code before main(). +_MCP_DANGEROUS_ENV = frozenset({ + "LD_PRELOAD", "LD_LIBRARY_PATH", "LD_AUDIT", "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", "DYLD_FRAMEWORK_PATH", "PYTHONPATH", "PYTHONSTARTUP", + "PYTHONHOME", "PYTHONEXECUTABLE", "NODE_OPTIONS", "NODE_PATH", "BASH_ENV", + "ENV", "SHELLOPTS", "PERL5LIB", "PERL5OPT", "RUBYOPT", "RUBYLIB", "GEM_PATH", + "R_PROFILE", "R_HOME", "PATH", "IFS", "PROMPT_COMMAND", +}) + + +def _mcp_allowed_commands() -> set: + """Operator-configured allowlist of safe MCP launcher basenames for the agent + path. Empty by default; set ODYSSEUS_MCP_ALLOWED_COMMANDS (comma-separated) + to opt specific trusted binaries in. Denied commands are rejected even if + listed here.""" + raw = os.environ.get("ODYSSEUS_MCP_ALLOWED_COMMANDS", "") + return {c.strip().lower() for c in raw.split(",") if c.strip()} + + +def _validate_mcp_command(command, args, env) -> Optional[str]: + """Validate a model-supplied stdio MCP registration. Returns an error string + if it must be rejected, else None. + + Closes the RCE where manage_mcp 'add' passed prompt-injection-controlled + command/args/env straight to a subprocess spawn (issue #438): a payload + smuggled into a skill description, memory entry, fetched page, or email body + could register a stdio server running arbitrary code as the app UID. + """ + if not isinstance(command, str) or not command.strip(): + return "command must be a non-empty string" + command = command.strip() + if "/" in command or "\\" in command: + return "command must be a bare executable name, not a path" + if any(ch in _MCP_SHELL_METACHARS for ch in command): + return "command contains shell metacharacters" + base = command.lower() + if base.endswith(".exe") or base.endswith(".cmd") or base.endswith(".bat"): + base = base.rsplit(".", 1)[0] + # Canonicalize a trailing version suffix so versioned aliases collapse to the + # family name (python3.11 -> python, node18 -> node, pip3 -> pip); both the + # raw basename and the canonical form are denied, so an operator cannot + # accidentally allowlist a runtime alias back into the path. + canon = re.sub(r"[-_.]?\d+(?:\.\d+)*$", "", base) + if base in _MCP_DENIED_COMMANDS or canon in _MCP_DENIED_COMMANDS: + return ( + f"command '{command}' is not allowed on the agent MCP path: " + "interpreters, runtimes, package runners, and shells can execute " + "arbitrary code. Register such a server via the admin route instead." + ) + if base not in _mcp_allowed_commands(): + return ( + f"command '{command}' is not in the MCP allowlist. Add it to " + "ODYSSEUS_MCP_ALLOWED_COMMANDS if you trust it, or register the " + "server via the admin route." + ) + + if args is not None: + if isinstance(args, str): + try: + args = json.loads(args) + except Exception: + return "args must be a JSON list" + if not isinstance(args, list): + return "args must be a list" + for a in args: + if not isinstance(a, str): + return "args must all be strings" + s = a.strip() + low = s.lower() + if any(s == f or s.startswith(f) for f in _MCP_CODE_EXEC_SHORT_FLAGS): + return f"arg '{a}' is a code-execution flag and is not allowed" + if any(low == f or low.startswith(f + "=") for f in _MCP_CODE_EXEC_LONG_FLAGS): + return f"arg '{a}' is a code-execution flag and is not allowed" + if any(low.startswith(u) for u in _MCP_URL_SCHEMES): + return f"arg '{a}' is a remote URL and is not allowed" + if any(ch in _MCP_SHELL_METACHARS for ch in a): + return f"arg '{a}' contains shell metacharacters" + + if env: + if isinstance(env, str): + try: + env = json.loads(env) + except Exception: + return "env must be a JSON object" + if not isinstance(env, dict): + return "env must be an object" + for k in env: + if str(k).strip().upper() in _MCP_DANGEROUS_ENV: + return f"env var '{k}' can inject code into the child process and is not allowed" + + return None + + async def do_manage_mcp(content: str, owner: Optional[str] = None) -> Dict: """Manage MCP servers: list, add, delete, enable, disable, reconnect.""" try: @@ -684,6 +827,12 @@ async def do_manage_mcp(content: str, owner: Optional[str] = None) -> Dict: env = args.get("env", {}) if not name or not command: return {"error": "name and command are required", "exit_code": 1} + # Validate BEFORE any DB write or spawn: a rejected registration must + # leave no enabled row (which would otherwise auto-reconnect on restart) + # and must not attempt a connection. + _mcp_err = _validate_mcp_command(command, cmd_args, env) + if _mcp_err: + return {"error": f"manage_mcp: refused unsafe server registration: {_mcp_err}", "exit_code": 1} sid = str(_uuid.uuid4())[:8] db = SessionLocal() try: @@ -1579,10 +1728,10 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: text = str(raw).strip().lower() if text in {"none", "no", "off", "false"}: return None - m = re.search(r"(\d+)\s*(?:m|min|minute|minutes)\b", text) + m = re.search(r"(\d+)\s*(?:minutes?|mins?|m)\b", text) if m: return max(0, int(m.group(1))) - m = re.search(r"(\d+)\s*(?:h|hr|hour|hours)\b", text) + m = re.search(r"(\d+)\s*(?:hours?|hrs?|h)\b", text) if m: return max(0, int(m.group(1)) * 60) if text.isdigit(): @@ -1595,7 +1744,7 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: return desc reminder_only = re.compile( r"^\s*(?:remind(?:er)?|alarm)\s*:?\s*\d+\s*" - r"(?:m|min|minute|minutes|h|hr|hour|hours)\b.*$", + r"(?:minutes?|mins?|m|hours?|hrs?|h)\b.*$", re.I, ) return "" if reminder_only.match(desc) else desc @@ -2900,6 +3049,10 @@ async def _cookbook_kill_session(session_id: str, *, remote_host: str = "", break if remote: + try: + remote, sport = _validate_cookbook_ssh_target(remote, sport) + except HTTPException as e: + return {"error": str(getattr(e, "detail", e)), "exit_code": 1} _pf = f"-p {shlex.quote(str(sport))} " if sport and str(sport) != "22" else "" cmd = ( f"ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no " @@ -2988,8 +3141,8 @@ async def do_tail_serve_output(content: str, owner: Optional[str] = None) -> Dic tail = 400 tail = max(20, min(tail, 4000)) headers = _internal_headers() - remote = (args.get("remote_host") or args.get("host") or "").strip() - sport = (args.get("ssh_port") or "").strip() + remote = _string_arg(args.get("remote_host") or args.get("host")) + sport = _string_arg(args.get("ssh_port")) # Resolve host from cookbook state if caller didn't pass one — same # lookup _cookbook_kill_session uses. if not remote: @@ -3007,6 +3160,12 @@ async def do_tail_serve_output(content: str, owner: Optional[str] = None) -> Dic if not sport: sport = t.get("sshPort") or "" break + if remote: + try: + remote, sport = _validate_cookbook_ssh_target(remote, sport) + except HTTPException as e: + return {"error": str(getattr(e, "detail", e)), "exit_code": 1} + # Prefer the persisted /tmp/odysseus-tmux/SESSION.log file over the # live tmux pane. The pane is what the user would see scrolling on # their screen — including the post-crash neofetch banner and the @@ -3193,7 +3352,7 @@ async def do_adopt_served_model(content: str, owner: Optional[str] = None) -> Di except ValueError: return {"error": "Invalid JSON arguments", "exit_code": 1} - host = (args.get("host") or args.get("remote_host") or "").strip() + host = _string_arg(args.get("host") or args.get("remote_host")) sess = (args.get("tmux_session") or args.get("session_id") or "").strip() model = (args.get("model") or args.get("repo_id") or "").strip() port = args.get("port") or 8000 @@ -3204,6 +3363,12 @@ async def do_adopt_served_model(content: str, owner: Optional[str] = None) -> Di return {"error": "tmux_session and model are required", "exit_code": 1} # Verify tmux session exists on the target host + if host: + try: + host, _ = _validate_cookbook_ssh_target(host) + except HTTPException as e: + return {"error": str(getattr(e, "detail", e)), "exit_code": 1} + headers = _internal_headers() if host: check = f"ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no {shlex.quote(host)} 'tmux has-session -t {shlex.quote(sess)} 2>&1'" @@ -3818,7 +3983,7 @@ async def do_resolve_contact(content: str, owner: Optional[str] = None) -> Dict: if not name: return {"error": "name is required", "exit_code": 1} - contacts = {} # email -> {name, source} + contacts = {} # email_or_phone -> {name, source, phone?} # 1. CardDAV (Radicale) — structured contacts. Call in-process: a # server-side httpx GET to /api/contacts/search carries no session @@ -3833,10 +3998,18 @@ async def do_resolve_contact(content: str, owner: Optional[str] = None) -> Dict: match = q in hay_name or any(q in (e or "").lower() for e in c.get("emails", [])) if not match: continue + has_email = False for email in (c.get("emails") or []): email = (email or "").strip().lower() if email and "@" in email: contacts[email] = {"name": c.get("name") or email, "source": "contacts"} + has_email = True + # Fall back to phone numbers when the contact has no email address + if not has_email: + for phone in (c.get("phones") or []): + phone = (phone or "").strip() + if phone: + contacts[phone] = {"name": c.get("name") or phone, "source": "contacts", "phone": phone} except Exception: pass @@ -3856,8 +4029,11 @@ async def do_resolve_contact(content: str, owner: Optional[str] = None) -> Dict: return {"output": f"No contacts found matching '{name}'.", "exit_code": 0} lines = [f"Contacts matching '{name}':"] - for email, info in contacts.items(): - lines.append(f"- {info['name']} <{email}> ({info['source']})") + for key, info in contacts.items(): + if info.get("phone"): + lines.append(f"- {info['name']} — phone: {info['phone']} ({info['source']})") + else: + lines.append(f"- {info['name']} <{key}> ({info['source']})") return {"output": "\n".join(lines), "exit_code": 0} diff --git a/src/tool_index.py b/src/tool_index.py index 5388fcbda..64640bcef 100644 --- a/src/tool_index.py +++ b/src/tool_index.py @@ -94,6 +94,7 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = { "manage_endpoints": "Endpoint management: list, add, delete, enable, or disable model API endpoints.", "manage_mcp": "MCP server management: list, add, delete, reconnect servers, or list available tools.", "manage_webhooks": "Webhook management: list, add, delete, enable, or disable webhooks.", + "api_call": "Call a configured API integration by name (Home Assistant, Miniflux, Gitea, Linkding, Jellyfin, RSS reader, git forge, bookmark manager, smart home, or any other registered service). Make a GET/POST/PUT/PATCH/DELETE request to the integration's endpoint path, with an optional JSON body. Use whenever the user asks to query or control one of their connected integrations/services.", "manage_tokens": "API token management: list, create, or delete API access tokens.", "manage_documents": "List, read, delete, or tidy documents in the editor panel. action='list' returns clickable rows (most-recent first) so the user can open any doc by clicking. action='read' (aka view/open/get) with document_id returns the content; supports offset= + limit= to page through large docs (response includes next_offset when more remains, so you can keep calling with offset=next_offset). action='delete' with document_id removes a doc (only way to delete). Use this for ANY 'show/read/list/open my documents/docs/files/notes' request — never shell or curl.", "manage_research": "List, read/open, or delete saved DEEP RESEARCH results from the Library. action='list' returns clickable [query](#research-) rows (most-recent first). action='read' (aka open/view/get) with id returns the report + sources. action='delete' with id removes it. Use this for ANY 'open/read/find/delete my research / that report / the research on X' request. NOTE: this is for EXISTING research; to START new research use trigger_research.", @@ -134,6 +135,7 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = { "app_api": "Generic loopback to allowed Odysseus internal endpoints. Use this when the user wants something the UI can do but there's no named tool for it. Covers calendar, gallery, library/documents, memory, notes, tasks, settings, research, compare, cookbook GPUs/state — allowed UI buttons hit /api/* endpoints and you can hit them too. Sensitive auth/user/admin/shell paths and host-control Cookbook mutation routes are blocked; do NOT use app_api for shell commands, package installs, engine rebuilds, or PID signalling. Use named command tooling for shell commands. action='endpoints' with filter= lists available endpoints. action='call' takes method+path+body. Hits same routes the UI uses — auth flows free. NOTE: themes are NOT an API endpoint — use the ui_control tool (create_theme / set_theme), not app_api. SESSIONS/CHATS: do NOT use app_api for these — GET /api/sessions returns EMPTY for tool calls (it's owner-filtered and tool calls authenticate as a different identity). EMAIL ACCOUNTS: do NOT use /api/email/accounts via app_api; use list_email_accounts, list_emails, and read_email instead. To list/rename/archive/delete/fork chats use the list_sessions and manage_session tools instead.", "edit_image": "Edit an image in the gallery: upscale (increase resolution), remove background (rembg), inpaint (fill selected area), or harmonize (blend edits). Specify image ID and action.", "trigger_research": "Start a deep research job on any topic — appears in the Deep Research sidebar, streams progress, produces a detailed report. Use for 'research X', 'look into Y', 'do deep research on Z', 'investigate'. NOT a scheduled task — it runs now and surfaces in the sidebar.", + "manage_bg_jobs": "Inspect and control detached background `bash` jobs (the ones started with a `#!bg` marker). action='list' shows this chat's jobs (id/status/age/command); action='output' returns a job's captured output so far (check on a long-running job, or re-read a finished one); action='kill' stops a runaway job by id. Use for 'is the background job done', 'check on that job', 'show the build output', 'kill the background job', 'stop the bg task'. output/kill need a job_id from list.", } @@ -348,6 +350,12 @@ class ToolIndex: {"list_email_accounts", "list_emails", "read_email", "send_email", "reply_to_email", "bulk_email", "delete_email", "archive_email", "mark_email_read", "resolve_contact", "ui_control"}, frozenset({"calendar", "event", "meeting", "schedule", "appointment"}): {"manage_calendar"}, + # Detached background `bash` jobs (#!bg): check on / read output / kill. + frozenset({"background job", "background jobs", "bg job", "bg jobs", + "background task", "is the job done", "check the job", + "check on that job", "job output", "kill the job", + "kill the background", "stop the background", "running job"}): + {"manage_bg_jobs"}, frozenset({"note", "todo", "reminder", "remind", "checklist", "remember to"}): {"manage_notes"}, # Chat/session management. "rename" alone maps to documents below, so a @@ -414,6 +422,14 @@ class ToolIndex: "my settings", "change setting", "change a setting", "set setting", "preference", "preferences", "configure"}): {"manage_settings", "ui_control"}, + # API-integration intent → the api_call tool. Mirrors the agent-loop + # "integrations" domain so api_call still surfaces on the retrieval and + # keyword-fallback paths (not just the deterministic domain seed) when a + # user names a connected service. + frozenset({"api_call", "api call", "integration", "integrations", + "home assistant", "homeassistant", "miniflux", "gitea", + "linkding", "jellyfin"}): + {"api_call"}, # Managing EXISTING research in the Library — open/read/find/delete. frozenset({"my research", "the research", "research on", "open research", "read research", "find research", "delete research", diff --git a/src/tool_parsing.py b/src/tool_parsing.py index 97d3f3477..c9548cce9 100644 --- a/src/tool_parsing.py +++ b/src/tool_parsing.py @@ -175,6 +175,9 @@ _TOOL_NAME_MAP = { "notes": "manage_notes", "todo": "manage_notes", "todos": "manage_notes", + "manage_bg_jobs": "manage_bg_jobs", + "bg_jobs": "manage_bg_jobs", + "background_jobs": "manage_bg_jobs", } _MISFENCED_WEB_TOOL_NAMES = { diff --git a/src/tool_schemas.py b/src/tool_schemas.py index 156ae34af..1d64b5db6 100644 --- a/src/tool_schemas.py +++ b/src/tool_schemas.py @@ -68,11 +68,12 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "web_fetch", - "description": "Fetch and read the text content of a specific URL the user names (e.g. 'check example.com', 'what's on this page '). Use when you already have a concrete URL/domain. NOT for open-ended searches (use web_search) or 'research X' jobs (use trigger_research).", + "description": "Fetch and read the text content of a specific URL the user names (e.g. 'check example.com', 'what's on this page '). Use when you already have a concrete URL/domain. NOT for open-ended searches (use web_search) or 'research X' jobs (use trigger_research). Downloads are size-budgeted; a '[partial content: ...]' notice in the result means the body was cut short and you can re-call with full=true for the rest.", "parameters": { "type": "object", "properties": { - "url": {"type": "string", "description": "The URL or domain to fetch (http/https; a bare domain like example.com is fine)"} + "url": {"type": "string", "description": "The URL or domain to fetch (http/https; a bare domain like example.com is fine)"}, + "full": {"type": "boolean", "description": "Raise the download budget to the hard cap for large pages/files. Use only after a result reported partial content."} }, "required": ["url"] } @@ -1008,7 +1009,7 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "resolve_contact", - "description": "Look up a contact's email address by name. Searches CardDAV address book and sent email history. Use when the user says 'message [name]' or 'email [name]' without an email address.", + "description": "Look up a contact by name. Searches CardDAV address book and sent email history. Returns email addresses (when available) or phone numbers. Use when the user says 'message [name]', 'email [name]', or asks for someone's contact details.", "parameters": { "type": "object", "properties": { @@ -1187,6 +1188,21 @@ FUNCTION_TOOL_SCHEMAS = [ } } }, + { + "type": "function", + "function": { + "name": "manage_bg_jobs", + "description": "Inspect and control detached background `bash` jobs (started with the `#!bg` marker). action='list' shows this chat's jobs with id/status/age/command; action='output' returns a job's captured output so far (use for a still-running job, or to re-read a finished one); action='kill' terminates a runaway job's process tree instead of waiting out its max-runtime. output and kill need job_id from list.", + "parameters": { + "type": "object", + "properties": { + "action": {"type": "string", "enum": ["list", "output", "kill"], "description": "list | output | kill (default: list)"}, + "job_id": {"type": "string", "description": "Background job id (required for output/kill; from action='list')"}, + }, + "required": ["action"] + } + } + }, ] @@ -1205,23 +1221,26 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock logger.error(f"Failed to parse function call arguments for {name}: {arguments}") return None + tool_type = _TOOL_NAME_MAP.get(name, name) + _BUILTIN_EMAIL_TOOLS = {"list_email_accounts", "send_email", "list_emails", "read_email", "reply_to_email", + "archive_email", "delete_email", "mark_email_read", "bulk_email", "download_attachment"} + # Some models emit valid JSON that isn't an object (e.g. a bare array - # ["ls -la"], string, or number) as the function arguments. Every branch - # below assumes a dict and calls args.get(...), so a non-dict would raise - # AttributeError and abort the whole agent stream. Coerce to {} instead. + # ["ls -la"], string, or number) as function arguments. Most local tools keep + # the legacy empty-object coercion for stream robustness, but email MCP tools + # must fail closed so a malformed call cannot read the default mailbox. if not isinstance(args, dict): + if tool_type.startswith("mcp__email__") or name in _BUILTIN_EMAIL_TOOLS: + logger.warning(f"Non-object email function call arguments for {name}: {args!r}; rejecting") + return None logger.warning(f"Non-object function call arguments for {name}: {args!r}; treating as empty") args = {} - tool_type = _TOOL_NAME_MAP.get(name, name) - # Allow MCP tools through (namespaced as mcp__serverid__toolname) if tool_type.startswith("mcp__"): content = json.dumps(args) if args else "{}" return ToolBlock(tool_type, content) # Email tools are implemented as MCP — route them to email - _BUILTIN_EMAIL_TOOLS = {"list_email_accounts", "send_email", "list_emails", "read_email", "reply_to_email", - "archive_email", "delete_email", "mark_email_read", "bulk_email", "download_attachment"} if name in _BUILTIN_EMAIL_TOOLS: return ToolBlock(f"mcp__email__{name}", json.dumps(args) if args else "{}") if tool_type not in TOOL_TAGS: diff --git a/src/tool_security.py b/src/tool_security.py index 3dc53ff26..2a7dca3c0 100644 --- a/src/tool_security.py +++ b/src/tool_security.py @@ -14,6 +14,7 @@ logger = logging.getLogger(__name__) NON_ADMIN_BLOCKED_TOOLS = { "bash", "python", + "manage_bg_jobs", "read_file", "write_file", "edit_file", @@ -114,6 +115,8 @@ _PLAN_MODE_KNOWN_MUTATORS = { # Shell is never read-only-safe; block it explicitly so it stays out of plan # mode even if the schema list fails to load. "bash", "python", + # Controls shell processes (kill); plan mode can't run bash anyway. + "manage_bg_jobs", } diff --git a/static/icon.ico b/static/icon.ico new file mode 100644 index 000000000..666a7e6ad Binary files /dev/null and b/static/icon.ico differ diff --git a/static/index.html b/static/index.html index e0e84bd0b..b20b73428 100644 --- a/static/index.html +++ b/static/index.html @@ -1918,7 +1918,7 @@

Change Password

- +
@@ -2054,7 +2054,7 @@

Add User

- +
Admin
diff --git a/static/js/admin.js b/static/js/admin.js index 61458f1c6..58b8765a5 100644 --- a/static/js/admin.js +++ b/static/js/admin.js @@ -13,6 +13,7 @@ let modalEl = null; // the endpoints list can flash a glow on that row. Cleared once the // animation fires. let _recentlyAddedEpId = null; +let _authPolicy = { password_min_length: 8, reserved_usernames: [] }; function el(id) { return document.getElementById(id); } function esc(s) { return uiModule.esc(s); } @@ -343,6 +344,15 @@ function initSignupToggle() { } function initAddUser() { + fetch('/api/auth/policy', { credentials: 'same-origin' }) + .then(r => r.ok ? r.json() : null) + .then(policy => { + if (!policy) return; + _authPolicy = policy; + const admPw = el('adm-newPassword'); + if (admPw) admPw.placeholder = `Password (min ${policy.password_min_length})`; + }) + .catch(() => {}); el('adm-addBtn').addEventListener('click', async () => { const msg = el('adm-addMsg'); msg.textContent = ''; msg.className = ''; @@ -350,7 +360,8 @@ function initAddUser() { const password = el('adm-newPassword').value; const is_admin = el('adm-newIsAdmin').checked; if (!username) { msg.textContent = 'Username required'; msg.className = 'admin-error'; return; } - if (password.length < 8) { msg.textContent = 'Password must be at least 8 characters'; msg.className = 'admin-error'; return; } + if (password.length < _authPolicy.password_min_length) { msg.textContent = `Password must be at least ${_authPolicy.password_min_length} characters`; msg.className = 'admin-error'; return; } + if (_authPolicy.reserved_usernames.includes(username.toLowerCase())) { msg.textContent = 'This username is reserved'; msg.className = 'admin-error'; return; } el('adm-addBtn').disabled = true; try { const res = await fetch('/api/auth/users', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ username, password, is_admin }) }); @@ -1745,7 +1756,6 @@ const TOOL_META = { manage_skills: { name: 'Skills', desc: 'Learn and use procedures', cat: 'Knowledge', ctx: '~200' }, manage_rag: { name: 'RAG / Docs', desc: 'Query indexed documents', cat: 'Knowledge', ctx: '~150' }, chat_with_model: { name: 'Chat with Model', desc: 'Talk to another AI model', cat: 'Multi-Agent', ctx: '~200' }, - second_opinion: { name: 'Second Opinion', desc: 'Get another model\'s take', cat: 'Multi-Agent', ctx: '~150' }, pipeline: { name: 'Pipeline', desc: 'Multi-step AI workflows', cat: 'Multi-Agent', ctx: '~200' }, ask_teacher: { name: 'Ask Teacher', desc: 'Query a more capable model', cat: 'Multi-Agent', ctx: '~150' }, send_to_session: { name: 'Send to Session', desc: 'Send message to another chat', cat: 'Sessions', ctx: '~100' }, diff --git a/static/js/assistant.js b/static/js/assistant.js index dca4bd55f..b4b9dc3cc 100644 --- a/static/js/assistant.js +++ b/static/js/assistant.js @@ -125,7 +125,7 @@ const TOOL_GROUPS = { 'Knowledge': ['web_search', 'read_file', 'manage_memory', 'manage_rag', 'search_chats'], 'Code': ['bash', 'python', 'write_file'], 'Documents': ['create_document', 'edit_document', 'update_document', 'suggest_document'], - 'AI & Models': ['chat_with_model', 'second_opinion', 'ask_teacher', 'pipeline', 'list_models', 'generate_image'], + 'AI & Models': ['chat_with_model', 'ask_teacher', 'pipeline', 'list_models', 'generate_image'], 'System': ['manage_session', 'manage_endpoints', 'manage_mcp', 'manage_settings', 'manage_skills', 'manage_webhooks', 'manage_tokens', 'manage_documents', 'create_session', 'list_sessions', 'send_to_session', 'ui_control'], }; diff --git a/static/js/calendar.js b/static/js/calendar.js index 4c5c38564..2b14d024a 100644 --- a/static/js/calendar.js +++ b/static/js/calendar.js @@ -413,7 +413,7 @@ function _calEventFg(ev) { // Returns '' for normal solid-color events. function _calItemBgStyle(ev) { if (!_isCalBgImage(ev.color)) return ''; - const url = _calBgImageUrl(ev.color).replace(/'/g, "\\'"); + const url = _calBgImageUrl(ev.color).replace(/'/g, "\\'").replace(/"/g, "%22"); return `background-image: linear-gradient(color-mix(in srgb, var(--bg) 70%, transparent), color-mix(in srgb, var(--bg) 70%, transparent)), url('${url}'); background-size: cover; background-position: center;`; } @@ -1260,7 +1260,7 @@ async function _renderWeek() { // events keep the original tinted treatment. let bgDecl; if (_isCalBgImage(ev.color)) { - const _url = _calBgImageUrl(ev.color).replace(/'/g, "\\'"); + const _url = _calBgImageUrl(ev.color).replace(/'/g, "\\'").replace(/"/g, "%22"); bgDecl = `background-image: linear-gradient(color-mix(in srgb, var(--bg) 55%, transparent), color-mix(in srgb, var(--bg) 55%, transparent)), url('${_url}'); background-size: cover; background-position: center;`; } else { bgDecl = `background:color-mix(in srgb, ${_calColor(ev)} 18%, var(--bg));`; diff --git a/static/js/chatRenderer.js b/static/js/chatRenderer.js index ce98be4b9..253fa5724 100644 --- a/static/js/chatRenderer.js +++ b/static/js/chatRenderer.js @@ -635,8 +635,8 @@ export function applyModelColor(roleEl, modelName) { popup.className = 'ctx-popup'; let html = '
'; if (logoHtml) html += ''; - html += short + '
'; - html += '
Model ' + modelName.split('/').pop() + '
'; + html += uiModule.esc(short) + '
'; + html += '
Model ' + uiModule.esc(modelName.split('/').pop()) + '
'; // Provider = the serving endpoint, distinct from the model vendor/logo // (e.g. the same model via OpenRouter vs Copilot vs Anthropic direct). const _epUrl = (window.sessionModule && window.sessionModule.getCurrentEndpointUrl) diff --git a/static/js/cookbook-deps-recipes.js b/static/js/cookbook-deps-recipes.js index afb1b7287..ba4f1b444 100644 --- a/static/js/cookbook-deps-recipes.js +++ b/static/js/cookbook-deps-recipes.js @@ -56,7 +56,7 @@ const _RECIPES = [ match: () => true, variants: { pip: { commands: ['CMAKE_ARGS="-DGGML_CUDA=on" uv pip install -U "llama-cpp-python[server]"'] }, - docker: { commands: ['docker pull ghcr.io/ggerganov/llama.cpp:server-cuda'] }, + docker: { commands: ['docker pull ghcr.io/ggml-org/llama.cpp:server-cuda'] }, }, }, ]; diff --git a/static/js/cookbook-diagnosis.js b/static/js/cookbook-diagnosis.js index 200803313..a8bb31419 100644 --- a/static/js/cookbook-diagnosis.js +++ b/static/js/cookbook-diagnosis.js @@ -612,24 +612,50 @@ export const ERROR_PATTERNS = [ ], }, { - // Tail-only + healthy-server suppression. tmux capture-pane returns the - // entire scrollback every poll, so a one-shot startup traceback would - // otherwise stick on the panel forever even while the server happily - // serves /v1/models. Only fire if the traceback is in recent output AND - // the server isn't currently logging healthy traffic. + // Dependency-install (pip) build failure — a required package failed to + // build its wheel (common when an old sdist's setup.py breaks on a newer + // Python, e.g. basicsr on 3.13). This is an install problem, NOT a serve + // problem, so it must never suggest killing vLLM. + match: (text) => { + const TAIL = text.slice(-6000); + // A serve script can run a fallback build and then start serving fine — + // don't flag a stale build error once the server is up. + if (/Application startup complete|"(?:GET|POST)\s+\/v1\/[^"]+ HTTP\/[\d.]+"\s*2\d\d|Uvicorn running on|server is listening on https?:\/\//i.test(TAIL)) return false; + return /Failed to build\b|subprocess-exited-with-error|Could not build wheels|metadata-generation-failed/i.test(TAIL); + }, + message: 'A dependency failed to build during install — usually an older package whose build breaks on this Python version, not a server problem. The install did not finish.', + suggestion: 'Suggested action: check the captured output for the package that failed to build; it may need a newer release or a patch to install on this Python version.', + fixes: [], + }, + { + // vLLM-specific traceback: only offer the kill-processes recovery when the + // output is actually about vLLM. Tail-only + healthy-server suppression so + // a one-shot startup traceback doesn't stick on the panel forever while + // the server happily serves /v1/models. match: (text) => { const TAIL = text.slice(-4096); if (!/Traceback \(most recent call last\)/i.test(TAIL)) return false; - // Healthy markers in the tail mean whatever blew up has been recovered - // from — the server is up and answering requests. if (/Application startup complete|"GET \/v1\/[^"]+ HTTP\/[\d.]+" 2\d\d|Uvicorn running on/i.test(TAIL)) return false; - return true; + return /vllm/i.test(TAIL); }, - message: 'Python traceback detected — may be a handled error, check logs.', + message: 'A vLLM process hit a Python traceback and may be wedged.', fixes: [ { label: 'Kill vLLM processes', action: (panel) => _runQuickCmd(panel, 'pkill -f vllm') }, ], }, + { + // Generic traceback (not vLLM, not a pip build): surface it without + // suggesting an unrelated vLLM kill. Same tail-only + healthy suppression. + match: (text) => { + const TAIL = text.slice(-4096); + if (!/Traceback \(most recent call last\)/i.test(TAIL)) return false; + if (/Application startup complete|"GET \/v1\/[^"]+ HTTP\/[\d.]+" 2\d\d|Uvicorn running on/i.test(TAIL)) return false; + return true; + }, + message: 'Python traceback detected — check the captured output below for the underlying error.', + suggestion: 'Suggested action: read the captured output for the failing step; copy the troubleshooting bundle if you need help.', + fixes: [], + }, ]; export function _diagnose(text) { diff --git a/static/js/cookbook.js b/static/js/cookbook.js index 5b95fad1e..b778c1df9 100644 --- a/static/js/cookbook.js +++ b/static/js/cookbook.js @@ -2958,10 +2958,13 @@ export async function open(opts) { // returned before hydration — and since close/reopen doesn't reset the page, // only a full reload recovered it. Re-rendering is cheap and the in-progress // Running tab is rendered separately just below. - _renderRecipes(); + // Guard the render passes: a single broken task card must not throw out of + // open() and leave the modal stuck hidden (it has no catch, so the panel + // would silently never appear). Show the window regardless; log and move on. + try { _renderRecipes(); } catch (e) { console.error('[cookbook] renderRecipes failed', e); } _rendered = true; _clearCookbookNotif(); - _renderRunningTab(); + try { _renderRunningTab(); } catch (e) { console.error('[cookbook] renderRunningTab failed', e); } // Self-heal: revive any download tasks whose tmux session is still alive // but were persisted as done/error (covers the "restarted server while a // big multi-shard download was in flight" case — the task survived in diff --git a/static/js/cookbookRunning.js b/static/js/cookbookRunning.js index 7672edfd2..fae84ce68 100644 --- a/static/js/cookbookRunning.js +++ b/static/js/cookbookRunning.js @@ -804,40 +804,47 @@ function _winSessionCmd(task, tmuxArgs) { const ps = host ? `Get-Content '${sd}\\${sid}.log' -Tail ${lines} -ErrorAction SilentlyContinue` : `Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.log') -Tail ${lines} -ErrorAction SilentlyContinue`; - return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`; + return _winPowerShellCmd(task, ps); } if (tmuxArgs.includes('has-session')) { const ps = host ? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Get-Process -Id $p -ErrorAction SilentlyContinue | Out-Null; if ($?) { exit 0 } else { exit 1 } } else { exit 1 }` : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Get-Process -Id $p -ErrorAction SilentlyContinue | Out-Null; if ($?) { exit 0 } else { exit 1 } } else { exit 1 }`; - return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`; + return _winPowerShellCmd(task, ps); } if (tmuxArgs.includes('kill-session')) { - const stopTree = `function Stop-Tree([int]$Id) { Get-CimInstance Win32_Process -Filter "ParentProcessId = $Id" -ErrorAction SilentlyContinue | ForEach-Object { Stop-Tree ([int]$_.ProcessId) }; Stop-Process -Id $Id -Force -ErrorAction SilentlyContinue }`; - const ps = host - ? `${stopTree}; $p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue` - : `${stopTree}; $p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`; - return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`; + const ps = _winSessionStopTreePs(task); + return _winPowerShellCmd(task, ps); } if (tmuxArgs.includes('send-keys') && tmuxArgs.includes('C-c')) { const ps = host ? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -ErrorAction SilentlyContinue }` : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -ErrorAction SilentlyContinue }`; - return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`; + return _winPowerShellCmd(task, ps); } return host ? `ssh ${pf}${host} 'tmux ${tmuxArgs}' 2>/dev/null` : `tmux ${tmuxArgs} 2>/dev/null`; } +function _winPowerShellCmd(task, ps) { + const command = `powershell -Command "${ps}"`; + if (!task.remoteHost) return command; + return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} ${_shQuote(command)}`; +} + +function _winSessionStopTreePs(task) { + const host = task.remoteHost; + const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux'; + const sid = task.sessionId; + const stopTree = `function Stop-Tree([int]$Id) { Get-CimInstance Win32_Process -Filter ('ParentProcessId = ' + $Id) -ErrorAction SilentlyContinue | ForEach-Object { Stop-Tree ([int]$_.ProcessId) }; Stop-Process -Id $Id -Force -ErrorAction SilentlyContinue }`; + return host + ? `${stopTree}; $p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue` + : `${stopTree}; $p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`; +} + export function _tmuxGracefulKill(task) { if (_isWindows(task)) { - const host = task.remoteHost; - const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux'; - const sid = task.sessionId; - const pf = _sshPrefix(_getPort(task)); - const ps = host - ? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue` - : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`; - return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`; + const ps = _winSessionStopTreePs(task); + return _winPowerShellCmd(task, ps); } if (task.remoteHost) { return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} 'tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null'`; diff --git a/static/js/cookbookServe.js b/static/js/cookbookServe.js index da48507f7..a22c73af3 100644 --- a/static/js/cookbookServe.js +++ b/static/js/cookbookServe.js @@ -499,13 +499,28 @@ function _selectedServeTarget(panel) { host, serverKey: server ? (_serverKey?.(server) || '') : (select?.value || ''), serverName: server?.name || '', - port: host ? (_getPort(host) || server?.port || '') : '', + env: server?.env || '', + port: host ? (server?.port || _getPort(host) || '') : '', venv, platform: server?.platform || _envState.platform || '', label, }; } +function _remoteWindowsDiffusersUnsupported(target) { + return !!(target?.host && target?.platform === 'windows'); +} + +function _backendChoicesForTarget(target) { + if (target?.platform === 'windows') { + if (_remoteWindowsDiffusersUnsupported(target)) return [['llamacpp','llama.cpp']]; + return [['llamacpp','llama.cpp'],['diffusers','Diffusers']]; + } + return _isMetal() + ? [['llamacpp','llama.cpp'],['ollama','Ollama']] + : [['vllm','vLLM'],['sglang','SGLang'],['llamacpp','llama.cpp'],['ollama','Ollama'],['diffusers','Diffusers']]; +} + async function _fetchServeRuntimePackage(panel, backend) { const packageByBackend = { vllm: 'vllm', @@ -948,13 +963,14 @@ function _rerenderCachedModels() { const _isMiniMaxM2 = _isMiniMaxM2Model({ ...m, repo_id: repo }); const _isMiniMaxMSeries = _isMiniMaxM3 || _isMiniMaxM2; const svm = (k, def) => (_modelSs && _hasOwn(_modelSs, k)) ? _modelSs[k] : def; + const _serveTarget = _selectedServeTarget(); + const _backendChoices = _backendChoicesForTarget(_serveTarget); + const _allowedBackends = new Set(_backendChoices.map(([v]) => v)); const detectedBackend = _detectBackend(m).backend; - const _allowedBackends = new Set(_isWindows() - ? ['llamacpp', 'diffusers'] - : (_isMetal() ? ['llamacpp', 'ollama'] : ['vllm', 'sglang', 'llamacpp', 'ollama', 'diffusers'])); - const defaultBackend = (_repoForcedBackend && ss.backend && _allowedBackends.has(ss.backend)) + let defaultBackend = (_repoForcedBackend && ss.backend && _allowedBackends.has(ss.backend)) ? ss.backend : detectedBackend; + if (!_allowedBackends.has(defaultBackend)) defaultBackend = _backendChoices[0]?.[0] || detectedBackend; const savedMatchesBackend = _repoForcedBackend || (ss.backend || 'vllm') === detectedBackend; const sv = (k, def) => (ss[k] !== undefined && savedMatchesBackend) ? ss[k] : def; const defaultTp = defaultBackend === 'llamacpp' ? '1' : sv('tp', _isMiniMaxMSeries ? '8' : '1'); @@ -1027,12 +1043,6 @@ function _rerenderCachedModels() { panelHtml += `
${_slotsHtml}
`; // Row 1: Engine + Server + Env panelHtml += `
`; - const _backendChoices = _isWindows() - ? [['llamacpp','llama.cpp'],['diffusers','Diffusers']] - : _isMetal() - // Diffusers (diffusion_server.py) is CUDA-only — omit it on Metal. - ? [['llamacpp','llama.cpp'],['ollama','Ollama']] - : [['vllm','vLLM'],['sglang','SGLang'],['llamacpp','llama.cpp'],['ollama','Ollama'],['diffusers','Diffusers']]; const backendOpts = _backendChoices.map(([v,l]) => ``).join(''); // Custom Backend picker — native
+
+
IMAP (Receiving)
-
+
SMTP (Sending) — optional, leave blank for read-only
@@ -2959,6 +2977,16 @@ async function initEmailAccountsSettings() {
`; + // Show/hide OAuth section and password fields based on provider selection. + function _syncOauthUI(providerKey) { + const p = PROVIDERS[providerKey]; + const isOauth = !!(p && p.oauth); + el('eaf-oauth-section').style.display = isOauth ? '' : 'none'; + formEl.querySelectorAll('.eaf-password-section').forEach(r => { + r.style.display = isOauth ? 'none' : ''; + }); + } + const eafProviderNotes = { outlook: { title: 'Outlook / Office 365 needs OAuth', @@ -2983,13 +3011,41 @@ async function initEmailAccountsSettings() { el('eaf-provider').addEventListener('change', (e) => { _renderEafProviderNote(e.target.value); const p = PROVIDERS[e.target.value]; - if (!p) return; + if (!p) { _syncOauthUI(''); return; } el('eaf-imap-host').value = p.imap.host; el('eaf-imap-port').value = p.imap.port; el('eaf-imap-starttls').checked = !!p.imap.starttls; el('eaf-smtp-host').value = p.smtp.host; el('eaf-smtp-port').value = p.smtp.port; el('eaf-smtp-security').value = p.smtp.security || ((parseInt(p.smtp.port || 465) === 587) ? 'starttls' : 'ssl'); + _syncOauthUI(e.target.value); + }); + + // Init OAuth UI for accounts already connected via OAuth. + if (a.oauth_provider === 'google') _syncOauthUI('google_workspace'); + + // "Connect with Google" button — save the account first, then redirect to OAuth. + el('eaf-oauth-btn').addEventListener('click', async () => { + // Must save the account first to get an account_id to pass to the OAuth flow. + const body = { + name: el('eaf-name').value.trim() || el('eaf-from').value.trim(), + from_address: el('eaf-from').value.trim(), + imap_host: el('eaf-imap-host').value.trim(), + imap_port: parseInt(el('eaf-imap-port').value) || 993, + imap_user: el('eaf-imap-user').value.trim(), + imap_starttls: el('eaf-imap-starttls').checked, + smtp_host: el('eaf-smtp-host').value.trim(), + smtp_port: parseInt(el('eaf-smtp-port').value) || 587, + smtp_user: el('eaf-imap-user').value.trim(), + }; + if (!body.name) { el('eaf-msg').textContent = 'Enter a Name or Email first'; el('eaf-msg').style.color = 'var(--red)'; return; } + const url = isEdit ? `/api/email/accounts/${a.id}` : '/api/email/accounts'; + const method = isEdit ? 'PUT' : 'POST'; + const r = await fetch(url, { method, credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const d = await r.json(); + if (!d.ok) { el('eaf-msg').textContent = d.error || 'Save failed'; el('eaf-msg').style.color = 'var(--red)'; return; } + const accId = isEdit ? a.id : d.id; + window.location.href = `/api/email/oauth/google/authorize?account_id=${encodeURIComponent(accId)}`; }); el('eaf-smtp-security').value = _smtpSecurity(a); @@ -3009,6 +3065,7 @@ async function initEmailAccountsSettings() { const body = { name: el('eaf-name').value.trim(), from_address: el('eaf-from').value.trim(), + display_name: el('eaf-display-name').value.trim(), imap_host: el('eaf-imap-host').value.trim(), imap_port: parseInt(el('eaf-imap-port').value) || 993, imap_user: el('eaf-imap-user').value.trim(), @@ -4317,6 +4374,7 @@ async function initUnifiedIntegrations() { // it may be remote (DNS, LAN, Tailscale), not localhost. const PROVIDERS = { gmail: { label: 'Gmail', emailEx: 'you@gmail.com', imap: { host: 'imap.gmail.com', port: 993, starttls: false }, smtp: { host: 'smtp.gmail.com', port: 465 } }, + google_workspace: { label: 'Google Workspace / .edu', emailEx: 'you@yourschool.edu', imap: { host: 'imap.gmail.com', port: 993, starttls: false }, smtp: { host: 'smtp.gmail.com', port: 587 }, oauth: 'google' }, migadu: { label: 'Migadu', emailEx: 'you@yourdomain.com', imap: { host: 'imap.migadu.com', port: 993, starttls: false }, smtp: { host: 'smtp.migadu.com', port: 465 } }, icloud: { label: 'iCloud', emailEx: 'you@icloud.com', imap: { host: 'imap.mail.me.com', port: 993, starttls: false }, smtp: { host: 'smtp.mail.me.com', port: 587 } }, outlook: { label: 'Outlook / Office 365', emailEx: 'you@outlook.com', imap: { host: 'outlook.office365.com', port: 993, starttls: false }, smtp: { host: 'smtp.office365.com', port: 587 } }, @@ -4334,6 +4392,7 @@ async function initUnifiedIntegrations() { const PROV_LOGO = { '': _customLogo, gmail: _letterLogo('G', '#ea4335'), + google_workspace: _letterLogo('G', '#ea4335'), migadu: _letterLogo('M', '#3aa39d'), icloud: _letterLogo('i', '#3693f3'), outlook: _letterLogo('O', '#0078d4'), @@ -4362,11 +4421,17 @@ async function initUnifiedIntegrations() {
+
+
IMAP (Receiving)
-
+
SMTP (Sending) — optional, leave blank for read-only
@@ -4491,6 +4556,16 @@ async function initUnifiedIntegrations() {
`; }; + // Show/hide the OAuth section and password fields based on provider selection. + function _syncOauthUI(providerKey) { + const p = PROVIDERS[providerKey]; + const isOauth = !!(p && p.oauth); + el('uf-oauth-section').style.display = isOauth ? '' : 'none'; + formEl.querySelectorAll('.uf-password-section').forEach(r => { + r.style.display = isOauth ? 'none' : ''; + }); + } + // Custom dropdown wire-up — the native