mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-08-07 22:18:50 -04:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 dependabot[bot]
|
d960fc758c |
@@ -21,7 +21,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
@@ -73,10 +73,10 @@ jobs:
|
||||
name: Python syntax (compileall)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
# Byte-compile sources — catches syntax errors without installing deps.
|
||||
@@ -86,10 +86,10 @@ jobs:
|
||||
name: JS syntax (node --check)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: "20"
|
||||
# Syntax-check our own JS (skip vendored libs in static/lib).
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
# ROADMAP "fresh install smoke tests" item; make this required once green.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
@@ -135,7 +135,7 @@ jobs:
|
||||
echo "docs_only=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
if: steps.docs-check.outputs.docs_only != 'true'
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
@@ -27,15 +27,15 @@ jobs:
|
||||
language: [actions, javascript-typescript, python]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
uses: github/codeql-action/init@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: none
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
uses: github/codeql-action/analyze@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
|
||||
@@ -37,12 +37,12 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Lint Dockerfile
|
||||
uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0
|
||||
uses: hadolint/hadolint-action@2a66e89f53d0771bb131a7fa31f3136336094aa6 # v3.4.0
|
||||
with:
|
||||
dockerfile: Dockerfile
|
||||
# DL3008: pinning apt package versions is impractical on a -slim base
|
||||
|
||||
@@ -52,17 +52,17 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Buildx
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
# Build without pushing so a broken Dockerfile is caught here, and the
|
||||
# exact image we ship is what gets scanned.
|
||||
- name: Build image
|
||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
push: false
|
||||
@@ -93,15 +93,15 @@ jobs:
|
||||
security-events: write # upload SARIF to the Security tab
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Buildx
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Build image
|
||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
push: false
|
||||
@@ -119,7 +119,7 @@ jobs:
|
||||
TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2
|
||||
|
||||
- name: Upload Trivy results
|
||||
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-image
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
@@ -55,12 +55,12 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
|
||||
@@ -45,20 +45,20 @@ jobs:
|
||||
arch: arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up Buildx
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
platforms: ${{ matrix.platform }}
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Read APP_VERSION + short sha
|
||||
@@ -103,16 +103,16 @@ jobs:
|
||||
pattern: digest-*
|
||||
merge-multiple: true
|
||||
- name: Set up Buildx
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Compute tags
|
||||
id: meta
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
# Skip bots (Dependabot, release-drafter, etc.)
|
||||
if: ${{ github.event.issue.user.type != 'Bot' }}
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
persist-credentials: false
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
# Skip bots: they open PRs programmatically and have their own process.
|
||||
if: github.event.pull_request.user.type != 'Bot'
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.base_ref }}
|
||||
sparse-checkout: .github/scripts
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
# Full history so a secret committed in an earlier commit (and later
|
||||
# deleted) is still caught -- deletion does not remove it from Git.
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
@@ -61,12 +61,12 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
|
||||
@@ -739,7 +739,7 @@ app.include_router(setup_stt_routes(stt_service))
|
||||
logger.info("STT service initialized (provider managed via settings)")
|
||||
|
||||
# Documents (artifacts/canvas)
|
||||
from routes.document.document_routes import setup_document_routes
|
||||
from routes.document_routes import setup_document_routes
|
||||
document_router = setup_document_routes(session_manager, upload_handler)
|
||||
app.include_router(document_router)
|
||||
|
||||
@@ -820,7 +820,7 @@ set_ai_rag_manager(rag_manager, personal_docs_mgr)
|
||||
logger.info("AI interaction tools initialized (session, memory, RAG, UI control)")
|
||||
|
||||
# Webhooks
|
||||
from routes.webhook.webhook_routes import setup_webhook_routes
|
||||
from routes.webhook_routes import setup_webhook_routes
|
||||
app.include_router(setup_webhook_routes(webhook_manager, auth_manager, session_manager, api_key_manager))
|
||||
|
||||
# API Tokens
|
||||
@@ -852,7 +852,7 @@ app.include_router(setup_codex_routes(
|
||||
))
|
||||
app.include_router(setup_claude_routes())
|
||||
|
||||
from routes.vault.vault_routes import setup_vault_routes
|
||||
from routes.vault_routes import setup_vault_routes
|
||||
app.include_router(setup_vault_routes())
|
||||
|
||||
# Contacts (CardDAV)
|
||||
|
||||
@@ -17,8 +17,6 @@ from mcp.types import Tool, TextContent
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from src.memory import MemoryStoreUnreadable
|
||||
|
||||
server = Server("memory")
|
||||
|
||||
# Late-initialized managers (set during first tool call)
|
||||
@@ -31,10 +29,6 @@ _OWNER_SCOPE_ERROR = (
|
||||
"Error: Memory MCP owner is not configured for an owner-scoped memory store. "
|
||||
"Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
|
||||
)
|
||||
_UNREADABLE_STORE_ERROR = (
|
||||
"Error: Memory store is temporarily unreadable — nothing was saved. "
|
||||
"Repair or restore memory.json, then retry."
|
||||
)
|
||||
|
||||
|
||||
def _configured_owner() -> str | None:
|
||||
@@ -57,20 +51,8 @@ def _owner_scoped_store(entries: list[dict]) -> bool:
|
||||
return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
|
||||
|
||||
|
||||
def _scope_entries(for_update: bool = False) -> tuple[str | None, list[dict], list[dict], str | None]:
|
||||
"""Return configured owner, all entries, visible entries, and optional error.
|
||||
|
||||
``for_update=True`` is for read-modify-write callers. They save the ``all
|
||||
entries`` list back, so an unreadable store must be reported as an error
|
||||
instead of degrading to ``[]`` — otherwise the save writes their one new
|
||||
entry over the whole store (issue #5673).
|
||||
"""
|
||||
if for_update:
|
||||
try:
|
||||
entries = _memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
return None, [], [], f"{_UNREADABLE_STORE_ERROR} ({e})"
|
||||
else:
|
||||
def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]:
|
||||
"""Return configured owner, all entries, visible entries, and optional error."""
|
||||
entries = _memory_manager.load_all()
|
||||
owner = _configured_owner()
|
||||
if owner is None and _owner_scoped_store(entries):
|
||||
@@ -179,7 +161,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
|
||||
category = arguments.get("category", "fact")
|
||||
if not text:
|
||||
return _text_result("Error: Memory text cannot be empty")
|
||||
owner, memories, _visible, scope_error = _scope_entries(for_update=True)
|
||||
owner, memories, _visible, scope_error = _scope_entries()
|
||||
if scope_error:
|
||||
return _text_result(scope_error)
|
||||
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
|
||||
|
||||
+1
-10
@@ -6,7 +6,6 @@ from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request, Response
|
||||
from core.middleware import require_admin
|
||||
from services.memory import MemoryStoreUnreadable
|
||||
from src.auth_helpers import get_current_user
|
||||
from src.settings import load_settings, save_settings, load_features, save_features
|
||||
|
||||
@@ -77,15 +76,7 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo
|
||||
|
||||
# ── Memories ──
|
||||
if "memories" in body and isinstance(body["memories"], list):
|
||||
# Strict load: importing on top of an unreadable store would write
|
||||
# only the incoming rows back and drop everything already saved.
|
||||
try:
|
||||
existing = memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
logger.error("Refusing to import memories: %s", e)
|
||||
raise HTTPException(
|
||||
503, "Memory store is temporarily unreadable — nothing was imported."
|
||||
)
|
||||
existing = memory_manager.load_all()
|
||||
# Dedup against THIS user's own memories only. Using every tenant's
|
||||
# rows (load_all) meant a memory whose text matched any other
|
||||
# user's was silently skipped, so the importing user lost their own
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
"""Document route domain package (slice 2m, #4082/#4071).
|
||||
|
||||
Contains document_routes.py and document_helpers.py, migrated from the flat
|
||||
routes/ directory. Backward-compat shims at routes/document_routes.py and
|
||||
routes/document_helpers.py re-export from here.
|
||||
"""
|
||||
@@ -1,243 +0,0 @@
|
||||
"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
|
||||
|
||||
"""Document routes — CRUD for living documents with version history."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.database import Document, DocumentVersion
|
||||
from core.database import Session as DbSession
|
||||
from src.auth_helpers import _auth_disabled
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---- Request schemas ----
|
||||
|
||||
class DocumentCreate(BaseModel):
|
||||
session_id: Optional[str] = None
|
||||
title: str = "Untitled"
|
||||
language: Optional[str] = None
|
||||
content: str = ""
|
||||
|
||||
class DocumentUpdate(BaseModel):
|
||||
content: str
|
||||
summary: Optional[str] = None
|
||||
force_version: bool = False
|
||||
|
||||
class DocumentPatch(BaseModel):
|
||||
title: Optional[str] = None
|
||||
language: Optional[str] = None
|
||||
session_id: Optional[str] = None # link/unlink document to a session
|
||||
|
||||
|
||||
# ---- Helpers ----
|
||||
|
||||
def _doc_to_dict(doc: Document) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": doc.id,
|
||||
"session_id": doc.session_id,
|
||||
"title": doc.title,
|
||||
"language": doc.language,
|
||||
"current_content": doc.current_content,
|
||||
"version_count": doc.version_count,
|
||||
"is_active": doc.is_active,
|
||||
"archived": bool(getattr(doc, "archived", False)),
|
||||
"created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
|
||||
"updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
|
||||
# Source-email provenance (set when doc was created from an email
|
||||
# attachment) — drives the "Send signed reply" menu item.
|
||||
"source_email_uid": getattr(doc, "source_email_uid", None),
|
||||
"source_email_folder": getattr(doc, "source_email_folder", None),
|
||||
"source_email_account_id": getattr(doc, "source_email_account_id", None),
|
||||
"source_email_message_id": getattr(doc, "source_email_message_id", None),
|
||||
}
|
||||
|
||||
def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": v.id,
|
||||
"document_id": v.document_id,
|
||||
"version_number": v.version_number,
|
||||
"content": v.content,
|
||||
"summary": v.summary,
|
||||
"source": v.source,
|
||||
"created_at": v.created_at.isoformat() if v.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _verify_doc_owner(db, doc: Document, user: str):
|
||||
"""Verify `user` owns this document. Raise 404 if not.
|
||||
|
||||
Documents now carry their own `owner` column, so a doc whose session
|
||||
was deleted (session_id → NULL) can still prove ownership and stay
|
||||
openable / cloneable. We trust that column first and only fall back to
|
||||
the session join for any not-yet-backfilled legacy row.
|
||||
"""
|
||||
if user is None:
|
||||
if _auth_disabled():
|
||||
return # Single-user / no-auth mode: allow access
|
||||
raise HTTPException(403, "Authentication required")
|
||||
if doc.owner is not None:
|
||||
if doc.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
return
|
||||
# Legacy fallback: derive ownership from the linked session.
|
||||
if not doc.session_id:
|
||||
raise HTTPException(404, "Document not found")
|
||||
session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
|
||||
if not session or session.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
|
||||
|
||||
def _owner_session_filter(q, user):
|
||||
"""Restrict a documents query to those owned by `user`.
|
||||
|
||||
Documents now carry their own `owner` column (backfilled at boot from
|
||||
the linked session, or assigned to the admin user for legacy/orphaned
|
||||
docs). We filter on that directly rather than on a session join, so a
|
||||
document whose session was deleted (session_id → NULL) still shows up
|
||||
for its owner instead of silently vanishing from the Library + search.
|
||||
|
||||
The owner backfill runs in init_db before the app serves requests, so
|
||||
by the time this filter is live there are no NULL-owner rows to leak;
|
||||
we therefore match the owner strictly for authenticated callers."""
|
||||
if not user:
|
||||
if user == "" or _auth_disabled():
|
||||
return q
|
||||
return q.filter(False)
|
||||
return q.filter(Document.owner == user)
|
||||
|
||||
|
||||
|
||||
def _slug(name: str) -> str:
|
||||
"""Filesystem-friendly version of a document title.
|
||||
|
||||
Whitespace becomes underscores; other unsafe punctuation is dropped.
|
||||
Preserves letters, digits, dot, hyphen, underscore. Idempotent.
|
||||
"""
|
||||
import re as _re
|
||||
s = (name or "").strip()
|
||||
# Drop the trailing extension if the title happens to include one
|
||||
s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
|
||||
s = _re.sub(r'\s+', '_', s)
|
||||
s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
|
||||
s = _re.sub(r'_+', '_', s).strip('_')
|
||||
return s or "form"
|
||||
|
||||
|
||||
# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
|
||||
_PDF_RENDER_SCALE = 2.0
|
||||
|
||||
|
||||
def _upload_path_inside(upload_dir: str, path: str) -> bool:
|
||||
base = os.path.realpath(upload_dir)
|
||||
p = os.path.realpath(path)
|
||||
try:
|
||||
return os.path.commonpath([base, p]) == base
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_user_upload_path(
|
||||
upload_handler: Any,
|
||||
upload_id: str,
|
||||
owner: Optional[str],
|
||||
auth_manager=None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve an upload id to a filesystem path the caller may read."""
|
||||
if upload_handler is None:
|
||||
return None
|
||||
resolved = upload_handler.resolve_upload(
|
||||
upload_id,
|
||||
owner=owner,
|
||||
auth_manager=auth_manager,
|
||||
)
|
||||
if not isinstance(resolved, dict) or not resolved:
|
||||
return None
|
||||
path = resolved.get("path")
|
||||
upload_dir = getattr(upload_handler, "upload_dir", None)
|
||||
if path and upload_dir and not _upload_path_inside(upload_dir, path):
|
||||
logger.warning("Upload path outside upload directory: %s", path)
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def _locate_upload(
|
||||
upload_dir: str,
|
||||
file_id: str,
|
||||
owner: Optional[str] = None,
|
||||
auth_manager=None,
|
||||
upload_handler: Any = None,
|
||||
):
|
||||
"""Find an upload by its filename ID via UploadHandler.resolve_upload."""
|
||||
if upload_handler is None:
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
base_dir = os.path.dirname(os.path.abspath(upload_dir))
|
||||
upload_handler = UploadHandler(base_dir, upload_dir)
|
||||
return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
|
||||
|
||||
|
||||
def _assert_pdf_marker_upload_owned(
|
||||
request: Request,
|
||||
content: str,
|
||||
user: Optional[str],
|
||||
upload_handler: Any,
|
||||
) -> None:
|
||||
"""Reject document content whose pdf_source marker points at another user's upload."""
|
||||
if upload_handler is None:
|
||||
return
|
||||
from src.pdf_form_doc import find_source_upload_id
|
||||
|
||||
upload_id = find_source_upload_id(content or "")
|
||||
if not upload_id:
|
||||
return
|
||||
auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
|
||||
if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
|
||||
raise HTTPException(
|
||||
400,
|
||||
"Document PDF marker references an upload you do not own",
|
||||
)
|
||||
|
||||
|
||||
def _derive_title(content: str) -> str:
|
||||
"""Derive a title from document content."""
|
||||
import re
|
||||
if not isinstance(content, str):
|
||||
return "Untitled"
|
||||
text = content.strip()
|
||||
if not text:
|
||||
return "Untitled"
|
||||
|
||||
# Markdown header
|
||||
md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
|
||||
if md:
|
||||
title = md.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# HTML heading
|
||||
html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
|
||||
if html:
|
||||
title = html.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# First non-empty line (if short enough)
|
||||
for line in text.split('\n'):
|
||||
line = line.strip()
|
||||
if line and 2 <= len(line) <= 60:
|
||||
title = re.sub(r'[:#*`]+$', '', line).strip()
|
||||
if title and len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title or "Untitled"
|
||||
|
||||
return "Untitled"
|
||||
File diff suppressed because it is too large
Load Diff
+239
-10
@@ -1,14 +1,243 @@
|
||||
"""Backward-compat shim — canonical location is routes/document/document_helpers.py.
|
||||
"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
|
||||
|
||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
||||
that ``import routes.document_helpers``, ``from routes.document_helpers import
|
||||
X``, and the ``sys.modules.pop("routes.document_helpers")`` + re-import
|
||||
pattern used by test_security_regressions.py all operate on the *same* object.
|
||||
Keeps existing import paths working after slice 2m (#4082/#4071).
|
||||
"""
|
||||
"""Document routes — CRUD for living documents with version history."""
|
||||
|
||||
import sys as _sys
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from routes.document import document_helpers as _canonical # noqa: F401
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
_sys.modules[__name__] = _canonical
|
||||
from core.database import Document, DocumentVersion
|
||||
from core.database import Session as DbSession
|
||||
from src.auth_helpers import _auth_disabled
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---- Request schemas ----
|
||||
|
||||
class DocumentCreate(BaseModel):
|
||||
session_id: Optional[str] = None
|
||||
title: str = "Untitled"
|
||||
language: Optional[str] = None
|
||||
content: str = ""
|
||||
|
||||
class DocumentUpdate(BaseModel):
|
||||
content: str
|
||||
summary: Optional[str] = None
|
||||
force_version: bool = False
|
||||
|
||||
class DocumentPatch(BaseModel):
|
||||
title: Optional[str] = None
|
||||
language: Optional[str] = None
|
||||
session_id: Optional[str] = None # link/unlink document to a session
|
||||
|
||||
|
||||
# ---- Helpers ----
|
||||
|
||||
def _doc_to_dict(doc: Document) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": doc.id,
|
||||
"session_id": doc.session_id,
|
||||
"title": doc.title,
|
||||
"language": doc.language,
|
||||
"current_content": doc.current_content,
|
||||
"version_count": doc.version_count,
|
||||
"is_active": doc.is_active,
|
||||
"archived": bool(getattr(doc, "archived", False)),
|
||||
"created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
|
||||
"updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
|
||||
# Source-email provenance (set when doc was created from an email
|
||||
# attachment) — drives the "Send signed reply" menu item.
|
||||
"source_email_uid": getattr(doc, "source_email_uid", None),
|
||||
"source_email_folder": getattr(doc, "source_email_folder", None),
|
||||
"source_email_account_id": getattr(doc, "source_email_account_id", None),
|
||||
"source_email_message_id": getattr(doc, "source_email_message_id", None),
|
||||
}
|
||||
|
||||
def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": v.id,
|
||||
"document_id": v.document_id,
|
||||
"version_number": v.version_number,
|
||||
"content": v.content,
|
||||
"summary": v.summary,
|
||||
"source": v.source,
|
||||
"created_at": v.created_at.isoformat() if v.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _verify_doc_owner(db, doc: Document, user: str):
|
||||
"""Verify `user` owns this document. Raise 404 if not.
|
||||
|
||||
Documents now carry their own `owner` column, so a doc whose session
|
||||
was deleted (session_id → NULL) can still prove ownership and stay
|
||||
openable / cloneable. We trust that column first and only fall back to
|
||||
the session join for any not-yet-backfilled legacy row.
|
||||
"""
|
||||
if user is None:
|
||||
if _auth_disabled():
|
||||
return # Single-user / no-auth mode: allow access
|
||||
raise HTTPException(403, "Authentication required")
|
||||
if doc.owner is not None:
|
||||
if doc.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
return
|
||||
# Legacy fallback: derive ownership from the linked session.
|
||||
if not doc.session_id:
|
||||
raise HTTPException(404, "Document not found")
|
||||
session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
|
||||
if not session or session.owner != user:
|
||||
raise HTTPException(404, "Document not found")
|
||||
|
||||
|
||||
def _owner_session_filter(q, user):
|
||||
"""Restrict a documents query to those owned by `user`.
|
||||
|
||||
Documents now carry their own `owner` column (backfilled at boot from
|
||||
the linked session, or assigned to the admin user for legacy/orphaned
|
||||
docs). We filter on that directly rather than on a session join, so a
|
||||
document whose session was deleted (session_id → NULL) still shows up
|
||||
for its owner instead of silently vanishing from the Library + search.
|
||||
|
||||
The owner backfill runs in init_db before the app serves requests, so
|
||||
by the time this filter is live there are no NULL-owner rows to leak;
|
||||
we therefore match the owner strictly for authenticated callers."""
|
||||
if not user:
|
||||
if user == "" or _auth_disabled():
|
||||
return q
|
||||
return q.filter(False)
|
||||
return q.filter(Document.owner == user)
|
||||
|
||||
|
||||
|
||||
def _slug(name: str) -> str:
|
||||
"""Filesystem-friendly version of a document title.
|
||||
|
||||
Whitespace becomes underscores; other unsafe punctuation is dropped.
|
||||
Preserves letters, digits, dot, hyphen, underscore. Idempotent.
|
||||
"""
|
||||
import re as _re
|
||||
s = (name or "").strip()
|
||||
# Drop the trailing extension if the title happens to include one
|
||||
s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
|
||||
s = _re.sub(r'\s+', '_', s)
|
||||
s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
|
||||
s = _re.sub(r'_+', '_', s).strip('_')
|
||||
return s or "form"
|
||||
|
||||
|
||||
# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
|
||||
_PDF_RENDER_SCALE = 2.0
|
||||
|
||||
|
||||
def _upload_path_inside(upload_dir: str, path: str) -> bool:
|
||||
base = os.path.realpath(upload_dir)
|
||||
p = os.path.realpath(path)
|
||||
try:
|
||||
return os.path.commonpath([base, p]) == base
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_user_upload_path(
|
||||
upload_handler: Any,
|
||||
upload_id: str,
|
||||
owner: Optional[str],
|
||||
auth_manager=None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve an upload id to a filesystem path the caller may read."""
|
||||
if upload_handler is None:
|
||||
return None
|
||||
resolved = upload_handler.resolve_upload(
|
||||
upload_id,
|
||||
owner=owner,
|
||||
auth_manager=auth_manager,
|
||||
)
|
||||
if not isinstance(resolved, dict) or not resolved:
|
||||
return None
|
||||
path = resolved.get("path")
|
||||
upload_dir = getattr(upload_handler, "upload_dir", None)
|
||||
if path and upload_dir and not _upload_path_inside(upload_dir, path):
|
||||
logger.warning("Upload path outside upload directory: %s", path)
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def _locate_upload(
|
||||
upload_dir: str,
|
||||
file_id: str,
|
||||
owner: Optional[str] = None,
|
||||
auth_manager=None,
|
||||
upload_handler: Any = None,
|
||||
):
|
||||
"""Find an upload by its filename ID via UploadHandler.resolve_upload."""
|
||||
if upload_handler is None:
|
||||
from src.upload_handler import UploadHandler
|
||||
|
||||
base_dir = os.path.dirname(os.path.abspath(upload_dir))
|
||||
upload_handler = UploadHandler(base_dir, upload_dir)
|
||||
return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
|
||||
|
||||
|
||||
def _assert_pdf_marker_upload_owned(
|
||||
request: Request,
|
||||
content: str,
|
||||
user: Optional[str],
|
||||
upload_handler: Any,
|
||||
) -> None:
|
||||
"""Reject document content whose pdf_source marker points at another user's upload."""
|
||||
if upload_handler is None:
|
||||
return
|
||||
from src.pdf_form_doc import find_source_upload_id
|
||||
|
||||
upload_id = find_source_upload_id(content or "")
|
||||
if not upload_id:
|
||||
return
|
||||
auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
|
||||
if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
|
||||
raise HTTPException(
|
||||
400,
|
||||
"Document PDF marker references an upload you do not own",
|
||||
)
|
||||
|
||||
|
||||
def _derive_title(content: str) -> str:
|
||||
"""Derive a title from document content."""
|
||||
import re
|
||||
if not isinstance(content, str):
|
||||
return "Untitled"
|
||||
text = content.strip()
|
||||
if not text:
|
||||
return "Untitled"
|
||||
|
||||
# Markdown header
|
||||
md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
|
||||
if md:
|
||||
title = md.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# HTML heading
|
||||
html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
|
||||
if html:
|
||||
title = html.group(1).strip()
|
||||
if len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title
|
||||
|
||||
# First non-empty line (if short enough)
|
||||
for line in text.split('\n'):
|
||||
line = line.strip()
|
||||
if line and 2 <= len(line) <= 60:
|
||||
title = re.sub(r'[:#*`]+$', '', line).strip()
|
||||
if title and len(title) > 50:
|
||||
title = title[:48] + "…"
|
||||
return title or "Untitled"
|
||||
|
||||
return "Untitled"
|
||||
|
||||
+1806
-13
File diff suppressed because it is too large
Load Diff
@@ -21,7 +21,7 @@ def _strip_list_prefix(text: str) -> str:
|
||||
return text
|
||||
return _LIST_PREFIX_RE.sub("", text, count=1).strip()
|
||||
|
||||
from services.memory import MemoryManager, MemoryStoreUnreadable
|
||||
from services.memory import MemoryManager
|
||||
from core.session_manager import SessionManager
|
||||
from src.request_models import MemoryAddRequest
|
||||
from core.database import SessionLocal
|
||||
@@ -35,22 +35,6 @@ from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _load_for_update(memory_manager) -> List[Dict[str, Any]]:
|
||||
"""Load the whole store for a read-modify-write cycle.
|
||||
|
||||
A transient read failure must not look like an empty store: the caller
|
||||
would append to ``[]`` and save that back, atomically destroying every
|
||||
existing memory (issue #5673). Surface it as a 503 and change nothing.
|
||||
"""
|
||||
try:
|
||||
return memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
logger.error("Refusing to rewrite the memory store: %s", e)
|
||||
raise HTTPException(
|
||||
503, "Memory store is temporarily unreadable — no changes were made."
|
||||
)
|
||||
|
||||
|
||||
def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
|
||||
"""Set up memory-related routes."""
|
||||
router = APIRouter(prefix="/api/memory", tags=["memory"])
|
||||
@@ -132,7 +116,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
|
||||
if memory_data.session_id:
|
||||
new_entry["session_id"] = memory_data.session_id
|
||||
all_mem = _load_for_update(memory_manager)
|
||||
all_mem = memory_manager.load_all()
|
||||
all_mem.append(new_entry)
|
||||
memory_manager.save(all_mem)
|
||||
# Sync vector index
|
||||
@@ -503,7 +487,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
|
||||
"""Pin or unpin a memory. Pinned memories are always included in context."""
|
||||
user = _owner(request)
|
||||
all_mem = _load_for_update(memory_manager)
|
||||
all_mem = memory_manager.load_all()
|
||||
for i, memory in enumerate(all_mem):
|
||||
if memory["id"] == memory_id:
|
||||
_verify_memory_owner(memory, user)
|
||||
@@ -528,7 +512,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
|
||||
"""Update an existing memory item with new text and optional category."""
|
||||
user = _owner(request)
|
||||
all_mem = _load_for_update(memory_manager)
|
||||
all_mem = memory_manager.load_all()
|
||||
for i, memory in enumerate(all_mem):
|
||||
if memory["id"] == memory_id:
|
||||
_verify_memory_owner(memory, user)
|
||||
@@ -550,7 +534,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
def delete_memory(request: Request, memory_id: str):
|
||||
"""Delete a memory item by its ID."""
|
||||
user = _owner(request)
|
||||
all_mem = _load_for_update(memory_manager)
|
||||
all_mem = memory_manager.load_all()
|
||||
|
||||
# Find and verify ownership before deleting
|
||||
target = next((m for m in all_mem if m["id"] == memory_id), None)
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Vault route domain package (slice 2k, #4082/#4071).
|
||||
|
||||
Contains vault_routes.py, migrated from the flat routes/ directory.
|
||||
Backward-compat shim at routes/vault_routes.py re-exports from here.
|
||||
"""
|
||||
@@ -1,242 +0,0 @@
|
||||
"""
|
||||
vault_routes.py
|
||||
|
||||
Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
|
||||
Stores the BW_SESSION key in data/vault.json with restrictive permissions.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.middleware import require_admin
|
||||
from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
|
||||
from src.constants import VAULT_FILE as _VAULT_FILE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VAULT_FILE = Path(_VAULT_FILE)
|
||||
|
||||
|
||||
def _find_bw() -> str:
|
||||
"""Locate the bw binary, checking PATH and common npm-global locations.
|
||||
|
||||
On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
|
||||
which_tool via PATHEXT.
|
||||
"""
|
||||
p = which_tool("bw")
|
||||
if p:
|
||||
return p
|
||||
if IS_WINDOWS:
|
||||
appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
|
||||
for candidate in (
|
||||
os.path.join(appdata, "npm", "bw.cmd"),
|
||||
os.path.join(appdata, "npm", "bw.exe"),
|
||||
):
|
||||
if os.path.isfile(candidate):
|
||||
return candidate
|
||||
return "bw"
|
||||
home = os.path.expanduser("~")
|
||||
for candidate in (
|
||||
f"{home}/.npm-global/bin/bw",
|
||||
f"{home}/.nvm/versions/node/*/bin/bw",
|
||||
"/usr/local/bin/bw",
|
||||
"/opt/homebrew/bin/bw",
|
||||
):
|
||||
if "*" in candidate:
|
||||
import glob
|
||||
for m in glob.glob(candidate):
|
||||
if os.path.isfile(m) and os.access(m, os.X_OK):
|
||||
return m
|
||||
elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||
return candidate
|
||||
return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below)
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
if VAULT_FILE.exists():
|
||||
try:
|
||||
data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def _save_config(cfg: dict):
|
||||
VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
|
||||
# POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
|
||||
# is ACL-restricted already).
|
||||
safe_chmod(str(VAULT_FILE), 0o600)
|
||||
|
||||
|
||||
async def _run_bw(args: list, session: str = None, input_text: str = None,
|
||||
bw_password: str = None) -> tuple:
|
||||
env = {}
|
||||
env.update(os.environ)
|
||||
if session:
|
||||
env["BW_SESSION"] = session
|
||||
# Secrets must never be passed as argv — process arguments are world-readable
|
||||
# via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
|
||||
# support for bw commands that need it; unlock/login callers should prefer
|
||||
# stdin so the master password is not left in the child environment either.
|
||||
if bw_password is not None:
|
||||
env["BW_PASSWORD"] = bw_password
|
||||
bw_path = _find_bw()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
bw_path, *args,
|
||||
stdin=asyncio.subprocess.PIPE if input_text else None,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
|
||||
except Exception as e:
|
||||
return "", f"Failed to launch bw: {e}", 1
|
||||
try:
|
||||
stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
|
||||
except Exception as e:
|
||||
return "", f"bw subprocess error: {e}", 1
|
||||
return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
|
||||
|
||||
|
||||
class VaultConfig(BaseModel):
|
||||
server_url: str = ""
|
||||
email: str = ""
|
||||
|
||||
|
||||
class VaultUnlockRequest(BaseModel):
|
||||
master_password: str
|
||||
|
||||
|
||||
class VaultLoginRequest(BaseModel):
|
||||
email: str
|
||||
master_password: str
|
||||
|
||||
|
||||
def setup_vault_routes():
|
||||
router = APIRouter(prefix="/api/vault", tags=["vault"])
|
||||
|
||||
@router.get("/config")
|
||||
async def get_config(request: Request):
|
||||
"""Return vault config (no sensitive fields)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
return {
|
||||
"server_url": cfg.get("server_url", ""),
|
||||
"email": cfg.get("email", ""),
|
||||
"unlocked": bool(cfg.get("session")),
|
||||
"unlocked_at": cfg.get("unlocked_at", ""),
|
||||
"bw_installed": await _check_bw_installed(),
|
||||
}
|
||||
|
||||
@router.post("/config")
|
||||
async def save_config(req: VaultConfig, request: Request):
|
||||
"""Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg["server_url"] = req.server_url.strip().rstrip("/")
|
||||
cfg["email"] = req.email.strip()
|
||||
|
||||
if cfg["server_url"]:
|
||||
_, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
|
||||
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/login")
|
||||
async def login(req: VaultLoginRequest, request: Request):
|
||||
"""Log in to Vaultwarden (required once per account)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
# Update email
|
||||
cfg["email"] = req.email
|
||||
_save_config(cfg)
|
||||
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["login", req.email, "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
# Already logged in is OK
|
||||
if "already logged in" in stderr.lower():
|
||||
return {"ok": True, "already": True}
|
||||
return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
|
||||
# bw login --raw prints session key on success (when 2FA disabled)
|
||||
if stdout:
|
||||
cfg["session"] = stdout
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/unlock")
|
||||
async def unlock(req: VaultUnlockRequest, request: Request):
|
||||
"""Unlock the vault and save the session key."""
|
||||
require_admin(request)
|
||||
# Pass the master password on stdin, not argv. argv is visible through
|
||||
# `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
|
||||
# the child process environment.
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["unlock", "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
|
||||
session = stdout.strip()
|
||||
if not session:
|
||||
return {"ok": False, "error": "bw returned empty session"}
|
||||
cfg = _load_config()
|
||||
cfg["session"] = session
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True, "message": "Vault unlocked"}
|
||||
|
||||
@router.post("/lock")
|
||||
async def lock(request: Request):
|
||||
"""Lock the vault (clear session from config)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
# Also tell bw to lock
|
||||
await _run_bw(["lock"])
|
||||
return {"ok": True, "message": "Vault locked"}
|
||||
|
||||
@router.post("/logout")
|
||||
async def logout(request: Request):
|
||||
"""Log out of the Bitwarden CLI completely."""
|
||||
require_admin(request)
|
||||
await _run_bw(["logout"])
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("email", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
async def _check_bw_installed() -> bool:
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
_find_bw(), "--version",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await proc.communicate()
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
+237
-9
@@ -1,14 +1,242 @@
|
||||
"""Backward-compat shim — canonical location is routes/vault/vault_routes.py.
|
||||
"""
|
||||
vault_routes.py
|
||||
|
||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
||||
that ``import routes.vault_routes``, ``from routes.vault_routes import X``,
|
||||
and the ``import ... as vr`` + ``monkeypatch.setattr(vr, ...)`` pattern used
|
||||
by test_vault_password_not_in_argv.py all operate on the *same* object.
|
||||
Keeps existing import paths working after slice 2k (#4082/#4071).
|
||||
Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
|
||||
Stores the BW_SESSION key in data/vault.json with restrictive permissions.
|
||||
"""
|
||||
|
||||
import sys as _sys
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from routes.vault import vault_routes as _canonical # noqa: F401
|
||||
from core.middleware import require_admin
|
||||
from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
|
||||
from src.constants import VAULT_FILE as _VAULT_FILE
|
||||
|
||||
_sys.modules[__name__] = _canonical
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VAULT_FILE = Path(_VAULT_FILE)
|
||||
|
||||
|
||||
def _find_bw() -> str:
|
||||
"""Locate the bw binary, checking PATH and common npm-global locations.
|
||||
|
||||
On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
|
||||
which_tool via PATHEXT.
|
||||
"""
|
||||
p = which_tool("bw")
|
||||
if p:
|
||||
return p
|
||||
if IS_WINDOWS:
|
||||
appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
|
||||
for candidate in (
|
||||
os.path.join(appdata, "npm", "bw.cmd"),
|
||||
os.path.join(appdata, "npm", "bw.exe"),
|
||||
):
|
||||
if os.path.isfile(candidate):
|
||||
return candidate
|
||||
return "bw"
|
||||
home = os.path.expanduser("~")
|
||||
for candidate in (
|
||||
f"{home}/.npm-global/bin/bw",
|
||||
f"{home}/.nvm/versions/node/*/bin/bw",
|
||||
"/usr/local/bin/bw",
|
||||
"/opt/homebrew/bin/bw",
|
||||
):
|
||||
if "*" in candidate:
|
||||
import glob
|
||||
for m in glob.glob(candidate):
|
||||
if os.path.isfile(m) and os.access(m, os.X_OK):
|
||||
return m
|
||||
elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||
return candidate
|
||||
return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below)
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
if VAULT_FILE.exists():
|
||||
try:
|
||||
data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def _save_config(cfg: dict):
|
||||
VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
|
||||
# POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
|
||||
# is ACL-restricted already).
|
||||
safe_chmod(str(VAULT_FILE), 0o600)
|
||||
|
||||
|
||||
async def _run_bw(args: list, session: str = None, input_text: str = None,
|
||||
bw_password: str = None) -> tuple:
|
||||
env = {}
|
||||
env.update(os.environ)
|
||||
if session:
|
||||
env["BW_SESSION"] = session
|
||||
# Secrets must never be passed as argv — process arguments are world-readable
|
||||
# via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
|
||||
# support for bw commands that need it; unlock/login callers should prefer
|
||||
# stdin so the master password is not left in the child environment either.
|
||||
if bw_password is not None:
|
||||
env["BW_PASSWORD"] = bw_password
|
||||
bw_path = _find_bw()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
bw_path, *args,
|
||||
stdin=asyncio.subprocess.PIPE if input_text else None,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
|
||||
except Exception as e:
|
||||
return "", f"Failed to launch bw: {e}", 1
|
||||
try:
|
||||
stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
|
||||
except Exception as e:
|
||||
return "", f"bw subprocess error: {e}", 1
|
||||
return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
|
||||
|
||||
|
||||
class VaultConfig(BaseModel):
|
||||
server_url: str = ""
|
||||
email: str = ""
|
||||
|
||||
|
||||
class VaultUnlockRequest(BaseModel):
|
||||
master_password: str
|
||||
|
||||
|
||||
class VaultLoginRequest(BaseModel):
|
||||
email: str
|
||||
master_password: str
|
||||
|
||||
|
||||
def setup_vault_routes():
|
||||
router = APIRouter(prefix="/api/vault", tags=["vault"])
|
||||
|
||||
@router.get("/config")
|
||||
async def get_config(request: Request):
|
||||
"""Return vault config (no sensitive fields)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
return {
|
||||
"server_url": cfg.get("server_url", ""),
|
||||
"email": cfg.get("email", ""),
|
||||
"unlocked": bool(cfg.get("session")),
|
||||
"unlocked_at": cfg.get("unlocked_at", ""),
|
||||
"bw_installed": await _check_bw_installed(),
|
||||
}
|
||||
|
||||
@router.post("/config")
|
||||
async def save_config(req: VaultConfig, request: Request):
|
||||
"""Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg["server_url"] = req.server_url.strip().rstrip("/")
|
||||
cfg["email"] = req.email.strip()
|
||||
|
||||
if cfg["server_url"]:
|
||||
_, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
|
||||
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/login")
|
||||
async def login(req: VaultLoginRequest, request: Request):
|
||||
"""Log in to Vaultwarden (required once per account)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
# Update email
|
||||
cfg["email"] = req.email
|
||||
_save_config(cfg)
|
||||
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["login", req.email, "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
# Already logged in is OK
|
||||
if "already logged in" in stderr.lower():
|
||||
return {"ok": True, "already": True}
|
||||
return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
|
||||
# bw login --raw prints session key on success (when 2FA disabled)
|
||||
if stdout:
|
||||
cfg["session"] = stdout
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/unlock")
|
||||
async def unlock(req: VaultUnlockRequest, request: Request):
|
||||
"""Unlock the vault and save the session key."""
|
||||
require_admin(request)
|
||||
# Pass the master password on stdin, not argv. argv is visible through
|
||||
# `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
|
||||
# the child process environment.
|
||||
stdout, stderr, rc = await _run_bw(
|
||||
["unlock", "--raw"],
|
||||
input_text=req.master_password + "\n",
|
||||
)
|
||||
if rc != 0:
|
||||
return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
|
||||
session = stdout.strip()
|
||||
if not session:
|
||||
return {"ok": False, "error": "bw returned empty session"}
|
||||
cfg = _load_config()
|
||||
cfg["session"] = session
|
||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||
_save_config(cfg)
|
||||
return {"ok": True, "message": "Vault unlocked"}
|
||||
|
||||
@router.post("/lock")
|
||||
async def lock(request: Request):
|
||||
"""Lock the vault (clear session from config)."""
|
||||
require_admin(request)
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
# Also tell bw to lock
|
||||
await _run_bw(["lock"])
|
||||
return {"ok": True, "message": "Vault locked"}
|
||||
|
||||
@router.post("/logout")
|
||||
async def logout(request: Request):
|
||||
"""Log out of the Bitwarden CLI completely."""
|
||||
require_admin(request)
|
||||
await _run_bw(["logout"])
|
||||
cfg = _load_config()
|
||||
cfg.pop("session", None)
|
||||
cfg.pop("email", None)
|
||||
cfg.pop("unlocked_at", None)
|
||||
_save_config(cfg)
|
||||
return {"ok": True}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
async def _check_bw_installed() -> bool:
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
_find_bw(), "--version",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await proc.communicate()
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Webhook route domain package (slice 2l, #4082/#4071).
|
||||
|
||||
Contains webhook_routes.py, migrated from the flat routes/ directory.
|
||||
Backward-compat shim at routes/webhook_routes.py re-exports from here.
|
||||
"""
|
||||
@@ -1,395 +0,0 @@
|
||||
"""Webhook, API Token, and sync chat routes."""
|
||||
|
||||
import uuid
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Request, Form
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from core.database import SessionLocal, Webhook, ModelEndpoint
|
||||
from src.auth_helpers import owner_filter
|
||||
from src.url_security import validate_public_http_url
|
||||
from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["webhooks"])
|
||||
|
||||
# Input limits
|
||||
MAX_NAME_LEN = 100
|
||||
MAX_URL_LEN = 2048
|
||||
MAX_SECRET_LEN = 256
|
||||
MAX_MESSAGE_LEN = 32_000
|
||||
|
||||
|
||||
from core.middleware import require_admin as _require_admin
|
||||
|
||||
|
||||
def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
|
||||
"""First enabled ModelEndpoint visible to token_owner — their own rows plus
|
||||
legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
|
||||
let a chat-scoped token fall back onto another user's private endpoint and
|
||||
silently spend that owner's API key/quota. Prefer owner rows before shared
|
||||
rows. Fails closed to null-owner rows only when token_owner is absent.
|
||||
Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
|
||||
"""
|
||||
query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712
|
||||
if token_owner:
|
||||
query = owner_filter(query, ModelEndpoint, token_owner)
|
||||
return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
|
||||
return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711
|
||||
|
||||
|
||||
def _caller_owns_session(sess_owner, caller) -> bool:
|
||||
"""Strict session-ownership gate for the token-authenticated sync-chat
|
||||
endpoint (`POST /api/v1/chat`).
|
||||
|
||||
Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
|
||||
gates in notes/calendar/gallery: a caller may resume a session ONLY when
|
||||
its owner matches them exactly. A null/empty session owner (legacy or
|
||||
migrated rows) is deliberately NOT resumable by an arbitrary token — the
|
||||
old ``sess_owner and sess_owner != caller`` form skipped the check whenever
|
||||
``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
|
||||
device) could resume such a session, inject a message, and read back its
|
||||
history and reuse the owner's endpoint credentials. Fail closed: an
|
||||
unresolvable caller also returns False.
|
||||
"""
|
||||
if not caller:
|
||||
return False
|
||||
return sess_owner == caller
|
||||
|
||||
|
||||
def setup_webhook_routes(
|
||||
webhook_manager: WebhookManager,
|
||||
auth_manager,
|
||||
session_manager=None,
|
||||
api_key_manager=None,
|
||||
) -> APIRouter:
|
||||
|
||||
@router.get("/webhooks")
|
||||
def list_webhooks(request: Request):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
hooks = db.query(Webhook).all()
|
||||
return [
|
||||
{
|
||||
"id": w.id,
|
||||
"name": w.name,
|
||||
"url": w.url,
|
||||
"has_secret": bool(w.secret),
|
||||
"events": w.events.split(",") if w.events else [],
|
||||
"is_active": w.is_active,
|
||||
"last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
|
||||
"last_status_code": w.last_status_code,
|
||||
"last_error": w.last_error,
|
||||
"created_at": w.created_at.isoformat() if w.created_at else None,
|
||||
}
|
||||
for w in hooks
|
||||
]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.post("/webhooks")
|
||||
def create_webhook(
|
||||
request: Request,
|
||||
name: str = Form(""),
|
||||
url: str = Form(""),
|
||||
secret: str = Form(""),
|
||||
events: str = Form(""),
|
||||
):
|
||||
_require_admin(request)
|
||||
name = name.strip()[:MAX_NAME_LEN]
|
||||
if not name:
|
||||
raise HTTPException(400, "Webhook name is required")
|
||||
try:
|
||||
url = validate_webhook_url(url)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
try:
|
||||
events = validate_events(events)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
|
||||
secret_val = secret.strip()[:MAX_SECRET_LEN] or None
|
||||
# Encrypt the secret at rest using the same Fernet key as API keys
|
||||
encrypted_secret = None
|
||||
if secret_val and api_key_manager:
|
||||
encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
|
||||
elif secret_val:
|
||||
encrypted_secret = secret_val # Fallback if no encryption available
|
||||
|
||||
webhook_id = str(uuid.uuid4())[:8]
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db.add(Webhook(
|
||||
id=webhook_id,
|
||||
name=name,
|
||||
url=url,
|
||||
secret=encrypted_secret,
|
||||
events=events,
|
||||
is_active=True,
|
||||
))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return {"id": webhook_id, "name": name}
|
||||
|
||||
@router.post("/webhooks/{webhook_id}/test")
|
||||
async def test_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
url, secret = wh.url, wh.secret
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
await webhook_manager.deliver_test(webhook_id, url, secret)
|
||||
return {"status": "sent"}
|
||||
|
||||
@router.patch("/webhooks/{webhook_id}")
|
||||
def toggle_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
wh.is_active = not wh.is_active
|
||||
db.commit()
|
||||
return {"id": webhook_id, "is_active": wh.is_active}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.delete("/webhooks/{webhook_id}")
|
||||
def delete_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
|
||||
db.commit()
|
||||
if not deleted:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
finally:
|
||||
db.close()
|
||||
return {"status": "deleted"}
|
||||
|
||||
# ================================================================
|
||||
# Sync Chat Endpoint (for n8n / Make / Activepieces)
|
||||
# ================================================================
|
||||
|
||||
# Known provider base URLs — auto-resolved from api_key prefix or model name
|
||||
KNOWN_PROVIDERS = {
|
||||
"deepseek": "https://api.deepseek.com/v1",
|
||||
"openai": "https://api.openai.com/v1",
|
||||
"mistral": "https://api.mistral.ai/v1",
|
||||
"groq": "https://api.groq.com/openai/v1",
|
||||
"together": "https://api.together.xyz/v1",
|
||||
"openrouter": "https://openrouter.ai/api/v1",
|
||||
"ollama": "https://ollama.com/api",
|
||||
"opencode-zen": "https://opencode.ai/zen/v1",
|
||||
"opencode-go": "https://opencode.ai/zen/go/v1",
|
||||
"fireworks": "https://api.fireworks.ai/inference/v1",
|
||||
"venice": "https://api.venice.ai/api/v1",
|
||||
"kimi-code": "https://api.kimi.com/coding/v1",
|
||||
"kimicode": "https://api.kimi.com/coding/v1",
|
||||
}
|
||||
|
||||
# Model prefix → provider mapping for auto-detection
|
||||
MODEL_PROVIDER_MAP = {
|
||||
"deepseek": "deepseek",
|
||||
"gpt-": "openai",
|
||||
"o1": "openai",
|
||||
"o3": "openai",
|
||||
"o4": "openai",
|
||||
"mistral": "mistral",
|
||||
"llama": "groq",
|
||||
"mixtral": "groq",
|
||||
"kimi-for-coding": "kimi-code",
|
||||
"kimi": "kimi-code",
|
||||
}
|
||||
|
||||
def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
|
||||
"""Try to auto-resolve a base URL from provider name or model prefix."""
|
||||
if provider and provider.lower() in KNOWN_PROVIDERS:
|
||||
return KNOWN_PROVIDERS[provider.lower()]
|
||||
if model:
|
||||
model_lower = model.lower()
|
||||
for prefix, prov in MODEL_PROVIDER_MAP.items():
|
||||
if model_lower.startswith(prefix):
|
||||
return KNOWN_PROVIDERS[prov]
|
||||
return None
|
||||
|
||||
class SyncChatRequest(BaseModel):
|
||||
message: str = Field(..., max_length=MAX_MESSAGE_LEN)
|
||||
model: Optional[str] = Field(None, max_length=200)
|
||||
session: Optional[str] = Field(None, max_length=100)
|
||||
api_key: Optional[str] = Field(None, max_length=256)
|
||||
base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
|
||||
provider: Optional[str] = Field(None, max_length=50)
|
||||
|
||||
@router.post("/v1/chat")
|
||||
async def sync_chat(request: Request, body: SyncChatRequest):
|
||||
if not getattr(request.state, "api_token", False):
|
||||
raise HTTPException(403, "This endpoint requires an API token")
|
||||
scopes = set(getattr(request.state, "api_token_scopes", []) or [])
|
||||
if "chat" not in scopes:
|
||||
raise HTTPException(403, "API token is not scoped for chat")
|
||||
token_owner = getattr(request.state, "api_token_owner", None)
|
||||
|
||||
from core.models import ChatMessage
|
||||
from src.llm_core import llm_call_async
|
||||
from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
|
||||
|
||||
message = body.message.strip()
|
||||
if not message:
|
||||
raise HTTPException(400, "Message is required")
|
||||
|
||||
session_id = body.session
|
||||
sess = None
|
||||
|
||||
# --- Case 1: Resume an existing session ---
|
||||
if session_id and session_manager:
|
||||
try:
|
||||
sess = session_manager.get_session(session_id)
|
||||
except (KeyError, Exception):
|
||||
raise HTTPException(404, "Session not found")
|
||||
# SECURITY: verify the API-token's user owns this session — without
|
||||
# this any token holder could resume any user's chat by passing its
|
||||
# ID. The token's user is on request.state.user (set by API-token
|
||||
# middleware); fall back to require_user if not present.
|
||||
try:
|
||||
from src.auth_helpers import get_current_user as _gcu
|
||||
_tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
|
||||
except Exception:
|
||||
_tok_user = None
|
||||
# Strict ownership (see _caller_owns_session): fail closed so a
|
||||
# null-owner / cross-owner session can't be resumed by an arbitrary
|
||||
# chat-scoped token.
|
||||
_sess_owner = getattr(sess, "owner", None)
|
||||
if not _caller_owns_session(_sess_owner, _tok_user):
|
||||
raise HTTPException(404, "Session not found")
|
||||
|
||||
# --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
|
||||
if not sess and body.api_key:
|
||||
api_key = body.api_key.strip()
|
||||
model = body.model or "deepseek-chat"
|
||||
|
||||
# Validate only token-supplied direct base_url; auto-resolved known-provider
|
||||
# URLs are not subject to extra local/LAN blocking beyond existing provider logic.
|
||||
direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
|
||||
if direct_base_url:
|
||||
try:
|
||||
base_url = validate_public_http_url(direct_base_url)
|
||||
except ValueError as e:
|
||||
detail = str(e).replace("URL", "base_url", 1)
|
||||
raise HTTPException(400, detail)
|
||||
else:
|
||||
base_url = _resolve_base_url(model, body.provider)
|
||||
if not base_url:
|
||||
raise HTTPException(400,
|
||||
"Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
|
||||
"or provider ('deepseek', 'openai', 'groq', etc.)")
|
||||
base_url = normalize_base(base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Case 3: Fall back to first configured ModelEndpoint ---
|
||||
if not sess:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ep = _select_api_chat_fallback_endpoint(db, token_owner)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if not ep:
|
||||
raise HTTPException(400,
|
||||
"No session, api_key, or configured endpoints. "
|
||||
"Pass api_key + model, or configure an endpoint in Admin.")
|
||||
|
||||
base_url = normalize_base(ep.base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
model = body.model or "auto"
|
||||
api_key = ep.api_key
|
||||
if getattr(ep, "provider_auth_id", None):
|
||||
try:
|
||||
from src.endpoint_resolver import resolve_endpoint_runtime
|
||||
base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not resolve endpoint credentials")
|
||||
|
||||
if model == "auto":
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
models_url = build_models_url(base_url)
|
||||
hdrs = build_headers(api_key, base_url)
|
||||
if models_url:
|
||||
resp = await client.get(models_url, headers=hdrs)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
items = data if isinstance(data, list) else (data.get("data") or [])
|
||||
ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
|
||||
if not ids and isinstance(data, dict):
|
||||
ids = [
|
||||
m.get("name") or m.get("model")
|
||||
for m in (data.get("models") or [])
|
||||
if m.get("name") or m.get("model")
|
||||
]
|
||||
else:
|
||||
import json as _json
|
||||
ids = _json.loads(ep.cached_models or "[]")
|
||||
model = ids[0] if ids else "auto"
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not discover models from endpoint")
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
if api_key:
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Send message and get response ---
|
||||
sess.add_message(ChatMessage("user", message))
|
||||
|
||||
messages = [{"role": m.role, "content": m.content} for m in sess.history]
|
||||
|
||||
reply = await llm_call_async(
|
||||
sess.endpoint_url, sess.model, messages,
|
||||
headers=sess.headers, timeout=120,
|
||||
)
|
||||
sess.add_message(ChatMessage("assistant", reply))
|
||||
session_manager.save_sessions()
|
||||
|
||||
webhook_manager.fire_and_forget("chat.completed", {
|
||||
"session_id": session_id, "model": sess.model,
|
||||
"user_message": message[:2000], "response": reply[:2000],
|
||||
})
|
||||
|
||||
return {"response": reply, "session_id": session_id, "model": sess.model}
|
||||
|
||||
return router
|
||||
+391
-12
@@ -1,16 +1,395 @@
|
||||
"""Backward-compat shim — canonical location is routes/webhook/webhook_routes.py.
|
||||
"""Webhook, API Token, and sync chat routes."""
|
||||
|
||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
||||
that ``import routes.webhook_routes``, ``from routes.webhook_routes import X``,
|
||||
``importlib.import_module("routes.webhook_routes")``, and the
|
||||
``__import__("routes.webhook_routes", fromlist=[...])`` + ``setattr(wh_mod,
|
||||
...)`` pattern used by test_null_owner_gates.py all operate on the *same*
|
||||
object. Keeps existing import paths working after slice 2l (#4082/#4071).
|
||||
Source-introspection tests read the canonical file by path.
|
||||
"""
|
||||
import uuid
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import sys as _sys
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Request, Form
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from routes.webhook import webhook_routes as _canonical # noqa: F401
|
||||
from core.database import SessionLocal, Webhook, ModelEndpoint
|
||||
from src.auth_helpers import owner_filter
|
||||
from src.url_security import validate_public_http_url
|
||||
from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
|
||||
|
||||
_sys.modules[__name__] = _canonical
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["webhooks"])
|
||||
|
||||
# Input limits
|
||||
MAX_NAME_LEN = 100
|
||||
MAX_URL_LEN = 2048
|
||||
MAX_SECRET_LEN = 256
|
||||
MAX_MESSAGE_LEN = 32_000
|
||||
|
||||
|
||||
from core.middleware import require_admin as _require_admin
|
||||
|
||||
|
||||
def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
|
||||
"""First enabled ModelEndpoint visible to token_owner — their own rows plus
|
||||
legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
|
||||
let a chat-scoped token fall back onto another user's private endpoint and
|
||||
silently spend that owner's API key/quota. Prefer owner rows before shared
|
||||
rows. Fails closed to null-owner rows only when token_owner is absent.
|
||||
Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
|
||||
"""
|
||||
query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712
|
||||
if token_owner:
|
||||
query = owner_filter(query, ModelEndpoint, token_owner)
|
||||
return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
|
||||
return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711
|
||||
|
||||
|
||||
def _caller_owns_session(sess_owner, caller) -> bool:
|
||||
"""Strict session-ownership gate for the token-authenticated sync-chat
|
||||
endpoint (`POST /api/v1/chat`).
|
||||
|
||||
Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
|
||||
gates in notes/calendar/gallery: a caller may resume a session ONLY when
|
||||
its owner matches them exactly. A null/empty session owner (legacy or
|
||||
migrated rows) is deliberately NOT resumable by an arbitrary token — the
|
||||
old ``sess_owner and sess_owner != caller`` form skipped the check whenever
|
||||
``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
|
||||
device) could resume such a session, inject a message, and read back its
|
||||
history and reuse the owner's endpoint credentials. Fail closed: an
|
||||
unresolvable caller also returns False.
|
||||
"""
|
||||
if not caller:
|
||||
return False
|
||||
return sess_owner == caller
|
||||
|
||||
|
||||
def setup_webhook_routes(
|
||||
webhook_manager: WebhookManager,
|
||||
auth_manager,
|
||||
session_manager=None,
|
||||
api_key_manager=None,
|
||||
) -> APIRouter:
|
||||
|
||||
@router.get("/webhooks")
|
||||
def list_webhooks(request: Request):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
hooks = db.query(Webhook).all()
|
||||
return [
|
||||
{
|
||||
"id": w.id,
|
||||
"name": w.name,
|
||||
"url": w.url,
|
||||
"has_secret": bool(w.secret),
|
||||
"events": w.events.split(",") if w.events else [],
|
||||
"is_active": w.is_active,
|
||||
"last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
|
||||
"last_status_code": w.last_status_code,
|
||||
"last_error": w.last_error,
|
||||
"created_at": w.created_at.isoformat() if w.created_at else None,
|
||||
}
|
||||
for w in hooks
|
||||
]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.post("/webhooks")
|
||||
def create_webhook(
|
||||
request: Request,
|
||||
name: str = Form(""),
|
||||
url: str = Form(""),
|
||||
secret: str = Form(""),
|
||||
events: str = Form(""),
|
||||
):
|
||||
_require_admin(request)
|
||||
name = name.strip()[:MAX_NAME_LEN]
|
||||
if not name:
|
||||
raise HTTPException(400, "Webhook name is required")
|
||||
try:
|
||||
url = validate_webhook_url(url)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
try:
|
||||
events = validate_events(events)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
|
||||
secret_val = secret.strip()[:MAX_SECRET_LEN] or None
|
||||
# Encrypt the secret at rest using the same Fernet key as API keys
|
||||
encrypted_secret = None
|
||||
if secret_val and api_key_manager:
|
||||
encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
|
||||
elif secret_val:
|
||||
encrypted_secret = secret_val # Fallback if no encryption available
|
||||
|
||||
webhook_id = str(uuid.uuid4())[:8]
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db.add(Webhook(
|
||||
id=webhook_id,
|
||||
name=name,
|
||||
url=url,
|
||||
secret=encrypted_secret,
|
||||
events=events,
|
||||
is_active=True,
|
||||
))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return {"id": webhook_id, "name": name}
|
||||
|
||||
@router.post("/webhooks/{webhook_id}/test")
|
||||
async def test_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
url, secret = wh.url, wh.secret
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
await webhook_manager.deliver_test(webhook_id, url, secret)
|
||||
return {"status": "sent"}
|
||||
|
||||
@router.patch("/webhooks/{webhook_id}")
|
||||
def toggle_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||
if not wh:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
wh.is_active = not wh.is_active
|
||||
db.commit()
|
||||
return {"id": webhook_id, "is_active": wh.is_active}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.delete("/webhooks/{webhook_id}")
|
||||
def delete_webhook(request: Request, webhook_id: str):
|
||||
_require_admin(request)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
|
||||
db.commit()
|
||||
if not deleted:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
finally:
|
||||
db.close()
|
||||
return {"status": "deleted"}
|
||||
|
||||
# ================================================================
|
||||
# Sync Chat Endpoint (for n8n / Make / Activepieces)
|
||||
# ================================================================
|
||||
|
||||
# Known provider base URLs — auto-resolved from api_key prefix or model name
|
||||
KNOWN_PROVIDERS = {
|
||||
"deepseek": "https://api.deepseek.com/v1",
|
||||
"openai": "https://api.openai.com/v1",
|
||||
"mistral": "https://api.mistral.ai/v1",
|
||||
"groq": "https://api.groq.com/openai/v1",
|
||||
"together": "https://api.together.xyz/v1",
|
||||
"openrouter": "https://openrouter.ai/api/v1",
|
||||
"ollama": "https://ollama.com/api",
|
||||
"opencode-zen": "https://opencode.ai/zen/v1",
|
||||
"opencode-go": "https://opencode.ai/zen/go/v1",
|
||||
"fireworks": "https://api.fireworks.ai/inference/v1",
|
||||
"venice": "https://api.venice.ai/api/v1",
|
||||
"kimi-code": "https://api.kimi.com/coding/v1",
|
||||
"kimicode": "https://api.kimi.com/coding/v1",
|
||||
}
|
||||
|
||||
# Model prefix → provider mapping for auto-detection
|
||||
MODEL_PROVIDER_MAP = {
|
||||
"deepseek": "deepseek",
|
||||
"gpt-": "openai",
|
||||
"o1": "openai",
|
||||
"o3": "openai",
|
||||
"o4": "openai",
|
||||
"mistral": "mistral",
|
||||
"llama": "groq",
|
||||
"mixtral": "groq",
|
||||
"kimi-for-coding": "kimi-code",
|
||||
"kimi": "kimi-code",
|
||||
}
|
||||
|
||||
def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
|
||||
"""Try to auto-resolve a base URL from provider name or model prefix."""
|
||||
if provider and provider.lower() in KNOWN_PROVIDERS:
|
||||
return KNOWN_PROVIDERS[provider.lower()]
|
||||
if model:
|
||||
model_lower = model.lower()
|
||||
for prefix, prov in MODEL_PROVIDER_MAP.items():
|
||||
if model_lower.startswith(prefix):
|
||||
return KNOWN_PROVIDERS[prov]
|
||||
return None
|
||||
|
||||
class SyncChatRequest(BaseModel):
|
||||
message: str = Field(..., max_length=MAX_MESSAGE_LEN)
|
||||
model: Optional[str] = Field(None, max_length=200)
|
||||
session: Optional[str] = Field(None, max_length=100)
|
||||
api_key: Optional[str] = Field(None, max_length=256)
|
||||
base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
|
||||
provider: Optional[str] = Field(None, max_length=50)
|
||||
|
||||
@router.post("/v1/chat")
|
||||
async def sync_chat(request: Request, body: SyncChatRequest):
|
||||
if not getattr(request.state, "api_token", False):
|
||||
raise HTTPException(403, "This endpoint requires an API token")
|
||||
scopes = set(getattr(request.state, "api_token_scopes", []) or [])
|
||||
if "chat" not in scopes:
|
||||
raise HTTPException(403, "API token is not scoped for chat")
|
||||
token_owner = getattr(request.state, "api_token_owner", None)
|
||||
|
||||
from core.models import ChatMessage
|
||||
from src.llm_core import llm_call_async
|
||||
from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
|
||||
|
||||
message = body.message.strip()
|
||||
if not message:
|
||||
raise HTTPException(400, "Message is required")
|
||||
|
||||
session_id = body.session
|
||||
sess = None
|
||||
|
||||
# --- Case 1: Resume an existing session ---
|
||||
if session_id and session_manager:
|
||||
try:
|
||||
sess = session_manager.get_session(session_id)
|
||||
except (KeyError, Exception):
|
||||
raise HTTPException(404, "Session not found")
|
||||
# SECURITY: verify the API-token's user owns this session — without
|
||||
# this any token holder could resume any user's chat by passing its
|
||||
# ID. The token's user is on request.state.user (set by API-token
|
||||
# middleware); fall back to require_user if not present.
|
||||
try:
|
||||
from src.auth_helpers import get_current_user as _gcu
|
||||
_tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
|
||||
except Exception:
|
||||
_tok_user = None
|
||||
# Strict ownership (see _caller_owns_session): fail closed so a
|
||||
# null-owner / cross-owner session can't be resumed by an arbitrary
|
||||
# chat-scoped token.
|
||||
_sess_owner = getattr(sess, "owner", None)
|
||||
if not _caller_owns_session(_sess_owner, _tok_user):
|
||||
raise HTTPException(404, "Session not found")
|
||||
|
||||
# --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
|
||||
if not sess and body.api_key:
|
||||
api_key = body.api_key.strip()
|
||||
model = body.model or "deepseek-chat"
|
||||
|
||||
# Validate only token-supplied direct base_url; auto-resolved known-provider
|
||||
# URLs are not subject to extra local/LAN blocking beyond existing provider logic.
|
||||
direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
|
||||
if direct_base_url:
|
||||
try:
|
||||
base_url = validate_public_http_url(direct_base_url)
|
||||
except ValueError as e:
|
||||
detail = str(e).replace("URL", "base_url", 1)
|
||||
raise HTTPException(400, detail)
|
||||
else:
|
||||
base_url = _resolve_base_url(model, body.provider)
|
||||
if not base_url:
|
||||
raise HTTPException(400,
|
||||
"Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
|
||||
"or provider ('deepseek', 'openai', 'groq', etc.)")
|
||||
base_url = normalize_base(base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Case 3: Fall back to first configured ModelEndpoint ---
|
||||
if not sess:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ep = _select_api_chat_fallback_endpoint(db, token_owner)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if not ep:
|
||||
raise HTTPException(400,
|
||||
"No session, api_key, or configured endpoints. "
|
||||
"Pass api_key + model, or configure an endpoint in Admin.")
|
||||
|
||||
base_url = normalize_base(ep.base_url)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
model = body.model or "auto"
|
||||
api_key = ep.api_key
|
||||
if getattr(ep, "provider_auth_id", None):
|
||||
try:
|
||||
from src.endpoint_resolver import resolve_endpoint_runtime
|
||||
base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
|
||||
endpoint_url = build_chat_url(base_url)
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not resolve endpoint credentials")
|
||||
|
||||
if model == "auto":
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
models_url = build_models_url(base_url)
|
||||
hdrs = build_headers(api_key, base_url)
|
||||
if models_url:
|
||||
resp = await client.get(models_url, headers=hdrs)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
items = data if isinstance(data, list) else (data.get("data") or [])
|
||||
ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
|
||||
if not ids and isinstance(data, dict):
|
||||
ids = [
|
||||
m.get("name") or m.get("model")
|
||||
for m in (data.get("models") or [])
|
||||
if m.get("name") or m.get("model")
|
||||
]
|
||||
else:
|
||||
import json as _json
|
||||
ids = _json.loads(ep.cached_models or "[]")
|
||||
model = ids[0] if ids else "auto"
|
||||
except Exception:
|
||||
raise HTTPException(500, "Could not discover models from endpoint")
|
||||
|
||||
if not session_manager:
|
||||
raise HTTPException(500, "Session manager not available")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
sess = session_manager.create_session(
|
||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||
model=model, owner=token_owner,
|
||||
)
|
||||
if api_key:
|
||||
sess.headers = build_headers(api_key, base_url)
|
||||
session_manager.save_sessions()
|
||||
session_id = sid
|
||||
|
||||
# --- Send message and get response ---
|
||||
sess.add_message(ChatMessage("user", message))
|
||||
|
||||
messages = [{"role": m.role, "content": m.content} for m in sess.history]
|
||||
|
||||
reply = await llm_call_async(
|
||||
sess.endpoint_url, sess.model, messages,
|
||||
headers=sess.headers, timeout=120,
|
||||
)
|
||||
sess.add_message(ChatMessage("assistant", reply))
|
||||
session_manager.save_sessions()
|
||||
|
||||
webhook_manager.fire_and_forget("chat.completed", {
|
||||
"session_id": session_id, "model": sess.model,
|
||||
"user_message": message[:2000], "response": reply[:2000],
|
||||
})
|
||||
|
||||
return {"response": reply, "session_id": session_id, "model": sess.model}
|
||||
|
||||
return router
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"""Memory service — persistent memory storage and retrieval."""
|
||||
|
||||
from .service import MemoryService, Memory, MemorySearchResult
|
||||
from .memory import MemoryManager, MemoryStoreUnreadable
|
||||
from .memory import MemoryManager
|
||||
from .memory_vector import MemoryVectorStore
|
||||
|
||||
__all__ = [
|
||||
@@ -10,6 +10,5 @@ __all__ = [
|
||||
"Memory",
|
||||
"MemorySearchResult",
|
||||
"MemoryManager",
|
||||
"MemoryStoreUnreadable",
|
||||
"MemoryVectorStore",
|
||||
]
|
||||
|
||||
@@ -5,16 +5,6 @@ application runtime instantiates ``src.memory.MemoryManager``, so keeping a
|
||||
parallel implementation here risks silent drift between import paths.
|
||||
"""
|
||||
|
||||
from src.memory import (
|
||||
MemoryManager,
|
||||
MemoryStoreUnreadable,
|
||||
get_text_similarity,
|
||||
tokenize,
|
||||
)
|
||||
from src.memory import MemoryManager, get_text_similarity, tokenize
|
||||
|
||||
__all__ = [
|
||||
"MemoryManager",
|
||||
"MemoryStoreUnreadable",
|
||||
"get_text_similarity",
|
||||
"tokenize",
|
||||
]
|
||||
__all__ = ["MemoryManager", "get_text_similarity", "tokenize"]
|
||||
|
||||
@@ -17,8 +17,6 @@ import os
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from src.memory import MemoryStoreUnreadable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -389,13 +387,7 @@ async def extract_and_store(
|
||||
# Get owner from session
|
||||
_owner = getattr(session, 'owner', None)
|
||||
|
||||
# Strict load: this is a read-modify-write. Degrading to [] here would
|
||||
# save only the newly extracted facts and drop the entire store.
|
||||
try:
|
||||
existing = memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
logger.error("Skipping auto memory extraction, store unreadable: %s", e)
|
||||
return
|
||||
existing = memory_manager.load_all()
|
||||
added = 0
|
||||
|
||||
for fact in facts:
|
||||
@@ -634,18 +626,7 @@ async def audit_memories(
|
||||
|
||||
# Merge audited entries back with other users' entries
|
||||
if owner:
|
||||
# Strict load: the merge below reconstructs the whole file. If this
|
||||
# degraded to [] we would save only this owner's audited slice and
|
||||
# destroy every other tenant's memories.
|
||||
try:
|
||||
all_entries = memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
logger.error("Aborting memory audit save, store unreadable: %s", e)
|
||||
return {
|
||||
"before": before_count,
|
||||
"after": before_count,
|
||||
"error": "store_unreadable",
|
||||
}
|
||||
all_entries = memory_manager.load_all()
|
||||
audited_ids = {e["id"] for e in final_entries}
|
||||
other_entries = [e for e in all_entries if e.get("owner") != owner and (e.get("owner") is not None)]
|
||||
# Also keep legacy entries that weren't part of this audit
|
||||
|
||||
+1
-10
@@ -22,7 +22,6 @@ import time
|
||||
from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
|
||||
|
||||
from src.constants import GENERATED_IMAGES_DIR
|
||||
from src.memory import MemoryStoreUnreadable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -385,15 +384,7 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner
|
||||
return {"error": "Memory text cannot be empty"}
|
||||
|
||||
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
|
||||
# Strict load: this is a read-modify-write, and it is the path an
|
||||
# ordinary "remember that I prefer X" takes. Degrading to [] here would
|
||||
# save just this one entry over a store we only failed to read,
|
||||
# atomically destroying every memory in it (issue #5673).
|
||||
try:
|
||||
memories = _memory_manager.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
logger.error("Refusing to add memory, store unreadable: %s", e)
|
||||
return {"error": "Memory store is temporarily unreadable — nothing was saved."}
|
||||
memories = _memory_manager.load_all()
|
||||
memories.append(entry)
|
||||
_memory_manager.save(memories)
|
||||
|
||||
|
||||
+3
-172
@@ -1,14 +1,11 @@
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import logging
|
||||
import re
|
||||
from typing import Dict, List, Optional, Any
|
||||
from urllib.parse import urljoin, urlparse, urlunparse
|
||||
|
||||
import httpcore
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
|
||||
@@ -357,152 +354,6 @@ def _find_integration(identifier: str) -> Optional[Dict[str, Any]]:
|
||||
return None
|
||||
|
||||
|
||||
# httpcore raises its own exception hierarchy; map the ones a simple request can
|
||||
# surface back to their httpx equivalents so the caller's `except httpx.*` blocks
|
||||
# below behave exactly as they did with the default transport.
|
||||
_HTTPCORE_TO_HTTPX_EXC = {
|
||||
httpcore.ConnectError: httpx.ConnectError,
|
||||
httpcore.ConnectTimeout: httpx.ConnectTimeout,
|
||||
httpcore.NetworkError: httpx.NetworkError,
|
||||
httpcore.PoolTimeout: httpx.PoolTimeout,
|
||||
httpcore.ProtocolError: httpx.ProtocolError,
|
||||
httpcore.ReadError: httpx.ReadError,
|
||||
httpcore.ReadTimeout: httpx.ReadTimeout,
|
||||
httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
|
||||
httpcore.TimeoutException: httpx.TimeoutException,
|
||||
httpcore.WriteError: httpx.WriteError,
|
||||
httpcore.WriteTimeout: httpx.WriteTimeout,
|
||||
}
|
||||
|
||||
|
||||
class _PinnedAsyncBackend(httpcore.AsyncNetworkBackend):
|
||||
"""Network backend that connects only to the pre-validated IPs, in order.
|
||||
|
||||
Every address here came out of the single SSRF resolution, so moving to the
|
||||
next one after a connect failure is not re-resolution — it's ordinary
|
||||
multi-address fallback restricted to the set the guard already approved.
|
||||
httpcore takes TLS SNI and the ``Host`` header from the request URL rather
|
||||
than the connect host, so pinning the socket destination leaves certificate
|
||||
validation and vhost routing pointed at the original hostname.
|
||||
"""
|
||||
|
||||
def __init__(self, ips: List[ipaddress._BaseAddress]):
|
||||
self._ips = [str(ip) for ip in ips]
|
||||
self._real = httpcore.AnyIOBackend()
|
||||
|
||||
async def connect_tcp(self, host, port, timeout=None, local_address=None,
|
||||
socket_options=None):
|
||||
# One shared connect budget: each attempt gets the time left until the
|
||||
# original deadline, so N dead addresses can't stretch the connect
|
||||
# phase to N * timeout.
|
||||
deadline = None if timeout is None else time.monotonic() + timeout
|
||||
last_exc: Optional[Exception] = None
|
||||
for ip in self._ips:
|
||||
remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
|
||||
try:
|
||||
return await self._real.connect_tcp(
|
||||
ip, port, remaining, local_address, socket_options
|
||||
)
|
||||
except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc:
|
||||
last_exc = exc
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
break
|
||||
raise last_exc
|
||||
|
||||
async def connect_unix_socket(self, path, timeout=None, socket_options=None):
|
||||
return await self._real.connect_unix_socket(path, timeout, socket_options)
|
||||
|
||||
async def sleep(self, seconds: float) -> None:
|
||||
return await self._real.sleep(seconds)
|
||||
|
||||
|
||||
class _PinnedAsyncTransport(httpx.AsyncBaseTransport):
|
||||
"""httpx transport that pins the TCP connect to the pre-resolved IP(s).
|
||||
|
||||
Kept local, mirroring the per-module pinned transports web fetch and
|
||||
webhook delivery already carry, rather than coupling api_call to the
|
||||
webhook subsystem. The request URL passes through unchanged, so SNI and the
|
||||
``Host`` header stay the original hostname; only the socket destination is
|
||||
pinned, which is what closes the rebinding window.
|
||||
"""
|
||||
|
||||
def __init__(self, ips: List[ipaddress._BaseAddress]):
|
||||
self._pinned_ips = list(ips)
|
||||
self._pool = httpcore.AsyncConnectionPool(
|
||||
# Reuse the CA trust the default httpx client would build (certifi
|
||||
# plus SSL_CERT_FILE / SSL_CERT_DIR when trust_env is set) so
|
||||
# swapping in this transport doesn't quietly change which chains
|
||||
# verify. ssl.create_default_context() would use system roots.
|
||||
ssl_context=httpx.create_ssl_context(),
|
||||
http1=True,
|
||||
http2=False,
|
||||
network_backend=_PinnedAsyncBackend(ips),
|
||||
)
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
core_req = httpcore.Request(
|
||||
method=request.method,
|
||||
url=httpcore.URL(
|
||||
scheme=request.url.raw_scheme,
|
||||
host=request.url.raw_host,
|
||||
port=request.url.port,
|
||||
target=request.url.raw_path,
|
||||
),
|
||||
headers=request.headers.raw,
|
||||
content=request.stream,
|
||||
extensions=request.extensions,
|
||||
)
|
||||
try:
|
||||
core_resp = await self._pool.handle_async_request(core_req)
|
||||
content = b"".join([chunk async for chunk in core_resp.aiter_stream()])
|
||||
await core_resp.aclose()
|
||||
except Exception as exc:
|
||||
mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
|
||||
if mapped is not None:
|
||||
raise mapped(str(exc)) from exc
|
||||
raise
|
||||
return httpx.Response(
|
||||
status_code=core_resp.status,
|
||||
headers=core_resp.headers,
|
||||
content=content,
|
||||
extensions=core_resp.extensions,
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._pool.aclose()
|
||||
|
||||
|
||||
def _validated_ips(raw_ips: List[str]) -> List[ipaddress._BaseAddress]:
|
||||
"""Return every entry that parses as an IP address, de-duplicated, order
|
||||
preserved.
|
||||
|
||||
check_outbound_url only reports ok when *all* of these classify as safe, so
|
||||
the whole list is guard-approved and any of them is a legitimate connect
|
||||
target. Skipping unparseable entries mirrors how the guard walks the same
|
||||
resolver output.
|
||||
|
||||
De-duplication matters because the resolver is getaddrinfo(host, None) with
|
||||
no socktype filter, so glibc reports the same address once per socktype
|
||||
(SOCK_STREAM/SOCK_DGRAM/SOCK_RAW) — a single-homed host comes back three
|
||||
times. Without this, the connect fallback would spend the shared deadline
|
||||
retrying one dead address instead of moving on to a genuinely different one.
|
||||
"""
|
||||
ips: List[ipaddress._BaseAddress] = []
|
||||
seen = set()
|
||||
for raw in raw_ips:
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
try:
|
||||
ip = ipaddress.ip_address(raw.split("%")[0]) # strip IPv6 zone id
|
||||
except ValueError:
|
||||
continue
|
||||
if ip in seen:
|
||||
continue
|
||||
seen.add(ip)
|
||||
ips.append(ip)
|
||||
return ips
|
||||
|
||||
|
||||
async def execute_api_call(
|
||||
integration_id: str,
|
||||
method: str,
|
||||
@@ -558,31 +409,13 @@ async def execute_api_call(
|
||||
# loopback for locked-down deployments. Private stays allowed by default
|
||||
# because LAN integrations (Home Assistant, Miniflux, ntfy) are the
|
||||
# primary use case.
|
||||
from src.url_safety import check_outbound_url, _default_resolver
|
||||
from src.url_safety import check_outbound_url
|
||||
block_private = os.getenv(
|
||||
"INTEGRATION_API_BLOCK_PRIVATE_IPS", "false"
|
||||
).lower() == "true"
|
||||
# Resolve the host exactly once and remember the IPs the guard validated so
|
||||
# the request below can be pinned to them. check_outbound_url only reports
|
||||
# (ok, reason); a plain httpx client re-resolves the host at connect time,
|
||||
# which reopens a DNS-rebinding TOCTOU — a base_url host that answers with a
|
||||
# public IP for the guard and then flips to 169.254.169.254 for the connect
|
||||
# would reach cloud metadata with the integration's auth headers attached.
|
||||
resolved_ips: List[str] = []
|
||||
|
||||
def _recording_resolver(host: str) -> List[str]:
|
||||
ips = _default_resolver(host)
|
||||
resolved_ips[:] = ips
|
||||
return ips
|
||||
|
||||
ok, reason = check_outbound_url(
|
||||
url, block_private=block_private, resolver=_recording_resolver
|
||||
)
|
||||
ok, reason = check_outbound_url(url, block_private=block_private)
|
||||
if not ok:
|
||||
return {"error": f"URL rejected: {reason}", "exit_code": 1}
|
||||
pinned_ips = _validated_ips(resolved_ips)
|
||||
if not pinned_ips:
|
||||
return {"error": "URL rejected: host did not resolve to a usable address", "exit_code": 1}
|
||||
|
||||
method = method.upper()
|
||||
|
||||
@@ -622,9 +455,7 @@ async def execute_api_call(
|
||||
auth = httpx.BasicAuth(parts[0], parts[1])
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=30.0, transport=_PinnedAsyncTransport(pinned_ips)
|
||||
) as client:
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
response = await client.request(
|
||||
method,
|
||||
url,
|
||||
|
||||
+8
-78
@@ -10,18 +10,6 @@ from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryStoreUnreadable(RuntimeError):
|
||||
"""memory.json exists on disk but could not be read or parsed.
|
||||
|
||||
"The contents are unknown" is categorically different from "there are no
|
||||
memories". A read-modify-write caller that conflates the two appends to an
|
||||
empty view and then persists it, destroying the whole store — the writes
|
||||
are atomic, so the loss is durable. Raised by
|
||||
:meth:`MemoryManager.load_all_for_update` so those callers fail closed.
|
||||
"""
|
||||
|
||||
|
||||
def tokenize(text: str) -> List[str]:
|
||||
"""Simple tokenizer that splits on whitespace and removes punctuation."""
|
||||
return [word.strip('.,!?";') for word in text.split()]
|
||||
@@ -122,70 +110,22 @@ class MemoryManager:
|
||||
with open(self.memory_file, 'w', encoding='utf-8') as f:
|
||||
json.dump([], f, ensure_ascii=False, indent=2)
|
||||
|
||||
def _read_entries(self) -> List[Dict]:
|
||||
"""Parse the store, or raise :class:`MemoryStoreUnreadable`.
|
||||
|
||||
Returns ``[]`` only when the file genuinely does not exist. Every other
|
||||
failure mode raises, so callers can tell "no memories" apart from
|
||||
"couldn't read the memories".
|
||||
"""
|
||||
def load_all(self) -> List[Dict]:
|
||||
"""Load all memory entries from JSON file (unfiltered)."""
|
||||
if not os.path.exists(self.memory_file):
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(self.memory_file, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except OSError as e:
|
||||
# PermissionError is an OSError (a scanner holding the file, a
|
||||
# permissions problem, bad media).
|
||||
raise MemoryStoreUnreadable(
|
||||
f"cannot read {self.memory_file}: {e}"
|
||||
) from e
|
||||
except json.JSONDecodeError as e:
|
||||
# This is the branch that actually destroyed stores: the file reads
|
||||
# back fine, so nothing stops the save that follows. A truncated
|
||||
# memory.json is reachable because core/database.py rewrites it with
|
||||
# a plain open(..,"w") + json.dump during migration.
|
||||
#
|
||||
# Preserved behaviour: a corrupt store still gets one shot at the
|
||||
# pre-JSON memory.txt migration. Only raise when that finds nothing,
|
||||
# so we never report "empty" for a store we simply failed to parse.
|
||||
legacy = self._migrate_from_legacy()
|
||||
if legacy:
|
||||
return legacy
|
||||
raise MemoryStoreUnreadable(
|
||||
f"{self.memory_file} is not valid JSON: {e}"
|
||||
) from e
|
||||
|
||||
if not isinstance(data, list):
|
||||
raise MemoryStoreUnreadable(
|
||||
f"{self.memory_file} is not a JSON array (got {type(data).__name__})"
|
||||
)
|
||||
if isinstance(data, list):
|
||||
return self._validate_entries(data)
|
||||
|
||||
def load_all(self) -> List[Dict]:
|
||||
"""Load all memory entries from JSON file (unfiltered).
|
||||
|
||||
Lenient by design: this feeds display, search, and context-injection
|
||||
paths, so an unreadable store degrades to an empty list rather than
|
||||
breaking chat. Never build a value from this that you intend to save
|
||||
back — use :meth:`load_all_for_update` for that.
|
||||
"""
|
||||
try:
|
||||
return self._read_entries()
|
||||
except MemoryStoreUnreadable as e:
|
||||
except (json.JSONDecodeError, PermissionError) as e:
|
||||
logger.error("Error loading memory.json: %s", e)
|
||||
return self._migrate_from_legacy()
|
||||
|
||||
return []
|
||||
|
||||
def load_all_for_update(self) -> List[Dict]:
|
||||
"""Load for a read-modify-write cycle.
|
||||
|
||||
Propagates :class:`MemoryStoreUnreadable` instead of degrading to ``[]``
|
||||
so a caller can never append to an empty view and persist it over a
|
||||
store that was only temporarily unreadable (issue #5673).
|
||||
"""
|
||||
return self._read_entries()
|
||||
|
||||
def load(self, owner: str = None) -> List[Dict]:
|
||||
"""Load memory entries, optionally filtered by owner."""
|
||||
entries = self.load_all()
|
||||
@@ -195,12 +135,7 @@ class MemoryManager:
|
||||
|
||||
def claim_ownerless(self, owner: str):
|
||||
"""Assign all ownerless memory entries to the given owner."""
|
||||
try:
|
||||
entries = self.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
# Skip the sweep rather than rewrite the store from an unknown view.
|
||||
logger.error("Skipping ownerless claim, memory store unreadable: %s", e)
|
||||
return
|
||||
entries = self.load_all()
|
||||
changed = False
|
||||
claimed = 0
|
||||
for entry in entries:
|
||||
@@ -300,12 +235,7 @@ class MemoryManager:
|
||||
if not ids:
|
||||
return
|
||||
id_set = set(ids)
|
||||
try:
|
||||
entries = self.load_all_for_update()
|
||||
except MemoryStoreUnreadable as e:
|
||||
# Best-effort counter; never worth rewriting the store blind.
|
||||
logger.error("Skipping uses bump, memory store unreadable: %s", e)
|
||||
return
|
||||
entries = self.load_all()
|
||||
changed = False
|
||||
for e in entries:
|
||||
if e.get("id") in id_set:
|
||||
|
||||
@@ -157,11 +157,7 @@ class NativeMemoryProvider(MemoryProvider):
|
||||
if metadata:
|
||||
entry["metadata"] = dict(metadata)
|
||||
|
||||
# Strict load: read-modify-write. `load_all` degrades an unreadable
|
||||
# store to [], which would save this single entry over everything
|
||||
# already stored (issue #5673). The provider API has no error channel,
|
||||
# so MemoryStoreUnreadable propagates to the caller.
|
||||
memories = self.memory_manager.load_all_for_update()
|
||||
memories = self.memory_manager.load_all()
|
||||
memories.append(entry)
|
||||
self.memory_manager.save(memories)
|
||||
|
||||
@@ -227,10 +223,7 @@ class NativeMemoryProvider(MemoryProvider):
|
||||
]
|
||||
|
||||
async def delete(self, memory_id: str, *, owner: Optional[str] = None) -> bool:
|
||||
# Strict load for the same reason: `remaining` is derived from this
|
||||
# list and saved back, so it must never be built from a store we
|
||||
# failed to read.
|
||||
memories = self.memory_manager.load_all_for_update()
|
||||
memories = self.memory_manager.load_all()
|
||||
remaining = []
|
||||
deleted_id = None
|
||||
|
||||
|
||||
+1
-5
@@ -187,12 +187,8 @@ _FUNCTION_MODEL_NAME_RE = re.compile(
|
||||
_FUNCTION_MODEL_PARAMS_OPEN_RE = re.compile(r"<parameters>\s*", re.IGNORECASE)
|
||||
_FUNCTION_MODEL_PARAMS_CLOSE_RE = re.compile(r"</parameters>", re.IGNORECASE)
|
||||
_QWEN_ROLE_MARKER_RE = re.compile(r"</?\|(?:assistant|assistan|user|system|tool)\|>?|</\|end\|>?", re.IGNORECASE)
|
||||
# At least one pipe is required around `end`. Both pipes used to be optional
|
||||
# (`\|?end\|?`), which also matched a bare `end` on its own line and deleted it
|
||||
# from ordinary prose and from Ruby/Lua/shell snippets that close blocks with
|
||||
# one; see #5547. `|end`, `end|`, `|end|` and `/|end|` still strip as before.
|
||||
_QWEN_BARE_MARKER_RE = re.compile(
|
||||
r"(?:^|[\t\r\n ])(?:/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|"
|
||||
r"(?:^|[\t\r\n ])(?:\|?end\|?|/?\|end\|)(?=[\t\r\n ]|$)|"
|
||||
r"(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
+2
-4
@@ -46,9 +46,7 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict:
|
||||
except ValueError:
|
||||
return {"error": "Invalid JSON arguments", "exit_code": 1}
|
||||
|
||||
action = (args.get("action") or "").strip().lower()
|
||||
if not action:
|
||||
return {"error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)", "exit_code": 1}
|
||||
action = (args.get("action") or "").lower()
|
||||
from services.memory.skills import SkillsManager
|
||||
from services.memory.skill_format import Skill, slugify
|
||||
from src.constants import DATA_DIR
|
||||
@@ -57,7 +55,7 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict:
|
||||
# Accept legacy `skill_id` as an alias for `name`.
|
||||
name = (args.get("name") or args.get("skill_id") or "").strip()
|
||||
|
||||
if action in ("list", "index"):
|
||||
if action in ("list", "index", ""):
|
||||
all_skills = sm.load(owner=owner)
|
||||
if not all_skills:
|
||||
return {"results": "No skills yet. Create one with action='add'."}
|
||||
|
||||
+83
-16
@@ -10,14 +10,14 @@ import modelsModule from './js/models.js?v=20260715startupcalm2';
|
||||
import ragModule from './js/rag.js';
|
||||
import presetsModule from './js/presets.js';
|
||||
import searchModule from './js/search.js';
|
||||
import chatModule from './js/chat.js?v=20260801fix1';
|
||||
import chatModule from './js/chat.js?v=20260722ctxheader4';
|
||||
import compareModule from './js/compare/index.js?v=20260723compareicon2';
|
||||
import documentModule from './js/document.js?v=20260722emailfastindex1';
|
||||
import searchChatModule from './js/search-chat.js';
|
||||
import { makeWindowDraggable } from './js/windowDrag.js';
|
||||
import markdownModule from './js/markdown.js';
|
||||
import chatRenderer from './js/chatRenderer.js?v=20260722emailfastindex1';
|
||||
import sessionModule from './js/sessions.js';
|
||||
import sessionModule from './js/sessions.js?v=20260722ctxheader4';
|
||||
import memoryModule from './js/memory.js?v=20260722memoryloading1';
|
||||
import voiceRecorderModule from './js/voiceRecorder.js';
|
||||
import censorModule from './js/censor.js';
|
||||
@@ -1689,20 +1689,12 @@ function initializeEventListeners() {
|
||||
|
||||
const newMemoryInput = el('new-memory-input');
|
||||
if (newMemoryInput) {
|
||||
// keydown, not the deprecated keypress: keypress is not guaranteed to
|
||||
// fire for Enter everywhere, which left the Add Memory form with no
|
||||
// working submit path (#5828).
|
||||
newMemoryInput.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Enter' && !e.isComposing) {
|
||||
e.preventDefault();
|
||||
newMemoryInput.addEventListener('keypress', (e) => {
|
||||
if (e.key === 'Enter') {
|
||||
memoryModule.addNewMemory();
|
||||
}
|
||||
});
|
||||
}
|
||||
const newMemoryAddBtn = el('new-memory-add-btn');
|
||||
if (newMemoryAddBtn) {
|
||||
newMemoryAddBtn.addEventListener('click', () => memoryModule.addNewMemory());
|
||||
}
|
||||
|
||||
// Voice recording is handled by the dual-purpose send/mic button (see below)
|
||||
|
||||
@@ -3916,10 +3908,85 @@ function startOdysseusApp() {
|
||||
const messageInput = el('message');
|
||||
const modelPickerWrap = document.getElementById('model-picker-wrap');
|
||||
|
||||
// ArrowUp/ArrowDown prompt recall on #message lives in
|
||||
// static/js/composerArrowUpRecall.js (wired from chat.js). Do not re-add a
|
||||
// copy here: two capture-phase listeners on the same textarea meant the one
|
||||
// without the draft guard won and ate unsent multi-line prompts (#5862).
|
||||
function _readComposerPromptHistory() {
|
||||
const chatBox = document.getElementById('chat-history');
|
||||
if (!chatBox) return [];
|
||||
return Array.from(chatBox.querySelectorAll('.msg-user'))
|
||||
.reverse()
|
||||
.map(msg => {
|
||||
const body = msg.querySelector('.body');
|
||||
return msg.dataset?.raw || (body ? body.textContent : '') || '';
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
if (messageInput && !messageInput._odysseusPromptRecallCapture) {
|
||||
messageInput._odysseusPromptRecallCapture = true;
|
||||
let recallHistory = [];
|
||||
let recallIndex = -1;
|
||||
let lastRecalled = '';
|
||||
const norm = (v) => String(v || '').replace(/\r\n/g, '\n').trimEnd();
|
||||
messageInput.addEventListener('input', () => {
|
||||
if (norm(messageInput.value) === norm(lastRecalled)) return;
|
||||
recallHistory = [];
|
||||
recallIndex = -1;
|
||||
lastRecalled = '';
|
||||
try { delete messageInput.dataset.odysseusRecallIndex; } catch {}
|
||||
}, true);
|
||||
messageInput.addEventListener('keydown', (e) => {
|
||||
if (e.key !== 'ArrowUp' && e.key !== 'ArrowDown') return;
|
||||
if (e.shiftKey || e.altKey || e.ctrlKey || e.metaKey || e.isComposing) return;
|
||||
if (window._ghostAutocomplete?.isActive?.()) return;
|
||||
const fresh = _readComposerPromptHistory();
|
||||
const history = fresh.length ? fresh : recallHistory;
|
||||
if (!history.length) return;
|
||||
const current = norm(messageInput.value);
|
||||
let currentIndex = current ? history.findIndex(item => norm(item) === current) : -1;
|
||||
if (current && currentIndex < 0 && current === norm(lastRecalled)) currentIndex = recallIndex;
|
||||
if (current && currentIndex < 0) {
|
||||
const markedIndex = Number(messageInput.dataset.odysseusRecallIndex);
|
||||
if (Number.isInteger(markedIndex) && markedIndex >= 0 && markedIndex < history.length) {
|
||||
currentIndex = markedIndex;
|
||||
}
|
||||
}
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
e.stopImmediatePropagation();
|
||||
if (e.key === 'ArrowDown') {
|
||||
if (currentIndex < 0) return;
|
||||
const nextIndex = currentIndex - 1;
|
||||
if (nextIndex < 0) {
|
||||
recallHistory = history;
|
||||
recallIndex = -1;
|
||||
lastRecalled = '';
|
||||
try { delete messageInput.dataset.odysseusRecallIndex; } catch {}
|
||||
messageInput.value = '';
|
||||
try { messageInput.selectionStart = messageInput.selectionEnd = 0; } catch {}
|
||||
try { uiModule.autoResize(messageInput); } catch {}
|
||||
return;
|
||||
}
|
||||
const recalled = history[nextIndex];
|
||||
recallHistory = history;
|
||||
recallIndex = nextIndex;
|
||||
lastRecalled = recalled;
|
||||
try { messageInput.dataset.odysseusRecallIndex = String(nextIndex); } catch {}
|
||||
messageInput.value = recalled;
|
||||
try { messageInput.selectionStart = messageInput.selectionEnd = recalled.length; } catch {}
|
||||
try { uiModule.autoResize(messageInput); } catch {}
|
||||
return;
|
||||
}
|
||||
const nextIndex = currentIndex >= 0 ? Math.min(currentIndex + 1, history.length - 1) : 0;
|
||||
const recalled = history[nextIndex];
|
||||
if (!recalled) return;
|
||||
recallHistory = history;
|
||||
recallIndex = nextIndex;
|
||||
lastRecalled = recalled;
|
||||
try { messageInput.dataset.odysseusRecallIndex = String(nextIndex); } catch {}
|
||||
messageInput.value = recalled;
|
||||
try { messageInput.selectionStart = messageInput.selectionEnd = recalled.length; } catch {}
|
||||
try { uiModule.autoResize(messageInput); } catch {}
|
||||
}, true);
|
||||
}
|
||||
|
||||
const _sendIcon = '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><path d="M12 19V5M5 12l7-7 7 7"/></svg>';
|
||||
const _micIcon = '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"/><path d="M19 10v2a7 7 0 0 1-14 0v-2"/><line x1="12" y1="19" x2="12" y2="23"/><line x1="8" y1="23" x2="16" y2="23"/></svg>';
|
||||
|
||||
+5
-6
@@ -250,9 +250,9 @@
|
||||
</script>
|
||||
<link rel="stylesheet" href="/static/style.css?v=20260723tasksbulkfeedback1">
|
||||
<link rel="modulepreload" href="/static/app.js?v=20260723tasksbulkfeedback1">
|
||||
<link rel="modulepreload" href="/static/js/chat.js?v=20260801fix1">
|
||||
<link rel="modulepreload" href="/static/js/chat.js?v=20260722ctxheader4">
|
||||
<link rel="modulepreload" href="/static/js/ui.js">
|
||||
<link rel="modulepreload" href="/static/js/sessions.js">
|
||||
<link rel="modulepreload" href="/static/js/sessions.js?v=20260722ctxheader4">
|
||||
<link rel="modulepreload" href="/static/js/markdown.js">
|
||||
</head>
|
||||
<body>
|
||||
@@ -365,7 +365,6 @@
|
||||
<span class="skill-rich-ph"><span class="k">Add a memory</span> — e.g. 'I prefer concise replies' <svg class="k" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-left:4px;" aria-hidden="true"><polyline points="9 10 4 15 9 20"/><path d="M20 4v7a4 4 0 0 1-4 4H4"/></svg></span>
|
||||
</div>
|
||||
<select id="new-memory-category" class="memory-edit-cat-select" aria-label="Memory category"></select>
|
||||
<button type="button" id="new-memory-add-btn" class="theme-io-btn" title="Save this memory" style="flex:none;height:28px;font-size:12px;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:4px;" aria-hidden="true"><line x1="12" y1="5" x2="12" y2="19"/><line x1="5" y1="12" x2="19" y2="12"/></svg>Add</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="admin-card">
|
||||
@@ -1006,7 +1005,7 @@
|
||||
var tips = mobile ? phone : desktop;
|
||||
var el = document.getElementById('welcome-tip');
|
||||
if (el) {
|
||||
el.textContent = tips[Math.floor(Math.random() * tips.length)];
|
||||
el.textContent = 'Pick a model if you want, or just type.';
|
||||
}
|
||||
fetch('/api/version').then(function(r){return r.json()}).then(function(d){
|
||||
if (d.version) window._appVersion = d.version;
|
||||
@@ -2505,7 +2504,7 @@
|
||||
<script type="module" src="/static/js/ui.js"></script>
|
||||
<script type="module" src="/static/js/markdown.js"></script>
|
||||
<script type="module" src="/static/js/dragSort.js"></script>
|
||||
<script type="module" src="/static/js/sessions.js"></script>
|
||||
<script type="module" src="/static/js/sessions.js?v=20260722ctxheader4"></script>
|
||||
<script type="module" src="/static/js/memory.js?v=20260722memoryloading1"></script>
|
||||
<script type="module" src="/static/js/skills.js"></script>
|
||||
<script type="module" src="/static/js/tourHints.js"></script>
|
||||
@@ -2523,7 +2522,7 @@
|
||||
<script type="module" src="/static/js/chatRenderer.js?v=20260722emailfastindex1"></script>
|
||||
<script type="module" src="/static/js/codeRunner.js"></script>
|
||||
<script type="module" src="/static/js/chatStream.js?v=20260722emailfastindex1"></script>
|
||||
<script type="module" src="/static/js/chat.js?v=20260801fix1"></script>
|
||||
<script type="module" src="/static/js/chat.js?v=20260722ctxheader4"></script>
|
||||
<script type="module" src="/static/js/cookbook.js"></script>
|
||||
<script src="/static/js/cookbookSchedule.js"></script>
|
||||
<script type="module" src="/static/js/search-chat.js"></script>
|
||||
|
||||
+3
-9
@@ -349,9 +349,6 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr
|
||||
|
||||
async function _adoptOpenedSessionBeforeAutoCreate() {
|
||||
if (!sessionModule || !sessionModule.getCurrentSessionId || sessionModule.getCurrentSessionId()) return true;
|
||||
// Don't adopt a stale session when the user explicitly started a New Chat
|
||||
// (pending state set) — the send path must materialize the pending session.
|
||||
if (sessionModule.hasPendingChat && sessionModule.hasPendingChat()) return false;
|
||||
const activeRowId = document.querySelector('.list-item.active-session[data-session-id], .session-item.active[data-session-id]')?.dataset?.sessionId || '';
|
||||
const hashId = _hashSessionCandidate();
|
||||
const lastSelectedId = String(window.__odysseusLastSelectedSessionId || '').trim();
|
||||
@@ -1406,8 +1403,6 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr
|
||||
currentAccumulated = '';
|
||||
currentHolder = null;
|
||||
|
||||
let abortCtrl = null;
|
||||
let streamingTTS = false;
|
||||
try {
|
||||
// Re-enable auto-scroll when user sends a message
|
||||
uiModule.setAutoScroll(true);
|
||||
@@ -1721,7 +1716,7 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr
|
||||
}
|
||||
|
||||
|
||||
abortCtrl = new AbortController();
|
||||
const abortCtrl = new AbortController();
|
||||
abortCtrl._reason = '';
|
||||
currentAbort = abortCtrl;
|
||||
|
||||
@@ -1902,7 +1897,7 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr
|
||||
let isThinking = false;
|
||||
let thinkingStartTime = null;
|
||||
// Streaming TTS: synthesize sentence-by-sentence during streaming
|
||||
streamingTTS = !!(window.aiTTSManager && window.aiTTSManager.autoPlay && window.aiTTSManager.available);
|
||||
const streamingTTS = !!(window.aiTTSManager && window.aiTTSManager.autoPlay && window.aiTTSManager.available);
|
||||
if (streamingTTS) window.aiTTSManager.streamingStart();
|
||||
// Multi-bubble agent tracking
|
||||
let roundHolder = holder; // Current AI text bubble (changes per round)
|
||||
@@ -4792,8 +4787,7 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr
|
||||
if (msgIndex < 0) return;
|
||||
|
||||
const bodyEl = userMsgElement.querySelector('.body');
|
||||
let currentText = (userMsgElement.dataset.raw || (bodyEl ? bodyEl.textContent : '') || '').trim();
|
||||
currentText = currentText.replace(/\s*\[\d+ attachment\(s\)\]$/, '');
|
||||
const currentText = bodyEl ? bodyEl.textContent.trim().replace(/\s*\[\d+ attachment\(s\)\]$/, '') : '';
|
||||
|
||||
// Replace body with an editable textarea
|
||||
const editor = document.createElement('textarea');
|
||||
|
||||
@@ -478,10 +478,7 @@ const DSML_STRAY_RE = /<\s*\/?\s*[｜|]+\s*DSML\s*[｜|]+[^>]*>/gi;
|
||||
const DSML_INVOKE_RE = /<\s*[｜|]+\s*DSML\s*[｜|]+\s*invoke\b[^>]*>[\s\S]*?(?:<\s*\/\s*[｜|]+\s*DSML\s*[｜|]+\s*invoke\s*>|$)/gi;
|
||||
const RAW_OPENAI_TOOL_JSON_RE = /(?:\[\s*)?\{\s*"function"\s*:\s*\{[\s\S]*?\}\s*,\s*"id"\s*:\s*"[^"]*"\s*,\s*"type"\s*:\s*"function"\s*\}\s*\]?/gi;
|
||||
const QWEN_ROLE_MARKER_RE = /<\/?\|(?:assistant|assistan|user|system|tool)\|>?|<\/\|end\|>?/gi;
|
||||
// Keep in sync with _QWEN_BARE_MARKER_RE in src/tool_parsing.py. At least one
|
||||
// pipe is required around `end`: with both optional (`\|?end\|?`) this also ate
|
||||
// a bare `end` on its own line, breaking Ruby/Lua/shell snippets (#5547).
|
||||
const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi;
|
||||
const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\|?end\|?|\/?\|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi;
|
||||
// Self-narration about tool results (model echoing stdout/exit_code)
|
||||
const TOOL_NARRATION_RE = /(?:The (?:result|output) shows?:?\s*)?-?\s*(?:stdout|stderr|exit_code):\s*.+/gi;
|
||||
|
||||
|
||||
@@ -143,9 +143,9 @@ export function wireArrowUpRecall(composer, getUserMessages, options = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
// ArrowUp walks older prompts. An unmatched draft already returned above,
|
||||
// so reaching here means the composer is empty or holds a recalled prompt
|
||||
// — the caret-navigation case is never hijacked.
|
||||
// ArrowUp owns prompt history in the chat composer. If the current text
|
||||
// is not already a recalled prompt, start from newest instead of letting
|
||||
// the browser move the caret inside the textarea.
|
||||
const nextIndex = currentIndex >= 0 ? Math.min(currentIndex + 1, history.length - 1) : 0;
|
||||
const recalled = history[nextIndex];
|
||||
if (!recalled) {
|
||||
|
||||
@@ -1847,10 +1847,6 @@ export async function selectSession(id, { keepSidebar = false, showLoading = tru
|
||||
const _isTransientChat = !!_meta && (_meta.folder === 'Assistant' || _meta.folder === 'Tasks');
|
||||
if (!_isTransientChat) {
|
||||
Storage.set('lastSessionId', id);
|
||||
// Update URL hash without triggering hashchange handler
|
||||
if (window.location.hash !== '#' + id) {
|
||||
history.replaceState(null, '', '#' + id);
|
||||
}
|
||||
}
|
||||
// Restore character preset for persistent chats
|
||||
try {
|
||||
@@ -2317,7 +2313,6 @@ export async function materializePendingSession() {
|
||||
currentSessionId = payload.id;
|
||||
if (!isIncognito) {
|
||||
Storage.set('lastSessionId', payload.id);
|
||||
history.replaceState(null, '', '#' + payload.id);
|
||||
}
|
||||
|
||||
// Reload the sidebar in the background. Awaiting this used to block the first
|
||||
|
||||
+5
-3
@@ -83,9 +83,11 @@ export async function loadSkills(cascade = false) {
|
||||
// Play the domino-in entrance on this load (set when the tab is opened,
|
||||
// not for the silent re-loads after an edit/delete).
|
||||
if (cascade) _cascadeNext = true;
|
||||
// Always re-fetch when the tab is explicitly opened — the cascade
|
||||
// animation is handled inside renderSkillsList() via _cascadeNext.
|
||||
// Skipping the fetch here caused stale data on panel close/reopen (#5870).
|
||||
if (cascade && loaded && !_loadPromise && _playSkillsCascade()) {
|
||||
_cascadeNext = false;
|
||||
updateCount();
|
||||
return;
|
||||
}
|
||||
if (_loadPromise) return _loadPromise;
|
||||
_loadPromise = (async () => {
|
||||
try {
|
||||
|
||||
@@ -76,7 +76,7 @@ def _load_webhook_routes_for_test(monkeypatch):
|
||||
module_name = "routes.webhook_routes_under_test"
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name,
|
||||
Path(__file__).resolve().parent.parent / "routes" / "webhook" / "webhook_routes.py",
|
||||
Path(__file__).resolve().parent.parent / "routes" / "webhook_routes.py",
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
@@ -27,9 +27,6 @@ def _setup(monkeypatch, store, user="alice"):
|
||||
|
||||
mem = MagicMock()
|
||||
mem.load_all.return_value = list(store)
|
||||
# import_data reads through the strict loader so a store it cannot read is
|
||||
# never overwritten (#5673); the double has to offer the same entry point.
|
||||
mem.load_all_for_update.return_value = list(store)
|
||||
saved = {}
|
||||
mem.save.side_effect = lambda entries: saved.__setitem__("entries", entries)
|
||||
|
||||
|
||||
@@ -306,24 +306,3 @@ def test_integration_recalls_from_chat_history_dom():
|
||||
)
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
assert json.loads(proc.stdout.strip()) == {"value": "stored prompt", "prevented": True}
|
||||
|
||||
|
||||
def test_prompt_recall_is_not_duplicated_in_app_js():
|
||||
"""Only composerArrowUpRecall.js may own ArrowUp on #message (issue #5862).
|
||||
|
||||
static/app.js once carried a near-verbatim copy of this recall logic, wired
|
||||
as a second capture-phase listener on the same textarea. That copy lacked
|
||||
the draft guard here, and because it called stopImmediatePropagation it won
|
||||
regardless of registration order — so a typed multi-line prompt was replaced
|
||||
by the last sent one instead of the caret moving up a line.
|
||||
"""
|
||||
app_js = (_REPO / "static" / "app.js").read_text(encoding="utf-8")
|
||||
for marker in (
|
||||
"_odysseusPromptRecallCapture",
|
||||
"_readComposerPromptHistory",
|
||||
"odysseusRecallIndex",
|
||||
):
|
||||
assert marker not in app_js, (
|
||||
f"static/app.js reintroduces prompt recall ({marker!r}); "
|
||||
"it belongs to static/js/composerArrowUpRecall.js alone"
|
||||
)
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
"""Regression test for the document route shim (slice 2m, #4082/#4071).
|
||||
|
||||
The backward-compat shims at ``routes/document_routes.py`` and
|
||||
``routes/document_helpers.py`` use ``sys.modules`` replacement so the legacy
|
||||
import paths and the canonical ``routes.document.*`` paths resolve to the
|
||||
*same* module objects. This is required because multiple tests do
|
||||
``import routes.document_routes as droutes`` followed by
|
||||
``droutes.SessionLocal = ...`` / ``monkeypatch.setattr(droutes, ...)`` and
|
||||
``sys.modules.pop("routes.document_helpers")`` + re-import — for those to
|
||||
take effect at runtime, the legacy and canonical module objects must be
|
||||
identical.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import routes.document_routes as _shim_routes # noqa: F401
|
||||
import routes.document_helpers as _shim_helpers # noqa: F401
|
||||
|
||||
|
||||
def test_legacy_and_canonical_routes_are_same_object():
|
||||
legacy = importlib.import_module("routes.document_routes")
|
||||
canonical = importlib.import_module("routes.document.document_routes")
|
||||
assert legacy is canonical
|
||||
|
||||
|
||||
def test_legacy_and_canonical_helpers_are_same_object():
|
||||
legacy = importlib.import_module("routes.document_helpers")
|
||||
canonical = importlib.import_module("routes.document.document_helpers")
|
||||
assert legacy is canonical
|
||||
@@ -87,7 +87,7 @@ def test_known_imap_mailbox_call_sites_are_quoted():
|
||||
assert "conn.select(sent_name" not in pollers
|
||||
assert "imap.append(sent_folder" not in pollers
|
||||
|
||||
document_routes = Path("routes/document/document_routes.py").read_text()
|
||||
document_routes = Path("routes/document_routes.py").read_text()
|
||||
assert "conn.select(doc.source_email_folder" not in document_routes
|
||||
|
||||
|
||||
|
||||
@@ -9,13 +9,8 @@ link-local/metadata is always rejected; RFC-1918/loopback only when
|
||||
INTEGRATION_API_BLOCK_PRIVATE_IPS=true (LAN integrations are the primary
|
||||
use case, so private stays allowed by default).
|
||||
"""
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import ssl
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpcore
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from src import integrations
|
||||
@@ -102,238 +97,3 @@ async def test_private_base_url_allowed_by_default_blocked_with_knob(monkeypatch
|
||||
assert result["exit_code"] == 1
|
||||
assert "rejected" in result["error"].lower()
|
||||
client.request.assert_not_called()
|
||||
|
||||
|
||||
async def _call_capturing_transport(base_url, path="/items"):
|
||||
"""Drive execute_api_call and return (result, transport) where transport is
|
||||
the object passed to httpx.AsyncClient(transport=...)."""
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.headers = {"content-type": "application/json"}
|
||||
resp.json.return_value = {"ok": True}
|
||||
resp.text = '{"ok": true}'
|
||||
|
||||
client = AsyncMock()
|
||||
client.__aenter__ = AsyncMock(return_value=client)
|
||||
client.__aexit__ = AsyncMock(return_value=None)
|
||||
client.request = AsyncMock(return_value=resp)
|
||||
|
||||
captured = {}
|
||||
|
||||
def _fake_async_client(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return client
|
||||
|
||||
with (
|
||||
patch.object(integrations, "_find_integration",
|
||||
return_value=_integration(base_url)),
|
||||
patch("httpx.AsyncClient", side_effect=_fake_async_client),
|
||||
):
|
||||
result = await integrations.execute_api_call("test_integ", "GET", path)
|
||||
return result, captured.get("transport"), client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connection_is_pinned_to_the_validated_ip(monkeypatch):
|
||||
"""DNS-rebinding defense: the guard resolves the host once to a benign
|
||||
public IP, and the request must be pinned to *that* IP so a host that
|
||||
rebinds to the metadata range at connect time can't be reached with the
|
||||
integration's auth headers. Static resolution passing the guard is not
|
||||
enough — a plain client would re-resolve at connect."""
|
||||
monkeypatch.setattr("src.url_safety._default_resolver",
|
||||
lambda host: ["93.184.216.34"])
|
||||
result, transport, client = await _call_capturing_transport(
|
||||
"http://rebinding.attacker.example")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
client.request.assert_called_once()
|
||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pin_carries_the_whole_validated_ip_set(monkeypatch):
|
||||
"""When a host resolves to several records the transport keeps all of them
|
||||
(check_outbound_url validated every one), in resolver order, so it can fall
|
||||
back past a dead first address instead of failing the whole call."""
|
||||
monkeypatch.setattr("src.url_safety._default_resolver",
|
||||
lambda host: ["93.184.216.34", "198.51.100.7"])
|
||||
result, transport, _ = await _call_capturing_transport("http://multi.example")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34", "198.51.100.7"]
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
"""Stand-in for the connected socket the real backend returns."""
|
||||
|
||||
|
||||
class _RecordingBackend:
|
||||
"""Fake httpcore backend: connect_tcp fails for the addresses in `dead`
|
||||
and succeeds for the rest, recording the order it was asked to connect."""
|
||||
|
||||
def __init__(self, dead):
|
||||
self.dead = set(dead)
|
||||
self.attempts = []
|
||||
|
||||
async def connect_tcp(self, host, port, timeout=None, local_address=None,
|
||||
socket_options=None):
|
||||
self.attempts.append((host, timeout))
|
||||
if host in self.dead:
|
||||
raise httpcore.ConnectError(f"connection refused: {host}")
|
||||
return _FakeStream()
|
||||
|
||||
|
||||
def _pinned_backend(ips, dead):
|
||||
"""A _PinnedAsyncBackend whose underlying connect is the recording fake."""
|
||||
backend = integrations._PinnedAsyncBackend(ips)
|
||||
backend._real = _RecordingBackend(dead)
|
||||
return backend
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_falls_back_from_dead_first_to_live_second():
|
||||
"""first-dead / second-live: the pinned backend must try the next validated
|
||||
address when the first refuses, rather than surfacing the failure. It also
|
||||
ignores the `host` httpcore passes (the original hostname) and connects to
|
||||
the pinned IPs, which is what keeps TLS SNI / Host on the real hostname."""
|
||||
ips = [ipaddress.ip_address("203.0.113.10"), ipaddress.ip_address("198.51.100.7")]
|
||||
backend = _pinned_backend(ips, dead={"203.0.113.10"})
|
||||
|
||||
stream = await backend.connect_tcp("original.hostname.example", 443, timeout=5.0)
|
||||
|
||||
assert isinstance(stream, _FakeStream)
|
||||
# Tried the dead address first, then the live one — never the hostname.
|
||||
assert [host for host, _ in backend._real.attempts] == ["203.0.113.10", "198.51.100.7"]
|
||||
# Fallback shared one budget: the second attempt got the time left, not a fresh 5s.
|
||||
assert backend._real.attempts[1][1] <= 5.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_raises_when_every_validated_address_is_dead():
|
||||
ips = [ipaddress.ip_address("203.0.113.10"), ipaddress.ip_address("198.51.100.7")]
|
||||
backend = _pinned_backend(ips, dead={"203.0.113.10", "198.51.100.7"})
|
||||
|
||||
with pytest.raises(httpcore.ConnectError):
|
||||
await backend.connect_tcp("original.hostname.example", 443, timeout=5.0)
|
||||
assert [host for host, _ in backend._real.attempts] == ["203.0.113.10", "198.51.100.7"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pinned_transport_reuses_httpx_ca_trust(monkeypatch):
|
||||
"""TLS trust must come from the same builder the default httpx client uses
|
||||
(certifi + SSL_CERT_FILE / SSL_CERT_DIR via trust_env), not from
|
||||
ssl.create_default_context()'s system roots — otherwise chains that verified
|
||||
under the old default client can silently stop verifying."""
|
||||
sentinel = ssl.create_default_context()
|
||||
calls = []
|
||||
|
||||
def _fake_create(*args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr(httpx, "create_ssl_context", _fake_create)
|
||||
transport = integrations._PinnedAsyncTransport([ipaddress.ip_address("93.184.216.34")])
|
||||
try:
|
||||
assert calls, "transport did not build its context via httpx.create_ssl_context"
|
||||
assert transport._pool._ssl_context is sentinel
|
||||
finally:
|
||||
await transport.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_socket_falls_back_from_dead_first_to_live_second():
|
||||
"""End-to-end over real loopback sockets: pin [127.0.0.2 (nothing
|
||||
listening), 127.0.0.1 (live)], and the request must succeed by falling back
|
||||
to the second address while the Host header stays the original hostname —
|
||||
i.e. only the socket destination moved, vhost/SNI routing did not."""
|
||||
captured = {}
|
||||
|
||||
async def handle(reader, writer):
|
||||
request = await reader.read(4096)
|
||||
for line in request.split(b"\r\n"):
|
||||
if line.lower().startswith(b"host:"):
|
||||
captured["host"] = line.split(b":", 1)[1].strip().decode()
|
||||
writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nhi")
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
|
||||
server = await asyncio.start_server(handle, "127.0.0.1", 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
async with server:
|
||||
await server.start_serving()
|
||||
transport = integrations._PinnedAsyncTransport(
|
||||
[ipaddress.ip_address("127.0.0.2"), ipaddress.ip_address("127.0.0.1")]
|
||||
)
|
||||
try:
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
resp = await client.get(f"http://pinned.example:{port}/health")
|
||||
finally:
|
||||
await transport.aclose()
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.text == "hi"
|
||||
assert captured.get("host") == f"pinned.example:{port}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ip_literal_base_url_still_pins_and_is_not_rejected():
|
||||
"""A base_url that is already an IP has nothing to rebind, but it must not
|
||||
trip the "did not resolve" guard either.
|
||||
|
||||
check_outbound_url resolves even a literal (getaddrinfo returns the address
|
||||
itself), so the captured list is populated and the pin is a no-op rather
|
||||
than a rejection. Uses the real resolver on purpose — no monkeypatch — so
|
||||
this would catch the fail-closed branch firing on a literal.
|
||||
"""
|
||||
result, transport, client = await _call_capturing_transport(
|
||||
"http://93.184.216.34")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ipv6_base_url_pins_every_validated_address(monkeypatch):
|
||||
"""IPv6 goes down the same path as v4.
|
||||
|
||||
Resolution is stubbed rather than using a literal so this doesn't depend on
|
||||
the runner having IPv6 configured.
|
||||
"""
|
||||
v6 = "2606:2800:220:1:248:1893:25c8:1946"
|
||||
monkeypatch.setattr("src.url_safety._default_resolver", lambda host: [v6])
|
||||
result, transport, client = await _call_capturing_transport("http://v6.example")
|
||||
|
||||
assert result.get("exit_code") == 0
|
||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
||||
assert [str(ip) for ip in transport._pinned_ips] == [v6]
|
||||
|
||||
|
||||
def test_validated_ips_strips_zone_id_and_drops_junk():
|
||||
"""getaddrinfo can hand back a scoped v6 address like 'fe80::1%eth0'."""
|
||||
got = integrations._validated_ips(
|
||||
["93.184.216.34", "fe80::1%eth0", "not-an-ip", None, "2001:db8::5"]
|
||||
)
|
||||
assert [str(ip) for ip in got] == ["93.184.216.34", "fe80::1", "2001:db8::5"]
|
||||
|
||||
|
||||
def test_validated_ips_deduplicates_repeated_addresses():
|
||||
"""The resolver is getaddrinfo(host, None) with no socktype filter, so glibc
|
||||
returns one record per socktype and a single-homed host arrives three times
|
||||
over. Duplicates must collapse (first-seen order kept) or the connect
|
||||
fallback wastes its shared deadline retrying one dead address."""
|
||||
got = integrations._validated_ips(
|
||||
["93.184.216.34", "93.184.216.34", "93.184.216.34"]
|
||||
)
|
||||
assert [str(ip) for ip in got] == ["93.184.216.34"]
|
||||
|
||||
# Order is first-seen, and distinct addresses all survive.
|
||||
got = integrations._validated_ips(
|
||||
["198.51.100.7", "93.184.216.34", "198.51.100.7", "2001:db8::5"]
|
||||
)
|
||||
assert [str(ip) for ip in got] == ["198.51.100.7", "93.184.216.34", "2001:db8::5"]
|
||||
|
||||
# A zone-id variant is the same address once stripped, so it collapses too.
|
||||
got = integrations._validated_ips(["fe80::1%eth0", "fe80::1%eth1", "fe80::1"])
|
||||
assert [str(ip) for ip in got] == ["fe80::1"]
|
||||
|
||||
@@ -83,10 +83,9 @@ async def _call(json_data, status=200):
|
||||
with (
|
||||
patch.object(integrations, "_find_integration", return_value=DUMMY_INTEGRATION),
|
||||
patch("httpx.AsyncClient", return_value=mock_client),
|
||||
# api.example.com doesn't resolve. Point the resolver at a public
|
||||
# address instead of stubbing the guard open, so the real check (and
|
||||
# the connect-IP pinning that reads its result) still runs.
|
||||
patch("src.url_safety._default_resolver", lambda host: ["93.184.216.34"]),
|
||||
# api.example.com doesn't resolve; the SSRF guard would fail closed.
|
||||
# These tests are about truncation, so stub the guard open.
|
||||
patch("src.url_safety.check_outbound_url", return_value=(True, "ok")),
|
||||
):
|
||||
return await integrations.execute_api_call("test_integ", "GET", "/items")
|
||||
|
||||
@@ -102,10 +101,9 @@ async def _call_with_integration(integration, path="/items"):
|
||||
with (
|
||||
patch.object(integrations, "_find_integration", return_value=integration),
|
||||
patch("httpx.AsyncClient", return_value=mock_client),
|
||||
# api.example.com doesn't resolve. Point the resolver at a public
|
||||
# address instead of stubbing the guard open, so the real check (and
|
||||
# the connect-IP pinning that reads its result) still runs.
|
||||
patch("src.url_safety._default_resolver", lambda host: ["93.184.216.34"]),
|
||||
# api.example.com doesn't resolve; the SSRF guard would fail closed.
|
||||
# These tests are about URL joining, so stub the guard open.
|
||||
patch("src.url_safety.check_outbound_url", return_value=(True, "ok")),
|
||||
):
|
||||
result = await integrations.execute_api_call("test_integ", "GET", path)
|
||||
return result, mock_client
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from src.tools.system import do_manage_skills
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
{},
|
||||
{"action": ""},
|
||||
{"action": " "},
|
||||
{"name": "demo", "description": "x", "procedure": ["step"]},
|
||||
],
|
||||
)
|
||||
async def test_manage_skills_requires_action(payload):
|
||||
result = await do_manage_skills(json.dumps(payload), owner="test")
|
||||
|
||||
assert result == {
|
||||
"error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)",
|
||||
"exit_code": 1,
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
"""The Brain > Add Memory form must be submittable (#5828).
|
||||
|
||||
The form previously had no submit button and relied on a deprecated
|
||||
``keypress`` listener for Enter, which is not guaranteed to fire on all
|
||||
platforms — leaving the form with no working submit path. Pins:
|
||||
|
||||
- a visible, keyboard-accessible submit button next to the category select;
|
||||
- the button wired to ``memoryModule.addNewMemory()``;
|
||||
- Enter handled via ``keydown`` with ``preventDefault()`` (and no lingering
|
||||
``keypress`` handler on the input).
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
APP_JS = Path("static/app.js")
|
||||
INDEX_HTML = Path("static/index.html")
|
||||
|
||||
|
||||
def _add_memory_row(html):
|
||||
start = html.index('id="new-memory-input"')
|
||||
end = html.index("</div>", html.index('id="new-memory-add-btn"', start))
|
||||
return html[start:end]
|
||||
|
||||
|
||||
def test_add_memory_form_renders_a_submit_button():
|
||||
html = INDEX_HTML.read_text()
|
||||
row = _add_memory_row(html)
|
||||
|
||||
assert 'id="new-memory-category"' in row, "button must sit in the same row as the form fields"
|
||||
btn_start = row.index('id="new-memory-add-btn"')
|
||||
btn_tag = row[row.rindex("<button", 0, btn_start):row.index(">", btn_start)]
|
||||
assert 'type="button"' in btn_tag, "must not rely on implicit submit semantics"
|
||||
|
||||
|
||||
def _new_memory_wiring_block(source):
|
||||
start = source.index("const newMemoryInput = el('new-memory-input');")
|
||||
end = source.index("// Voice recording", start)
|
||||
return source[start:end]
|
||||
|
||||
|
||||
def test_submit_button_is_wired_to_add_new_memory():
|
||||
block = _new_memory_wiring_block(APP_JS.read_text())
|
||||
|
||||
assert "el('new-memory-add-btn')" in block
|
||||
assert "addEventListener('click', () => memoryModule.addNewMemory())" in block
|
||||
|
||||
|
||||
def test_enter_uses_keydown_with_prevent_default():
|
||||
block = _new_memory_wiring_block(APP_JS.read_text())
|
||||
|
||||
assert "addEventListener('keydown'" in block
|
||||
assert "addEventListener('keypress'" not in block, "keypress is deprecated and unreliable for Enter"
|
||||
assert "e.preventDefault();" in block
|
||||
assert "!e.isComposing" in block, "IME composition must not submit the form"
|
||||
assert "memoryModule.addNewMemory();" in block
|
||||
@@ -67,12 +67,6 @@ class FakeMemoryManager:
|
||||
def load_all(self):
|
||||
return list(self.rows)
|
||||
|
||||
def load_all_for_update(self):
|
||||
# Mirrors the real MemoryManager: extraction is a read-modify-write and
|
||||
# goes through the strict loader (#5673). A healthy store behaves the
|
||||
# same as load_all.
|
||||
return list(self.rows)
|
||||
|
||||
def load(self, owner=None):
|
||||
return [r for r in self.rows if r.get("owner") == owner]
|
||||
|
||||
|
||||
@@ -1,255 +0,0 @@
|
||||
"""A memory store that cannot be READ must never be overwritten (issue #5673).
|
||||
|
||||
`MemoryManager.save` is atomic, and the add/import/extract paths are all
|
||||
read-modify-write: load the whole store, append, save it back. `load_all`
|
||||
used to answer a *failed read* with `[]` — indistinguishable from "no
|
||||
memories" — so a failed read turned into
|
||||
|
||||
load_all() -> [] -> [].append(new) -> save([new])
|
||||
|
||||
which atomically replaced the entire store with one entry.
|
||||
|
||||
The trigger that actually bites is a store that is **readable but not
|
||||
parseable** — a truncated file, or one holding `{}` instead of `[]`. Nothing
|
||||
obstructs the write, so the request succeeds with HTTP 200 and every existing
|
||||
memory is destroyed silently. Truncation is reachable: `core/database.py`
|
||||
rewrites memory.json during migration with a plain `open(..., "w")` +
|
||||
`json.dump`, which is not atomic.
|
||||
|
||||
A live exclusive lock is NOT the dangerous case: it blocks the read and the
|
||||
`os.replace` alike, so the save fails too and the store survives (verified
|
||||
end-to-end — clean dev returns 500 there and loses nothing).
|
||||
|
||||
`load_all_for_update` is the strict loader those callers now use: it raises
|
||||
`MemoryStoreUnreadable` rather than reporting an empty store.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import builtins
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from src.memory import MemoryManager, MemoryStoreUnreadable
|
||||
|
||||
_SEED = [
|
||||
{"id": "m1", "text": "user prefers dark mode", "owner": "alice"},
|
||||
{"id": "m2", "text": "user lives in Berlin", "owner": "alice"},
|
||||
{"id": "m3", "text": "bob's cat is called Mila", "owner": "bob"},
|
||||
]
|
||||
|
||||
|
||||
def _seeded(tmp_path):
|
||||
m = MemoryManager(str(tmp_path))
|
||||
m.save([dict(e) for e in _SEED])
|
||||
return m
|
||||
|
||||
|
||||
def _break_reads_of(monkeypatch, target, exc):
|
||||
"""Make open() raise `exc` for `target` only, leaving every other path alone."""
|
||||
real_open = builtins.open
|
||||
|
||||
def fake_open(file, mode="r", *args, **kwargs):
|
||||
if os.path.abspath(str(file)) == os.path.abspath(target) and "r" in mode:
|
||||
raise exc
|
||||
return real_open(file, mode, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "open", fake_open)
|
||||
|
||||
|
||||
# ── the strict loader signals, rather than reporting "empty" ──────────────
|
||||
|
||||
def test_strict_load_raises_on_permission_error(tmp_path, monkeypatch):
|
||||
m = _seeded(tmp_path)
|
||||
_break_reads_of(monkeypatch, m.memory_file, PermissionError(13, "locked"))
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
m.load_all_for_update()
|
||||
|
||||
|
||||
def test_strict_load_raises_on_corrupt_json(tmp_path):
|
||||
m = _seeded(tmp_path)
|
||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
||||
f.write('[{"id": "m1", "text": "truncated mid-writ')
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
m.load_all_for_update()
|
||||
|
||||
|
||||
def test_strict_load_raises_when_store_is_not_a_list(tmp_path):
|
||||
# A file holding `{}` or `null` is not an empty store, it is a broken one.
|
||||
m = _seeded(tmp_path)
|
||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
||||
json.dump({}, f)
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
m.load_all_for_update()
|
||||
|
||||
|
||||
def test_strict_load_returns_entries_when_healthy(tmp_path):
|
||||
m = _seeded(tmp_path)
|
||||
assert {e["id"] for e in m.load_all_for_update()} == {"m1", "m2", "m3"}
|
||||
|
||||
|
||||
def test_strict_load_returns_empty_when_file_genuinely_absent(tmp_path):
|
||||
m = _seeded(tmp_path)
|
||||
os.remove(m.memory_file)
|
||||
# Absent is the one case that legitimately means "no memories yet".
|
||||
assert m.load_all_for_update() == []
|
||||
|
||||
|
||||
# ── read paths stay lenient, so an unreadable store can't break chat ──────
|
||||
|
||||
def test_read_path_still_degrades_to_empty(tmp_path, monkeypatch):
|
||||
m = _seeded(tmp_path)
|
||||
_break_reads_of(monkeypatch, m.memory_file, PermissionError(13, "locked"))
|
||||
# Context injection / search must not raise; they just see nothing.
|
||||
assert m.load_all() == []
|
||||
assert m.load(owner="alice") == []
|
||||
|
||||
|
||||
# ── the actual #5673 regression: the store survives ───────────────────────
|
||||
|
||||
def test_add_cycle_under_transient_read_error_does_not_wipe(tmp_path, monkeypatch):
|
||||
"""Mirrors routes/memory/memory_routes.py api_add_memory exactly."""
|
||||
m = _seeded(tmp_path)
|
||||
new_entry = m.add_entry("a brand new fact", owner="alice")
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
all_mem = m.load_all_for_update()
|
||||
all_mem.append(new_entry)
|
||||
m.save(all_mem)
|
||||
|
||||
# Reads work again; every original memory is still there and the file was
|
||||
# never replaced by the single new entry.
|
||||
assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"}
|
||||
|
||||
|
||||
def test_audit_merge_cannot_drop_other_tenants(tmp_path, monkeypatch):
|
||||
"""The audit path rebuilds the whole file from load_all + one owner's slice.
|
||||
|
||||
Reading [] there would save only the audited owner's entries and destroy
|
||||
every other tenant's memories, so it has to fail closed too.
|
||||
"""
|
||||
m = _seeded(tmp_path)
|
||||
alice_slice = [e for e in _SEED if e["owner"] == "alice"]
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
all_entries = m.load_all_for_update()
|
||||
others = [e for e in all_entries if e.get("owner") != "alice"]
|
||||
m.save(alice_slice + others)
|
||||
|
||||
assert any(e["id"] == "m3" for e in m.load_all()), "bob's memory was destroyed"
|
||||
|
||||
|
||||
def test_uses_bump_skips_write_when_unreadable(tmp_path, monkeypatch):
|
||||
m = _seeded(tmp_path)
|
||||
with monkeypatch.context() as mp:
|
||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
||||
m.increment_uses(["m1"]) # must not raise, must not write
|
||||
assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"}
|
||||
|
||||
|
||||
def test_claim_ownerless_skips_write_when_unreadable(tmp_path, monkeypatch):
|
||||
m = _seeded(tmp_path)
|
||||
with monkeypatch.context() as mp:
|
||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
||||
m.claim_ownerless("alice")
|
||||
assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"}
|
||||
|
||||
|
||||
# ── the add sinks users actually reach ────────────────────────────────────
|
||||
#
|
||||
# The tests above replay the read-modify-write shape. These drive the real
|
||||
# entry points end to end, because those are what #5673 reports: "remember
|
||||
# that I prefer X" in ordinary chat (src/ai_interaction.py do_manage_memory,
|
||||
# routed from src/tool_execution.py) and the built-in memory MCP server
|
||||
# (mcp_servers/memory_server.py, registered in src/builtin_mcp.py).
|
||||
#
|
||||
# They use a truncated store rather than a read error on purpose: it reads
|
||||
# fine, so nothing stops the save, which is the case that silently destroyed
|
||||
# stores. The assertion is that the file is left byte-identical — still broken,
|
||||
# but still holding the user's memories, so it can be repaired by hand.
|
||||
|
||||
|
||||
def _truncated_store(tmp_path):
|
||||
"""Seed a store that reads back fine but no longer parses."""
|
||||
m = _seeded(tmp_path)
|
||||
good = json.dumps([dict(e) for e in _SEED], indent=2)
|
||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
||||
f.write(good[:good.rindex("]")]) # drop the closing bracket only
|
||||
with open(m.memory_file, "rb") as f:
|
||||
return m, f.read()
|
||||
|
||||
|
||||
def _on_disk(manager) -> bytes:
|
||||
with open(manager.memory_file, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def test_agent_memory_add_does_not_overwrite_unreadable_store(tmp_path, monkeypatch):
|
||||
"""src/ai_interaction.py do_manage_memory, action "add"."""
|
||||
from src import ai_interaction
|
||||
|
||||
manager, before = _truncated_store(tmp_path)
|
||||
monkeypatch.setattr(ai_interaction, "_memory_manager", manager)
|
||||
monkeypatch.setattr(ai_interaction, "_memory_vector", None)
|
||||
|
||||
result = asyncio.run(ai_interaction.do_manage_memory("add\nuser prefers tabs"))
|
||||
|
||||
assert _on_disk(manager) == before, "the unreadable store was overwritten"
|
||||
assert b"m3" in _on_disk(manager)
|
||||
assert "error" in result, "the add reported success over an unreadable store"
|
||||
|
||||
|
||||
def test_mcp_memory_add_does_not_overwrite_unreadable_store(tmp_path, monkeypatch):
|
||||
"""mcp_servers/memory_server.py, action "add"."""
|
||||
import mcp_servers.memory_server as memory_server
|
||||
|
||||
manager, before = _truncated_store(tmp_path)
|
||||
monkeypatch.setattr(memory_server, "_memory_manager", manager)
|
||||
monkeypatch.setattr(memory_server, "_memory_vector", None)
|
||||
monkeypatch.setattr(memory_server, "_initialized", True)
|
||||
for key in memory_server._OWNER_ENV_KEYS:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
result = asyncio.run(memory_server.call_tool(
|
||||
"manage_memory", {"action": "add", "text": "user prefers tabs"}
|
||||
))
|
||||
|
||||
assert _on_disk(manager) == before, "the unreadable store was overwritten"
|
||||
assert b"m3" in _on_disk(manager)
|
||||
assert result[0].text.startswith("Error:")
|
||||
|
||||
|
||||
def test_native_provider_remember_does_not_overwrite_unreadable_store(tmp_path):
|
||||
"""src/memory_provider.py NativeMemoryProvider.remember.
|
||||
|
||||
Registered into app state in src/app_initializer.py but not yet consumed
|
||||
outside tests, so this is the pattern held in place before it goes live.
|
||||
"""
|
||||
from src.memory_provider import NativeMemoryProvider
|
||||
|
||||
manager, before = _truncated_store(tmp_path)
|
||||
provider = NativeMemoryProvider(manager)
|
||||
|
||||
with pytest.raises(MemoryStoreUnreadable):
|
||||
asyncio.run(provider.remember("user prefers tabs", owner="alice"))
|
||||
|
||||
assert _on_disk(manager) == before
|
||||
|
||||
|
||||
# ── the legacy memory.txt migration is preserved ──────────────────────────
|
||||
|
||||
def test_corrupt_store_still_migrates_from_legacy_txt(tmp_path):
|
||||
m = _seeded(tmp_path)
|
||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
||||
f.write("{ not json")
|
||||
legacy = os.path.join(str(tmp_path), "memory.txt")
|
||||
with open(legacy, "w", encoding="utf-8") as f:
|
||||
f.write("recovered fact one\nrecovered fact two\n")
|
||||
|
||||
entries = m.load_all_for_update()
|
||||
assert [e["text"] for e in entries] == ["recovered fact one", "recovered fact two"]
|
||||
@@ -14,7 +14,7 @@ def _function_source(path: str, name: str) -> str:
|
||||
|
||||
|
||||
def test_document_ai_tidy_resolves_with_owner_scope():
|
||||
body = _function_source("routes/document/document_routes.py", "ai_tidy_documents")
|
||||
body = _function_source("routes/document_routes.py", "ai_tidy_documents")
|
||||
assert "resolve_task_endpoint(owner=user or None)" in body
|
||||
assert 'resolve_endpoint("default", owner=user or None)' in body
|
||||
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
"""Regression: the Qwen bare-marker scrub must not eat a lone `end` (#5547).
|
||||
|
||||
`_QWEN_BARE_MARKER_RE` cleans Qwen turn markers that leak into content. Its
|
||||
`end` branch was `\\|?end\\|?` — both pipes optional — so it also matched a bare
|
||||
`end` surrounded by whitespace and replaced it with a space. Any message
|
||||
containing Ruby, Lua or shell code that closes a block with a lone `end` had
|
||||
those lines silently deleted, in the stored text and in the rendered message.
|
||||
|
||||
Requiring at least one pipe keeps every real marker (`|end`, `end|`, `|end|`,
|
||||
`/|end|`) stripping as before. The same pattern is duplicated in
|
||||
static/js/chatRenderer.js, so the JS copy is checked here too — the two must
|
||||
not drift.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import src.agent_tools # noqa: F401 (break agent_tools<->tool_parsing import cycle)
|
||||
from src.tool_parsing import strip_tool_blocks
|
||||
|
||||
_REPO = Path(__file__).resolve().parent.parent
|
||||
_CHAT_RENDERER = _REPO / "static" / "js" / "chatRenderer.js"
|
||||
|
||||
# Inputs that must survive untouched, and the substring that proves they did.
|
||||
KEPT = [
|
||||
("loop do\n puts \"yo\"\nend\n", "\nend"), # the reported Ruby case
|
||||
("if x then\nend", "\nend"),
|
||||
("function f()\nend\n", "\nend"),
|
||||
("a end b", "a end b"),
|
||||
("append end", "append end"),
|
||||
("END", "END"),
|
||||
("\nEnd\n", "End"),
|
||||
]
|
||||
|
||||
# Real markers — at least one pipe, plus the role word — with the exact output
|
||||
# they must still produce. Asserted as equality rather than "marker not in out"
|
||||
# so narrowing the pattern can't pass by deleting more than it should.
|
||||
STRIPPED = [
|
||||
("a |end| b", "a b"),
|
||||
("a /|end| b", "a b"),
|
||||
("a |end b", "a b"),
|
||||
("a end| b", "a b"),
|
||||
("x assistant y", "x y"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text,kept", KEPT)
|
||||
def test_bare_end_survives_stripping(text, kept):
|
||||
assert kept in strip_tool_blocks(text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text,expected", STRIPPED)
|
||||
def test_piped_end_markers_are_still_stripped(text, expected):
|
||||
assert strip_tool_blocks(text) == expected
|
||||
|
||||
|
||||
def test_bare_end_inside_a_fenced_block_survives():
|
||||
"""The scrub runs over the whole message, fenced regions included."""
|
||||
out = strip_tool_blocks("Here:\n```ruby\nloop do\n puts 1\nend\n```\nDone.")
|
||||
assert "\nend\n" in out
|
||||
|
||||
|
||||
def _js_bare_marker_regex_source():
|
||||
src = _CHAT_RENDERER.read_text(encoding="utf-8")
|
||||
m = re.search(r"^const QWEN_BARE_MARKER_RE = (/.*/[gimsuy]*);$", src, re.MULTILINE)
|
||||
assert m, "QWEN_BARE_MARKER_RE literal not found in chatRenderer.js"
|
||||
return m.group(1)
|
||||
|
||||
|
||||
def test_js_copy_of_the_pattern_matches_the_python_one():
|
||||
"""Guard the duplication: the JS branch must require a pipe too."""
|
||||
if shutil.which("node") is None:
|
||||
pytest.skip("node binary not on PATH")
|
||||
|
||||
cases = [text for text, _ in KEPT] + [text for text, _ in STRIPPED]
|
||||
script = (
|
||||
"const RE = %s;\n"
|
||||
"const cases = JSON.parse(process.argv[1]);\n"
|
||||
"console.log(JSON.stringify(cases.map(c => c.replace(RE, ' '))));"
|
||||
% _js_bare_marker_regex_source()
|
||||
)
|
||||
result = subprocess.run(
|
||||
["node", "--input-type=module", "-e", script, json.dumps(cases)],
|
||||
cwd=_REPO, capture_output=True, timeout=15, text=True,
|
||||
)
|
||||
assert result.returncode == 0, f"node failed:\n{result.stderr}"
|
||||
got = json.loads(result.stdout.splitlines()[-1])
|
||||
|
||||
for (text, kept), out in zip(KEPT, got):
|
||||
assert kept in out, f"JS regex dropped {kept!r} from {text!r}"
|
||||
for (text, expected), out in zip(STRIPPED, got[len(KEPT):]):
|
||||
assert out == expected, f"JS regex: {text!r} -> {out!r}, expected {expected!r}"
|
||||
@@ -1,11 +0,0 @@
|
||||
"""Regression test for the vault route shim (slice 2k, #4082/#4071)."""
|
||||
|
||||
import importlib
|
||||
|
||||
import routes.vault_routes as _shim_vault # noqa: F401
|
||||
|
||||
|
||||
def test_legacy_and_canonical_vault_module_are_same_object():
|
||||
legacy = importlib.import_module("routes.vault_routes")
|
||||
canonical = importlib.import_module("routes.vault.vault_routes")
|
||||
assert legacy is canonical
|
||||
@@ -88,7 +88,7 @@ def test_request_vision_call_sites_pass_owner():
|
||||
chat_source = (ROOT / "src" / "chat_handler.py").read_text()
|
||||
processor_source = (ROOT / "src" / "document_processor.py").read_text()
|
||||
upload_source = (ROOT / "routes" / "upload_routes.py").read_text()
|
||||
document_source = (ROOT / "routes" / "document" / "document_routes.py").read_text()
|
||||
document_source = (ROOT / "routes" / "document_routes.py").read_text()
|
||||
gallery_source = (ROOT / "routes" / "gallery" / "gallery_routes.py").read_text()
|
||||
memory_source = (ROOT / "routes" / "memory" / "memory_routes.py").read_text()
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
"""Regression test for the webhook route shim (slice 2l, #4082/#4071)."""
|
||||
|
||||
import importlib
|
||||
|
||||
import routes.webhook_routes as _shim_webhook # noqa: F401
|
||||
|
||||
|
||||
def test_legacy_and_canonical_webhook_module_are_same_object():
|
||||
legacy = importlib.import_module("routes.webhook_routes")
|
||||
canonical = importlib.import_module("routes.webhook.webhook_routes")
|
||||
assert legacy is canonical
|
||||
Reference in New Issue
Block a user