mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-08-08 06:28:37 -04:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d960fc758c |
@@ -21,7 +21,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
@@ -73,10 +73,10 @@ jobs:
|
|||||||
name: Python syntax (compileall)
|
name: Python syntax (compileall)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
# Byte-compile sources — catches syntax errors without installing deps.
|
# Byte-compile sources — catches syntax errors without installing deps.
|
||||||
@@ -86,10 +86,10 @@ jobs:
|
|||||||
name: JS syntax (node --check)
|
name: JS syntax (node --check)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "20"
|
||||||
# Syntax-check our own JS (skip vendored libs in static/lib).
|
# Syntax-check our own JS (skip vendored libs in static/lib).
|
||||||
@@ -108,7 +108,7 @@ jobs:
|
|||||||
# ROADMAP "fresh install smoke tests" item; make this required once green.
|
# ROADMAP "fresh install smoke tests" item; make this required once green.
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
@@ -135,7 +135,7 @@ jobs:
|
|||||||
echo "docs_only=false" >> "$GITHUB_OUTPUT"
|
echo "docs_only=false" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||||
if: steps.docs-check.outputs.docs_only != 'true'
|
if: steps.docs-check.outputs.docs_only != 'true'
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
|
|||||||
@@ -27,15 +27,15 @@ jobs:
|
|||||||
language: [actions, javascript-typescript, python]
|
language: [actions, javascript-typescript, python]
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- name: Initialize CodeQL
|
- name: Initialize CodeQL
|
||||||
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
uses: github/codeql-action/init@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
|
||||||
with:
|
with:
|
||||||
languages: ${{ matrix.language }}
|
languages: ${{ matrix.language }}
|
||||||
build-mode: none
|
build-mode: none
|
||||||
- name: Perform CodeQL Analysis
|
- name: Perform CodeQL Analysis
|
||||||
uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
uses: github/codeql-action/analyze@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
|
||||||
with:
|
with:
|
||||||
category: "/language:${{ matrix.language }}"
|
category: "/language:${{ matrix.language }}"
|
||||||
|
|||||||
@@ -37,12 +37,12 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Lint Dockerfile
|
- name: Lint Dockerfile
|
||||||
uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0
|
uses: hadolint/hadolint-action@2a66e89f53d0771bb131a7fa31f3136336094aa6 # v3.4.0
|
||||||
with:
|
with:
|
||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
# DL3008: pinning apt package versions is impractical on a -slim base
|
# DL3008: pinning apt package versions is impractical on a -slim base
|
||||||
|
|||||||
@@ -52,17 +52,17 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Set up Buildx
|
- name: Set up Buildx
|
||||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||||
|
|
||||||
# Build without pushing so a broken Dockerfile is caught here, and the
|
# Build without pushing so a broken Dockerfile is caught here, and the
|
||||||
# exact image we ship is what gets scanned.
|
# exact image we ship is what gets scanned.
|
||||||
- name: Build image
|
- name: Build image
|
||||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
push: false
|
push: false
|
||||||
@@ -93,15 +93,15 @@ jobs:
|
|||||||
security-events: write # upload SARIF to the Security tab
|
security-events: write # upload SARIF to the Security tab
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Set up Buildx
|
- name: Set up Buildx
|
||||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||||
|
|
||||||
- name: Build image
|
- name: Build image
|
||||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
push: false
|
push: false
|
||||||
@@ -119,7 +119,7 @@ jobs:
|
|||||||
TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2
|
TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2
|
||||||
|
|
||||||
- name: Upload Trivy results
|
- name: Upload Trivy results
|
||||||
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
|
||||||
with:
|
with:
|
||||||
sarif_file: trivy-results.sarif
|
sarif_file: trivy-results.sarif
|
||||||
category: trivy-image
|
category: trivy-image
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
@@ -55,12 +55,12 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
|
|
||||||
|
|||||||
@@ -45,20 +45,20 @@ jobs:
|
|||||||
arch: arm64
|
arch: arm64
|
||||||
runner: ubuntu-24.04-arm
|
runner: ubuntu-24.04-arm
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- name: Set up Buildx
|
- name: Set up Buildx
|
||||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||||
- name: Log in to GHCR
|
- name: Log in to GHCR
|
||||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||||
with:
|
with:
|
||||||
registry: ${{ env.REGISTRY }}
|
registry: ${{ env.REGISTRY }}
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
- name: Build and push by digest
|
- name: Build and push by digest
|
||||||
id: build
|
id: build
|
||||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
platforms: ${{ matrix.platform }}
|
platforms: ${{ matrix.platform }}
|
||||||
@@ -86,7 +86,7 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
packages: write
|
packages: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- name: Read APP_VERSION + short sha
|
- name: Read APP_VERSION + short sha
|
||||||
@@ -103,16 +103,16 @@ jobs:
|
|||||||
pattern: digest-*
|
pattern: digest-*
|
||||||
merge-multiple: true
|
merge-multiple: true
|
||||||
- name: Set up Buildx
|
- name: Set up Buildx
|
||||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||||
- name: Log in to GHCR
|
- name: Log in to GHCR
|
||||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||||
with:
|
with:
|
||||||
registry: ${{ env.REGISTRY }}
|
registry: ${{ env.REGISTRY }}
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
- name: Compute tags
|
- name: Compute tags
|
||||||
id: meta
|
id: meta
|
||||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
|
||||||
with:
|
with:
|
||||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||||
tags: |
|
tags: |
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ jobs:
|
|||||||
# Skip bots (Dependabot, release-drafter, etc.)
|
# Skip bots (Dependabot, release-drafter, etc.)
|
||||||
if: ${{ github.event.issue.user.type != 'Bot' }}
|
if: ${{ github.event.issue.user.type != 'Bot' }}
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
sparse-checkout: .github/scripts
|
sparse-checkout: .github/scripts
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ jobs:
|
|||||||
# Skip bots: they open PRs programmatically and have their own process.
|
# Skip bots: they open PRs programmatically and have their own process.
|
||||||
if: github.event.pull_request.user.type != 'Bot'
|
if: github.event.pull_request.user.type != 'Bot'
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
ref: ${{ github.base_ref }}
|
ref: ${{ github.base_ref }}
|
||||||
sparse-checkout: .github/scripts
|
sparse-checkout: .github/scripts
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
# Full history so a secret committed in an earlier commit (and later
|
# Full history so a secret committed in an earlier commit (and later
|
||||||
# deleted) is still caught -- deletion does not remove it from Git.
|
# deleted) is still caught -- deletion does not remove it from Git.
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
@@ -61,12 +61,12 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
|
|
||||||
|
|||||||
@@ -739,7 +739,7 @@ app.include_router(setup_stt_routes(stt_service))
|
|||||||
logger.info("STT service initialized (provider managed via settings)")
|
logger.info("STT service initialized (provider managed via settings)")
|
||||||
|
|
||||||
# Documents (artifacts/canvas)
|
# Documents (artifacts/canvas)
|
||||||
from routes.document.document_routes import setup_document_routes
|
from routes.document_routes import setup_document_routes
|
||||||
document_router = setup_document_routes(session_manager, upload_handler)
|
document_router = setup_document_routes(session_manager, upload_handler)
|
||||||
app.include_router(document_router)
|
app.include_router(document_router)
|
||||||
|
|
||||||
@@ -820,7 +820,7 @@ set_ai_rag_manager(rag_manager, personal_docs_mgr)
|
|||||||
logger.info("AI interaction tools initialized (session, memory, RAG, UI control)")
|
logger.info("AI interaction tools initialized (session, memory, RAG, UI control)")
|
||||||
|
|
||||||
# Webhooks
|
# Webhooks
|
||||||
from routes.webhook.webhook_routes import setup_webhook_routes
|
from routes.webhook_routes import setup_webhook_routes
|
||||||
app.include_router(setup_webhook_routes(webhook_manager, auth_manager, session_manager, api_key_manager))
|
app.include_router(setup_webhook_routes(webhook_manager, auth_manager, session_manager, api_key_manager))
|
||||||
|
|
||||||
# API Tokens
|
# API Tokens
|
||||||
@@ -852,7 +852,7 @@ app.include_router(setup_codex_routes(
|
|||||||
))
|
))
|
||||||
app.include_router(setup_claude_routes())
|
app.include_router(setup_claude_routes())
|
||||||
|
|
||||||
from routes.vault.vault_routes import setup_vault_routes
|
from routes.vault_routes import setup_vault_routes
|
||||||
app.include_router(setup_vault_routes())
|
app.include_router(setup_vault_routes())
|
||||||
|
|
||||||
# Contacts (CardDAV)
|
# Contacts (CardDAV)
|
||||||
|
|||||||
@@ -17,8 +17,6 @@ from mcp.types import Tool, TextContent
|
|||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||||
|
|
||||||
from src.memory import MemoryStoreUnreadable
|
|
||||||
|
|
||||||
server = Server("memory")
|
server = Server("memory")
|
||||||
|
|
||||||
# Late-initialized managers (set during first tool call)
|
# Late-initialized managers (set during first tool call)
|
||||||
@@ -31,10 +29,6 @@ _OWNER_SCOPE_ERROR = (
|
|||||||
"Error: Memory MCP owner is not configured for an owner-scoped memory store. "
|
"Error: Memory MCP owner is not configured for an owner-scoped memory store. "
|
||||||
"Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
|
"Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
|
||||||
)
|
)
|
||||||
_UNREADABLE_STORE_ERROR = (
|
|
||||||
"Error: Memory store is temporarily unreadable — nothing was saved. "
|
|
||||||
"Repair or restore memory.json, then retry."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _configured_owner() -> str | None:
|
def _configured_owner() -> str | None:
|
||||||
@@ -57,21 +51,9 @@ def _owner_scoped_store(entries: list[dict]) -> bool:
|
|||||||
return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
|
return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
|
||||||
|
|
||||||
|
|
||||||
def _scope_entries(for_update: bool = False) -> tuple[str | None, list[dict], list[dict], str | None]:
|
def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]:
|
||||||
"""Return configured owner, all entries, visible entries, and optional error.
|
"""Return configured owner, all entries, visible entries, and optional error."""
|
||||||
|
entries = _memory_manager.load_all()
|
||||||
``for_update=True`` is for read-modify-write callers. They save the ``all
|
|
||||||
entries`` list back, so an unreadable store must be reported as an error
|
|
||||||
instead of degrading to ``[]`` — otherwise the save writes their one new
|
|
||||||
entry over the whole store (issue #5673).
|
|
||||||
"""
|
|
||||||
if for_update:
|
|
||||||
try:
|
|
||||||
entries = _memory_manager.load_all_for_update()
|
|
||||||
except MemoryStoreUnreadable as e:
|
|
||||||
return None, [], [], f"{_UNREADABLE_STORE_ERROR} ({e})"
|
|
||||||
else:
|
|
||||||
entries = _memory_manager.load_all()
|
|
||||||
owner = _configured_owner()
|
owner = _configured_owner()
|
||||||
if owner is None and _owner_scoped_store(entries):
|
if owner is None and _owner_scoped_store(entries):
|
||||||
return None, entries, [], _OWNER_SCOPE_ERROR
|
return None, entries, [], _OWNER_SCOPE_ERROR
|
||||||
@@ -179,7 +161,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
|
|||||||
category = arguments.get("category", "fact")
|
category = arguments.get("category", "fact")
|
||||||
if not text:
|
if not text:
|
||||||
return _text_result("Error: Memory text cannot be empty")
|
return _text_result("Error: Memory text cannot be empty")
|
||||||
owner, memories, _visible, scope_error = _scope_entries(for_update=True)
|
owner, memories, _visible, scope_error = _scope_entries()
|
||||||
if scope_error:
|
if scope_error:
|
||||||
return _text_result(scope_error)
|
return _text_result(scope_error)
|
||||||
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
|
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
|
||||||
|
|||||||
+1
-10
@@ -6,7 +6,6 @@ from datetime import datetime
|
|||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, Request, Response
|
from fastapi import APIRouter, HTTPException, Request, Response
|
||||||
from core.middleware import require_admin
|
from core.middleware import require_admin
|
||||||
from services.memory import MemoryStoreUnreadable
|
|
||||||
from src.auth_helpers import get_current_user
|
from src.auth_helpers import get_current_user
|
||||||
from src.settings import load_settings, save_settings, load_features, save_features
|
from src.settings import load_settings, save_settings, load_features, save_features
|
||||||
|
|
||||||
@@ -77,15 +76,7 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo
|
|||||||
|
|
||||||
# ── Memories ──
|
# ── Memories ──
|
||||||
if "memories" in body and isinstance(body["memories"], list):
|
if "memories" in body and isinstance(body["memories"], list):
|
||||||
# Strict load: importing on top of an unreadable store would write
|
existing = memory_manager.load_all()
|
||||||
# only the incoming rows back and drop everything already saved.
|
|
||||||
try:
|
|
||||||
existing = memory_manager.load_all_for_update()
|
|
||||||
except MemoryStoreUnreadable as e:
|
|
||||||
logger.error("Refusing to import memories: %s", e)
|
|
||||||
raise HTTPException(
|
|
||||||
503, "Memory store is temporarily unreadable — nothing was imported."
|
|
||||||
)
|
|
||||||
# Dedup against THIS user's own memories only. Using every tenant's
|
# Dedup against THIS user's own memories only. Using every tenant's
|
||||||
# rows (load_all) meant a memory whose text matched any other
|
# rows (load_all) meant a memory whose text matched any other
|
||||||
# user's was silently skipped, so the importing user lost their own
|
# user's was silently skipped, so the importing user lost their own
|
||||||
|
|||||||
@@ -1,6 +0,0 @@
|
|||||||
"""Document route domain package (slice 2m, #4082/#4071).
|
|
||||||
|
|
||||||
Contains document_routes.py and document_helpers.py, migrated from the flat
|
|
||||||
routes/ directory. Backward-compat shims at routes/document_routes.py and
|
|
||||||
routes/document_helpers.py re-export from here.
|
|
||||||
"""
|
|
||||||
@@ -1,243 +0,0 @@
|
|||||||
"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
|
|
||||||
|
|
||||||
"""Document routes — CRUD for living documents with version history."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
from typing import Any, Dict, Optional
|
|
||||||
|
|
||||||
from fastapi import HTTPException, Request
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
from core.database import Document, DocumentVersion
|
|
||||||
from core.database import Session as DbSession
|
|
||||||
from src.auth_helpers import _auth_disabled
|
|
||||||
from src.upload_handler import UploadHandler
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
# ---- Request schemas ----
|
|
||||||
|
|
||||||
class DocumentCreate(BaseModel):
|
|
||||||
session_id: Optional[str] = None
|
|
||||||
title: str = "Untitled"
|
|
||||||
language: Optional[str] = None
|
|
||||||
content: str = ""
|
|
||||||
|
|
||||||
class DocumentUpdate(BaseModel):
|
|
||||||
content: str
|
|
||||||
summary: Optional[str] = None
|
|
||||||
force_version: bool = False
|
|
||||||
|
|
||||||
class DocumentPatch(BaseModel):
|
|
||||||
title: Optional[str] = None
|
|
||||||
language: Optional[str] = None
|
|
||||||
session_id: Optional[str] = None # link/unlink document to a session
|
|
||||||
|
|
||||||
|
|
||||||
# ---- Helpers ----
|
|
||||||
|
|
||||||
def _doc_to_dict(doc: Document) -> Dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"id": doc.id,
|
|
||||||
"session_id": doc.session_id,
|
|
||||||
"title": doc.title,
|
|
||||||
"language": doc.language,
|
|
||||||
"current_content": doc.current_content,
|
|
||||||
"version_count": doc.version_count,
|
|
||||||
"is_active": doc.is_active,
|
|
||||||
"archived": bool(getattr(doc, "archived", False)),
|
|
||||||
"created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
|
|
||||||
"updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
|
|
||||||
# Source-email provenance (set when doc was created from an email
|
|
||||||
# attachment) — drives the "Send signed reply" menu item.
|
|
||||||
"source_email_uid": getattr(doc, "source_email_uid", None),
|
|
||||||
"source_email_folder": getattr(doc, "source_email_folder", None),
|
|
||||||
"source_email_account_id": getattr(doc, "source_email_account_id", None),
|
|
||||||
"source_email_message_id": getattr(doc, "source_email_message_id", None),
|
|
||||||
}
|
|
||||||
|
|
||||||
def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"id": v.id,
|
|
||||||
"document_id": v.document_id,
|
|
||||||
"version_number": v.version_number,
|
|
||||||
"content": v.content,
|
|
||||||
"summary": v.summary,
|
|
||||||
"source": v.source,
|
|
||||||
"created_at": v.created_at.isoformat() if v.created_at else None,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _verify_doc_owner(db, doc: Document, user: str):
|
|
||||||
"""Verify `user` owns this document. Raise 404 if not.
|
|
||||||
|
|
||||||
Documents now carry their own `owner` column, so a doc whose session
|
|
||||||
was deleted (session_id → NULL) can still prove ownership and stay
|
|
||||||
openable / cloneable. We trust that column first and only fall back to
|
|
||||||
the session join for any not-yet-backfilled legacy row.
|
|
||||||
"""
|
|
||||||
if user is None:
|
|
||||||
if _auth_disabled():
|
|
||||||
return # Single-user / no-auth mode: allow access
|
|
||||||
raise HTTPException(403, "Authentication required")
|
|
||||||
if doc.owner is not None:
|
|
||||||
if doc.owner != user:
|
|
||||||
raise HTTPException(404, "Document not found")
|
|
||||||
return
|
|
||||||
# Legacy fallback: derive ownership from the linked session.
|
|
||||||
if not doc.session_id:
|
|
||||||
raise HTTPException(404, "Document not found")
|
|
||||||
session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
|
|
||||||
if not session or session.owner != user:
|
|
||||||
raise HTTPException(404, "Document not found")
|
|
||||||
|
|
||||||
|
|
||||||
def _owner_session_filter(q, user):
|
|
||||||
"""Restrict a documents query to those owned by `user`.
|
|
||||||
|
|
||||||
Documents now carry their own `owner` column (backfilled at boot from
|
|
||||||
the linked session, or assigned to the admin user for legacy/orphaned
|
|
||||||
docs). We filter on that directly rather than on a session join, so a
|
|
||||||
document whose session was deleted (session_id → NULL) still shows up
|
|
||||||
for its owner instead of silently vanishing from the Library + search.
|
|
||||||
|
|
||||||
The owner backfill runs in init_db before the app serves requests, so
|
|
||||||
by the time this filter is live there are no NULL-owner rows to leak;
|
|
||||||
we therefore match the owner strictly for authenticated callers."""
|
|
||||||
if not user:
|
|
||||||
if user == "" or _auth_disabled():
|
|
||||||
return q
|
|
||||||
return q.filter(False)
|
|
||||||
return q.filter(Document.owner == user)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def _slug(name: str) -> str:
|
|
||||||
"""Filesystem-friendly version of a document title.
|
|
||||||
|
|
||||||
Whitespace becomes underscores; other unsafe punctuation is dropped.
|
|
||||||
Preserves letters, digits, dot, hyphen, underscore. Idempotent.
|
|
||||||
"""
|
|
||||||
import re as _re
|
|
||||||
s = (name or "").strip()
|
|
||||||
# Drop the trailing extension if the title happens to include one
|
|
||||||
s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
|
|
||||||
s = _re.sub(r'\s+', '_', s)
|
|
||||||
s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
|
|
||||||
s = _re.sub(r'_+', '_', s).strip('_')
|
|
||||||
return s or "form"
|
|
||||||
|
|
||||||
|
|
||||||
# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
|
|
||||||
_PDF_RENDER_SCALE = 2.0
|
|
||||||
|
|
||||||
|
|
||||||
def _upload_path_inside(upload_dir: str, path: str) -> bool:
|
|
||||||
base = os.path.realpath(upload_dir)
|
|
||||||
p = os.path.realpath(path)
|
|
||||||
try:
|
|
||||||
return os.path.commonpath([base, p]) == base
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _resolve_user_upload_path(
|
|
||||||
upload_handler: Any,
|
|
||||||
upload_id: str,
|
|
||||||
owner: Optional[str],
|
|
||||||
auth_manager=None,
|
|
||||||
) -> Optional[str]:
|
|
||||||
"""Resolve an upload id to a filesystem path the caller may read."""
|
|
||||||
if upload_handler is None:
|
|
||||||
return None
|
|
||||||
resolved = upload_handler.resolve_upload(
|
|
||||||
upload_id,
|
|
||||||
owner=owner,
|
|
||||||
auth_manager=auth_manager,
|
|
||||||
)
|
|
||||||
if not isinstance(resolved, dict) or not resolved:
|
|
||||||
return None
|
|
||||||
path = resolved.get("path")
|
|
||||||
upload_dir = getattr(upload_handler, "upload_dir", None)
|
|
||||||
if path and upload_dir and not _upload_path_inside(upload_dir, path):
|
|
||||||
logger.warning("Upload path outside upload directory: %s", path)
|
|
||||||
return None
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def _locate_upload(
|
|
||||||
upload_dir: str,
|
|
||||||
file_id: str,
|
|
||||||
owner: Optional[str] = None,
|
|
||||||
auth_manager=None,
|
|
||||||
upload_handler: Any = None,
|
|
||||||
):
|
|
||||||
"""Find an upload by its filename ID via UploadHandler.resolve_upload."""
|
|
||||||
if upload_handler is None:
|
|
||||||
from src.upload_handler import UploadHandler
|
|
||||||
|
|
||||||
base_dir = os.path.dirname(os.path.abspath(upload_dir))
|
|
||||||
upload_handler = UploadHandler(base_dir, upload_dir)
|
|
||||||
return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_pdf_marker_upload_owned(
|
|
||||||
request: Request,
|
|
||||||
content: str,
|
|
||||||
user: Optional[str],
|
|
||||||
upload_handler: Any,
|
|
||||||
) -> None:
|
|
||||||
"""Reject document content whose pdf_source marker points at another user's upload."""
|
|
||||||
if upload_handler is None:
|
|
||||||
return
|
|
||||||
from src.pdf_form_doc import find_source_upload_id
|
|
||||||
|
|
||||||
upload_id = find_source_upload_id(content or "")
|
|
||||||
if not upload_id:
|
|
||||||
return
|
|
||||||
auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
|
|
||||||
if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
|
|
||||||
raise HTTPException(
|
|
||||||
400,
|
|
||||||
"Document PDF marker references an upload you do not own",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _derive_title(content: str) -> str:
|
|
||||||
"""Derive a title from document content."""
|
|
||||||
import re
|
|
||||||
if not isinstance(content, str):
|
|
||||||
return "Untitled"
|
|
||||||
text = content.strip()
|
|
||||||
if not text:
|
|
||||||
return "Untitled"
|
|
||||||
|
|
||||||
# Markdown header
|
|
||||||
md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
|
|
||||||
if md:
|
|
||||||
title = md.group(1).strip()
|
|
||||||
if len(title) > 50:
|
|
||||||
title = title[:48] + "…"
|
|
||||||
return title
|
|
||||||
|
|
||||||
# HTML heading
|
|
||||||
html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
|
|
||||||
if html:
|
|
||||||
title = html.group(1).strip()
|
|
||||||
if len(title) > 50:
|
|
||||||
title = title[:48] + "…"
|
|
||||||
return title
|
|
||||||
|
|
||||||
# First non-empty line (if short enough)
|
|
||||||
for line in text.split('\n'):
|
|
||||||
line = line.strip()
|
|
||||||
if line and 2 <= len(line) <= 60:
|
|
||||||
title = re.sub(r'[:#*`]+$', '', line).strip()
|
|
||||||
if title and len(title) > 50:
|
|
||||||
title = title[:48] + "…"
|
|
||||||
return title or "Untitled"
|
|
||||||
|
|
||||||
return "Untitled"
|
|
||||||
File diff suppressed because it is too large
Load Diff
+239
-10
@@ -1,14 +1,243 @@
|
|||||||
"""Backward-compat shim — canonical location is routes/document/document_helpers.py.
|
"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
|
||||||
|
|
||||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
"""Document routes — CRUD for living documents with version history."""
|
||||||
that ``import routes.document_helpers``, ``from routes.document_helpers import
|
|
||||||
X``, and the ``sys.modules.pop("routes.document_helpers")`` + re-import
|
|
||||||
pattern used by test_security_regressions.py all operate on the *same* object.
|
|
||||||
Keeps existing import paths working after slice 2m (#4082/#4071).
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys as _sys
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
from routes.document import document_helpers as _canonical # noqa: F401
|
from fastapi import HTTPException, Request
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
_sys.modules[__name__] = _canonical
|
from core.database import Document, DocumentVersion
|
||||||
|
from core.database import Session as DbSession
|
||||||
|
from src.auth_helpers import _auth_disabled
|
||||||
|
from src.upload_handler import UploadHandler
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# ---- Request schemas ----
|
||||||
|
|
||||||
|
class DocumentCreate(BaseModel):
|
||||||
|
session_id: Optional[str] = None
|
||||||
|
title: str = "Untitled"
|
||||||
|
language: Optional[str] = None
|
||||||
|
content: str = ""
|
||||||
|
|
||||||
|
class DocumentUpdate(BaseModel):
|
||||||
|
content: str
|
||||||
|
summary: Optional[str] = None
|
||||||
|
force_version: bool = False
|
||||||
|
|
||||||
|
class DocumentPatch(BaseModel):
|
||||||
|
title: Optional[str] = None
|
||||||
|
language: Optional[str] = None
|
||||||
|
session_id: Optional[str] = None # link/unlink document to a session
|
||||||
|
|
||||||
|
|
||||||
|
# ---- Helpers ----
|
||||||
|
|
||||||
|
def _doc_to_dict(doc: Document) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": doc.id,
|
||||||
|
"session_id": doc.session_id,
|
||||||
|
"title": doc.title,
|
||||||
|
"language": doc.language,
|
||||||
|
"current_content": doc.current_content,
|
||||||
|
"version_count": doc.version_count,
|
||||||
|
"is_active": doc.is_active,
|
||||||
|
"archived": bool(getattr(doc, "archived", False)),
|
||||||
|
"created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
|
||||||
|
"updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
|
||||||
|
# Source-email provenance (set when doc was created from an email
|
||||||
|
# attachment) — drives the "Send signed reply" menu item.
|
||||||
|
"source_email_uid": getattr(doc, "source_email_uid", None),
|
||||||
|
"source_email_folder": getattr(doc, "source_email_folder", None),
|
||||||
|
"source_email_account_id": getattr(doc, "source_email_account_id", None),
|
||||||
|
"source_email_message_id": getattr(doc, "source_email_message_id", None),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": v.id,
|
||||||
|
"document_id": v.document_id,
|
||||||
|
"version_number": v.version_number,
|
||||||
|
"content": v.content,
|
||||||
|
"summary": v.summary,
|
||||||
|
"source": v.source,
|
||||||
|
"created_at": v.created_at.isoformat() if v.created_at else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_doc_owner(db, doc: Document, user: str):
|
||||||
|
"""Verify `user` owns this document. Raise 404 if not.
|
||||||
|
|
||||||
|
Documents now carry their own `owner` column, so a doc whose session
|
||||||
|
was deleted (session_id → NULL) can still prove ownership and stay
|
||||||
|
openable / cloneable. We trust that column first and only fall back to
|
||||||
|
the session join for any not-yet-backfilled legacy row.
|
||||||
|
"""
|
||||||
|
if user is None:
|
||||||
|
if _auth_disabled():
|
||||||
|
return # Single-user / no-auth mode: allow access
|
||||||
|
raise HTTPException(403, "Authentication required")
|
||||||
|
if doc.owner is not None:
|
||||||
|
if doc.owner != user:
|
||||||
|
raise HTTPException(404, "Document not found")
|
||||||
|
return
|
||||||
|
# Legacy fallback: derive ownership from the linked session.
|
||||||
|
if not doc.session_id:
|
||||||
|
raise HTTPException(404, "Document not found")
|
||||||
|
session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
|
||||||
|
if not session or session.owner != user:
|
||||||
|
raise HTTPException(404, "Document not found")
|
||||||
|
|
||||||
|
|
||||||
|
def _owner_session_filter(q, user):
|
||||||
|
"""Restrict a documents query to those owned by `user`.
|
||||||
|
|
||||||
|
Documents now carry their own `owner` column (backfilled at boot from
|
||||||
|
the linked session, or assigned to the admin user for legacy/orphaned
|
||||||
|
docs). We filter on that directly rather than on a session join, so a
|
||||||
|
document whose session was deleted (session_id → NULL) still shows up
|
||||||
|
for its owner instead of silently vanishing from the Library + search.
|
||||||
|
|
||||||
|
The owner backfill runs in init_db before the app serves requests, so
|
||||||
|
by the time this filter is live there are no NULL-owner rows to leak;
|
||||||
|
we therefore match the owner strictly for authenticated callers."""
|
||||||
|
if not user:
|
||||||
|
if user == "" or _auth_disabled():
|
||||||
|
return q
|
||||||
|
return q.filter(False)
|
||||||
|
return q.filter(Document.owner == user)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _slug(name: str) -> str:
|
||||||
|
"""Filesystem-friendly version of a document title.
|
||||||
|
|
||||||
|
Whitespace becomes underscores; other unsafe punctuation is dropped.
|
||||||
|
Preserves letters, digits, dot, hyphen, underscore. Idempotent.
|
||||||
|
"""
|
||||||
|
import re as _re
|
||||||
|
s = (name or "").strip()
|
||||||
|
# Drop the trailing extension if the title happens to include one
|
||||||
|
s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
|
||||||
|
s = _re.sub(r'\s+', '_', s)
|
||||||
|
s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
|
||||||
|
s = _re.sub(r'_+', '_', s).strip('_')
|
||||||
|
return s or "form"
|
||||||
|
|
||||||
|
|
||||||
|
# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
|
||||||
|
_PDF_RENDER_SCALE = 2.0
|
||||||
|
|
||||||
|
|
||||||
|
def _upload_path_inside(upload_dir: str, path: str) -> bool:
|
||||||
|
base = os.path.realpath(upload_dir)
|
||||||
|
p = os.path.realpath(path)
|
||||||
|
try:
|
||||||
|
return os.path.commonpath([base, p]) == base
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_user_upload_path(
|
||||||
|
upload_handler: Any,
|
||||||
|
upload_id: str,
|
||||||
|
owner: Optional[str],
|
||||||
|
auth_manager=None,
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Resolve an upload id to a filesystem path the caller may read."""
|
||||||
|
if upload_handler is None:
|
||||||
|
return None
|
||||||
|
resolved = upload_handler.resolve_upload(
|
||||||
|
upload_id,
|
||||||
|
owner=owner,
|
||||||
|
auth_manager=auth_manager,
|
||||||
|
)
|
||||||
|
if not isinstance(resolved, dict) or not resolved:
|
||||||
|
return None
|
||||||
|
path = resolved.get("path")
|
||||||
|
upload_dir = getattr(upload_handler, "upload_dir", None)
|
||||||
|
if path and upload_dir and not _upload_path_inside(upload_dir, path):
|
||||||
|
logger.warning("Upload path outside upload directory: %s", path)
|
||||||
|
return None
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _locate_upload(
|
||||||
|
upload_dir: str,
|
||||||
|
file_id: str,
|
||||||
|
owner: Optional[str] = None,
|
||||||
|
auth_manager=None,
|
||||||
|
upload_handler: Any = None,
|
||||||
|
):
|
||||||
|
"""Find an upload by its filename ID via UploadHandler.resolve_upload."""
|
||||||
|
if upload_handler is None:
|
||||||
|
from src.upload_handler import UploadHandler
|
||||||
|
|
||||||
|
base_dir = os.path.dirname(os.path.abspath(upload_dir))
|
||||||
|
upload_handler = UploadHandler(base_dir, upload_dir)
|
||||||
|
return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_pdf_marker_upload_owned(
|
||||||
|
request: Request,
|
||||||
|
content: str,
|
||||||
|
user: Optional[str],
|
||||||
|
upload_handler: Any,
|
||||||
|
) -> None:
|
||||||
|
"""Reject document content whose pdf_source marker points at another user's upload."""
|
||||||
|
if upload_handler is None:
|
||||||
|
return
|
||||||
|
from src.pdf_form_doc import find_source_upload_id
|
||||||
|
|
||||||
|
upload_id = find_source_upload_id(content or "")
|
||||||
|
if not upload_id:
|
||||||
|
return
|
||||||
|
auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
|
||||||
|
if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
|
||||||
|
raise HTTPException(
|
||||||
|
400,
|
||||||
|
"Document PDF marker references an upload you do not own",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _derive_title(content: str) -> str:
|
||||||
|
"""Derive a title from document content."""
|
||||||
|
import re
|
||||||
|
if not isinstance(content, str):
|
||||||
|
return "Untitled"
|
||||||
|
text = content.strip()
|
||||||
|
if not text:
|
||||||
|
return "Untitled"
|
||||||
|
|
||||||
|
# Markdown header
|
||||||
|
md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
|
||||||
|
if md:
|
||||||
|
title = md.group(1).strip()
|
||||||
|
if len(title) > 50:
|
||||||
|
title = title[:48] + "…"
|
||||||
|
return title
|
||||||
|
|
||||||
|
# HTML heading
|
||||||
|
html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
|
||||||
|
if html:
|
||||||
|
title = html.group(1).strip()
|
||||||
|
if len(title) > 50:
|
||||||
|
title = title[:48] + "…"
|
||||||
|
return title
|
||||||
|
|
||||||
|
# First non-empty line (if short enough)
|
||||||
|
for line in text.split('\n'):
|
||||||
|
line = line.strip()
|
||||||
|
if line and 2 <= len(line) <= 60:
|
||||||
|
title = re.sub(r'[:#*`]+$', '', line).strip()
|
||||||
|
if title and len(title) > 50:
|
||||||
|
title = title[:48] + "…"
|
||||||
|
return title or "Untitled"
|
||||||
|
|
||||||
|
return "Untitled"
|
||||||
|
|||||||
+1806
-13
File diff suppressed because it is too large
Load Diff
@@ -21,7 +21,7 @@ def _strip_list_prefix(text: str) -> str:
|
|||||||
return text
|
return text
|
||||||
return _LIST_PREFIX_RE.sub("", text, count=1).strip()
|
return _LIST_PREFIX_RE.sub("", text, count=1).strip()
|
||||||
|
|
||||||
from services.memory import MemoryManager, MemoryStoreUnreadable
|
from services.memory import MemoryManager
|
||||||
from core.session_manager import SessionManager
|
from core.session_manager import SessionManager
|
||||||
from src.request_models import MemoryAddRequest
|
from src.request_models import MemoryAddRequest
|
||||||
from core.database import SessionLocal
|
from core.database import SessionLocal
|
||||||
@@ -35,22 +35,6 @@ from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _load_for_update(memory_manager) -> List[Dict[str, Any]]:
|
|
||||||
"""Load the whole store for a read-modify-write cycle.
|
|
||||||
|
|
||||||
A transient read failure must not look like an empty store: the caller
|
|
||||||
would append to ``[]`` and save that back, atomically destroying every
|
|
||||||
existing memory (issue #5673). Surface it as a 503 and change nothing.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
return memory_manager.load_all_for_update()
|
|
||||||
except MemoryStoreUnreadable as e:
|
|
||||||
logger.error("Refusing to rewrite the memory store: %s", e)
|
|
||||||
raise HTTPException(
|
|
||||||
503, "Memory store is temporarily unreadable — no changes were made."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
|
def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
|
||||||
"""Set up memory-related routes."""
|
"""Set up memory-related routes."""
|
||||||
router = APIRouter(prefix="/api/memory", tags=["memory"])
|
router = APIRouter(prefix="/api/memory", tags=["memory"])
|
||||||
@@ -132,7 +116,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
|||||||
new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
|
new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
|
||||||
if memory_data.session_id:
|
if memory_data.session_id:
|
||||||
new_entry["session_id"] = memory_data.session_id
|
new_entry["session_id"] = memory_data.session_id
|
||||||
all_mem = _load_for_update(memory_manager)
|
all_mem = memory_manager.load_all()
|
||||||
all_mem.append(new_entry)
|
all_mem.append(new_entry)
|
||||||
memory_manager.save(all_mem)
|
memory_manager.save(all_mem)
|
||||||
# Sync vector index
|
# Sync vector index
|
||||||
@@ -503,7 +487,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
|||||||
def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
|
def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
|
||||||
"""Pin or unpin a memory. Pinned memories are always included in context."""
|
"""Pin or unpin a memory. Pinned memories are always included in context."""
|
||||||
user = _owner(request)
|
user = _owner(request)
|
||||||
all_mem = _load_for_update(memory_manager)
|
all_mem = memory_manager.load_all()
|
||||||
for i, memory in enumerate(all_mem):
|
for i, memory in enumerate(all_mem):
|
||||||
if memory["id"] == memory_id:
|
if memory["id"] == memory_id:
|
||||||
_verify_memory_owner(memory, user)
|
_verify_memory_owner(memory, user)
|
||||||
@@ -528,7 +512,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
|||||||
def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
|
def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
|
||||||
"""Update an existing memory item with new text and optional category."""
|
"""Update an existing memory item with new text and optional category."""
|
||||||
user = _owner(request)
|
user = _owner(request)
|
||||||
all_mem = _load_for_update(memory_manager)
|
all_mem = memory_manager.load_all()
|
||||||
for i, memory in enumerate(all_mem):
|
for i, memory in enumerate(all_mem):
|
||||||
if memory["id"] == memory_id:
|
if memory["id"] == memory_id:
|
||||||
_verify_memory_owner(memory, user)
|
_verify_memory_owner(memory, user)
|
||||||
@@ -550,7 +534,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
|||||||
def delete_memory(request: Request, memory_id: str):
|
def delete_memory(request: Request, memory_id: str):
|
||||||
"""Delete a memory item by its ID."""
|
"""Delete a memory item by its ID."""
|
||||||
user = _owner(request)
|
user = _owner(request)
|
||||||
all_mem = _load_for_update(memory_manager)
|
all_mem = memory_manager.load_all()
|
||||||
|
|
||||||
# Find and verify ownership before deleting
|
# Find and verify ownership before deleting
|
||||||
target = next((m for m in all_mem if m["id"] == memory_id), None)
|
target = next((m for m in all_mem if m["id"] == memory_id), None)
|
||||||
|
|||||||
@@ -1,5 +0,0 @@
|
|||||||
"""Vault route domain package (slice 2k, #4082/#4071).
|
|
||||||
|
|
||||||
Contains vault_routes.py, migrated from the flat routes/ directory.
|
|
||||||
Backward-compat shim at routes/vault_routes.py re-exports from here.
|
|
||||||
"""
|
|
||||||
@@ -1,242 +0,0 @@
|
|||||||
"""
|
|
||||||
vault_routes.py
|
|
||||||
|
|
||||||
Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
|
|
||||||
Stores the BW_SESSION key in data/vault.json with restrictive permissions.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import shutil
|
|
||||||
import asyncio
|
|
||||||
from pathlib import Path
|
|
||||||
from datetime import datetime
|
|
||||||
from fastapi import APIRouter, Request
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
from core.middleware import require_admin
|
|
||||||
from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
|
|
||||||
from src.constants import VAULT_FILE as _VAULT_FILE
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
VAULT_FILE = Path(_VAULT_FILE)
|
|
||||||
|
|
||||||
|
|
||||||
def _find_bw() -> str:
|
|
||||||
"""Locate the bw binary, checking PATH and common npm-global locations.
|
|
||||||
|
|
||||||
On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
|
|
||||||
which_tool via PATHEXT.
|
|
||||||
"""
|
|
||||||
p = which_tool("bw")
|
|
||||||
if p:
|
|
||||||
return p
|
|
||||||
if IS_WINDOWS:
|
|
||||||
appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
|
|
||||||
for candidate in (
|
|
||||||
os.path.join(appdata, "npm", "bw.cmd"),
|
|
||||||
os.path.join(appdata, "npm", "bw.exe"),
|
|
||||||
):
|
|
||||||
if os.path.isfile(candidate):
|
|
||||||
return candidate
|
|
||||||
return "bw"
|
|
||||||
home = os.path.expanduser("~")
|
|
||||||
for candidate in (
|
|
||||||
f"{home}/.npm-global/bin/bw",
|
|
||||||
f"{home}/.nvm/versions/node/*/bin/bw",
|
|
||||||
"/usr/local/bin/bw",
|
|
||||||
"/opt/homebrew/bin/bw",
|
|
||||||
):
|
|
||||||
if "*" in candidate:
|
|
||||||
import glob
|
|
||||||
for m in glob.glob(candidate):
|
|
||||||
if os.path.isfile(m) and os.access(m, os.X_OK):
|
|
||||||
return m
|
|
||||||
elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
|
||||||
return candidate
|
|
||||||
return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below)
|
|
||||||
|
|
||||||
|
|
||||||
def _load_config() -> dict:
|
|
||||||
if VAULT_FILE.exists():
|
|
||||||
try:
|
|
||||||
data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
|
|
||||||
return data if isinstance(data, dict) else {}
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def _save_config(cfg: dict):
|
|
||||||
VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
|
|
||||||
# POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
|
|
||||||
# is ACL-restricted already).
|
|
||||||
safe_chmod(str(VAULT_FILE), 0o600)
|
|
||||||
|
|
||||||
|
|
||||||
async def _run_bw(args: list, session: str = None, input_text: str = None,
|
|
||||||
bw_password: str = None) -> tuple:
|
|
||||||
env = {}
|
|
||||||
env.update(os.environ)
|
|
||||||
if session:
|
|
||||||
env["BW_SESSION"] = session
|
|
||||||
# Secrets must never be passed as argv — process arguments are world-readable
|
|
||||||
# via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
|
|
||||||
# support for bw commands that need it; unlock/login callers should prefer
|
|
||||||
# stdin so the master password is not left in the child environment either.
|
|
||||||
if bw_password is not None:
|
|
||||||
env["BW_PASSWORD"] = bw_password
|
|
||||||
bw_path = _find_bw()
|
|
||||||
try:
|
|
||||||
proc = await asyncio.create_subprocess_exec(
|
|
||||||
bw_path, *args,
|
|
||||||
stdin=asyncio.subprocess.PIPE if input_text else None,
|
|
||||||
stdout=asyncio.subprocess.PIPE,
|
|
||||||
stderr=asyncio.subprocess.PIPE,
|
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
except FileNotFoundError:
|
|
||||||
return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
|
|
||||||
except Exception as e:
|
|
||||||
return "", f"Failed to launch bw: {e}", 1
|
|
||||||
try:
|
|
||||||
stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
|
|
||||||
except Exception as e:
|
|
||||||
return "", f"bw subprocess error: {e}", 1
|
|
||||||
return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
|
|
||||||
|
|
||||||
|
|
||||||
class VaultConfig(BaseModel):
|
|
||||||
server_url: str = ""
|
|
||||||
email: str = ""
|
|
||||||
|
|
||||||
|
|
||||||
class VaultUnlockRequest(BaseModel):
|
|
||||||
master_password: str
|
|
||||||
|
|
||||||
|
|
||||||
class VaultLoginRequest(BaseModel):
|
|
||||||
email: str
|
|
||||||
master_password: str
|
|
||||||
|
|
||||||
|
|
||||||
def setup_vault_routes():
|
|
||||||
router = APIRouter(prefix="/api/vault", tags=["vault"])
|
|
||||||
|
|
||||||
@router.get("/config")
|
|
||||||
async def get_config(request: Request):
|
|
||||||
"""Return vault config (no sensitive fields)."""
|
|
||||||
require_admin(request)
|
|
||||||
cfg = _load_config()
|
|
||||||
return {
|
|
||||||
"server_url": cfg.get("server_url", ""),
|
|
||||||
"email": cfg.get("email", ""),
|
|
||||||
"unlocked": bool(cfg.get("session")),
|
|
||||||
"unlocked_at": cfg.get("unlocked_at", ""),
|
|
||||||
"bw_installed": await _check_bw_installed(),
|
|
||||||
}
|
|
||||||
|
|
||||||
@router.post("/config")
|
|
||||||
async def save_config(req: VaultConfig, request: Request):
|
|
||||||
"""Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
|
|
||||||
require_admin(request)
|
|
||||||
cfg = _load_config()
|
|
||||||
cfg["server_url"] = req.server_url.strip().rstrip("/")
|
|
||||||
cfg["email"] = req.email.strip()
|
|
||||||
|
|
||||||
if cfg["server_url"]:
|
|
||||||
_, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
|
|
||||||
if rc != 0:
|
|
||||||
return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
|
|
||||||
|
|
||||||
_save_config(cfg)
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
@router.post("/login")
|
|
||||||
async def login(req: VaultLoginRequest, request: Request):
|
|
||||||
"""Log in to Vaultwarden (required once per account)."""
|
|
||||||
require_admin(request)
|
|
||||||
cfg = _load_config()
|
|
||||||
# Update email
|
|
||||||
cfg["email"] = req.email
|
|
||||||
_save_config(cfg)
|
|
||||||
|
|
||||||
stdout, stderr, rc = await _run_bw(
|
|
||||||
["login", req.email, "--raw"],
|
|
||||||
input_text=req.master_password + "\n",
|
|
||||||
)
|
|
||||||
if rc != 0:
|
|
||||||
# Already logged in is OK
|
|
||||||
if "already logged in" in stderr.lower():
|
|
||||||
return {"ok": True, "already": True}
|
|
||||||
return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
|
|
||||||
# bw login --raw prints session key on success (when 2FA disabled)
|
|
||||||
if stdout:
|
|
||||||
cfg["session"] = stdout
|
|
||||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
|
||||||
_save_config(cfg)
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
@router.post("/unlock")
|
|
||||||
async def unlock(req: VaultUnlockRequest, request: Request):
|
|
||||||
"""Unlock the vault and save the session key."""
|
|
||||||
require_admin(request)
|
|
||||||
# Pass the master password on stdin, not argv. argv is visible through
|
|
||||||
# `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
|
|
||||||
# the child process environment.
|
|
||||||
stdout, stderr, rc = await _run_bw(
|
|
||||||
["unlock", "--raw"],
|
|
||||||
input_text=req.master_password + "\n",
|
|
||||||
)
|
|
||||||
if rc != 0:
|
|
||||||
return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
|
|
||||||
session = stdout.strip()
|
|
||||||
if not session:
|
|
||||||
return {"ok": False, "error": "bw returned empty session"}
|
|
||||||
cfg = _load_config()
|
|
||||||
cfg["session"] = session
|
|
||||||
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
|
||||||
_save_config(cfg)
|
|
||||||
return {"ok": True, "message": "Vault unlocked"}
|
|
||||||
|
|
||||||
@router.post("/lock")
|
|
||||||
async def lock(request: Request):
|
|
||||||
"""Lock the vault (clear session from config)."""
|
|
||||||
require_admin(request)
|
|
||||||
cfg = _load_config()
|
|
||||||
cfg.pop("session", None)
|
|
||||||
cfg.pop("unlocked_at", None)
|
|
||||||
_save_config(cfg)
|
|
||||||
# Also tell bw to lock
|
|
||||||
await _run_bw(["lock"])
|
|
||||||
return {"ok": True, "message": "Vault locked"}
|
|
||||||
|
|
||||||
@router.post("/logout")
|
|
||||||
async def logout(request: Request):
|
|
||||||
"""Log out of the Bitwarden CLI completely."""
|
|
||||||
require_admin(request)
|
|
||||||
await _run_bw(["logout"])
|
|
||||||
cfg = _load_config()
|
|
||||||
cfg.pop("session", None)
|
|
||||||
cfg.pop("email", None)
|
|
||||||
cfg.pop("unlocked_at", None)
|
|
||||||
_save_config(cfg)
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
return router
|
|
||||||
|
|
||||||
|
|
||||||
async def _check_bw_installed() -> bool:
|
|
||||||
try:
|
|
||||||
proc = await asyncio.create_subprocess_exec(
|
|
||||||
_find_bw(), "--version",
|
|
||||||
stdout=asyncio.subprocess.PIPE,
|
|
||||||
stderr=asyncio.subprocess.PIPE,
|
|
||||||
)
|
|
||||||
await proc.communicate()
|
|
||||||
return proc.returncode == 0
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
+237
-9
@@ -1,14 +1,242 @@
|
|||||||
"""Backward-compat shim — canonical location is routes/vault/vault_routes.py.
|
"""
|
||||||
|
vault_routes.py
|
||||||
|
|
||||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
|
||||||
that ``import routes.vault_routes``, ``from routes.vault_routes import X``,
|
Stores the BW_SESSION key in data/vault.json with restrictive permissions.
|
||||||
and the ``import ... as vr`` + ``monkeypatch.setattr(vr, ...)`` pattern used
|
|
||||||
by test_vault_password_not_in_argv.py all operate on the *same* object.
|
|
||||||
Keeps existing import paths working after slice 2k (#4082/#4071).
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import sys as _sys
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import asyncio
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
|
from fastapi import APIRouter, Request
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
from routes.vault import vault_routes as _canonical # noqa: F401
|
from core.middleware import require_admin
|
||||||
|
from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
|
||||||
|
from src.constants import VAULT_FILE as _VAULT_FILE
|
||||||
|
|
||||||
_sys.modules[__name__] = _canonical
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
VAULT_FILE = Path(_VAULT_FILE)
|
||||||
|
|
||||||
|
|
||||||
|
def _find_bw() -> str:
|
||||||
|
"""Locate the bw binary, checking PATH and common npm-global locations.
|
||||||
|
|
||||||
|
On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
|
||||||
|
which_tool via PATHEXT.
|
||||||
|
"""
|
||||||
|
p = which_tool("bw")
|
||||||
|
if p:
|
||||||
|
return p
|
||||||
|
if IS_WINDOWS:
|
||||||
|
appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
|
||||||
|
for candidate in (
|
||||||
|
os.path.join(appdata, "npm", "bw.cmd"),
|
||||||
|
os.path.join(appdata, "npm", "bw.exe"),
|
||||||
|
):
|
||||||
|
if os.path.isfile(candidate):
|
||||||
|
return candidate
|
||||||
|
return "bw"
|
||||||
|
home = os.path.expanduser("~")
|
||||||
|
for candidate in (
|
||||||
|
f"{home}/.npm-global/bin/bw",
|
||||||
|
f"{home}/.nvm/versions/node/*/bin/bw",
|
||||||
|
"/usr/local/bin/bw",
|
||||||
|
"/opt/homebrew/bin/bw",
|
||||||
|
):
|
||||||
|
if "*" in candidate:
|
||||||
|
import glob
|
||||||
|
for m in glob.glob(candidate):
|
||||||
|
if os.path.isfile(m) and os.access(m, os.X_OK):
|
||||||
|
return m
|
||||||
|
elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||||
|
return candidate
|
||||||
|
return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_config() -> dict:
|
||||||
|
if VAULT_FILE.exists():
|
||||||
|
try:
|
||||||
|
data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
|
||||||
|
return data if isinstance(data, dict) else {}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _save_config(cfg: dict):
|
||||||
|
VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
|
||||||
|
# POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
|
||||||
|
# is ACL-restricted already).
|
||||||
|
safe_chmod(str(VAULT_FILE), 0o600)
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_bw(args: list, session: str = None, input_text: str = None,
|
||||||
|
bw_password: str = None) -> tuple:
|
||||||
|
env = {}
|
||||||
|
env.update(os.environ)
|
||||||
|
if session:
|
||||||
|
env["BW_SESSION"] = session
|
||||||
|
# Secrets must never be passed as argv — process arguments are world-readable
|
||||||
|
# via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
|
||||||
|
# support for bw commands that need it; unlock/login callers should prefer
|
||||||
|
# stdin so the master password is not left in the child environment either.
|
||||||
|
if bw_password is not None:
|
||||||
|
env["BW_PASSWORD"] = bw_password
|
||||||
|
bw_path = _find_bw()
|
||||||
|
try:
|
||||||
|
proc = await asyncio.create_subprocess_exec(
|
||||||
|
bw_path, *args,
|
||||||
|
stdin=asyncio.subprocess.PIPE if input_text else None,
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.PIPE,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
|
||||||
|
except Exception as e:
|
||||||
|
return "", f"Failed to launch bw: {e}", 1
|
||||||
|
try:
|
||||||
|
stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
|
||||||
|
except Exception as e:
|
||||||
|
return "", f"bw subprocess error: {e}", 1
|
||||||
|
return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
|
||||||
|
|
||||||
|
|
||||||
|
class VaultConfig(BaseModel):
|
||||||
|
server_url: str = ""
|
||||||
|
email: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
class VaultUnlockRequest(BaseModel):
|
||||||
|
master_password: str
|
||||||
|
|
||||||
|
|
||||||
|
class VaultLoginRequest(BaseModel):
|
||||||
|
email: str
|
||||||
|
master_password: str
|
||||||
|
|
||||||
|
|
||||||
|
def setup_vault_routes():
|
||||||
|
router = APIRouter(prefix="/api/vault", tags=["vault"])
|
||||||
|
|
||||||
|
@router.get("/config")
|
||||||
|
async def get_config(request: Request):
|
||||||
|
"""Return vault config (no sensitive fields)."""
|
||||||
|
require_admin(request)
|
||||||
|
cfg = _load_config()
|
||||||
|
return {
|
||||||
|
"server_url": cfg.get("server_url", ""),
|
||||||
|
"email": cfg.get("email", ""),
|
||||||
|
"unlocked": bool(cfg.get("session")),
|
||||||
|
"unlocked_at": cfg.get("unlocked_at", ""),
|
||||||
|
"bw_installed": await _check_bw_installed(),
|
||||||
|
}
|
||||||
|
|
||||||
|
@router.post("/config")
|
||||||
|
async def save_config(req: VaultConfig, request: Request):
|
||||||
|
"""Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
|
||||||
|
require_admin(request)
|
||||||
|
cfg = _load_config()
|
||||||
|
cfg["server_url"] = req.server_url.strip().rstrip("/")
|
||||||
|
cfg["email"] = req.email.strip()
|
||||||
|
|
||||||
|
if cfg["server_url"]:
|
||||||
|
_, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
|
||||||
|
if rc != 0:
|
||||||
|
return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
|
||||||
|
|
||||||
|
_save_config(cfg)
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
@router.post("/login")
|
||||||
|
async def login(req: VaultLoginRequest, request: Request):
|
||||||
|
"""Log in to Vaultwarden (required once per account)."""
|
||||||
|
require_admin(request)
|
||||||
|
cfg = _load_config()
|
||||||
|
# Update email
|
||||||
|
cfg["email"] = req.email
|
||||||
|
_save_config(cfg)
|
||||||
|
|
||||||
|
stdout, stderr, rc = await _run_bw(
|
||||||
|
["login", req.email, "--raw"],
|
||||||
|
input_text=req.master_password + "\n",
|
||||||
|
)
|
||||||
|
if rc != 0:
|
||||||
|
# Already logged in is OK
|
||||||
|
if "already logged in" in stderr.lower():
|
||||||
|
return {"ok": True, "already": True}
|
||||||
|
return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
|
||||||
|
# bw login --raw prints session key on success (when 2FA disabled)
|
||||||
|
if stdout:
|
||||||
|
cfg["session"] = stdout
|
||||||
|
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||||
|
_save_config(cfg)
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
@router.post("/unlock")
|
||||||
|
async def unlock(req: VaultUnlockRequest, request: Request):
|
||||||
|
"""Unlock the vault and save the session key."""
|
||||||
|
require_admin(request)
|
||||||
|
# Pass the master password on stdin, not argv. argv is visible through
|
||||||
|
# `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
|
||||||
|
# the child process environment.
|
||||||
|
stdout, stderr, rc = await _run_bw(
|
||||||
|
["unlock", "--raw"],
|
||||||
|
input_text=req.master_password + "\n",
|
||||||
|
)
|
||||||
|
if rc != 0:
|
||||||
|
return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
|
||||||
|
session = stdout.strip()
|
||||||
|
if not session:
|
||||||
|
return {"ok": False, "error": "bw returned empty session"}
|
||||||
|
cfg = _load_config()
|
||||||
|
cfg["session"] = session
|
||||||
|
cfg["unlocked_at"] = datetime.utcnow().isoformat()
|
||||||
|
_save_config(cfg)
|
||||||
|
return {"ok": True, "message": "Vault unlocked"}
|
||||||
|
|
||||||
|
@router.post("/lock")
|
||||||
|
async def lock(request: Request):
|
||||||
|
"""Lock the vault (clear session from config)."""
|
||||||
|
require_admin(request)
|
||||||
|
cfg = _load_config()
|
||||||
|
cfg.pop("session", None)
|
||||||
|
cfg.pop("unlocked_at", None)
|
||||||
|
_save_config(cfg)
|
||||||
|
# Also tell bw to lock
|
||||||
|
await _run_bw(["lock"])
|
||||||
|
return {"ok": True, "message": "Vault locked"}
|
||||||
|
|
||||||
|
@router.post("/logout")
|
||||||
|
async def logout(request: Request):
|
||||||
|
"""Log out of the Bitwarden CLI completely."""
|
||||||
|
require_admin(request)
|
||||||
|
await _run_bw(["logout"])
|
||||||
|
cfg = _load_config()
|
||||||
|
cfg.pop("session", None)
|
||||||
|
cfg.pop("email", None)
|
||||||
|
cfg.pop("unlocked_at", None)
|
||||||
|
_save_config(cfg)
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
return router
|
||||||
|
|
||||||
|
|
||||||
|
async def _check_bw_installed() -> bool:
|
||||||
|
try:
|
||||||
|
proc = await asyncio.create_subprocess_exec(
|
||||||
|
_find_bw(), "--version",
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.PIPE,
|
||||||
|
)
|
||||||
|
await proc.communicate()
|
||||||
|
return proc.returncode == 0
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|||||||
@@ -1,5 +0,0 @@
|
|||||||
"""Webhook route domain package (slice 2l, #4082/#4071).
|
|
||||||
|
|
||||||
Contains webhook_routes.py, migrated from the flat routes/ directory.
|
|
||||||
Backward-compat shim at routes/webhook_routes.py re-exports from here.
|
|
||||||
"""
|
|
||||||
@@ -1,395 +0,0 @@
|
|||||||
"""Webhook, API Token, and sync chat routes."""
|
|
||||||
|
|
||||||
import uuid
|
|
||||||
import logging
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
from fastapi import APIRouter, HTTPException, Request, Form
|
|
||||||
from pydantic import BaseModel, Field
|
|
||||||
|
|
||||||
from core.database import SessionLocal, Webhook, ModelEndpoint
|
|
||||||
from src.auth_helpers import owner_filter
|
|
||||||
from src.url_security import validate_public_http_url
|
|
||||||
from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api", tags=["webhooks"])
|
|
||||||
|
|
||||||
# Input limits
|
|
||||||
MAX_NAME_LEN = 100
|
|
||||||
MAX_URL_LEN = 2048
|
|
||||||
MAX_SECRET_LEN = 256
|
|
||||||
MAX_MESSAGE_LEN = 32_000
|
|
||||||
|
|
||||||
|
|
||||||
from core.middleware import require_admin as _require_admin
|
|
||||||
|
|
||||||
|
|
||||||
def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
|
|
||||||
"""First enabled ModelEndpoint visible to token_owner — their own rows plus
|
|
||||||
legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
|
|
||||||
let a chat-scoped token fall back onto another user's private endpoint and
|
|
||||||
silently spend that owner's API key/quota. Prefer owner rows before shared
|
|
||||||
rows. Fails closed to null-owner rows only when token_owner is absent.
|
|
||||||
Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
|
|
||||||
"""
|
|
||||||
query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712
|
|
||||||
if token_owner:
|
|
||||||
query = owner_filter(query, ModelEndpoint, token_owner)
|
|
||||||
return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
|
|
||||||
return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711
|
|
||||||
|
|
||||||
|
|
||||||
def _caller_owns_session(sess_owner, caller) -> bool:
|
|
||||||
"""Strict session-ownership gate for the token-authenticated sync-chat
|
|
||||||
endpoint (`POST /api/v1/chat`).
|
|
||||||
|
|
||||||
Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
|
|
||||||
gates in notes/calendar/gallery: a caller may resume a session ONLY when
|
|
||||||
its owner matches them exactly. A null/empty session owner (legacy or
|
|
||||||
migrated rows) is deliberately NOT resumable by an arbitrary token — the
|
|
||||||
old ``sess_owner and sess_owner != caller`` form skipped the check whenever
|
|
||||||
``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
|
|
||||||
device) could resume such a session, inject a message, and read back its
|
|
||||||
history and reuse the owner's endpoint credentials. Fail closed: an
|
|
||||||
unresolvable caller also returns False.
|
|
||||||
"""
|
|
||||||
if not caller:
|
|
||||||
return False
|
|
||||||
return sess_owner == caller
|
|
||||||
|
|
||||||
|
|
||||||
def setup_webhook_routes(
|
|
||||||
webhook_manager: WebhookManager,
|
|
||||||
auth_manager,
|
|
||||||
session_manager=None,
|
|
||||||
api_key_manager=None,
|
|
||||||
) -> APIRouter:
|
|
||||||
|
|
||||||
@router.get("/webhooks")
|
|
||||||
def list_webhooks(request: Request):
|
|
||||||
_require_admin(request)
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
hooks = db.query(Webhook).all()
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"id": w.id,
|
|
||||||
"name": w.name,
|
|
||||||
"url": w.url,
|
|
||||||
"has_secret": bool(w.secret),
|
|
||||||
"events": w.events.split(",") if w.events else [],
|
|
||||||
"is_active": w.is_active,
|
|
||||||
"last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
|
|
||||||
"last_status_code": w.last_status_code,
|
|
||||||
"last_error": w.last_error,
|
|
||||||
"created_at": w.created_at.isoformat() if w.created_at else None,
|
|
||||||
}
|
|
||||||
for w in hooks
|
|
||||||
]
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
@router.post("/webhooks")
|
|
||||||
def create_webhook(
|
|
||||||
request: Request,
|
|
||||||
name: str = Form(""),
|
|
||||||
url: str = Form(""),
|
|
||||||
secret: str = Form(""),
|
|
||||||
events: str = Form(""),
|
|
||||||
):
|
|
||||||
_require_admin(request)
|
|
||||||
name = name.strip()[:MAX_NAME_LEN]
|
|
||||||
if not name:
|
|
||||||
raise HTTPException(400, "Webhook name is required")
|
|
||||||
try:
|
|
||||||
url = validate_webhook_url(url)
|
|
||||||
except ValueError as e:
|
|
||||||
raise HTTPException(400, str(e))
|
|
||||||
try:
|
|
||||||
events = validate_events(events)
|
|
||||||
except ValueError as e:
|
|
||||||
raise HTTPException(400, str(e))
|
|
||||||
|
|
||||||
secret_val = secret.strip()[:MAX_SECRET_LEN] or None
|
|
||||||
# Encrypt the secret at rest using the same Fernet key as API keys
|
|
||||||
encrypted_secret = None
|
|
||||||
if secret_val and api_key_manager:
|
|
||||||
encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
|
|
||||||
elif secret_val:
|
|
||||||
encrypted_secret = secret_val # Fallback if no encryption available
|
|
||||||
|
|
||||||
webhook_id = str(uuid.uuid4())[:8]
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
db.add(Webhook(
|
|
||||||
id=webhook_id,
|
|
||||||
name=name,
|
|
||||||
url=url,
|
|
||||||
secret=encrypted_secret,
|
|
||||||
events=events,
|
|
||||||
is_active=True,
|
|
||||||
))
|
|
||||||
db.commit()
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
return {"id": webhook_id, "name": name}
|
|
||||||
|
|
||||||
@router.post("/webhooks/{webhook_id}/test")
|
|
||||||
async def test_webhook(request: Request, webhook_id: str):
|
|
||||||
_require_admin(request)
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
|
||||||
if not wh:
|
|
||||||
raise HTTPException(404, "Webhook not found")
|
|
||||||
url, secret = wh.url, wh.secret
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
await webhook_manager.deliver_test(webhook_id, url, secret)
|
|
||||||
return {"status": "sent"}
|
|
||||||
|
|
||||||
@router.patch("/webhooks/{webhook_id}")
|
|
||||||
def toggle_webhook(request: Request, webhook_id: str):
|
|
||||||
_require_admin(request)
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
|
||||||
if not wh:
|
|
||||||
raise HTTPException(404, "Webhook not found")
|
|
||||||
wh.is_active = not wh.is_active
|
|
||||||
db.commit()
|
|
||||||
return {"id": webhook_id, "is_active": wh.is_active}
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
@router.delete("/webhooks/{webhook_id}")
|
|
||||||
def delete_webhook(request: Request, webhook_id: str):
|
|
||||||
_require_admin(request)
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
|
|
||||||
db.commit()
|
|
||||||
if not deleted:
|
|
||||||
raise HTTPException(404, "Webhook not found")
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
return {"status": "deleted"}
|
|
||||||
|
|
||||||
# ================================================================
|
|
||||||
# Sync Chat Endpoint (for n8n / Make / Activepieces)
|
|
||||||
# ================================================================
|
|
||||||
|
|
||||||
# Known provider base URLs — auto-resolved from api_key prefix or model name
|
|
||||||
KNOWN_PROVIDERS = {
|
|
||||||
"deepseek": "https://api.deepseek.com/v1",
|
|
||||||
"openai": "https://api.openai.com/v1",
|
|
||||||
"mistral": "https://api.mistral.ai/v1",
|
|
||||||
"groq": "https://api.groq.com/openai/v1",
|
|
||||||
"together": "https://api.together.xyz/v1",
|
|
||||||
"openrouter": "https://openrouter.ai/api/v1",
|
|
||||||
"ollama": "https://ollama.com/api",
|
|
||||||
"opencode-zen": "https://opencode.ai/zen/v1",
|
|
||||||
"opencode-go": "https://opencode.ai/zen/go/v1",
|
|
||||||
"fireworks": "https://api.fireworks.ai/inference/v1",
|
|
||||||
"venice": "https://api.venice.ai/api/v1",
|
|
||||||
"kimi-code": "https://api.kimi.com/coding/v1",
|
|
||||||
"kimicode": "https://api.kimi.com/coding/v1",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Model prefix → provider mapping for auto-detection
|
|
||||||
MODEL_PROVIDER_MAP = {
|
|
||||||
"deepseek": "deepseek",
|
|
||||||
"gpt-": "openai",
|
|
||||||
"o1": "openai",
|
|
||||||
"o3": "openai",
|
|
||||||
"o4": "openai",
|
|
||||||
"mistral": "mistral",
|
|
||||||
"llama": "groq",
|
|
||||||
"mixtral": "groq",
|
|
||||||
"kimi-for-coding": "kimi-code",
|
|
||||||
"kimi": "kimi-code",
|
|
||||||
}
|
|
||||||
|
|
||||||
def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
|
|
||||||
"""Try to auto-resolve a base URL from provider name or model prefix."""
|
|
||||||
if provider and provider.lower() in KNOWN_PROVIDERS:
|
|
||||||
return KNOWN_PROVIDERS[provider.lower()]
|
|
||||||
if model:
|
|
||||||
model_lower = model.lower()
|
|
||||||
for prefix, prov in MODEL_PROVIDER_MAP.items():
|
|
||||||
if model_lower.startswith(prefix):
|
|
||||||
return KNOWN_PROVIDERS[prov]
|
|
||||||
return None
|
|
||||||
|
|
||||||
class SyncChatRequest(BaseModel):
|
|
||||||
message: str = Field(..., max_length=MAX_MESSAGE_LEN)
|
|
||||||
model: Optional[str] = Field(None, max_length=200)
|
|
||||||
session: Optional[str] = Field(None, max_length=100)
|
|
||||||
api_key: Optional[str] = Field(None, max_length=256)
|
|
||||||
base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
|
|
||||||
provider: Optional[str] = Field(None, max_length=50)
|
|
||||||
|
|
||||||
@router.post("/v1/chat")
|
|
||||||
async def sync_chat(request: Request, body: SyncChatRequest):
|
|
||||||
if not getattr(request.state, "api_token", False):
|
|
||||||
raise HTTPException(403, "This endpoint requires an API token")
|
|
||||||
scopes = set(getattr(request.state, "api_token_scopes", []) or [])
|
|
||||||
if "chat" not in scopes:
|
|
||||||
raise HTTPException(403, "API token is not scoped for chat")
|
|
||||||
token_owner = getattr(request.state, "api_token_owner", None)
|
|
||||||
|
|
||||||
from core.models import ChatMessage
|
|
||||||
from src.llm_core import llm_call_async
|
|
||||||
from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
|
|
||||||
|
|
||||||
message = body.message.strip()
|
|
||||||
if not message:
|
|
||||||
raise HTTPException(400, "Message is required")
|
|
||||||
|
|
||||||
session_id = body.session
|
|
||||||
sess = None
|
|
||||||
|
|
||||||
# --- Case 1: Resume an existing session ---
|
|
||||||
if session_id and session_manager:
|
|
||||||
try:
|
|
||||||
sess = session_manager.get_session(session_id)
|
|
||||||
except (KeyError, Exception):
|
|
||||||
raise HTTPException(404, "Session not found")
|
|
||||||
# SECURITY: verify the API-token's user owns this session — without
|
|
||||||
# this any token holder could resume any user's chat by passing its
|
|
||||||
# ID. The token's user is on request.state.user (set by API-token
|
|
||||||
# middleware); fall back to require_user if not present.
|
|
||||||
try:
|
|
||||||
from src.auth_helpers import get_current_user as _gcu
|
|
||||||
_tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
|
|
||||||
except Exception:
|
|
||||||
_tok_user = None
|
|
||||||
# Strict ownership (see _caller_owns_session): fail closed so a
|
|
||||||
# null-owner / cross-owner session can't be resumed by an arbitrary
|
|
||||||
# chat-scoped token.
|
|
||||||
_sess_owner = getattr(sess, "owner", None)
|
|
||||||
if not _caller_owns_session(_sess_owner, _tok_user):
|
|
||||||
raise HTTPException(404, "Session not found")
|
|
||||||
|
|
||||||
# --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
|
|
||||||
if not sess and body.api_key:
|
|
||||||
api_key = body.api_key.strip()
|
|
||||||
model = body.model or "deepseek-chat"
|
|
||||||
|
|
||||||
# Validate only token-supplied direct base_url; auto-resolved known-provider
|
|
||||||
# URLs are not subject to extra local/LAN blocking beyond existing provider logic.
|
|
||||||
direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
|
|
||||||
if direct_base_url:
|
|
||||||
try:
|
|
||||||
base_url = validate_public_http_url(direct_base_url)
|
|
||||||
except ValueError as e:
|
|
||||||
detail = str(e).replace("URL", "base_url", 1)
|
|
||||||
raise HTTPException(400, detail)
|
|
||||||
else:
|
|
||||||
base_url = _resolve_base_url(model, body.provider)
|
|
||||||
if not base_url:
|
|
||||||
raise HTTPException(400,
|
|
||||||
"Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
|
|
||||||
"or provider ('deepseek', 'openai', 'groq', etc.)")
|
|
||||||
base_url = normalize_base(base_url)
|
|
||||||
endpoint_url = build_chat_url(base_url)
|
|
||||||
|
|
||||||
if not session_manager:
|
|
||||||
raise HTTPException(500, "Session manager not available")
|
|
||||||
|
|
||||||
sid = str(uuid.uuid4())
|
|
||||||
sess = session_manager.create_session(
|
|
||||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
|
||||||
model=model, owner=token_owner,
|
|
||||||
)
|
|
||||||
sess.headers = build_headers(api_key, base_url)
|
|
||||||
session_manager.save_sessions()
|
|
||||||
session_id = sid
|
|
||||||
|
|
||||||
# --- Case 3: Fall back to first configured ModelEndpoint ---
|
|
||||||
if not sess:
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
ep = _select_api_chat_fallback_endpoint(db, token_owner)
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
if not ep:
|
|
||||||
raise HTTPException(400,
|
|
||||||
"No session, api_key, or configured endpoints. "
|
|
||||||
"Pass api_key + model, or configure an endpoint in Admin.")
|
|
||||||
|
|
||||||
base_url = normalize_base(ep.base_url)
|
|
||||||
endpoint_url = build_chat_url(base_url)
|
|
||||||
model = body.model or "auto"
|
|
||||||
api_key = ep.api_key
|
|
||||||
if getattr(ep, "provider_auth_id", None):
|
|
||||||
try:
|
|
||||||
from src.endpoint_resolver import resolve_endpoint_runtime
|
|
||||||
base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
|
|
||||||
endpoint_url = build_chat_url(base_url)
|
|
||||||
except Exception:
|
|
||||||
raise HTTPException(500, "Could not resolve endpoint credentials")
|
|
||||||
|
|
||||||
if model == "auto":
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(timeout=5) as client:
|
|
||||||
models_url = build_models_url(base_url)
|
|
||||||
hdrs = build_headers(api_key, base_url)
|
|
||||||
if models_url:
|
|
||||||
resp = await client.get(models_url, headers=hdrs)
|
|
||||||
resp.raise_for_status()
|
|
||||||
data = resp.json()
|
|
||||||
items = data if isinstance(data, list) else (data.get("data") or [])
|
|
||||||
ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
|
|
||||||
if not ids and isinstance(data, dict):
|
|
||||||
ids = [
|
|
||||||
m.get("name") or m.get("model")
|
|
||||||
for m in (data.get("models") or [])
|
|
||||||
if m.get("name") or m.get("model")
|
|
||||||
]
|
|
||||||
else:
|
|
||||||
import json as _json
|
|
||||||
ids = _json.loads(ep.cached_models or "[]")
|
|
||||||
model = ids[0] if ids else "auto"
|
|
||||||
except Exception:
|
|
||||||
raise HTTPException(500, "Could not discover models from endpoint")
|
|
||||||
|
|
||||||
if not session_manager:
|
|
||||||
raise HTTPException(500, "Session manager not available")
|
|
||||||
|
|
||||||
sid = str(uuid.uuid4())
|
|
||||||
sess = session_manager.create_session(
|
|
||||||
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
|
||||||
model=model, owner=token_owner,
|
|
||||||
)
|
|
||||||
if api_key:
|
|
||||||
sess.headers = build_headers(api_key, base_url)
|
|
||||||
session_manager.save_sessions()
|
|
||||||
session_id = sid
|
|
||||||
|
|
||||||
# --- Send message and get response ---
|
|
||||||
sess.add_message(ChatMessage("user", message))
|
|
||||||
|
|
||||||
messages = [{"role": m.role, "content": m.content} for m in sess.history]
|
|
||||||
|
|
||||||
reply = await llm_call_async(
|
|
||||||
sess.endpoint_url, sess.model, messages,
|
|
||||||
headers=sess.headers, timeout=120,
|
|
||||||
)
|
|
||||||
sess.add_message(ChatMessage("assistant", reply))
|
|
||||||
session_manager.save_sessions()
|
|
||||||
|
|
||||||
webhook_manager.fire_and_forget("chat.completed", {
|
|
||||||
"session_id": session_id, "model": sess.model,
|
|
||||||
"user_message": message[:2000], "response": reply[:2000],
|
|
||||||
})
|
|
||||||
|
|
||||||
return {"response": reply, "session_id": session_id, "model": sess.model}
|
|
||||||
|
|
||||||
return router
|
|
||||||
+391
-12
@@ -1,16 +1,395 @@
|
|||||||
"""Backward-compat shim — canonical location is routes/webhook/webhook_routes.py.
|
"""Webhook, API Token, and sync chat routes."""
|
||||||
|
|
||||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
import uuid
|
||||||
that ``import routes.webhook_routes``, ``from routes.webhook_routes import X``,
|
import logging
|
||||||
``importlib.import_module("routes.webhook_routes")``, and the
|
from typing import Optional
|
||||||
``__import__("routes.webhook_routes", fromlist=[...])`` + ``setattr(wh_mod,
|
|
||||||
...)`` pattern used by test_null_owner_gates.py all operate on the *same*
|
|
||||||
object. Keeps existing import paths working after slice 2l (#4082/#4071).
|
|
||||||
Source-introspection tests read the canonical file by path.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys as _sys
|
import httpx
|
||||||
|
from fastapi import APIRouter, HTTPException, Request, Form
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from routes.webhook import webhook_routes as _canonical # noqa: F401
|
from core.database import SessionLocal, Webhook, ModelEndpoint
|
||||||
|
from src.auth_helpers import owner_filter
|
||||||
|
from src.url_security import validate_public_http_url
|
||||||
|
from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
|
||||||
|
|
||||||
_sys.modules[__name__] = _canonical
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api", tags=["webhooks"])
|
||||||
|
|
||||||
|
# Input limits
|
||||||
|
MAX_NAME_LEN = 100
|
||||||
|
MAX_URL_LEN = 2048
|
||||||
|
MAX_SECRET_LEN = 256
|
||||||
|
MAX_MESSAGE_LEN = 32_000
|
||||||
|
|
||||||
|
|
||||||
|
from core.middleware import require_admin as _require_admin
|
||||||
|
|
||||||
|
|
||||||
|
def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
|
||||||
|
"""First enabled ModelEndpoint visible to token_owner — their own rows plus
|
||||||
|
legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
|
||||||
|
let a chat-scoped token fall back onto another user's private endpoint and
|
||||||
|
silently spend that owner's API key/quota. Prefer owner rows before shared
|
||||||
|
rows. Fails closed to null-owner rows only when token_owner is absent.
|
||||||
|
Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
|
||||||
|
"""
|
||||||
|
query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712
|
||||||
|
if token_owner:
|
||||||
|
query = owner_filter(query, ModelEndpoint, token_owner)
|
||||||
|
return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
|
||||||
|
return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711
|
||||||
|
|
||||||
|
|
||||||
|
def _caller_owns_session(sess_owner, caller) -> bool:
|
||||||
|
"""Strict session-ownership gate for the token-authenticated sync-chat
|
||||||
|
endpoint (`POST /api/v1/chat`).
|
||||||
|
|
||||||
|
Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
|
||||||
|
gates in notes/calendar/gallery: a caller may resume a session ONLY when
|
||||||
|
its owner matches them exactly. A null/empty session owner (legacy or
|
||||||
|
migrated rows) is deliberately NOT resumable by an arbitrary token — the
|
||||||
|
old ``sess_owner and sess_owner != caller`` form skipped the check whenever
|
||||||
|
``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
|
||||||
|
device) could resume such a session, inject a message, and read back its
|
||||||
|
history and reuse the owner's endpoint credentials. Fail closed: an
|
||||||
|
unresolvable caller also returns False.
|
||||||
|
"""
|
||||||
|
if not caller:
|
||||||
|
return False
|
||||||
|
return sess_owner == caller
|
||||||
|
|
||||||
|
|
||||||
|
def setup_webhook_routes(
|
||||||
|
webhook_manager: WebhookManager,
|
||||||
|
auth_manager,
|
||||||
|
session_manager=None,
|
||||||
|
api_key_manager=None,
|
||||||
|
) -> APIRouter:
|
||||||
|
|
||||||
|
@router.get("/webhooks")
|
||||||
|
def list_webhooks(request: Request):
|
||||||
|
_require_admin(request)
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
hooks = db.query(Webhook).all()
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"id": w.id,
|
||||||
|
"name": w.name,
|
||||||
|
"url": w.url,
|
||||||
|
"has_secret": bool(w.secret),
|
||||||
|
"events": w.events.split(",") if w.events else [],
|
||||||
|
"is_active": w.is_active,
|
||||||
|
"last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
|
||||||
|
"last_status_code": w.last_status_code,
|
||||||
|
"last_error": w.last_error,
|
||||||
|
"created_at": w.created_at.isoformat() if w.created_at else None,
|
||||||
|
}
|
||||||
|
for w in hooks
|
||||||
|
]
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
@router.post("/webhooks")
|
||||||
|
def create_webhook(
|
||||||
|
request: Request,
|
||||||
|
name: str = Form(""),
|
||||||
|
url: str = Form(""),
|
||||||
|
secret: str = Form(""),
|
||||||
|
events: str = Form(""),
|
||||||
|
):
|
||||||
|
_require_admin(request)
|
||||||
|
name = name.strip()[:MAX_NAME_LEN]
|
||||||
|
if not name:
|
||||||
|
raise HTTPException(400, "Webhook name is required")
|
||||||
|
try:
|
||||||
|
url = validate_webhook_url(url)
|
||||||
|
except ValueError as e:
|
||||||
|
raise HTTPException(400, str(e))
|
||||||
|
try:
|
||||||
|
events = validate_events(events)
|
||||||
|
except ValueError as e:
|
||||||
|
raise HTTPException(400, str(e))
|
||||||
|
|
||||||
|
secret_val = secret.strip()[:MAX_SECRET_LEN] or None
|
||||||
|
# Encrypt the secret at rest using the same Fernet key as API keys
|
||||||
|
encrypted_secret = None
|
||||||
|
if secret_val and api_key_manager:
|
||||||
|
encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
|
||||||
|
elif secret_val:
|
||||||
|
encrypted_secret = secret_val # Fallback if no encryption available
|
||||||
|
|
||||||
|
webhook_id = str(uuid.uuid4())[:8]
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
db.add(Webhook(
|
||||||
|
id=webhook_id,
|
||||||
|
name=name,
|
||||||
|
url=url,
|
||||||
|
secret=encrypted_secret,
|
||||||
|
events=events,
|
||||||
|
is_active=True,
|
||||||
|
))
|
||||||
|
db.commit()
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
return {"id": webhook_id, "name": name}
|
||||||
|
|
||||||
|
@router.post("/webhooks/{webhook_id}/test")
|
||||||
|
async def test_webhook(request: Request, webhook_id: str):
|
||||||
|
_require_admin(request)
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||||
|
if not wh:
|
||||||
|
raise HTTPException(404, "Webhook not found")
|
||||||
|
url, secret = wh.url, wh.secret
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
await webhook_manager.deliver_test(webhook_id, url, secret)
|
||||||
|
return {"status": "sent"}
|
||||||
|
|
||||||
|
@router.patch("/webhooks/{webhook_id}")
|
||||||
|
def toggle_webhook(request: Request, webhook_id: str):
|
||||||
|
_require_admin(request)
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
|
||||||
|
if not wh:
|
||||||
|
raise HTTPException(404, "Webhook not found")
|
||||||
|
wh.is_active = not wh.is_active
|
||||||
|
db.commit()
|
||||||
|
return {"id": webhook_id, "is_active": wh.is_active}
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
@router.delete("/webhooks/{webhook_id}")
|
||||||
|
def delete_webhook(request: Request, webhook_id: str):
|
||||||
|
_require_admin(request)
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
|
||||||
|
db.commit()
|
||||||
|
if not deleted:
|
||||||
|
raise HTTPException(404, "Webhook not found")
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
return {"status": "deleted"}
|
||||||
|
|
||||||
|
# ================================================================
|
||||||
|
# Sync Chat Endpoint (for n8n / Make / Activepieces)
|
||||||
|
# ================================================================
|
||||||
|
|
||||||
|
# Known provider base URLs — auto-resolved from api_key prefix or model name
|
||||||
|
KNOWN_PROVIDERS = {
|
||||||
|
"deepseek": "https://api.deepseek.com/v1",
|
||||||
|
"openai": "https://api.openai.com/v1",
|
||||||
|
"mistral": "https://api.mistral.ai/v1",
|
||||||
|
"groq": "https://api.groq.com/openai/v1",
|
||||||
|
"together": "https://api.together.xyz/v1",
|
||||||
|
"openrouter": "https://openrouter.ai/api/v1",
|
||||||
|
"ollama": "https://ollama.com/api",
|
||||||
|
"opencode-zen": "https://opencode.ai/zen/v1",
|
||||||
|
"opencode-go": "https://opencode.ai/zen/go/v1",
|
||||||
|
"fireworks": "https://api.fireworks.ai/inference/v1",
|
||||||
|
"venice": "https://api.venice.ai/api/v1",
|
||||||
|
"kimi-code": "https://api.kimi.com/coding/v1",
|
||||||
|
"kimicode": "https://api.kimi.com/coding/v1",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Model prefix → provider mapping for auto-detection
|
||||||
|
MODEL_PROVIDER_MAP = {
|
||||||
|
"deepseek": "deepseek",
|
||||||
|
"gpt-": "openai",
|
||||||
|
"o1": "openai",
|
||||||
|
"o3": "openai",
|
||||||
|
"o4": "openai",
|
||||||
|
"mistral": "mistral",
|
||||||
|
"llama": "groq",
|
||||||
|
"mixtral": "groq",
|
||||||
|
"kimi-for-coding": "kimi-code",
|
||||||
|
"kimi": "kimi-code",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
|
||||||
|
"""Try to auto-resolve a base URL from provider name or model prefix."""
|
||||||
|
if provider and provider.lower() in KNOWN_PROVIDERS:
|
||||||
|
return KNOWN_PROVIDERS[provider.lower()]
|
||||||
|
if model:
|
||||||
|
model_lower = model.lower()
|
||||||
|
for prefix, prov in MODEL_PROVIDER_MAP.items():
|
||||||
|
if model_lower.startswith(prefix):
|
||||||
|
return KNOWN_PROVIDERS[prov]
|
||||||
|
return None
|
||||||
|
|
||||||
|
class SyncChatRequest(BaseModel):
|
||||||
|
message: str = Field(..., max_length=MAX_MESSAGE_LEN)
|
||||||
|
model: Optional[str] = Field(None, max_length=200)
|
||||||
|
session: Optional[str] = Field(None, max_length=100)
|
||||||
|
api_key: Optional[str] = Field(None, max_length=256)
|
||||||
|
base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
|
||||||
|
provider: Optional[str] = Field(None, max_length=50)
|
||||||
|
|
||||||
|
@router.post("/v1/chat")
|
||||||
|
async def sync_chat(request: Request, body: SyncChatRequest):
|
||||||
|
if not getattr(request.state, "api_token", False):
|
||||||
|
raise HTTPException(403, "This endpoint requires an API token")
|
||||||
|
scopes = set(getattr(request.state, "api_token_scopes", []) or [])
|
||||||
|
if "chat" not in scopes:
|
||||||
|
raise HTTPException(403, "API token is not scoped for chat")
|
||||||
|
token_owner = getattr(request.state, "api_token_owner", None)
|
||||||
|
|
||||||
|
from core.models import ChatMessage
|
||||||
|
from src.llm_core import llm_call_async
|
||||||
|
from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
|
||||||
|
|
||||||
|
message = body.message.strip()
|
||||||
|
if not message:
|
||||||
|
raise HTTPException(400, "Message is required")
|
||||||
|
|
||||||
|
session_id = body.session
|
||||||
|
sess = None
|
||||||
|
|
||||||
|
# --- Case 1: Resume an existing session ---
|
||||||
|
if session_id and session_manager:
|
||||||
|
try:
|
||||||
|
sess = session_manager.get_session(session_id)
|
||||||
|
except (KeyError, Exception):
|
||||||
|
raise HTTPException(404, "Session not found")
|
||||||
|
# SECURITY: verify the API-token's user owns this session — without
|
||||||
|
# this any token holder could resume any user's chat by passing its
|
||||||
|
# ID. The token's user is on request.state.user (set by API-token
|
||||||
|
# middleware); fall back to require_user if not present.
|
||||||
|
try:
|
||||||
|
from src.auth_helpers import get_current_user as _gcu
|
||||||
|
_tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
|
||||||
|
except Exception:
|
||||||
|
_tok_user = None
|
||||||
|
# Strict ownership (see _caller_owns_session): fail closed so a
|
||||||
|
# null-owner / cross-owner session can't be resumed by an arbitrary
|
||||||
|
# chat-scoped token.
|
||||||
|
_sess_owner = getattr(sess, "owner", None)
|
||||||
|
if not _caller_owns_session(_sess_owner, _tok_user):
|
||||||
|
raise HTTPException(404, "Session not found")
|
||||||
|
|
||||||
|
# --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
|
||||||
|
if not sess and body.api_key:
|
||||||
|
api_key = body.api_key.strip()
|
||||||
|
model = body.model or "deepseek-chat"
|
||||||
|
|
||||||
|
# Validate only token-supplied direct base_url; auto-resolved known-provider
|
||||||
|
# URLs are not subject to extra local/LAN blocking beyond existing provider logic.
|
||||||
|
direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
|
||||||
|
if direct_base_url:
|
||||||
|
try:
|
||||||
|
base_url = validate_public_http_url(direct_base_url)
|
||||||
|
except ValueError as e:
|
||||||
|
detail = str(e).replace("URL", "base_url", 1)
|
||||||
|
raise HTTPException(400, detail)
|
||||||
|
else:
|
||||||
|
base_url = _resolve_base_url(model, body.provider)
|
||||||
|
if not base_url:
|
||||||
|
raise HTTPException(400,
|
||||||
|
"Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
|
||||||
|
"or provider ('deepseek', 'openai', 'groq', etc.)")
|
||||||
|
base_url = normalize_base(base_url)
|
||||||
|
endpoint_url = build_chat_url(base_url)
|
||||||
|
|
||||||
|
if not session_manager:
|
||||||
|
raise HTTPException(500, "Session manager not available")
|
||||||
|
|
||||||
|
sid = str(uuid.uuid4())
|
||||||
|
sess = session_manager.create_session(
|
||||||
|
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||||
|
model=model, owner=token_owner,
|
||||||
|
)
|
||||||
|
sess.headers = build_headers(api_key, base_url)
|
||||||
|
session_manager.save_sessions()
|
||||||
|
session_id = sid
|
||||||
|
|
||||||
|
# --- Case 3: Fall back to first configured ModelEndpoint ---
|
||||||
|
if not sess:
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
ep = _select_api_chat_fallback_endpoint(db, token_owner)
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
if not ep:
|
||||||
|
raise HTTPException(400,
|
||||||
|
"No session, api_key, or configured endpoints. "
|
||||||
|
"Pass api_key + model, or configure an endpoint in Admin.")
|
||||||
|
|
||||||
|
base_url = normalize_base(ep.base_url)
|
||||||
|
endpoint_url = build_chat_url(base_url)
|
||||||
|
model = body.model or "auto"
|
||||||
|
api_key = ep.api_key
|
||||||
|
if getattr(ep, "provider_auth_id", None):
|
||||||
|
try:
|
||||||
|
from src.endpoint_resolver import resolve_endpoint_runtime
|
||||||
|
base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
|
||||||
|
endpoint_url = build_chat_url(base_url)
|
||||||
|
except Exception:
|
||||||
|
raise HTTPException(500, "Could not resolve endpoint credentials")
|
||||||
|
|
||||||
|
if model == "auto":
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=5) as client:
|
||||||
|
models_url = build_models_url(base_url)
|
||||||
|
hdrs = build_headers(api_key, base_url)
|
||||||
|
if models_url:
|
||||||
|
resp = await client.get(models_url, headers=hdrs)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
items = data if isinstance(data, list) else (data.get("data") or [])
|
||||||
|
ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
|
||||||
|
if not ids and isinstance(data, dict):
|
||||||
|
ids = [
|
||||||
|
m.get("name") or m.get("model")
|
||||||
|
for m in (data.get("models") or [])
|
||||||
|
if m.get("name") or m.get("model")
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
import json as _json
|
||||||
|
ids = _json.loads(ep.cached_models or "[]")
|
||||||
|
model = ids[0] if ids else "auto"
|
||||||
|
except Exception:
|
||||||
|
raise HTTPException(500, "Could not discover models from endpoint")
|
||||||
|
|
||||||
|
if not session_manager:
|
||||||
|
raise HTTPException(500, "Session manager not available")
|
||||||
|
|
||||||
|
sid = str(uuid.uuid4())
|
||||||
|
sess = session_manager.create_session(
|
||||||
|
session_id=sid, name="API Chat", endpoint_url=endpoint_url,
|
||||||
|
model=model, owner=token_owner,
|
||||||
|
)
|
||||||
|
if api_key:
|
||||||
|
sess.headers = build_headers(api_key, base_url)
|
||||||
|
session_manager.save_sessions()
|
||||||
|
session_id = sid
|
||||||
|
|
||||||
|
# --- Send message and get response ---
|
||||||
|
sess.add_message(ChatMessage("user", message))
|
||||||
|
|
||||||
|
messages = [{"role": m.role, "content": m.content} for m in sess.history]
|
||||||
|
|
||||||
|
reply = await llm_call_async(
|
||||||
|
sess.endpoint_url, sess.model, messages,
|
||||||
|
headers=sess.headers, timeout=120,
|
||||||
|
)
|
||||||
|
sess.add_message(ChatMessage("assistant", reply))
|
||||||
|
session_manager.save_sessions()
|
||||||
|
|
||||||
|
webhook_manager.fire_and_forget("chat.completed", {
|
||||||
|
"session_id": session_id, "model": sess.model,
|
||||||
|
"user_message": message[:2000], "response": reply[:2000],
|
||||||
|
})
|
||||||
|
|
||||||
|
return {"response": reply, "session_id": session_id, "model": sess.model}
|
||||||
|
|
||||||
|
return router
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
"""Memory service — persistent memory storage and retrieval."""
|
"""Memory service — persistent memory storage and retrieval."""
|
||||||
|
|
||||||
from .service import MemoryService, Memory, MemorySearchResult
|
from .service import MemoryService, Memory, MemorySearchResult
|
||||||
from .memory import MemoryManager, MemoryStoreUnreadable
|
from .memory import MemoryManager
|
||||||
from .memory_vector import MemoryVectorStore
|
from .memory_vector import MemoryVectorStore
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
@@ -10,6 +10,5 @@ __all__ = [
|
|||||||
"Memory",
|
"Memory",
|
||||||
"MemorySearchResult",
|
"MemorySearchResult",
|
||||||
"MemoryManager",
|
"MemoryManager",
|
||||||
"MemoryStoreUnreadable",
|
|
||||||
"MemoryVectorStore",
|
"MemoryVectorStore",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -5,16 +5,6 @@ application runtime instantiates ``src.memory.MemoryManager``, so keeping a
|
|||||||
parallel implementation here risks silent drift between import paths.
|
parallel implementation here risks silent drift between import paths.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from src.memory import (
|
from src.memory import MemoryManager, get_text_similarity, tokenize
|
||||||
MemoryManager,
|
|
||||||
MemoryStoreUnreadable,
|
|
||||||
get_text_similarity,
|
|
||||||
tokenize,
|
|
||||||
)
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = ["MemoryManager", "get_text_similarity", "tokenize"]
|
||||||
"MemoryManager",
|
|
||||||
"MemoryStoreUnreadable",
|
|
||||||
"get_text_similarity",
|
|
||||||
"tokenize",
|
|
||||||
]
|
|
||||||
|
|||||||
@@ -17,8 +17,6 @@ import os
|
|||||||
import re
|
import re
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from src.memory import MemoryStoreUnreadable
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@@ -389,13 +387,7 @@ async def extract_and_store(
|
|||||||
# Get owner from session
|
# Get owner from session
|
||||||
_owner = getattr(session, 'owner', None)
|
_owner = getattr(session, 'owner', None)
|
||||||
|
|
||||||
# Strict load: this is a read-modify-write. Degrading to [] here would
|
existing = memory_manager.load_all()
|
||||||
# save only the newly extracted facts and drop the entire store.
|
|
||||||
try:
|
|
||||||
existing = memory_manager.load_all_for_update()
|
|
||||||
except MemoryStoreUnreadable as e:
|
|
||||||
logger.error("Skipping auto memory extraction, store unreadable: %s", e)
|
|
||||||
return
|
|
||||||
added = 0
|
added = 0
|
||||||
|
|
||||||
for fact in facts:
|
for fact in facts:
|
||||||
@@ -634,18 +626,7 @@ async def audit_memories(
|
|||||||
|
|
||||||
# Merge audited entries back with other users' entries
|
# Merge audited entries back with other users' entries
|
||||||
if owner:
|
if owner:
|
||||||
# Strict load: the merge below reconstructs the whole file. If this
|
all_entries = memory_manager.load_all()
|
||||||
# degraded to [] we would save only this owner's audited slice and
|
|
||||||
# destroy every other tenant's memories.
|
|
||||||
try:
|
|
||||||
all_entries = memory_manager.load_all_for_update()
|
|
||||||
except MemoryStoreUnreadable as e:
|
|
||||||
logger.error("Aborting memory audit save, store unreadable: %s", e)
|
|
||||||
return {
|
|
||||||
"before": before_count,
|
|
||||||
"after": before_count,
|
|
||||||
"error": "store_unreadable",
|
|
||||||
}
|
|
||||||
audited_ids = {e["id"] for e in final_entries}
|
audited_ids = {e["id"] for e in final_entries}
|
||||||
other_entries = [e for e in all_entries if e.get("owner") != owner and (e.get("owner") is not None)]
|
other_entries = [e for e in all_entries if e.get("owner") != owner and (e.get("owner") is not None)]
|
||||||
# Also keep legacy entries that weren't part of this audit
|
# Also keep legacy entries that weren't part of this audit
|
||||||
|
|||||||
+1
-10
@@ -22,7 +22,6 @@ import time
|
|||||||
from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
|
from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
|
||||||
|
|
||||||
from src.constants import GENERATED_IMAGES_DIR
|
from src.constants import GENERATED_IMAGES_DIR
|
||||||
from src.memory import MemoryStoreUnreadable
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -385,15 +384,7 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner
|
|||||||
return {"error": "Memory text cannot be empty"}
|
return {"error": "Memory text cannot be empty"}
|
||||||
|
|
||||||
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
|
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
|
||||||
# Strict load: this is a read-modify-write, and it is the path an
|
memories = _memory_manager.load_all()
|
||||||
# ordinary "remember that I prefer X" takes. Degrading to [] here would
|
|
||||||
# save just this one entry over a store we only failed to read,
|
|
||||||
# atomically destroying every memory in it (issue #5673).
|
|
||||||
try:
|
|
||||||
memories = _memory_manager.load_all_for_update()
|
|
||||||
except MemoryStoreUnreadable as e:
|
|
||||||
logger.error("Refusing to add memory, store unreadable: %s", e)
|
|
||||||
return {"error": "Memory store is temporarily unreadable — nothing was saved."}
|
|
||||||
memories.append(entry)
|
memories.append(entry)
|
||||||
_memory_manager.save(memories)
|
_memory_manager.save(memories)
|
||||||
|
|
||||||
|
|||||||
+3
-172
@@ -1,14 +1,11 @@
|
|||||||
import ipaddress
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import time
|
|
||||||
import uuid
|
import uuid
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
from typing import Dict, List, Optional, Any
|
from typing import Dict, List, Optional, Any
|
||||||
from urllib.parse import urljoin, urlparse, urlunparse
|
from urllib.parse import urljoin, urlparse, urlunparse
|
||||||
|
|
||||||
import httpcore
|
|
||||||
import httpx
|
import httpx
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
|
|
||||||
@@ -357,152 +354,6 @@ def _find_integration(identifier: str) -> Optional[Dict[str, Any]]:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
# httpcore raises its own exception hierarchy; map the ones a simple request can
|
|
||||||
# surface back to their httpx equivalents so the caller's `except httpx.*` blocks
|
|
||||||
# below behave exactly as they did with the default transport.
|
|
||||||
_HTTPCORE_TO_HTTPX_EXC = {
|
|
||||||
httpcore.ConnectError: httpx.ConnectError,
|
|
||||||
httpcore.ConnectTimeout: httpx.ConnectTimeout,
|
|
||||||
httpcore.NetworkError: httpx.NetworkError,
|
|
||||||
httpcore.PoolTimeout: httpx.PoolTimeout,
|
|
||||||
httpcore.ProtocolError: httpx.ProtocolError,
|
|
||||||
httpcore.ReadError: httpx.ReadError,
|
|
||||||
httpcore.ReadTimeout: httpx.ReadTimeout,
|
|
||||||
httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
|
|
||||||
httpcore.TimeoutException: httpx.TimeoutException,
|
|
||||||
httpcore.WriteError: httpx.WriteError,
|
|
||||||
httpcore.WriteTimeout: httpx.WriteTimeout,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class _PinnedAsyncBackend(httpcore.AsyncNetworkBackend):
|
|
||||||
"""Network backend that connects only to the pre-validated IPs, in order.
|
|
||||||
|
|
||||||
Every address here came out of the single SSRF resolution, so moving to the
|
|
||||||
next one after a connect failure is not re-resolution — it's ordinary
|
|
||||||
multi-address fallback restricted to the set the guard already approved.
|
|
||||||
httpcore takes TLS SNI and the ``Host`` header from the request URL rather
|
|
||||||
than the connect host, so pinning the socket destination leaves certificate
|
|
||||||
validation and vhost routing pointed at the original hostname.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, ips: List[ipaddress._BaseAddress]):
|
|
||||||
self._ips = [str(ip) for ip in ips]
|
|
||||||
self._real = httpcore.AnyIOBackend()
|
|
||||||
|
|
||||||
async def connect_tcp(self, host, port, timeout=None, local_address=None,
|
|
||||||
socket_options=None):
|
|
||||||
# One shared connect budget: each attempt gets the time left until the
|
|
||||||
# original deadline, so N dead addresses can't stretch the connect
|
|
||||||
# phase to N * timeout.
|
|
||||||
deadline = None if timeout is None else time.monotonic() + timeout
|
|
||||||
last_exc: Optional[Exception] = None
|
|
||||||
for ip in self._ips:
|
|
||||||
remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
|
|
||||||
try:
|
|
||||||
return await self._real.connect_tcp(
|
|
||||||
ip, port, remaining, local_address, socket_options
|
|
||||||
)
|
|
||||||
except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc:
|
|
||||||
last_exc = exc
|
|
||||||
if deadline is not None and time.monotonic() >= deadline:
|
|
||||||
break
|
|
||||||
raise last_exc
|
|
||||||
|
|
||||||
async def connect_unix_socket(self, path, timeout=None, socket_options=None):
|
|
||||||
return await self._real.connect_unix_socket(path, timeout, socket_options)
|
|
||||||
|
|
||||||
async def sleep(self, seconds: float) -> None:
|
|
||||||
return await self._real.sleep(seconds)
|
|
||||||
|
|
||||||
|
|
||||||
class _PinnedAsyncTransport(httpx.AsyncBaseTransport):
|
|
||||||
"""httpx transport that pins the TCP connect to the pre-resolved IP(s).
|
|
||||||
|
|
||||||
Kept local, mirroring the per-module pinned transports web fetch and
|
|
||||||
webhook delivery already carry, rather than coupling api_call to the
|
|
||||||
webhook subsystem. The request URL passes through unchanged, so SNI and the
|
|
||||||
``Host`` header stay the original hostname; only the socket destination is
|
|
||||||
pinned, which is what closes the rebinding window.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, ips: List[ipaddress._BaseAddress]):
|
|
||||||
self._pinned_ips = list(ips)
|
|
||||||
self._pool = httpcore.AsyncConnectionPool(
|
|
||||||
# Reuse the CA trust the default httpx client would build (certifi
|
|
||||||
# plus SSL_CERT_FILE / SSL_CERT_DIR when trust_env is set) so
|
|
||||||
# swapping in this transport doesn't quietly change which chains
|
|
||||||
# verify. ssl.create_default_context() would use system roots.
|
|
||||||
ssl_context=httpx.create_ssl_context(),
|
|
||||||
http1=True,
|
|
||||||
http2=False,
|
|
||||||
network_backend=_PinnedAsyncBackend(ips),
|
|
||||||
)
|
|
||||||
|
|
||||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
||||||
core_req = httpcore.Request(
|
|
||||||
method=request.method,
|
|
||||||
url=httpcore.URL(
|
|
||||||
scheme=request.url.raw_scheme,
|
|
||||||
host=request.url.raw_host,
|
|
||||||
port=request.url.port,
|
|
||||||
target=request.url.raw_path,
|
|
||||||
),
|
|
||||||
headers=request.headers.raw,
|
|
||||||
content=request.stream,
|
|
||||||
extensions=request.extensions,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
core_resp = await self._pool.handle_async_request(core_req)
|
|
||||||
content = b"".join([chunk async for chunk in core_resp.aiter_stream()])
|
|
||||||
await core_resp.aclose()
|
|
||||||
except Exception as exc:
|
|
||||||
mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
|
|
||||||
if mapped is not None:
|
|
||||||
raise mapped(str(exc)) from exc
|
|
||||||
raise
|
|
||||||
return httpx.Response(
|
|
||||||
status_code=core_resp.status,
|
|
||||||
headers=core_resp.headers,
|
|
||||||
content=content,
|
|
||||||
extensions=core_resp.extensions,
|
|
||||||
)
|
|
||||||
|
|
||||||
async def aclose(self) -> None:
|
|
||||||
await self._pool.aclose()
|
|
||||||
|
|
||||||
|
|
||||||
def _validated_ips(raw_ips: List[str]) -> List[ipaddress._BaseAddress]:
|
|
||||||
"""Return every entry that parses as an IP address, de-duplicated, order
|
|
||||||
preserved.
|
|
||||||
|
|
||||||
check_outbound_url only reports ok when *all* of these classify as safe, so
|
|
||||||
the whole list is guard-approved and any of them is a legitimate connect
|
|
||||||
target. Skipping unparseable entries mirrors how the guard walks the same
|
|
||||||
resolver output.
|
|
||||||
|
|
||||||
De-duplication matters because the resolver is getaddrinfo(host, None) with
|
|
||||||
no socktype filter, so glibc reports the same address once per socktype
|
|
||||||
(SOCK_STREAM/SOCK_DGRAM/SOCK_RAW) — a single-homed host comes back three
|
|
||||||
times. Without this, the connect fallback would spend the shared deadline
|
|
||||||
retrying one dead address instead of moving on to a genuinely different one.
|
|
||||||
"""
|
|
||||||
ips: List[ipaddress._BaseAddress] = []
|
|
||||||
seen = set()
|
|
||||||
for raw in raw_ips:
|
|
||||||
if not isinstance(raw, str):
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
ip = ipaddress.ip_address(raw.split("%")[0]) # strip IPv6 zone id
|
|
||||||
except ValueError:
|
|
||||||
continue
|
|
||||||
if ip in seen:
|
|
||||||
continue
|
|
||||||
seen.add(ip)
|
|
||||||
ips.append(ip)
|
|
||||||
return ips
|
|
||||||
|
|
||||||
|
|
||||||
async def execute_api_call(
|
async def execute_api_call(
|
||||||
integration_id: str,
|
integration_id: str,
|
||||||
method: str,
|
method: str,
|
||||||
@@ -558,31 +409,13 @@ async def execute_api_call(
|
|||||||
# loopback for locked-down deployments. Private stays allowed by default
|
# loopback for locked-down deployments. Private stays allowed by default
|
||||||
# because LAN integrations (Home Assistant, Miniflux, ntfy) are the
|
# because LAN integrations (Home Assistant, Miniflux, ntfy) are the
|
||||||
# primary use case.
|
# primary use case.
|
||||||
from src.url_safety import check_outbound_url, _default_resolver
|
from src.url_safety import check_outbound_url
|
||||||
block_private = os.getenv(
|
block_private = os.getenv(
|
||||||
"INTEGRATION_API_BLOCK_PRIVATE_IPS", "false"
|
"INTEGRATION_API_BLOCK_PRIVATE_IPS", "false"
|
||||||
).lower() == "true"
|
).lower() == "true"
|
||||||
# Resolve the host exactly once and remember the IPs the guard validated so
|
ok, reason = check_outbound_url(url, block_private=block_private)
|
||||||
# the request below can be pinned to them. check_outbound_url only reports
|
|
||||||
# (ok, reason); a plain httpx client re-resolves the host at connect time,
|
|
||||||
# which reopens a DNS-rebinding TOCTOU — a base_url host that answers with a
|
|
||||||
# public IP for the guard and then flips to 169.254.169.254 for the connect
|
|
||||||
# would reach cloud metadata with the integration's auth headers attached.
|
|
||||||
resolved_ips: List[str] = []
|
|
||||||
|
|
||||||
def _recording_resolver(host: str) -> List[str]:
|
|
||||||
ips = _default_resolver(host)
|
|
||||||
resolved_ips[:] = ips
|
|
||||||
return ips
|
|
||||||
|
|
||||||
ok, reason = check_outbound_url(
|
|
||||||
url, block_private=block_private, resolver=_recording_resolver
|
|
||||||
)
|
|
||||||
if not ok:
|
if not ok:
|
||||||
return {"error": f"URL rejected: {reason}", "exit_code": 1}
|
return {"error": f"URL rejected: {reason}", "exit_code": 1}
|
||||||
pinned_ips = _validated_ips(resolved_ips)
|
|
||||||
if not pinned_ips:
|
|
||||||
return {"error": "URL rejected: host did not resolve to a usable address", "exit_code": 1}
|
|
||||||
|
|
||||||
method = method.upper()
|
method = method.upper()
|
||||||
|
|
||||||
@@ -622,9 +455,7 @@ async def execute_api_call(
|
|||||||
auth = httpx.BasicAuth(parts[0], parts[1])
|
auth = httpx.BasicAuth(parts[0], parts[1])
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(
|
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||||
timeout=30.0, transport=_PinnedAsyncTransport(pinned_ips)
|
|
||||||
) as client:
|
|
||||||
response = await client.request(
|
response = await client.request(
|
||||||
method,
|
method,
|
||||||
url,
|
url,
|
||||||
|
|||||||
+9
-79
@@ -10,18 +10,6 @@ from datetime import datetime
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class MemoryStoreUnreadable(RuntimeError):
|
|
||||||
"""memory.json exists on disk but could not be read or parsed.
|
|
||||||
|
|
||||||
"The contents are unknown" is categorically different from "there are no
|
|
||||||
memories". A read-modify-write caller that conflates the two appends to an
|
|
||||||
empty view and then persists it, destroying the whole store — the writes
|
|
||||||
are atomic, so the loss is durable. Raised by
|
|
||||||
:meth:`MemoryManager.load_all_for_update` so those callers fail closed.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def tokenize(text: str) -> List[str]:
|
def tokenize(text: str) -> List[str]:
|
||||||
"""Simple tokenizer that splits on whitespace and removes punctuation."""
|
"""Simple tokenizer that splits on whitespace and removes punctuation."""
|
||||||
return [word.strip('.,!?";') for word in text.split()]
|
return [word.strip('.,!?";') for word in text.split()]
|
||||||
@@ -122,69 +110,21 @@ class MemoryManager:
|
|||||||
with open(self.memory_file, 'w', encoding='utf-8') as f:
|
with open(self.memory_file, 'w', encoding='utf-8') as f:
|
||||||
json.dump([], f, ensure_ascii=False, indent=2)
|
json.dump([], f, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
def _read_entries(self) -> List[Dict]:
|
def load_all(self) -> List[Dict]:
|
||||||
"""Parse the store, or raise :class:`MemoryStoreUnreadable`.
|
"""Load all memory entries from JSON file (unfiltered)."""
|
||||||
|
|
||||||
Returns ``[]`` only when the file genuinely does not exist. Every other
|
|
||||||
failure mode raises, so callers can tell "no memories" apart from
|
|
||||||
"couldn't read the memories".
|
|
||||||
"""
|
|
||||||
if not os.path.exists(self.memory_file):
|
if not os.path.exists(self.memory_file):
|
||||||
return []
|
return []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(self.memory_file, "r", encoding="utf-8") as f:
|
with open(self.memory_file, "r", encoding="utf-8") as f:
|
||||||
data = json.load(f)
|
data = json.load(f)
|
||||||
except OSError as e:
|
if isinstance(data, list):
|
||||||
# PermissionError is an OSError (a scanner holding the file, a
|
return self._validate_entries(data)
|
||||||
# permissions problem, bad media).
|
except (json.JSONDecodeError, PermissionError) as e:
|
||||||
raise MemoryStoreUnreadable(
|
|
||||||
f"cannot read {self.memory_file}: {e}"
|
|
||||||
) from e
|
|
||||||
except json.JSONDecodeError as e:
|
|
||||||
# This is the branch that actually destroyed stores: the file reads
|
|
||||||
# back fine, so nothing stops the save that follows. A truncated
|
|
||||||
# memory.json is reachable because core/database.py rewrites it with
|
|
||||||
# a plain open(..,"w") + json.dump during migration.
|
|
||||||
#
|
|
||||||
# Preserved behaviour: a corrupt store still gets one shot at the
|
|
||||||
# pre-JSON memory.txt migration. Only raise when that finds nothing,
|
|
||||||
# so we never report "empty" for a store we simply failed to parse.
|
|
||||||
legacy = self._migrate_from_legacy()
|
|
||||||
if legacy:
|
|
||||||
return legacy
|
|
||||||
raise MemoryStoreUnreadable(
|
|
||||||
f"{self.memory_file} is not valid JSON: {e}"
|
|
||||||
) from e
|
|
||||||
|
|
||||||
if not isinstance(data, list):
|
|
||||||
raise MemoryStoreUnreadable(
|
|
||||||
f"{self.memory_file} is not a JSON array (got {type(data).__name__})"
|
|
||||||
)
|
|
||||||
return self._validate_entries(data)
|
|
||||||
|
|
||||||
def load_all(self) -> List[Dict]:
|
|
||||||
"""Load all memory entries from JSON file (unfiltered).
|
|
||||||
|
|
||||||
Lenient by design: this feeds display, search, and context-injection
|
|
||||||
paths, so an unreadable store degrades to an empty list rather than
|
|
||||||
breaking chat. Never build a value from this that you intend to save
|
|
||||||
back — use :meth:`load_all_for_update` for that.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
return self._read_entries()
|
|
||||||
except MemoryStoreUnreadable as e:
|
|
||||||
logger.error("Error loading memory.json: %s", e)
|
logger.error("Error loading memory.json: %s", e)
|
||||||
return []
|
return self._migrate_from_legacy()
|
||||||
|
|
||||||
def load_all_for_update(self) -> List[Dict]:
|
return []
|
||||||
"""Load for a read-modify-write cycle.
|
|
||||||
|
|
||||||
Propagates :class:`MemoryStoreUnreadable` instead of degrading to ``[]``
|
|
||||||
so a caller can never append to an empty view and persist it over a
|
|
||||||
store that was only temporarily unreadable (issue #5673).
|
|
||||||
"""
|
|
||||||
return self._read_entries()
|
|
||||||
|
|
||||||
def load(self, owner: str = None) -> List[Dict]:
|
def load(self, owner: str = None) -> List[Dict]:
|
||||||
"""Load memory entries, optionally filtered by owner."""
|
"""Load memory entries, optionally filtered by owner."""
|
||||||
@@ -195,12 +135,7 @@ class MemoryManager:
|
|||||||
|
|
||||||
def claim_ownerless(self, owner: str):
|
def claim_ownerless(self, owner: str):
|
||||||
"""Assign all ownerless memory entries to the given owner."""
|
"""Assign all ownerless memory entries to the given owner."""
|
||||||
try:
|
entries = self.load_all()
|
||||||
entries = self.load_all_for_update()
|
|
||||||
except MemoryStoreUnreadable as e:
|
|
||||||
# Skip the sweep rather than rewrite the store from an unknown view.
|
|
||||||
logger.error("Skipping ownerless claim, memory store unreadable: %s", e)
|
|
||||||
return
|
|
||||||
changed = False
|
changed = False
|
||||||
claimed = 0
|
claimed = 0
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
@@ -300,12 +235,7 @@ class MemoryManager:
|
|||||||
if not ids:
|
if not ids:
|
||||||
return
|
return
|
||||||
id_set = set(ids)
|
id_set = set(ids)
|
||||||
try:
|
entries = self.load_all()
|
||||||
entries = self.load_all_for_update()
|
|
||||||
except MemoryStoreUnreadable as e:
|
|
||||||
# Best-effort counter; never worth rewriting the store blind.
|
|
||||||
logger.error("Skipping uses bump, memory store unreadable: %s", e)
|
|
||||||
return
|
|
||||||
changed = False
|
changed = False
|
||||||
for e in entries:
|
for e in entries:
|
||||||
if e.get("id") in id_set:
|
if e.get("id") in id_set:
|
||||||
|
|||||||
@@ -157,11 +157,7 @@ class NativeMemoryProvider(MemoryProvider):
|
|||||||
if metadata:
|
if metadata:
|
||||||
entry["metadata"] = dict(metadata)
|
entry["metadata"] = dict(metadata)
|
||||||
|
|
||||||
# Strict load: read-modify-write. `load_all` degrades an unreadable
|
memories = self.memory_manager.load_all()
|
||||||
# store to [], which would save this single entry over everything
|
|
||||||
# already stored (issue #5673). The provider API has no error channel,
|
|
||||||
# so MemoryStoreUnreadable propagates to the caller.
|
|
||||||
memories = self.memory_manager.load_all_for_update()
|
|
||||||
memories.append(entry)
|
memories.append(entry)
|
||||||
self.memory_manager.save(memories)
|
self.memory_manager.save(memories)
|
||||||
|
|
||||||
@@ -227,10 +223,7 @@ class NativeMemoryProvider(MemoryProvider):
|
|||||||
]
|
]
|
||||||
|
|
||||||
async def delete(self, memory_id: str, *, owner: Optional[str] = None) -> bool:
|
async def delete(self, memory_id: str, *, owner: Optional[str] = None) -> bool:
|
||||||
# Strict load for the same reason: `remaining` is derived from this
|
memories = self.memory_manager.load_all()
|
||||||
# list and saved back, so it must never be built from a store we
|
|
||||||
# failed to read.
|
|
||||||
memories = self.memory_manager.load_all_for_update()
|
|
||||||
remaining = []
|
remaining = []
|
||||||
deleted_id = None
|
deleted_id = None
|
||||||
|
|
||||||
|
|||||||
+1
-5
@@ -187,12 +187,8 @@ _FUNCTION_MODEL_NAME_RE = re.compile(
|
|||||||
_FUNCTION_MODEL_PARAMS_OPEN_RE = re.compile(r"<parameters>\s*", re.IGNORECASE)
|
_FUNCTION_MODEL_PARAMS_OPEN_RE = re.compile(r"<parameters>\s*", re.IGNORECASE)
|
||||||
_FUNCTION_MODEL_PARAMS_CLOSE_RE = re.compile(r"</parameters>", re.IGNORECASE)
|
_FUNCTION_MODEL_PARAMS_CLOSE_RE = re.compile(r"</parameters>", re.IGNORECASE)
|
||||||
_QWEN_ROLE_MARKER_RE = re.compile(r"</?\|(?:assistant|assistan|user|system|tool)\|>?|</\|end\|>?", re.IGNORECASE)
|
_QWEN_ROLE_MARKER_RE = re.compile(r"</?\|(?:assistant|assistan|user|system|tool)\|>?|</\|end\|>?", re.IGNORECASE)
|
||||||
# At least one pipe is required around `end`. Both pipes used to be optional
|
|
||||||
# (`\|?end\|?`), which also matched a bare `end` on its own line and deleted it
|
|
||||||
# from ordinary prose and from Ruby/Lua/shell snippets that close blocks with
|
|
||||||
# one; see #5547. `|end`, `end|`, `|end|` and `/|end|` still strip as before.
|
|
||||||
_QWEN_BARE_MARKER_RE = re.compile(
|
_QWEN_BARE_MARKER_RE = re.compile(
|
||||||
r"(?:^|[\t\r\n ])(?:/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|"
|
r"(?:^|[\t\r\n ])(?:\|?end\|?|/?\|end\|)(?=[\t\r\n ]|$)|"
|
||||||
r"(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)",
|
r"(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)",
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
|
|||||||
+2
-4
@@ -46,9 +46,7 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict:
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
return {"error": "Invalid JSON arguments", "exit_code": 1}
|
return {"error": "Invalid JSON arguments", "exit_code": 1}
|
||||||
|
|
||||||
action = (args.get("action") or "").strip().lower()
|
action = (args.get("action") or "").lower()
|
||||||
if not action:
|
|
||||||
return {"error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)", "exit_code": 1}
|
|
||||||
from services.memory.skills import SkillsManager
|
from services.memory.skills import SkillsManager
|
||||||
from services.memory.skill_format import Skill, slugify
|
from services.memory.skill_format import Skill, slugify
|
||||||
from src.constants import DATA_DIR
|
from src.constants import DATA_DIR
|
||||||
@@ -57,7 +55,7 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict:
|
|||||||
# Accept legacy `skill_id` as an alias for `name`.
|
# Accept legacy `skill_id` as an alias for `name`.
|
||||||
name = (args.get("name") or args.get("skill_id") or "").strip()
|
name = (args.get("name") or args.get("skill_id") or "").strip()
|
||||||
|
|
||||||
if action in ("list", "index"):
|
if action in ("list", "index", ""):
|
||||||
all_skills = sm.load(owner=owner)
|
all_skills = sm.load(owner=owner)
|
||||||
if not all_skills:
|
if not all_skills:
|
||||||
return {"results": "No skills yet. Create one with action='add'."}
|
return {"results": "No skills yet. Create one with action='add'."}
|
||||||
|
|||||||
+83
-16
@@ -10,14 +10,14 @@ import modelsModule from './js/models.js?v=20260715startupcalm2';
|
|||||||
import ragModule from './js/rag.js';
|
import ragModule from './js/rag.js';
|
||||||
import presetsModule from './js/presets.js';
|
import presetsModule from './js/presets.js';
|
||||||
import searchModule from './js/search.js';
|
import searchModule from './js/search.js';
|
||||||
import chatModule from './js/chat.js?v=20260801fix1';
|
import chatModule from './js/chat.js?v=20260722ctxheader4';
|
||||||
import compareModule from './js/compare/index.js?v=20260723compareicon2';
|
import compareModule from './js/compare/index.js?v=20260723compareicon2';
|
||||||
import documentModule from './js/document.js?v=20260722emailfastindex1';
|
import documentModule from './js/document.js?v=20260722emailfastindex1';
|
||||||
import searchChatModule from './js/search-chat.js';
|
import searchChatModule from './js/search-chat.js';
|
||||||
import { makeWindowDraggable } from './js/windowDrag.js';
|
import { makeWindowDraggable } from './js/windowDrag.js';
|
||||||
import markdownModule from './js/markdown.js';
|
import markdownModule from './js/markdown.js';
|
||||||
import chatRenderer from './js/chatRenderer.js?v=20260722emailfastindex1';
|
import chatRenderer from './js/chatRenderer.js?v=20260722emailfastindex1';
|
||||||
import sessionModule from './js/sessions.js';
|
import sessionModule from './js/sessions.js?v=20260722ctxheader4';
|
||||||
import memoryModule from './js/memory.js?v=20260722memoryloading1';
|
import memoryModule from './js/memory.js?v=20260722memoryloading1';
|
||||||
import voiceRecorderModule from './js/voiceRecorder.js';
|
import voiceRecorderModule from './js/voiceRecorder.js';
|
||||||
import censorModule from './js/censor.js';
|
import censorModule from './js/censor.js';
|
||||||
@@ -1689,20 +1689,12 @@ function initializeEventListeners() {
|
|||||||
|
|
||||||
const newMemoryInput = el('new-memory-input');
|
const newMemoryInput = el('new-memory-input');
|
||||||
if (newMemoryInput) {
|
if (newMemoryInput) {
|
||||||
// keydown, not the deprecated keypress: keypress is not guaranteed to
|
newMemoryInput.addEventListener('keypress', (e) => {
|
||||||
// fire for Enter everywhere, which left the Add Memory form with no
|
if (e.key === 'Enter') {
|
||||||
// working submit path (#5828).
|
|
||||||
newMemoryInput.addEventListener('keydown', (e) => {
|
|
||||||
if (e.key === 'Enter' && !e.isComposing) {
|
|
||||||
e.preventDefault();
|
|
||||||
memoryModule.addNewMemory();
|
memoryModule.addNewMemory();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
const newMemoryAddBtn = el('new-memory-add-btn');
|
|
||||||
if (newMemoryAddBtn) {
|
|
||||||
newMemoryAddBtn.addEventListener('click', () => memoryModule.addNewMemory());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Voice recording is handled by the dual-purpose send/mic button (see below)
|
// Voice recording is handled by the dual-purpose send/mic button (see below)
|
||||||
|
|
||||||
@@ -3916,10 +3908,85 @@ function startOdysseusApp() {
|
|||||||
const messageInput = el('message');
|
const messageInput = el('message');
|
||||||
const modelPickerWrap = document.getElementById('model-picker-wrap');
|
const modelPickerWrap = document.getElementById('model-picker-wrap');
|
||||||
|
|
||||||
// ArrowUp/ArrowDown prompt recall on #message lives in
|
function _readComposerPromptHistory() {
|
||||||
// static/js/composerArrowUpRecall.js (wired from chat.js). Do not re-add a
|
const chatBox = document.getElementById('chat-history');
|
||||||
// copy here: two capture-phase listeners on the same textarea meant the one
|
if (!chatBox) return [];
|
||||||
// without the draft guard won and ate unsent multi-line prompts (#5862).
|
return Array.from(chatBox.querySelectorAll('.msg-user'))
|
||||||
|
.reverse()
|
||||||
|
.map(msg => {
|
||||||
|
const body = msg.querySelector('.body');
|
||||||
|
return msg.dataset?.raw || (body ? body.textContent : '') || '';
|
||||||
|
})
|
||||||
|
.filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (messageInput && !messageInput._odysseusPromptRecallCapture) {
|
||||||
|
messageInput._odysseusPromptRecallCapture = true;
|
||||||
|
let recallHistory = [];
|
||||||
|
let recallIndex = -1;
|
||||||
|
let lastRecalled = '';
|
||||||
|
const norm = (v) => String(v || '').replace(/\r\n/g, '\n').trimEnd();
|
||||||
|
messageInput.addEventListener('input', () => {
|
||||||
|
if (norm(messageInput.value) === norm(lastRecalled)) return;
|
||||||
|
recallHistory = [];
|
||||||
|
recallIndex = -1;
|
||||||
|
lastRecalled = '';
|
||||||
|
try { delete messageInput.dataset.odysseusRecallIndex; } catch {}
|
||||||
|
}, true);
|
||||||
|
messageInput.addEventListener('keydown', (e) => {
|
||||||
|
if (e.key !== 'ArrowUp' && e.key !== 'ArrowDown') return;
|
||||||
|
if (e.shiftKey || e.altKey || e.ctrlKey || e.metaKey || e.isComposing) return;
|
||||||
|
if (window._ghostAutocomplete?.isActive?.()) return;
|
||||||
|
const fresh = _readComposerPromptHistory();
|
||||||
|
const history = fresh.length ? fresh : recallHistory;
|
||||||
|
if (!history.length) return;
|
||||||
|
const current = norm(messageInput.value);
|
||||||
|
let currentIndex = current ? history.findIndex(item => norm(item) === current) : -1;
|
||||||
|
if (current && currentIndex < 0 && current === norm(lastRecalled)) currentIndex = recallIndex;
|
||||||
|
if (current && currentIndex < 0) {
|
||||||
|
const markedIndex = Number(messageInput.dataset.odysseusRecallIndex);
|
||||||
|
if (Number.isInteger(markedIndex) && markedIndex >= 0 && markedIndex < history.length) {
|
||||||
|
currentIndex = markedIndex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
e.preventDefault();
|
||||||
|
e.stopPropagation();
|
||||||
|
e.stopImmediatePropagation();
|
||||||
|
if (e.key === 'ArrowDown') {
|
||||||
|
if (currentIndex < 0) return;
|
||||||
|
const nextIndex = currentIndex - 1;
|
||||||
|
if (nextIndex < 0) {
|
||||||
|
recallHistory = history;
|
||||||
|
recallIndex = -1;
|
||||||
|
lastRecalled = '';
|
||||||
|
try { delete messageInput.dataset.odysseusRecallIndex; } catch {}
|
||||||
|
messageInput.value = '';
|
||||||
|
try { messageInput.selectionStart = messageInput.selectionEnd = 0; } catch {}
|
||||||
|
try { uiModule.autoResize(messageInput); } catch {}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const recalled = history[nextIndex];
|
||||||
|
recallHistory = history;
|
||||||
|
recallIndex = nextIndex;
|
||||||
|
lastRecalled = recalled;
|
||||||
|
try { messageInput.dataset.odysseusRecallIndex = String(nextIndex); } catch {}
|
||||||
|
messageInput.value = recalled;
|
||||||
|
try { messageInput.selectionStart = messageInput.selectionEnd = recalled.length; } catch {}
|
||||||
|
try { uiModule.autoResize(messageInput); } catch {}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const nextIndex = currentIndex >= 0 ? Math.min(currentIndex + 1, history.length - 1) : 0;
|
||||||
|
const recalled = history[nextIndex];
|
||||||
|
if (!recalled) return;
|
||||||
|
recallHistory = history;
|
||||||
|
recallIndex = nextIndex;
|
||||||
|
lastRecalled = recalled;
|
||||||
|
try { messageInput.dataset.odysseusRecallIndex = String(nextIndex); } catch {}
|
||||||
|
messageInput.value = recalled;
|
||||||
|
try { messageInput.selectionStart = messageInput.selectionEnd = recalled.length; } catch {}
|
||||||
|
try { uiModule.autoResize(messageInput); } catch {}
|
||||||
|
}, true);
|
||||||
|
}
|
||||||
|
|
||||||
const _sendIcon = '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><path d="M12 19V5M5 12l7-7 7 7"/></svg>';
|
const _sendIcon = '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><path d="M12 19V5M5 12l7-7 7 7"/></svg>';
|
||||||
const _micIcon = '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"/><path d="M19 10v2a7 7 0 0 1-14 0v-2"/><line x1="12" y1="19" x2="12" y2="23"/><line x1="8" y1="23" x2="16" y2="23"/></svg>';
|
const _micIcon = '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"/><path d="M19 10v2a7 7 0 0 1-14 0v-2"/><line x1="12" y1="19" x2="12" y2="23"/><line x1="8" y1="23" x2="16" y2="23"/></svg>';
|
||||||
|
|||||||
+5
-6
@@ -250,9 +250,9 @@
|
|||||||
</script>
|
</script>
|
||||||
<link rel="stylesheet" href="/static/style.css?v=20260723tasksbulkfeedback1">
|
<link rel="stylesheet" href="/static/style.css?v=20260723tasksbulkfeedback1">
|
||||||
<link rel="modulepreload" href="/static/app.js?v=20260723tasksbulkfeedback1">
|
<link rel="modulepreload" href="/static/app.js?v=20260723tasksbulkfeedback1">
|
||||||
<link rel="modulepreload" href="/static/js/chat.js?v=20260801fix1">
|
<link rel="modulepreload" href="/static/js/chat.js?v=20260722ctxheader4">
|
||||||
<link rel="modulepreload" href="/static/js/ui.js">
|
<link rel="modulepreload" href="/static/js/ui.js">
|
||||||
<link rel="modulepreload" href="/static/js/sessions.js">
|
<link rel="modulepreload" href="/static/js/sessions.js?v=20260722ctxheader4">
|
||||||
<link rel="modulepreload" href="/static/js/markdown.js">
|
<link rel="modulepreload" href="/static/js/markdown.js">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
@@ -365,7 +365,6 @@
|
|||||||
<span class="skill-rich-ph"><span class="k">Add a memory</span> — e.g. 'I prefer concise replies' <svg class="k" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-left:4px;" aria-hidden="true"><polyline points="9 10 4 15 9 20"/><path d="M20 4v7a4 4 0 0 1-4 4H4"/></svg></span>
|
<span class="skill-rich-ph"><span class="k">Add a memory</span> — e.g. 'I prefer concise replies' <svg class="k" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-left:4px;" aria-hidden="true"><polyline points="9 10 4 15 9 20"/><path d="M20 4v7a4 4 0 0 1-4 4H4"/></svg></span>
|
||||||
</div>
|
</div>
|
||||||
<select id="new-memory-category" class="memory-edit-cat-select" aria-label="Memory category"></select>
|
<select id="new-memory-category" class="memory-edit-cat-select" aria-label="Memory category"></select>
|
||||||
<button type="button" id="new-memory-add-btn" class="theme-io-btn" title="Save this memory" style="flex:none;height:28px;font-size:12px;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:4px;" aria-hidden="true"><line x1="12" y1="5" x2="12" y2="19"/><line x1="5" y1="12" x2="19" y2="12"/></svg>Add</button>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="admin-card">
|
<div class="admin-card">
|
||||||
@@ -1006,7 +1005,7 @@
|
|||||||
var tips = mobile ? phone : desktop;
|
var tips = mobile ? phone : desktop;
|
||||||
var el = document.getElementById('welcome-tip');
|
var el = document.getElementById('welcome-tip');
|
||||||
if (el) {
|
if (el) {
|
||||||
el.textContent = tips[Math.floor(Math.random() * tips.length)];
|
el.textContent = 'Pick a model if you want, or just type.';
|
||||||
}
|
}
|
||||||
fetch('/api/version').then(function(r){return r.json()}).then(function(d){
|
fetch('/api/version').then(function(r){return r.json()}).then(function(d){
|
||||||
if (d.version) window._appVersion = d.version;
|
if (d.version) window._appVersion = d.version;
|
||||||
@@ -2505,7 +2504,7 @@
|
|||||||
<script type="module" src="/static/js/ui.js"></script>
|
<script type="module" src="/static/js/ui.js"></script>
|
||||||
<script type="module" src="/static/js/markdown.js"></script>
|
<script type="module" src="/static/js/markdown.js"></script>
|
||||||
<script type="module" src="/static/js/dragSort.js"></script>
|
<script type="module" src="/static/js/dragSort.js"></script>
|
||||||
<script type="module" src="/static/js/sessions.js"></script>
|
<script type="module" src="/static/js/sessions.js?v=20260722ctxheader4"></script>
|
||||||
<script type="module" src="/static/js/memory.js?v=20260722memoryloading1"></script>
|
<script type="module" src="/static/js/memory.js?v=20260722memoryloading1"></script>
|
||||||
<script type="module" src="/static/js/skills.js"></script>
|
<script type="module" src="/static/js/skills.js"></script>
|
||||||
<script type="module" src="/static/js/tourHints.js"></script>
|
<script type="module" src="/static/js/tourHints.js"></script>
|
||||||
@@ -2523,7 +2522,7 @@
|
|||||||
<script type="module" src="/static/js/chatRenderer.js?v=20260722emailfastindex1"></script>
|
<script type="module" src="/static/js/chatRenderer.js?v=20260722emailfastindex1"></script>
|
||||||
<script type="module" src="/static/js/codeRunner.js"></script>
|
<script type="module" src="/static/js/codeRunner.js"></script>
|
||||||
<script type="module" src="/static/js/chatStream.js?v=20260722emailfastindex1"></script>
|
<script type="module" src="/static/js/chatStream.js?v=20260722emailfastindex1"></script>
|
||||||
<script type="module" src="/static/js/chat.js?v=20260801fix1"></script>
|
<script type="module" src="/static/js/chat.js?v=20260722ctxheader4"></script>
|
||||||
<script type="module" src="/static/js/cookbook.js"></script>
|
<script type="module" src="/static/js/cookbook.js"></script>
|
||||||
<script src="/static/js/cookbookSchedule.js"></script>
|
<script src="/static/js/cookbookSchedule.js"></script>
|
||||||
<script type="module" src="/static/js/search-chat.js"></script>
|
<script type="module" src="/static/js/search-chat.js"></script>
|
||||||
|
|||||||
+3
-9
@@ -349,9 +349,6 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr
|
|||||||
|
|
||||||
async function _adoptOpenedSessionBeforeAutoCreate() {
|
async function _adoptOpenedSessionBeforeAutoCreate() {
|
||||||
if (!sessionModule || !sessionModule.getCurrentSessionId || sessionModule.getCurrentSessionId()) return true;
|
if (!sessionModule || !sessionModule.getCurrentSessionId || sessionModule.getCurrentSessionId()) return true;
|
||||||
// Don't adopt a stale session when the user explicitly started a New Chat
|
|
||||||
// (pending state set) — the send path must materialize the pending session.
|
|
||||||
if (sessionModule.hasPendingChat && sessionModule.hasPendingChat()) return false;
|
|
||||||
const activeRowId = document.querySelector('.list-item.active-session[data-session-id], .session-item.active[data-session-id]')?.dataset?.sessionId || '';
|
const activeRowId = document.querySelector('.list-item.active-session[data-session-id], .session-item.active[data-session-id]')?.dataset?.sessionId || '';
|
||||||
const hashId = _hashSessionCandidate();
|
const hashId = _hashSessionCandidate();
|
||||||
const lastSelectedId = String(window.__odysseusLastSelectedSessionId || '').trim();
|
const lastSelectedId = String(window.__odysseusLastSelectedSessionId || '').trim();
|
||||||
@@ -1406,8 +1403,6 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr
|
|||||||
currentAccumulated = '';
|
currentAccumulated = '';
|
||||||
currentHolder = null;
|
currentHolder = null;
|
||||||
|
|
||||||
let abortCtrl = null;
|
|
||||||
let streamingTTS = false;
|
|
||||||
try {
|
try {
|
||||||
// Re-enable auto-scroll when user sends a message
|
// Re-enable auto-scroll when user sends a message
|
||||||
uiModule.setAutoScroll(true);
|
uiModule.setAutoScroll(true);
|
||||||
@@ -1721,7 +1716,7 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
abortCtrl = new AbortController();
|
const abortCtrl = new AbortController();
|
||||||
abortCtrl._reason = '';
|
abortCtrl._reason = '';
|
||||||
currentAbort = abortCtrl;
|
currentAbort = abortCtrl;
|
||||||
|
|
||||||
@@ -1902,7 +1897,7 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr
|
|||||||
let isThinking = false;
|
let isThinking = false;
|
||||||
let thinkingStartTime = null;
|
let thinkingStartTime = null;
|
||||||
// Streaming TTS: synthesize sentence-by-sentence during streaming
|
// Streaming TTS: synthesize sentence-by-sentence during streaming
|
||||||
streamingTTS = !!(window.aiTTSManager && window.aiTTSManager.autoPlay && window.aiTTSManager.available);
|
const streamingTTS = !!(window.aiTTSManager && window.aiTTSManager.autoPlay && window.aiTTSManager.available);
|
||||||
if (streamingTTS) window.aiTTSManager.streamingStart();
|
if (streamingTTS) window.aiTTSManager.streamingStart();
|
||||||
// Multi-bubble agent tracking
|
// Multi-bubble agent tracking
|
||||||
let roundHolder = holder; // Current AI text bubble (changes per round)
|
let roundHolder = holder; // Current AI text bubble (changes per round)
|
||||||
@@ -4792,8 +4787,7 @@ import { wireArrowUpRecall, getUserMessagesFromChatHistory } from './composerArr
|
|||||||
if (msgIndex < 0) return;
|
if (msgIndex < 0) return;
|
||||||
|
|
||||||
const bodyEl = userMsgElement.querySelector('.body');
|
const bodyEl = userMsgElement.querySelector('.body');
|
||||||
let currentText = (userMsgElement.dataset.raw || (bodyEl ? bodyEl.textContent : '') || '').trim();
|
const currentText = bodyEl ? bodyEl.textContent.trim().replace(/\s*\[\d+ attachment\(s\)\]$/, '') : '';
|
||||||
currentText = currentText.replace(/\s*\[\d+ attachment\(s\)\]$/, '');
|
|
||||||
|
|
||||||
// Replace body with an editable textarea
|
// Replace body with an editable textarea
|
||||||
const editor = document.createElement('textarea');
|
const editor = document.createElement('textarea');
|
||||||
|
|||||||
@@ -478,10 +478,7 @@ const DSML_STRAY_RE = /<\s*\/?\s*[||]+\s*DSML\s*[||]+[^>]*>/gi;
|
|||||||
const DSML_INVOKE_RE = /<\s*[||]+\s*DSML\s*[||]+\s*invoke\b[^>]*>[\s\S]*?(?:<\s*\/\s*[||]+\s*DSML\s*[||]+\s*invoke\s*>|$)/gi;
|
const DSML_INVOKE_RE = /<\s*[||]+\s*DSML\s*[||]+\s*invoke\b[^>]*>[\s\S]*?(?:<\s*\/\s*[||]+\s*DSML\s*[||]+\s*invoke\s*>|$)/gi;
|
||||||
const RAW_OPENAI_TOOL_JSON_RE = /(?:\[\s*)?\{\s*"function"\s*:\s*\{[\s\S]*?\}\s*,\s*"id"\s*:\s*"[^"]*"\s*,\s*"type"\s*:\s*"function"\s*\}\s*\]?/gi;
|
const RAW_OPENAI_TOOL_JSON_RE = /(?:\[\s*)?\{\s*"function"\s*:\s*\{[\s\S]*?\}\s*,\s*"id"\s*:\s*"[^"]*"\s*,\s*"type"\s*:\s*"function"\s*\}\s*\]?/gi;
|
||||||
const QWEN_ROLE_MARKER_RE = /<\/?\|(?:assistant|assistan|user|system|tool)\|>?|<\/\|end\|>?/gi;
|
const QWEN_ROLE_MARKER_RE = /<\/?\|(?:assistant|assistan|user|system|tool)\|>?|<\/\|end\|>?/gi;
|
||||||
// Keep in sync with _QWEN_BARE_MARKER_RE in src/tool_parsing.py. At least one
|
const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\|?end\|?|\/?\|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi;
|
||||||
// pipe is required around `end`: with both optional (`\|?end\|?`) this also ate
|
|
||||||
// a bare `end` on its own line, breaking Ruby/Lua/shell snippets (#5547).
|
|
||||||
const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi;
|
|
||||||
// Self-narration about tool results (model echoing stdout/exit_code)
|
// Self-narration about tool results (model echoing stdout/exit_code)
|
||||||
const TOOL_NARRATION_RE = /(?:The (?:result|output) shows?:?\s*)?-?\s*(?:stdout|stderr|exit_code):\s*.+/gi;
|
const TOOL_NARRATION_RE = /(?:The (?:result|output) shows?:?\s*)?-?\s*(?:stdout|stderr|exit_code):\s*.+/gi;
|
||||||
|
|
||||||
|
|||||||
@@ -143,9 +143,9 @@ export function wireArrowUpRecall(composer, getUserMessages, options = {}) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArrowUp walks older prompts. An unmatched draft already returned above,
|
// ArrowUp owns prompt history in the chat composer. If the current text
|
||||||
// so reaching here means the composer is empty or holds a recalled prompt
|
// is not already a recalled prompt, start from newest instead of letting
|
||||||
// — the caret-navigation case is never hijacked.
|
// the browser move the caret inside the textarea.
|
||||||
const nextIndex = currentIndex >= 0 ? Math.min(currentIndex + 1, history.length - 1) : 0;
|
const nextIndex = currentIndex >= 0 ? Math.min(currentIndex + 1, history.length - 1) : 0;
|
||||||
const recalled = history[nextIndex];
|
const recalled = history[nextIndex];
|
||||||
if (!recalled) {
|
if (!recalled) {
|
||||||
|
|||||||
@@ -1847,10 +1847,6 @@ export async function selectSession(id, { keepSidebar = false, showLoading = tru
|
|||||||
const _isTransientChat = !!_meta && (_meta.folder === 'Assistant' || _meta.folder === 'Tasks');
|
const _isTransientChat = !!_meta && (_meta.folder === 'Assistant' || _meta.folder === 'Tasks');
|
||||||
if (!_isTransientChat) {
|
if (!_isTransientChat) {
|
||||||
Storage.set('lastSessionId', id);
|
Storage.set('lastSessionId', id);
|
||||||
// Update URL hash without triggering hashchange handler
|
|
||||||
if (window.location.hash !== '#' + id) {
|
|
||||||
history.replaceState(null, '', '#' + id);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
// Restore character preset for persistent chats
|
// Restore character preset for persistent chats
|
||||||
try {
|
try {
|
||||||
@@ -2317,7 +2313,6 @@ export async function materializePendingSession() {
|
|||||||
currentSessionId = payload.id;
|
currentSessionId = payload.id;
|
||||||
if (!isIncognito) {
|
if (!isIncognito) {
|
||||||
Storage.set('lastSessionId', payload.id);
|
Storage.set('lastSessionId', payload.id);
|
||||||
history.replaceState(null, '', '#' + payload.id);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reload the sidebar in the background. Awaiting this used to block the first
|
// Reload the sidebar in the background. Awaiting this used to block the first
|
||||||
|
|||||||
+5
-3
@@ -83,9 +83,11 @@ export async function loadSkills(cascade = false) {
|
|||||||
// Play the domino-in entrance on this load (set when the tab is opened,
|
// Play the domino-in entrance on this load (set when the tab is opened,
|
||||||
// not for the silent re-loads after an edit/delete).
|
// not for the silent re-loads after an edit/delete).
|
||||||
if (cascade) _cascadeNext = true;
|
if (cascade) _cascadeNext = true;
|
||||||
// Always re-fetch when the tab is explicitly opened — the cascade
|
if (cascade && loaded && !_loadPromise && _playSkillsCascade()) {
|
||||||
// animation is handled inside renderSkillsList() via _cascadeNext.
|
_cascadeNext = false;
|
||||||
// Skipping the fetch here caused stale data on panel close/reopen (#5870).
|
updateCount();
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (_loadPromise) return _loadPromise;
|
if (_loadPromise) return _loadPromise;
|
||||||
_loadPromise = (async () => {
|
_loadPromise = (async () => {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ def _load_webhook_routes_for_test(monkeypatch):
|
|||||||
module_name = "routes.webhook_routes_under_test"
|
module_name = "routes.webhook_routes_under_test"
|
||||||
spec = importlib.util.spec_from_file_location(
|
spec = importlib.util.spec_from_file_location(
|
||||||
module_name,
|
module_name,
|
||||||
Path(__file__).resolve().parent.parent / "routes" / "webhook" / "webhook_routes.py",
|
Path(__file__).resolve().parent.parent / "routes" / "webhook_routes.py",
|
||||||
)
|
)
|
||||||
module = importlib.util.module_from_spec(spec)
|
module = importlib.util.module_from_spec(spec)
|
||||||
spec.loader.exec_module(module)
|
spec.loader.exec_module(module)
|
||||||
|
|||||||
@@ -27,9 +27,6 @@ def _setup(monkeypatch, store, user="alice"):
|
|||||||
|
|
||||||
mem = MagicMock()
|
mem = MagicMock()
|
||||||
mem.load_all.return_value = list(store)
|
mem.load_all.return_value = list(store)
|
||||||
# import_data reads through the strict loader so a store it cannot read is
|
|
||||||
# never overwritten (#5673); the double has to offer the same entry point.
|
|
||||||
mem.load_all_for_update.return_value = list(store)
|
|
||||||
saved = {}
|
saved = {}
|
||||||
mem.save.side_effect = lambda entries: saved.__setitem__("entries", entries)
|
mem.save.side_effect = lambda entries: saved.__setitem__("entries", entries)
|
||||||
|
|
||||||
|
|||||||
@@ -306,24 +306,3 @@ def test_integration_recalls_from_chat_history_dom():
|
|||||||
)
|
)
|
||||||
assert proc.returncode == 0, proc.stderr
|
assert proc.returncode == 0, proc.stderr
|
||||||
assert json.loads(proc.stdout.strip()) == {"value": "stored prompt", "prevented": True}
|
assert json.loads(proc.stdout.strip()) == {"value": "stored prompt", "prevented": True}
|
||||||
|
|
||||||
|
|
||||||
def test_prompt_recall_is_not_duplicated_in_app_js():
|
|
||||||
"""Only composerArrowUpRecall.js may own ArrowUp on #message (issue #5862).
|
|
||||||
|
|
||||||
static/app.js once carried a near-verbatim copy of this recall logic, wired
|
|
||||||
as a second capture-phase listener on the same textarea. That copy lacked
|
|
||||||
the draft guard here, and because it called stopImmediatePropagation it won
|
|
||||||
regardless of registration order — so a typed multi-line prompt was replaced
|
|
||||||
by the last sent one instead of the caret moving up a line.
|
|
||||||
"""
|
|
||||||
app_js = (_REPO / "static" / "app.js").read_text(encoding="utf-8")
|
|
||||||
for marker in (
|
|
||||||
"_odysseusPromptRecallCapture",
|
|
||||||
"_readComposerPromptHistory",
|
|
||||||
"odysseusRecallIndex",
|
|
||||||
):
|
|
||||||
assert marker not in app_js, (
|
|
||||||
f"static/app.js reintroduces prompt recall ({marker!r}); "
|
|
||||||
"it belongs to static/js/composerArrowUpRecall.js alone"
|
|
||||||
)
|
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
"""Regression test for the document route shim (slice 2m, #4082/#4071).
|
|
||||||
|
|
||||||
The backward-compat shims at ``routes/document_routes.py`` and
|
|
||||||
``routes/document_helpers.py`` use ``sys.modules`` replacement so the legacy
|
|
||||||
import paths and the canonical ``routes.document.*`` paths resolve to the
|
|
||||||
*same* module objects. This is required because multiple tests do
|
|
||||||
``import routes.document_routes as droutes`` followed by
|
|
||||||
``droutes.SessionLocal = ...`` / ``monkeypatch.setattr(droutes, ...)`` and
|
|
||||||
``sys.modules.pop("routes.document_helpers")`` + re-import — for those to
|
|
||||||
take effect at runtime, the legacy and canonical module objects must be
|
|
||||||
identical.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import importlib
|
|
||||||
|
|
||||||
import routes.document_routes as _shim_routes # noqa: F401
|
|
||||||
import routes.document_helpers as _shim_helpers # noqa: F401
|
|
||||||
|
|
||||||
|
|
||||||
def test_legacy_and_canonical_routes_are_same_object():
|
|
||||||
legacy = importlib.import_module("routes.document_routes")
|
|
||||||
canonical = importlib.import_module("routes.document.document_routes")
|
|
||||||
assert legacy is canonical
|
|
||||||
|
|
||||||
|
|
||||||
def test_legacy_and_canonical_helpers_are_same_object():
|
|
||||||
legacy = importlib.import_module("routes.document_helpers")
|
|
||||||
canonical = importlib.import_module("routes.document.document_helpers")
|
|
||||||
assert legacy is canonical
|
|
||||||
@@ -87,7 +87,7 @@ def test_known_imap_mailbox_call_sites_are_quoted():
|
|||||||
assert "conn.select(sent_name" not in pollers
|
assert "conn.select(sent_name" not in pollers
|
||||||
assert "imap.append(sent_folder" not in pollers
|
assert "imap.append(sent_folder" not in pollers
|
||||||
|
|
||||||
document_routes = Path("routes/document/document_routes.py").read_text()
|
document_routes = Path("routes/document_routes.py").read_text()
|
||||||
assert "conn.select(doc.source_email_folder" not in document_routes
|
assert "conn.select(doc.source_email_folder" not in document_routes
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -9,13 +9,8 @@ link-local/metadata is always rejected; RFC-1918/loopback only when
|
|||||||
INTEGRATION_API_BLOCK_PRIVATE_IPS=true (LAN integrations are the primary
|
INTEGRATION_API_BLOCK_PRIVATE_IPS=true (LAN integrations are the primary
|
||||||
use case, so private stays allowed by default).
|
use case, so private stays allowed by default).
|
||||||
"""
|
"""
|
||||||
import asyncio
|
|
||||||
import ipaddress
|
|
||||||
import ssl
|
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import httpcore
|
|
||||||
import httpx
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from src import integrations
|
from src import integrations
|
||||||
@@ -102,238 +97,3 @@ async def test_private_base_url_allowed_by_default_blocked_with_knob(monkeypatch
|
|||||||
assert result["exit_code"] == 1
|
assert result["exit_code"] == 1
|
||||||
assert "rejected" in result["error"].lower()
|
assert "rejected" in result["error"].lower()
|
||||||
client.request.assert_not_called()
|
client.request.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
async def _call_capturing_transport(base_url, path="/items"):
|
|
||||||
"""Drive execute_api_call and return (result, transport) where transport is
|
|
||||||
the object passed to httpx.AsyncClient(transport=...)."""
|
|
||||||
resp = MagicMock()
|
|
||||||
resp.status_code = 200
|
|
||||||
resp.headers = {"content-type": "application/json"}
|
|
||||||
resp.json.return_value = {"ok": True}
|
|
||||||
resp.text = '{"ok": true}'
|
|
||||||
|
|
||||||
client = AsyncMock()
|
|
||||||
client.__aenter__ = AsyncMock(return_value=client)
|
|
||||||
client.__aexit__ = AsyncMock(return_value=None)
|
|
||||||
client.request = AsyncMock(return_value=resp)
|
|
||||||
|
|
||||||
captured = {}
|
|
||||||
|
|
||||||
def _fake_async_client(*args, **kwargs):
|
|
||||||
captured.update(kwargs)
|
|
||||||
return client
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch.object(integrations, "_find_integration",
|
|
||||||
return_value=_integration(base_url)),
|
|
||||||
patch("httpx.AsyncClient", side_effect=_fake_async_client),
|
|
||||||
):
|
|
||||||
result = await integrations.execute_api_call("test_integ", "GET", path)
|
|
||||||
return result, captured.get("transport"), client
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_connection_is_pinned_to_the_validated_ip(monkeypatch):
|
|
||||||
"""DNS-rebinding defense: the guard resolves the host once to a benign
|
|
||||||
public IP, and the request must be pinned to *that* IP so a host that
|
|
||||||
rebinds to the metadata range at connect time can't be reached with the
|
|
||||||
integration's auth headers. Static resolution passing the guard is not
|
|
||||||
enough — a plain client would re-resolve at connect."""
|
|
||||||
monkeypatch.setattr("src.url_safety._default_resolver",
|
|
||||||
lambda host: ["93.184.216.34"])
|
|
||||||
result, transport, client = await _call_capturing_transport(
|
|
||||||
"http://rebinding.attacker.example")
|
|
||||||
|
|
||||||
assert result.get("exit_code") == 0
|
|
||||||
client.request.assert_called_once()
|
|
||||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
|
||||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_pin_carries_the_whole_validated_ip_set(monkeypatch):
|
|
||||||
"""When a host resolves to several records the transport keeps all of them
|
|
||||||
(check_outbound_url validated every one), in resolver order, so it can fall
|
|
||||||
back past a dead first address instead of failing the whole call."""
|
|
||||||
monkeypatch.setattr("src.url_safety._default_resolver",
|
|
||||||
lambda host: ["93.184.216.34", "198.51.100.7"])
|
|
||||||
result, transport, _ = await _call_capturing_transport("http://multi.example")
|
|
||||||
|
|
||||||
assert result.get("exit_code") == 0
|
|
||||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34", "198.51.100.7"]
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeStream:
|
|
||||||
"""Stand-in for the connected socket the real backend returns."""
|
|
||||||
|
|
||||||
|
|
||||||
class _RecordingBackend:
|
|
||||||
"""Fake httpcore backend: connect_tcp fails for the addresses in `dead`
|
|
||||||
and succeeds for the rest, recording the order it was asked to connect."""
|
|
||||||
|
|
||||||
def __init__(self, dead):
|
|
||||||
self.dead = set(dead)
|
|
||||||
self.attempts = []
|
|
||||||
|
|
||||||
async def connect_tcp(self, host, port, timeout=None, local_address=None,
|
|
||||||
socket_options=None):
|
|
||||||
self.attempts.append((host, timeout))
|
|
||||||
if host in self.dead:
|
|
||||||
raise httpcore.ConnectError(f"connection refused: {host}")
|
|
||||||
return _FakeStream()
|
|
||||||
|
|
||||||
|
|
||||||
def _pinned_backend(ips, dead):
|
|
||||||
"""A _PinnedAsyncBackend whose underlying connect is the recording fake."""
|
|
||||||
backend = integrations._PinnedAsyncBackend(ips)
|
|
||||||
backend._real = _RecordingBackend(dead)
|
|
||||||
return backend
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_connect_falls_back_from_dead_first_to_live_second():
|
|
||||||
"""first-dead / second-live: the pinned backend must try the next validated
|
|
||||||
address when the first refuses, rather than surfacing the failure. It also
|
|
||||||
ignores the `host` httpcore passes (the original hostname) and connects to
|
|
||||||
the pinned IPs, which is what keeps TLS SNI / Host on the real hostname."""
|
|
||||||
ips = [ipaddress.ip_address("203.0.113.10"), ipaddress.ip_address("198.51.100.7")]
|
|
||||||
backend = _pinned_backend(ips, dead={"203.0.113.10"})
|
|
||||||
|
|
||||||
stream = await backend.connect_tcp("original.hostname.example", 443, timeout=5.0)
|
|
||||||
|
|
||||||
assert isinstance(stream, _FakeStream)
|
|
||||||
# Tried the dead address first, then the live one — never the hostname.
|
|
||||||
assert [host for host, _ in backend._real.attempts] == ["203.0.113.10", "198.51.100.7"]
|
|
||||||
# Fallback shared one budget: the second attempt got the time left, not a fresh 5s.
|
|
||||||
assert backend._real.attempts[1][1] <= 5.0
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_connect_raises_when_every_validated_address_is_dead():
|
|
||||||
ips = [ipaddress.ip_address("203.0.113.10"), ipaddress.ip_address("198.51.100.7")]
|
|
||||||
backend = _pinned_backend(ips, dead={"203.0.113.10", "198.51.100.7"})
|
|
||||||
|
|
||||||
with pytest.raises(httpcore.ConnectError):
|
|
||||||
await backend.connect_tcp("original.hostname.example", 443, timeout=5.0)
|
|
||||||
assert [host for host, _ in backend._real.attempts] == ["203.0.113.10", "198.51.100.7"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_pinned_transport_reuses_httpx_ca_trust(monkeypatch):
|
|
||||||
"""TLS trust must come from the same builder the default httpx client uses
|
|
||||||
(certifi + SSL_CERT_FILE / SSL_CERT_DIR via trust_env), not from
|
|
||||||
ssl.create_default_context()'s system roots — otherwise chains that verified
|
|
||||||
under the old default client can silently stop verifying."""
|
|
||||||
sentinel = ssl.create_default_context()
|
|
||||||
calls = []
|
|
||||||
|
|
||||||
def _fake_create(*args, **kwargs):
|
|
||||||
calls.append(kwargs)
|
|
||||||
return sentinel
|
|
||||||
|
|
||||||
monkeypatch.setattr(httpx, "create_ssl_context", _fake_create)
|
|
||||||
transport = integrations._PinnedAsyncTransport([ipaddress.ip_address("93.184.216.34")])
|
|
||||||
try:
|
|
||||||
assert calls, "transport did not build its context via httpx.create_ssl_context"
|
|
||||||
assert transport._pool._ssl_context is sentinel
|
|
||||||
finally:
|
|
||||||
await transport.aclose()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_real_socket_falls_back_from_dead_first_to_live_second():
|
|
||||||
"""End-to-end over real loopback sockets: pin [127.0.0.2 (nothing
|
|
||||||
listening), 127.0.0.1 (live)], and the request must succeed by falling back
|
|
||||||
to the second address while the Host header stays the original hostname —
|
|
||||||
i.e. only the socket destination moved, vhost/SNI routing did not."""
|
|
||||||
captured = {}
|
|
||||||
|
|
||||||
async def handle(reader, writer):
|
|
||||||
request = await reader.read(4096)
|
|
||||||
for line in request.split(b"\r\n"):
|
|
||||||
if line.lower().startswith(b"host:"):
|
|
||||||
captured["host"] = line.split(b":", 1)[1].strip().decode()
|
|
||||||
writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nhi")
|
|
||||||
await writer.drain()
|
|
||||||
writer.close()
|
|
||||||
|
|
||||||
server = await asyncio.start_server(handle, "127.0.0.1", 0)
|
|
||||||
port = server.sockets[0].getsockname()[1]
|
|
||||||
async with server:
|
|
||||||
await server.start_serving()
|
|
||||||
transport = integrations._PinnedAsyncTransport(
|
|
||||||
[ipaddress.ip_address("127.0.0.2"), ipaddress.ip_address("127.0.0.1")]
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(transport=transport) as client:
|
|
||||||
resp = await client.get(f"http://pinned.example:{port}/health")
|
|
||||||
finally:
|
|
||||||
await transport.aclose()
|
|
||||||
|
|
||||||
assert resp.status_code == 200
|
|
||||||
assert resp.text == "hi"
|
|
||||||
assert captured.get("host") == f"pinned.example:{port}"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_ip_literal_base_url_still_pins_and_is_not_rejected():
|
|
||||||
"""A base_url that is already an IP has nothing to rebind, but it must not
|
|
||||||
trip the "did not resolve" guard either.
|
|
||||||
|
|
||||||
check_outbound_url resolves even a literal (getaddrinfo returns the address
|
|
||||||
itself), so the captured list is populated and the pin is a no-op rather
|
|
||||||
than a rejection. Uses the real resolver on purpose — no monkeypatch — so
|
|
||||||
this would catch the fail-closed branch firing on a literal.
|
|
||||||
"""
|
|
||||||
result, transport, client = await _call_capturing_transport(
|
|
||||||
"http://93.184.216.34")
|
|
||||||
|
|
||||||
assert result.get("exit_code") == 0
|
|
||||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
|
||||||
assert [str(ip) for ip in transport._pinned_ips] == ["93.184.216.34"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_ipv6_base_url_pins_every_validated_address(monkeypatch):
|
|
||||||
"""IPv6 goes down the same path as v4.
|
|
||||||
|
|
||||||
Resolution is stubbed rather than using a literal so this doesn't depend on
|
|
||||||
the runner having IPv6 configured.
|
|
||||||
"""
|
|
||||||
v6 = "2606:2800:220:1:248:1893:25c8:1946"
|
|
||||||
monkeypatch.setattr("src.url_safety._default_resolver", lambda host: [v6])
|
|
||||||
result, transport, client = await _call_capturing_transport("http://v6.example")
|
|
||||||
|
|
||||||
assert result.get("exit_code") == 0
|
|
||||||
assert isinstance(transport, integrations._PinnedAsyncTransport)
|
|
||||||
assert [str(ip) for ip in transport._pinned_ips] == [v6]
|
|
||||||
|
|
||||||
|
|
||||||
def test_validated_ips_strips_zone_id_and_drops_junk():
|
|
||||||
"""getaddrinfo can hand back a scoped v6 address like 'fe80::1%eth0'."""
|
|
||||||
got = integrations._validated_ips(
|
|
||||||
["93.184.216.34", "fe80::1%eth0", "not-an-ip", None, "2001:db8::5"]
|
|
||||||
)
|
|
||||||
assert [str(ip) for ip in got] == ["93.184.216.34", "fe80::1", "2001:db8::5"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_validated_ips_deduplicates_repeated_addresses():
|
|
||||||
"""The resolver is getaddrinfo(host, None) with no socktype filter, so glibc
|
|
||||||
returns one record per socktype and a single-homed host arrives three times
|
|
||||||
over. Duplicates must collapse (first-seen order kept) or the connect
|
|
||||||
fallback wastes its shared deadline retrying one dead address."""
|
|
||||||
got = integrations._validated_ips(
|
|
||||||
["93.184.216.34", "93.184.216.34", "93.184.216.34"]
|
|
||||||
)
|
|
||||||
assert [str(ip) for ip in got] == ["93.184.216.34"]
|
|
||||||
|
|
||||||
# Order is first-seen, and distinct addresses all survive.
|
|
||||||
got = integrations._validated_ips(
|
|
||||||
["198.51.100.7", "93.184.216.34", "198.51.100.7", "2001:db8::5"]
|
|
||||||
)
|
|
||||||
assert [str(ip) for ip in got] == ["198.51.100.7", "93.184.216.34", "2001:db8::5"]
|
|
||||||
|
|
||||||
# A zone-id variant is the same address once stripped, so it collapses too.
|
|
||||||
got = integrations._validated_ips(["fe80::1%eth0", "fe80::1%eth1", "fe80::1"])
|
|
||||||
assert [str(ip) for ip in got] == ["fe80::1"]
|
|
||||||
|
|||||||
@@ -83,10 +83,9 @@ async def _call(json_data, status=200):
|
|||||||
with (
|
with (
|
||||||
patch.object(integrations, "_find_integration", return_value=DUMMY_INTEGRATION),
|
patch.object(integrations, "_find_integration", return_value=DUMMY_INTEGRATION),
|
||||||
patch("httpx.AsyncClient", return_value=mock_client),
|
patch("httpx.AsyncClient", return_value=mock_client),
|
||||||
# api.example.com doesn't resolve. Point the resolver at a public
|
# api.example.com doesn't resolve; the SSRF guard would fail closed.
|
||||||
# address instead of stubbing the guard open, so the real check (and
|
# These tests are about truncation, so stub the guard open.
|
||||||
# the connect-IP pinning that reads its result) still runs.
|
patch("src.url_safety.check_outbound_url", return_value=(True, "ok")),
|
||||||
patch("src.url_safety._default_resolver", lambda host: ["93.184.216.34"]),
|
|
||||||
):
|
):
|
||||||
return await integrations.execute_api_call("test_integ", "GET", "/items")
|
return await integrations.execute_api_call("test_integ", "GET", "/items")
|
||||||
|
|
||||||
@@ -102,10 +101,9 @@ async def _call_with_integration(integration, path="/items"):
|
|||||||
with (
|
with (
|
||||||
patch.object(integrations, "_find_integration", return_value=integration),
|
patch.object(integrations, "_find_integration", return_value=integration),
|
||||||
patch("httpx.AsyncClient", return_value=mock_client),
|
patch("httpx.AsyncClient", return_value=mock_client),
|
||||||
# api.example.com doesn't resolve. Point the resolver at a public
|
# api.example.com doesn't resolve; the SSRF guard would fail closed.
|
||||||
# address instead of stubbing the guard open, so the real check (and
|
# These tests are about URL joining, so stub the guard open.
|
||||||
# the connect-IP pinning that reads its result) still runs.
|
patch("src.url_safety.check_outbound_url", return_value=(True, "ok")),
|
||||||
patch("src.url_safety._default_resolver", lambda host: ["93.184.216.34"]),
|
|
||||||
):
|
):
|
||||||
result = await integrations.execute_api_call("test_integ", "GET", path)
|
result = await integrations.execute_api_call("test_integ", "GET", path)
|
||||||
return result, mock_client
|
return result, mock_client
|
||||||
|
|||||||
@@ -1,24 +0,0 @@
|
|||||||
import json
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from src.tools.system import do_manage_skills
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"payload",
|
|
||||||
[
|
|
||||||
{},
|
|
||||||
{"action": ""},
|
|
||||||
{"action": " "},
|
|
||||||
{"name": "demo", "description": "x", "procedure": ["step"]},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
async def test_manage_skills_requires_action(payload):
|
|
||||||
result = await do_manage_skills(json.dumps(payload), owner="test")
|
|
||||||
|
|
||||||
assert result == {
|
|
||||||
"error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)",
|
|
||||||
"exit_code": 1,
|
|
||||||
}
|
|
||||||
@@ -1,54 +0,0 @@
|
|||||||
"""The Brain > Add Memory form must be submittable (#5828).
|
|
||||||
|
|
||||||
The form previously had no submit button and relied on a deprecated
|
|
||||||
``keypress`` listener for Enter, which is not guaranteed to fire on all
|
|
||||||
platforms — leaving the form with no working submit path. Pins:
|
|
||||||
|
|
||||||
- a visible, keyboard-accessible submit button next to the category select;
|
|
||||||
- the button wired to ``memoryModule.addNewMemory()``;
|
|
||||||
- Enter handled via ``keydown`` with ``preventDefault()`` (and no lingering
|
|
||||||
``keypress`` handler on the input).
|
|
||||||
"""
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
APP_JS = Path("static/app.js")
|
|
||||||
INDEX_HTML = Path("static/index.html")
|
|
||||||
|
|
||||||
|
|
||||||
def _add_memory_row(html):
|
|
||||||
start = html.index('id="new-memory-input"')
|
|
||||||
end = html.index("</div>", html.index('id="new-memory-add-btn"', start))
|
|
||||||
return html[start:end]
|
|
||||||
|
|
||||||
|
|
||||||
def test_add_memory_form_renders_a_submit_button():
|
|
||||||
html = INDEX_HTML.read_text()
|
|
||||||
row = _add_memory_row(html)
|
|
||||||
|
|
||||||
assert 'id="new-memory-category"' in row, "button must sit in the same row as the form fields"
|
|
||||||
btn_start = row.index('id="new-memory-add-btn"')
|
|
||||||
btn_tag = row[row.rindex("<button", 0, btn_start):row.index(">", btn_start)]
|
|
||||||
assert 'type="button"' in btn_tag, "must not rely on implicit submit semantics"
|
|
||||||
|
|
||||||
|
|
||||||
def _new_memory_wiring_block(source):
|
|
||||||
start = source.index("const newMemoryInput = el('new-memory-input');")
|
|
||||||
end = source.index("// Voice recording", start)
|
|
||||||
return source[start:end]
|
|
||||||
|
|
||||||
|
|
||||||
def test_submit_button_is_wired_to_add_new_memory():
|
|
||||||
block = _new_memory_wiring_block(APP_JS.read_text())
|
|
||||||
|
|
||||||
assert "el('new-memory-add-btn')" in block
|
|
||||||
assert "addEventListener('click', () => memoryModule.addNewMemory())" in block
|
|
||||||
|
|
||||||
|
|
||||||
def test_enter_uses_keydown_with_prevent_default():
|
|
||||||
block = _new_memory_wiring_block(APP_JS.read_text())
|
|
||||||
|
|
||||||
assert "addEventListener('keydown'" in block
|
|
||||||
assert "addEventListener('keypress'" not in block, "keypress is deprecated and unreliable for Enter"
|
|
||||||
assert "e.preventDefault();" in block
|
|
||||||
assert "!e.isComposing" in block, "IME composition must not submit the form"
|
|
||||||
assert "memoryModule.addNewMemory();" in block
|
|
||||||
@@ -67,12 +67,6 @@ class FakeMemoryManager:
|
|||||||
def load_all(self):
|
def load_all(self):
|
||||||
return list(self.rows)
|
return list(self.rows)
|
||||||
|
|
||||||
def load_all_for_update(self):
|
|
||||||
# Mirrors the real MemoryManager: extraction is a read-modify-write and
|
|
||||||
# goes through the strict loader (#5673). A healthy store behaves the
|
|
||||||
# same as load_all.
|
|
||||||
return list(self.rows)
|
|
||||||
|
|
||||||
def load(self, owner=None):
|
def load(self, owner=None):
|
||||||
return [r for r in self.rows if r.get("owner") == owner]
|
return [r for r in self.rows if r.get("owner") == owner]
|
||||||
|
|
||||||
|
|||||||
@@ -1,255 +0,0 @@
|
|||||||
"""A memory store that cannot be READ must never be overwritten (issue #5673).
|
|
||||||
|
|
||||||
`MemoryManager.save` is atomic, and the add/import/extract paths are all
|
|
||||||
read-modify-write: load the whole store, append, save it back. `load_all`
|
|
||||||
used to answer a *failed read* with `[]` — indistinguishable from "no
|
|
||||||
memories" — so a failed read turned into
|
|
||||||
|
|
||||||
load_all() -> [] -> [].append(new) -> save([new])
|
|
||||||
|
|
||||||
which atomically replaced the entire store with one entry.
|
|
||||||
|
|
||||||
The trigger that actually bites is a store that is **readable but not
|
|
||||||
parseable** — a truncated file, or one holding `{}` instead of `[]`. Nothing
|
|
||||||
obstructs the write, so the request succeeds with HTTP 200 and every existing
|
|
||||||
memory is destroyed silently. Truncation is reachable: `core/database.py`
|
|
||||||
rewrites memory.json during migration with a plain `open(..., "w")` +
|
|
||||||
`json.dump`, which is not atomic.
|
|
||||||
|
|
||||||
A live exclusive lock is NOT the dangerous case: it blocks the read and the
|
|
||||||
`os.replace` alike, so the save fails too and the store survives (verified
|
|
||||||
end-to-end — clean dev returns 500 there and loses nothing).
|
|
||||||
|
|
||||||
`load_all_for_update` is the strict loader those callers now use: it raises
|
|
||||||
`MemoryStoreUnreadable` rather than reporting an empty store.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import builtins
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from src.memory import MemoryManager, MemoryStoreUnreadable
|
|
||||||
|
|
||||||
_SEED = [
|
|
||||||
{"id": "m1", "text": "user prefers dark mode", "owner": "alice"},
|
|
||||||
{"id": "m2", "text": "user lives in Berlin", "owner": "alice"},
|
|
||||||
{"id": "m3", "text": "bob's cat is called Mila", "owner": "bob"},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _seeded(tmp_path):
|
|
||||||
m = MemoryManager(str(tmp_path))
|
|
||||||
m.save([dict(e) for e in _SEED])
|
|
||||||
return m
|
|
||||||
|
|
||||||
|
|
||||||
def _break_reads_of(monkeypatch, target, exc):
|
|
||||||
"""Make open() raise `exc` for `target` only, leaving every other path alone."""
|
|
||||||
real_open = builtins.open
|
|
||||||
|
|
||||||
def fake_open(file, mode="r", *args, **kwargs):
|
|
||||||
if os.path.abspath(str(file)) == os.path.abspath(target) and "r" in mode:
|
|
||||||
raise exc
|
|
||||||
return real_open(file, mode, *args, **kwargs)
|
|
||||||
|
|
||||||
monkeypatch.setattr(builtins, "open", fake_open)
|
|
||||||
|
|
||||||
|
|
||||||
# ── the strict loader signals, rather than reporting "empty" ──────────────
|
|
||||||
|
|
||||||
def test_strict_load_raises_on_permission_error(tmp_path, monkeypatch):
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
_break_reads_of(monkeypatch, m.memory_file, PermissionError(13, "locked"))
|
|
||||||
with pytest.raises(MemoryStoreUnreadable):
|
|
||||||
m.load_all_for_update()
|
|
||||||
|
|
||||||
|
|
||||||
def test_strict_load_raises_on_corrupt_json(tmp_path):
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
|
||||||
f.write('[{"id": "m1", "text": "truncated mid-writ')
|
|
||||||
with pytest.raises(MemoryStoreUnreadable):
|
|
||||||
m.load_all_for_update()
|
|
||||||
|
|
||||||
|
|
||||||
def test_strict_load_raises_when_store_is_not_a_list(tmp_path):
|
|
||||||
# A file holding `{}` or `null` is not an empty store, it is a broken one.
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
|
||||||
json.dump({}, f)
|
|
||||||
with pytest.raises(MemoryStoreUnreadable):
|
|
||||||
m.load_all_for_update()
|
|
||||||
|
|
||||||
|
|
||||||
def test_strict_load_returns_entries_when_healthy(tmp_path):
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
assert {e["id"] for e in m.load_all_for_update()} == {"m1", "m2", "m3"}
|
|
||||||
|
|
||||||
|
|
||||||
def test_strict_load_returns_empty_when_file_genuinely_absent(tmp_path):
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
os.remove(m.memory_file)
|
|
||||||
# Absent is the one case that legitimately means "no memories yet".
|
|
||||||
assert m.load_all_for_update() == []
|
|
||||||
|
|
||||||
|
|
||||||
# ── read paths stay lenient, so an unreadable store can't break chat ──────
|
|
||||||
|
|
||||||
def test_read_path_still_degrades_to_empty(tmp_path, monkeypatch):
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
_break_reads_of(monkeypatch, m.memory_file, PermissionError(13, "locked"))
|
|
||||||
# Context injection / search must not raise; they just see nothing.
|
|
||||||
assert m.load_all() == []
|
|
||||||
assert m.load(owner="alice") == []
|
|
||||||
|
|
||||||
|
|
||||||
# ── the actual #5673 regression: the store survives ───────────────────────
|
|
||||||
|
|
||||||
def test_add_cycle_under_transient_read_error_does_not_wipe(tmp_path, monkeypatch):
|
|
||||||
"""Mirrors routes/memory/memory_routes.py api_add_memory exactly."""
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
new_entry = m.add_entry("a brand new fact", owner="alice")
|
|
||||||
|
|
||||||
with monkeypatch.context() as mp:
|
|
||||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
|
||||||
with pytest.raises(MemoryStoreUnreadable):
|
|
||||||
all_mem = m.load_all_for_update()
|
|
||||||
all_mem.append(new_entry)
|
|
||||||
m.save(all_mem)
|
|
||||||
|
|
||||||
# Reads work again; every original memory is still there and the file was
|
|
||||||
# never replaced by the single new entry.
|
|
||||||
assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"}
|
|
||||||
|
|
||||||
|
|
||||||
def test_audit_merge_cannot_drop_other_tenants(tmp_path, monkeypatch):
|
|
||||||
"""The audit path rebuilds the whole file from load_all + one owner's slice.
|
|
||||||
|
|
||||||
Reading [] there would save only the audited owner's entries and destroy
|
|
||||||
every other tenant's memories, so it has to fail closed too.
|
|
||||||
"""
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
alice_slice = [e for e in _SEED if e["owner"] == "alice"]
|
|
||||||
|
|
||||||
with monkeypatch.context() as mp:
|
|
||||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
|
||||||
with pytest.raises(MemoryStoreUnreadable):
|
|
||||||
all_entries = m.load_all_for_update()
|
|
||||||
others = [e for e in all_entries if e.get("owner") != "alice"]
|
|
||||||
m.save(alice_slice + others)
|
|
||||||
|
|
||||||
assert any(e["id"] == "m3" for e in m.load_all()), "bob's memory was destroyed"
|
|
||||||
|
|
||||||
|
|
||||||
def test_uses_bump_skips_write_when_unreadable(tmp_path, monkeypatch):
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
with monkeypatch.context() as mp:
|
|
||||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
|
||||||
m.increment_uses(["m1"]) # must not raise, must not write
|
|
||||||
assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"}
|
|
||||||
|
|
||||||
|
|
||||||
def test_claim_ownerless_skips_write_when_unreadable(tmp_path, monkeypatch):
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
with monkeypatch.context() as mp:
|
|
||||||
_break_reads_of(mp, m.memory_file, PermissionError(13, "locked"))
|
|
||||||
m.claim_ownerless("alice")
|
|
||||||
assert {e["id"] for e in m.load_all()} == {"m1", "m2", "m3"}
|
|
||||||
|
|
||||||
|
|
||||||
# ── the add sinks users actually reach ────────────────────────────────────
|
|
||||||
#
|
|
||||||
# The tests above replay the read-modify-write shape. These drive the real
|
|
||||||
# entry points end to end, because those are what #5673 reports: "remember
|
|
||||||
# that I prefer X" in ordinary chat (src/ai_interaction.py do_manage_memory,
|
|
||||||
# routed from src/tool_execution.py) and the built-in memory MCP server
|
|
||||||
# (mcp_servers/memory_server.py, registered in src/builtin_mcp.py).
|
|
||||||
#
|
|
||||||
# They use a truncated store rather than a read error on purpose: it reads
|
|
||||||
# fine, so nothing stops the save, which is the case that silently destroyed
|
|
||||||
# stores. The assertion is that the file is left byte-identical — still broken,
|
|
||||||
# but still holding the user's memories, so it can be repaired by hand.
|
|
||||||
|
|
||||||
|
|
||||||
def _truncated_store(tmp_path):
|
|
||||||
"""Seed a store that reads back fine but no longer parses."""
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
good = json.dumps([dict(e) for e in _SEED], indent=2)
|
|
||||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
|
||||||
f.write(good[:good.rindex("]")]) # drop the closing bracket only
|
|
||||||
with open(m.memory_file, "rb") as f:
|
|
||||||
return m, f.read()
|
|
||||||
|
|
||||||
|
|
||||||
def _on_disk(manager) -> bytes:
|
|
||||||
with open(manager.memory_file, "rb") as f:
|
|
||||||
return f.read()
|
|
||||||
|
|
||||||
|
|
||||||
def test_agent_memory_add_does_not_overwrite_unreadable_store(tmp_path, monkeypatch):
|
|
||||||
"""src/ai_interaction.py do_manage_memory, action "add"."""
|
|
||||||
from src import ai_interaction
|
|
||||||
|
|
||||||
manager, before = _truncated_store(tmp_path)
|
|
||||||
monkeypatch.setattr(ai_interaction, "_memory_manager", manager)
|
|
||||||
monkeypatch.setattr(ai_interaction, "_memory_vector", None)
|
|
||||||
|
|
||||||
result = asyncio.run(ai_interaction.do_manage_memory("add\nuser prefers tabs"))
|
|
||||||
|
|
||||||
assert _on_disk(manager) == before, "the unreadable store was overwritten"
|
|
||||||
assert b"m3" in _on_disk(manager)
|
|
||||||
assert "error" in result, "the add reported success over an unreadable store"
|
|
||||||
|
|
||||||
|
|
||||||
def test_mcp_memory_add_does_not_overwrite_unreadable_store(tmp_path, monkeypatch):
|
|
||||||
"""mcp_servers/memory_server.py, action "add"."""
|
|
||||||
import mcp_servers.memory_server as memory_server
|
|
||||||
|
|
||||||
manager, before = _truncated_store(tmp_path)
|
|
||||||
monkeypatch.setattr(memory_server, "_memory_manager", manager)
|
|
||||||
monkeypatch.setattr(memory_server, "_memory_vector", None)
|
|
||||||
monkeypatch.setattr(memory_server, "_initialized", True)
|
|
||||||
for key in memory_server._OWNER_ENV_KEYS:
|
|
||||||
monkeypatch.delenv(key, raising=False)
|
|
||||||
|
|
||||||
result = asyncio.run(memory_server.call_tool(
|
|
||||||
"manage_memory", {"action": "add", "text": "user prefers tabs"}
|
|
||||||
))
|
|
||||||
|
|
||||||
assert _on_disk(manager) == before, "the unreadable store was overwritten"
|
|
||||||
assert b"m3" in _on_disk(manager)
|
|
||||||
assert result[0].text.startswith("Error:")
|
|
||||||
|
|
||||||
|
|
||||||
def test_native_provider_remember_does_not_overwrite_unreadable_store(tmp_path):
|
|
||||||
"""src/memory_provider.py NativeMemoryProvider.remember.
|
|
||||||
|
|
||||||
Registered into app state in src/app_initializer.py but not yet consumed
|
|
||||||
outside tests, so this is the pattern held in place before it goes live.
|
|
||||||
"""
|
|
||||||
from src.memory_provider import NativeMemoryProvider
|
|
||||||
|
|
||||||
manager, before = _truncated_store(tmp_path)
|
|
||||||
provider = NativeMemoryProvider(manager)
|
|
||||||
|
|
||||||
with pytest.raises(MemoryStoreUnreadable):
|
|
||||||
asyncio.run(provider.remember("user prefers tabs", owner="alice"))
|
|
||||||
|
|
||||||
assert _on_disk(manager) == before
|
|
||||||
|
|
||||||
|
|
||||||
# ── the legacy memory.txt migration is preserved ──────────────────────────
|
|
||||||
|
|
||||||
def test_corrupt_store_still_migrates_from_legacy_txt(tmp_path):
|
|
||||||
m = _seeded(tmp_path)
|
|
||||||
with open(m.memory_file, "w", encoding="utf-8") as f:
|
|
||||||
f.write("{ not json")
|
|
||||||
legacy = os.path.join(str(tmp_path), "memory.txt")
|
|
||||||
with open(legacy, "w", encoding="utf-8") as f:
|
|
||||||
f.write("recovered fact one\nrecovered fact two\n")
|
|
||||||
|
|
||||||
entries = m.load_all_for_update()
|
|
||||||
assert [e["text"] for e in entries] == ["recovered fact one", "recovered fact two"]
|
|
||||||
@@ -14,7 +14,7 @@ def _function_source(path: str, name: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def test_document_ai_tidy_resolves_with_owner_scope():
|
def test_document_ai_tidy_resolves_with_owner_scope():
|
||||||
body = _function_source("routes/document/document_routes.py", "ai_tidy_documents")
|
body = _function_source("routes/document_routes.py", "ai_tidy_documents")
|
||||||
assert "resolve_task_endpoint(owner=user or None)" in body
|
assert "resolve_task_endpoint(owner=user or None)" in body
|
||||||
assert 'resolve_endpoint("default", owner=user or None)' in body
|
assert 'resolve_endpoint("default", owner=user or None)' in body
|
||||||
|
|
||||||
|
|||||||
@@ -1,96 +0,0 @@
|
|||||||
"""Regression: the Qwen bare-marker scrub must not eat a lone `end` (#5547).
|
|
||||||
|
|
||||||
`_QWEN_BARE_MARKER_RE` cleans Qwen turn markers that leak into content. Its
|
|
||||||
`end` branch was `\\|?end\\|?` — both pipes optional — so it also matched a bare
|
|
||||||
`end` surrounded by whitespace and replaced it with a space. Any message
|
|
||||||
containing Ruby, Lua or shell code that closes a block with a lone `end` had
|
|
||||||
those lines silently deleted, in the stored text and in the rendered message.
|
|
||||||
|
|
||||||
Requiring at least one pipe keeps every real marker (`|end`, `end|`, `|end|`,
|
|
||||||
`/|end|`) stripping as before. The same pattern is duplicated in
|
|
||||||
static/js/chatRenderer.js, so the JS copy is checked here too — the two must
|
|
||||||
not drift.
|
|
||||||
"""
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
import src.agent_tools # noqa: F401 (break agent_tools<->tool_parsing import cycle)
|
|
||||||
from src.tool_parsing import strip_tool_blocks
|
|
||||||
|
|
||||||
_REPO = Path(__file__).resolve().parent.parent
|
|
||||||
_CHAT_RENDERER = _REPO / "static" / "js" / "chatRenderer.js"
|
|
||||||
|
|
||||||
# Inputs that must survive untouched, and the substring that proves they did.
|
|
||||||
KEPT = [
|
|
||||||
("loop do\n puts \"yo\"\nend\n", "\nend"), # the reported Ruby case
|
|
||||||
("if x then\nend", "\nend"),
|
|
||||||
("function f()\nend\n", "\nend"),
|
|
||||||
("a end b", "a end b"),
|
|
||||||
("append end", "append end"),
|
|
||||||
("END", "END"),
|
|
||||||
("\nEnd\n", "End"),
|
|
||||||
]
|
|
||||||
|
|
||||||
# Real markers — at least one pipe, plus the role word — with the exact output
|
|
||||||
# they must still produce. Asserted as equality rather than "marker not in out"
|
|
||||||
# so narrowing the pattern can't pass by deleting more than it should.
|
|
||||||
STRIPPED = [
|
|
||||||
("a |end| b", "a b"),
|
|
||||||
("a /|end| b", "a b"),
|
|
||||||
("a |end b", "a b"),
|
|
||||||
("a end| b", "a b"),
|
|
||||||
("x assistant y", "x y"),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("text,kept", KEPT)
|
|
||||||
def test_bare_end_survives_stripping(text, kept):
|
|
||||||
assert kept in strip_tool_blocks(text)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("text,expected", STRIPPED)
|
|
||||||
def test_piped_end_markers_are_still_stripped(text, expected):
|
|
||||||
assert strip_tool_blocks(text) == expected
|
|
||||||
|
|
||||||
|
|
||||||
def test_bare_end_inside_a_fenced_block_survives():
|
|
||||||
"""The scrub runs over the whole message, fenced regions included."""
|
|
||||||
out = strip_tool_blocks("Here:\n```ruby\nloop do\n puts 1\nend\n```\nDone.")
|
|
||||||
assert "\nend\n" in out
|
|
||||||
|
|
||||||
|
|
||||||
def _js_bare_marker_regex_source():
|
|
||||||
src = _CHAT_RENDERER.read_text(encoding="utf-8")
|
|
||||||
m = re.search(r"^const QWEN_BARE_MARKER_RE = (/.*/[gimsuy]*);$", src, re.MULTILINE)
|
|
||||||
assert m, "QWEN_BARE_MARKER_RE literal not found in chatRenderer.js"
|
|
||||||
return m.group(1)
|
|
||||||
|
|
||||||
|
|
||||||
def test_js_copy_of_the_pattern_matches_the_python_one():
|
|
||||||
"""Guard the duplication: the JS branch must require a pipe too."""
|
|
||||||
if shutil.which("node") is None:
|
|
||||||
pytest.skip("node binary not on PATH")
|
|
||||||
|
|
||||||
cases = [text for text, _ in KEPT] + [text for text, _ in STRIPPED]
|
|
||||||
script = (
|
|
||||||
"const RE = %s;\n"
|
|
||||||
"const cases = JSON.parse(process.argv[1]);\n"
|
|
||||||
"console.log(JSON.stringify(cases.map(c => c.replace(RE, ' '))));"
|
|
||||||
% _js_bare_marker_regex_source()
|
|
||||||
)
|
|
||||||
result = subprocess.run(
|
|
||||||
["node", "--input-type=module", "-e", script, json.dumps(cases)],
|
|
||||||
cwd=_REPO, capture_output=True, timeout=15, text=True,
|
|
||||||
)
|
|
||||||
assert result.returncode == 0, f"node failed:\n{result.stderr}"
|
|
||||||
got = json.loads(result.stdout.splitlines()[-1])
|
|
||||||
|
|
||||||
for (text, kept), out in zip(KEPT, got):
|
|
||||||
assert kept in out, f"JS regex dropped {kept!r} from {text!r}"
|
|
||||||
for (text, expected), out in zip(STRIPPED, got[len(KEPT):]):
|
|
||||||
assert out == expected, f"JS regex: {text!r} -> {out!r}, expected {expected!r}"
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
"""Regression test for the vault route shim (slice 2k, #4082/#4071)."""
|
|
||||||
|
|
||||||
import importlib
|
|
||||||
|
|
||||||
import routes.vault_routes as _shim_vault # noqa: F401
|
|
||||||
|
|
||||||
|
|
||||||
def test_legacy_and_canonical_vault_module_are_same_object():
|
|
||||||
legacy = importlib.import_module("routes.vault_routes")
|
|
||||||
canonical = importlib.import_module("routes.vault.vault_routes")
|
|
||||||
assert legacy is canonical
|
|
||||||
@@ -88,7 +88,7 @@ def test_request_vision_call_sites_pass_owner():
|
|||||||
chat_source = (ROOT / "src" / "chat_handler.py").read_text()
|
chat_source = (ROOT / "src" / "chat_handler.py").read_text()
|
||||||
processor_source = (ROOT / "src" / "document_processor.py").read_text()
|
processor_source = (ROOT / "src" / "document_processor.py").read_text()
|
||||||
upload_source = (ROOT / "routes" / "upload_routes.py").read_text()
|
upload_source = (ROOT / "routes" / "upload_routes.py").read_text()
|
||||||
document_source = (ROOT / "routes" / "document" / "document_routes.py").read_text()
|
document_source = (ROOT / "routes" / "document_routes.py").read_text()
|
||||||
gallery_source = (ROOT / "routes" / "gallery" / "gallery_routes.py").read_text()
|
gallery_source = (ROOT / "routes" / "gallery" / "gallery_routes.py").read_text()
|
||||||
memory_source = (ROOT / "routes" / "memory" / "memory_routes.py").read_text()
|
memory_source = (ROOT / "routes" / "memory" / "memory_routes.py").read_text()
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
"""Regression test for the webhook route shim (slice 2l, #4082/#4071)."""
|
|
||||||
|
|
||||||
import importlib
|
|
||||||
|
|
||||||
import routes.webhook_routes as _shim_webhook # noqa: F401
|
|
||||||
|
|
||||||
|
|
||||||
def test_legacy_and_canonical_webhook_module_are_same_object():
|
|
||||||
legacy = importlib.import_module("routes.webhook_routes")
|
|
||||||
canonical = importlib.import_module("routes.webhook.webhook_routes")
|
|
||||||
assert legacy is canonical
|
|
||||||
Reference in New Issue
Block a user