fix(routes): normalize session owner fallback helpers (#4313)

* fix(memory): normalize import session fallback

* fix(chat): use token owner for compaction scope

* fix(background): honor session endpoint fallback

.dockerignore

@@ -10,6 +10,16 @@ dist/
 build/
 .env
 .env.bak.*
+# Secrets: keep plaintext and every transient secrets.env variant out of
+# the build context. If an encrypted secrets.env is used, it is mounted
+# at runtime — never baked into the image. Mirrored in .gitignore.
+secrets.env
+secrets.env.*
+secrets.env~
+.secrets.env.swp
+.secrets.env.swo
+**/#secrets.env#
+!secrets.env.example
 /data/
 /logs/
 .git/

.env.example

@@ -190,3 +190,10 @@ SEARXNG_INSTANCE=http://localhost:8080
 # These overlays only expose the GPU devices. The slim Odysseus image
 # still needs CUDA/ROCm userspace via Cookbook -> Dependencies (vLLM,
 # llama-cpp-python, etc.) before models can actually serve on GPU.
+
+# ============================================================
+# Storage Paths (Docker Compose)
+# ============================================================
+
+# APP_DATA_DIR=./data
+# APP_LOGS_DIR=./logs

.github/CODEOWNERS

@@ -0,0 +1,9 @@
+# Code owners.
+#
+# Intentionally empty for now. The catch-all rule that mapped every path to a
+# single owner froze all merges the moment "Require review from Code Owners"
+# was enabled, because no other maintainer's approval could satisfy the gate.
+# A per-area ownership map (security/auth, CI, frontend, agent internals, with
+# multiple named owners per line) is being worked out in issue #593; once
+# agreed it replaces this file. Until then, required reviews and the security
+# CI gate (docs/security-ci.md) remain in force via branch protection.

.github/dependabot.yml

@@ -0,0 +1,48 @@
+# Dependabot keeps dependencies and pinned action versions current.
+#
+# Why this matters for security: every workflow in this repo pins its GitHub
+# Actions to an exact commit (a SHA), which is safe but freezes them in time.
+# Dependabot opens a small, reviewable pull request whenever a newer version
+# exists -- for Python packages, npm packages, the Docker base image, and the
+# pinned Actions themselves -- so staying patched does not require manual work.
+# Updates are grouped so a week's bumps arrive as one PR per ecosystem, not a
+# flood of separate ones.
+
+version: 2
+updates:
+  # Python dependencies (requirements.txt + requirements-optional.txt).
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    groups:
+      python:
+        patterns: ["*"]
+
+  # Frontend / tooling npm packages (package.json).
+  - package-ecosystem: npm
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    groups:
+      npm:
+        patterns: ["*"]
+
+  # The pinned action SHAs used across .github/workflows.
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    groups:
+      actions:
+        patterns: ["*"]
+
+  # The Docker base image in the Dockerfile.
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5

.github/pull_request_review_template.md

@@ -0,0 +1,123 @@
+# Pull Request Review Template
+
+Use this shape as a copyable reference for substantive PR reviews; GitHub does
+not auto-apply this file to review comments. Omit sections that do not add
+useful signal. Lead with confirmed findings; keep speculative notes out of the
+public review unless they are framed as a concrete open question.
+
+## Small PR Path
+
+For narrow docs, typo, test-only, or obvious local fixes, a short review is
+enough:
+
+```md
+LGTM after checking:
+- scope:
+- validation:
+- residual risk:
+```
+
+Use the fuller structure below for larger, risky, multi-finding, or
+security-sensitive reviews.
+
+## Findings
+
+**<sub><sub>![P2 Badge](https://img.shields.io/badge/P2-yellow?style=flat)</sub></sub> issue (test): Short issue title**
+
+- **Problem:** Concrete broken flow, contract, input, or risk.
+
+- **Impact:** Why this matters to users, CI, maintainers, data, security, or scale.
+
+- **Ask:** Smallest practical correction or decision the author should make.
+
+- **Location:** `path:line`
+
+## Open Questions
+
+- **question (scope, non-blocking): Short author question** Ask the concrete
+  intent, scope, or tradeoff question.
+
+## Validation
+
+- Ran:
+- Not run:
+- Residual risk:
+
+## PR Hygiene
+
+- Target/template/checks:
+- Related, duplicate, or superseding context:
+
+## No Findings Variant
+
+```md
+## Findings
+
+none confirmed
+
+## Validation
+
+- Ran:
+- Not run:
+- Residual risk:
+```
+
+## Legend
+
+- **Findings:** Verified, author-actionable issues that should be fixed or
+  consciously accepted before merge.
+- **Priority badges:** The shields.io badges below are optional formatting for
+  priority labels. Plain `P0`, `P1`, `P2`, or `P3` text is also acceptable when
+  an external image dependency is undesirable or may not render.
+  - **P0:** `![P0 Badge](https://img.shields.io/badge/P0-red?style=flat)` -
+    release-blocking or actively dangerous.
+  - **P1:** `![P1 Badge](https://img.shields.io/badge/P1-orange?style=flat)` -
+    serious bug, security risk, data-loss risk, or broken primary flow.
+  - **P2:** `![P2 Badge](https://img.shields.io/badge/P2-yellow?style=flat)` -
+    meaningful correctness, test, maintainability, or edge-case issue.
+  - **P3:** `![P3 Badge](https://img.shields.io/badge/P3-lightgrey?style=flat)` -
+    minor polish or low-risk cleanup.
+- **Intent labels:**
+  - **`issue`:** A confirmed defect, regression, broken contract, or concrete
+    risk.
+  - **`suggestion`:** A non-blocking improvement that would make the PR clearer,
+    safer, or easier to maintain.
+  - **`nit`:** A tiny, non-blocking cleanup or style note. Use it only when the
+    author can safely ignore it without changing the review outcome.
+  - **`question`:** A real author-facing clarification about intent, scope, or
+    tradeoffs. Do not use questions to hide an issue that should be stated
+    directly.
+  - **`LGTM`:** "Looks good to me." Use only when the review found no blocking
+    issues, or when any remaining notes are clearly optional.
+- **Decorations:** Optional labels in parentheses that clarify the finding type,
+  scope, or merge impact.
+  - **`security`:** Auth, authorization, ownership, secrets, SSRF, injection,
+    unsafe external input, or other trust-boundary concerns.
+  - **`test`:** Missing, failing, misleading, brittle, or insufficient tests.
+  - **`scope`:** PR scope, feature boundaries, unrelated churn, or work that
+    should be split into a separate issue or PR.
+  - **`ci`:** CI configuration, workflow failures, flaky checks, or validation
+    signal quality.
+  - **`api`:** Route, request/response, public function, schema, persistence, or
+    integration contract changes.
+  - **`docs`:** User-facing docs, contributor docs, examples, or comments that
+    need to change with the code.
+  - **`non-blocking`:** Useful feedback that should not prevent merge by
+    itself.
+- **Finding fields:**
+  - **Problem:** What is wrong, what contract is ambiguous, or what risk the PR
+    introduces.
+  - **Impact:** Why the problem matters in practical terms.
+  - **Ask:** The smallest concrete fix, test, or decision requested from the PR
+    author.
+  - **Location:** The most useful repo-relative file and line reference for the
+    finding, using `path:line`.
+- **Optional sections:**
+  - **Open Questions:** Genuine scope or intent questions; omit when there are
+    no real questions.
+  - **Validation:** What the reviewer ran, what was intentionally not run, and
+    what risk remains after review.
+  - **PR Hygiene:** Target-branch, template, CI/check, duplicate, related-work,
+    or superseding-PR notes.
+- **`none confirmed`:** Use only when no review-worthy findings were confirmed;
+  still list validation gaps or residual risk when relevant.

.github/workflows/ci.yml

@@ -19,10 +19,10 @@ jobs:
     name: Python syntax (compileall)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
         with:
           persist-credentials: false
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
         with:
           python-version: "3.11"
       # Byte-compile sources — catches syntax errors without installing deps.
@@ -32,10 +32,10 @@ jobs:
     name: JS syntax (node --check)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
         with:
           persist-credentials: false
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e  # v6.4.0
         with:
           node-version: "20"
       # Syntax-check our own JS (skip vendored libs in static/lib).
@@ -54,7 +54,7 @@ jobs:
     # ROADMAP "fresh install smoke tests" item; make this required once green.
     continue-on-error: true
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
         with:
           fetch-depth: 0
           persist-credentials: false
@@ -81,7 +81,7 @@ jobs:
             echo "docs_only=false" >> "$GITHUB_OUTPUT"
           fi
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
         if: steps.docs-check.outputs.docs_only != 'true'
         with:
           python-version: "3.11"

.github/workflows/container-scan.yml

@@ -0,0 +1,52 @@
+# Container security: Dockerfile lint
+#
+# Purpose: the Docker image is how most people run Odysseus, so it is part of
+# the attack surface. hadolint lints the Dockerfile for mistakes and insecure
+# patterns (running as root longer than needed, unpinned base image, bad apt
+# usage). Blocking.
+#
+# The image vulnerability scan (Trivy, advisory) lives in its own file,
+# container-trivy.yml. Keeping it separate lets that advisory scan be
+# path-filtered and held to a read-only token on pull requests without
+# weakening this blocking gate, which must always report so a required check
+# never hangs.
+#
+# Note: a separate open PR (#120) proposes a local `scripts/scan_image.py`.
+# This job is complementary -- it is a CI gate, not a script a contributor has
+# to remember to run.
+
+name: Container scan
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: container-scan-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  hadolint:
+    name: hadolint (Dockerfile lint)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Lint Dockerfile
+        uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5  # v3.3.0
+        with:
+          dockerfile: Dockerfile
+          # DL3008: pinning apt package versions is impractical on a -slim base
+          # image. Debian purges old package versions from its repos, so a
+          # pinned version breaks future rebuilds. The base image itself is
+          # what should be pinned (tracked by Dependabot's docker ecosystem).
+          ignore: DL3008

.github/workflows/container-trivy.yml

@@ -0,0 +1,125 @@
+# Container image vulnerability scan (advisory)
+#
+# Trivy builds the application image and scans it for known-vulnerable OS and
+# Python packages. Advisory only -- it reports findings to the repo's Security
+# tab without blocking a merge, because the image inevitably contains
+# already-known CVEs in upstream packages that are not this project's bug.
+#
+# Split from the Dockerfile lint (container-scan.yml) for two reasons:
+#
+#   - Least privilege. The image build runs Dockerfile instructions, which on a
+#     pull request are attacker-influenceable. That path (the `scan` job) is
+#     held to a read-only token and never publishes results. Only `publish`,
+#     which runs on push to main (curated, fast-forwarded from reviewed dev),
+#     gets security-events:write to upload SARIF.
+#   - Cost. Docs-only changes do not rebuild the image (paths-ignore below),
+#     matching docker-publish.yml. hadolint stays on the broad trigger in
+#     container-scan.yml so the blocking gate always reports.
+
+name: Container scan (Trivy)
+
+on:
+  pull_request:
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+      - '.github/ISSUE_TEMPLATE/**'
+  push:
+    branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+      - '.github/ISSUE_TEMPLATE/**'
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: container-trivy-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Pull requests and manual runs: build and scan under a read-only token.
+  # The build executes PR-supplied Dockerfile instructions, so this job must
+  # not hold any write scope, and it does not upload to the Security tab.
+  scan:
+    name: Trivy (image scan, advisory)
+    if: github.event_name != 'push'
+    runs-on: ubuntu-latest
+    # Advisory: a CVE in an upstream package must not block a PR.
+    continue-on-error: true
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4.1.0
+
+      # Build without pushing so a broken Dockerfile is caught here, and the
+      # exact image we ship is what gets scanned.
+      - name: Build image
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7.2.0
+        with:
+          context: .
+          push: false
+          load: true
+          tags: odysseus:ci
+
+      - name: Scan image with Trivy
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25  # v0.36.0
+        with:
+          image-ref: odysseus:ci
+          format: table
+          ignore-unfixed: true
+        env:
+          # Pin the vuln DB source to GHCR to avoid rate-limited Docker Hub
+          # mirrors that flake on shared runners.
+          TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2
+
+  # Push to main only: build, scan, and publish SARIF to the Security tab.
+  # This is the only path that runs trusted code, so it is the only one granted
+  # security-events:write.
+  publish:
+    name: Trivy (image scan + SARIF upload)
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    permissions:
+      contents: read
+      security-events: write  # upload SARIF to the Security tab
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4.1.0
+
+      - name: Build image
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7.2.0
+        with:
+          context: .
+          push: false
+          load: true
+          tags: odysseus:ci
+
+      - name: Scan image with Trivy
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25  # v0.36.0
+        with:
+          image-ref: odysseus:ci
+          format: sarif
+          output: trivy-results.sarif
+          ignore-unfixed: true
+        env:
+          TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2
+
+      - name: Upload Trivy results
+        uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        with:
+          sarif_file: trivy-results.sarif
+          category: trivy-image

.github/workflows/dependency-review.yml

@@ -0,0 +1,71 @@
+# Supply-chain review
+#
+# Purpose: defend against "side-chain" / supply-chain attacks -- a pull request
+# that adds (or bumps) a dependency to a version with a known vulnerability or a
+# disallowed license. Two layers:
+#
+#   - dependency-review: runs ONLY on pull requests. It compares the
+#     dependencies before and after the PR and blocks the merge if the change
+#     pulls in a package with a known security advisory. This is the gate.
+#   - pip-audit: scans the project's current Python requirements against the
+#     advisory database. Advisory only (it never blocks a merge), because it can
+#     flag a pre-existing issue in an already-shipped dependency.
+
+name: Dependency review
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+# Default-deny token; jobs grant only read access.
+permissions: {}
+
+concurrency:
+  group: dependency-review-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dependency-review:
+    name: dependency-review (PR gate)
+    # Only meaningful on a pull request -- it needs a base..head diff to review.
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Review dependency changes
+        uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294  # v5.0.0
+        with:
+          # Fail the PR on any newly introduced moderate-or-worse advisory.
+          fail-on-severity: moderate
+
+  pip-audit:
+    name: pip-audit (advisory)
+    runs-on: ubuntu-latest
+    # Advisory: report known-vulnerable Python deps without blocking the merge.
+    continue-on-error: true
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        with:
+          python-version: '3.12'
+
+      - name: Run pip-audit on requirements
+        run: |
+          set -euo pipefail
+          pip install pip-audit==2.10.0
+          pip-audit -r requirements.txt -r requirements-optional.txt --strict

.github/workflows/secret-scan.yml

@@ -0,0 +1,60 @@
+# Secret scanning
+#
+# Purpose: stop credentials (API keys, tokens, passwords, private keys) from
+# ever living in the Git history. Odysseus deliberately keeps real secrets in
+# files that are gitignored (.env, data/), but a slip in a future commit -- or a
+# malicious pull request that sneaks one in -- would otherwise go unnoticed.
+# This job reads the repository and the full commit history and fails if it
+# finds anything that looks like a secret.
+#
+# It runs the official gitleaks BINARY directly (pinned to an exact version and
+# verified against the project's published SHA-256 checksum) rather than the
+# gitleaks GitHub Action, because the Action asks for a paid license on
+# organization-owned repos. The binary is free and behaves identically.
+
+name: Secret scan
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+# Start with zero permissions; the single job opts back in to read-only.
+permissions: {}
+
+concurrency:
+  group: secret-scan-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  gitleaks:
+    name: gitleaks
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
+        with:
+          # Full history so a secret committed in an earlier commit (and later
+          # deleted) is still caught -- deletion does not remove it from Git.
+          fetch-depth: 0
+          persist-credentials: false
+
+      # Pinned version + checksum so a tampered release binary cannot run here.
+      # Bump VERSION/SHA256 together; the checksum comes from the matching
+      # gitleaks_<version>_checksums.txt on the GitHub release.
+      - name: Run gitleaks (pinned, checksum-verified)
+        env:
+          GITLEAKS_VERSION: 8.30.1
+          GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb
+        run: |
+          set -euo pipefail
+          TARBALL="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
+          curl -fsSL -o "${TARBALL}" \
+            "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/${TARBALL}"
+          echo "${GITLEAKS_SHA256}  ${TARBALL}" | sha256sum -c -
+          tar -xzf "${TARBALL}" gitleaks
+          # Scan the whole history. Findings print to the log and fail the job.
+          ./gitleaks git --no-banner --redact --verbose .

.github/workflows/workflow-security.yml

@@ -0,0 +1,80 @@
+# Workflow security (CI that audits the CI)
+#
+# Purpose: the GitHub Actions workflows themselves are an attack surface. A
+# poorly written workflow can leak the repository token, run attacker-supplied
+# code from a pull request, or pull in a tampered third-party action. These two
+# tools check every workflow file in this repo for those mistakes:
+#
+#   - actionlint: catches workflow syntax errors and shell-script bugs inside
+#     `run:` steps before they reach main.
+#   - zizmor: a security linter for Actions. Flags template-injection holes,
+#     unpinned actions, credential persistence, and over-broad token
+#     permissions -- exactly the patterns the rest of this CI is built to avoid.
+#
+# Add this early: it then audits every workflow added after it.
+
+name: Workflow security
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+# Default-deny token; each job grants only read access to the code.
+permissions: {}
+
+concurrency:
+  group: workflow-security-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  actionlint:
+    name: actionlint
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
+        with:
+          persist-credentials: false
+
+      # Pinned version + checksum so a tampered binary cannot run here.
+      - name: Run actionlint (pinned, checksum-verified)
+        env:
+          ACTIONLINT_VERSION: 1.7.12
+          ACTIONLINT_SHA256: 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8
+        run: |
+          set -euo pipefail
+          TARBALL="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz"
+          curl -fsSL -o "${TARBALL}" \
+            "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/${TARBALL}"
+          echo "${ACTIONLINT_SHA256}  ${TARBALL}" | sha256sum -c -
+          tar -xzf "${TARBALL}" actionlint
+          ./actionlint -color
+
+  zizmor:
+    name: zizmor (Actions SAST)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        with:
+          python-version: '3.12'
+
+      # Pinned zizmor release. --offline keeps the audit hermetic (no network
+      # calls about the actions it inspects); --min-severity=low surfaces
+      # everything so nothing slips through under the gate.
+      - name: Run zizmor
+        run: |
+          set -euo pipefail
+          pip install zizmor==1.25.2
+          zizmor --offline --min-severity=low .github/workflows/

.gitignore

@@ -14,6 +14,15 @@ venv/
 .env
 .env.bak.*
 !.env.example
+# Local uv lockfile (optional, per-platform — see "Faster installs with uv" in README)
+requirements.lock
+
+# SOPS workflow — encrypted `secrets.env` is intentionally committable,
+# but every variant (plaintext, manual decrypt copy, editor backup)
+# must stay out of git. Mirrored in .dockerignore so the same artifacts
+# also cannot enter image build layers.
+secrets.env.*
+!secrets.env.example
 
 # Data — all user data stays local
 data/
@@ -61,6 +70,9 @@ output.txt.txt
 *.tiff
 *.pdf
 
+# …except shipped static assets
+!static/icons/*.png
+
 # …except shipped demo assets in docs/ that the README links to.
 !docs/*.jpg
 !docs/*.jpeg

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-slim
+FROM python:3.14-slim
 
 # System deps. tmux is required by Cookbook for background downloads/serves.
 # openssh-client is required for Cookbook remote server tests, setup, probes,

Odysseus.spec

@@ -0,0 +1,45 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+
+a = Analysis(
+    ['launcher.py'],
+    pathex=[],
+    binaries=[],
+    datas=[('static', 'static'), ('scripts', 'scripts'), ('mcp_servers', 'mcp_servers'), ('services/hwfit/data', 'services/hwfit/data'), ('config', 'config'), ('.env.example', '.env.example')],
+    hiddenimports=[],
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    noarchive=False,
+    optimize=0,
+)
+pyz = PYZ(a.pure)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='Odysseus',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    console=False,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+    icon=['static\\icon.ico'],
+)
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name='Odysseus',
+)

README.md

@@ -1,444 +1,65 @@
-# Odysseus
+<p align="center">
+  <img src="docs/odysseus-wordmark.png" alt="Odysseus" width="280">
+</p>
 
-> **Branch note:** `dev` is the default branch and contains the latest development changes, but it may be unstable. For the more stable curated branch, use [`main`](https://github.com/pewdiepie-archdaemon/odysseus/tree/main).
+<p align="center">
+  A self-hosted AI workspace for chat, agents, research, documents, email, notes, calendar, and local model workflows.
+</p>
 
-```
-───────────────────────────────────────────────
- ⊹ ࣪ ˖ ૮( ˶ᵔ ᵕ ᵔ˶ )っ  Odysseus vers. 1.0
-───────────────────────────────────────────────
-```
+<p align="center">
+  <a href="#quick-start">Quick Start</a> ·
+  <a href="docs/setup.md">Setup Guide</a> ·
+  <a href="CONTRIBUTING.md">Contributing</a> ·
+  <a href="ROADMAP.md">Roadmap</a>
+</p>
 
-![Odysseus](docs/odysseus.jpg)
+<p align="center">
+  <a href="https://repology.org/project/odysseus-ai/versions"><img src="https://repology.org/badge/vertical-allrepos/odysseus-ai.svg" alt="Packaging status"></a>
+</p>
 
-A self-hosted AI workspace -- meant to be the self-hosted version of the UI experience you get from ChatGPT and Claude. But with more jank and fun. Running on your own hardware, with your own data -- local-first, privacy-first, and no trojan.
+<p align="center">
+  <img src="docs/odysseus.jpg" alt="Odysseus interface">
+</p>
 
-## Features
-  - **Chat** -- chat with any local model or API; adding them is super simple.<br>　<sub>vLLM · llama.cpp · Ollama · OpenRouter · OpenAI · GitHub Copilot</sub>
-  - **Agent** -- hand it tools and let it run the whole task itself.<br>　<sub>built on [opencode](https://github.com/anomalyco/opencode) · MCP · web · files · shell · skills · memory</sub>
-  - **Cookbook** -- Scans your hardware, recommends models, click to download and serve.. easy!<br>　<sub>built on [llmfit](https://github.com/AlexsJones/llmfit) · VRAM-aware · GGUF / FP8 / AWQ · fit scoring · vLLM / llama.cpp serving</sub>
-  - **Deep Research** -- multi-step runs that gather, read, and synthesize sources into a nice visual report.<br>　<sub>adapted from [Tongyi DeepResearch](https://github.com/Alibaba-NLP/DeepResearch)</sub>
-  - **Compare** -- a fun tool to compare models side by side. Test completely blind, no bias!<br>　<sub>multi-model · blind test · synthesis</sub>
-  - **Documents** -- YOU write the text, AI is there to assist, not the opposite.<br>　<sub>multi-tab editor · markdown · HTML · CSV · syntax highlighting · AI edits · suggestions</sub>
-  - **Memory / Skills** -- Persistent memory and skills, your agent evolves over time as it better understands you and your tasks!<br>　<sub>ChromaDB · fastembed (ONNX) · vector + keyword retrieval · import/export</sub>
-  - **Email** -- IMAP/SMTP inbox with AI triage built in: urgency reminders, auto-tag, auto-summary, auto-reply drafts, auto-spam.<br>　<sub>IMAP · SMTP · per-account routing · CalDAV-aware</sub>
-  - **Notes & Tasks** -- Quick notes with reminders, a todo list, and scheduled tasks the agent can act on.<br>　<sub>note pings · checklist · cron-style tasks · ntfy / browser / email channels</sub>
-  - **Calendar** -- Local-first calendar with CalDAV sync to Radicale / Nextcloud / Apple / Fastmail.<br>　<sub>CalDAV pull · .ics import/export · per-calendar colors · agent-aware</sub>
-  - **Works on mobile** -- looks and runs great on your phone, not just desktop.<br>　<sub>responsive · installable (PWA) · touch gestures</sub>
-  - **Extras** -- more to explore, happy if you give it a go!<br>　<sub>image editor · theme editor · file uploads (vision + PDF) · web search · presets · sessions · 2FA</sub>
-
-## Demo
-A full, hover-to-play tour lives on the landing page (`docs/index.html`).
-
-<details>
-<summary>Screenshots / clips</summary>
-
-### Chat & Agents
-![Chat & Agents](docs/chat.gif)
-### Deep Research
-![Deep Research](docs/research.gif)
-### Compare
-![Compare](docs/compare.gif)
-### Documents
-![Documents](docs/document.gif)
-### Notes & Tasks
-![Notes & Tasks](docs/notes.gif)
-
-</details>
+---
 
 ## Quick Start
 
-Defaults work out of the box: clone, run, then configure models/search/email
-inside **Settings**. Only edit `.env` for deployment-level overrides like
-`APP_BIND`, `APP_PORT`, `AUTH_ENABLED`, `DATABASE_URL`, or a pre-seeded admin password.
+> `dev` is the default branch and gets the newest changes first. Use [`main`](https://github.com/pewdiepie-archdaemon/odysseus/tree/main) if you want the more curated branch.
 
-On first setup, Odysseus creates an admin account (`admin` unless
-`ODYSSEUS_ADMIN_USER` is set) and prints a temporary password in the terminal.
-For Docker installs, the same line is in `docker compose logs odysseus`.
-Use that for the first login, then change it in **Settings**.
-
-Contributing? See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, testing, and
-pull request guidelines.
-
-### Docker (recommended)
 ```bash
 git clone https://github.com/pewdiepie-archdaemon/odysseus.git
 cd odysseus
-cp .env.example .env       # optional, but recommended for explicit defaults
+cp .env.example .env
 docker compose up -d --build
 ```
-To include optional extras in the image (PDF viewer, Office extraction; includes AGPL PyMuPDF), build with `docker compose build --build-arg INSTALL_OPTIONAL=true` before `up`.
 
-Open `http://localhost:7000` when the containers are healthy. Docker Compose
-binds the web UI to `127.0.0.1` by default. If the port is taken, set
-`APP_PORT=7001` in `.env` and recreate the container. Set `APP_BIND=0.0.0.0`
-only when you intentionally want LAN/reverse-proxy access.
+Open `http://localhost:7000` when the containers are healthy. The first admin password is printed in `docker compose logs odysseus`.
 
-### Native Linux / macOS
-```bash
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
-cd odysseus
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements.txt
-python setup.py
-python -m uvicorn app:app --host 127.0.0.1 --port 7000
-```
-Requirements: Python 3.11+. Cookbook also needs `tmux` for background model
-downloads and serves. The app itself is lightweight; local model serving is the
-heavy part and depends on the model, runtime, GPU, and VRAM, so small hosts can
-connect to API or remote model servers instead. Use `--host 0.0.0.0` only when you intentionally want LAN/reverse-proxy access.
+Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration live in the [setup guide](docs/setup.md).
 
-### Apple Silicon
-Docker on macOS cannot use the Metal GPU. For GPU-accelerated Cookbook on an
-M-series Mac, run Odysseus natively:
+## Features
 
-```bash
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
-cd odysseus
-./start-macos.sh
-```
+- **Chat + Agents** — local/API models, tools, MCP, files, shell, skills, and memory.
+- **Cookbook** — hardware-aware model recommendations, downloads, and serving.
+- **Deep Research** — multi-step web research with source reading and report generation.
+- **Compare** — blind side-by-side model testing and synthesis.
+- **Documents** — writing-first editor with AI edits, suggestions, Markdown, HTML, CSV, and syntax highlighting.
+- **Email** — IMAP/SMTP inbox with triage, tags, summaries, reminders, and reply drafts.
+- **Notes, Tasks + Calendar** — reminders, todos, scheduled agent tasks, and CalDAV sync.
+- **Extras** — gallery/image editor, themes, uploads, web search, presets, sessions, and 2FA.
 
-It launches at `http://127.0.0.1:7860`. To expose it to your phone over a trusted LAN/VPN such as Tailscale, bind all interfaces:
+## Demo
 
-```bash
-ODYSSEUS_HOST=0.0.0.0 ./start-macos.sh
-# then open http://<tailscale-ip>:7860
-```
-
-The script also reads `.env` at startup, so `APP_BIND=0.0.0.0` and `APP_PORT`
-set there are picked up automatically without a command-line override each run.
-
-Keep `AUTH_ENABLED=true` (the default) before binding outside loopback. Do not
-expose this port directly to the public internet. To build a clickable app wrapper:
-
-```bash
-./build-macos-app.sh
-```
-
-<details>
-<summary>Cookbook, GPU, Ollama, and troubleshooting notes</summary>
-
-**Docker bundled services.** Compose starts Odysseus, ChromaDB, SearXNG, and
-ntfy. Odysseus and the bundled service ports bind to `127.0.0.1` by default, so
-they are reachable from the host but not exposed to your LAN/public internet
-unless you opt in.
-
-**Cookbook storage in Docker.** Downloads live in `./data/huggingface`
-(`~/.cache/huggingface` in the container). Cookbook-installed Python CLIs and
-serve engines live in `./data/local` (`~/.local` in the container), so they
-survive container recreation.
-
-**Remote servers.** In **Cookbook -> Settings -> Servers**, generate the
-Odysseus SSH key and add the public key to the remote server's
-`~/.ssh/authorized_keys`. From the host you can also run:
-
-```bash
-ssh-copy-id -i data/ssh/id_ed25519.pub user@server
-```
-
-**Docker GPU overlays.** CPU-only users can skip this section. Cookbook can
-only detect GPUs that Docker exposes to the container — if the host runtime or
-device passthrough is not configured, Cookbook sees the iGPU, another card, or
-CPU instead of your intended GPU.
-
-For NVIDIA, `scripts/check-docker-gpu.sh` diagnoses GPU passthrough and can
-optionally install the host runtime or update `.env`.
-
-```bash
-# Read-only diagnostic (default — installs nothing, never edits .env):
-scripts/check-docker-gpu.sh
-
-# Print OS-specific install commands without running them:
-scripts/check-docker-gpu.sh --print-install-commands
-
-# Install NVIDIA Container Toolkit on Ubuntu/Debian (requires sudo):
-scripts/check-docker-gpu.sh --install-nvidia-toolkit
-
-# Write COMPOSE_FILE to .env (only when GPU passthrough is confirmed working):
-scripts/check-docker-gpu.sh --enable-nvidia-overlay
-
-# Full assisted setup — install toolkit, then enable overlay if passthrough works:
-scripts/check-docker-gpu.sh --install-nvidia-toolkit --enable-nvidia-overlay
-```
-
-Safety notes:
-- The app never installs host GPU runtime automatically.
-- The app never edits `.env` automatically.
-- `.env` is only modified when `--enable-nvidia-overlay` is explicitly passed,
-  and only after GPU passthrough succeeds. `--yes` skips prompts but does not
-  bypass the passthrough gate.
-- `.env.bak.*` backups created by `--enable-nvidia-overlay` are ignored by
-  Git and the Docker build context.
-
-To enable manually without the script, add this to `.env`:
-
-```bash
-COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml
-```
-
-**AMD / ROCm.** AMD setup is read-only diagnostic plus manual `.env` edit. Run:
-
-```bash
-scripts/check-docker-amd-gpu.sh
-```
-
-Then add the reported values to `.env`, replacing `RENDER_GID` with your host's
-numeric render group id:
-
-```bash
-COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml
-RENDER_GID=989
-```
-
-For NVIDIA/AMD GPU support, also read the comments in the selected overlay file: docker/gpu.nvidia.yml or docker/gpu.amd.yml.
-
-**Stack-management UIs (Portainer, Coolify, Dockhand, etc.).** These tools
-often accept only a single Compose file and do not reliably honor `COMPOSE_FILE`
-or multiple `-f` overlays. CLI users should keep using the `COMPOSE_FILE`
-overlay workflow above. For stack UIs, point the stack at one of the standalone
-files instead, which bundle the base stack plus the GPU settings:
-
-- `docker-compose.gpu-nvidia.yml` — still requires the NVIDIA Container Toolkit
-  on the host.
-- `docker-compose.gpu-amd.yml` — still requires host ROCm/kfd/DRI setup, the
-  `video`/`render` group membership, and `RENDER_GID` when needed.
-
-The base `docker-compose.yml` plus the `docker/gpu.*.yml` overlays remain the
-source of truth; the standalone files mirror them for single-file deployments.
-
-Verify after enabling either overlay:
-
-```bash
-docker compose exec odysseus nvidia-smi -L   # NVIDIA
-docker compose exec odysseus sh -lc 'test -e /dev/kfd && test -d /dev/dri && ls -l /dev/kfd /dev/dri/renderD*'  # AMD
-```
-
-> **GPU passthrough ≠ llama.cpp CUDA.** `nvidia-smi` passing inside the
-> container confirms Docker GPU access, but llama.cpp also needs `cudart` and
-> the CUDA Toolkit at runtime. If Cookbook logs show `Unable to find cudart
-> library`, `Could NOT find CUDAToolkit`, `CUDA Toolkit not found`, or
-> tensors/layers assigned to CPU, that is a Cookbook/llama.cpp build issue —
-> not a Docker passthrough failure. Re-install the serve engine via
-> **Cookbook → Dependencies** to get a CUDA-enabled build.
->
-> The same split applies to AMD/ROCm: seeing `/dev/kfd` and `/dev/dri` inside
-> the container confirms device passthrough, not ROCm userspace or a
-> ROCm-enabled vLLM/llama.cpp build. `rocm-smi` and `rocminfo` are not expected
-> inside the slim Odysseus image.
-
-**Ollama with Docker.** If Ollama runs on the host, add this endpoint in
-Settings:
-
-```text
-http://host.docker.internal:11434/v1
-```
-
-Ollama must listen outside its own loopback interface:
-
-```bash
-OLLAMA_HOST=0.0.0.0:11434 ollama serve
-```
-
-This connects Odysseus in Docker to an Ollama server that is already running on
-your host machine; it does not start Ollama inside the container.
-`host.docker.internal` is Docker's hostname for the host machine from inside the
-container. Cookbook **Serve** is a separate workflow for serving downloaded
-models through Odysseus/llama.cpp, so Windows users with an existing Ollama
-install usually only need to add the endpoint in Settings.
-
-**Useful checks.**
-
-```bash
-docker compose ps
-docker compose logs --tail=120 odysseus
-docker compose logs odysseus | grep -E 'ChromaDB|MemoryVectorStore|DEGRADED'
-```
-
-**macOS details.** `start-macos.sh` installs Homebrew deps, creates the venv,
-runs setup, and starts uvicorn on port `7860` because AirPlay often holds
-`7000`. It uses llama.cpp/Ollama for Metal. vLLM/SGLang are CUDA/ROCm-only and
-do not run on macOS. MLX-only models are not served by Odysseus.
-
-</details>
-
-### Native Windows
-
-**One-command launcher** (creates the venv, installs deps, runs setup, starts the
-server; safe to re-run):
-
-```powershell
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
-cd odysseus
-powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1
-```
-
-Or do it by hand:
-
-```powershell
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
-cd odysseus
-py -3.11 -m venv venv
-venv\Scripts\Activate.ps1
-pip install -r requirements.txt
-python setup.py
-python -m uvicorn app:app --host 127.0.0.1 --port 7000
-```
-
-If `python` points at an older interpreter, use `py -3.12` (or another installed
-3.11+ version) for the venv step.
-
-**Requirements:** Python 3.11+. The core app (chat, agent, memory, documents,
-email, calendar, deep research) runs fully native. For full **Cookbook** background
-model downloads and the agent shell tool, also install
-[Git for Windows](https://git-scm.com/download/win) (provides `bash.exe`).
-Local GPU *serving* of vLLM/SGLang needs Linux/WSL2; for a local model on Windows,
-[Ollama](https://ollama.com/download) is the easiest path — point Odysseus at
-`http://localhost:11434/v1` in Settings.
-
-Open `http://localhost:7000`, log in with the generated admin password,
-and configure everything else inside **Settings**.
-
-## Troubleshooting & Advanced Setup
-
-### `chromadb-client` conflicts with embedded ChromaDB
-If `chromadb-client` (the lightweight HTTP-only package) is installed alongside the full `chromadb` package, Odysseus starts but ChromaDB silently falls back to HTTP-only mode and fails.
-
-**Fix:** uninstall `chromadb-client` and force-reinstall the full package:
-```bash
-./venv/bin/pip uninstall chromadb-client -y
-./venv/bin/pip install --force-reinstall chromadb
-```
-
-### HTTPS + LAN/Tailscale exposure
-To expose Odysseus on a local network or Tailscale with HTTPS:
-1. Change the bind address to `0.0.0.0` in `.env` (`APP_BIND=0.0.0.0` or `ODYSSEUS_HOST=0.0.0.0`).
-2. Generate a locally-trusted cert for your LAN/Tailscale IPs using [mkcert](https://github.com/FiloSottile/mkcert):
-   ```bash
-   mkcert -install
-   mkcert -cert-file cert.pem -key-file key.pem 192.168.1.100 tailscale-ip
-   ```
-3. Run `uvicorn` with the generated certs:
-   ```bash
-   python -m uvicorn app:app --host 0.0.0.0 --port 7000 --ssl-certfile=cert.pem --ssl-keyfile=key.pem
-   ```
-4. Install the `mkcert` CA on any other device you want to access Odysseus from (e.g., for iOS, email the `rootCA.pem` to yourself, install the profile, and trust it in Certificate Trust Settings).
-
-### Optional Dependencies
-`requirements-optional.txt` contains packages that unlock extra features. It is not installed by default.
-
-| Package | Feature unlocked |
-|---------|-----------------|
-| `faster-whisper` | Local speech-to-text (microphone -> text) via the "local" STT provider. |
-| `ddgs` | DuckDuckGo as a search provider option. |
-| `PyMuPDF` | PDF page rendering in the side viewer panel and form-filling. (Note: AGPL-3.0) |
-| `markitdown` | Office/EPUB document text extraction (converts .docx/.xlsx/.pptx/.xls/.epub to Markdown). |
-
-### Outlook / Office 365 email
-Odysseus email accounts currently use IMAP/SMTP username-password auth. Outlook
-and Microsoft 365 generally require OAuth instead, so normal Microsoft mailbox
-passwords will fail. See [docs/email-outlook.md](docs/email-outlook.md) for the
-current limitation and the planned integration direction.
-
-## Security Notes
-Odysseus is a self-hosted workspace with powerful local tools: shell access, file uploads, model downloads, web research, email/calendar integrations, and API tokens. Treat it like an admin console.
-
-- Keep `AUTH_ENABLED=true` for any network-accessible deployment.
-- Keep `LOCALHOST_BYPASS=false` outside local development.
-- Use `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway.
-- Do not expose it directly to the public internet without HTTPS and a trusted reverse proxy or private access layer.
-- Keep `.env`, `data/`, `logs/`, databases, uploads, generated media, backups, auth/session files, API keys, and model/provider tokens out of Git and private shares. They are ignored by default.
-- Review `data/auth.json` after first boot: disable open signup unless you intentionally want it, make only your own account admin, and keep demo/test accounts non-admin.
-- Non-admin users do not get shell/Python/file read/write by default, and admin-only routes/tools such as MCP management, API tokens, webhooks, model/cookbook serving, backup/vault, and app settings are admin-gated. Other features are controlled by per-user privileges, so review each user's privileges before exposing a deployment.
-- Rotate any API keys or tokens that were ever pasted into a shared chat, demo, screenshot, or log.
-- If you enable API tokens or webhooks, create separate tokens per integration and delete unused ones.
-- Prefer binding manual development runs to `127.0.0.1`; bind to `0.0.0.0` only when you intentionally want LAN/reverse-proxy access.
-- Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only. Expose only the authenticated Odysseus web/API entrypoint through your trusted proxy or private access layer.
-- Before publishing a fork, run `git status --short` and confirm no private files from `.env`, `data/`, `logs/`, uploads, backups, or local databases are staged.
-
-### Private or proxied deployments
-Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and the bundled services to `127.0.0.1` by default, so a typical production/private setup is:
-
-1. Keep Odysseus on localhost, for example `127.0.0.1:7000`.
-2. Terminate HTTPS at a trusted reverse proxy or private access gateway.
-3. Put the authenticated Odysseus web/API entrypoint behind that layer.
-4. Keep raw service and model ports internal-only.
-
-Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true`, `LOCALHOST_BYPASS=false`, and `SECURE_COOKIES=true`.
-
-Common internal-only ports from the default docs/compose setup:
-
-| Port | Service |
-|---|---|
-| `7000` | Odysseus raw app port |
-| `8080` | SearXNG |
-| `8091` | ntfy |
-| `8100` | ChromaDB host port for manual/compose access |
-| `11434` | Ollama |
-| `8000-8020` | Common local model/provider APIs |
+A full hover-to-play tour lives on the landing page: [`docs/index.html`](docs/index.html).
 
 ## Contributing
-Help is welcome. The best entry points are fresh-install testing, provider setup
-bugs, mobile/editor polish, docs, and small focused refactors. See
-[ROADMAP.md](ROADMAP.md) for the current help-wanted list.
 
-## Configuration
-Most setup is done inside the app with `/setup` or **Settings**. Use `.env`
-for deployment-level defaults and secrets you want present before first boot.
-Key settings:
+Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, docs, and small focused refactors. See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md).
 
-| Variable | Default | Description |
-|---|---|---|
-| `LLM_HOST` | `localhost` | Your LLM server (e.g. `llm-host.local:8000`) |
-| `LLM_HOSTS` | -- | Comma-separated list for model discovery |
-| `OPENAI_API_KEY` | -- | Optional OpenAI key. Prefer adding providers in the app unless pre-seeding. |
-| `SEARXNG_INSTANCE` | `http://localhost:8080` | SearXNG URL. Docker overrides this to `http://searxng:8080`. |
-| `SEARXNG_SECRET` | generated on first Docker boot | Optional SearXNG cookie/CSRF secret. Leave blank unless you need to pin it. |
-| `APP_BIND` | `127.0.0.1` | Docker Compose host bind address for the web UI. Use `0.0.0.0` only for intentional LAN/reverse-proxy access. |
-| `APP_PORT` | `7000` | Docker Compose host port for the web UI. |
-| `AUTH_ENABLED` | `true` | Enable/disable login |
-| `LOCALHOST_BYPASS` | `false` | Development-only auth bypass for loopback requests. Keep false for shared/network deployments. |
-| `SECURE_COOKIES` | `false` | Set true when serving Odysseus through HTTPS at a trusted proxy or private access gateway. |
-| `DATABASE_URL` | `sqlite:///./data/app.db` | Database connection string |
-| `CHROMADB_HOST` | `localhost` | ChromaDB host for vector memory. Docker overrides this to `chromadb`. |
-| `CHROMADB_PORT` | `8100` | ChromaDB port for manual host runs. Docker overrides this to `8000`. |
-| `EMBEDDING_URL` | -- | OpenAI-compatible embeddings endpoint |
-| `ODYSSEUS_CHAT_UPLOAD_MAX_BYTES` | `10485760` | Chat/agent attachment cap in bytes. Raise for larger local PDFs or text documents. |
-| `ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES` | `104857600` | Gallery image upload cap in bytes (100 MB). |
-| `ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES` | `26214400` | Gallery transform input cap in bytes (25 MB). |
-| `ODYSSEUS_MEMORY_IMPORT_MAX_BYTES` | `10485760` | Memory import file cap in bytes (10 MB). |
-| `ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES` | `26214400` | Personal document upload cap in bytes (25 MB). |
-| `ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES` | `26214400` | Email compose attachment cap in bytes (25 MB). |
-| `ODYSSEUS_STT_MAX_AUDIO_BYTES` | `26214400` | Speech-to-text audio cap in bytes (25 MB). |
-| `ODYSSEUS_ICS_MAX_BYTES` | `10485760` | Calendar `.ics` import cap in bytes (10 MB). |
+## Security
 
-All upload-limit vars are validated (must be a positive integer) and optional; an invalid value fails fast at startup.
-
-### Built-in MCP servers (optional setup)
-
-Odysseus auto-registers a few built-in MCP servers at startup. The npx-based ones (currently the browser server, `@playwright/mcp`) only start when their npm package is already in the local npx cache. If a package isn't cached, that server is skipped with a startup log message explaining what to do, so a fresh install does not block on a multi-minute npm download or hang if Playwright system deps are missing.
-
-To enable the browser MCP (page navigation, screenshots, vision), run once:
-
-```bash
-npx -y @playwright/mcp@latest --version
-```
-
-That installs `@playwright/mcp` plus Playwright (~300MB total). Restart Odysseus and the server will register at startup.
-
-## Architecture
-```
-app.py                   # FastAPI entry point
-core/      auth, database, middleware, constants
-src/       llm_core, agent_loop, agent_tools, chat_processor, search/
-routes/    chat, session, document, memory, model … endpoints
-services/  docs, memory, search, hwfit (Cookbook) …
-static/    index.html + app.js + style.css + js/ (modular front-end)
-docs/      landing page (index.html) + preview clips
-```
-
-## Data
-All user data lives in `data/` (gitignored): `app.db` (sessions, messages, documents),
-`memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`.
+Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model/service ports publicly. Deployment details are in the [setup guide](docs/setup.md#security-notes).
 
 ## Star History
 
@@ -451,19 +72,5 @@ All user data lives in `data/` (gitignored): `app.db` (sessions, messages, docum
 </a>
 
 ## License
-AGPL-3.0-or-later -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md).
 
-```
-                                  |
-                                 |||
-                                |||||
-                  |    |    |   |||||||
-                 )_)  )_)  )_)   ~|~
-                )___))___))___)\  |
-               )____)____)_____)\\|
-             _____|____|____|_____\\\__
-             \                       /
-       ~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~
-               ~^~  all aboard!  ~^~
-       ~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~
-```
+AGPL-3.0-or-later -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md).

app.py

@@ -1,6 +1,7 @@
 # app.py — slim orchestrator
 import mimetypes
 import os
+import sys
 
 
 def register_static_mime_types() -> None:
@@ -69,10 +70,37 @@ from src.generated_images import GENERATED_IMAGE_HEADERS, resolve_generated_imag
 from starlette.responses import RedirectResponse
 
 # ========= LOGGING =========
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-)
+import logging.handlers
+from core.constants import DATA_DIR
+
+_root_logger = logging.getLogger()
+_root_logger.setLevel(logging.INFO)
+_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+# Clear existing handlers to avoid duplicates
+for _h in list(_root_logger.handlers):
+    _root_logger.removeHandler(_h)
+
+_console_h = logging.StreamHandler()
+_console_h.setFormatter(_formatter)
+_root_logger.addHandler(_console_h)
+
+try:
+    _log_dir = os.path.join(DATA_DIR, "logs")
+    os.makedirs(_log_dir, exist_ok=True)
+    _log_file = os.path.join(_log_dir, "app.log")
+
+    # RotatingFileHandler is not multi-process safe (e.g. if uvicorn is run with --workers N).
+    # Odysseus is single-process by convention, so this is acceptable, but be aware that
+    # concurrent log rotation issues can arise if multiple workers are configured.
+    _file_h = logging.handlers.RotatingFileHandler(
+        _log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8"
+    )
+    _file_h.setFormatter(_formatter)
+    _root_logger.addHandler(_file_h)
+except Exception as e:
+    _root_logger.warning(f"Failed to initialize file logging handler (falling back to console-only): {e}")
+
 logger = logging.getLogger(__name__)
 
 # ========= APP =========
@@ -86,12 +114,13 @@ app = FastAPI(
 )
 
 # ========= CORS =========
+CORS_ALLOW_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"]
 allowed_origins = os.getenv("ALLOWED_ORIGINS", "http://localhost,http://127.0.0.1").split(",")
 app.add_middleware(
     CORSMiddleware,
     allow_origins=allowed_origins,
     allow_credentials=True,
-    allow_methods=["GET", "POST", "PUT", "DELETE"],
+    allow_methods=CORS_ALLOW_METHODS,
     allow_headers=[
         "Accept",
         "Authorization",
@@ -140,6 +169,7 @@ _TIMEOUT_EXEMPT_PREFIXES = (
     "/api/cookbook/setup",  # remote pacman/apt installs
     "/api/upload",          # large files
     "/api/image",           # diffusion proxies (inpaint/harmonize/upscale/etc.) — own 120s httpx timeout
+    "/api/memory/audit",    # retains own 120s LLM inactivity timeout
 )
 
 
@@ -303,8 +333,8 @@ if AUTH_ENABLED:
                         request.state.current_user = "internal-tool"
                     request.state.api_token = False
                     return await call_next(request)
-            except Exception:
-                pass
+            except Exception as _e:
+                logger.warning("Internal tool auth header check failed", exc_info=_e)
             # Allow DIRECT localhost requests (internal service calls from
             # heartbeats etc.). Tunnel/proxy-forwarded requests are excluded by
             # _is_trusted_loopback so LOCALHOST_BYPASS can't be abused over a
@@ -357,11 +387,10 @@ if AUTH_ENABLED:
                                     _db.close()
                             try:
                                 await _asyncio.to_thread(_do)
-                            except Exception:
-                                pass
+                            except Exception as _e:
+                                logger.debug("Failed to update token last_used_at", exc_info=_e)
                         _asyncio.create_task(_touch_last_used(matched_id))
                         # Keep bearer-token callers out of normal cookie/user
-                        # routes. API-aware routes can read api_token_owner.
                         request.state.current_user = "api"
                         request.state.api_token = True
                         request.state.api_token_id = matched_id
@@ -410,7 +439,7 @@ class _RevalidatingStatic(StaticFiles):
         return resp
 
 
-app.mount("/static", _RevalidatingStatic(directory="static"), name="static")
+app.mount("/static", _RevalidatingStatic(directory=STATIC_DIR), name="static")
 
 # ========= GENERATED IMAGES =========
 @app.get("/api/generated-image/{filename}")
@@ -436,8 +465,8 @@ async def serve_generated_image(filename: str, request: Request):
                 _db.close()
     except HTTPException:
         raise
-    except Exception:
-        pass
+    except Exception as _e:
+        logger.warning("Image ownership verification failed for %r", filename, exc_info=_e)
     ext = filename.rsplit('.', 1)[-1].lower()
     mime = {
         "png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg",
@@ -498,7 +527,9 @@ app.state.session_manager = session_manager
 memory_manager    = components["memory_manager"]
 memory_vector     = components.get("memory_vector")
 upload_handler    = components["upload_handler"]
+app.state.upload_handler = upload_handler
 personal_docs_mgr = components["personal_docs_manager"]
+app.state.personal_docs_manager = personal_docs_mgr
 api_key_manager   = components["api_key_manager"]
 preset_manager    = components["preset_manager"]
 chat_processor    = components["chat_processor"]
@@ -675,6 +706,9 @@ app.include_router(setup_shell_routes())
 from routes.cookbook_routes import setup_cookbook_routes
 app.include_router(setup_cookbook_routes())
 
+from routes.workspace_routes import setup_workspace_routes
+app.include_router(setup_workspace_routes())
+
 # Hardware model fitting (cookbook "What Fits?" tab)
 from routes.hwfit_routes import setup_hwfit_routes
 app.include_router(setup_hwfit_routes())
@@ -1139,3 +1173,12 @@ async def _shutdown_event():
     except Exception as e:
         logger.warning(f"MCP shutdown error: {e}")
     logger.info("Application shutdown complete")
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    bind_host = os.getenv("APP_BIND", "127.0.0.1")
+    bind_port = int(os.getenv("APP_PORT", "7000"))
+
+    uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")

build-windows-portable.ps1

@@ -0,0 +1,72 @@
+#Requires -Version 5.1
+<#
+  Build a portable Windows distribution for Odysseus.
+
+  Output layout:
+    dist\Odysseus\Odysseus.exe
+    dist\Odysseus\static\...
+    dist\Odysseus\scripts\...
+    dist\Odysseus\mcp_servers\...
+    dist\Odysseus\services\hwfit\data\...
+
+  The app then keeps using its normal filesystem layout when frozen.
+
+  Usage:
+    powershell -ExecutionPolicy Bypass -File .\build-windows-portable.ps1
+#>
+
+$ErrorActionPreference = "Stop"
+Set-Location -Path $PSScriptRoot
+
+function Write-Step($msg) { Write-Host ""; Write-Host ("==> " + $msg) -ForegroundColor Cyan }
+function Fail($msg) {
+    Write-Host ""
+    Write-Host ("ERROR: " + $msg) -ForegroundColor Red
+    exit 1
+}
+
+Write-Step "Checking for Python"
+$pyExe = $null
+if (Test-Path ".\.venv\Scripts\python.exe") {
+    $pyExe = (Resolve-Path ".\.venv\Scripts\python.exe").Path
+} else {
+    foreach ($c in @("py", "python")) {
+        $cmd = Get-Command $c -ErrorAction SilentlyContinue
+        if ($cmd) { $pyExe = $cmd.Source; break }
+    }
+    if ($pyExe -like "*WindowsApps*python.exe") {
+        $pyCmd = Get-Command py -ErrorAction SilentlyContinue
+        if ($pyCmd) {
+            $pyExe = $pyCmd.Source
+        }
+    }
+}
+if (-not $pyExe) {
+    Fail "Python not found on PATH. Install Python 3.11+ first."
+}
+Write-Host ("Using Python: " + $pyExe)
+
+Write-Step "Installing build dependencies"
+& $pyExe -m pip install --upgrade pip --quiet
+& $pyExe -m pip install -r requirements.txt pyinstaller pystray Pillow
+if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed." }
+
+Write-Step "Building portable exe bundle"
+Remove-Item -Recurse -Force build, dist -ErrorAction SilentlyContinue
+
+$dataArgs = @(
+    "--add-data", "static;static",
+    "--add-data", "scripts;scripts",
+    "--add-data", "mcp_servers;mcp_servers",
+    "--add-data", "services/hwfit/data;services/hwfit/data",
+    "--add-data", "config;config",
+    "--add-data", ".env.example;.env.example"
+)
+
+& $pyExe -m PyInstaller --noconfirm --clean --onedir --noconsole --icon=static/icon.ico --name Odysseus @dataArgs launcher.py
+if ($LASTEXITCODE -ne 0) { Fail "PyInstaller build failed." }
+
+Write-Host ""
+Write-Host "Build complete." -ForegroundColor Green
+Write-Host "Portable app folder: $PSScriptRoot\dist\Odysseus" -ForegroundColor Green
+Write-Host "Distribute the whole folder (or zip it) so static assets and scripts stay with the exe." -ForegroundColor Green

companion/routes.py

@@ -5,8 +5,9 @@ offers and pair to it, without duplicating any LLM logic.
 
 Auth is enforced globally by AuthMiddleware (app.py), so reaching a handler here
 means the caller is authenticated by either a cookie session or a Bearer `ody_`
-API token. The read endpoints (ping/info/models) accept either; the pairing
-endpoints are admin-cookie only.
+API token. Ping/info accept either credential type, models requires a chat-
+scoped API token for bearer callers, and the pairing endpoints are admin-cookie
+only.
 
 Pairing CSRF posture: minting happens ONLY on POST. The session cookie is
 SameSite=Lax (routes/auth_routes.py), which a browser does not send on a
@@ -18,7 +19,7 @@ on a GET would be unsafe (Lax cookies ride top-level GET navigations), so GET
 
 import html
 
-from fastapi import APIRouter, Request
+from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import HTMLResponse
 
 from core.middleware import require_admin
@@ -52,6 +53,18 @@ def owner_can_see(row_owner, owner) -> bool:
     return row_owner is None or row_owner == owner
 
 
+def require_models_scope(request: Request) -> None:
+    """Require the companion chat scope for bearer-token model inventory."""
+    if not getattr(request.state, "api_token", False):
+        return
+    scopes = getattr(request.state, "api_token_scopes", None) or []
+    if isinstance(scopes, str):
+        scopes = [scope.strip() for scope in scopes.split(",")]
+    scope_set = {str(scope).strip() for scope in scopes if str(scope).strip()}
+    if _pairing.COMPANION_SCOPE not in scope_set:
+        raise HTTPException(403, "API token requires chat scope")
+
+
 def mint_pairing_token(owner: str, invalidate=None) -> tuple[str, str]:
     """Mint a pairing token AND invalidate the auth middleware's in-memory token
     cache, so the new token is accepted on the very next request without a server
@@ -103,6 +116,7 @@ def setup_companion_routes() -> APIRouter:
         rows -- the same rule as owner_filter. Read-only; never returns api_key
         material.
         """
+        require_models_scope(request)
         import json as _json
 
         from core.database import SessionLocal, ModelEndpoint

core/auth.py

@@ -3,6 +3,7 @@ Authentication module — multi-user password hashing, session tokens, config pe
 Config stored in data/auth.json. Uses bcrypt directly.
 """
 
+import enum
 import json
 import os
 import secrets
@@ -83,6 +84,15 @@ def _verify_password(password: str, hashed: str) -> bool:
     return bcrypt.checkpw(password.encode("utf-8"), hashed.encode("utf-8"))
 
 
+class SetAdminResult(enum.Enum):
+    """Outcome of AuthManager.set_admin, so callers can map each case to a
+    precise response instead of guessing from a bare bool."""
+    OK = "ok"
+    USER_NOT_FOUND = "user_not_found"
+    NOT_AUTHORIZED = "not_authorized"   # requester is not an admin
+    LAST_ADMIN = "last_admin"           # would remove the last remaining admin
+
+
 class AuthManager:
     """Manages multi-user password + session-token auth system."""
 
@@ -387,6 +397,69 @@ class AuthManager:
         logger.info(f"Updated privileges for '{username}': {current}")
         return True
 
+    def set_admin(self, username: str, is_admin: bool,
+                  requesting_user: str) -> SetAdminResult:
+        """Promote/demote an existing user to/from admin. Admin only.
+
+        Refuses to remove the last remaining admin so the instance can never
+        be locked out of admin access; self-demotion is allowed as long as
+        another admin remains. Admin status is re-checked live on every
+        request, so unlike delete/rename no session or token revocation is
+        needed — a demoted admin simply fails the next is_admin() gate.
+
+        Promotion stashes the user's current privilege map and demotion
+        restores it, so a temporary admin stint can't silently broaden a
+        user's non-admin access; users without a stash (created as admin,
+        or promoted before stashing existed) demote to DEFAULT_PRIVILEGES.
+
+        Counting admins and flipping the flag happen in one critical section
+        so two concurrent demotions can't race the admin count to zero.
+        """
+        username = (username or "").strip().lower()
+        requesting_user = (requesting_user or "").strip().lower()
+        is_admin = bool(is_admin)
+        with self._config_lock:
+            target = self._config.get("users", {}).get(username)
+            if target is None:
+                return SetAdminResult.USER_NOT_FOUND
+            if not self.users.get(requesting_user, {}).get("is_admin"):
+                return SetAdminResult.NOT_AUTHORIZED
+            currently_admin = bool(target.get("is_admin"))
+            if currently_admin == is_admin:
+                return SetAdminResult.OK  # no-op; leave privileges untouched
+            if currently_admin and not is_admin:
+                admin_count = sum(1 for d in self.users.values() if d.get("is_admin"))
+                if admin_count <= 1:
+                    return SetAdminResult.LAST_ADMIN
+            # Write order matters for lock-free readers: get_privileges()
+            # reads without _config_lock and trusts is_admin, so the admin
+            # flag must be flipped while the stored map is safe to expose —
+            # before writing admin privileges on promote, after restoring
+            # the pre-admin map on demote.
+            if is_admin:
+                target["is_admin"] = True
+                # Stash the pre-admin map so a later demotion can restore it.
+                # While is_admin is set the stored map is inert: get_privileges
+                # short-circuits to ADMIN_PRIVILEGES and set_privileges refuses
+                # admins, so only set_admin ever touches the stash.
+                target["privileges_before_admin"] = dict(
+                    target.get("privileges") or DEFAULT_PRIVILEGES
+                )
+                target["privileges"] = dict(ADMIN_PRIVILEGES)
+            else:
+                # Restore the stashed pre-admin map. Fall back to defaults for
+                # users created as admins (their stored map is ADMIN_PRIVILEGES,
+                # which must not leak past demotion — e.g. can_use_bash) and
+                # for admins promoted before the stash existed.
+                target["privileges"] = dict(
+                    target.pop("privileges_before_admin", None)
+                    or DEFAULT_PRIVILEGES
+                )
+                target["is_admin"] = False
+            self._save()
+        logger.info("Set is_admin=%s for '%s' (by '%s')", is_admin, username, requesting_user)
+        return SetAdminResult.OK
+
     def change_password(self, username: str, current_password: str, new_password: str) -> bool:
         username = username.strip().lower()
         if username not in self.users:
@@ -500,16 +573,20 @@ class AuthManager:
             return None
         return self.create_session_trusted(username)
 
-    def create_session_trusted(self, username: str) -> str:
+    def create_session_trusted(self, username: str) -> Optional[str]:
         """Issue a session token for an already-verified user.
         Call only after verify_password (and TOTP if enabled) have passed."""
         username = username.strip().lower()
         token = secrets.token_hex(32)
-        with self._sessions_lock:
-            self._sessions[token] = {
-                "username": username,
-                "expiry": time.time() + TOKEN_TTL,
-            }
+        with self._config_lock:
+            if username not in self.users:
+                logger.warning("Refused to issue session for missing user '%s'", username)
+                return None
+            with self._sessions_lock:
+                self._sessions[token] = {
+                    "username": username,
+                    "expiry": time.time() + TOKEN_TTL,
+                }
         self._save_sessions()
         return token
 

core/database.py

@@ -2,12 +2,15 @@ import os
 import logging
 import sqlite3
 from datetime import datetime, timezone
+from pathlib import Path
 from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text
 from sqlalchemy.engine import Engine
 from sqlalchemy.types import TypeDecorator
 from sqlalchemy.ext.declarative import declarative_base, declared_attr
 from sqlalchemy.orm import relationship, sessionmaker, backref
 
+from src.runtime_paths import get_app_root
+
 logger = logging.getLogger(__name__)
 
 # Create base class for declarative models
@@ -29,9 +32,26 @@ class TimestampMixin:
     def updated_at(cls):
         return Column(DateTime, default=utcnow_naive, onupdate=utcnow_naive, nullable=False)
 
-# Get database URL from environment, default to SQLite in DATA_DIR
+# Ensure the writable data directory exists before SQLite connects.
 from src.constants import DATA_DIR, AUTH_FILE, MEMORY_FILE, USER_PREFS_FILE, SETTINGS_FILE
-DATABASE_URL = os.getenv("DATABASE_URL", f"sqlite:///{DATA_DIR}/app.db")
+Path(DATA_DIR).mkdir(parents=True, exist_ok=True)
+
+
+def _default_database_url() -> str:
+    return f"sqlite:///{Path(DATA_DIR) / 'app.db'}"
+
+
+def _normalize_sqlite_url(url: str) -> str:
+    if not url.startswith("sqlite:///"):
+        return url
+    db_path = url.replace("sqlite:///", "", 1)
+    if db_path == ":memory:" or os.path.isabs(db_path):
+        return url
+    return f"sqlite:///{(Path(get_app_root()) / db_path).resolve().as_posix()}"
+
+
+# Get database URL from environment, default to SQLite in DATA_DIR
+DATABASE_URL = _normalize_sqlite_url(os.getenv("DATABASE_URL", _default_database_url()))
 
 # Create engine
 engine = create_engine(
@@ -324,6 +344,13 @@ class EmailAccount(TimestampMixin, Base):
     smtp_password  = Column(String, default="")
 
     from_address   = Column(String, default="")
+    display_name   = Column(String, nullable=True)   # "Hriday Ranka" — used in From: header
+
+    # OAuth2 (Google / Google Workspace). Tokens stored encrypted via secret_storage.
+    oauth_provider      = Column(String, nullable=True)   # "google" or None
+    oauth_access_token  = Column(String, nullable=True)   # encrypted
+    oauth_refresh_token = Column(String, nullable=True)   # encrypted
+    oauth_token_expiry  = Column(String, nullable=True)   # unix timestamp string
 
     __table_args__ = (
         Index('ix_email_accounts_owner_default', 'owner', 'is_default'),
@@ -1427,6 +1454,25 @@ def _migrate_add_task_automation_columns():
     except Exception as e:
         logging.getLogger(__name__).warning(f"task automation migration: {e}")
 
+def _migrate_add_email_oauth_columns():
+    """Add Google OAuth and display_name columns to email_accounts if missing."""
+    try:
+        with engine.connect() as conn:
+            cols = [r[1] for r in conn.execute(text("PRAGMA table_info(email_accounts)"))]
+            for col, typedef in [
+                ("oauth_provider",      "TEXT"),
+                ("oauth_access_token",  "TEXT"),
+                ("oauth_refresh_token", "TEXT"),
+                ("oauth_token_expiry",  "TEXT"),
+                ("display_name",        "TEXT"),
+            ]:
+                if col not in cols:
+                    conn.execute(text(f"ALTER TABLE email_accounts ADD COLUMN {col} {typedef}"))
+            conn.commit()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"email oauth columns migration: {e}")
+
+
 def _migrate_add_oauth_config():
     """Add oauth_config column to mcp_servers table if missing."""
     try:
@@ -1602,6 +1648,7 @@ class CalendarCal(TimestampMixin, Base):
     # NULL for local calendars and for CalDAV calendars created before
     # multi-account support was added (treated as "use any configured account").
     account_id = Column(String, nullable=True, index=True)
+    caldav_base_url = Column(String, nullable=True)
 
     events = relationship("CalendarEvent", back_populates="calendar", cascade="all, delete-orphan")
 
@@ -1632,10 +1679,27 @@ class CalendarEvent(TimestampMixin, Base):
     # vanishes upstream). NULL/local = created locally (agent, email triage, or
     # a UI event whose write-back failed) and must NOT be pruned by the sync.
     origin      = Column(String, nullable=True, index=True)
+    remote_href = Column(String, nullable=True)        # CalDAV object URL for updates/deletes
+    remote_etag = Column(String, nullable=True)        # Last seen CalDAV ETag, when available
+    caldav_sync_pending = Column(String, nullable=True) # create | update | delete retry marker
 
     calendar = relationship("CalendarCal", back_populates="events")
 
 
+class CalendarDeletedEvent(TimestampMixin, Base):
+    """Hidden CalDAV delete tombstone retained until remote delete succeeds."""
+    __tablename__ = "caldav_deleted_events"
+
+    uid = Column(String, primary_key=True, index=True)
+    owner = Column(String, nullable=True, index=True)
+    calendar_id = Column(String, nullable=True, index=True)
+    remote_href = Column(String, nullable=True)
+    remote_etag = Column(String, nullable=True)
+    caldav_base_url = Column(String, nullable=True)
+    summary = Column(String, nullable=True)
+    last_error = Column(Text, nullable=True)
+
+
 class Integration(TimestampMixin, Base):
     """An external service connection (email, RSS, webhook, etc.)."""
     __tablename__ = "integrations"
@@ -1753,6 +1817,7 @@ def init_db():
     _migrate_add_tidy_verdict()
     _migrate_add_doc_source_email_cols()
     _migrate_add_oauth_config()
+    _migrate_add_email_oauth_columns()
     _migrate_add_task_automation_columns()
     _migrate_add_disabled_tools()
     _migrate_add_mcp_oauth_tokens_column()
@@ -1767,6 +1832,7 @@ def init_db():
     _migrate_add_calendar_is_utc()
     _migrate_add_calendar_origin()
     _migrate_add_calendar_account_id()
+    _migrate_add_caldav_sync_columns()
     _migrate_chat_messages_fts()
     _migrate_encrypt_email_passwords()
     _migrate_encrypt_signatures()
@@ -2067,6 +2133,31 @@ def _migrate_add_calendar_account_id():
             pass
 
 
+def _migrate_add_caldav_sync_columns():
+    """Add remote CalDAV metadata used for bidirectional sync."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    try:
+        conn = sqlite3.connect(db_path)
+        ev_columns = [row[1] for row in conn.execute("PRAGMA table_info(calendar_events)").fetchall()]
+        if ev_columns and "remote_href" not in ev_columns:
+            conn.execute("ALTER TABLE calendar_events ADD COLUMN remote_href TEXT")
+        if ev_columns and "remote_etag" not in ev_columns:
+            conn.execute("ALTER TABLE calendar_events ADD COLUMN remote_etag TEXT")
+        if ev_columns and "caldav_sync_pending" not in ev_columns:
+            conn.execute("ALTER TABLE calendar_events ADD COLUMN caldav_sync_pending TEXT")
+
+        cal_columns = [row[1] for row in conn.execute("PRAGMA table_info(calendars)").fetchall()]
+        if cal_columns and "caldav_base_url" not in cal_columns:
+            conn.execute("ALTER TABLE calendars ADD COLUMN caldav_base_url TEXT")
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"CalDAV sync metadata migration failed: {e}")
+
+
 def _migrate_add_calendar_metadata():
     """Add importance/event_type/last_pinged columns to calendar_events table."""
     import sqlite3

core/platform_compat.py

@@ -191,6 +191,8 @@ def _windows_bash_fallbacks() -> List[str]:
         base = os.environ.get(env_name)
         if base:
             roots.append(ntpath.join(base, "Git"))
+            if env_name == "LocalAppData":
+                roots.append(ntpath.join(base, "Programs", "Git"))
     roots.extend(_WINDOWS_BASH_DEFAULT_ROOTS)
 
     paths: List[str] = []
@@ -298,7 +300,7 @@ def is_wsl() -> bool:
     import sys
     if sys.platform.startswith("linux") or os.name == "posix":
         try:
-            with open("/proc/version", "r") as f:
+            with open("/proc/version", "r", encoding="utf-8", errors="ignore") as f:
                 if "microsoft" in f.read().lower():
                     return True
         except Exception:

docker-compose.gpu-amd.yml

@@ -16,18 +16,18 @@ services:
     ports:
       - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
     volumes:
-      - ./data:/app/data:z
-      - ./logs:/app/logs:z
+      - ${APP_DATA_DIR:-./data}:/app/data:z
+      - ${APP_LOGS_DIR:-./logs}:/app/logs:z
       # Cookbook remote-server SSH identity. Odysseus can generate a key here;
       # add the shown public key to each remote server's authorized_keys.
-      - ./data/ssh:/app/.ssh:z
+      - ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z
       # Cookbook local model cache. Inside Docker, "Local" means the Odysseus
       # container, so persist its HuggingFace cache under ./data/huggingface.
-      - ./data/huggingface:/app/.cache/huggingface:z
+      - ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z
       # Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.)
       # land under /app/.local for the odysseus user. Persist them so a
       # container recreate does not silently remove installed serve engines.
-      - ./data/local:/app/.local:z
+      - ${APP_DATA_DIR:-./data}/local:/app/.local:z
     extra_hosts:
       # Lets the container reach local services on the Docker host, including
       # Ollama at http://host.docker.internal:11434.
@@ -60,6 +60,13 @@ services:
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
+      - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
+      - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
+      - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
+      - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker-compose.gpu-nvidia.yml

@@ -15,18 +15,18 @@ services:
     ports:
       - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
     volumes:
-      - ./data:/app/data:z
-      - ./logs:/app/logs:z
+      - ${APP_DATA_DIR:-./data}:/app/data:z
+      - ${APP_LOGS_DIR:-./logs}:/app/logs:z
       # Cookbook remote-server SSH identity. Odysseus can generate a key here;
       # add the shown public key to each remote server's authorized_keys.
-      - ./data/ssh:/app/.ssh:z
+      - ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z
       # Cookbook local model cache. Inside Docker, "Local" means the Odysseus
       # container, so persist its HuggingFace cache under ./data/huggingface.
-      - ./data/huggingface:/app/.cache/huggingface:z
+      - ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z
       # Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.)
       # land under /app/.local for the odysseus user. Persist them so a
       # container recreate does not silently remove installed serve engines.
-      - ./data/local:/app/.local:z
+      - ${APP_DATA_DIR:-./data}/local:/app/.local:z
     extra_hosts:
       # Lets the container reach local services on the Docker host, including
       # Ollama at http://host.docker.internal:11434.
@@ -59,6 +59,13 @@ services:
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
+      - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
+      - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
+      - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
+      - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker-compose.yml

@@ -4,18 +4,18 @@ services:
     ports:
       - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
     volumes:
-      - ./data:/app/data:z
-      - ./logs:/app/logs:z
+      - ${APP_DATA_DIR:-./data}:/app/data:z
+      - ${APP_LOGS_DIR:-./logs}:/app/logs:z
       # Cookbook remote-server SSH identity. Odysseus can generate a key here;
       # add the shown public key to each remote server's authorized_keys.
-      - ./data/ssh:/app/.ssh:z
+      - ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z
       # Cookbook local model cache. Inside Docker, "Local" means the Odysseus
       # container, so persist its HuggingFace cache under ./data/huggingface.
-      - ./data/huggingface:/app/.cache/huggingface:z
+      - ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z
       # Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.)
       # land under /app/.local for the odysseus user. Persist them so a
       # container recreate does not silently remove installed serve engines.
-      - ./data/local:/app/.local:z
+      - ${APP_DATA_DIR:-./data}/local:/app/.local:z
     extra_hosts:
       # Lets the container reach local services on the Docker host, including
       # Ollama at http://host.docker.internal:11434.
@@ -48,6 +48,13 @@ services:
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
+      - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
+      - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
+      - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
+      - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker/entrypoint.sh

@@ -13,6 +13,8 @@ set -e
 
 PUID="${PUID:-1000}"
 PGID="${PGID:-1000}"
+GOSU_BIN="$(command -v gosu)"
+PYTHON_BIN="$(command -v python)"
 
 # Reuse an existing matching group/user if the host's UID/GID already
 # corresponds to one in /etc/passwd (e.g. when the image is rebuilt
@@ -24,26 +26,57 @@ if ! getent passwd "$PUID" >/dev/null 2>&1; then
     useradd -u "$PUID" -g "$PGID" -M -s /bin/sh -d /app odysseus
 fi
 
-# Repair ownership on every writable path the app touches at runtime.
-#
-# Bind-mounted dirs (/app/data, /app/logs) are the obvious ones, but
-# the app ALSO writes inside the image's own source tree at runtime:
-#   - services/cache/{search,content}/*  (search cache LRU)
-#   - services/search_analytics.json
-#   - services/search_engine_error.log
-#   - services/tts cache, etc.
-# These dirs were created as root during `docker build`, so dropping
-# to PUID:PGID would otherwise crash on the first import that tries
-# to mkdir them. Chown the whole /app tree — fast (<1s on this size)
-# and idempotent via the `-not -uid` filter so we only touch files
-# that need fixing.
-for dir in /app /app/data /app/logs; do
+mount_root_for() {
+    awk -v target="$1" '$5 == target { print $4; exit }' /proc/self/mountinfo 2>/dev/null || true
+}
+
+is_broad_mount_root() {
+    case "$1" in
+        /|/home|/srv|/var|/usr|/opt|/tmp|/mnt|/media)
+            return 0
+            ;;
+    esac
+    return 1
+}
+
+repair_tree_ownership() {
+    dir="$1"
     if [ -d "$dir" ]; then
-        # `find ... -not -uid` keeps this O(touched-files), not
-        # O(everything), so terabyte-sized maildirs don't slow startup.
-        find "$dir" -not -uid "$PUID" -print0 2>/dev/null \
+        find "$dir" -xdev -not -uid "$PUID" -print0 2>/dev/null \
             | xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true
     fi
+}
+
+repair_app_tree_ownership() {
+    if [ -d /app ]; then
+        find /app -xdev \
+            \( -path /app/data -o -path /app/logs -o -path /app/.ssh -o -path /app/.cache -o -path /app/.local \) -prune \
+            -o -not -uid "$PUID" -print0 2>/dev/null \
+            | xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true
+    fi
+}
+
+repair_bind_mount_ownership() {
+    dir="$1"
+    if [ ! -d "$dir" ]; then
+        return
+    fi
+
+    mount_root="$(mount_root_for "$dir")"
+    if is_broad_mount_root "$mount_root"; then
+        echo "Skipping recursive ownership repair for $dir because it maps to broad host path $mount_root" >&2
+        chown "$PUID:$PGID" "$dir" 2>/dev/null || true
+        return
+    fi
+
+    repair_tree_ownership "$dir"
+}
+
+# Repair image-owned writable paths without walking into bind-mounted host
+# trees, then repair the app-owned mount roots separately.
+repair_app_tree_ownership
+for dir in /app/data /app/logs /app/.ssh /app/.cache/huggingface /app/.local; do
+    repair_bind_mount_ownership "$dir"
 done
 
 # Cookbook installs vllm/etc. via `pip install --user`, which pulls
@@ -83,9 +116,9 @@ export PATH="/app/.local/bin:$PATH"
 # Run first-time setup as the app user so data/ files get the right ownership.
 # setup.py is idempotent — skips auth.json / .env if they already exist.
 # || true so a setup failure never prevents the container from starting.
-gosu "$PUID:$PGID" python /app/setup.py || true
+"$GOSU_BIN" "$PUID:$PGID" "$PYTHON_BIN" /app/setup.py || true
 
 # Drop root and run the actual app. `gosu` is preferred over `su` /
 # `sudo` because it cleans up the process tree (no extra shell layer)
 # so signals (SIGTERM from `docker stop`) reach uvicorn directly.
-exec gosu "$PUID:$PGID" "$@"
+exec "$GOSU_BIN" "$PUID:$PGID" "$@"

docs/agent-migration.md

@@ -0,0 +1,194 @@
+# Agent migration manifests
+
+Odysseus should be able to learn from another agent without blindly trusting
+that agent's whole state. The safe migration path is:
+
+```text
+source agent export -> source adapter -> agent-migration.v1 manifest -> preview -> apply
+```
+
+The manifest is intentionally source-neutral. OpenClaw, Hermes, a folder of
+Markdown notes, or any other agent can have its own adapter, but Odysseus only
+needs to understand the normalized manifest.
+
+## Why not import everything as memory?
+
+Durable memory should stay compact and useful. Long notes, logs, session
+transcripts, and project archives are useful context, but they are not all
+memories. A good migration keeps two layers separate:
+
+- **Archive documents** preserve source material for search, reading, and later
+  extraction.
+- **Memory candidates** are short facts or preferences that can be reviewed
+  before being saved into Odysseus memory.
+
+This keeps Odysseus' existing memory-review flow intact while giving it better
+source material to review.
+
+## Manifest shape
+
+`agent-migration.v1` is a JSON object:
+
+```json
+{
+  "schema_version": "agent-migration.v1",
+  "generated_at": "2026-06-06T00:00:00Z",
+  "source": {
+    "name": "example-agent",
+    "kind": "generic"
+  },
+  "summary": {
+    "item_count": 3,
+    "counts_by_kind": {
+      "memory": 1,
+      "skill": 1,
+      "conversation_thread": 1,
+      "archive_document": 1
+    },
+    "warning_count": 0
+  },
+  "items": [],
+  "warnings": []
+}
+```
+
+Each item has a stable `id`, a `kind`, source metadata, and enough content for a
+future importer to preview it before applying.
+
+Supported item kinds in the first pass:
+
+- `memory` — a candidate memory with `text`, `category`, `source`, and
+  provenance metadata.
+- `skill` — a `SKILL.md` file with content and parsed frontmatter metadata.
+- `conversation_thread` — a normalized transcript thread from an exported chat
+  history. Message content is optional; adapters can preserve only thread
+  metadata, message counts, timestamps, and hashes when a manifest should stay
+  small or avoid embedding private transcript text.
+- `archive_document` — long-form source material. Content is optional; adapters
+  can preserve only path/hash/size metadata when a manifest should stay small.
+
+## Build a manifest
+
+Use the read-only helper:
+
+```bash
+python3 scripts/agent_migration_manifest.py \
+  --source-name old-agent \
+  --source-kind generic \
+  --memory-json /path/to/memories.json \
+  --skills-dir /path/to/skills \
+  --conversation-json /path/to/conversations.json \
+  --archive /path/to/notes \
+  --output /tmp/agent-migration.json
+```
+
+The helper does not write to `data/`, call an LLM, import Odysseus modules, or
+modify the source. It only writes JSON.
+
+Memory JSON may be:
+
+```json
+[
+  "A plain memory string",
+  {
+    "text": "A categorized memory",
+    "category": "preference",
+    "source": "old-agent"
+  }
+]
+```
+
+or an object containing a list under `memories`, `memory`, `items`, or `data`.
+
+Skills are scanned recursively for `SKILL.md`:
+
+```bash
+python3 scripts/agent_migration_manifest.py \
+  --source-name hermes \
+  --source-kind hermes \
+  --skills-dir ~/.hermes/skills \
+  --output /tmp/hermes-skills-manifest.json
+```
+
+Archive documents are metadata-only by default. To embed text content:
+
+```bash
+python3 scripts/agent_migration_manifest.py \
+  --source-name notes-export \
+  --archive /path/to/markdown-notes \
+  --include-archive-content \
+  --output /tmp/notes-manifest.json
+```
+
+Conversation exports are also metadata-only by default:
+
+```bash
+python3 scripts/agent_migration_manifest.py \
+  --source-name chatgpt-export \
+  --source-kind chatgpt \
+  --conversation-json /path/to/conversations.json \
+  --output /tmp/chatgpt-conversations-manifest.json
+```
+
+The first pass supports generic conversation JSON such as:
+
+```json
+[
+  {
+    "id": "thread-1",
+    "title": "Project plan",
+    "messages": [
+      {"role": "user", "content": "Can we design this?"},
+      {"role": "assistant", "content": "Yes, start with a narrow slice."}
+    ]
+  }
+]
+```
+
+It also recognizes ChatGPT-style `mapping` exports from `conversations.json`.
+To embed normalized messages:
+
+```bash
+python3 scripts/agent_migration_manifest.py \
+  --source-name chatgpt-export \
+  --source-kind chatgpt \
+  --conversation-json /path/to/conversations.json \
+  --include-conversation-content \
+  --max-conversation-messages 2000 \
+  --output /tmp/chatgpt-conversations-with-content.json
+```
+
+Content embedding is explicit because exported chat histories can be huge and
+private. A future source-specific adapter can add ZIP traversal, attachment
+metadata, and provider-specific project/workspace fields while still emitting
+the same `conversation_thread` manifest item.
+
+## Recommended apply behavior
+
+A future Odysseus importer should treat the manifest as untrusted user-provided
+data and apply it in stages:
+
+1. Show a dry-run summary with counts, warnings, duplicates, and sample items.
+2. Back up current `data/` state before writing anything.
+3. Import archive documents as documents or another searchable source, not as
+   memory.
+4. Import conversation threads as searchable archived context first, with
+   citations back to the source thread. Do not turn whole transcripts into
+   memory.
+5. Show memory candidates for review before saving through the normal memory
+   path.
+6. Import skills only after name/category conflict checks.
+7. Skip secrets by default. Credentials need explicit, provider-specific flows.
+
+## What belongs in source adapters?
+
+Adapters can be source-specific. The core manifest should not be.
+
+For example, an OpenClaw adapter may know about OpenClaw's workspace files. A
+Hermes adapter may know about `~/.hermes/config.yaml` and `~/.hermes/skills`.
+A ChatGPT adapter may know about `conversations.json`, uploaded-file metadata,
+and image attachment directories. A Claude adapter may know about Claude's
+export shape and project boundaries. A generic adapter may only know about
+memory JSON, conversation JSON, `SKILL.md`, and Markdown folders.
+
+Nonstandard folders should be adapter details, not required Odysseus concepts.

docs/backup-restore.md

@@ -0,0 +1,129 @@
+# Backup & Restore
+
+Odysseus keeps all of your state in the `data/` directory — the SQLite database
+(`app.db`), the Fernet encryption key (`data/.app_key`), the vault, memory, RAG
+indexes, personal documents, and uploads. The `scripts/odysseus-backup` tool
+snapshots that directory into a single gzip tarball and restores it later.
+
+Snapshots are safe to take while the app is running: SQLite databases are copied
+through SQLite's own `.backup` API rather than a raw file copy, so an in-flight
+write can't corrupt the snapshot.
+
+> **A snapshot contains your secrets.** The tarball includes the Fernet
+> encryption key (`data/.app_key`), the vault, sessions, and any stored
+> provider/API tokens — so treat it like a password. Store backups somewhere
+> private, never commit them to Git, and prefer an encrypted destination when
+> copying them offsite.
+
+## Quick start
+
+Run the tool from the repository root:
+
+```bash
+# Create a snapshot → backups/odysseus-backup-<YYYYMMDD-HHMMSS>.tar.gz
+./scripts/odysseus-backup snapshot
+
+# List existing snapshots (most recent first)
+./scripts/odysseus-backup list
+
+# Check a tarball's integrity without extracting it
+./scripts/odysseus-backup verify backups/odysseus-backup-20260101-120000.tar.gz
+
+# Restore (destructive — see the warning below)
+./scripts/odysseus-backup restore backups/odysseus-backup-20260101-120000.tar.gz --yes
+```
+
+The script depends only on the Python standard library, so any `python3` on your
+`PATH` will run it — you don't need the app's virtualenv active.
+
+Every command prints a JSON result. Add `--pretty` for indented output.
+
+## Commands
+
+### `snapshot`
+
+Writes a `tar.gz` of `data/` to `backups/<timestamp>.tar.gz`.
+
+| Flag | Effect |
+| --- | --- |
+| `--out PATH` | Write to a specific path instead of the default `backups/` location. Must be **outside** `data/`. |
+| `--include-research` | Include `data/deep_research/` (skipped by default — research runs are large). |
+| `--include-attachments` | Include `data/mail-attachments/` (skipped by default — cached IMAP extractions, re-derivable). |
+
+By default the snapshot includes everything under `data/` **except**
+`deep_research/` and `mail-attachments/`. Personal uploads and documents are
+included.
+
+```bash
+# Snapshot straight to a mounted NAS path
+./scripts/odysseus-backup snapshot --out /mnt/nas/odysseus-$(date +%F).tar.gz
+
+# Full snapshot including research runs and mail attachments
+./scripts/odysseus-backup snapshot --include-research --include-attachments
+```
+
+### `list`
+
+Lists the tarballs in `backups/`, most recent first, with size and modification
+time.
+
+### `verify PATH`
+
+Opens the tarball read-only and walks every member to confirm it is intact and
+safe to restore. Nothing is extracted. Use this before relying on an old backup
+or after copying one across machines.
+
+### `restore PATH --yes`
+
+Overwrites `data/` from a tarball.
+
+> **Restore is destructive.** It replaces the current `data/` directory. `--yes`
+> is required so a mistyped command can't wipe your live state.
+
+Restore is not a blind delete: before extracting, the tool **renames your current
+`data/` to `data.before-restore-<timestamp>`** in the repository root. If a
+restore turns out to be wrong, your previous state is still there — delete the
+restored `data/` and rename the stashed directory back. The restore path is also
+validated entry-by-entry: archives containing absolute paths, `..` segments,
+symlinks, or anything outside `data/` are rejected.
+
+## Scheduling offsite backups
+
+The tarball output composes cleanly with cron and any copy tool. For example, a
+nightly snapshot copied offsite:
+
+```cron
+0 3 * * *  cd /path/to/odysseus && ./scripts/odysseus-backup snapshot --out "/mnt/nas/odysseus-$(date +\%F).tar.gz"
+```
+
+Swap the `--out` target for `scp`, `rclone`, `s3cmd`, or similar to push the
+snapshot to remote storage.
+
+## Docker vs native installs
+
+The tool reads `data/` and writes `backups/` relative to the repository root, so
+where you run it matters:
+
+- **Native installs** — run it from the repo root as shown above. `data/` and
+  `backups/` are both in the repo directory.
+- **Docker** — `docker-compose.yml` bind-mounts the host's `./data` to
+  `/app/data`, so the live data is also present on the host. **Run the tool on
+  the host** from the repo root; the snapshot reads the bind-mounted `./data` and
+  writes to `./backups` on the host. Running it *inside* the container is not
+  recommended, because `backups/` is not a mounted volume and the tarball would
+  be lost when the container is recreated.
+
+> **ChromaDB caveat (Docker only).** In the Docker setup, ChromaDB stores its
+> vectors in a separate Compose-managed volume (declared as `chromadb-data`),
+> **not** under `./data`. `odysseus-backup` therefore does not capture the Docker
+> ChromaDB store. Back it up separately if you need it. Compose prefixes the
+> volume with the project name, so find the real name first
+> (`docker volume ls | grep chromadb`), then archive it — for example:
+>
+> ```bash
+> docker run --rm -v <project>_chromadb-data:/data -v "$PWD":/backup \
+>   alpine tar czf /backup/chromadb.tar.gz -C /data .
+> ```
+>
+> On native installs ChromaDB lives at `data/chroma/` and is included in the
+> snapshot normally.

docs/index.html

@@ -25,9 +25,16 @@
     --radius: 8px;
   }
   * { box-sizing: border-box; }
-  html { scroll-behavior: smooth; scroll-snap-type: y proximity; scroll-padding-top: 60px; }
-  /* Each section is a full-viewport "page" with its content centered, so only
-     one shows at a time and the snap is obvious. */
+  html { scroll-behavior: smooth; scroll-padding-top: 60px; }
+  /* REMOVED: "scroll-snap-type: y proximity"
+     The idea was: >>Each section is a full-viewport "page" with its content centered,
+     so only one shows at a time and the snap is obvious.<<
+
+     PROBLEM: sections easily grow taller than 100vh IRL
+     This cause forced jumps mid-read. It's intrusive UX.
+     The landing-page is not a PowerPoint presentation!
+
+     Preserved: CSS snap-points to avoid destroying code meta-data*/
   .hero, section {
     scroll-snap-align: start; min-height: 100vh;
     display: flex; flex-direction: column; justify-content: center;

docs/security-ci.md

@@ -0,0 +1,107 @@
+# Security CI guide
+
+This project runs a set of automated security checks on pull requests and
+selected branch pushes. This page explains what each one does, whether it can
+block a merge, and the few one-time settings you should turn on to get the full
+benefit.
+
+## What runs, and why
+
+Most checks live in files under `.github/workflows/`. CodeQL is configured
+through GitHub's code scanning default setup, so it appears as a dynamic GitHub
+workflow instead of a checked-in workflow file. They run automatically; you do
+not start them.
+
+| Check | What it protects against | Blocks a merge? |
+|---|---|---|
+| **Secret scan** (gitleaks) | An API key, token, or password being committed by mistake or on purpose | Yes |
+| **Workflow security** (actionlint + zizmor) | A broken or insecure automation file that could leak the repo's access token | Yes |
+| **Dependency review** | A pull request that adds a software library with a known security hole | Yes |
+| **pip-audit** | Known security holes in the Python libraries already used | No (advisory) |
+| **Container scan: hadolint** | Mistakes and insecure patterns in the `Dockerfile` | Yes |
+| **Container scan: Trivy** | Known security holes in the Docker image | No (advisory) |
+| **CodeQL** | Real bugs in the app's own code: injection, auth mistakes, path traversal | No (advisory) |
+
+"Blocks a merge" means a red X appears on the pull request and, once you enable
+the setting below, the **Merge** button is disabled until it is fixed.
+
+"Advisory" means it reports problems into the repository's **Security** tab so
+you can review them on your own schedule, but it never stops a merge. These are
+advisory on purpose: they often flag long-standing issues in other people's
+libraries, not something a given pull request introduced.
+
+## Where results appear
+
+- **Checks tab of a pull request**: the pass/fail of each check. A green tick is
+  good; a red X needs attention.
+- **Security tab of the repository**: detailed findings from the advisory
+  scanners (Trivy and CodeQL). This is your dashboard.
+
+## If a check fails
+
+- **Secret scan failed**: a real credential may have been committed. Treat it as
+  leaked: rotate (regenerate) that key or token immediately, then remove it from
+  the file. Do not just delete the commit; assume it was seen.
+- **Dependency review failed**: the pull request adds a library with a known
+  vulnerability. Ask the contributor to use a patched version, or decline the
+  change.
+- **hadolint / workflow security failed**: the contributor changed the
+  `Dockerfile` or an automation file in a way the linter rejects. Ask them to
+  address the message shown in the failed check.
+
+## One-time settings to turn on
+
+These two settings unlock the full value. You only do them once.
+
+### 1. Require the blocking checks before merging
+
+This makes the **Merge** button refuse to work until the gating checks pass.
+
+1. Go to the repository on GitHub.
+2. Click **Settings** (top right of the repo).
+3. In the left sidebar, click **Branches**.
+4. Under **Branch protection rules**, click **Add branch ruleset** (or **Add
+   rule**), and set the branch name pattern to `dev` (this is the branch all
+   pull requests target; `main` is fast-forwarded at releases).
+5. Enable **Require status checks to pass before merging**.
+6. In the search box that appears, add these checks by name:
+   - `Python syntax (compileall)`
+   - `JS syntax (node --check)`
+   - `gitleaks`
+   - `actionlint`
+   - `zizmor (Actions SAST)`
+   - `hadolint (Dockerfile lint)`
+   - `dependency-review (PR gate)`
+
+   The first two come from the correctness CI (`ci.yml`); the rest are this
+   security suite. Leave pytest, pip-audit, Trivy, and CodeQL unchecked so they
+   stay advisory.
+7. Also enable **Require a pull request before merging** and **Require review
+   from Code Owners** (this uses the `.github/CODEOWNERS` file so every change
+   needs your sign-off).
+8. Click **Create** / **Save changes**.
+
+Note: a check name only appears in the list after it has run at least once, so
+let the workflows run on one pull request first, then add them here.
+
+### 2. Turn on the Security tab features
+
+1. **Settings -> Code security** (or **Code security and analysis**).
+2. Turn on **Dependency graph** (usually on by default for public repos) -- this
+   powers Dependency review and Dependabot.
+3. Turn on **Dependabot alerts** and **Dependabot security updates**.
+4. Under **Code scanning**, use **Set up -> Default** for CodeQL. GitHub then
+   runs CodeQL as a dynamic workflow without the fork-token limitations that
+   affect checked-in advanced workflows.
+
+   Do not also add a checked-in CodeQL workflow while default setup is enabled:
+   GitHub rejects advanced CodeQL uploads when default setup is active. If the
+   project later needs an advanced CodeQL workflow, disable default setup first
+   and keep only one CodeQL publishing path active.
+
+## Keeping it current
+
+`.github/dependabot.yml` opens small weekly pull requests to update Python and
+npm packages, the Docker base image, and the pinned automation actions
+themselves. Review and merge those like any other pull request; they keep the
+project patched without manual tracking.

docs/setup.md

@@ -0,0 +1,425 @@
+# Odysseus Setup Guide
+
+This page keeps the detailed install, deployment, troubleshooting, and configuration notes out of the front README.
+
+## Quick Start
+
+> **Branch note:** `dev` is the default branch and contains the latest development changes, but it may be unstable. For the more stable curated branch, use [`main`](https://github.com/pewdiepie-archdaemon/odysseus/tree/main).
+
+Defaults work out of the box: clone, run, then configure models/search/email
+inside **Settings**. Only edit `.env` for deployment-level overrides like
+`APP_BIND`, `APP_PORT`, `AUTH_ENABLED`, `DATABASE_URL`, or a pre-seeded admin password.
+
+On first setup, Odysseus creates an admin account (`admin` unless
+`ODYSSEUS_ADMIN_USER` is set) and prints a temporary password in the terminal.
+For Docker installs, the same line is in `docker compose logs odysseus`.
+Use that for the first login, then change it in **Settings**.
+
+Contributing? See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, testing, and
+pull request guidelines.
+
+### Docker (recommended)
+```bash
+git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+cd odysseus
+cp .env.example .env       # optional, but recommended for explicit defaults
+docker compose up -d --build
+```
+To include optional extras in the image (PDF viewer, Office extraction; includes AGPL PyMuPDF), build with `docker compose build --build-arg INSTALL_OPTIONAL=true` before `up`.
+
+Open `http://localhost:7000` when the containers are healthy. Docker Compose
+binds the web UI to `127.0.0.1` by default. If the port is taken, set
+`APP_PORT=7001` in `.env` and recreate the container. Set `APP_BIND=0.0.0.0`
+only when you intentionally want LAN/reverse-proxy access.
+
+> **On Apple Silicon (M-series) Macs:** Docker can't reach the Metal GPU, so
+> Cookbook serves local models on CPU only. For GPU-accelerated model serving,
+> run natively instead — see [Apple Silicon](#apple-silicon) below.
+
+### Native Linux / macOS
+```bash
+git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+cd odysseus
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+python setup.py
+python -m uvicorn app:app --host 127.0.0.1 --port 7000
+```
+Requirements: Python 3.11+. Cookbook also needs `tmux` for background model
+downloads and serves. The app itself is lightweight; local model serving is the
+heavy part and depends on the model, runtime, GPU, and VRAM, so small hosts can
+connect to API or remote model servers instead. Use `--host 0.0.0.0` only when you intentionally want LAN/reverse-proxy access.
+
+### Apple Silicon
+Docker on macOS cannot use the Metal GPU. For GPU-accelerated Cookbook on an
+M-series Mac, run Odysseus natively:
+
+```bash
+git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+cd odysseus
+./start-macos.sh
+```
+
+It launches at `http://127.0.0.1:7860`. To expose it to your phone over a trusted LAN/VPN such as Tailscale, bind all interfaces:
+
+```bash
+ODYSSEUS_HOST=0.0.0.0 ./start-macos.sh
+# then open http://<tailscale-ip>:7860
+```
+
+The script also reads `.env` at startup, so `APP_BIND=0.0.0.0` and `APP_PORT`
+set there are picked up automatically without a command-line override each run.
+
+Keep `AUTH_ENABLED=true` (the default) before binding outside loopback. Do not
+expose this port directly to the public internet. To build a clickable app wrapper:
+
+```bash
+./build-macos-app.sh
+```
+
+<details>
+<summary>Cookbook, GPU, Ollama, and troubleshooting notes</summary>
+
+**Docker bundled services.** Compose starts Odysseus, ChromaDB, SearXNG, and
+ntfy. Odysseus and the bundled service ports bind to `127.0.0.1` by default, so
+they are reachable from the host but not exposed to your LAN/public internet
+unless you opt in.
+
+**Cookbook storage in Docker.** Downloads live in `./data/huggingface`
+(`~/.cache/huggingface` in the container). Cookbook-installed Python CLIs and
+serve engines live in `./data/local` (`~/.local` in the container), so they
+survive container recreation.
+
+**Remote servers.** In **Cookbook -> Settings -> Servers**, generate the
+Odysseus SSH key and add the public key to the remote server's
+`~/.ssh/authorized_keys`. From the host you can also run:
+
+```bash
+ssh-copy-id -i data/ssh/id_ed25519.pub user@server
+```
+
+**Docker GPU overlays.** CPU-only users can skip this section. Cookbook can
+only detect GPUs that Docker exposes to the container — if the host runtime or
+device passthrough is not configured, Cookbook sees the iGPU, another card, or
+CPU instead of your intended GPU.
+
+For NVIDIA, `scripts/check-docker-gpu.sh` diagnoses GPU passthrough and can
+optionally install the host runtime or update `.env`.
+
+```bash
+# Read-only diagnostic (default — installs nothing, never edits .env):
+scripts/check-docker-gpu.sh
+
+# Print OS-specific install commands without running them:
+scripts/check-docker-gpu.sh --print-install-commands
+
+# Install NVIDIA Container Toolkit on Ubuntu/Debian (requires sudo):
+scripts/check-docker-gpu.sh --install-nvidia-toolkit
+
+# Write COMPOSE_FILE to .env (only when GPU passthrough is confirmed working):
+scripts/check-docker-gpu.sh --enable-nvidia-overlay
+
+# Full assisted setup — install toolkit, then enable overlay if passthrough works:
+scripts/check-docker-gpu.sh --install-nvidia-toolkit --enable-nvidia-overlay
+```
+
+Safety notes:
+- The app never installs host GPU runtime automatically.
+- The app never edits `.env` automatically.
+- `.env` is only modified when `--enable-nvidia-overlay` is explicitly passed,
+  and only after GPU passthrough succeeds. `--yes` skips prompts but does not
+  bypass the passthrough gate.
+- `.env.bak.*` backups created by `--enable-nvidia-overlay` are ignored by
+  Git and the Docker build context.
+
+To enable manually without the script, add this to `.env`:
+
+```bash
+COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml
+```
+
+**AMD / ROCm.** AMD setup is read-only diagnostic plus manual `.env` edit. Run:
+
+```bash
+scripts/check-docker-amd-gpu.sh
+```
+
+Then add the reported values to `.env`, replacing `RENDER_GID` with your host's
+numeric render group id:
+
+```bash
+COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml
+RENDER_GID=989
+```
+
+For NVIDIA/AMD GPU support, also read the comments in the selected overlay file: docker/gpu.nvidia.yml or docker/gpu.amd.yml.
+
+**Stack-management UIs (Portainer, Coolify, Dockhand, etc.).** These tools
+often accept only a single Compose file and do not reliably honor `COMPOSE_FILE`
+or multiple `-f` overlays. CLI users should keep using the `COMPOSE_FILE`
+overlay workflow above. For stack UIs, point the stack at one of the standalone
+files instead, which bundle the base stack plus the GPU settings:
+
+- `docker-compose.gpu-nvidia.yml` — still requires the NVIDIA Container Toolkit
+  on the host.
+- `docker-compose.gpu-amd.yml` — still requires host ROCm/kfd/DRI setup, the
+  `video`/`render` group membership, and `RENDER_GID` when needed.
+
+The base `docker-compose.yml` plus the `docker/gpu.*.yml` overlays remain the
+source of truth; the standalone files mirror them for single-file deployments.
+
+Verify after enabling either overlay:
+
+```bash
+docker compose exec odysseus nvidia-smi -L   # NVIDIA
+docker compose exec odysseus sh -lc 'test -e /dev/kfd && test -d /dev/dri && ls -l /dev/kfd /dev/dri/renderD*'  # AMD
+```
+
+> **GPU passthrough ≠ llama.cpp CUDA.** `nvidia-smi` passing inside the
+> container confirms Docker GPU access, but llama.cpp also needs `cudart` and
+> the CUDA Toolkit at runtime. If Cookbook logs show `Unable to find cudart
+> library`, `Could NOT find CUDAToolkit`, `CUDA Toolkit not found`, or
+> tensors/layers assigned to CPU, that is a Cookbook/llama.cpp build issue —
+> not a Docker passthrough failure. Reinstall the serve engine via
+> **Cookbook → Dependencies** to get a CUDA-enabled build.
+>
+> The same split applies to AMD/ROCm: seeing `/dev/kfd` and `/dev/dri` inside
+> the container confirms device passthrough, not ROCm userspace or a
+> ROCm-enabled vLLM/llama.cpp build. `rocm-smi` and `rocminfo` are not expected
+> inside the slim Odysseus image.
+
+**Ollama with Docker.** If Ollama runs on the host, add this endpoint in
+Settings:
+
+```text
+http://host.docker.internal:11434/v1
+```
+
+Ollama must listen outside its own loopback interface:
+
+```bash
+OLLAMA_HOST=0.0.0.0:11434 ollama serve
+```
+
+This connects Odysseus in Docker to an Ollama server that is already running on
+your host machine; it does not start Ollama inside the container.
+`host.docker.internal` is Docker's hostname for the host machine from inside the
+container. Cookbook **Serve** is a separate workflow for serving downloaded
+models through Odysseus/llama.cpp, so Windows users with an existing Ollama
+install usually only need to add the endpoint in Settings.
+
+**Useful checks.**
+
+```bash
+docker compose ps
+docker compose logs --tail=120 odysseus
+docker compose logs odysseus | grep -E 'ChromaDB|MemoryVectorStore|DEGRADED'
+```
+
+**macOS details.** `start-macos.sh` installs Homebrew deps, creates the venv,
+runs setup, and starts uvicorn on port `7860` because AirPlay often holds
+`7000`. It uses llama.cpp/Ollama for Metal. vLLM/SGLang are CUDA/ROCm-only and
+do not run on macOS. MLX-only models are not served by Odysseus.
+
+</details>
+
+### Native Windows
+
+**One-command launcher** (creates the venv, installs deps, runs setup, starts the
+server; safe to re-run):
+
+```powershell
+git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+cd odysseus
+powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1
+```
+
+Or do it by hand:
+
+```powershell
+git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+cd odysseus
+py -3.11 -m venv venv
+venv\Scripts\Activate.ps1
+pip install -r requirements.txt
+python setup.py
+python -m uvicorn app:app --host 127.0.0.1 --port 7000
+```
+
+If `python` points at an older interpreter, use `py -3.12` (or another installed
+3.11+ version) for the venv step.
+
+**Requirements:** Python 3.11+. The core app (chat, agent, memory, documents,
+email, calendar, deep research) runs fully native. For full **Cookbook** background
+model downloads and the agent shell tool, also install
+[Git for Windows](https://git-scm.com/download/win) (provides `bash.exe`).
+Local GPU *serving* of vLLM/SGLang needs Linux/WSL2; for a local model on Windows,
+[Ollama](https://ollama.com/download) is the easiest path — point Odysseus at
+`http://localhost:11434/v1` in Settings.
+
+Open `http://localhost:7000`, log in with the generated admin password,
+and configure everything else inside **Settings**.
+
+## Troubleshooting & Advanced Setup
+
+### `chromadb-client` conflicts with embedded ChromaDB
+If `chromadb-client` (the lightweight HTTP-only package) is installed alongside the full `chromadb` package, Odysseus starts but ChromaDB silently falls back to HTTP-only mode and fails.
+
+**Fix:** uninstall `chromadb-client` and force-reinstall the full package:
+```bash
+./venv/bin/pip uninstall chromadb-client -y
+./venv/bin/pip install --force-reinstall chromadb
+```
+
+### HTTPS + LAN/Tailscale exposure
+To expose Odysseus on a local network or Tailscale with HTTPS:
+1. Change the bind address to `0.0.0.0` in `.env` (`APP_BIND=0.0.0.0` or `ODYSSEUS_HOST=0.0.0.0`).
+2. Generate a locally-trusted cert for your LAN/Tailscale IPs using [mkcert](https://github.com/FiloSottile/mkcert):
+   ```bash
+   mkcert -install
+   mkcert -cert-file cert.pem -key-file key.pem 192.168.1.100 tailscale-ip
+   ```
+3. Run `uvicorn` with the generated certs:
+   ```bash
+   python -m uvicorn app:app --host 0.0.0.0 --port 7000 --ssl-certfile=cert.pem --ssl-keyfile=key.pem
+   ```
+4. Install the `mkcert` CA on any other device you want to access Odysseus from (e.g., for iOS, email the `rootCA.pem` to yourself, install the profile, and trust it in Certificate Trust Settings).
+
+### Optional Dependencies
+`requirements-optional.txt` contains packages that unlock extra features. It is not installed by default.
+
+| Package | Feature unlocked |
+|---------|-----------------|
+| `faster-whisper` | Local speech-to-text (microphone -> text) via the "local" STT provider. |
+| `ddgs` | DuckDuckGo as a search provider option. |
+| `PyMuPDF` | PDF page rendering in the side viewer panel and form-filling. (Note: AGPL-3.0) |
+| `markitdown` | Office/EPUB document text extraction (converts .docx/.xlsx/.pptx/.xls/.epub to Markdown). |
+
+### Faster, reproducible installs with uv (optional)
+[uv](https://docs.astral.sh/uv/) works as a drop-in replacement for the
+venv + pip steps in the native install guides, no project changes are needed but this change results in faster installs along with a lockfile for reproducible environments. After [installing `uv`](https://docs.astral.sh/uv/getting-started/installation/), use:
+
+```bash
+uv venv venv --python 3.13
+uv pip install -r requirements.txt
+# then continue as usual: python setup.py, uvicorn, ...
+```
+
+`requirements.txt` is intentionally unpinned, so two installs at different times can produce different package versions. If you want a reproducible environment (e.g. across your own machines, or to roll back after a bad upgrade), snapshot and restore exact versions with:
+
+```bash
+uv pip compile requirements.txt -o requirements.lock   # snapshot current resolution
+uv pip sync requirements.lock                          # reproduce it exactly later
+```
+
+`requirements.lock` is gitignored and platform-specific (compile it on the OS you deploy to). Regenerate it deliberately when you want to take upgrades. The plain `uv pip install -r requirements.txt` keeps following the unpinned requirements like pip does.
+
+### Outlook / Office 365 email
+Odysseus email accounts currently use IMAP/SMTP username-password auth. Outlook
+and Microsoft 365 generally require OAuth instead, so normal Microsoft mailbox
+passwords will fail. See [docs/email-outlook.md](docs/email-outlook.md) for the
+current limitation and the planned integration direction.
+
+## Security Notes
+Odysseus is a self-hosted workspace with powerful local tools: shell access, file uploads, model downloads, web research, email/calendar integrations, and API tokens. Treat it like an admin console.
+
+- Keep `AUTH_ENABLED=true` for any network-accessible deployment.
+- Keep `LOCALHOST_BYPASS=false` outside local development.
+- Use `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway.
+- Do not expose it directly to the public internet without HTTPS and a trusted reverse proxy or private access layer.
+- Keep `.env`, `data/`, `logs/`, databases, uploads, generated media, backups, auth/session files, API keys, and model/provider tokens out of Git and private shares. They are ignored by default.
+- Review `data/auth.json` after first boot: disable open signup unless you intentionally want it, make only your own account admin, and keep demo/test accounts non-admin.
+- Non-admin users do not get shell/Python/file read/write by default, and admin-only routes/tools such as MCP management, API tokens, webhooks, model/cookbook serving, backup/vault, and app settings are admin-gated. Other features are controlled by per-user privileges, so review each user's privileges before exposing a deployment.
+- Rotate any API keys or tokens that were ever pasted into a shared chat, demo, screenshot, or log.
+- If you enable API tokens or webhooks, create separate tokens per integration and delete unused ones.
+- Prefer binding manual development runs to `127.0.0.1`; bind to `0.0.0.0` only when you intentionally want LAN/reverse-proxy access.
+- Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only. Expose only the authenticated Odysseus web/API entrypoint through your trusted proxy or private access layer.
+- Before publishing a fork, run `git status --short` and confirm no private files from `.env`, `data/`, `logs/`, uploads, backups, or local databases are staged.
+
+### Private or proxied deployments
+Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and the bundled services to `127.0.0.1` by default, so a typical production/private setup is:
+
+1. Keep Odysseus on localhost, for example `127.0.0.1:7000`.
+2. Terminate HTTPS at a trusted reverse proxy or private access gateway.
+3. Put the authenticated Odysseus web/API entrypoint behind that layer.
+4. Keep raw service and model ports internal-only.
+
+Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true`, `LOCALHOST_BYPASS=false`, and `SECURE_COOKIES=true`.
+`ALLOWED_ORIGINS` lists exact permitted origins for cross-origin browser/API clients; ordinary same-origin reverse-proxy access usually does not need a special CORS entry.
+
+Common internal-only ports from the default docs/compose setup:
+
+| Port | Service |
+|---|---|
+| `7000` | Odysseus raw app port |
+| `8080` | SearXNG |
+| `8091` | ntfy |
+| `8100` | ChromaDB host port for manual/compose access |
+| `11434` | Ollama |
+| `8000-8020` | Common local model/provider APIs |
+
+## Configuration
+Most setup is done inside the app with `/setup` or **Settings**. Use `.env`
+for deployment-level defaults and secrets you want present before first boot.
+Key settings:
+
+| Variable | Default | Description |
+|---|---|---|
+| `LLM_HOST` | `localhost` | Your LLM server (e.g. `llm-host.local:8000`) |
+| `LLM_HOSTS` | -- | Comma-separated list for model discovery |
+| `OPENAI_API_KEY` | -- | Optional OpenAI key. Prefer adding providers in the app unless pre-seeding. |
+| `SEARXNG_INSTANCE` | `http://localhost:8080` | SearXNG URL. Docker overrides this to `http://searxng:8080`. |
+| `SEARXNG_SECRET` | generated on first Docker boot | Optional SearXNG cookie/CSRF secret. Leave blank unless you need to pin it. |
+| `APP_BIND` | `127.0.0.1` | Docker Compose host bind address for the web UI. Use `0.0.0.0` only for intentional LAN/reverse-proxy access. |
+| `APP_PORT` | `7000` | Docker Compose host port for the web UI. |
+| `APP_DATA_DIR` | `./data` | Docker Compose host directory for application data volumes. |
+| `APP_LOGS_DIR` | `./logs` | Docker Compose host directory for application logs. |
+| `AUTH_ENABLED` | `true` | Enable/disable login |
+| `LOCALHOST_BYPASS` | `false` | Development-only auth bypass for loopback requests. Keep false for shared/network deployments. |
+| `ALLOWED_ORIGINS` | `http://localhost,http://127.0.0.1` | Comma-separated exact permitted origins for cross-origin browser/API clients. |
+| `SECURE_COOKIES` | `false` | Set true when serving Odysseus through HTTPS at a trusted proxy or private access gateway. |
+| `DATABASE_URL` | `sqlite:///./data/app.db` | Database connection string |
+| `CHROMADB_HOST` | `localhost` | ChromaDB host for vector memory. Docker overrides this to `chromadb`. |
+| `CHROMADB_PORT` | `8100` | ChromaDB port for manual host runs. Docker overrides this to `8000`. |
+| `EMBEDDING_URL` | -- | OpenAI-compatible embeddings endpoint |
+| `ODYSSEUS_CHAT_UPLOAD_MAX_BYTES` | `10485760` | Chat/agent attachment cap in bytes. Raise for larger local PDFs or text documents. |
+| `ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES` | `104857600` | Gallery image upload cap in bytes (100 MB). |
+| `ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES` | `26214400` | Gallery transform input cap in bytes (25 MB). |
+| `ODYSSEUS_MEMORY_IMPORT_MAX_BYTES` | `10485760` | Memory import file cap in bytes (10 MB). |
+| `ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES` | `26214400` | Personal document upload cap in bytes (25 MB). |
+| `ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES` | `26214400` | Email compose attachment cap in bytes (25 MB). |
+| `ODYSSEUS_STT_MAX_AUDIO_BYTES` | `26214400` | Speech-to-text audio cap in bytes (25 MB). |
+| `ODYSSEUS_ICS_MAX_BYTES` | `10485760` | Calendar `.ics` import cap in bytes (10 MB). |
+
+All upload-limit vars are validated (must be a positive integer) and optional; an invalid value fails fast at startup.
+
+### Built-in MCP servers (optional setup)
+
+Odysseus auto-registers a few built-in MCP servers at startup. The npx-based ones (currently the browser server, `@playwright/mcp`) only start when their npm package is already in the local npx cache. If a package isn't cached, that server is skipped with a startup log message explaining what to do, so a fresh install does not block on a multi-minute npm download or hang if Playwright system deps are missing.
+
+To enable the browser MCP (page navigation, screenshots, vision), run once:
+
+```bash
+npx -y @playwright/mcp@latest --version
+```
+
+That installs `@playwright/mcp` plus Playwright (~300MB total). Restart Odysseus and the server will register at startup.
+
+## Architecture
+```
+app.py                   # FastAPI entry point
+core/      auth, database, middleware, constants
+src/       llm_core, agent_loop, agent_tools, chat_processor, search/
+routes/    chat, session, document, memory, model … endpoints
+services/  docs, memory, search, hwfit (Cookbook) …
+static/    index.html + app.js + style.css + js/ (modular front-end)
+docs/      landing page (index.html) + preview clips
+```
+
+## Data
+All user data lives in `data/` (gitignored): `app.db` (sessions, messages, documents),
+`memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`.
+
+To back up or restore everything in `data/`, see the
+[Backup & Restore guide](docs/backup-restore.md).

integrations/claude/skills/odysseus/SKILL.md

@@ -102,6 +102,7 @@ python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py POST /api/codex/memory
 
 ## Email draft + send
 
+- Prefer `POST /api/codex/emails/draft-document` for agent-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send.
 - `POST /api/codex/emails/draft` — body matches `SendEmailRequest` (`to`, `cc`, `bcc`, `subject`, `body`, `body_html`, `attachments`, `account_id`, `in_reply_to`, `references`). Requires `email:draft` (or `email:send`).
 - `POST /api/codex/emails/send` — same body. Requires `email:send`. Never send without explicit user instruction.
 

integrations/claude/skills/odysseus/scripts/odysseus_api.py

@@ -17,6 +17,11 @@ def _usage() -> int:
     print("  odysseus_api.py todos add TITLE", file=sys.stderr)
     print("  odysseus_api.py emails list [limit]", file=sys.stderr)
     print("  odysseus_api.py emails read UID", file=sys.stderr)
+    print("  odysseus_api.py emails draft-doc JSON_PAYLOAD", file=sys.stderr)
+    print("  odysseus_api.py documents list [limit]", file=sys.stderr)
+    print("  odysseus_api.py documents read DOC_ID", file=sys.stderr)
+    print("  odysseus_api.py documents create JSON_PAYLOAD", file=sys.stderr)
+    print("  odysseus_api.py documents delete DOC_ID", file=sys.stderr)
     print("  odysseus_api.py cookbook tasks", file=sys.stderr)
     print("  odysseus_api.py cookbook servers", file=sys.stderr)
     print("  odysseus_api.py cookbook cached [HOST]", file=sys.stderr)
@@ -79,6 +84,33 @@ def main() -> int:
             method = "GET"
             path = f"/api/codex/emails/{sys.argv[3]}"
             body = None
+        elif action in ("draft-doc", "draft_document") and len(sys.argv) >= 4:
+            method = "POST"
+            path = "/api/codex/emails/draft-document"
+            body = " ".join(sys.argv[3:])
+        else:
+            return _usage()
+    elif command in ("documents", "docs"):
+        if len(sys.argv) < 3:
+            return _usage()
+        action = sys.argv[2].lower()
+        if action == "list":
+            method = "GET"
+            limit = sys.argv[3] if len(sys.argv) >= 4 else "50"
+            path = f"/api/codex/documents?limit={limit}"
+            body = None
+        elif action == "read" and len(sys.argv) >= 4:
+            method = "GET"
+            path = f"/api/codex/documents/{sys.argv[3]}"
+            body = None
+        elif action == "create" and len(sys.argv) >= 4:
+            method = "POST"
+            path = "/api/codex/documents"
+            body = " ".join(sys.argv[3:])
+        elif action == "delete" and len(sys.argv) >= 4:
+            method = "DELETE"
+            path = f"/api/codex/documents/{sys.argv[3]}"
+            body = None
         else:
             return _usage()
     elif command == "cookbook":

integrations/codex/scripts/odysseus_api.py

@@ -17,6 +17,11 @@ def _usage() -> int:
     print("  odysseus_api.py todos add TITLE", file=sys.stderr)
     print("  odysseus_api.py emails list [limit]", file=sys.stderr)
     print("  odysseus_api.py emails read UID", file=sys.stderr)
+    print("  odysseus_api.py emails draft-doc JSON_PAYLOAD", file=sys.stderr)
+    print("  odysseus_api.py documents list [limit]", file=sys.stderr)
+    print("  odysseus_api.py documents read DOC_ID", file=sys.stderr)
+    print("  odysseus_api.py documents create JSON_PAYLOAD", file=sys.stderr)
+    print("  odysseus_api.py documents delete DOC_ID", file=sys.stderr)
     print("  odysseus_api.py cookbook tasks", file=sys.stderr)
     print("  odysseus_api.py cookbook servers", file=sys.stderr)
     print("  odysseus_api.py cookbook cached [HOST]", file=sys.stderr)
@@ -79,6 +84,33 @@ def main() -> int:
             method = "GET"
             path = f"/api/codex/emails/{sys.argv[3]}"
             body = None
+        elif action in ("draft-doc", "draft_document") and len(sys.argv) >= 4:
+            method = "POST"
+            path = "/api/codex/emails/draft-document"
+            body = " ".join(sys.argv[3:])
+        else:
+            return _usage()
+    elif command in ("documents", "docs"):
+        if len(sys.argv) < 3:
+            return _usage()
+        action = sys.argv[2].lower()
+        if action == "list":
+            method = "GET"
+            limit = sys.argv[3] if len(sys.argv) >= 4 else "50"
+            path = f"/api/codex/documents?limit={limit}"
+            body = None
+        elif action == "read" and len(sys.argv) >= 4:
+            method = "GET"
+            path = f"/api/codex/documents/{sys.argv[3]}"
+            body = None
+        elif action == "create" and len(sys.argv) >= 4:
+            method = "POST"
+            path = "/api/codex/documents"
+            body = " ".join(sys.argv[3:])
+        elif action == "delete" and len(sys.argv) >= 4:
+            method = "DELETE"
+            path = f"/api/codex/documents/{sys.argv[3]}"
+            body = None
         else:
             return _usage()
     elif command == "cookbook":

integrations/codex/skills/odysseus/SKILL.md

@@ -102,6 +102,7 @@ python3 integrations/codex/scripts/odysseus_api.py POST /api/codex/memory '{"tex
 
 ## Email draft + send
 
+- Prefer `POST /api/codex/emails/draft-document` for Codex-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send.
 - `POST /api/codex/emails/draft` — body matches `SendEmailRequest` (`to`, `cc`, `bcc`, `subject`, `body`, `body_html`, `attachments`, `account_id`, `in_reply_to`, `references`). Requires `email:draft` (or `email:send`).
 - `POST /api/codex/emails/send` — same body. Requires `email:send`. Never send without explicit user instruction.
 

launch-windows.ps1

@@ -30,14 +30,26 @@ function Fail($msg) {
     exit 1
 }
 
+function Test-WindowsBashStub($path) {
+    if (-not $path) { return $false }
+    $lowered = $path.ToLowerInvariant()
+    foreach ($stub in @("system32\bash.exe", "sysnative\bash.exe", "windowsapps\bash.exe")) {
+        if ($lowered.Contains($stub)) { return $true }
+    }
+    return $false
+}
+
 function Find-GitBash {
     $cmd = Get-Command bash -ErrorAction SilentlyContinue
-    if ($cmd) { return $cmd.Source }
+    if ($cmd -and -not (Test-WindowsBashStub $cmd.Source)) { return $cmd.Source }
 
     $roots = @()
     foreach ($name in @("ProgramFiles", "ProgramW6432", "ProgramFiles(x86)", "LocalAppData")) {
         $base = [Environment]::GetEnvironmentVariable($name)
-        if ($base) { $roots += (Join-Path $base "Git") }
+        if ($base) {
+            $roots += (Join-Path $base "Git")
+            if ($name -eq "LocalAppData") { $roots += (Join-Path $base "Programs\Git") }
+        }
     }
     $roots += @("C:\Program Files\Git", "C:\Program Files (x86)\Git")
 
@@ -93,6 +105,14 @@ if (-not $pyExe) {
     }
 }
 
+if ($pyExe -like "*WindowsApps*python.exe") {
+    $pyCmd = Get-Command py -ErrorAction SilentlyContinue
+    if ($pyCmd) {
+        $pyExe = $pyCmd.Source
+        $pyArgs = @("-3.11")
+    }
+}
+
 if (-not $pyExe) {
     Fail "Couldn't find Python 3.11+ for Windows setup. Install Python 3.11+ (or open the Python launcher with 'py -3.11') from https://www.python.org/downloads/, then re-run this script."
 }
@@ -129,7 +149,20 @@ if (-not (Find-GitBash)) {
     Write-Host "      https://git-scm.com/download/win" -ForegroundColor Yellow
 }
 
-# 6. Start the server (use `python -m uvicorn` - bare `uvicorn` may not be on PATH)
+# 6. Point CUDA_PATH at a real CUDA toolkit so GPU llama-cpp-python can import.
+$cudaBase = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA"
+if (Test-Path $cudaBase) {
+    $cudaBest = Get-ChildItem $cudaBase -Directory -ErrorAction SilentlyContinue |
+        Where-Object { Test-Path (Join-Path $_.FullName "bin") } |
+        Sort-Object { try { [version]($_.Name -replace "^v", "") } catch { [version]"0.0" } } -Descending |
+        Select-Object -First 1
+    if ($cudaBest) {
+        $env:CUDA_PATH = $cudaBest.FullName
+        Write-Host ("Using CUDA_PATH = " + $cudaBest.FullName) -ForegroundColor Cyan
+    }
+}
+
+# 7. Start the server (use `python -m uvicorn` - bare `uvicorn` may not be on PATH)
 Write-Step ("Starting Odysseus at http://{0}:{1}" -f $BindHost, $Port)
 Write-Host "Press Ctrl+C to stop."
 Write-Host ""

launcher.py

@@ -0,0 +1,142 @@
+# launcher.py
+"""Dedicated entrypoint for the standalone Windows portable launcher.
+
+Handles:
+- Immediate GUI splash screen creation using tkinter.
+- Suppressing console stream crashes in windowed GUI mode via NullWriter.
+- Spawning system tray icon via pystray and Pillow (lazy-loaded).
+- Auto-opening default browser pointing to the running backend.
+- Launching the FastAPI server (importing and running app.py).
+"""
+import os
+import sys
+import threading
+import time
+import webbrowser
+
+# Define a dummy NullWriter to suppress standard stream crashes (isatty etc.) in GUI mode
+class NullWriter:
+    def write(self, text):
+        pass
+    def flush(self):
+        pass
+    def isatty(self):
+        return False
+
+if sys.stdout is None:
+    sys.stdout = NullWriter()
+if sys.stderr is None:
+    sys.stderr = NullWriter()
+
+
+splash_root = None
+
+# If running from a frozen PyInstaller bundle, launch the splash screen IMMEDIATELY
+if getattr(sys, 'frozen', False):
+    import tkinter as tk
+
+    def show_splash_instantly():
+        global splash_root
+        try:
+            splash_root = tk.Tk()
+            splash_root.title("Odysseus")
+            splash_root.overrideredirect(True)
+            splash_root.configure(bg="#1a1c23")
+
+            # Accented borders
+            splash_root.config(highlightbackground="#e06c75", highlightcolor="#e06c75", highlightthickness=1)
+
+            w, h = 360, 160
+            ws = splash_root.winfo_screenwidth()
+            hs = splash_root.winfo_screenheight()
+            x = (ws - w) // 2
+            y = (hs - h) // 2
+            splash_root.geometry(f"{w}x{h}+{x}+{y}")
+
+            tk.Label(splash_root, text="⛵ Odysseus", font=("Segoe UI", 22, "bold"), bg="#1a1c23", fg="#e06c75").pack(pady=(22, 2))
+            tk.Label(splash_root, text="Launching background services...", font=("Segoe UI", 10), bg="#1a1c23", fg="#d1d4e0").pack(pady=2)
+            tk.Label(splash_root, text="Please wait, this will take a few seconds.", font=("Segoe UI", 8, "italic"), bg="#1a1c23", fg="#5c6370").pack(pady=(12, 0))
+
+            splash_root.attributes("-topmost", True)
+            splash_root.mainloop()
+        except Exception:
+            pass
+
+    # Launch the GUI splash screen immediately on a background thread
+    threading.Thread(target=show_splash_instantly, daemon=True).start()
+
+
+def create_tray_image():
+    # Generate a beautiful 64x64 icon matching Odysseus brand red accent (#e06c75)
+    from PIL import Image, ImageDraw
+    image = Image.new('RGBA', (64, 64), (0, 0, 0, 0))
+    dc = ImageDraw.Draw(image)
+    accent_red = (224, 108, 117, 255)
+    light_red = (224, 108, 117, 150)
+
+    # Draw premium sailing boat
+    dc.polygon([(32, 10), (32, 45), (12, 45)], fill=accent_red)
+    dc.polygon([(32, 18), (32, 45), (48, 45)], fill=light_red)
+    dc.polygon([(8, 48), (56, 48), (44, 56), (20, 56)], fill=accent_red)
+    return image
+
+
+def on_open_browser(icon, item, url):
+    webbrowser.open(url)
+
+
+def on_exit(icon, item):
+    icon.stop()
+    os._exit(0)
+
+
+def setup_system_tray(url):
+    try:
+        import pystray
+        icon_img = create_tray_image()
+        menu = (
+            pystray.MenuItem('Open Odysseus', lambda icon, item: on_open_browser(icon, item, url), default=True),
+            pystray.MenuItem('Exit', on_exit)
+        )
+        tray_icon = pystray.Icon(
+            "Odysseus",
+            icon_img,
+            "Odysseus",
+            menu
+        )
+        tray_icon.run()
+    except Exception:
+        pass
+
+
+def open_browser(url):
+    # Allow uvicorn and app lifecycles to complete warmups
+    time.sleep(3.5)
+
+    # Safely close the splash screen
+    try:
+        global splash_root
+        if splash_root:
+            splash_root.after(0, splash_root.destroy)
+    except Exception:
+        pass
+
+    webbrowser.open(url)
+
+
+if __name__ == "__main__":
+    import uvicorn
+    # Import the FastAPI app from app.py
+    from app import app
+
+    bind_host = os.getenv("APP_BIND", "127.0.0.1")
+    bind_port = int(os.getenv("APP_PORT", "7000"))
+    url = f"http://{bind_host}:{bind_port}"
+
+    if getattr(sys, 'frozen', False):
+        # Start browser manager thread
+        threading.Thread(target=open_browser, args=(url,), daemon=True).start()
+        # Start system tray manager thread
+        threading.Thread(target=setup_system_tray, args=(url,), daemon=True).start()
+
+    uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")

mcp_servers/email_server.py

@@ -23,6 +23,7 @@ import os.path
 from pathlib import Path
 from datetime import datetime, timedelta
 import uuid
+from contextvars import ContextVar
 
 from mcp.server import Server
 from mcp.server.stdio import stdio_server
@@ -55,6 +56,8 @@ def _uid_fetch_rows(data) -> list:
 # flat keys when no DB row matches (legacy single-account behaviour).
 
 _ACCOUNT_CACHE: dict = {}  # key = normalized account selector -> config dict
+_MCP_OWNER_ARG = "_odysseus_owner"
+_CURRENT_OWNER: ContextVar[str | None] = ContextVar("email_mcp_owner", default=None)
 
 
 def _clean_header_value(value) -> str:
@@ -68,6 +71,45 @@ def _db_path() -> Path:
     return Path(APP_DB)
 
 
+def _current_owner() -> str:
+    owner = _CURRENT_OWNER.get()
+    return str(owner or "").strip()
+
+
+def _account_visible_to_owner(row: dict, owner: str) -> bool:
+    row_owner = str(row.get("owner") or "").strip()
+    if row_owner == owner:
+        return True
+    if row_owner:
+        return False
+    # Legacy ownerless accounts are only visible to a scoped caller when the
+    # mailbox itself matches the owner, mirroring the HTTP email route fallback.
+    owner_l = owner.lower()
+    return owner_l in {
+        str(row.get("imap_user") or "").strip().lower(),
+        str(row.get("from_address") or "").strip().lower(),
+    }
+
+
+def _filter_accounts_for_owner(rows: list[dict]) -> list[dict]:
+    owner = _current_owner()
+    if owner:
+        return [r for r in rows if _account_visible_to_owner(r, owner)]
+
+    owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()}
+    if len(owners) > 1:
+        return []
+    return rows
+
+
+def _mcp_owner_required(rows: list[dict] | None = None) -> bool:
+    if _current_owner():
+        return False
+    rows = rows if rows is not None else _read_accounts_from_db()
+    owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()}
+    return len(owners) > 1
+
+
 def _load_email_writing_style() -> str:
     """Return the existing Settings > Email > Writing Style value."""
     try:
@@ -121,9 +163,8 @@ def _default_document_owner() -> str | None:
         return None
 
 
-def _list_accounts_raw() -> list:
-    """Return list of dicts from the email_accounts table. Empty list if table
-    missing or empty. Never raises."""
+def _read_accounts_from_db() -> list:
+    """Return all enabled email account rows. Empty list if missing. Never raises."""
     path = _db_path()
     if not path.exists():
         return []
@@ -131,9 +172,10 @@ def _list_accounts_raw() -> list:
         conn = sqlite3.connect(str(path))
         conn.row_factory = sqlite3.Row
         columns = {r[1] for r in conn.execute("PRAGMA table_info(email_accounts)").fetchall()}
+        owner_select = "owner" if "owner" in columns else "NULL AS owner"
         smtp_security_select = "smtp_security" if "smtp_security" in columns else "'' AS smtp_security"
         rows = conn.execute(f"""
-            SELECT id, name, is_default, enabled,
+            SELECT id, {owner_select}, name, is_default, enabled,
                    imap_host, imap_port, imap_user, imap_password, imap_starttls,
                    smtp_host, smtp_port, {smtp_security_select}, smtp_user, smtp_password, from_address
             FROM email_accounts WHERE enabled = 1
@@ -147,11 +189,15 @@ def _list_accounts_raw() -> list:
         return []
 
 
-def _resolve_account(selector: str | None) -> dict | None:
+def _list_accounts_raw() -> list:
+    """Return owner-visible email account rows for the active MCP call."""
+    return _filter_accounts_for_owner(_read_accounts_from_db())
+
+
+def _resolve_account_from_rows(rows: list[dict], selector: str | None) -> dict | None:
     """Given a selector (None = default, or a name/user/id string), return the
     matching row or None. Matching is case-insensitive substring on name +
     imap_user + from_address, plus exact id match."""
-    rows = _list_accounts_raw()
     if not rows:
         return None
     if not selector:
@@ -186,6 +232,10 @@ def _resolve_account(selector: str | None) -> dict | None:
     return None
 
 
+def _resolve_account(selector: str | None) -> dict | None:
+    return _resolve_account_from_rows(_list_accounts_raw(), selector)
+
+
 def _load_config(account: str | None = None) -> dict:
     """Return the full config dict for the requested account (or default).
 
@@ -194,7 +244,7 @@ def _load_config(account: str | None = None) -> dict:
       2. env vars + settings.json flat keys (legacy)
       3. hardcoded fallbacks (localhost:31143 etc.)
     """
-    cache_key = (account or "").strip().lower() or "__default__"
+    cache_key = (_current_owner(), (account or "").strip().lower() or "__default__")
     if cache_key in _ACCOUNT_CACHE:
         return _ACCOUNT_CACHE[cache_key]
 
@@ -223,8 +273,11 @@ def _load_config(account: str | None = None) -> dict:
         "account_name": None,
     }
 
-    rows = _list_accounts_raw()
-    row = _resolve_account(account)
+    raw_rows = _read_accounts_from_db()
+    rows = _filter_accounts_for_owner(raw_rows)
+    row = _resolve_account_from_rows(rows, account)
+    if _current_owner() and raw_rows and not rows:
+        raise ValueError("No email account is configured for the authenticated owner")
     if account and rows and not row:
         available = ", ".join(
             f"{r.get('name') or r.get('imap_user')} <{r.get('imap_user') or r.get('from_address') or '?'}>"
@@ -885,8 +938,109 @@ def _smtp_connect(account=None, cfg=None):
     return conn
 
 
+def _read_agent_email_confirm_setting() -> bool:
+    """True if the user wants agent send_email/reply_to_email calls to be
+    queued for manual approval instead of SMTPed immediately. Defaults to
+    True so a fresh install is safe — agents have been observed inventing
+    signatures and sending to real recipients without the user's review."""
+    try:
+        from src.settings import get_setting
+        return bool(get_setting("agent_email_confirm", True))
+    except Exception:
+        return True
+
+
+def _stash_agent_draft(*, to, subject, body, in_reply_to=None, references=None,
+                      cc=None, bcc=None, account=None) -> dict:
+    """Insert the composed email into scheduled_emails with status
+    'agent_draft' and a far-future send_at so the scheduled-send poller
+    never picks it up. Returns the pending payload the model surfaces to
+    the user (and that the chat UI can render as an approval card)."""
+    try:
+        from src.constants import SCHEDULED_EMAILS_DB
+    except Exception:
+        return {"success": False, "error": "Pending-email storage unavailable"}
+    pending_id = uuid.uuid4().hex[:16]
+    far_future = "9999-12-31T00:00:00"
+    now = datetime.utcnow().isoformat()
+    try:
+        conn = sqlite3.connect(SCHEDULED_EMAILS_DB)
+        # Touch the schema in case the email-routes init hasn't run yet
+        # (MCP server can boot independently).
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS scheduled_emails (
+                id TEXT PRIMARY KEY,
+                to_addr TEXT NOT NULL,
+                cc TEXT,
+                bcc TEXT,
+                subject TEXT,
+                body TEXT NOT NULL,
+                in_reply_to TEXT,
+                references_hdr TEXT,
+                attachments TEXT,
+                send_at TEXT NOT NULL,
+                created_at TEXT NOT NULL,
+                status TEXT NOT NULL DEFAULT 'pending',
+                error TEXT,
+                owner TEXT DEFAULT '',
+                account_id TEXT,
+                odysseus_kind TEXT
+            )
+        """)
+        conn.execute("""
+            INSERT INTO scheduled_emails
+            (id, to_addr, cc, bcc, subject, body, in_reply_to, references_hdr,
+             attachments, send_at, created_at, status, account_id, odysseus_kind, owner)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'agent_draft', ?, ?, ?)
+        """, (
+            pending_id,
+            to if isinstance(to, str) else ", ".join(to),
+            cc if isinstance(cc, str) else (", ".join(cc) if cc else None),
+            bcc if isinstance(bcc, str) else (", ".join(bcc) if bcc else None),
+            subject or "",
+            body or "",
+            in_reply_to or None,
+            references if isinstance(references, str) else (" ".join(references) if references else None),
+            "[]",
+            far_future,
+            now,
+            account or None,
+            "agent_draft",
+            _current_owner(),
+        ))
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        return {"success": False, "error": f"Failed to stash draft: {e}"}
+    return {
+        "success": True,
+        "pending": True,
+        "pending_id": pending_id,
+        "to": to if isinstance(to, str) else ", ".join(to),
+        "subject": subject or "",
+        "body": body or "",
+        "message": (
+            "✋ Draft staged for your approval — nothing has been sent yet.\n"
+            "Review the To/Subject/Body above. Reply 'send' to deliver, or "
+            "'cancel' to discard."
+        ),
+    }
+
+
 def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, bcc=None, account=None):
-    """Send an email via SMTP. Returns dict with status."""
+    """Send an email via SMTP. Returns dict with status.
+
+    When the `agent_email_confirm` setting is on (the default), the email
+    is NOT SMTPed — instead it lands in scheduled_emails as an
+    `agent_draft` row and the user reviews + approves it from the chat
+    UI. This closes the auto-send hole that let earlier models invent
+    signatures and ship them to real recipients without confirmation."""
+    if _read_agent_email_confirm_setting():
+        return _stash_agent_draft(
+            to=to, subject=subject, body=body,
+            in_reply_to=in_reply_to, references=references,
+            cc=cc, bcc=bcc, account=account,
+        )
     send_account, cfg = _resolve_send_config(account)
     msg = EmailMessage()
     msg["From"] = _clean_header_value(cfg["from_address"])
@@ -1038,7 +1192,7 @@ def _create_email_draft_document(
     doc_id = str(uuid.uuid4())
     ver_id = str(uuid.uuid4())
     doc_title = (title or subject or "Email draft").strip() or "Email draft"
-    doc_owner = _default_document_owner()
+    doc_owner = _current_owner() or _default_document_owner()
 
     db = SessionLocal()
     try:
@@ -1824,10 +1978,22 @@ async def list_tools() -> list[Tool]:
 
 @server.call_tool()
 async def call_tool(name: str, arguments: dict) -> list[TextContent]:
+    arguments = dict(arguments) if isinstance(arguments, dict) else {}
+    owner = str(arguments.pop(_MCP_OWNER_ARG, "") or "").strip()
+    owner_token = _CURRENT_OWNER.set(owner or None)
     try:
+        all_db_accounts = _read_accounts_from_db()
+        if _mcp_owner_required(all_db_accounts):
+            return [TextContent(
+                type="text",
+                text="Error: email MCP requires an authenticated owner when multiple email account owners are configured.",
+            )]
+
         if name == "list_email_accounts":
-            rows = _list_accounts_raw()
+            rows = _filter_accounts_for_owner(all_db_accounts)
             if not rows:
+                if all_db_accounts and owner:
+                    return [TextContent(type="text", text="No email accounts configured for this owner.")]
                 return [TextContent(type="text", text="No email accounts configured. Legacy single-account mode active.")]
             lines = [f"Found {len(rows)} email account(s):\n"]
             for r in rows:
@@ -2007,6 +2173,16 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
                 bcc=arguments.get("bcc"),
                 account=acct,
             )
+            if "error" in result:
+                return [TextContent(type="text", text=f"Error: {result['error']}")]
+            if result.get("pending"):
+                return [TextContent(
+                    type="text",
+                    text=(
+                        f"Draft staged for approval (pending id: {result.get('pending_id')}). "
+                        "Nothing has been sent yet. Review and approve it in Odysseus before delivery."
+                    ),
+                )]
             acct_note = f" (from {result['account']})" if result.get("account") else ""
             return [TextContent(type="text", text=f"Sent email to {result['to']} with subject '{result['subject']}'{acct_note}.")]
 
@@ -2182,6 +2358,8 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
 
     except Exception as e:
         return [TextContent(type="text", text=f"Error: {e}")]
+    finally:
+        _CURRENT_OWNER.reset(owner_token)
 
 
 # ── Main ──

mcp_servers/memory_server.py

@@ -6,6 +6,7 @@ Imports MemoryManager and MemoryVectorStore from the Odysseus codebase.
 """
 
 import asyncio
+import os
 import sys
 import time
 from pathlib import Path
@@ -23,6 +24,55 @@ _memory_manager = None
 _memory_vector = None
 _initialized = False
 
+_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_MEMORY_OWNER", "ODYSSEUS_MEMORY_OWNER")
+_OWNER_SCOPE_ERROR = (
+    "Error: Memory MCP owner is not configured for an owner-scoped memory store. "
+    "Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
+)
+
+
+def _configured_owner() -> str | None:
+    for key in _OWNER_ENV_KEYS:
+        owner = os.environ.get(key, "").strip()
+        if owner:
+            return owner
+    return None
+
+
+def _entry_owner(entry: dict) -> str | None:
+    owner = entry.get("owner")
+    if owner is None:
+        return None
+    owner_text = str(owner).strip()
+    return owner_text or None
+
+
+def _owner_scoped_store(entries: list[dict]) -> bool:
+    return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
+
+
+def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]:
+    """Return configured owner, all entries, visible entries, and optional error."""
+    entries = _memory_manager.load_all()
+    owner = _configured_owner()
+    if owner is None and _owner_scoped_store(entries):
+        return None, entries, [], _OWNER_SCOPE_ERROR
+    if owner is None:
+        visible = [
+            entry for entry in entries
+            if isinstance(entry, dict) and _entry_owner(entry) is None
+        ]
+    else:
+        visible = [
+            entry for entry in entries
+            if isinstance(entry, dict) and _entry_owner(entry) == owner
+        ]
+    return owner, entries, visible, None
+
+
+def _text_result(text: str) -> list[TextContent]:
+    return [TextContent(type="text", text=text)]
+
 
 def _ensure_init():
     """Lazy-init memory managers on first use."""
@@ -75,43 +125,46 @@ async def list_tools() -> list[Tool]:
 @server.call_tool()
 async def call_tool(name: str, arguments: dict) -> list[TextContent]:
     if name != "manage_memory":
-        return [TextContent(type="text", text=f"Unknown tool: {name}")]
+        return _text_result(f"Unknown tool: {name}")
 
     _ensure_init()
     if not _memory_manager:
-        return [TextContent(type="text", text="Error: Memory manager not available")]
+        return _text_result("Error: Memory manager not available")
 
     action = arguments.get("action", "")
 
     if action == "list":
         category_filter = arguments.get("category", "")
-        memories = _memory_manager.load()
+        _owner, _all_memories, memories, scope_error = _scope_entries()
+        if scope_error:
+            return _text_result(scope_error)
         if category_filter:
             memories = [m for m in memories if m.get("category", "").lower() == category_filter.lower()]
         if not memories:
             msg = "No memories found"
             if category_filter:
                 msg += f" in category '{category_filter}'"
-            return [TextContent(type="text", text=msg + ".")]
+            return _text_result(msg + ".")
+
         lines = [f"Found {len(memories)} memory entries:\n"]
-        for m in memories[:100]:
+        for m in memories:
             cat = m.get("category", "fact")
             mid = m.get("id", "?")[:8]
             text = m.get("text", "")
             if len(text) > 150:
                 text = text[:150] + "..."
             lines.append(f"- [{cat}] `{mid}` — {text}")
-        if len(memories) > 100:
-            lines.append(f"... and {len(memories) - 100} more")
-        return [TextContent(type="text", text="\n".join(lines))]
+        return _text_result("\n".join(lines))
 
     elif action == "add":
         text = arguments.get("text", "")
         category = arguments.get("category", "fact")
         if not text:
-            return [TextContent(type="text", text="Error: Memory text cannot be empty")]
-        entry = _memory_manager.add_entry(text, source="ai_agent", category=category)
-        memories = _memory_manager.load_all()
+            return _text_result("Error: Memory text cannot be empty")
+        owner, memories, _visible, scope_error = _scope_entries()
+        if scope_error:
+            return _text_result(scope_error)
+        entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
         memories.append(entry)
         _memory_manager.save(memories)
         if _memory_vector and _memory_vector.healthy:
@@ -119,25 +172,28 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
                 _memory_vector.add(entry["id"], text)
             except Exception:
                 pass
-        return [TextContent(type="text", text=f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")]
+        return _text_result(f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")
 
     elif action == "edit":
         memory_id = arguments.get("memory_id", "")
         new_text = arguments.get("text", "")
         if not memory_id or not new_text:
-            return [TextContent(type="text", text="Error: edit needs memory_id and text")]
-        memories = _memory_manager.load_all()
-        found = False
+            return _text_result("Error: edit needs memory_id and text")
+        _owner, memories, visible, scope_error = _scope_entries()
+        if scope_error:
+            return _text_result(scope_error)
         full_id = None
-        for m in memories:
+        for m in visible:
             if m.get("id", "").startswith(memory_id):
-                m["text"] = new_text
-                m["timestamp"] = int(time.time())
-                found = True
                 full_id = m["id"]
                 break
-        if not found:
-            return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")]
+        if not full_id:
+            return _text_result(f"Error: Memory '{memory_id}' not found")
+        for m in memories:
+            if m.get("id") == full_id:
+                m["text"] = new_text
+                m["timestamp"] = int(time.time())
+                break
         _memory_manager.save(memories)
         if _memory_vector and _memory_vector.healthy and full_id:
             try:
@@ -145,24 +201,26 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
                 _memory_vector.add(full_id, new_text)
             except Exception:
                 pass
-        return [TextContent(type="text", text=f"Memory updated: {new_text}")]
+        return _text_result(f"Memory updated: {new_text}")
 
     elif action == "delete":
         memory_id = arguments.get("memory_id", "")
         if not memory_id:
-            return [TextContent(type="text", text="Error: delete needs memory_id")]
-        memories = _memory_manager.load_all()
+            return _text_result("Error: delete needs memory_id")
+        _owner, memories, visible, scope_error = _scope_entries()
+        if scope_error:
+            return _text_result(scope_error)
         full_id = None
         deleted_text = ""
         deleted_category = ""
-        for m in memories:
+        for m in visible:
             if m.get("id", "").startswith(memory_id):
                 full_id = m["id"]
                 deleted_text = m.get("text", "")
                 deleted_category = m.get("category", "")
                 break
         if not full_id:
-            return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")]
+            return _text_result(f"Error: Memory '{memory_id}' not found")
         memories = [m for m in memories if m.get("id") != full_id]
         _memory_manager.save(memories)
         if _memory_vector and _memory_vector.healthy and full_id:
@@ -172,30 +230,32 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
                 pass
         cat = f"[{deleted_category}] " if deleted_category else ""
         snippet = deleted_text if len(deleted_text) <= 120 else deleted_text[:117] + "..."
-        return [TextContent(type="text", text=f"Memory deleted: {cat}{snippet} (id: {memory_id})")]
+        return _text_result(f"Memory deleted: {cat}{snippet} (id: {memory_id})")
 
     elif action == "search":
         query = arguments.get("text", "")
         if not query:
-            return [TextContent(type="text", text="Error: search needs text (query)")]
-        memories = _memory_manager.load()
+            return _text_result("Error: search needs text (query)")
+        _owner, _all_memories, memories, scope_error = _scope_entries()
+        if scope_error:
+            return _text_result(scope_error)
         if hasattr(_memory_manager, 'get_relevant_memories'):
             results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
         else:
             query_lower = query.lower()
             results = [m for m in memories if query_lower in m.get("text", "").lower()][:20]
         if not results:
-            return [TextContent(type="text", text=f"No memories found matching '{query}'.")]
+            return _text_result(f"No memories found matching '{query}'.")
         lines = [f"Found {len(results)} matching memories:\n"]
         for m in results:
             cat = m.get("category", "fact")
             mid = m.get("id", "?")[:8]
             text = m.get("text", "")
             lines.append(f"- [{cat}] `{mid}` — {text}")
-        return [TextContent(type="text", text="\n".join(lines))]
+        return _text_result("\n".join(lines))
 
     else:
-        return [TextContent(type="text", text=f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")]
+        return _text_result(f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")
 
 
 async def run():

package-lock.json

@@ -5,16 +5,16 @@
   "packages": {
     "": {
       "dependencies": {
-        "@anthropic-ai/sdk": "^0.98.0"
+        "@anthropic-ai/sdk": "^0.104.1"
       },
       "devDependencies": {
-        "@antithesishq/bombadil": "^0.3.2"
+        "@antithesishq/bombadil": "^0.5.0"
       }
     },
     "node_modules/@anthropic-ai/sdk": {
-      "version": "0.98.0",
-      "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.98.0.tgz",
-      "integrity": "sha512-N7aXtCvC5g6T1Y4V29lJjceu/zTkVkIZF0jdBvagr0TRFHuKeImffalGWEfqZKrvjH+IQbzJWw6TmSmUzrlMgg==",
+      "version": "0.104.1",
+      "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.104.1.tgz",
+      "integrity": "sha512-gGACa/+IaiXzRRmF96aOhamoBgapKRBiFWbmmTFP8aMkpaEcuStF+Q61bjo4vPxBM7gqWJNZqsngslRdnLHv0Q==",
       "license": "MIT",
       "dependencies": {
         "json-schema-to-ts": "^3.1.1",
@@ -33,11 +33,14 @@
       }
     },
     "node_modules/@antithesishq/bombadil": {
-      "version": "0.3.2",
-      "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.3.2.tgz",
-      "integrity": "sha512-ATy1w9ZY5gbny1H8DFc7rxZitT7DLLLFDiGcRZe+8TQiUrV5tLO+IJGOVNNLp3RpCqjZqSsxGiKoQsx31ipV1g==",
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.5.0.tgz",
+      "integrity": "sha512-s0zImmr0iyvSP6QcVLvf40CUiZYIdWBAxiq20uhzujwvfitYa3PGJN652k/pLtVccHM/JrGQxZdvLnihZpltHA==",
       "dev": true,
-      "license": "MIT"
+      "license": "MIT",
+      "bin": {
+        "bombadil": "bin/bombadil.js"
+      }
     },
     "node_modules/@babel/runtime": {
       "version": "7.29.7",

package.json

@@ -4,9 +4,9 @@
     "url": "https://github.com/pewdiepie-archdaemon/odysseus.git"
   },
   "devDependencies": {
-    "@antithesishq/bombadil": "^0.3.2"
+    "@antithesishq/bombadil": "^0.5.0"
   },
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.98.0"
+    "@anthropic-ai/sdk": "^0.104.1"
   }
 }

requirements-optional.txt

@@ -33,4 +33,4 @@ PyMuPDF
 # magika (onnxruntime), already a core dep via fastembed. We avoid the
 # [all]/Azure/audio extras (cloud + heavy). Pinned to a release >30 days old per
 # the dependency-age discussion in issue #485.
-markitdown[docx,pptx,xlsx,xls]==0.1.5
+markitdown[docx,pptx,xlsx,xls]==0.1.6

requirements.txt

@@ -3,8 +3,8 @@ uvicorn
 python-multipart
 python-dotenv
 httpx
-pydantic>=2.0
-pydantic-settings>=2.0
+pydantic>=2.13.4
+pydantic-settings>=2.14.1
 SQLAlchemy
 pypdf
 beautifulsoup4
@@ -43,3 +43,7 @@ qrcode[pil]
 croniter
 pytest
 pytest-asyncio
+# starlette.testclient prefers httpx2 since Starlette 1.2.0 and warns on every
+# TestClient import when only classic httpx is present. Runtime code keeps
+# using `httpx` above; this is test-client only.
+httpx2

routes/api_token_routes.py

@@ -31,6 +31,7 @@ ALLOWED_SCOPES = {
 TOKEN_PROFILES = {
     "chat": ["chat"],
     "codex_todos": ["todos:read", "todos:write"],
+    "codex_documents": ["documents:read", "documents:write"],
     "codex_email_drafts": ["email:read", "email:draft", "documents:read", "documents:write"],
 }
 
@@ -154,14 +155,19 @@ def setup_api_token_routes() -> APIRouter:
     @router.patch("/tokens/{token_id}")
     async def update_token(request: Request, token_id: str):
         require_admin(request)
+        current_user = get_current_user(request)
         try:
             payload = await request.json()
         except Exception:
             payload = {}
+        if not isinstance(payload, dict):
+            payload = {}
         with get_db_session() as db:
             token = db.query(ApiToken).filter(ApiToken.id == token_id).first()
             if not token:
                 raise HTTPException(404, "Token not found")
+            if current_user and token.owner != current_user:
+                raise HTTPException(403, "Not your token")
             if isinstance(payload.get("name"), str) and payload["name"].strip():
                 token.name = payload["name"].strip()[:MAX_NAME_LEN]
             # Only touch scopes when the caller actually sent them. A partial
@@ -189,10 +195,14 @@ def setup_api_token_routes() -> APIRouter:
     @router.delete("/tokens/{token_id}")
     def delete_token(request: Request, token_id: str):
         require_admin(request)
+        current_user = get_current_user(request)
         with get_db_session() as db:
-            deleted = db.query(ApiToken).filter(ApiToken.id == token_id).delete()
-            if not deleted:
+            token = db.query(ApiToken).filter(ApiToken.id == token_id).first()
+            if not token:
                 raise HTTPException(404, "Token not found")
+            if current_user and token.owner != current_user:
+                raise HTTPException(403, "Not your token")
+            db.delete(token)
         _invalidate_cache(request)
         return {"status": "deleted"}
 

routes/auth_routes.py

@@ -12,7 +12,7 @@ import re
 from pathlib import Path
 
 from core.atomic_io import atomic_write_json, atomic_write_text
-from core.auth import AuthManager
+from core.auth import AuthManager, SetAdminResult
 from src.constants import DEEP_RESEARCH_DIR, MEMORY_FILE, SKILLS_DIR
 from src.rate_limiter import RateLimiter
 from src.settings_scrub import scrub_settings
@@ -73,6 +73,11 @@ class DeleteUserRequest(BaseModel):
 class RenameUserRequest(BaseModel):
     username: str
 
+
+class SetAdminRequest(BaseModel):
+    is_admin: bool
+
+
 class SetOpenRegistrationRequest(BaseModel):
     enabled: bool
 
@@ -139,6 +144,8 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
                 raise HTTPException(401, "Invalid 2FA code")
         # All checks passed — create session (password already verified above)
         token = await asyncio.to_thread(auth_manager.create_session_trusted, username)
+        if not token:
+            raise HTTPException(401, "Invalid credentials")
         cookie_kwargs = dict(
             key=SESSION_COOKIE,
             value=token,
@@ -416,6 +423,34 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         except Exception as e:
             logger.warning("Failed to rename memory.json owner references %s -> %s: %s", old_username, new_username, e)
 
+        # uploads.json: upload rows use owner metadata for access checks and
+        # owner-prefixed index keys for dedupe. Rename both so attachments keep
+        # resolving after the account username changes.
+        try:
+            upload_handler = getattr(request.app.state, "upload_handler", None)
+            rename_owner = getattr(upload_handler, "rename_owner", None)
+            if callable(rename_owner):
+                rename_owner(old_username, new_username)
+        except Exception as e:
+            logger.warning("Failed to rename upload owner references %s -> %s: %s", old_username, new_username, e)
+
+        # direct personal RAG uploads live in per-owner directories and the
+        # vector metadata also carries the username used for owner-filtered
+        # search. Keep both in sync with the auth rename.
+        try:
+            from routes.personal_routes import rename_personal_upload_owner
+            personal_docs_manager = getattr(request.app.state, "personal_docs_manager", None)
+            if personal_docs_manager is not None:
+                rag_manager = getattr(personal_docs_manager, "rag_manager", None)
+                rename_personal_upload_owner(
+                    old_username,
+                    new_username,
+                    personal_docs_manager=personal_docs_manager,
+                    rag_manager=rag_manager,
+                )
+        except Exception as e:
+            logger.warning("Failed to rename personal RAG upload owner references %s -> %s: %s", old_username, new_username, e)
+
         # skills: SKILL.md frontmatter carries owner: <username>; the usage
         # sidecar (_usage.json) keys entries as owner::skill-name. Both must
         # be updated or the renamed user's Skills panel goes empty.
@@ -476,6 +511,31 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             invalidator()
         return {"ok": True, "username": new_username, "renamed_self": old_username == user}
 
+    @router.put("/users/{username}/admin")
+    async def set_user_admin(username: str, body: SetAdminRequest, request: Request):
+        """Promote/demote a user to/from admin. Admin only.
+
+        The last remaining admin can't be demoted (no lockout). Self-demotion
+        is allowed while another admin exists; the `self` flag tells the UI to
+        reload the acting user into the normal-user view.
+        """
+        user = _get_current_user(request)
+        if not user or not auth_manager.is_admin(user):
+            raise HTTPException(403, "Admin only")
+        result = auth_manager.set_admin(username, body.is_admin, user)
+        if result is SetAdminResult.USER_NOT_FOUND:
+            raise HTTPException(404, "User not found")
+        if result is SetAdminResult.NOT_AUTHORIZED:
+            raise HTTPException(403, "Admin only")
+        if result is SetAdminResult.LAST_ADMIN:
+            raise HTTPException(400, "Cannot demote the last admin")
+        target = (username or "").strip().lower()
+        return {
+            "ok": True,
+            "is_admin": body.is_admin,
+            "self": target == (user or "").strip().lower(),
+        }
+
     @router.post("/signup-toggle", deprecated=True)
     async def toggle_signup(request: Request):
         """

routes/calendar_routes.py

@@ -11,7 +11,7 @@ from pydantic import BaseModel
 from sqlalchemy import or_, and_
 from dateutil.rrule import rrulestr
 
-from core.database import SessionLocal, CalendarCal, CalendarEvent
+from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent
 from src.auth_helpers import require_user
 from src.upload_limits import read_upload_limited, ICS_MAX_BYTES
 
@@ -126,6 +126,54 @@ def _resolve_base_uid(uid: str) -> str:
         raise ValueError("malformed compound UID: missing base before ::")
     return base
 
+
+async def _push_caldav_event_after_commit(owner: str, uid: str, action: str):
+    """Best-effort CalDAV write-through. Local writes stay authoritative if
+    the remote server is unreachable; pending flags let /sync retry later."""
+    try:
+        result = {"ok": True}
+        if action == "create":
+            from src.caldav_sync import push_event_create
+            result = await push_event_create(owner, uid)
+        elif action == "update":
+            from src.caldav_sync import push_event_update
+            result = await push_event_update(owner, uid)
+        elif action == "delete":
+            from src.caldav_sync import push_event_delete
+            result = await push_event_delete(owner, uid)
+        if result and not result.get("ok") and not result.get("skipped"):
+            raise RuntimeError(result.get("error") or result)
+    except Exception as e:
+        logger.warning("CalDAV %s push failed for uid=%s: %s", action, uid, e)
+        if action in {"create", "update"}:
+            db = SessionLocal()
+            try:
+                ev = _get_or_404_event(db, uid, owner)
+                ev.caldav_sync_pending = action
+                db.commit()
+            except Exception:
+                db.rollback()
+            finally:
+                db.close()
+
+
+def _record_caldav_delete_tombstone(db, ev: CalendarEvent, owner: str) -> None:
+    if not (ev.calendar and ev.calendar.source == "caldav"):
+        return
+    tombstone = db.query(CalendarDeletedEvent).filter(
+        CalendarDeletedEvent.uid == ev.uid,
+        CalendarDeletedEvent.owner == owner,
+    ).first()
+    if not tombstone:
+        tombstone = CalendarDeletedEvent(uid=ev.uid, owner=owner)
+        db.add(tombstone)
+    tombstone.calendar_id = ev.calendar_id
+    tombstone.remote_href = ev.remote_href
+    tombstone.remote_etag = ev.remote_etag
+    tombstone.caldav_base_url = getattr(ev.calendar, "caldav_base_url", None)
+    tombstone.summary = ev.summary or ""
+    tombstone.last_error = None
+
 # ── Pydantic models ──
 
 class EventCreate(BaseModel):
@@ -843,13 +891,13 @@ def setup_calendar_routes() -> APIRouter:
             return {"ok": False, "error": str(e)[:200]}
 
     @router.post("/sync")
-    async def sync_caldav_endpoint(request: Request):
-        """Pull events from the configured CalDAV server into local DB.
+    async def sync_caldav_endpoint(request: Request, direction: str = "pull"):
+        """Sync events with the configured CalDAV server.
         Returns counts + any per-calendar errors. Called by the frontend
         on calendar open and by the periodic scheduler loop."""
         owner = _require_user(request)
-        from src.caldav_sync import sync_caldav
-        return await sync_caldav(owner)
+        from src.caldav_sync import sync_caldav_direction
+        return await sync_caldav_direction(owner, direction)
 
 
     @router.delete("/calendars/{cal_id}")
@@ -1002,19 +1050,12 @@ def setup_calendar_routes() -> APIRouter:
                 is_utc=_is_utc and not data.all_day,
                 rrule=data.rrule or "",
                 color=data.color or None,
+                caldav_sync_pending="create" if cal.source == "caldav" else None,
             )
             db.add(ev)
             db.commit()
             if cal.source == "caldav":
-                # Push the new event to the remote so it appears on the user's
-                # other devices — the sync is otherwise pull-only (#800).
-                from src.caldav_writeback import writeback_event
-                await writeback_event(owner, cal.source, cal.id, {
-                    "uid": uid, "summary": data.summary, "description": data.description,
-                    "location": data.location, "dtstart": dtstart, "dtend": dtend,
-                    "all_day": data.all_day, "is_utc": _is_utc and not data.all_day,
-                    "rrule": data.rrule or "",
-                })
+                await _push_caldav_event_after_commit(owner, uid, "create")
             return {"ok": True, "uid": uid}
         except HTTPException:
             raise
@@ -1060,15 +1101,12 @@ def setup_calendar_routes() -> APIRouter:
                 ev.rrule = data.rrule
             if data.color is not None:
                 ev.color = data.color if data.color else None
+            is_caldav = ev.calendar and ev.calendar.source == "caldav"
+            if is_caldav:
+                ev.caldav_sync_pending = "update"
             db.commit()
-            cal = db.query(CalendarCal).filter(CalendarCal.id == ev.calendar_id).first()
-            if cal and cal.source == "caldav":
-                from src.caldav_writeback import writeback_event
-                await writeback_event(owner, cal.source, cal.id, {
-                    "uid": ev.uid, "summary": ev.summary, "description": ev.description,
-                    "location": ev.location, "dtstart": ev.dtstart, "dtend": ev.dtend,
-                    "all_day": ev.all_day, "is_utc": ev.is_utc, "rrule": ev.rrule or "",
-                })
+            if is_caldav:
+                await _push_caldav_event_after_commit(owner, base_uid, "update")
             return {"ok": True}
         except HTTPException:
             raise
@@ -1089,15 +1127,13 @@ def setup_calendar_routes() -> APIRouter:
         db = SessionLocal()
         try:
             ev = _get_or_404_event(db, base_uid, owner)
-            # Capture what the remote push needs BEFORE the row is gone.
-            _cal = db.query(CalendarCal).filter(CalendarCal.id == ev.calendar_id).first()
-            _is_caldav = bool(_cal and _cal.source == "caldav")
-            _cal_id, _ev_uid = ev.calendar_id, ev.uid
+            is_caldav = ev.calendar and ev.calendar.source == "caldav"
+            if is_caldav:
+                _record_caldav_delete_tombstone(db, ev, owner)
             db.delete(ev)
             db.commit()
-            if _is_caldav:
-                from src.caldav_writeback import writeback_event
-                await writeback_event(owner, "caldav", _cal_id, {"uid": _ev_uid}, delete=True)
+            if is_caldav:
+                await _push_caldav_event_after_commit(owner, base_uid, "delete")
             return {"ok": True}
         except HTTPException:
             raise

routes/chat_helpers.py

@@ -14,7 +14,7 @@ from core.database import Session as DBSession, ModelEndpoint
 from src.llm_core import normalize_model_id
 from src.endpoint_resolver import normalize_base
 from src.context_compactor import maybe_compact, trim_for_context
-from src.auth_helpers import get_current_user
+from src.auth_helpers import effective_user
 from src.prompt_security import untrusted_context_message
 from routes.prefs_routes import _load_for_user as load_prefs_for_user
 
@@ -78,7 +78,7 @@ def _enforce_chat_privileges(request, sess) -> None:
     which means unrestricted allowed_models / zero cap -> no-op for them.
     """
     try:
-        user = get_current_user(request)
+        user = effective_user(request)
     except Exception:
         user = None
     if not user:
@@ -160,7 +160,7 @@ async def auto_name_session(session_manager, sess):
 
         owner = getattr(sess, "owner", None)
         t_url, t_model, t_headers = resolve_task_endpoint(
-            sess.endpoint_url, sess.model, sess.headers, owner=owner,
+            sess.endpoint_url, sess.model, sess.headers, owner=owner
         )
         if not t_model:
             logger.debug("[auto-name] No model provided, skipping")
@@ -338,11 +338,11 @@ def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, inco
 def fire_message_event(request, webhook_manager, session_id: str, sess, message: str, compare_mode: bool = False):
     """Fire webhook and event_bus events for a new user message."""
     if webhook_manager and not compare_mode:
-        asyncio.create_task(webhook_manager.fire("chat.message", {
+        webhook_manager.fire_and_forget("chat.message", {
             "session_id": session_id, "model": sess.model, "message": message[:2000],
-        }))
+        })
     from src.event_bus import fire_event
-    user = get_current_user(request)
+    user = effective_user(request)
     fire_event("message_sent", user)
 
 
@@ -497,6 +497,29 @@ def _normalize_model_id_from_cache(sess) -> Optional[str]:
     return None
 
 
+def _session_is_research_spinoff(sess) -> bool:
+    """True if this session was created via research "Discuss" spin-off.
+
+    Detected by the primer system message the spin-off endpoint seeds into
+    history (metadata ``research_spinoff_from``). Such sessions are grounded
+    on the seeded report, so global memory + personal-doc RAG injection is
+    suppressed for them (the report is the sole knowledge base). Handles both
+    ChatMessage objects and plain dicts.
+    """
+    for m in getattr(sess, "history", []) or []:
+        role = getattr(m, "role", None)
+        if role is None and isinstance(m, dict):
+            role = m.get("role")
+        if role != "system":
+            continue
+        md = getattr(m, "metadata", None)
+        if md is None and isinstance(m, dict):
+            md = m.get("metadata")
+        if (md or {}).get("research_spinoff_from"):
+            return True
+    return False
+
+
 async def build_chat_context(
     sess,
     request,
@@ -545,8 +568,9 @@ async def build_chat_context(
     if not incognito:
         fire_message_event(request, webhook_manager, session_id, sess, message, compare_mode)
 
-    # Resolve user prefs
-    user = get_current_user(request)
+    # Resolve owner-scoped prefs/context. Browser requests keep the cookie user;
+    # bearer-token chat requests use the token owner instead of the "api" sentinel.
+    user = effective_user(request)
     uprefs = load_prefs_for_user(user)
 
     # Memory enabled?
@@ -562,9 +586,17 @@ async def build_chat_context(
         mem_enabled, user, incognito, no_memory, uprefs.get("memory_enabled", "NOT_SET"),
     )
 
+    # Research-spinoff ("Discuss") sessions are grounded on the seeded report:
+    # the primer system message IS the knowledge base. Injecting global memory
+    # or personal-doc RAG on every turn pulls in keyword-matched but off-topic
+    # facts ("wrong data") and competes with the report, so suppress both here.
+    is_research_spinoff = _session_is_research_spinoff(sess)
+    if is_research_spinoff:
+        mem_enabled = False
+
     # Use RAG?
     use_rag_val = (str(use_rag).lower() != "false") if use_rag is not None else True
-    if incognito or not allow_tool_preprocessing:
+    if incognito or not allow_tool_preprocessing or is_research_spinoff:
         use_rag_val = False
 
     # If pre-fetched search context was provided (compare mode), skip live web search
@@ -587,7 +619,7 @@ async def build_chat_context(
         incognito=incognito,
         use_skills=skills_enabled,
     )
-    if use_rag is not None:
+    if use_rag is not None or is_research_spinoff:
         _preface_kwargs["use_rag"] = use_rag_val
     preface, rag_sources, web_sources = chat_processor.build_context_preface(**_preface_kwargs)
 
@@ -1081,10 +1113,10 @@ def run_post_response_tasks(
 
     # Webhook
     if webhook_manager and not compare_mode:
-        asyncio.create_task(webhook_manager.fire("chat.completed", {
+        webhook_manager.fire_and_forget("chat.completed", {
             "session_id": session_id, "model": sess.model,
             "user_message": message, "response": full_response[:2000],
-        }))
+        })
 
     # Auto-name
     if needs_auto_name(sess.name):

routes/chat_routes.py

@@ -6,7 +6,7 @@ import os
 import time
 import logging
 from datetime import datetime
-from typing import Dict, Any, AsyncGenerator, List
+from typing import Dict, Any, AsyncGenerator, List, Optional
 
 from fastapi import APIRouter, Request, HTTPException, Form, Query
 from fastapi.responses import StreamingResponse
@@ -23,7 +23,7 @@ from src.endpoint_resolver import normalize_base as _normalize_base, build_chat_
 from src.session_search import search_session_messages
 from src.prompt_security import untrusted_context_message
 from core.exceptions import SessionNotFoundError
-from src.auth_helpers import get_current_user
+from src.auth_helpers import effective_user, get_current_user
 from routes.session_routes import _verify_session_owner
 from routes.document_helpers import _owner_session_filter
 from core.database import SessionLocal, get_session_mode, set_session_mode
@@ -62,6 +62,33 @@ def _stream_set(session_id: str, **fields) -> None:
     rec.update(fields)
 
 
+def _resolve_request_workspace(request, raw_value) -> tuple:
+    """Resolve the posted workspace for this request: (workspace, rejected).
+
+    Privilege is checked BEFORE the path ever touches the filesystem. Only
+    admin/single-user callers can use the workspace-backed file/shell tools,
+    so only they get vet_workspace() and the workspace_rejected signal. For
+    any other caller the submitted value is dropped uniformly, with no vetting
+    and no event: otherwise the presence/absence of workspace_rejected would
+    let a non-admin chat caller probe which host paths exist.
+
+    vet_workspace rejects non-directories, sensitive roots (.ssh, .gnupg,
+    ...), and filesystem roots; on rejection there is no confinement and the
+    default tool-path allowlist applies. The rejected value is surfaced so the
+    stream can tell an admin client (which believes a workspace is active)
+    that it was dropped.
+    """
+    requested = (raw_value or "").strip()
+    if not requested:
+        return "", ""
+    from src.tool_security import owner_is_admin_or_single_user
+    if not owner_is_admin_or_single_user(get_current_user(request)):
+        return "", ""
+    from src.tool_execution import vet_workspace
+    workspace = vet_workspace(requested) or ""
+    return workspace, (requested if not workspace else "")
+
+
 def _session_url_matches_endpoint(session_url: str, endpoint_base: str) -> bool:
     if not session_url or not endpoint_base:
         return False
@@ -99,7 +126,8 @@ def _clear_orphaned_session_endpoint(sess, owner: str | None = None) -> bool:
         sess.model = ""
         sess.headers = {}
         return True
-    except Exception:
+    except Exception as e:
+        logger.warning("Failed to clear orphaned session endpoint", exc_info=e)
         db.rollback()
         return False
     finally:
@@ -117,7 +145,8 @@ def _endpoint_cache_contains_model(endpoint, model: str) -> bool:
         return True
     try:
         models = json.loads(raw) if isinstance(raw, str) else raw
-    except Exception:
+    except Exception as e:
+        logger.warning("Failed to parse cached models list, treating as containing model", exc_info=e)
         return True
     if not isinstance(models, list) or not models:
         return True
@@ -209,7 +238,8 @@ def _recover_empty_session_model(sess, session_id: str, owner: str | None = None
                 is_chatgpt_subscription = False
         try:
             cached = json.loads(ep.cached_models) if isinstance(ep.cached_models, str) else (ep.cached_models or [])
-        except Exception:
+        except Exception as e:
+            logger.warning("Failed to parse cached_models for endpoint %r", getattr(ep, "id", "?"), exc_info=e)
             cached = []
         if not cached:
             visible = []
@@ -333,7 +363,7 @@ def setup_chat_routes(
             sess = session_manager.get_session(session)
         except KeyError:
             raise HTTPException(404, f"Session '{session}' not found")
-        owner = get_current_user(request)
+        owner = effective_user(request)
         if _clear_orphaned_session_endpoint(sess, owner=owner):
             raise HTTPException(400, "Selected model endpoint was removed. Pick another model in Settings.")
 
@@ -447,8 +477,11 @@ def setup_chat_routes(
         use_research = form_data.get("use_research")
         time_filter = form_data.get("time_filter")
         preset_id = form_data.get("preset_id")
-        allow_bash = form_data.get("allow_bash")
-        allow_web_search = form_data.get("allow_web_search")
+        # Issue #3229: API callers send JSON, not FormData.  Read from the
+        # JSON body as fallback so callers who send {"allow_bash": true}
+        # actually get bash enabled.
+        allow_bash = form_data.get("allow_bash") or (body or {}).get("allow_bash")
+        allow_web_search = form_data.get("allow_web_search") or (body or {}).get("allow_web_search")
         use_rag = form_data.get("use_rag")
         search_context = form_data.get("search_context")  # pre-fetched web search results (compare mode)
         compare_mode = str(form_data.get("compare_mode", "")).lower() == "true"
@@ -457,6 +490,10 @@ def setup_chat_routes(
         # manual form posts that still send plan_mode=true.
         plan_mode = False
         chat_mode = str(form_data.get("mode", "")).lower()  # 'chat' or 'agent'
+        # Workspace: confine the agent's file/shell tools to this folder.
+        workspace, workspace_rejected = _resolve_request_workspace(
+            request, form_data.get("workspace")
+        )
         # Plan mode is a modifier on agent mode — it only makes sense with tools.
         if plan_mode:
             chat_mode = "agent"
@@ -492,6 +529,66 @@ def setup_chat_routes(
         active_doc_id = form_data.get("active_doc_id", "").strip()
         logger.info(f"[doc-inject] chat_mode={chat_mode}, active_doc_id={active_doc_id!r}")
 
+        # Active email reader — when the user has an email open in the UI, the
+        # frontend passes its uid/folder/account so "reply", "summarize this",
+        # etc. resolve to the real email instead of the agent inventing a
+        # fake markdown draft.
+        active_email_uid = form_data.get("active_email_uid", "").strip()
+        active_email_folder = form_data.get("active_email_folder", "INBOX").strip() or "INBOX"
+        active_email_account = form_data.get("active_email_account", "").strip()
+        active_email_ctx: Optional[Dict[str, str]] = None
+        # Always reset between requests so a stale active-email pointer from
+        # a previous turn (different reader closed, different account, etc.)
+        # can't leak in when the user has no email open this turn.
+        try:
+            from src.tool_implementations import clear_active_email
+            clear_active_email()
+        except Exception:
+            pass
+        if active_email_uid:
+            active_email_ctx = {
+                "uid": active_email_uid,
+                "folder": active_email_folder,
+                "account": active_email_account,
+            }
+            # Try to enrich with subject + from so the agent's system prompt
+            # block can quote them. Best-effort: a stale cache is fine, a
+            # missing email just means we pass uid/folder/account only.
+            try:
+                from routes.email_routes import _read_cache_get, _read_cache_key
+                _ck = _read_cache_key(active_email_account or None, active_email_folder, active_email_uid, owner=get_current_user(request))
+                _cached_email = _read_cache_get(_ck)
+                if _cached_email and isinstance(_cached_email, dict):
+                    active_email_ctx["subject"] = str(_cached_email.get("subject") or "")
+                    active_email_ctx["from"] = str(
+                        _cached_email.get("from_address")
+                        or _cached_email.get("from")
+                        or _cached_email.get("from_name")
+                        or ""
+                    )
+                    _body_preview = (_cached_email.get("body") or "")[:2000]
+                    if _body_preview:
+                        active_email_ctx["body_preview"] = _body_preview
+            except Exception as _e:
+                logger.debug(f"[email-inject] cache enrich skipped: {_e}")
+            # Stash so email tools can resolve "this email" without UID guessing.
+            try:
+                from src.tool_implementations import set_active_email
+                set_active_email(
+                    uid=active_email_uid,
+                    folder=active_email_folder,
+                    account=active_email_account or None,
+                    subject=active_email_ctx.get("subject"),
+                    sender=active_email_ctx.get("from"),
+                )
+            except Exception as _e:
+                logger.debug(f"[email-inject] set_active_email failed: {_e}")
+            logger.info(
+                "[email-inject] active_email uid=%s folder=%s account=%s subject=%r",
+                active_email_uid, active_email_folder, active_email_account or "(default)",
+                active_email_ctx.get("subject", ""),
+            )
+
         try:
             # Attachment-only sends: skip the message-required check when the
             # user has attached one or more files (the attachment IS the action).
@@ -506,7 +603,7 @@ def setup_chat_routes(
             # but BEFORE loading. Prevents cross-user session hijack.
             _verify_session_owner(request, session)
             sess = session_manager.get_session(session)
-            owner = get_current_user(request)
+            owner = effective_user(request)
             if _clear_orphaned_session_endpoint(sess, owner=owner):
                 raise HTTPException(400, "Selected model endpoint was removed. Pick another model in Settings.")
             # Issue #587: picker shows a model from the endpoint cache but
@@ -537,7 +634,7 @@ def setup_chat_routes(
         _enforce_chat_privileges(request, sess)
 
         # Ensure session has auth headers
-        resolve_session_auth(sess, session, owner=get_current_user(request))
+        resolve_session_auth(sess, session, owner=effective_user(request))
 
         # Check for research_pending BEFORE mode persist overwrites it
         do_research = str(use_research).lower() == "true"
@@ -552,8 +649,8 @@ def setup_chat_routes(
         elif attachments:
             try:
                 att_ids = [str(x) for x in json.loads(attachments)]
-            except Exception:
-                pass
+            except Exception as e:
+                logger.warning("Failed to parse attachments JSON, ignoring attachments", exc_info=e)
 
         no_memory = str(form_data.get("no_memory", "")).lower() == "true"
         pre_context_tool_policy = build_effective_tool_policy(
@@ -607,15 +704,27 @@ def setup_chat_routes(
                             active_doc_id,
                         )
                         active_doc = None
-                    elif doc_session and doc_session != session:
-                        logger.warning(
-                            "[doc-inject] ignoring stale active_doc_id %s from session %s while in session %s",
-                            active_doc_id,
-                            doc_session,
-                            session,
-                        )
-                        active_doc = None
                     else:
+                        # NOTE: previously dropped the doc when doc.session_id
+                        # != current chat session — but that broke the common
+                        # case of "open an email draft from one chat, ask a
+                        # different chat to write into it". The frontend only
+                        # sends active_doc_id for docs currently visible in
+                        # the UI, and we already owner-checked above, so trust
+                        # the explicit signal. We just log the mismatch and
+                        # re-bind the doc to the current session so future
+                        # turns find it via the session-fallback path too.
+                        if doc_session and doc_session != session:
+                            logger.info(
+                                "[doc-inject] cross-session active_doc_id %s (was session %s, now %s) — accepting and rebinding",
+                                active_doc_id, doc_session, session,
+                            )
+                            try:
+                                active_doc.session_id = session
+                                _doc_db.commit()
+                            except Exception as _e:
+                                _doc_db.rollback()
+                                logger.warning(f"[doc-inject] session rebind failed: {_e}")
                         logger.info(f"[doc-inject] found by ID: title={active_doc.title!r}, lang={active_doc.language!r}, is_active={active_doc.is_active}, content_len={len(active_doc.current_content or '')}")
                 else:
                     logger.warning(f"[doc-inject] NOT FOUND by ID {active_doc_id}")
@@ -656,9 +765,18 @@ def setup_chat_routes(
 
         # Build disabled-tools set from frontend toggles + user privileges
         disabled_tools = set()
-        if str(allow_bash).lower() != "true":
+        # Only disable bash/web_search when the caller *explicitly* set them
+        # to a falsy value.  When unset (None), defer to per-user privilege
+        # checks below — this lets admins with can_use_bash=True use bash
+        # by default without having to send allow_bash in every request.
+        if allow_bash is not None and str(allow_bash).lower() != "true":
             disabled_tools.add("bash")
-        if str(allow_web_search).lower() != "true":
+        _explicit_web_intent = bool(_tool_intent and _tool_intent.category == "web")
+        if (
+            allow_web_search is not None
+            and str(allow_web_search).lower() != "true"
+            and not _explicit_web_intent
+        ):
             disabled_tools.add("web_search")
             disabled_tools.add("web_fetch")
 
@@ -671,6 +789,21 @@ def setup_chat_routes(
                 "manage_skills",      # skill presets tied to user
             })
 
+        # Active email reader open → strip the tools that let the agent
+        # "drift" to a new compose: create_document (writes a fake email-
+        # shaped .md file) and send_email (sends fresh to a recipient the
+        # agent invented). With those gone, the only paths left for "write
+        # email saying X" are ui_control open_email_reply (draft) and
+        # reply_to_email (immediate send) — both of which use the open
+        # email's UID. Code-level enforcement instead of relying on a
+        # prompt rule the model can ignore.
+        if active_email_ctx and active_email_ctx.get("uid"):
+            disabled_tools.update({
+                "create_document",
+                "send_email",
+                "mcp__email__send_email",
+            })
+
         # Enforce per-user privileges
         _privs = {}
         _user = ctx.user
@@ -761,6 +894,13 @@ def setup_chat_routes(
             # Register active stream for partial-save safety net
             _active_streams[session] = {"status": "streaming", "partial": "", "query": message, "is_research": effective_do_research, "mode": _effective_mode}
 
+            # The client sent a workspace the server refused to bind (deleted
+            # folder, file path, sensitive dir, filesystem root). Tell it up
+            # front so the UI can clear the pill instead of displaying a
+            # confinement that is not actually in effect.
+            if workspace_rejected:
+                yield f"data: {json.dumps({'type': 'workspace_rejected', 'data': {'path': workspace_rejected}})}\n\n"
+
             if ctx.preprocessed.attachment_meta:
                 yield f"data: {json.dumps({'type': 'attachments', 'data': ctx.preprocessed.attachment_meta})}\n\n"
 
@@ -1131,6 +1271,7 @@ def setup_chat_routes(
                         max_rounds=_max_rounds,
                         context_length=ctx.context_length,
                         active_document=active_doc,
+                        active_email=active_email_ctx,
                         session_id=session,
                         disabled_tools=disabled_tools if disabled_tools else None,
                         tool_policy=tool_policy,
@@ -1138,6 +1279,7 @@ def setup_chat_routes(
                         fallbacks=_fallback_candidates,
                         plan_mode=plan_mode,
                         approved_plan=approved_plan or None,
+                        workspace=workspace or None,
                     ):
                         if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"):
                             try:
@@ -1343,7 +1485,7 @@ def setup_chat_routes(
         if not q or not q.strip():
             return []
 
-        _user = get_current_user(request)
+        _user = effective_user(request)
         return [
             result.to_dict()
             for result in search_session_messages(

routes/codex_routes.py

@@ -18,6 +18,7 @@ from fastapi.responses import StreamingResponse
 from src.auth_helpers import require_authenticated_request, require_user
 from src.tool_implementations import do_manage_notes
 from src.constants import COOKBOOK_STATE_FILE
+from routes._validators import validate_remote_host, validate_ssh_port
 
 
 COOKBOOK_READ_SCOPES = {"cookbook:read", "cookbook:launch"}
@@ -36,6 +37,25 @@ DOCS_WRITE_SCOPES = {"documents:write"}
 WRITE_ACTIONS = {"add", "create", "new", "save", "remind", "update", "delete", "toggle_item", "remove", "remove_item"}
 
 
+def _ssh_prefix_for_task(task: dict) -> tuple[str, str]:
+    """Resolve a cookbook task's stored SSH target into ``(host, port_flag)``.
+
+    ``host`` is ``""`` for a local task. ``remoteHost`` / ``sshPort`` come from
+    cookbook_state.json and get interpolated into an ``ssh`` command string, so
+    validate them the same way the cookbook routes do. A tampered entry with
+    shell metacharacters in ``remoteHost`` is rejected with 400 rather than
+    injected.
+    """
+    raw_host = task.get("remoteHost")
+    raw_port = task.get("sshPort")
+    host_value = str(raw_host).strip() if raw_host is not None else None
+    port_value = str(raw_port).strip() if raw_port is not None else None
+    host = validate_remote_host(host_value or None) or ""
+    ssh_port = validate_ssh_port(port_value or None) or ""
+    port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
+    return host, port_flag
+
+
 async def _as_owner(request: Request, owner: str, fn, *args, **kwargs):
     """Run an existing route handler with request.state.current_user temporarily
     set to ``owner`` so its internal get_current_user/require_user calls see
@@ -75,6 +95,20 @@ def _scope_owner(request: Request, allowed: set[str]) -> str:
     return require_user(request)
 
 
+def _scope_owner_all(request: Request, required: set[str]) -> str:
+    """Return owner only when an API token has every required scope."""
+    if getattr(request.state, "api_token", False):
+        scopes = set(getattr(request.state, "api_token_scopes", []) or [])
+        missing = required - scopes
+        if missing:
+            raise HTTPException(403, f"API token missing required scope: {' and '.join(sorted(missing))}")
+        owner = getattr(request.state, "api_token_owner", None)
+        if not owner:
+            raise HTTPException(403, "API token has no owner")
+        return owner
+    return require_user(request)
+
+
 def _find_endpoint(router: APIRouter | None, method: str, path: str):
     if router is None:
         return None
@@ -122,7 +156,7 @@ def setup_codex_routes(
                     "read": scoped(EMAIL_READ_SCOPES),
                     "draft": scoped(EMAIL_DRAFT_SCOPES),
                     "send": scoped(EMAIL_SEND_SCOPES),
-                    "actions": ["list", "read", "draft", "send"],
+                    "actions": ["list", "read", "draft_document", "draft", "send"],
                 },
                 "memory": {
                     "read": scoped(MEMORY_READ_SCOPES),
@@ -246,6 +280,59 @@ def setup_codex_routes(
     # Both handlers in routes/email_routes.py already accept `owner=` via
     # FastAPI Depends, so we call them directly without patching state.
 
+    def _email_draft_document_content(body: dict[str, Any]) -> str:
+        def clean(v: Any) -> str:
+            if isinstance(v, list):
+                return ", ".join(str(x).strip() for x in v if str(x).strip())
+            return str(v or "").strip()
+
+        to = clean(body.get("to"))
+        cc = clean(body.get("cc"))
+        bcc = clean(body.get("bcc"))
+        subject = clean(body.get("subject"))
+        in_reply_to = clean(body.get("in_reply_to"))
+        references = clean(body.get("references"))
+        body_text = str(body.get("body") or body.get("body_html") or "").strip()
+        lines = [
+            f"To: {to}",
+        ]
+        if cc:
+            lines.append(f"Cc: {cc}")
+        if bcc:
+            lines.append(f"Bcc: {bcc}")
+        lines.append(f"Subject: {subject}")
+        if in_reply_to:
+            lines.append(f"In-Reply-To: {in_reply_to}")
+        if references:
+            lines.append(f"References: {references}")
+        lines.extend(["---", body_text])
+        return "\n".join(lines).rstrip() + "\n"
+
+    @router.post("/emails/draft-document")
+    async def codex_email_draft_document(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
+        owner = _scope_owner(request, EMAIL_DRAFT_SCOPES)
+        docs_owner = _scope_owner_all(request, DOCS_WRITE_SCOPES)
+        if docs_owner != owner:
+            raise HTTPException(403, "API token owner mismatch")
+        if documents_create_endpoint is None:
+            raise HTTPException(503, "Documents integration is not available")
+        from routes.document_routes import DocumentCreate
+
+        subject = str(body.get("subject") or "Email draft").strip() or "Email draft"
+        title = str(body.get("title") or subject).strip() or "Email draft"
+        req = DocumentCreate(
+            session_id=body.get("session_id"),
+            title=title,
+            language="email",
+            content=_email_draft_document_content(body),
+        )
+        result = await _as_owner(request, owner, documents_create_endpoint, request, req)
+        if isinstance(result, dict):
+            result = dict(result)
+            result["draft_type"] = "document"
+            result["send_required_confirmation"] = True
+        return result
+
     @router.post("/emails/draft")
     async def codex_email_draft(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
         owner = _scope_owner(request, EMAIL_DRAFT_SCOPES)
@@ -486,8 +573,7 @@ def setup_codex_routes(
         task = next((t for t in tasks if t.get("sessionId") == session_id), None)
         if task is None:
             raise HTTPException(404, "task not found")
-        host = (task.get("remoteHost") or "").strip()
-        ssh_port = (task.get("sshPort") or "").strip()
+        host, port_flag = _ssh_prefix_for_task(task)
         # Prefer the persisted log file over the tmux pane. The pane gets
         # overwritten by the post-crash neofetch banner + bash prompt the
         # moment vllm exits; the log file is the raw stdout/stderr and
@@ -499,7 +585,6 @@ def setup_codex_routes(
             f"else tmux capture-pane -t {session_id} -p -S -{tail}; fi"
         )
         if host:
-            port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
             import shlex
             cmd = f"ssh {port_flag}{host} {shlex.quote(inner)}"
         else:
@@ -561,10 +646,8 @@ def setup_codex_routes(
         state = _read_cookbook_state()
         tasks = state.get("tasks") or []
         task = next((t for t in tasks if t.get("sessionId") == session_id), None)
-        host = ((task or {}).get("remoteHost") or "").strip()
-        ssh_port = ((task or {}).get("sshPort") or "").strip()
+        host, port_flag = _ssh_prefix_for_task(task or {})
         if host:
-            port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
             cmd = f"ssh {port_flag}{host} \"tmux kill-session -t {session_id}\""
         else:
             cmd = f"tmux kill-session -t {session_id}"
@@ -714,7 +797,7 @@ def setup_codex_routes(
         norm = dict(body or {})
         sess = (norm.get("tmux_session") or norm.get("session_id") or "").strip()
         model = (norm.get("model") or norm.get("repo_id") or "").strip()
-        host = (norm.get("host") or norm.get("remote_host") or "").strip()
+        host = validate_remote_host((norm.get("host") or norm.get("remote_host") or "").strip() or None) or ""
         port = norm.get("port") or 8000
         import re as _re
         if not sess or not _re.fullmatch(r"[a-zA-Z0-9_-]+", sess):

routes/contacts_routes.py

@@ -12,6 +12,7 @@ import json
 import csv
 import io
 import os
+import inspect
 import httpx
 from pathlib import Path
 from datetime import datetime
@@ -45,10 +46,14 @@ def _save_settings(settings):
 def _get_carddav_config():
     import os
     settings = _load_settings()
+    password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", ""))
+    if password and "carddav_password" in settings:
+        from src.secret_storage import decrypt
+        password = decrypt(password)
     return {
         "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
         "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
-        "password": settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", "")),
+        "password": password,
     }
 
 
@@ -86,11 +91,13 @@ def _normalize_contact(contact: Dict) -> Dict:
     name = str(contact.get("name") or "").strip()
     if not name and emails:
         name = emails[0].split("@")[0]
+    address = str(contact.get("address") or "").strip()
     return {
         "uid": str(contact.get("uid") or uuid.uuid4()),
         "name": name,
         "emails": emails,
         "phones": phones,
+        "address": address,
     }
 
 
@@ -146,7 +153,7 @@ def _parse_vcards(text: str) -> List[Dict]:
     for block in re.split(r"BEGIN:VCARD", text):
         if not block.strip():
             continue
-        contact = {"name": "", "emails": [], "phones": [], "uid": ""}
+        contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""}
         for line in block.split("\n"):
             line = line.strip()
             # Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
@@ -169,6 +176,15 @@ def _parse_vcards(text: str) -> List[Dict]:
                     phone = _vunesc(name_part.split(":", 1)[1])
                     if phone and phone not in contact["phones"]:
                         contact["phones"].append(phone)
+            elif name_part.startswith("ADR"):
+                # vCard ADR is 7 semicolon-separated components:
+                # post-office-box;extended-address;street;locality;region;postal-code;country.
+                # Recover a human-readable string by joining non-empty
+                # components with ", ".
+                if ":" in name_part:
+                    raw = name_part.split(":", 1)[1]
+                    parts = [_vunesc(p).strip() for p in raw.split(";")]
+                    contact["address"] = ", ".join(p for p in parts if p)
             elif name_part.startswith("UID:"):
                 contact["uid"] = _vunesc(name_part[4:])
         if contact["name"] or contact["emails"]:
@@ -193,7 +209,8 @@ def _vesc(value: str) -> str:
 
 def _build_vcard(name: str, email: str, uid: Optional[str] = None,
                  emails: Optional[List[str]] = None,
-                 phones: Optional[List[str]] = None) -> str:
+                 phones: Optional[List[str]] = None,
+                 address: Optional[str] = None) -> str:
     """Build a vCard. Accepts either a single `email` (legacy callers) or
     full `emails`/`phones` lists (edit path). The first email is marked
     PREF=1. All values are RFC-6350-escaped."""
@@ -226,6 +243,12 @@ def _build_vcard(name: str, email: str, uid: Optional[str] = None,
         lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
     for ph in phone_list:
         lines.append(f"TEL:{_vesc(ph)}")
+    # Address: stuff the whole human-readable string into the street
+    # component of ADR. vCard ADR has 7 semicolon-separated components:
+    # post-office-box;extended-address;street;locality;region;postal-code;country.
+    addr = (address or "").strip()
+    if addr:
+        lines.append(f"ADR:;;{_vesc(addr)};;;;")
     lines.append("END:VCARD")
     return "\r\n".join(lines) + "\r\n"
 
@@ -362,7 +385,7 @@ def _resolve_resource_url(uid: str) -> str:
     return _lookup() or _vcard_url(uid)
 
 
-def _create_contact(name: str, email: str) -> bool:
+def _create_contact(name: str, email: str, address: str = "") -> bool:
     """Add a new contact via CardDAV or local contacts."""
     cfg = _get_carddav_config()
     if not _carddav_configured(cfg):
@@ -371,12 +394,12 @@ def _create_contact(name: str, email: str) -> bool:
         for c in contacts:
             if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
                 return True
-        contacts.append(_normalize_contact({"name": name, "emails": [email]}))
+        contacts.append(_normalize_contact({"name": name, "emails": [email], "address": address}))
         _save_local_contacts(contacts)
         return True
 
     contact_uid = str(uuid.uuid4())
-    vcard = _build_vcard(name, email, contact_uid)
+    vcard = _build_vcard(name, email, contact_uid, address=address)
     try:
         url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
         auth = None
@@ -609,7 +632,7 @@ def _contacts_to_csv(contacts: List[Dict]) -> str:
     return out.getvalue()
 
 
-def _update_contact(uid: str, name: str, emails: List[str], phones: List[str]) -> bool:
+def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
     """Rewrite an existing contact via CardDAV or local contacts."""
     cfg = _get_carddav_config()
     if not _carddav_configured(cfg):
@@ -618,16 +641,19 @@ def _update_contact(uid: str, name: str, emails: List[str], phones: List[str]) -
         out = []
         for c in contacts:
             if c.get("uid") == uid:
-                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones}))
+                # Preserve existing address when caller passes "" (only
+                # updating name/emails/phones, not touching address).
+                addr = address if address else c.get("address", "")
+                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
                 found = True
             else:
                 out.append(c)
         if not found:
-            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones}))
+            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
         _save_local_contacts(out)
         return True
 
-    vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones)
+    vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address)
     # Use the real resource href (handles externally-created contacts whose
     # filename != UID); falls back to the <uid>.vcf guess.
     try:
@@ -714,16 +740,39 @@ def setup_contacts_routes():
         """Add a new contact."""
         name = (data.get("name") or "").strip()
         email = (data.get("email") or "").strip()
+        phone = (data.get("phone") or "").strip()
+        address = (data.get("address") or "").strip()
         if not email:
             return {"success": False, "error": "Email required"}
-        # Check if already exists
-        contacts = _fetch_contacts()
-        for c in contacts:
-            if email.lower() in [e.lower() for e in c["emails"]]:
-                return {"success": True, "message": "Already exists", "contact": c}
+        # Check if already exists by email
+        if email:
+            contacts = _fetch_contacts()
+            for c in contacts:
+                if email.lower() in [e.lower() for e in c["emails"]]:
+                    return {"success": True, "message": "Already exists", "contact": c}
         if not name:
             name = email.split("@")[0]
-        ok = _create_contact(name, email)
+        create_params = inspect.signature(_create_contact).parameters
+        if len(create_params) >= 3:
+            ok = _create_contact(name, email, address)
+        else:
+            ok = _create_contact(name, email)
+        # If a phone was provided, do an immediate update to thread it
+        # through (the simple _create_contact signature only takes name +
+        # email + address; phones happen via update).
+        if ok and phone:
+            try:
+                fresh = _fetch_contacts(force=True)
+                created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
+                if created:
+                    _update_contact(
+                        created["uid"], name,
+                        created.get("emails", []),
+                        [phone],
+                        address,
+                    )
+            except Exception:
+                pass
         return {"success": ok}
 
     @router.post("/import")
@@ -785,7 +834,11 @@ def setup_contacts_routes():
                     except ValueError as e:
                         raise HTTPException(400, str(e))
                 else:
-                    settings[key] = data[key]
+                    value = data[key]
+                    if key == "carddav_password" and value:
+                        from src.secret_storage import encrypt
+                        value = encrypt(value)
+                    settings[key] = value
         _save_settings(settings)
         # Force re-fetch
         _contact_cache["fetched_at"] = None
@@ -802,7 +855,7 @@ def setup_contacts_routes():
     # match PUT /{uid} with uid="config".
     @router.put("/{uid}")
     async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
-        """Edit an existing contact — name / emails / phones."""
+        """Edit an existing contact — name / emails / phones / address."""
         name = (data.get("name") or "").strip()
         emails = data.get("emails")
         phones = data.get("phones")
@@ -810,11 +863,12 @@ def setup_contacts_routes():
             emails = [data["email"]]
         emails = [e.strip() for e in (emails or []) if e and e.strip()]
         phones = [p.strip() for p in (phones or []) if p and p.strip()]
-        if not name and not emails:
-            return {"success": False, "error": "Name or email required"}
+        address = (data.get("address") or "").strip()
+        if not name and not emails and not address:
+            return {"success": False, "error": "Name, email, or address required"}
         if not name and emails:
             name = emails[0].split("@")[0]
-        ok = _update_contact(uid, name, emails, phones)
+        ok = _update_contact(uid, name, emails, phones, address)
         return {"success": ok}
 
     @router.delete("/{uid}")

routes/cookbook_helpers.py

@@ -1,12 +1,14 @@
 """cookbook_helpers.py — validators + small helpers shared by the cookbook routes.
 Extracted from cookbook_routes.py; the routes module imports the symbols it needs."""
 
+import json
 import logging
 import ntpath
 import os
 import posixpath
 import re
 import shlex
+from pathlib import Path
 
 from fastapi import HTTPException
 from pydantic import BaseModel
@@ -90,6 +92,24 @@ def _validate_token(v: str | None) -> str | None:
     return v
 
 
+def load_stored_hf_token(*, state_path: Path | str | None = None) -> str:
+    """Return the decrypted HF token from cookbook_state.json, else env fallback."""
+    path = Path(state_path) if state_path else Path(os.environ.get("DATA_DIR", "data")) / "cookbook_state.json"
+    token = ""
+    if path.exists():
+        try:
+            state = json.loads(path.read_text(encoding="utf-8"))
+            env = state.get("env") if isinstance(state, dict) else {}
+            if isinstance(env, dict) and env.get("hfToken"):
+                from src.secret_storage import decrypt
+                token = decrypt(env.get("hfToken") or "")
+        except Exception:
+            token = ""
+    if not token:
+        token = (os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "").strip()
+    return token
+
+
 def _validate_local_dir(v: str | None) -> str | None:
     if v is None or v == "":
         return None
@@ -342,7 +362,12 @@ def _user_shell_path_bootstrap() -> list[str]:
         '  ODYSSEUS_USER_PATH="$("$ODYSSEUS_USER_SHELL" -ic \'printf "__ODYSSEUS_PATH__%s\\n" "$PATH"\' 2>/dev/null | sed -n \'s/^__ODYSSEUS_PATH__//p\' | tail -n 1 || true)"',
         '  if [ -n "$ODYSSEUS_USER_PATH" ]; then export PATH="$ODYSSEUS_USER_PATH:$PATH"; fi',
         'fi',
-        'command -v python3 >/dev/null 2>&1 || python3() { python "$@"; }',
+        # Windows can expose python3 as a Microsoft Store App Execution Alias
+        # under WindowsApps. Git Bash sees that stub as present, but it exits
+        # before running Python. A Windows venv usually has python.exe, not
+        # python3.exe, so treat a missing or WindowsApps python3 as absent.
+        '_odys_py3="$(command -v python3 2>/dev/null || true)"',
+        'case "$_odys_py3" in ""|*[Ww]indows[Aa]pps*) python3() { python "$@"; } ;; esac',
         'command -v python >/dev/null 2>&1 || python() { python3 "$@"; }',
     ]
 
@@ -480,6 +505,8 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "    if u.startswith('KB'): return int(n * 1024)",
         "    return int(n)",
         "def scan_ollama():",
+        "    if any(m.get('is_ollama') for m in models): return",
+        "    if os.name == 'nt' and not os.environ.get('ODYSSEUS_ALLOW_OLLAMA_CLI_SCAN'): return",
         "    if not shutil.which('ollama'): return",
         "    try:",
         "        p = subprocess.run(['ollama', 'list'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=6)",
@@ -510,8 +537,8 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "            models.append({'repo_id':name,'size_bytes':size_bytes,'nb_files':1,'has_incomplete':False,'path':'ollama','backend':'ollama','is_ollama':True})",
         "        return",
         "for _hf_cache in hf_cache_paths(): scan_hf(_hf_cache)",
-        "scan_ollama()",
         "scan_ollama_api()",
+        "scan_ollama()",
     ]
     for model_dir in model_dirs or []:
         lines.append(f"scan_dir(os.path.expanduser({model_dir!r}))")
@@ -553,6 +580,36 @@ _GGUF_PRELUDE_RE = re.compile(
 _OLLAMA_HOST_ASSIGNMENT_RE = re.compile(r"(?:^|\s)OLLAMA_HOST=([^\s]+)")
 _OLLAMA_BIND_RE = re.compile(r"^\[([^\]]+)\]:(\d+)$|^([^:]+):(\d+)$")
 _OLLAMA_BIND_HOST_RE = re.compile(r"^[A-Za-z0-9._:-]+$")
+_LLAMA_CPP_PYTHON_GGML_TYPES = {
+    "f32": "0",
+    "f16": "1",
+    "q4_0": "2",
+    "q4_1": "3",
+    "q5_0": "6",
+    "q5_1": "7",
+    "q8_0": "8",
+    "q8_1": "9",
+    "q2_k": "10",
+    "q3_k": "11",
+    "q4_k": "12",
+    "q5_k": "13",
+    "q6_k": "14",
+    "q8_k": "15",
+    "iq2_xxs": "16",
+    "iq2_xs": "17",
+    "iq3_xxs": "18",
+    "iq1_s": "19",
+    "iq4_nl": "20",
+    "iq3_s": "21",
+    "iq2_s": "22",
+    "iq4_xs": "23",
+    "mxfp4": "39",
+    "nvfp4": "40",
+    "q1_0": "41",
+}
+_LLAMA_CPP_PYTHON_TYPE_FLAG_RE = re.compile(
+    r"(?P<flag>--type_[kv])(?P<sep>\s+|=)(?P<quote>['\"]?)(?P<value>[A-Za-z0-9_]+)(?P=quote)"
+)
 
 
 def _ollama_bind_from_cmd(cmd: str | None, *, default_host: str = "127.0.0.1") -> tuple[str, str]:
@@ -584,6 +641,22 @@ def _ollama_bind_from_cmd(cmd: str | None, *, default_host: str = "127.0.0.1") -
     return f"[{host}]" if bracketed_host else host, port
 
 
+def _normalize_llama_cpp_python_cache_types(cmd: str | None) -> str | None:
+    """Map llama.cpp KV cache type names to llama-cpp-python's integer enum."""
+    if not cmd or "llama_cpp.server" not in cmd:
+        return cmd
+
+    def repl(match: re.Match[str]) -> str:
+        value = match.group("value")
+        mapped = _LLAMA_CPP_PYTHON_GGML_TYPES.get(value.lower())
+        if not mapped:
+            return match.group(0)
+        quote = match.group("quote")
+        return f"{match.group('flag')}{match.group('sep')}{quote}{mapped}{quote}"
+
+    return _LLAMA_CPP_PYTHON_TYPE_FLAG_RE.sub(repl, cmd)
+
+
 def _check_serve_binary(seg: str) -> None:
     """Validate that a single command segment starts with an allowlisted binary
     (after skipping leading env-var assignments like `CUDA_VISIBLE_DEVICES=0`)."""
@@ -722,6 +795,7 @@ def _append_llama_cpp_linux_accel_build_lines(runner_lines: list[str]) -> None:
     runner_lines.append('    done')
     # rm -rf build so a prior poisoned CMakeCache.txt (e.g. from a failed CUDA
     # or HIP attempt) doesn't cause the next configure to reuse stale settings.
+    runner_lines.append('    mkdir -p ~/bin')
     runner_lines.append('    cd ~/llama.cpp && rm -rf build')
     runner_lines.append('    if command -v hipconfig &>/dev/null || [ -d /opt/rocm ] || [ -n "$ROCM_PATH" ] || [ -n "$HIP_PATH" ]; then')
     runner_lines.append('      if command -v hipconfig &>/dev/null; then')
@@ -1026,6 +1100,16 @@ def _diagnose_serve_output(text: str) -> dict | None:
             "vLLM is not installed or not in PATH on this server.",
             [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}],
         ),
+        (
+            r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|"
+            r"(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|"
+            r"Please ensure sgl_kernel is properly installed",
+            "SGLang native dependencies are missing on this server.",
+            [
+                {"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"},
+                {"label": "upgrade sglang-kernel after OS packages are installed", "op": "manual"},
+            ],
+        ),
         (
             r"sglang.*command not found|No module named sglang|SGLang is not installed",
             "SGLang is not installed or not in PATH on this server.",

routes/cookbook_output.py

@@ -0,0 +1,75 @@
+"""Pure helpers for shaping cookbook task output for the status response.
+
+Kept dependency-free (no FastAPI / SQLAlchemy imports) so the behavior can be
+unit-tested without standing up the whole app.
+"""
+
+import re
+
+_FETCHING_ZERO_FILES_RE = re.compile(r"Fetching\s+0\s+files", re.IGNORECASE)
+
+# Probe scripts for the dead-session download check, run as
+# `python3 -c <PROBE> <repo_id> <cache_root>` (locally or over SSH).
+# cache_root is the task's custom download dir, '' for the default HF cache.
+# It has to be passed explicitly: the download runner exports
+# HF_HOME=<local_dir>, so that task's cache lives under <local_dir>/hub, and
+# the probe process's own environment knows nothing about it.
+HF_CACHE_COMPLETE_PROBE = (
+    "import os,sys;"
+    "repo=sys.argv[1];"
+    "root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';"
+    "base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));"
+    "d=os.path.join(base,'models--'+repo.replace('/','--'));"
+    "snap=os.path.join(d,'snapshots');"
+    "ok=os.path.isdir(snap) and any(os.path.isdir(os.path.join(snap,x)) and os.listdir(os.path.join(snap,x)) for x in os.listdir(snap));"
+    "inc=False;"
+    "blobs=os.path.join(d,'blobs');"
+    "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));"
+    "sys.exit(0 if ok and not inc else 1)"
+)
+
+HF_CACHE_INCOMPLETE_PROBE = (
+    "import os,sys;"
+    "repo=sys.argv[1];"
+    "root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';"
+    "base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));"
+    "d=os.path.join(base,'models--'+repo.replace('/','--'));"
+    "blobs=os.path.join(d,'blobs');"
+    "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));"
+    "sys.exit(0 if inc else 1)"
+)
+
+
+def classify_dead_download(full_snapshot: str):
+    """Resolve a dead download session's status from its runner markers.
+
+    The runner prints DOWNLOAD_OK only after exiting 0 (and DOWNLOAD_FAILED
+    otherwise), so the markers stay trustworthy after the tmux pane is gone.
+    Returns (status, zero_files), or None when the snapshot carries no marker
+    and the caller has to fall back to the cache probe. Same precedence as
+    the live-session branch: DOWNLOAD_OK wins, except a "Fetching 0 files"
+    run is an error (nothing matched the include/quant pattern).
+    """
+    if not full_snapshot:
+        return None
+    if "DOWNLOAD_OK" in full_snapshot:
+        if _FETCHING_ZERO_FILES_RE.search(full_snapshot):
+            return ("error", True)
+        return ("completed", False)
+    if "DOWNLOAD_FAILED" in full_snapshot:
+        return ("error", False)
+    return None
+
+
+def error_aware_output_tail(full_snapshot: str, status: str) -> str:
+    """Return the trailing slice of a task log for the status response.
+
+    Failed tasks return the last 50 lines so the "Copy last 50 lines" action
+    surfaces the actual error context (stack traces, build output). Running and
+    other non-error tasks keep the cheaper 12-line tail to limit the payload on
+    the 10s polling interval.
+    """
+    if not full_snapshot:
+        return ""
+    tail_lines = 50 if status == "error" else 12
+    return "\n".join(full_snapshot.splitlines()[-tail_lines:])

routes/cookbook_routes.py

@@ -30,6 +30,10 @@ from core.platform_compat import (
     which_tool,
 )
 from routes.shell_routes import TMUX_LOG_DIR
+from routes.cookbook_output import (
+    error_aware_output_tail, classify_dead_download,
+    HF_CACHE_COMPLETE_PROBE, HF_CACHE_INCOMPLETE_PROBE,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -39,8 +43,13 @@ from routes.cookbook_helpers import (
     _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase,
     _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
     _append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script,
+    load_stored_hf_token,
+    _append_vllm_linux_preflight_lines, _ollama_bind_from_cmd, _pip_install_fallback_chain,
+    _pip_install_no_cache, _user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd,
+    _diagnose_serve_output, run_ssh_command_async,
     _ollama_bind_from_cmd, _pip_install_fallback_chain, _pip_install_no_cache,
     _user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd,
+    _normalize_llama_cpp_python_cache_types,
     ModelDownloadRequest, ServeRequest,
 )
 
@@ -49,7 +58,7 @@ _HF_TOKEN_STATUS_SNIPPET = (
     'echo "[odysseus] HF token: applied"; '
     'else '
     'echo "[odysseus] HF token: NOT SET — gated/private models will be denied. '
-    'Add one in Odysseus Settings -> Cookbook -> HuggingFace Token."; '
+    'Add one in Odysseus Cookbook -> Settings -> HuggingFace Token."; '
     'fi'
 )
 
@@ -165,6 +174,16 @@ def setup_cookbook_routes() -> APIRouter:
                 "vLLM is not installed or not in PATH on this server.",
                 [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}],
             ),
+            (
+                r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|"
+                r"(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|"
+                r"Please ensure sgl_kernel is properly installed",
+                "SGLang native dependencies are missing on this server.",
+                [
+                    {"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"},
+                    {"label": "upgrade sglang-kernel after OS packages are installed", "op": "manual"},
+                ],
+            ),
             (
                 r"sglang.*command not found|No module named sglang|SGLang is not installed",
                 "SGLang is not installed or not in PATH on this server.",
@@ -233,14 +252,7 @@ def setup_cookbook_routes() -> APIRouter:
         return state
 
     def _load_stored_hf_token() -> str:
-        if not _cookbook_state_path.exists():
-            return ""
-        try:
-            state = json.loads(_cookbook_state_path.read_text(encoding="utf-8"))
-            env = state.get("env") if isinstance(state, dict) else {}
-            return _decrypt_secret(env.get("hfToken") if isinstance(env, dict) else "")
-        except Exception:
-            return ""
+        return load_stored_hf_token(state_path=_cookbook_state_path)
 
     def _cookbook_ssh_dir() -> Path:
         # The Docker image keeps cookbook keys under /app/.ssh; that path only
@@ -355,7 +367,11 @@ def setup_cookbook_routes() -> APIRouter:
             # all output to the log the poller reads. Paths handed to bash use
             # POSIX form + shell-quoting so drive paths / spaces survive.
             inner = TMUX_LOG_DIR / f"{session_id}_run.sh"
-            inner.write_text("\n".join(bash_lines) + "\n", encoding="utf-8")
+            pp = shlex.quote(pid_path.as_posix())
+            inner.write_text(
+                f"printf '%s\\n' \"$$\" > {pp}\n" + "\n".join(bash_lines) + "\n",
+                encoding="utf-8",
+            )
             lp = shlex.quote(log_path.as_posix())
             ip = shlex.quote(inner.as_posix())
             script_path = TMUX_LOG_DIR / f"{session_id}.sh"
@@ -660,7 +676,7 @@ def setup_cookbook_routes() -> APIRouter:
             _spf = f"-p {_port} " if _port and _port != "22" else ""
             setup_cmd = (
                 f"scp -O {_pf}-q '{runner_path}' {remote}:{remote_runner} && "
-                f"ssh {_spf}{remote} 'chmod +x {remote_runner} && tmux new-session -d -s {session_id} \"./{remote_runner}\"'"
+                f"ssh {_spf}{remote} 'chmod +x {remote_runner} && tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} \"./{remote_runner}\"'"
             )
         else:
             # Local: run hf download in the background (tmux on POSIX, a detached
@@ -692,7 +708,7 @@ def setup_cookbook_routes() -> APIRouter:
                 lines.append('exec "${SHELL:-/bin/bash}"')
                 wrapper_script.write_text("\n".join(lines) + "\n", encoding="utf-8")
                 wrapper_script.chmod(0o755)
-            setup_cmd = None if IS_WINDOWS else f"tmux new-session -d -s {session_id} {shlex.quote(str(wrapper_script))}"
+            setup_cmd = None if IS_WINDOWS else f"tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} {shlex.quote(str(wrapper_script))}"
 
         logger.info(f"Model download: {req.repo_id} (backend={'ollama' if is_ollama_download else 'hf'}, include={req.include}, session={session_id}, remote={remote})")
         logger.info(f"Download setup_cmd: {setup_cmd}")
@@ -968,9 +984,9 @@ def setup_cookbook_routes() -> APIRouter:
             ssh_args = ["ssh"]
             if ssh_port and ssh_port != "22":
                 ssh_args.extend(["-p", str(ssh_port)])
-            capture_cmd = ssh_args + [remote, "tmux", "capture-pane", "-t", session_id, "-p", "-S", "-200"]
+            capture_cmd = ssh_args + [remote, "tmux", "capture-pane", "-t", session_id, "-p", "-S", "-2000"]
         else:
-            capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-200"]
+            capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-2000"]
 
         _exit_re = re.compile(r"=== Process exited with code (-?\d+) ===")
         for wait_s in _waits:
@@ -1213,6 +1229,7 @@ def setup_cookbook_routes() -> APIRouter:
         # many downstream `"engine" in req.cmd` membership checks can't hit
         # `TypeError: argument of type 'NoneType'` (a 500 instead of a clean 400).
         req.cmd = _validate_serve_cmd(req.cmd) or ""
+        req.cmd = _normalize_llama_cpp_python_cache_types(req.cmd) or ""
         req.cmd = _venv_safe_local_pip_install_cmd(
             req.cmd,
             local=not bool(req.remote_host),
@@ -1267,6 +1284,11 @@ def setup_cookbook_routes() -> APIRouter:
         # LOCAL execution on a native-Windows host never uses tmux (detached
         # process path below), regardless of the UI-supplied platform.
         local_windows = IS_WINDOWS and not remote
+        if is_windows and remote and "diffusion_server.py" in req.cmd:
+            raise HTTPException(
+                400,
+                "Remote Windows Diffusers serving is not supported yet; use local Windows or a Linux remote server.",
+            )
 
         if not is_windows and not local_windows and not await _binary_available("tmux", remote, req.ssh_port):
             return {
@@ -1560,10 +1582,10 @@ def setup_cookbook_routes() -> APIRouter:
                 setup_cmd = (
                     f"{scp_extras}"
                     f"scp -O {_Pf}-q '{runner_path}' {remote}:{remote_runner} && "
-                    f"ssh {_pf}{remote} 'chmod +x {remote_runner} && tmux new-session -d -s {session_id} \"./{remote_runner}\"'"
+                    f"ssh {_pf}{remote} 'chmod +x {remote_runner} && tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} \"./{remote_runner}\"'"
                 )
             else:
-                setup_cmd = f"tmux new-session -d -s {session_id} {shlex.quote(str(runner_path))}"
+                setup_cmd = f"tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} {shlex.quote(str(runner_path))}"
 
         if setup_cmd is None:
             # LOCAL Windows: launch the bash runner detached; no tmux setup_cmd.
@@ -2608,6 +2630,193 @@ def setup_cookbook_routes() -> APIRouter:
             "error": _ollama_library_cache["error"],
         }
 
+    # ── vLLM recipe scraper ─────────────────────────────────────────────
+    # Fetches the official YAML recipe for a model from vllm-project/recipes
+    # and normalizes it into a small JSON the frontend can consume. Cached
+    # per-repo so the GitHub raw endpoint isn't hammered.
+    _vllm_recipe_cache: dict[str, tuple[float, dict | None]] = {}
+    # Manifest of all <org>/<model> ids that have a recipe in the upstream
+    # repo. Cheap to fetch (one Git Tree API call), so we cache the whole
+    # set for ~12h. Per-row "does this model have a recipe?" lookups hit
+    # this set instead of doing 912 individual recipe fetches.
+    _vllm_recipe_manifest: dict = {"fetched_at": 0.0, "models": set(), "error": ""}
+
+    @router.get("/api/cookbook/vllm-recipe-manifest")
+    async def vllm_recipe_manifest(refresh: int = 0):
+        """Return the set of <org>/<model> ids known to have a vLLM recipe.
+        One GitHub Tree API call, 12h cache. The frontend uses this to badge
+        rows in the model list before the user expands them."""
+        import time as _time
+        import httpx as _httpx
+        TTL = 12 * 3600.0
+        now = _time.time()
+        if (
+            refresh
+            or (now - _vllm_recipe_manifest["fetched_at"]) > TTL
+            or not _vllm_recipe_manifest["models"]
+        ):
+            url = (
+                "https://api.github.com/repos/vllm-project/recipes/"
+                "git/trees/main?recursive=1"
+            )
+            def _fetch_sync() -> tuple[int, dict | None, str]:
+                try:
+                    headers = {"Accept": "application/vnd.github+json"}
+                    with _httpx.Client(timeout=10.0, follow_redirects=True) as client:
+                        r = client.get(url, headers=headers)
+                        if r.status_code != 200:
+                            return r.status_code, None, r.text[:200]
+                        return 200, r.json(), ""
+                except Exception as e:
+                    return 0, None, f"fetch error: {e}"
+            status, data, err = await asyncio.to_thread(_fetch_sync)
+            if status == 200 and isinstance(data, dict):
+                models: set[str] = set()
+                for entry in data.get("tree") or []:
+                    path = (entry or {}).get("path") or ""
+                    if not path.startswith("models/") or not path.endswith(".yaml"):
+                        continue
+                    # path = "models/<org>/<model>.yaml" → "<org>/<model>"
+                    body = path[len("models/"):-len(".yaml")]
+                    if "/" in body:
+                        models.add(body)
+                _vllm_recipe_manifest["models"] = models
+                _vllm_recipe_manifest["fetched_at"] = now
+                _vllm_recipe_manifest["error"] = ""
+            else:
+                _vllm_recipe_manifest["error"] = (
+                    f"HTTP {status}: {err}" if status else err
+                )
+                # Don't clobber a stale-but-usable list on transient failures.
+                if not _vllm_recipe_manifest["models"]:
+                    return {
+                        "models": [],
+                        "count": 0,
+                        "error": _vllm_recipe_manifest["error"],
+                    }
+        return {
+            "models": sorted(_vllm_recipe_manifest["models"]),
+            "count": len(_vllm_recipe_manifest["models"]),
+            "fetched_at": _vllm_recipe_manifest["fetched_at"],
+            "error": _vllm_recipe_manifest["error"],
+        }
+
+    @router.get("/api/cookbook/vllm-recipe")
+    async def vllm_recipe(repo: str, refresh: int = 0):
+        """Return the vLLM official recipe for a HuggingFace repo, if one
+        exists at vllm-project/recipes. `repo` is the full HF id like
+        'MiniMaxAI/MiniMax-M2'. Cached 6h."""
+        import time as _time
+        import httpx as _httpx
+        import yaml as _yaml
+
+        TTL = 6 * 3600.0
+        now = _time.time()
+        repo = (repo or "").strip().strip("/")
+        if "/" not in repo:
+            return {"exists": False, "error": "repo must be <org>/<model>"}
+
+        cached = _vllm_recipe_cache.get(repo)
+        if cached and not refresh and (now - cached[0]) < TTL:
+            return cached[1] or {"exists": False, "cached": True}
+
+        url = (
+            f"https://raw.githubusercontent.com/vllm-project/recipes/"
+            f"main/models/{repo}.yaml"
+        )
+
+        def _fetch_sync() -> tuple[int, str]:
+            try:
+                with _httpx.Client(timeout=8.0, follow_redirects=True) as client:
+                    r = client.get(url)
+                    return r.status_code, r.text
+            except Exception as e:
+                return 0, f"fetch error: {e}"
+
+        status, text = await asyncio.to_thread(_fetch_sync)
+        if status == 404:
+            _vllm_recipe_cache[repo] = (now, {"exists": False})
+            return {"exists": False}
+        if status != 200:
+            return {"exists": False, "error": f"HTTP {status}", "transient": True}
+
+        try:
+            doc = _yaml.safe_load(text) or {}
+        except Exception as e:
+            return {"exists": False, "error": f"yaml parse: {e}"}
+
+        meta = doc.get("meta") or {}
+        model = doc.get("model") or {}
+        features = doc.get("features") or {}
+        deps = doc.get("dependencies") or []
+        variants = doc.get("variants") or {}
+        hw_overrides = doc.get("hardware_overrides") or {}
+        strat_overrides = doc.get("strategy_overrides") or {}
+
+        # Tool-call + reasoning parsers, as flat arg arrays, so the frontend
+        # can drop them straight into the launch command.
+        tool_calling = features.get("tool_calling") or {}
+        reasoning = features.get("reasoning") or {}
+
+        normalized = {
+            "exists": True,
+            "source_url": url,
+            "title": meta.get("title") or "",
+            "provider": meta.get("provider") or "",
+            "description": meta.get("description") or "",
+            "date_updated": str(meta.get("date_updated") or ""),
+            "hardware_support": meta.get("hardware") or {},
+            "model_id": model.get("model_id") or repo,
+            "min_vllm_version": model.get("min_vllm_version") or "",
+            "architecture": model.get("architecture") or "",
+            "parameter_count": model.get("parameter_count") or "",
+            "active_parameters": model.get("active_parameters") or "",
+            "context_length": model.get("context_length") or 0,
+            "base_args": list(model.get("base_args") or []),
+            "base_env": dict(model.get("base_env") or {}),
+            "tool_calling": {
+                "description": tool_calling.get("description") or "",
+                "args": list(tool_calling.get("args") or []),
+            } if tool_calling else None,
+            "reasoning": {
+                "description": reasoning.get("description") or "",
+                "args": list(reasoning.get("args") or []),
+            } if reasoning else None,
+            "dependencies": [
+                {
+                    "note": (d.get("note") or "").strip(),
+                    "command": (d.get("command") or "").strip(),
+                    "optional": bool(d.get("optional", False)),
+                }
+                for d in deps if isinstance(d, dict)
+            ],
+            "variants": {
+                k: {
+                    "model_id": v.get("model_id") or model.get("model_id") or repo,
+                    "precision": v.get("precision") or "",
+                    "vram_minimum_gb": v.get("vram_minimum_gb") or 0,
+                    "description": v.get("description") or "",
+                    "extra_args": list(v.get("extra_args") or []),
+                    "extra_env": dict(v.get("extra_env") or {}),
+                }
+                for k, v in variants.items() if isinstance(v, dict)
+            },
+            "hardware_overrides": {
+                hw: {
+                    "extra_args": list((ov or {}).get("extra_args") or []),
+                    "extra_env": dict((ov or {}).get("extra_env") or {}),
+                }
+                for hw, ov in hw_overrides.items() if isinstance(ov, dict)
+            },
+            "strategy_overrides": {
+                strat: dict(ov or {})
+                for strat, ov in strat_overrides.items() if isinstance(ov, dict)
+            },
+            "compatible_strategies": list(doc.get("compatible_strategies") or []),
+        }
+        _vllm_recipe_cache[repo] = (now, normalized)
+        return normalized
+
     @router.get("/api/cookbook/tasks/status")
     async def cookbook_tasks_status(request: Request):
         """Check status of all active cookbook tmux sessions.
@@ -2622,30 +2831,20 @@ def setup_cookbook_routes() -> APIRouter:
     def _cookbook_tasks_status_sync():
         import subprocess
 
-        def _download_cache_complete(repo_id: str, remote_host: str = "", ssh_port: str = "") -> bool:
+        def _download_cache_complete(repo_id: str, remote_host: str = "", ssh_port: str = "", cache_root: str = "") -> bool:
             """Best-effort check for a completed HF cache entry.
 
             tmux output can stop at a stale progress line if the pane/session
             disappears before Cookbook captures the final DOWNLOAD_OK marker.
             In that case, trust the cache shape: a snapshot directory with files
             and no *.incomplete blobs means HuggingFace finished materializing the
-            model.
+            model. cache_root is the task's custom download dir — the runner
+            pointed HF_HOME there, so the cache lives under <cache_root>/hub,
+            not wherever this probe's environment says.
             """
             if not repo_id or "/" not in repo_id:
                 return False
-            py = (
-                "import os,sys;"
-                "repo=sys.argv[1];"
-                "base=os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub');"
-                "d=os.path.join(base,'models--'+repo.replace('/','--'));"
-                "snap=os.path.join(d,'snapshots');"
-                "ok=os.path.isdir(snap) and any(os.path.isdir(os.path.join(snap,x)) and os.listdir(os.path.join(snap,x)) for x in os.listdir(snap));"
-                "inc=False;"
-                "blobs=os.path.join(d,'blobs');"
-                "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));"
-                "sys.exit(0 if ok and not inc else 1)"
-            )
-            cmd = ["python3", "-c", py, repo_id]
+            cmd = ["python3", "-c", HF_CACHE_COMPLETE_PROBE, repo_id, cache_root or ""]
             try:
                 if remote_host:
                     ssh_base = ["ssh"]
@@ -2659,7 +2858,7 @@ def setup_cookbook_routes() -> APIRouter:
             except Exception:
                 return False
 
-        def _download_cache_incomplete(repo_id: str, remote_host: str = "", ssh_port: str = "") -> bool:
+        def _download_cache_incomplete(repo_id: str, remote_host: str = "", ssh_port: str = "", cache_root: str = "") -> bool:
             """Best-effort check for resumable HF partial blobs.
 
             A lost SSH/tmux session can leave a real download still incomplete.
@@ -2668,16 +2867,7 @@ def setup_cookbook_routes() -> APIRouter:
             """
             if not repo_id or "/" not in repo_id:
                 return False
-            py = (
-                "import os,sys;"
-                "repo=sys.argv[1];"
-                "base=os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub');"
-                "d=os.path.join(base,'models--'+repo.replace('/','--'));"
-                "blobs=os.path.join(d,'blobs');"
-                "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));"
-                "sys.exit(0 if inc else 1)"
-            )
-            cmd = ["python3", "-c", py, repo_id]
+            cmd = ["python3", "-c", HF_CACHE_INCOMPLETE_PROBE, repo_id, cache_root or ""]
             try:
                 if remote_host:
                     ssh_base = ["ssh"]
@@ -2873,6 +3063,7 @@ def setup_cookbook_routes() -> APIRouter:
             # snapshot to classify (DOWNLOAD_OK / exit marker) — evaluate it even
             # when the PID is gone instead of blindly reporting "stopped".
             download_zero_files = False
+            exit_code = None
             status = "unknown"
             download_has_ok = task_type == "download" and "DOWNLOAD_OK" in full_snapshot
             download_has_failed = task_type == "download" and "DOWNLOAD_FAILED" in full_snapshot
@@ -2881,7 +3072,7 @@ def setup_cookbook_routes() -> APIRouter:
                 and (
                     ".incomplete" in full_snapshot
                     or bool(re.search(r'model-\d+-of-\d+\.[A-Za-z0-9_.-]+:\s+(?:[0-9]|[1-8][0-9])%', full_snapshot))
-                    or _download_cache_incomplete(_payload.get("repo_id") or model, remote, str(_tport or ""))
+                    or _download_cache_incomplete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
                 )
             )
             if is_alive or (local_win_task and full_snapshot):
@@ -2922,11 +3113,19 @@ def setup_cookbook_routes() -> APIRouter:
                 else:
                     status = "running"
             else:
-                # Session is dead — check if it completed or crashed
-                if (
+                # Session is dead — check if it completed or crashed. The
+                # runner markers in the retained output are conclusive
+                # (DOWNLOAD_OK only prints after exit 0), so check them before
+                # the cache probe, which can't see ollama pulls at all.
+                marker = classify_dead_download(full_snapshot) if task_type == "download" else None
+                if marker is not None:
+                    status, download_zero_files = marker
+                    if status == "completed" and not progress_text:
+                        progress_text = "Download complete"
+                elif (
                     task_type == "download"
                     and not download_has_incomplete_evidence
-                    and _download_cache_complete(_payload.get("repo_id") or model, remote, str(_tport or ""))
+                    and _download_cache_complete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
                 ):
                     status = "completed"
                     if not progress_text:
@@ -2946,7 +3145,7 @@ def setup_cookbook_routes() -> APIRouter:
                 status = "error"
             if download_zero_files:
                 diagnosis = {"message": "No matching files were downloaded. The model repo or filename/quant pattern may be wrong (for example a ':Q4_K_M' tag that does not exist in the repo). Check the repo and the include/quant pattern."}
-            output_tail = "\n".join(full_snapshot.splitlines()[-12:]) if full_snapshot else ""
+            output_tail = error_aware_output_tail(full_snapshot, status)
 
             results.append({
                 "session_id": session_id,
@@ -2957,6 +3156,7 @@ def setup_cookbook_routes() -> APIRouter:
                 "phase": serve_phase,
                 "diagnosis": diagnosis,
                 "output_tail": output_tail,
+                "exit_code": exit_code,
                 "cmd": _payload.get("_cmd") or "",
                 "tps": phase_info.get("tps"),
                 "reqs": phase_info.get("reqs"),

routes/diagnostics_routes.py

@@ -1,12 +1,13 @@
 """Diagnostics routes — /api/db/stats, /api/rag/stats, /api/test/youtube, /api/test-research."""
 
 import logging
+import os
 from typing import Dict, Any
 
 from fastapi import APIRouter, HTTPException, Form, Request
 
 from services.youtube.youtube_handler import extract_youtube_id, extract_transcript_async
-from core.constants import DEFAULT_HOST
+from core.constants import DEFAULT_HOST, DATA_DIR
 from core.middleware import require_admin
 
 logger = logging.getLogger(__name__)
@@ -28,6 +29,30 @@ def setup_diagnostics_routes(
         from src.service_health import collect_service_health
         return await collect_service_health(rag_manager, memory_vector)
 
+    @router.get("/api/diagnostics/logs")
+    async def get_diagnostics_logs(request: Request, limit: int = 200) -> Dict[str, Any]:
+        require_admin(request)
+        limit = max(1, min(limit, 1000))
+        try:
+            log_file = os.path.join(DATA_DIR, "logs", "app.log")
+            if not os.path.exists(log_file):
+                return {"status": "success", "logs": []}
+
+            # Safe tail read of the log file (max 5MB via rotation)
+            with open(log_file, "r", encoding="utf-8", errors="ignore") as f:
+                lines = f.readlines()
+
+            tail_lines = lines[-limit:] if len(lines) > limit else lines
+            tail_lines = [line.rstrip('\r\n') for line in tail_lines]
+
+            return {
+                "status": "success",
+                "logs": tail_lines
+            }
+        except Exception as e:
+            logger.error(f"Diagnostics logs retrieval error: {e}")
+            raise HTTPException(500, f"Failed to retrieve logs: {str(e)}")
+
     @router.get("/api/db/stats")
     async def get_database_stats(request: Request) -> Dict[str, Any]:
         require_admin(request)

routes/document_helpers.py

@@ -102,8 +102,11 @@ def _owner_session_filter(q, user):
 
     The owner backfill runs in init_db before the app serves requests, so
     by the time this filter is live there are no NULL-owner rows to leak;
-    we therefore match the owner strictly."""
-    if user is None:
+    we therefore match the owner strictly for authenticated callers."""
+    if not user:
+        from src.auth_helpers import _auth_disabled
+        if user == "" or _auth_disabled():
+            return q
         return q.filter(False)
     return q.filter(Document.owner == user)
 

routes/document_routes.py

@@ -503,7 +503,8 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
         user = get_current_user(request)
         try:
             data = await request.json()
-        except Exception:
+        except Exception as e:
+            logger.warning("Failed to parse export request body, defaulting to empty", exc_info=e)
             data = {}
         ids = data.get("ids") or []
         if not ids:
@@ -645,8 +646,8 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                     try:
                         from src.agent_tools.document_tools import clear_active_document
                         clear_active_document(doc_id)
-                    except Exception:
-                        pass
+                    except Exception as e:
+                        logger.warning("Failed to clear active document %r on detach", doc_id, exc_info=e)
             db.commit()
             db.refresh(doc)
             return _doc_to_dict(doc)

routes/email_helpers.py

@@ -13,6 +13,8 @@ and `email_pollers.py` (the background loops):
 """
 
 import os
+import base64
+import time
 import imaplib
 import smtplib
 import email as email_mod
@@ -38,6 +40,106 @@ from src.secret_storage import decrypt as _decrypt
 logger = logging.getLogger(__name__)
 
 
+def _xoauth2_raw(user: str, access_token: str) -> str:
+    """The SASL XOAUTH2 initial-response string (unencoded).
+
+    Both smtplib.SMTP.auth() and imaplib.IMAP4.authenticate() base64-encode
+    the value their callback returns, so callers pass this raw form — never
+    pre-encoded — to avoid double base64.
+    """
+    return f"user={user}\x01auth=Bearer {access_token}\x01\x01"
+
+
+def _xoauth2_bytes(user: str, access_token: str) -> bytes:
+    """Raw XOAUTH2 bytes for imaplib's authenticate() callback."""
+    return _xoauth2_raw(user, access_token).encode()
+
+
+def make_oauth_state(account_id: str, owner: str) -> str:
+    """Return an HMAC-signed, base64-encoded OAuth state token.
+
+    Encodes account_id + owner + a random nonce, signed with the app secret
+    so the callback can validate that the flow was initiated by an
+    authenticated, owning user (CSRF / state-forgery protection).
+    """
+    import hmac as _hmac, hashlib as _hl, secrets as _sec
+    from src.secret_storage import _load_or_create_key
+    nonce = _sec.token_hex(16)
+    payload = json.dumps({"a": account_id, "o": owner, "n": nonce}, separators=(",", ":"))
+    sig = _hmac.new(_load_or_create_key(), payload.encode(), _hl.sha256).hexdigest()
+    return base64.urlsafe_b64encode(f"{payload}|{sig}".encode()).decode()
+
+
+def verify_oauth_state(state: str) -> dict | None:
+    """Verify an OAuth state token's HMAC signature.
+
+    Returns the decoded payload dict ({"a", "o", "n"}) on success, or None if
+    the token is malformed, tampered, or signed with a different key.
+    """
+    import hmac as _hmac, hashlib as _hl
+    from src.secret_storage import _load_or_create_key
+    try:
+        decoded = base64.urlsafe_b64decode(state.encode()).decode()
+        payload, sig = decoded.rsplit("|", 1)
+        expected = _hmac.new(_load_or_create_key(), payload.encode(), _hl.sha256).hexdigest()
+        if not _hmac.compare_digest(sig, expected):
+            return None
+        return json.loads(payload)
+    except Exception:
+        return None
+
+
+def _refresh_google_token(account_id: str) -> str | None:
+    """Exchange the stored refresh token for a new access token and persist it."""
+    import httpx
+    from core.database import SessionLocal as _SL, EmailAccount as _EA
+    from src.secret_storage import encrypt as _enc, decrypt as _dec
+    client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "")
+    client_secret = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET", "")
+    if not client_id or not client_secret:
+        return None
+    db = _SL()
+    try:
+        row = db.get(_EA, account_id)
+        if not row or not row.oauth_refresh_token:
+            return None
+        refresh_token = _dec(row.oauth_refresh_token or "")
+        if not refresh_token:
+            return None
+        resp = httpx.post("https://oauth2.googleapis.com/token", data={
+            "client_id": client_id,
+            "client_secret": client_secret,
+            "refresh_token": refresh_token,
+            "grant_type": "refresh_token",
+        }, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        access_token = data["access_token"]
+        row.oauth_access_token = _enc(access_token)
+        row.oauth_token_expiry = str(int(time.time()) + data.get("expires_in", 3600))
+        db.commit()
+        return access_token
+    except Exception:
+        logger.warning(f"Google token refresh failed for account {account_id}")
+        return None
+    finally:
+        db.close()
+
+
+def _get_valid_google_token(account_id: str, cfg: dict) -> str | None:
+    """Return a valid Google access token, refreshing if expired or missing."""
+    from src.secret_storage import decrypt as _dec
+    access_token = _dec(cfg.get("oauth_access_token") or "")
+    expiry_str = cfg.get("oauth_token_expiry") or ""
+    if access_token and expiry_str:
+        try:
+            if int(expiry_str) - 60 > time.time():
+                return access_token
+        except (ValueError, TypeError):
+            pass
+    return _refresh_google_token(account_id)
+
+
 def _smtp_security_mode(cfg: dict) -> str:
     raw = str(cfg.get("smtp_security") or "").strip().lower()
     if raw in {"ssl", "starttls", "none"}:
@@ -54,20 +156,29 @@ def _send_smtp_message(cfg: dict, from_addr: str, recipients: list[str], message
     port = int(cfg.get("smtp_port") or 465)
     user = cfg.get("smtp_user") or ""
     password = cfg.get("smtp_password") or ""
+
+    def _auth_smtp(smtp):
+        if cfg.get("oauth_provider") == "google":
+            token = _get_valid_google_token(cfg.get("account_id"), cfg)
+            if not token:
+                raise RuntimeError("Google OAuth token unavailable — reconnect the account")
+            smtp.ehlo()
+            smtp.auth("XOAUTH2", lambda challenge=None: _xoauth2_raw(user, token), initial_response_ok=True)
+        elif user and password:
+            smtp.login(user, password)
+
     security = _smtp_security_mode(cfg)
 
     if security == "ssl":
         with smtplib.SMTP_SSL(host, port, timeout=timeout) as smtp:
-            if user and password:
-                smtp.login(user, password)
+            _auth_smtp(smtp)
             smtp.sendmail(from_addr, recipients, message)
         return
 
     with smtplib.SMTP(host, port, timeout=timeout) as smtp:
         if security == "starttls":
             smtp.starttls()
-        if user and password:
-            smtp.login(user, password)
+        _auth_smtp(smtp)
         smtp.sendmail(from_addr, recipients, message)
 
 
@@ -304,6 +415,7 @@ OWNER_SCOPED_EMAIL_CACHE_TABLES = {
     "email_ai_replies",
     "email_calendar_extractions",
     "email_urgency_alerts",
+    "sender_signatures",
 }
 
 
@@ -341,6 +453,55 @@ def _ensure_owner_scoped_email_cache_table(conn, table: str, create_sql: str, co
         _lg.getLogger(__name__).warning(f"{table} owner-migration skipped: {_mig_e}")
 
 
+def _ensure_sender_signatures_table(conn):
+    """Create/migrate learned sender signatures to an owner-scoped cache."""
+    create_sql = """
+        CREATE TABLE IF NOT EXISTS sender_signatures (
+            from_address TEXT,
+            owner TEXT DEFAULT '',
+            signature_text TEXT,
+            sample_count INTEGER,
+            last_built_at TEXT NOT NULL,
+            model_used TEXT,
+            source TEXT,
+            PRIMARY KEY (from_address, owner)
+        )
+    """
+    conn.execute(create_sql)
+    try:
+        info = conn.execute("PRAGMA table_info(sender_signatures)").fetchall()
+        cols = [r[1] for r in info]
+        pk_cols = [r[1] for r in sorted((r for r in info if r[5]), key=lambda r: r[5])]
+        if "owner" in cols and pk_cols == ["from_address", "owner"]:
+            return
+
+        conn.execute("ALTER TABLE sender_signatures RENAME TO sender_signatures__old")
+        conn.execute(create_sql)
+        old_cols = [r[1] for r in conn.execute("PRAGMA table_info(sender_signatures__old)").fetchall()]
+        copy_cols = [
+            c for c in (
+                "from_address",
+                "signature_text",
+                "sample_count",
+                "last_built_at",
+                "model_used",
+                "source",
+            )
+            if c in old_cols
+        ]
+        source_owner = "COALESCE(owner, '')" if "owner" in old_cols else "''"
+        conn.execute(
+            f"INSERT OR IGNORE INTO sender_signatures "
+            f"({', '.join([*copy_cols, 'owner'])}) "
+            f"SELECT {', '.join([*copy_cols, source_owner])} "
+            f"FROM sender_signatures__old"
+        )
+        conn.execute("DROP TABLE sender_signatures__old")
+    except Exception as _mig_e:
+        import logging as _lg
+        _lg.getLogger(__name__).warning(f"sender_signatures owner-migration skipped: {_mig_e}")
+
+
 def attachment_extract_dir(folder: str, uid: str) -> Path:
     """Containment-safe extraction directory for an attachment.
 
@@ -559,20 +720,10 @@ def _init_scheduled_db():
             conn.execute("ALTER TABLE email_boundaries ADD COLUMN turns_json TEXT")
     except Exception:
         pass
-    # Per-sender signature cache. Populated by `learn_sender_signatures`
-    # action: the LLM extracts the common trailing block across N emails
-    # from each sender; the renderer folds it consistently for every
-    # future email from that address.
-    conn.execute("""
-        CREATE TABLE IF NOT EXISTS sender_signatures (
-            from_address TEXT PRIMARY KEY,
-            signature_text TEXT,
-            sample_count INTEGER,
-            last_built_at TEXT NOT NULL,
-            model_used TEXT,
-            source TEXT
-        )
-    """)
+    # Per-sender signature cache. Populated by `learn_sender_signatures`.
+    # Message sender addresses are global, so signatures must be scoped to the
+    # mailbox owner before `/read` returns them to the renderer.
+    _ensure_sender_signatures_table(conn)
     conn.commit()
     conn.close()
 
@@ -661,10 +812,16 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict:
                     "imap_password": _decrypt(row.imap_password or ""),
                     "imap_starttls": bool(row.imap_starttls),
                     "from_address": row.from_address or row.imap_user or "",
+                    "oauth_provider": row.oauth_provider or "",
+                    "oauth_access_token": row.oauth_access_token or "",
+                    "oauth_refresh_token": row.oauth_refresh_token or "",
+                    "oauth_token_expiry": row.oauth_token_expiry or "",
+                    "display_name": row.display_name or "",
                 }
-                if not (cfg["smtp_host"] and cfg["smtp_user"] and cfg["smtp_password"]):
+                is_oauth = bool(cfg.get("oauth_provider"))
+                if not is_oauth and not (cfg["smtp_host"] and cfg["smtp_user"] and cfg["smtp_password"]):
                     logger.warning(f"SMTP not configured for account {row.name!r}")
-                if not (cfg["imap_host"] and cfg["imap_user"] and cfg["imap_password"]):
+                if not is_oauth and not (cfg["imap_host"] and cfg["imap_user"] and cfg["imap_password"]):
                     logger.warning(f"IMAP not configured for account {row.name!r}")
                 return cfg
         finally:
@@ -785,12 +942,19 @@ def _imap_connect(account_id: str | None = None, owner: str = "",
         timeout=timeout,
     )
     try:
-        conn.login(cfg["imap_user"], cfg["imap_password"])
+        if cfg.get("oauth_provider") == "google":
+            token = _get_valid_google_token(cfg.get("account_id"), cfg)
+            if not token:
+                raise RuntimeError("Google OAuth token unavailable — reconnect the account in Settings → Integrations")
+            conn.authenticate("XOAUTH2", lambda x: _xoauth2_bytes(cfg["imap_user"], token))
+        else:
+            conn.login(cfg["imap_user"], cfg["imap_password"])
     except Exception:
         # A failed AUTHENTICATE (e.g. an Office 365 app password on an
-        # MFA-enabled tenant, #3174) otherwise orphans the already-connected
-        # socket; close it before propagating so a misconfigured account
-        # can't leak one descriptor per retry / background poller pass.
+        # MFA-enabled tenant, #3174, or an expired/revoked OAuth token)
+        # otherwise orphans the already-connected socket; close it before
+        # propagating so a misconfigured account can't leak one descriptor
+        # per retry / background poller pass.
         try:
             conn.shutdown()
         except Exception:

routes/email_routes.py

@@ -13,7 +13,9 @@ handlers need. The split is mechanical — no behavior change.
 """
 
 import asyncio
+import os
 import sqlite3 as _sql3
+import time
 import email as email_mod
 import email.header
 import email.utils
@@ -43,6 +45,7 @@ from routes.email_helpers import (
     _load_settings, _save_settings, _get_email_config,
     _send_smtp_message, _smtp_security_mode,
     _IMAP_TIMEOUT_SECONDS, _open_imap_connection,
+    make_oauth_state, verify_oauth_state,
     _imap_connect, _imap, _decode_header, _detect_sent_folder, _detect_drafts_folder,
     _extract_attachment_text, _list_attachments_from_msg,
     _extract_attachment_to_disk, _extract_html, _extract_text,
@@ -76,15 +79,16 @@ def _email_tag_owner_aliases(account_id: str | None, owner: str = "") -> list[st
                         cfg.get("smtp_user") or "",
                         cfg.get("from_address") or "",
                     ])
-                except Exception:
+                except Exception as _e:
+                    logger.warning("Failed to resolve email account alias", exc_info=_e)
                     resolved_account_id = None
             row = db.get(_EA, resolved_account_id) if resolved_account_id else None
             if row:
                 aliases.extend([row.owner or "", row.imap_user or "", row.from_address or ""])
         finally:
             db.close()
-    except Exception:
-        pass
+    except Exception as _e:
+        logger.warning("Failed to load email aliases", exc_info=_e)
     out = []
     for a in aliases:
         a = (a or "").strip()
@@ -249,8 +253,45 @@ def _uid_from_fetch_meta(meta_b: bytes) -> str:
     return m.group(1).decode() if m else ""
 
 
+_FETCH_SEQ_RE = re.compile(rb"^(\d+)\s+\(")
+
+
+def _group_uid_fetch_records(msg_data) -> list:
+    """Group an imaplib UID FETCH response into per-message (meta, payload).
+
+    imaplib yields an interleaved list: ``(meta, literal)`` tuples for
+    attributes that carry a literal (``RFC822.HEADER {n}`` etc.) plus bare
+    ``bytes`` elements for everything the server sends outside a literal.
+    Where each attribute lands is server-specific: Dovecot sends FLAGS
+    *before* the header literal (so it ends up inside the tuple meta), while
+    Gmail sends FLAGS *after* it, arriving as a bare ``b' FLAGS (\\Seen))'``
+    element. Dropping bare elements therefore silently loses FLAGS on Gmail
+    and every message renders as unread/unflagged.
+
+    A tuple whose meta starts with a sequence number opens a new record;
+    every other part — continuation tuple or bare bytes — is folded into the
+    current record's meta so attribute regexes see the full meta text.
+    Plain ``b')'`` terminators get folded in too, which is harmless.
+    """
+    grouped: list = []  # list of (meta_bytes, payload_bytes_or_None)
+    for part in (msg_data or []):
+        if isinstance(part, tuple):
+            meta_b = part[0] if isinstance(part[0], (bytes, bytearray)) else str(part[0]).encode()
+            if _FETCH_SEQ_RE.match(meta_b):
+                grouped.append((meta_b, part[1]))
+            elif grouped:
+                cur_meta, cur_payload = grouped[-1]
+                grouped[-1] = (cur_meta + b" " + meta_b, cur_payload or part[1])
+        elif isinstance(part, (bytes, bytearray)) and grouped:
+            cur_meta, cur_payload = grouped[-1]
+            grouped[-1] = (cur_meta + b" " + bytes(part), cur_payload)
+    return grouped
+
+
 def _smtp_ready(cfg: dict) -> bool:
-    return bool(cfg.get("smtp_host") and cfg.get("smtp_user") and cfg.get("smtp_password"))
+    if not cfg.get("smtp_host") or not cfg.get("smtp_user"):
+        return False
+    return bool(cfg.get("smtp_password") or cfg.get("oauth_provider"))
 
 
 def _resolve_send_config(account_id: str | None = None, owner: str = "") -> dict:
@@ -799,20 +840,11 @@ def setup_email_routes():
                 except Exception as e:
                     logger.warning(f"Batch fetch failed, falling back to per-UID: {e}")
                     status, msg_data = "NO", []
-                # imaplib batch responses interleave (meta, payload) tuples and
-                # `b')'` terminators. Group by message: each tuple where the
-                # meta begins with a seq number starts a new message record.
-                seq_re = re.compile(rb'^(\d+)\s+\(')
-                grouped = []  # list of (meta_str, payload_bytes)
-                for part in (msg_data or []):
-                    if isinstance(part, tuple):
-                        meta_b = part[0] if isinstance(part[0], (bytes, bytearray)) else str(part[0]).encode()
-                        if seq_re.match(meta_b):
-                            grouped.append((meta_b, part[1]))
-                        elif grouped:
-                            # continuation of previous message — concatenate meta info if any
-                            cur_meta, cur_payload = grouped[-1]
-                            grouped[-1] = (cur_meta + b" " + meta_b, cur_payload or part[1])
+                # Group the batched response into per-message (meta, payload)
+                # records. Bare bytes parts must be kept: Gmail returns FLAGS
+                # after the header literal as a bare element, and dropping it
+                # rendered every Gmail message as unread/unflagged.
+                grouped = _group_uid_fetch_records(msg_data)
 
                 if status != "OK" and not grouped:
                     conn.logout()
@@ -1061,14 +1093,22 @@ def setup_email_routes():
             return {"contacts": [], "error": "Mail operation failed"}
 
     @router.get("/search")
-    async def search_emails(
+    # Sync def: the body is blocking IMAP I/O with no awaits. As `async def` it ran
+    # directly on the event loop and stalled the whole app during a search; as a sync
+    # def FastAPI runs it in a threadpool, keeping the loop responsive.
+    def search_emails(
         q: str = Query(""),
         folder: str = Query("INBOX"),
         limit: int = Query(50),
         account_id: str | None = Query(None),
         owner: str = Depends(require_owner),
     ):
-        """Search emails server-side via IMAP SEARCH. Matches subject, from, or body text."""
+        """Search emails server-side via IMAP SEARCH. Matches subject, from, or body text.
+
+        When the caller asks for INBOX and the account has an "All Mail"
+        folder (Gmail does), we transparently swap to All Mail so the
+        search surfaces archived / labelled emails too. Plain IMAP
+        accounts fall back to whatever folder the caller specified."""
         if not q or len(q) < 2:
             return {"emails": [], "total": 0, "query": q}
         # CRLF in q would terminate the IMAP command early — reject defensively.
@@ -1076,7 +1116,27 @@ def setup_email_routes():
             raise HTTPException(400, "Invalid query")
         try:
             with _imap(account_id, owner=owner) as conn:
-                conn.select(_q(folder), readonly=True)
+                # If the user asked for INBOX, try to upgrade to All Mail —
+                # one folder == every email on Gmail-class servers.
+                effective_folder = folder
+                if (folder or "").upper() == "INBOX":
+                    try:
+                        status, folder_lines = conn.list()
+                        if status == "OK" and folder_lines:
+                            for raw in folder_lines:
+                                if isinstance(raw, bytes):
+                                    raw = raw.decode("utf-8", errors="replace")
+                                m = re.match(r"\((?P<flags>[^)]*)\)\s+\"[^\"]*\"\s+(?P<name>.+)", raw)
+                                if not m:
+                                    continue
+                                flags = (m.group("flags") or "").lower()
+                                name = m.group("name").strip().strip('"')
+                                if "\\all" in flags or "all mail" in name.lower():
+                                    effective_folder = name
+                                    break
+                    except Exception:
+                        pass
+                conn.select(_q(effective_folder), readonly=True)
 
                 # Escape backslash and quote for the IMAP-SEARCH quoted-string.
                 q_escaped = q.replace('\\', '\\\\').replace('"', '\\"')
@@ -1084,7 +1144,7 @@ def setup_email_routes():
 
                 status, data = _imap_uid_search(conn, search_cmd)
                 if status != "OK" or not data[0]:
-                    return {"emails": [], "total": 0, "query": q}
+                    return {"emails": [], "total": 0, "query": q, "folder": effective_folder}
 
                 uid_list = data[0].split()
                 total = len(uid_list)
@@ -1098,14 +1158,15 @@ def setup_email_routes():
                             continue
                         raw_header = None
                         flags = ""
-                        for part in msg_data:
-                            if isinstance(part, tuple):
-                                meta = part[0].decode() if isinstance(part[0], bytes) else str(part[0])
-                                if b"RFC822.HEADER" in part[0] if isinstance(part[0], bytes) else "RFC822.HEADER" in meta:
-                                    raw_header = part[1]
-                                flag_match = re.search(r'FLAGS \(([^)]*)\)', meta)
-                                if flag_match:
-                                    flags = flag_match.group(1)
+                        # Same Gmail caveat as the list route: FLAGS may
+                        # arrive after the header literal, so group bare
+                        # parts back into the message meta before scanning.
+                        for meta_b, payload in _group_uid_fetch_records(msg_data):
+                            if payload and b"RFC822.HEADER" in meta_b:
+                                raw_header = payload
+                            flag_match = re.search(rb'FLAGS \(([^)]*)\)', meta_b)
+                            if flag_match:
+                                flags = flag_match.group(1).decode(errors="replace")
                         if not raw_header:
                             continue
                         msg = email_mod.message_from_bytes(raw_header)
@@ -1148,6 +1209,13 @@ def setup_email_routes():
                             "is_flagged": "\\Flagged" in flags,
                             "flags": flags,
                             "has_attachments": has_attachments,
+                            # Stamp the folder so the frontend opens each
+                            # email from the folder it actually lives in
+                            # (the search may have run against All Mail
+                            # even though the caller asked for INBOX),
+                            # otherwise clicks open whatever happens to
+                            # have the same UID in INBOX → wrong email.
+                            "folder": effective_folder,
                         })
                     except Exception as e:
                         logger.warning(f"Error parsing search result {uid}: {e}")
@@ -1247,8 +1315,9 @@ def setup_email_routes():
                 try:
                     if sender_addr:
                         _rs = _c.execute(
-                            "SELECT signature_text FROM sender_signatures WHERE from_address = ?",
-                            (sender_addr.lower().strip(),),
+                            f"SELECT signature_text FROM sender_signatures "
+                            f"WHERE from_address = ? AND {owner_clause}",
+                            (sender_addr.lower().strip(), *owner_params),
                         ).fetchone()
                         if _rs and _rs[0]:
                             cached_sender_sig = _rs[0]
@@ -1693,6 +1762,22 @@ def setup_email_routes():
             logger.error(f"Failed to mark unread {uid}: {e}")
             return {"success": False, "error": "Mail operation failed"}
 
+    @router.post("/flag/{uid}")
+    async def flag_email(uid: str, folder: str = Query("INBOX"), account_id: str | None = Query(None),
+                         on: bool = Query(True), owner: str = Depends(require_owner)):
+        """Toggle the \\Flagged flag (a.k.a. favorite / star) on an email.
+        Pass `on=true` to favorite, `on=false` to unfavorite."""
+        try:
+            with _imap(account_id, owner=owner) as conn:
+                conn.select(_q(folder))
+                if not _store_email_flag(conn, uid, "\\Flagged", add=bool(on)):
+                    return {"success": False, "error": "Email not found"}
+            _invalidate_list_cache(account_id, folder)
+            return {"success": True, "flagged": bool(on)}
+        except Exception as e:
+            logger.error(f"Failed to flag {uid}: {e}")
+            return {"success": False, "error": "Mail operation failed"}
+
     @router.post("/mark-read/{uid}")
     async def mark_read(uid: str, folder: str = Query("INBOX"), account_id: str | None = Query(None), owner: str = Depends(require_owner)):
         """Mark an email as read (set \\Seen flag)."""
@@ -1708,7 +1793,9 @@ def setup_email_routes():
             return {"success": False, "error": "Mail operation failed"}
 
     @router.post("/archive/{uid}")
-    async def archive_email(uid: str, folder: str = Query("INBOX"), account_id: str | None = Query(None), owner: str = Depends(require_owner)):
+    # Sync def: blocking IMAP I/O with no awaits — see search_emails above. Runs in a
+    # threadpool instead of blocking the event loop.
+    def archive_email(uid: str, folder: str = Query("INBOX"), account_id: str | None = Query(None), owner: str = Depends(require_owner)):
         """Move email to Archive folder."""
         try:
             with _imap(account_id, owner=owner) as conn:
@@ -1940,7 +2027,7 @@ def setup_email_routes():
             outer = MIMEMultipart("alternative")
             body_container = outer
 
-        outer["From"] = cfg["from_address"]
+        outer["From"] = email.utils.formataddr((cfg.get("display_name") or "", cfg["from_address"]))
         outer["To"] = to
         if cc:
             outer["Cc"] = cc
@@ -2071,6 +2158,77 @@ def setup_email_routes():
             logger.error(f"cancel_scheduled {sid!r} failed: {e}")
             return {"success": False, "error": "Mail operation failed"}
 
+    # ── Agent send-confirm: list/approve/cancel ──────────────────────────
+    # When `agent_email_confirm` is on, the MCP send_email tool drops the
+    # composed email into scheduled_emails with status='agent_draft' (a
+    # far-future send_at so the poller never picks it up). These endpoints
+    # let the chat UI surface them for the user and either approve (flip
+    # to status='pending' with send_at=now so the poller delivers it) or
+    # cancel (status='cancelled').
+    @router.get("/pending")
+    async def list_pending_agent_drafts(owner: str = Depends(require_owner)):
+        import sqlite3
+        try:
+            conn = sqlite3.connect(SCHEDULED_DB)
+            conn.row_factory = sqlite3.Row
+            rows = conn.execute(
+                """SELECT id, to_addr, subject, body, created_at, account_id
+                   FROM scheduled_emails
+                   WHERE status = 'agent_draft' AND owner = ?
+                   ORDER BY created_at DESC""",
+                (owner or "",),
+            ).fetchall()
+            conn.close()
+            return {"pending": [dict(r) for r in rows]}
+        except Exception as e:
+            logger.error(f"list_pending_agent_drafts failed: {e}")
+            return {"pending": [], "error": "Mail operation failed"}
+
+    @router.post("/pending/{sid}/approve")
+    async def approve_agent_draft(sid: str, owner: str = Depends(require_owner)):
+        """Approve a draft staged by the agent: flip status → pending and
+        backdate send_at so the scheduled-send poller picks it up
+        immediately."""
+        import sqlite3
+        try:
+            conn = sqlite3.connect(SCHEDULED_DB)
+            cur = conn.execute(
+                """UPDATE scheduled_emails
+                   SET status = 'pending', send_at = ?
+                   WHERE id = ? AND status = 'agent_draft' AND owner = ?""",
+                (datetime.utcnow().isoformat(), sid, owner or ""),
+            )
+            conn.commit()
+            affected = cur.rowcount
+            conn.close()
+            if not affected:
+                return {"success": False, "error": "Draft not found or already handled"}
+            return {"success": True}
+        except Exception as e:
+            logger.error(f"approve_agent_draft {sid!r} failed: {e}")
+            return {"success": False, "error": "Mail operation failed"}
+
+    @router.delete("/pending/{sid}")
+    async def cancel_agent_draft(sid: str, owner: str = Depends(require_owner)):
+        """Discard a draft the agent staged for approval."""
+        import sqlite3
+        try:
+            conn = sqlite3.connect(SCHEDULED_DB)
+            cur = conn.execute(
+                """UPDATE scheduled_emails SET status = 'cancelled'
+                   WHERE id = ? AND status = 'agent_draft' AND owner = ?""",
+                (sid, owner or ""),
+            )
+            conn.commit()
+            affected = cur.rowcount
+            conn.close()
+            if not affected:
+                return {"success": False, "error": "Draft not found or already handled"}
+            return {"success": True}
+        except Exception as e:
+            logger.error(f"cancel_agent_draft {sid!r} failed: {e}")
+            return {"success": False, "error": "Mail operation failed"}
+
     @router.get("/resolve-contact")
     async def resolve_contact(name: str = Query(..., description="Name to search for"), owner: str = Depends(require_owner)):
         """Search Sent folder for a contact by name. Returns matching email addresses."""
@@ -2131,6 +2289,7 @@ def setup_email_routes():
         try:
             cfg = _resolve_send_config(req.account_id, owner=owner)
         except Exception as e:
+            logger.warning(f"No SMTP-capable account resolved: {e}")
             return {"success": False, "error": str(e) or "No SMTP-capable email account configured"}
 
         # Use 'mixed' if we have attachments, 'alternative' otherwise
@@ -2143,7 +2302,7 @@ def setup_email_routes():
             outer = MIMEMultipart("alternative")
             body_container = outer
 
-        outer["From"] = cfg["from_address"]
+        outer["From"] = email.utils.formataddr((cfg.get("display_name") or "", cfg["from_address"]))
         outer["To"] = req.to
         if req.cc:
             outer["Cc"] = req.cc
@@ -2194,6 +2353,10 @@ def setup_email_routes():
 
         _account_id = cfg.get("account_id") or req.account_id  # capture for the IMAP append in the closure
         _in_reply_to = (req.in_reply_to or "").strip()
+        _oauth_provider = cfg.get("oauth_provider") or ""
+        _oauth_access_token = cfg.get("oauth_access_token") or ""
+        _oauth_refresh_token = cfg.get("oauth_refresh_token") or ""
+        _oauth_token_expiry = cfg.get("oauth_token_expiry") or ""
 
         def _deliver():
             try:
@@ -2204,6 +2367,11 @@ def setup_email_routes():
                         "smtp_security": _smtp_security,
                         "smtp_user": _smtp_user,
                         "smtp_password": _smtp_pw,
+                        "account_id": _account_id,
+                        "oauth_provider": _oauth_provider,
+                        "oauth_access_token": _oauth_access_token,
+                        "oauth_refresh_token": _oauth_refresh_token,
+                        "oauth_token_expiry": _oauth_token_expiry,
                     },
                     _from,
                     _recipients,
@@ -2316,7 +2484,7 @@ def setup_email_routes():
             msg.attach(MIMEText(_draft_html, "html", "utf-8"))
         else:
             msg = MIMEText(req.body, "plain", "utf-8")
-        msg["From"] = cfg["from_address"]
+        msg["From"] = email.utils.formataddr((cfg.get("display_name") or "", cfg["from_address"]))
         msg["To"] = req.to
         if req.cc:
             msg["Cc"] = req.cc
@@ -2584,11 +2752,15 @@ def setup_email_routes():
             source_uid = (data.get("uid") or "").strip()
             source_folder = (data.get("folder") or "INBOX").strip()
             fast_reply = bool(data.get("fast", False))
+            user_hint = (data.get("user_hint") or "").strip()
 
             if not original_body:
                 return {"success": False, "error": "No email body provided"}
 
-            if message_id:
+            # Skip cache lookup when the caller supplied a user_hint — the
+            # cached generic reply doesn't reflect the instructions and
+            # would silently override them.
+            if message_id and not user_hint:
                 try:
                     _c = _sql3.connect(SCHEDULED_DB)
                     owner_clause, owner_params = _email_cache_owner_clause(owner)
@@ -2728,8 +2900,13 @@ def setup_email_routes():
             user_msg = (
                 f"Recipient: {to}\nSubject: {subject}\n\n"
                 f"Original email and any current draft:\n{original_body[:6000]}\n\n"
-                f"Draft a reply. Return only the reply body text."
             )
+            if user_hint:
+                user_msg += (
+                    f"User's instructions for THIS reply (follow these — they override "
+                    f"defaults like length/tone):\n{user_hint[:2000]}\n\n"
+                )
+            user_msg += "Draft a reply. Return only the reply body text."
 
             # Build a candidate chain so a stale session-stored API key
             # (the most common cause of "authentication failed" here)
@@ -2959,6 +3136,8 @@ def setup_email_routes():
                     "from_address": r.from_address or "",
                     "has_imap_password": bool(r.imap_password),
                     "has_smtp_password": bool(r.smtp_password),
+                    "oauth_provider": r.oauth_provider or "",
+                    "display_name": r.display_name or "",
                 })
             return {"accounts": out}
         finally:
@@ -2991,6 +3170,7 @@ def setup_email_routes():
                 smtp_user=(data.get("smtp_user") or "").strip(),
                 smtp_password=_enc(data.get("smtp_password") or ""),
                 from_address=(data.get("from_address") or "").strip(),
+                display_name=(data.get("display_name") or "").strip(),
                 # SECURITY: stamp the creator so all subsequent reads / mutations
                 # can filter by user. Without this every new account leaks to
                 # every other user.
@@ -3025,7 +3205,7 @@ def setup_email_routes():
             if not row:
                 return {"ok": False, "error": "Account not found"}
             # Simple fields
-            for key in ("name", "imap_host", "imap_user", "smtp_host", "smtp_user", "from_address"):
+            for key in ("name", "imap_host", "imap_user", "smtp_host", "smtp_user", "from_address", "display_name"):
                 if key in data:
                     setattr(row, key, (data[key] or "").strip())
             for key in ("imap_port", "smtp_port"):
@@ -3214,4 +3394,123 @@ def setup_email_routes():
         finally:
             db.close()
 
+    # ── Google OAuth2 routes ──
+
+    @router.get("/oauth/google/authorize")
+    async def google_oauth_authorize(account_id: str = Query(...), request: Request = None, owner: str = Depends(require_user)):
+        import urllib.parse
+        _assert_owns_account(account_id, owner)
+        client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "")
+        if not client_id:
+            raise HTTPException(400, "GOOGLE_OAUTH_CLIENT_ID not set — add it to .env")
+        redirect_uri = (
+            os.environ.get("GOOGLE_OAUTH_REDIRECT_URI")
+            or f"http://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback"
+        )
+        state = make_oauth_state(account_id, owner)
+        params = urllib.parse.urlencode({
+            "client_id": client_id,
+            "redirect_uri": redirect_uri,
+            "response_type": "code",
+            "scope": "https://mail.google.com/ email",
+            "access_type": "offline",
+            "prompt": "consent",
+            "state": state,
+        })
+        from fastapi.responses import RedirectResponse as _RR
+        return _RR(f"https://accounts.google.com/o/oauth2/v2/auth?{params}")
+
+    @router.get("/oauth/google/callback")
+    async def google_oauth_callback(
+        code: str = Query(None),
+        state: str = Query(None),
+        error: str = Query(None),
+        request: Request = None,
+    ):
+        import urllib.parse
+        from fastapi.responses import RedirectResponse as _RR
+        if error:
+            return _RR("/?section=integrations&email_oauth_error=google_error")
+        if not code or not state:
+            return _RR("/?section=integrations&email_oauth_error=missing_code")
+        state_data = verify_oauth_state(state)
+        if not state_data:
+            return _RR("/?section=integrations&email_oauth_error=invalid_state")
+        account_id = state_data.get("a", "")
+        owner = state_data.get("o", "")
+        client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "")
+        client_secret = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET", "")
+        redirect_uri = (
+            os.environ.get("GOOGLE_OAUTH_REDIRECT_URI")
+            or f"http://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback"
+        )
+        import httpx as _httpx
+        try:
+            resp = _httpx.post("https://oauth2.googleapis.com/token", data={
+                "code": code,
+                "client_id": client_id,
+                "client_secret": client_secret,
+                "redirect_uri": redirect_uri,
+                "grant_type": "authorization_code",
+            }, timeout=10)
+            resp.raise_for_status()
+            data = resp.json()
+        except Exception:
+            logger.warning("Google token exchange failed")
+            return _RR("/?section=integrations&email_oauth_error=token_exchange_failed")
+        access_token = data.get("access_token", "")
+        refresh_token = data.get("refresh_token", "")
+        expiry = str(int(time.time()) + data.get("expires_in", 3600))
+        # Fetch the email address from userinfo so we can auto-fill imap_user.
+        email_addr = ""
+        display_name = ""
+        try:
+            ui = _httpx.get("https://www.googleapis.com/oauth2/v1/userinfo",
+                            headers={"Authorization": f"Bearer {access_token}"}, timeout=10)
+            if ui.is_success:
+                ui_data = ui.json()
+                email_addr = ui_data.get("email", "")
+                display_name = ui_data.get("name", "")
+        except Exception:
+            pass
+        from core.database import SessionLocal, EmailAccount
+        from src.secret_storage import encrypt as _enc
+        db = SessionLocal()
+        try:
+            row = db.query(EmailAccount).filter(EmailAccount.id == account_id).first()
+            if not row:
+                return _RR("/?section=integrations&email_oauth_error=account_not_found")
+            # SECURITY: verify the account belongs to the initiating user.
+            if owner and row.owner and row.owner != owner:
+                logger.warning("OAuth callback owner mismatch — rejecting token write")
+                return _RR("/?section=integrations&email_oauth_error=ownership_error")
+            row.oauth_provider = "google"
+            row.oauth_access_token = _enc(access_token)
+            if refresh_token:
+                row.oauth_refresh_token = _enc(refresh_token)
+            row.oauth_token_expiry = expiry
+            # Auto-fill Google IMAP/SMTP settings if not already configured.
+            if not row.imap_host:
+                row.imap_host = "imap.gmail.com"
+                row.imap_port = 993
+                row.imap_starttls = False
+            if not row.smtp_host:
+                row.smtp_host = "smtp.gmail.com"
+                row.smtp_port = 587
+            if email_addr:
+                if not row.imap_user:
+                    row.imap_user = email_addr
+                if not row.smtp_user:
+                    row.smtp_user = email_addr
+                if not row.from_address:
+                    row.from_address = email_addr
+                if not row.name or row.name == row.id:
+                    row.name = email_addr
+            if display_name and not row.display_name:
+                row.display_name = display_name
+            db.commit()
+        finally:
+            db.close()
+        return _RR("/?section=integrations&email_oauth_success=1")
+
     return router

routes/embedding_routes.py

@@ -9,6 +9,7 @@ from pathlib import Path
 from fastapi import APIRouter, HTTPException, Form, Depends
 from core.constants import EMBEDDING_ENDPOINT_FILE, FASTEMBED_CACHE_DIR
 from core.middleware import require_admin
+from src.runtime_paths import get_app_root
 
 logger = logging.getLogger(__name__)
 

routes/gallery_routes.py

@@ -19,6 +19,7 @@ from src.upload_limits import (
     GALLERY_TRANSFORM_UPLOAD_MAX_BYTES,
 )
 from src.constants import GENERATED_IMAGES_DIR
+from src.optional_deps import patch_realesrgan_torchvision_compat
 
 from routes.gallery_helpers import (
     GalleryPatch, _extract_exif, _image_to_dict, _owner_filter, _human_size,
@@ -108,6 +109,32 @@ def _visible_image_endpoint_for_base(db, base: str, owner: str | None):
     return fallback
 
 
+async def _fetch_result_image_b64(url: str) -> Optional[str]:
+    """Fetch an image URL returned in an upstream response body, base64-encoded
+    (or None on a non-200).
+
+    The URL comes from the diffusion/OpenAI server's response, not from our own
+    config, so a malicious or compromised endpoint could otherwise steer this
+    fetch at an internal or cloud-metadata address. Validate it the same way the
+    client-supplied endpoint is validated before the first request.
+    """
+    import base64
+    import httpx
+    from src.url_safety import check_outbound_url
+
+    ok, reason = check_outbound_url(
+        url,
+        block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true",
+    )
+    if not ok:
+        raise HTTPException(502, f"Upstream returned an unsafe image URL: {reason}")
+    async with httpx.AsyncClient(timeout=60) as c2:
+        ir = await c2.get(url)
+        if ir.status_code == 200:
+            return base64.b64encode(ir.content).decode()
+    return None
+
+
 def setup_gallery_routes() -> APIRouter:
     router = APIRouter(tags=["gallery"])
 
@@ -197,8 +224,6 @@ def setup_gallery_routes() -> APIRouter:
     @router.post("/api/gallery/{image_id}/replace")
     async def gallery_replace(request: Request, image_id: str):
         """Replace an existing gallery image file with a new one."""
-        from pathlib import Path
-
         user = get_current_user(request)
         db = SessionLocal()
         try:
@@ -214,9 +239,8 @@ def setup_gallery_routes() -> APIRouter:
                 raise HTTPException(400, "No image provided")
 
             content = await read_upload_limited(file, GALLERY_UPLOAD_MAX_BYTES, "Gallery replacement")
-            img_dir = Path(GENERATED_IMAGES_DIR)
-            img_dir.mkdir(parents=True, exist_ok=True)
-            img_path = img_dir / _sanitize_gallery_filename(img.filename)
+            GALLERY_IMAGE_DIR.mkdir(parents=True, exist_ok=True)
+            img_path = _gallery_image_path(img.filename)
             img_path.write_bytes(content)
 
             # Refresh dimensions in case the editor resized the canvas.
@@ -904,15 +928,23 @@ def setup_gallery_routes() -> APIRouter:
                 raise HTTPException(404, "Image not found")
 
             img_filename = img.filename
-            # Remove the file from disk
-            img_path = _gallery_image_path(img_filename)
-            if img_path.exists():
-                img_path.unlink()
-
-            # Soft-delete the record
+            # Soft-delete the record first; the DB is the source of truth.
             img.is_active = False
             db.commit()
 
+            # Only after the soft-delete commit succeeds do we remove the file.
+            # If the file were deleted first and the commit then failed/rolled
+            # back, the still-active record would point at a missing file.
+            # Best-effort so a missing or locked file can't 500 a delete that
+            # already succeeded logically. Uses the path-confined resolver so a
+            # malformed stored filename can't escape generated_images.
+            try:
+                img_path = _gallery_image_path(img_filename)
+                if img_path.exists():
+                    img_path.unlink()
+            except Exception as e:
+                logger.warning(f"Could not remove gallery image file for {img_filename}: {e}")
+
             # Strip stale chat-history references so the image bubble
             # (and its prompt caption) doesn't come back after a server
             # reboot replays the session. We remove the matching tool
@@ -1142,10 +1174,7 @@ def setup_gallery_routes() -> APIRouter:
                         if item.get("b64_json"):
                             raw_b64 = item["b64_json"]
                         elif item.get("url"):
-                            async with httpx.AsyncClient(timeout=60) as c2:
-                                img_r = await c2.get(item["url"])
-                                if img_r.status_code == 200:
-                                    raw_b64 = base64.b64encode(img_r.content).decode()
+                            raw_b64 = await _fetch_result_image_b64(item["url"])
                     if not raw_b64:
                         raise HTTPException(502, "OpenAI returned no image")
 
@@ -1206,7 +1235,7 @@ def setup_gallery_routes() -> APIRouter:
         original and regenerates `strength` fraction. With strength ~0.4
         you get edge blending + lighting unification while keeping the
         composition recognisable."""
-        import httpx, base64 as _b64
+        import httpx
         user = require_privilege(request, "can_generate_images")
         body = await request.json()
 
@@ -1382,10 +1411,9 @@ def setup_gallery_routes() -> APIRouter:
                             if item.get("b64_json"):
                                 return {"image": item["b64_json"]}
                             if item.get("url"):
-                                async with httpx.AsyncClient(timeout=60) as c2:
-                                    ir = await c2.get(item["url"])
-                                    if ir.status_code == 200:
-                                        return {"image": _b64.b64encode(ir.content).decode()}
+                                img_b64 = await _fetch_result_image_b64(item["url"])
+                                if img_b64:
+                                    return {"image": img_b64}
                     last_err = f"{path}: server returned no image"
                 except httpx.ConnectError as e:
                     raise HTTPException(502, f"Can't reach diffusion server at {base}: {e}")
@@ -1445,6 +1473,7 @@ def setup_gallery_routes() -> APIRouter:
         img_bytes = base64.b64decode(image_b64)
         src = Image.open(io.BytesIO(img_bytes)).convert("RGB")
         try:
+            patch_realesrgan_torchvision_compat()
             from realesrgan import RealESRGANer
         except ImportError:
             return {"error": "realesrgan not installed. Install it from Cookbook → Dependencies (search 'realesrgan')."}
@@ -1494,6 +1523,7 @@ def setup_gallery_routes() -> APIRouter:
         img_bytes = base64.b64decode(image_b64)
         src = Image.open(io.BytesIO(img_bytes)).convert("RGB")
         try:
+            patch_realesrgan_torchvision_compat()
             from basicsr.archs.rrdbnet_arch import RRDBNet
             from realesrgan import RealESRGANer
         except ImportError:

routes/hwfit_routes.py

@@ -119,7 +119,7 @@ def setup_hwfit_routes():
         return detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh)
 
     @router.get("/models")
-    def get_models(use_case: str = "", sort: str = "score", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
+    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
         """Rank LLM models against detected hardware and return scored results.
         gpu_count: override GPU count (0 = CPU only, 1-N = simulate N GPUs of the
             active group). gpu_group: index into system.gpu_groups (the homogeneous

routes/mcp_routes.py

@@ -108,6 +108,12 @@ def _load_disabled_map():
         db.close()
 
 
+def _mcp_oauth_redirect_uri() -> str:
+    """Shared callback URL for legacy Google and generic MCP OAuth flows."""
+    from src.mcp_oauth import REDIRECT_URI
+    return REDIRECT_URI
+
+
 def setup_mcp_routes(mcp_manager: McpManager):
     """Setup MCP routes with the provided manager."""
 
@@ -445,9 +451,9 @@ def setup_mcp_routes(mcp_manager: McpManager):
             client_id = keys["client_id"]
             scopes = oauth_cfg.get("scopes", [])
 
-            # For Desktop App creds, redirect to localhost — the user will
+            # For Desktop App creds, default to localhost — the user will
             # paste the resulting URL back if they're on a different device.
-            redirect_uri = "http://localhost:7000/api/mcp/oauth/callback"
+            redirect_uri = _mcp_oauth_redirect_uri()
 
             params = {
                 "client_id": client_id,
@@ -469,7 +475,7 @@ def setup_mcp_routes(mcp_manager: McpManager):
                 return RedirectResponse(auth_url)
             else:
                 # Remote device — show paste-back page
-                return HTMLResponse(_oauth_authorize_page(auth_url, server_id, host))
+                return HTMLResponse(_oauth_authorize_page(auth_url, server_id, host, redirect_uri))
         finally:
             db.close()
 
@@ -536,7 +542,7 @@ def setup_mcp_routes(mcp_manager: McpManager):
             client_id = keys["client_id"]
             client_secret = keys["client_secret"]
 
-            redirect_uri = "http://localhost:7000/api/mcp/oauth/callback"
+            redirect_uri = _mcp_oauth_redirect_uri()
 
             async with httpx.AsyncClient() as client:
                 resp = await client.post(
@@ -603,13 +609,19 @@ def setup_mcp_routes(mcp_manager: McpManager):
     return router
 
 
-def _oauth_authorize_page(auth_url: str, server_id: str, host: str) -> str:
+def _oauth_authorize_page(
+    auth_url: str,
+    server_id: str,
+    host: str,
+    redirect_uri: str = "http://localhost:7000/api/mcp/oauth/callback",
+) -> str:
     """Page with Google sign-in link and URL paste-back form for remote access."""
     # Escape values interpolated into the page: `host` comes from the request
     # Host header and `server_id` from the OAuth state — neither is trusted.
     auth_url = html.escape(auth_url, quote=True)
     server_id = html.escape(server_id, quote=True)
     host = html.escape(host, quote=True)
+    redirect_uri = html.escape(redirect_uri, quote=True)
     return f"""<!DOCTYPE html>
 <html><head>
 <meta charset="UTF-8"><title>Authorize — Odysseus</title>
@@ -654,7 +666,7 @@ def _oauth_authorize_page(auth_url: str, server_id: str, host: str) -> str:
   <div class="divider"></div>
   <form method="POST" action="http://{host}/api/mcp/oauth/exchange/{server_id}">
     <p>Paste the URL from your browser after signing in:</p>
-    <input type="text" name="callback_url" placeholder="http://localhost:7000/api/mcp/oauth/callback?code=..." required>
+    <input type="text" name="callback_url" placeholder="{redirect_uri}?code=..." required>
     <br><button type="submit">Connect</button>
   </form>
 </div></body></html>"""

routes/memory_routes.py

@@ -29,6 +29,7 @@ from src.llm_core import llm_call_async
 from services.memory.memory_extractor import audit_memories
 from src.auth_helpers import get_current_user, require_user
 from src.endpoint_resolver import resolve_endpoint
+from src.task_endpoint import resolve_task_endpoint
 from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
 
 logger = logging.getLogger(__name__)
@@ -105,6 +106,13 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
         if memory_manager.find_duplicates(text, user_mem):
             return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
 
+        if memory_data.session_id:
+            try:
+                session_obj = session_manager.get_session(memory_data.session_id)
+            except KeyError:
+                raise HTTPException(404, "Session not found")
+            _assert_session_owner(session_obj, user)
+
         new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
         if memory_data.session_id:
             new_entry["session_id"] = memory_data.session_id
@@ -163,8 +171,17 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
 
             session_id = memory.get("session_id")
             if session_id and session_id in session_manager.sessions:
-                session = session_manager.get_session(session_id)
-                memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
+                try:
+                    session = session_manager.get_session(session_id)
+                    if session:
+                        _assert_session_owner(session, user)
+                    memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
+                except KeyError:
+                    memory["session_name"] = "Unknown"
+                except HTTPException as exc:
+                    if exc.status_code != 404:
+                        raise
+                    memory["session_name"] = "Unknown"
             else:
                 memory["session_name"] = "Unknown"
 
@@ -224,14 +241,18 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
         }
         messages = [system_msg] + sess.get_context_messages()
 
+        t_url, t_model, t_headers = resolve_task_endpoint(
+            sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
+        )
+
         try:
             suggestion_text = await llm_call_async(
-                sess.endpoint_url,
-                sess.model,
+                t_url,
+                t_model,
                 messages,
                 temperature=0.2,
                 max_tokens=500,
-                headers=sess.headers,
+                headers=t_headers,
             )
             try:
                 suggestions = json.loads(suggestion_text)
@@ -252,57 +273,30 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
     async def api_audit_memories(request: Request, session: str = Form(None)):
         """Deduplicate and consolidate memories via LLM.
 
-        Uses the default model from settings, or falls back to a session's model.
+        Uses task/utility/default settings through the shared resolver, with
+        the active session as fallback when no task or utility model is set.
         Returns before and after memory counts.
         """
-        from routes.model_routes import _load_settings, _normalize_base, build_chat_url
-        from core.database import ModelEndpoint
-        import json as _json
-
-        endpoint_url = model = None
-        headers = {}
-
-        # Try default model from settings first
-        settings = _load_settings()
-        ep_id = settings.get("default_endpoint_id", "")
-        default_model = settings.get("default_model", "")
-        if ep_id:
-            db = SessionLocal()
-            try:
-                ep = db.query(ModelEndpoint).filter(
-                    ModelEndpoint.id == ep_id, ModelEndpoint.is_enabled == True
-                ).first()
-                if ep:
-                    base = _normalize_base(ep.base_url)
-                    endpoint_url = build_chat_url(base)
-                    model = default_model
-                    if not model and ep.models:
-                        try:
-                            models = _json.loads(ep.models) if isinstance(ep.models, str) else ep.models
-                            if models:
-                                model = models[0]
-                        except Exception:
-                            pass
-                    if ep.api_key:
-                        headers = {"Authorization": f"Bearer {ep.api_key}"}
-            finally:
-                db.close()
-
-        # Fall back to session model if no default configured
-        if not endpoint_url and session:
+        user = _owner(request)
+        fallback_url = fallback_model = None
+        fallback_headers = None
+        if session:
             try:
                 sess = session_manager.get_session(session)
-                _assert_session_owner(sess, _owner(request))
-                endpoint_url = sess.endpoint_url
-                model = sess.model
-                headers = sess.headers
+                _assert_session_owner(sess, user)
+                fallback_url = sess.endpoint_url
+                fallback_model = sess.model
+                fallback_headers = sess.headers
             except KeyError:
                 pass
 
+        endpoint_url, model, headers = resolve_task_endpoint(
+            fallback_url, fallback_model, fallback_headers, owner=user
+        )
+
         if not endpoint_url or not model:
             raise HTTPException(400, "No default model configured — set one in Settings")
 
-        user = _owner(request)
         result = await audit_memories(
             memory_manager,
             memory_vector,
@@ -340,17 +334,28 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
         model = None
         headers = {}
 
+        user = _owner(request)
+
         if session:
             try:
                 sess = session_manager.get_session(session)
-                _assert_session_owner(sess, _owner(request))
-                endpoint_url = sess.endpoint_url
-                model = sess.model
-                headers = sess.headers
+                _assert_session_owner(sess, user)
             except KeyError:
-                 raise HTTPException(404, "Session not found — needed for LLM config")
+                sess = None
+            except HTTPException as exc:
+                if exc.status_code != 404:
+                    raise
+                sess = None
+
+            if sess is None:
+                logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
+                endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
+            else:
+                endpoint_url, model, headers = resolve_task_endpoint(
+                    sess.endpoint_url, sess.model, sess.headers, owner=user
+                )
         else:
-            endpoint_url, model, headers = resolve_endpoint("utility", owner=_owner(request))
+            endpoint_url, model, headers = resolve_task_endpoint(owner=user)
     
         if not endpoint_url or not model:
             raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")

routes/model_routes.py

@@ -5,6 +5,7 @@ import re
 import uuid
 import json
 import hashlib
+import ipaddress
 import socket
 import time as _time
 import logging
@@ -26,7 +27,7 @@ from src.endpoint_resolver import (
     build_models_url,
     build_headers,
 )
-from src.auth_helpers import _auth_disabled, owner_filter
+from src.auth_helpers import _auth_disabled, effective_user, owner_filter
 
 logger = logging.getLogger(__name__)
 
@@ -123,6 +124,21 @@ def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
     return cleared_users
 
 
+def _default_endpoint_needs_assignment(current_default_id: str, enabled_endpoint_ids) -> bool:
+    """Whether the global default chat endpoint should be (re)assigned.
+
+    True when nothing is configured yet, or the configured default no longer
+    resolves to an enabled endpoint (e.g. the user disabled it). Without the
+    second case, adding a new endpoint after disabling the previous default
+    leaves `default_endpoint_id` pointing at the disabled endpoint, so features
+    that read the raw setting (Memory → Tidy) fail with "No default model
+    configured" even though an enabled endpoint exists. See #3586.
+    """
+    if not current_default_id:
+        return True
+    return current_default_id not in enabled_endpoint_ids
+
+
 # Loopback hosts a user might type for a local model server (LM Studio,
 # llama.cpp, vLLM, …). Inside Docker these point at the *container*, not the
 # host the server actually runs on.
@@ -233,6 +249,9 @@ _PROVIDER_CURATED = {
     "zai-coding": [
         "glm-5.1", "glm-5v-turbo", "glm-5-turbo", "glm-4.7", "glm-4.5-air",
     ],
+    "kimi-code": [
+        "kimi-for-coding",
+    ],
     "deepseek": [
         "deepseek-chat", "deepseek-reasoner",
     ],
@@ -300,6 +319,8 @@ def _match_provider_curated(base_url: str, provider: str) -> str:
     parsed = urlparse(base_url)
     if _host_match(base_url, "z.ai") and "/api/coding" in (parsed.path or ""):
         return "zai-coding"
+    if _host_match(base_url, "kimi.com") and "/coding" in (parsed.path or ""):
+        return "kimi-code"
     for domain, key in _HOST_TO_CURATED:
         if _host_match(base_url, domain):
             return key
@@ -542,6 +563,8 @@ def _safe_build_models_url(base_url: str) -> str:
     """Build a /models URL without letting optional provider imports break probes."""
     try:
         return build_models_url(base_url)
+    except ValueError:
+        raise
     except Exception as exc:
         logger.debug("Model URL detection failed for %s: %s", base_url, exc)
         return f"{(base_url or '').rstrip('/')}/models"
@@ -613,7 +636,7 @@ def _probe_single_model(base: str, api_key: str, model_id: str, timeout: int = 1
 
     try:
         t0 = _time.time()
-        r = httpx.post(target_url, headers=h, json=payload, timeout=timeout)
+        r = httpx.post(target_url, headers=h, json=payload, timeout=timeout, verify=llm_verify())
         latency = round((_time.time() - t0) * 1000)
         if r.is_success:
             return {"status": "ok", "latency_ms": latency}
@@ -639,13 +662,20 @@ def _probe_single_model(base: str, api_key: str, model_id: str, timeout: int = 1
 
 # Hostnames / IP prefixes that indicate a local endpoint
 _LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"}
-_PRIVATE_PREFIXES = ("10.", "172.16.", "172.17.", "172.18.", "172.19.",
-                     "172.20.", "172.21.", "172.22.", "172.23.", "172.24.",
-                     "172.25.", "172.26.", "172.27.", "172.28.", "172.29.",
-                     "172.30.", "172.31.", "192.168.")
+_PRIVATE_NETWORKS = (
+    ipaddress.ip_network("10.0.0.0/8"),
+    ipaddress.ip_network("172.16.0.0/12"),
+    ipaddress.ip_network("192.168.0.0/16"),
+)
+_TAILSCALE_CGNAT = ipaddress.ip_network("100.64.0.0/10")
 
 
-_TAILSCALE_RE = re.compile(r"^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\.")
+def _local_ip_literal(host: str) -> bool:
+    try:
+        ip = ipaddress.ip_address(host)
+    except ValueError:
+        return False
+    return any(ip in network for network in _PRIVATE_NETWORKS) or ip in _TAILSCALE_CGNAT
 
 
 def _classify_endpoint(base_url: str, endpoint_kind: str = "auto") -> str:
@@ -659,9 +689,7 @@ def _classify_endpoint(base_url: str, endpoint_kind: str = "auto") -> str:
         return "api"
     try:
         host = urlparse(base_url).hostname or ""
-        if host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES):
-            return "local"
-        if _TAILSCALE_RE.match(host):
+        if host in _LOCAL_HOSTS or _local_ip_literal(host):
             return "local"
     except Exception:
         pass
@@ -688,6 +716,7 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
     """Probe a base URL's /models endpoint and return list of model IDs.
     For Anthropic, queries their /v1/models API, falling back to hardcoded list."""
     from src.endpoint_resolver import resolve_url
+    from src.llm_core import httpx_get_kimi_aware
     base = resolve_url(_normalize_base(base_url))
     provider = _safe_detect_provider(base)
     if provider == "chatgpt-subscription":
@@ -723,7 +752,7 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
     url = _safe_build_models_url(base)
     headers = _safe_build_headers(api_key, base)
     try:
-        r = httpx.get(url, headers=headers, timeout=timeout, verify=llm_verify())
+        r = httpx_get_kimi_aware(url, headers, timeout=timeout, verify=llm_verify())
         r.raise_for_status()
         data = r.json()
         # OpenAI format: {"data": [{"id": "model-name"}]}
@@ -739,6 +768,11 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
                 for _e in _PROVIDER_CURATED.get(_ck, []):
                     if _e not in set(models) and not any(m.startswith(_e) for m in models):
                         models.append(_e)
+            if _host_match(base, "kimi.com") and "/coding" in (urlparse(base).path or ""):
+                _ck = _match_provider_curated(base, None)
+                for _e in _PROVIDER_CURATED.get(_ck, []):
+                    if _e not in set(models) and not any(m.startswith(_e) for m in models):
+                        models.append(_e)
             return [m for m in models if _is_chat_model(m)]
     except httpx.HTTPStatusError as e:
         if api_key:
@@ -855,15 +889,52 @@ def _ping_endpoint(base_url: str, api_key: str = None, timeout: float = 1.5) ->
 
 
 def _model_endpoint_error_message(base_url: str, ping: Dict[str, Any] = None) -> str:
-    """Return a provider-aware error message for failed endpoint probes."""
+    """Return a provider-aware error message for failed endpoint probes.
+
+    Surfaces the URL we actually probed and, when the endpoint looks like
+    LM Studio (port 1234 or hostname match), adds a hint about loading a
+    model and confirming the Developer Server is running. The user previously
+    saw a generic "No models found for that provider/key" with no way to
+    tell whether the URL was wrong, the server was down, or the server was
+    reachable but had no model loaded (issue #25).
+    """
     ping = ping or {}
     error = ping.get("error")
+    from src.endpoint_resolver import build_models_url
+    try:
+        probed = build_models_url(base_url) or base_url
+    except Exception:
+        probed = base_url
     parsed = urlparse(base_url)
     host = (parsed.hostname or "").lower()
     is_ollama = parsed.port == 11434 or "ollama" in host or "ollama" in base_url.lower()
+    is_lmstudio = (
+        parsed.port == 1234
+        or "lmstudio" in host
+        or "lm-studio" in host
+        or "lm_studio" in host
+    )
+
+    if is_lmstudio:
+        parts = [
+            "LM Studio is reachable, but no models were reported.",
+            f"Probed {probed}.",
+        ]
+        if error:
+            parts.append(f"Last probe error: {error}.")
+        parts.append(
+            "Open LM Studio, load at least one model, and confirm the "
+            "Developer Server is running on port 1234."
+        )
+        parts.append(
+            "Base URL should be http://localhost:1234/v1 (native) or "
+            "http://host.docker.internal:1234/v1 (Docker)."
+        )
+        return " ".join(parts)
 
     if is_ollama:
         parts = ["No Ollama models found for that endpoint."]
+        parts.append(f"Probed {probed}.")
         if error:
             parts.append(f"Last probe error: {error}.")
         parts.append("Check that Ollama is running and that the base URL is correct.")
@@ -873,9 +944,9 @@ def _model_endpoint_error_message(base_url: str, ping: Dict[str, Any] = None) ->
         return " ".join(parts)
 
     if error:
-        return f"No models found for that provider/key. Last probe error: {error}."
+        return f"No models found for that provider/key. Probed {probed}. Last probe error: {error}."
 
-    return "No models found for that provider/key."
+    return f"No models found for that provider/key. Probed {probed}."
 
 
 def _normalize_model_ids(value):
@@ -1192,13 +1263,16 @@ def setup_model_routes(model_discovery):
         # Require auth; "" is the unconfigured single-user mode, treated as
         # "see everything" by _fetch_models.
         try:
-            from src.auth_helpers import get_current_user as _gcu
-            owner = _gcu(request) or ""
-        except Exception:
-            owner = ""
-        # Reject anonymous in configured deployments — no leaking the model
-        # list to unauthenticated callers.
-        try:
+            if getattr(request.state, "api_token", False):
+                scopes = set(getattr(request.state, "api_token_scopes", []) or [])
+                if "chat" not in scopes:
+                    raise HTTPException(403, "API token is not scoped for chat")
+                if not getattr(request.state, "api_token_owner", None):
+                    raise HTTPException(403, "API token has no owner")
+            owner = effective_user(request) or ""
+
+            # Reject anonymous in configured deployments — no leaking the model
+            # list to unauthenticated callers.
             auth_mgr = getattr(request.app.state, "auth_manager", None)
             if not owner and not _auth_disabled() and auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
                 raise HTTPException(401, "Not authenticated")
@@ -1727,12 +1801,19 @@ def setup_model_routes(model_discovery):
             )
             db.add(ep)
             db.commit()
-            # Auto-set as default chat endpoint if none configured yet. Seed
-            # the first CHAT model (not raw model_ids[0]) so we don't pin the
-            # global default to an embedding/tts/etc. entry a provider happens
-            # to list first.
+            # Auto-set as default chat endpoint when none is usable yet — either
+            # nothing is configured, or the configured default points at an
+            # endpoint that is now missing/disabled (#3586). Seed the first CHAT
+            # model (not raw model_ids[0]) so we don't pin the global default to
+            # an embedding/tts/etc. entry a provider happens to list first.
             settings = _load_settings()
-            if not settings.get("default_endpoint_id"):
+            enabled_ids = {
+                e.id
+                for e in db.query(ModelEndpoint).filter(
+                    ModelEndpoint.is_enabled == True  # noqa: E712
+                ).all()
+            }
+            if _default_endpoint_needs_assignment(settings.get("default_endpoint_id") or "", enabled_ids):
                 from src.endpoint_resolver import _first_chat_model
                 settings["default_endpoint_id"] = ep.id
                 settings["default_model"] = _first_chat_model(model_ids) or ""

routes/note_routes.py

@@ -10,7 +10,7 @@ from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
 
 from core.database import SessionLocal, Note
-from src.auth_helpers import get_current_user
+from src.auth_helpers import require_user
 from src.constants import DATA_DIR
 from sqlalchemy.orm.attributes import flag_modified
 
@@ -208,14 +208,17 @@ async def dispatch_reminder(
         try:
             from src.endpoint_resolver import resolve_endpoint
             from src.llm_core import llm_call_async
+            from src.reminder_personas import synthesis_system_prompt
             url, model, headers = resolve_endpoint("utility", owner=owner or None)
             if not url:
                 url, model, headers = resolve_endpoint("default", owner=owner or None)
             if url and model:
+                persona_id = (settings.get("reminder_llm_persona") or "").strip()
+                sys_prompt = synthesis_system_prompt(persona_id)
                 raw = await llm_call_async(
                     url=url, model=model,
                     messages=[
-                        {"role": "system", "content": "You are a reminder assistant. Write a single short, warm, motivating sentence (max 25 words) reminding the user about the note below. Do not add greetings, preamble, or hashtags. Output only the sentence."},
+                        {"role": "system", "content": sys_prompt},
                         {"role": "user", "content": f"Title: {title}\n\n{note_body}".strip()},
                     ],
                     temperature=0.7, max_tokens=200, headers=headers, timeout=30,
@@ -567,7 +570,16 @@ def setup_note_routes(task_scheduler=None):
     router = APIRouter(prefix="/api/notes", tags=["notes"])
 
     def _owner(request: Request) -> Optional[str]:
-        return get_current_user(request)
+        # require_user, not bare get_current_user: a request that reaches
+        # these owner-scoped routes with NO identity (auth-middleware
+        # regression, SSRF from a sibling service) must fail closed (401)
+        # when auth is configured — not be treated as the single-user mode
+        # and handed blanket access to every account's notes. The documented
+        # anonymous modes (AUTH_ENABLED=false, LOCALHOST_BYPASS on loopback,
+        # unconfigured first-run) still resolve to None, the single-user
+        # path. fire_reminder below already gated this way; the CRUD routes
+        # did not.
+        return require_user(request) or None
 
     def _is_admin_or_single_user(request: Request, user: str | None) -> bool:
         if user == "internal-tool":
@@ -802,8 +814,7 @@ def setup_note_routes(task_scheduler=None):
         Returns {synthesis, email_sent}.
         """
         # Gate against anonymous callers — LLM synthesis can burn tokens.
-        from src.auth_helpers import require_user as _ru
-        user = _ru(request)
+        user = require_user(request)
         body = await request.json()
         note_id = str(body.get("note_id") or "").strip()
         if not note_id:
@@ -826,6 +837,12 @@ def setup_note_routes(task_scheduler=None):
                 _override["reminder_webhook_integration_id"] = body["webhook_integration_id"]
             if body.get("webhook_payload_template"):
                 _override["reminder_webhook_payload_template"] = body["webhook_payload_template"]
+            # Mirror the in-UI AI Synthesis toggle + persona so the test
+            # actually exercises the synthesis path before/without a Save.
+            if "llm_synthesis" in body:
+                _override["reminder_llm_synthesis"] = bool(body["llm_synthesis"])
+            if "llm_persona" in body:
+                _override["reminder_llm_persona"] = str(body["llm_persona"] or "")
         else:
             db = SessionLocal()
             try:

routes/personal_routes.py

@@ -2,8 +2,9 @@
 """Routes for personal documents management."""
 import os
 import logging
+import shutil
 import uuid
-from typing import List, Tuple
+from typing import Any, Dict, List, Tuple
 from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Depends
 from src.request_models import DirectoryRequest
 from core.constants import BASE_DIR, PERSONAL_DIR, PERSONAL_UPLOADS_DIR
@@ -18,14 +19,15 @@ UPLOADS_DIR = PERSONAL_UPLOADS_DIR
 logger = logging.getLogger(__name__)
 
 
-def _personal_upload_dir_for_owner(owner: str | None) -> str:
+def _personal_upload_dir_for_owner(owner: str | None, *, create: bool = True) -> str:
     """Return the per-owner upload directory used for direct RAG uploads."""
     owner_segment = secure_filename((owner or "local").strip())[:80] or "local"
     upload_dir = os.path.abspath(os.path.join(UPLOADS_DIR, owner_segment))
     base_abs = os.path.abspath(UPLOADS_DIR)
     if os.path.commonpath([upload_dir, base_abs]) != base_abs:
         raise ValueError("Unsafe upload owner path")
-    os.makedirs(upload_dir, exist_ok=True)
+    if create:
+        os.makedirs(upload_dir, exist_ok=True)
     return upload_dir
 
 
@@ -44,6 +46,87 @@ def _unique_personal_upload_path(upload_dir: str, original_name: str | None) ->
         raise ValueError("Unsafe upload filename")
     return file_path, filename, safe_name
 
+
+def _unique_existing_target(path: str) -> str:
+    """Return a non-existing sibling path for rename collision handling."""
+    if not os.path.exists(path):
+        return path
+    stem, ext = os.path.splitext(path)
+    while True:
+        candidate = f"{stem}-{uuid.uuid4().hex[:10]}{ext}"
+        if not os.path.exists(candidate):
+            return candidate
+
+
+def _remove_empty_tree(path: str) -> None:
+    """Best-effort removal of empty directories under ``path``."""
+    if not os.path.isdir(path):
+        return
+    for root, dirs, _files in os.walk(path, topdown=False):
+        for dirname in dirs:
+            candidate = os.path.join(root, dirname)
+            try:
+                os.rmdir(candidate)
+            except OSError:
+                pass
+    try:
+        os.rmdir(path)
+    except OSError:
+        pass
+
+
+def rename_personal_upload_owner(
+    old_owner: str,
+    new_owner: str,
+    *,
+    personal_docs_manager: Any = None,
+    rag_manager: Any = None,
+) -> Dict[str, Any]:
+    """Move direct personal uploads and rewrite RAG owner metadata on user rename."""
+    old_dir = _personal_upload_dir_for_owner(old_owner, create=False)
+    new_dir = _personal_upload_dir_for_owner(new_owner, create=False)
+    path_map: Dict[str, str] = {}
+    moved_files = 0
+
+    if os.path.isdir(old_dir) and old_dir != new_dir:
+        os.makedirs(new_dir, exist_ok=True)
+        for root, _dirs, files in os.walk(old_dir):
+            rel_root = os.path.relpath(root, old_dir)
+            target_root = new_dir if rel_root == "." else os.path.join(new_dir, rel_root)
+            os.makedirs(target_root, exist_ok=True)
+            for filename in files:
+                source = os.path.abspath(os.path.join(root, filename))
+                target = _unique_existing_target(os.path.abspath(os.path.join(target_root, filename)))
+                shutil.move(source, target)
+                path_map[source] = target
+                moved_files += 1
+        _remove_empty_tree(old_dir)
+
+    if personal_docs_manager is not None:
+        rename_directory = getattr(personal_docs_manager, "rename_directory", None)
+        if callable(rename_directory):
+            rename_directory(old_dir, new_dir, path_map=path_map)
+
+    rag_result = None
+    if rag_manager is not None:
+        rename_owner = getattr(rag_manager, "rename_owner", None)
+        if callable(rename_owner):
+            rag_result = rename_owner(
+                old_owner,
+                new_owner,
+                path_map=path_map,
+                path_prefixes=[(old_dir, new_dir)],
+            )
+
+    return {
+        "old_dir": old_dir,
+        "new_dir": new_dir,
+        "moved_files": moved_files,
+        "path_map": path_map,
+        "rag_result": rag_result,
+    }
+
+
 def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
     """
     Setup personal documents related routes.
@@ -160,8 +243,11 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
             JSON response confirming removal
         """
         try:
-            if not directory:
-                raise HTTPException(400, "Directory path is required")
+            # Confine to PERSONAL_DIR — parity with add_directory_to_rag (which
+            # resolves the path the same way). Without this, an arbitrary or
+            # `..`-escaping path is passed straight to
+            # personal_docs_manager.remove_directory / rag.remove_directory.
+            directory = _resolve_allowed_personal_dir(directory)
 
             logger.info(f"Removing directory from RAG: {directory}")
 
@@ -275,8 +361,8 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
             # Delete file from disk if it's in uploads dir
             deleted_from_disk = False
             try:
-                abs_target = os.path.abspath(filepath)
-                base_abs = os.path.abspath(UPLOADS_DIR)
+                abs_target = os.path.realpath(filepath)
+                base_abs = os.path.realpath(UPLOADS_DIR)
                 in_uploads = (
                     abs_target == base_abs
                     or os.path.commonpath([abs_target, base_abs]) == base_abs

routes/session_routes.py

@@ -11,7 +11,7 @@ from core.session_manager import SessionManager
 from core.models import ChatMessage
 from src.request_models import SessionResponse
 from core.database import Session as DbSession, SessionLocal, Document, GalleryImage, utcnow_naive
-from src.auth_helpers import get_current_user, effective_user, _auth_disabled, owner_filter
+from src.auth_helpers import effective_user, _auth_disabled, owner_filter
 from src.session_actions import is_session_recently_active
 
 
@@ -328,7 +328,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         endpoint_id: str = Form(""),
     ):
         skip_val = str(skip_validation).lower() == "true"
-        user = get_current_user(request)
+        user = effective_user(request)
         endpoint_api_key = ""
         endpoint_base_url = ""
         _reject_raw_endpoint_url_for_non_admin(request, user, endpoint_id, endpoint_url)
@@ -477,7 +477,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db.close()
         # Switch model/endpoint mid-session
         if model is not None and endpoint_url is not None:
-            user = get_current_user(request)
+            user = effective_user(request)
             _reject_raw_endpoint_url_for_non_admin(request, user, endpoint_id, endpoint_url)
             endpoint_api_key = ""
             endpoint_base_url = ""
@@ -1004,6 +1004,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         """
         from src.llm_core import llm_call
         user = effective_user(request)
+        single_user_mode = not user and _auth_disabled()
         user_sessions = session_manager.get_sessions_for_user(user)
 
         # Delete empty and throwaway sessions before sorting
@@ -1022,7 +1023,12 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         }
         _THROWAWAY_MAX_MESSAGES = 4  # only delete if <= this many messages
         try:
-            rows = db.query(DbSession).filter(DbSession.archived == False, DbSession.owner == user).limit(2000).all()
+            rows_q = db.query(DbSession).filter(DbSession.archived == False)
+            if user:
+                rows_q = rows_q.filter(DbSession.owner == user)
+            elif not single_user_mode:
+                rows_q = rows_q.filter(DbSession.owner == user)
+            rows = rows_q.limit(2000).all()
             folder_map = {r.id: r.folder for r in rows}
             # Precompute per-session message counts in TWO aggregate queries
             # instead of 1–3 queries PER session — with many chats the per-row
@@ -1242,7 +1248,12 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         db = SessionLocal()
         try:
             for sid, folder_name in assignments.items():
-                db_session = db.query(DbSession).filter(DbSession.id == sid, DbSession.owner == user).first()
+                db_session_q = db.query(DbSession).filter(DbSession.id == sid)
+                if user:
+                    db_session_q = db_session_q.filter(DbSession.owner == user)
+                elif not single_user_mode:
+                    db_session_q = db_session_q.filter(DbSession.owner == user)
+                db_session = db_session_q.first()
                 if db_session:
                     db_session.folder = folder_name
                     db_session.updated_at = datetime.utcnow()

routes/shell_routes.py

@@ -1,6 +1,7 @@
 """Shell routes — user-facing command execution endpoint."""
 
 import asyncio
+import importlib
 import json
 import logging
 import os
@@ -14,6 +15,7 @@ from collections import namedtuple
 from pathlib import Path
 from typing import Dict, Any
 from core.platform_compat import IS_APPLE_SILICON, which_tool
+from src.optional_deps import prepare_optional_dependency_import
 
 # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist
 # on Windows, so importing them unconditionally crashed app startup there
@@ -149,6 +151,11 @@ def _pip_dist_name(pkg: dict) -> str:
     return (pkg.get("name") or "").replace("_", "-")
 
 
+def _import_optional_dependency_for_status(name: str):
+    prepare_optional_dependency_import(name)
+    return importlib.import_module(name)
+
+
 def _package_installed_from_probe(name: str, probe: dict) -> bool:
     """Return whether an optional dependency is usable by Cookbook.
 
@@ -970,7 +977,6 @@ def setup_shell_routes() -> APIRouter:
         """
         _require_admin(request)
         _reject_cross_site(request)
-        import importlib
         import importlib.metadata as importlib_metadata
         import shlex
         import json as _json
@@ -1057,6 +1063,13 @@ def setup_shell_routes() -> APIRouter:
                 "category": "Image",
                 "target": "remote",
             },
+            {
+                "name": "transformers",
+                "pip": "transformers",
+                "desc": "Hugging Face model components used by SD/Flux pipelines and image tools",
+                "category": "Image",
+                "target": "remote",
+            },
             {
                 "name": "rembg",
                 "pip": "rembg[gpu]",
@@ -1202,7 +1215,7 @@ def setup_shell_routes() -> APIRouter:
                     pkg["status_note"] = _package_status_note("vllm", probe)
             else:
                 try:
-                    importlib.import_module(pkg["name"])
+                    _import_optional_dependency_for_status(pkg["name"])
                     importlib_metadata.version(_pip_dist_name(pkg))
                     pkg["installed"] = True
                 except ImportError:
@@ -1251,6 +1264,7 @@ def setup_shell_routes() -> APIRouter:
             "sglang[all]",
             "diffusers",
             "diffusers[torch]",
+            "transformers",
             "TTS",
             "bark",
             "faster-whisper",

routes/skills_routes.py

@@ -691,8 +691,12 @@ async def _run_skill_test_once(md: str, task: str, url, model, headers, owner) -
         {"role": "user", "content": task},
     ]
     try:
+        # max_tokens explicitly set: passing 0 lets some upstreams (Ollama,
+        # OpenAI-compat) generate an empty completion, which manifested as
+        # the skill test returning nothing while chat (which carries its
+        # preset's max_tokens) worked. 4096 matches the chat default.
         async for chunk in stream_agent_loop(url, model, messages, headers=headers,
-                                             temperature=0.3, max_tokens=0, max_rounds=8, owner=owner):
+                                             temperature=0.3, max_tokens=4096, max_rounds=8, owner=owner):
             if not chunk.startswith("data: ") or chunk.strip() == "data: [DONE]":
                 continue
             try:

routes/task_routes.py

@@ -151,6 +151,7 @@ class TaskCreate(BaseModel):
     endpoint_url: Optional[str] = None
     then_task_id: Optional[str] = None            # chain: run this task after success
     notifications_enabled: Optional[bool] = None  # None lets action-specific defaults apply
+    character_id: Optional[str] = None             # built-in persona id (PERSONAS) — biases output voice
 
 
 class TaskUpdate(BaseModel):
@@ -171,6 +172,7 @@ class TaskUpdate(BaseModel):
     endpoint_url: Optional[str] = None
     then_task_id: Optional[str] = None
     notifications_enabled: Optional[bool] = None
+    character_id: Optional[str] = None
 
 
 def _display_task_name(t: ScheduledTask) -> str:
@@ -203,6 +205,7 @@ def _task_to_dict(t: ScheduledTask, include_last_run_result: bool = False) -> di
         "output_target": t.output_target,
         "session_id": t.session_id,
         "crew_member_id": getattr(t, "crew_member_id", None),
+        "character_id": getattr(t, "character_id", None),
         "model": t.model,
         "endpoint_url": t.endpoint_url,
         "run_count": t.run_count or 0,
@@ -552,6 +555,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 then_task_id=then_task_id,
                 webhook_token=webhook_token,
                 notifications_enabled=notifications_enabled,
+                character_id=(req.character_id or None),
             )
             db.add(task)
             db.commit()
@@ -705,6 +709,9 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 task.then_task_id = _validate_then_task_id(db, req.then_task_id, user, current_task_id=task.id)
             if req.notifications_enabled is not None:
                 task.notifications_enabled = bool(req.notifications_enabled)
+            if req.character_id is not None:
+                # Empty string clears the persona; non-empty stores the id.
+                task.character_id = req.character_id or None
             if req.cron_expression is not None:
                 if req.cron_expression:
                     try:

routes/upload_routes.py

@@ -7,7 +7,7 @@ from fastapi import APIRouter, Request, File, UploadFile, HTTPException
 from typing import List
 import logging
 from core.middleware import require_admin
-from src.auth_helpers import get_current_user
+from src.auth_helpers import effective_user
 from src.upload_handler import count_recent_uploads
 
 logger = logging.getLogger(__name__)
@@ -78,7 +78,7 @@ def setup_upload_routes(upload_handler):
         
         for u in files:
             try:
-                meta = upload_handler.save_upload(u, client_ip, owner=get_current_user(request))
+                meta = upload_handler.save_upload(u, client_ip, owner=effective_user(request))
                 out.append({
                     "id": meta["id"],
                     "name": meta["name"],
@@ -138,7 +138,7 @@ def setup_upload_routes(upload_handler):
                 original_name = info.get("name", file_id)
         auth_mgr = getattr(request.app.state, "auth_manager", None)
         auth_configured = bool(auth_mgr and auth_mgr.is_configured)
-        current_user = get_current_user(request)
+        current_user = effective_user(request)
         file_owner = info.get("owner") if info else None
         if auth_configured:
             if not current_user:
@@ -204,7 +204,7 @@ def setup_upload_routes(upload_handler):
         info = _load_upload_info(file_id)
         auth_mgr = getattr(request.app.state, "auth_manager", None)
         auth_configured = bool(auth_mgr and auth_mgr.is_configured)
-        current_user = get_current_user(request)
+        current_user = effective_user(request)
         file_owner = info.get("owner") if info else None
         if auth_configured:
             if not current_user:
@@ -247,7 +247,7 @@ def setup_upload_routes(upload_handler):
             raise HTTPException(404, "File not found")
         auth_mgr = getattr(request.app.state, "auth_manager", None)
         auth_configured = bool(auth_mgr and auth_mgr.is_configured)
-        current_user = get_current_user(request)
+        current_user = effective_user(request)
         file_owner = info.get("owner")
         if auth_configured:
             if not current_user:

routes/webhook_routes.py

@@ -1,6 +1,5 @@
 """Webhook, API Token, and sync chat routes."""
 
-import asyncio
 import uuid
 import logging
 from typing import Optional
@@ -198,6 +197,8 @@ def setup_webhook_routes(
         "opencode-go": "https://opencode.ai/zen/go/v1",
         "fireworks": "https://api.fireworks.ai/inference/v1",
         "venice": "https://api.venice.ai/api/v1",
+        "kimi-code": "https://api.kimi.com/coding/v1",
+        "kimicode": "https://api.kimi.com/coding/v1",
     }
 
     # Model prefix → provider mapping for auto-detection
@@ -210,6 +211,8 @@ def setup_webhook_routes(
         "mistral": "mistral",
         "llama": "groq",
         "mixtral": "groq",
+        "kimi-for-coding": "kimi-code",
+        "kimi": "kimi-code",
     }
 
     def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
@@ -381,10 +384,10 @@ def setup_webhook_routes(
         sess.add_message(ChatMessage("assistant", reply))
         session_manager.save_sessions()
 
-        asyncio.create_task(webhook_manager.fire("chat.completed", {
+        webhook_manager.fire_and_forget("chat.completed", {
             "session_id": session_id, "model": sess.model,
             "user_message": message[:2000], "response": reply[:2000],
-        }))
+        })
 
         return {"response": reply, "session_id": session_id, "model": sess.model}
 

routes/workspace_routes.py

@@ -0,0 +1,85 @@
+"""Workspace API - browse server directories to pick a tool workspace folder."""
+import os
+from fastapi import APIRouter, Request, HTTPException, Query
+
+from src.auth_helpers import get_current_user
+from src.tool_security import owner_is_admin_or_single_user
+
+# Cap entries returned per directory (mirrors filesystem_tools._CODENAV_MAX_HITS).
+# A huge directory shouldn't dump thousands of rows into the picker; the user can
+# type/paste a path to jump straight in instead.
+_MAX_BROWSE_DIRS = 500
+
+
+def setup_workspace_routes():
+    router = APIRouter(prefix="/api/workspace", tags=["workspace"])
+
+    @router.get("/browse")
+    def browse(request: Request, path: str = Query(default="")):
+        """List subdirectories of `path` (default: home) so the UI can navigate
+        the server filesystem and pick a workspace folder. Directories only.
+
+        ADMIN-ONLY: this enumerates the server filesystem, so it is gated the
+        same way the file/shell tools are (read_file/write_file/bash are in
+        NON_ADMIN_BLOCKED_TOOLS). A non-admin who can't use those tools must not
+        be able to map the host's directory tree either.
+        """
+        owner = get_current_user(request)
+        if not owner_is_admin_or_single_user(owner):
+            raise HTTPException(status_code=403, detail="Workspace browsing is admin-only")
+
+        # Resolve symlinks so the reported path is canonical and the UI navigates
+        # real directories (defends against symlink games in displayed paths).
+        target = os.path.realpath(os.path.expanduser(path.strip() or "~"))
+        if not os.path.isdir(target):
+            target = os.path.realpath(os.path.expanduser("~"))
+
+        dirs = []
+        try:
+            with os.scandir(target) as it:
+                for entry in it:
+                    try:
+                        # Don't follow symlinks when classifying - a symlinked
+                        # dir is skipped rather than letting the browser wander
+                        # off via a link. Hidden entries are omitted.
+                        if entry.is_dir(follow_symlinks=False) and not entry.name.startswith("."):
+                            # Build the child path server-side with os.path.join
+                            # so it's correct on Windows (backslashes) and Linux.
+                            dirs.append({"name": entry.name, "path": os.path.join(target, entry.name)})
+                    except OSError:
+                        continue
+        except (PermissionError, OSError):
+            dirs = []
+
+        dirs_sorted = sorted(dirs, key=lambda d: d["name"].lower())
+        truncated = len(dirs_sorted) > _MAX_BROWSE_DIRS
+        parent = os.path.dirname(target)
+        from src.tool_execution import vet_workspace
+        return {
+            "path": target,
+            "parent": parent if parent and parent != target else None,
+            "dirs": dirs_sorted[:_MAX_BROWSE_DIRS],
+            "truncated": truncated,
+            # Whether this directory may be bound as a workspace (filesystem
+            # roots and sensitive dirs may be browsed through but not chosen).
+            "selectable": vet_workspace(target) is not None,
+        }
+
+    @router.get("/vet")
+    def vet(request: Request, path: str = Query(default="")):
+        """Validate a workspace path without binding it.
+
+        The UI calls this before persisting a manually typed path (/workspace
+        set) so a typo, file path, deleted folder, sensitive dir, or filesystem
+        root is rejected up front with the canonical path returned on success,
+        instead of being stored client-side and silently dropped at chat time.
+        Admin-gated like /browse: it confirms path existence on the host.
+        """
+        owner = get_current_user(request)
+        if not owner_is_admin_or_single_user(owner):
+            raise HTTPException(status_code=403, detail="Workspace selection is admin-only")
+        from src.tool_execution import vet_workspace
+        resolved = vet_workspace(path)
+        return {"ok": resolved is not None, "path": resolved}
+
+    return router

scripts/agent_migration_manifest.py

@@ -0,0 +1,635 @@
+#!/usr/bin/env python3
+"""Build a neutral agent migration manifest.
+
+This helper is intentionally read-only. It does not import the Odysseus
+application package, write to data/, call an LLM, or apply anything. It turns
+common agent export shapes into a portable JSON manifest that Odysseus can
+preview or import later.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import mimetypes
+import sys
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterable
+
+
+SCHEMA_VERSION = "agent-migration.v1"
+TEXT_EXTENSIONS = {
+    ".cfg",
+    ".conf",
+    ".csv",
+    ".json",
+    ".log",
+    ".md",
+    ".markdown",
+    ".py",
+    ".rst",
+    ".toml",
+    ".txt",
+    ".yaml",
+    ".yml",
+}
+
+
+@dataclass(frozen=True)
+class InputWarning:
+    path: str
+    message: str
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def sha256_text(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def sha256_path(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(65536), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def stable_id(kind: str, source_name: str, *parts: Any) -> str:
+    raw = "\x1f".join([kind, source_name, *[str(part) for part in parts]])
+    return f"{kind}:{hashlib.sha256(raw.encode('utf-8')).hexdigest()[:16]}"
+
+
+def read_json(path: Path) -> Any:
+    with path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def normalize_category(value: Any) -> str:
+    category = str(value or "fact").strip().lower()
+    return category or "fact"
+
+
+def normalize_memory_text(item: Any) -> str:
+    if isinstance(item, str):
+        return item.strip()
+    if isinstance(item, dict):
+        for key in ("text", "content", "memory", "value"):
+            value = item.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip()
+    return ""
+
+
+def memory_metadata(item: Any, source_path: Path, index: int) -> dict[str, Any]:
+    metadata: dict[str, Any] = {
+        "source_path": str(source_path),
+        "source_index": index,
+    }
+    if isinstance(item, dict):
+        for key in ("id", "timestamp", "created_at", "updated_at", "source", "tags", "pinned"):
+            if key in item:
+                metadata[f"source_{key}"] = item.get(key)
+    return metadata
+
+
+def payload_items(payload: Any, keys: tuple[str, ...]) -> Any:
+    if isinstance(payload, dict):
+        for key in keys:
+            if isinstance(payload.get(key), list):
+                return payload[key]
+    return payload
+
+
+def collect_memory_json(path: Path, source_name: str) -> tuple[list[dict[str, Any]], list[InputWarning]]:
+    warnings: list[InputWarning] = []
+    try:
+        payload = read_json(path)
+    except Exception as exc:
+        return [], [InputWarning(str(path), f"could not read JSON: {exc}")]
+
+    payload = payload_items(payload, ("memories", "memory", "items", "data"))
+
+    if not isinstance(payload, list):
+        return [], [InputWarning(str(path), "expected a JSON list or an object containing a memory list")]
+
+    items: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for index, item in enumerate(payload):
+        text = normalize_memory_text(item)
+        if not text:
+            warnings.append(InputWarning(str(path), f"skipped memory at index {index}: missing text"))
+            continue
+        digest = sha256_text(text.strip().lower())
+        if digest in seen:
+            warnings.append(InputWarning(str(path), f"skipped duplicate memory at index {index}"))
+            continue
+        seen.add(digest)
+        category = normalize_category(item.get("category") if isinstance(item, dict) else "fact")
+        source = str(item.get("source") or source_name) if isinstance(item, dict) else source_name
+        items.append(
+            {
+                "id": stable_id("memory", source_name, path, index, digest),
+                "kind": "memory",
+                "text": text,
+                "category": category,
+                "source": source,
+                "metadata": memory_metadata(item, path, index),
+            }
+        )
+    return items, warnings
+
+
+def normalize_timestamp(value: Any) -> str | None:
+    if value is None or value == "":
+        return None
+    if isinstance(value, (int, float)):
+        try:
+            return (
+                datetime.fromtimestamp(float(value), timezone.utc)
+                .replace(microsecond=0)
+                .isoformat()
+                .replace("+00:00", "Z")
+            )
+        except (OverflowError, OSError, ValueError):
+            return str(value)
+    return str(value)
+
+
+def normalize_role(value: Any) -> str:
+    role = str(value or "unknown").strip().lower()
+    if role in {"human", "user"}:
+        return "user"
+    if role in {"assistant", "ai", "bot", "model"}:
+        return "assistant"
+    if role in {"system", "tool"}:
+        return role
+    return role or "unknown"
+
+
+def content_part_text(part: Any) -> str:
+    if isinstance(part, str):
+        return part
+    if isinstance(part, dict):
+        for key in ("text", "content", "value"):
+            value = part.get(key)
+            if isinstance(value, str):
+                return value
+        if part.get("type") == "text" and isinstance(part.get("text"), str):
+            return part["text"]
+    return ""
+
+
+def normalize_message_text(message: dict[str, Any]) -> str:
+    content = message.get("content")
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "\n".join(text for text in (content_part_text(part).strip() for part in content) if text)
+    if isinstance(content, dict):
+        parts = content.get("parts")
+        if isinstance(parts, list):
+            return "\n".join(text for text in (content_part_text(part).strip() for part in parts) if text)
+        for key in ("text", "content", "value"):
+            value = content.get(key)
+            if isinstance(value, str):
+                return value
+    for key in ("text", "body", "message"):
+        value = message.get(key)
+        if isinstance(value, str):
+            return value
+    return ""
+
+
+def normalize_message(message: dict[str, Any]) -> dict[str, Any] | None:
+    author = message.get("author") if isinstance(message.get("author"), dict) else {}
+    role = (
+        message.get("role")
+        or message.get("sender")
+        or message.get("speaker")
+        or author.get("role")
+        or author.get("name")
+    )
+    text = normalize_message_text(message).strip()
+    if not text:
+        return None
+    normalized: dict[str, Any] = {
+        "role": normalize_role(role),
+        "text": text,
+    }
+    timestamp = normalize_timestamp(message.get("created_at") or message.get("create_time") or message.get("timestamp"))
+    if timestamp:
+        normalized["created_at"] = timestamp
+    message_id = message.get("id")
+    if message_id is not None:
+        normalized["source_id"] = str(message_id)
+    return normalized
+
+
+def chatgpt_mapping_messages(conversation: dict[str, Any]) -> list[dict[str, Any]]:
+    mapping = conversation.get("mapping")
+    if not isinstance(mapping, dict):
+        return []
+    rows: list[tuple[float, int, dict[str, Any]]] = []
+    for index, node in enumerate(mapping.values()):
+        if not isinstance(node, dict) or not isinstance(node.get("message"), dict):
+            continue
+        message = node["message"]
+        sort_value = message.get("create_time")
+        try:
+            sort_key = float(sort_value)
+        except (TypeError, ValueError):
+            sort_key = float(index)
+        normalized = normalize_message(message)
+        if normalized:
+            rows.append((sort_key, index, normalized))
+    return [row[2] for row in sorted(rows, key=lambda row: (row[0], row[1]))]
+
+
+def conversation_messages(conversation: dict[str, Any]) -> tuple[list[dict[str, Any]], str]:
+    mapped = chatgpt_mapping_messages(conversation)
+    if mapped:
+        return mapped, "chatgpt_mapping"
+    for key in ("messages", "chat_messages", "turns"):
+        raw_messages = conversation.get(key)
+        if isinstance(raw_messages, list):
+            messages = [
+                normalized
+                for raw in raw_messages
+                if isinstance(raw, dict)
+                for normalized in [normalize_message(raw)]
+                if normalized
+            ]
+            return messages, key
+    return [], "unknown"
+
+
+def conversation_title(conversation: dict[str, Any], index: int) -> str:
+    for key in ("title", "name", "summary"):
+        value = conversation.get(key)
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+    return f"Conversation {index + 1}"
+
+
+def collect_conversation_json(
+    path: Path,
+    source_name: str,
+    *,
+    include_content: bool = False,
+    max_messages: int = 2000,
+) -> tuple[list[dict[str, Any]], list[InputWarning]]:
+    warnings: list[InputWarning] = []
+    try:
+        payload = read_json(path)
+    except Exception as exc:
+        return [], [InputWarning(str(path), f"could not read JSON: {exc}")]
+
+    payload = payload_items(payload, ("conversations", "conversation", "items", "data"))
+    if isinstance(payload, dict):
+        payload = [payload]
+    if not isinstance(payload, list):
+        return [], [InputWarning(str(path), "expected a JSON list or an object containing a conversation list")]
+
+    items: list[dict[str, Any]] = []
+    for index, conversation in enumerate(payload):
+        if not isinstance(conversation, dict):
+            warnings.append(InputWarning(str(path), f"skipped conversation at index {index}: expected object"))
+            continue
+        messages, format_hint = conversation_messages(conversation)
+        if not messages:
+            warnings.append(InputWarning(str(path), f"skipped conversation at index {index}: no text messages found"))
+            continue
+        title = conversation_title(conversation, index)
+        source_id = conversation.get("id") or conversation.get("uuid") or conversation.get("conversation_id")
+        text_digest = sha256_text("\n".join(f"{msg['role']}:{msg['text']}" for msg in messages))
+        metadata: dict[str, Any] = {
+            "source_path": str(path),
+            "source_index": index,
+            "source_format": format_hint,
+            "message_count": len(messages),
+            "text_sha256": text_digest,
+            "content_included": False,
+        }
+        if source_id is not None:
+            metadata["source_id"] = str(source_id)
+        for key in ("create_time", "created_at", "update_time", "updated_at"):
+            timestamp = normalize_timestamp(conversation.get(key))
+            if timestamp:
+                metadata[f"source_{key}"] = timestamp
+        item: dict[str, Any] = {
+            "id": stable_id("conversation", source_name, path, source_id or index, text_digest),
+            "kind": "conversation_thread",
+            "title": title,
+            "source": source_name,
+            "metadata": metadata,
+        }
+        if include_content:
+            if len(messages) > max_messages:
+                warnings.append(
+                    InputWarning(
+                        str(path),
+                        f"skipped conversation content at index {index}: over {max_messages} messages",
+                    )
+                )
+            else:
+                item["messages"] = messages
+                item["metadata"]["content_included"] = True
+        items.append(item)
+    return items, warnings
+
+
+def parse_skill_frontmatter(text: str) -> dict[str, Any]:
+    if not text.startswith("---"):
+        return {}
+    end = text.find("\n---", 3)
+    if end < 0:
+        return {}
+    frontmatter: dict[str, Any] = {}
+    for line in text[3:end].strip().splitlines():
+        if not line.strip() or line.lstrip().startswith("#") or ":" not in line:
+            continue
+        key, value = line.split(":", 1)
+        key = key.strip()
+        value = value.strip().strip('"').strip("'")
+        if key:
+            frontmatter[key] = value
+    return frontmatter
+
+
+def collect_skill_dir(path: Path, source_name: str) -> tuple[list[dict[str, Any]], list[InputWarning]]:
+    warnings: list[InputWarning] = []
+    if path.is_symlink():
+        return [], [InputWarning(str(path), "skills path is a symlink; skipped")]
+    if not path.exists():
+        return [], [InputWarning(str(path), "skills directory does not exist")]
+    if not path.is_dir():
+        return [], [InputWarning(str(path), "skills path is not a directory")]
+
+    items: list[dict[str, Any]] = []
+    for skill_path in sorted(path.rglob("SKILL.md")):
+        if skill_path.is_symlink():
+            warnings.append(InputWarning(str(skill_path), "skipped symlinked skill file"))
+            continue
+        try:
+            text = skill_path.read_text(encoding="utf-8")
+        except Exception as exc:
+            warnings.append(InputWarning(str(skill_path), f"could not read skill: {exc}"))
+            continue
+        frontmatter = parse_skill_frontmatter(text)
+        name = str(frontmatter.get("name") or skill_path.parent.name).strip() or skill_path.parent.name
+        items.append(
+            {
+                "id": stable_id("skill", source_name, skill_path, sha256_text(text)),
+                "kind": "skill",
+                "name": name,
+                "category": str(frontmatter.get("category") or "general"),
+                "source": source_name,
+                "format": "SKILL.md",
+                "content": text,
+                "metadata": {
+                    "source_path": str(skill_path),
+                    "sha256": sha256_text(text),
+                    "frontmatter": frontmatter,
+                },
+            }
+        )
+    return items, warnings
+
+
+def looks_textual(path: Path) -> bool:
+    if path.suffix.lower() in TEXT_EXTENSIONS:
+        return True
+    guessed, _ = mimetypes.guess_type(str(path))
+    return bool(guessed and (guessed.startswith("text/") or guessed in {"application/json"}))
+
+
+def iter_archive_dir(path: Path) -> Iterable[Path | InputWarning]:
+    try:
+        children = sorted(path.iterdir())
+    except Exception as exc:
+        yield InputWarning(str(path), f"could not scan archive directory: {exc}")
+        return
+    for child in children:
+        if child.is_symlink():
+            yield InputWarning(str(child), "skipped symlinked archive path")
+            continue
+        if child.is_file():
+            yield child
+        elif child.is_dir():
+            yield from iter_archive_dir(child)
+
+
+def iter_archive_files(paths: Iterable[Path]) -> Iterable[Path | InputWarning]:
+    for path in paths:
+        if path.is_symlink():
+            yield InputWarning(str(path), "skipped symlinked archive path")
+            continue
+        if path.is_file():
+            yield path
+        elif path.is_dir():
+            yield from iter_archive_dir(path)
+
+
+def collect_archive_paths(
+    paths: list[Path],
+    source_name: str,
+    *,
+    include_content: bool = False,
+    max_bytes: int = 256_000,
+) -> tuple[list[dict[str, Any]], list[InputWarning]]:
+    warnings: list[InputWarning] = []
+    items: list[dict[str, Any]] = []
+    existing_paths: list[Path] = []
+    for path in paths:
+        if path.is_symlink():
+            warnings.append(InputWarning(str(path), "archive path is a symlink; skipped"))
+            continue
+        if not path.exists():
+            warnings.append(InputWarning(str(path), "archive path does not exist"))
+            continue
+        if not path.is_file() and not path.is_dir():
+            warnings.append(InputWarning(str(path), "archive path is not a file or directory"))
+            continue
+        existing_paths.append(path)
+
+    for entry in iter_archive_files(existing_paths):
+        if isinstance(entry, InputWarning):
+            warnings.append(entry)
+            continue
+        path = entry
+        if not looks_textual(path):
+            warnings.append(InputWarning(str(path), "skipped non-text archive file"))
+            continue
+        try:
+            st = path.stat()
+        except Exception as exc:
+            warnings.append(InputWarning(str(path), f"could not stat archive file: {exc}"))
+            continue
+        size = st.st_size
+        try:
+            file_hash = sha256_path(path)
+        except Exception as exc:
+            warnings.append(InputWarning(str(path), f"could not hash archive file: {exc}"))
+            continue
+        if include_content and size > max_bytes:
+            warnings.append(InputWarning(str(path), f"skipped archive content over {max_bytes} bytes"))
+        archive_item: dict[str, Any] = {
+            "id": stable_id("archive", source_name, path, file_hash),
+            "kind": "archive_document",
+            "title": path.name,
+            "source": source_name,
+            "metadata": {
+                "source_path": str(path),
+                "size_bytes": size,
+                "sha256": file_hash,
+            },
+        }
+        if include_content and size <= max_bytes:
+            try:
+                archive_item["content"] = path.read_text(encoding="utf-8")
+            except UnicodeDecodeError:
+                archive_item["content"] = path.read_text(encoding="utf-8", errors="replace")
+                archive_item["metadata"]["decoded_with_replacement"] = True
+        items.append(archive_item)
+    return items, warnings
+
+
+def build_manifest(args) -> dict[str, Any]:
+    warnings: list[InputWarning] = []
+    items: list[dict[str, Any]] = []
+
+    for path in args.memory_json:
+        collected, got_warnings = collect_memory_json(path, args.source_name)
+        items.extend(collected)
+        warnings.extend(got_warnings)
+
+    for path in args.skills_dir:
+        collected, got_warnings = collect_skill_dir(path, args.source_name)
+        items.extend(collected)
+        warnings.extend(got_warnings)
+
+    for path in args.conversation_json:
+        collected, got_warnings = collect_conversation_json(
+            path,
+            args.source_name,
+            include_content=args.include_conversation_content,
+            max_messages=args.max_conversation_messages,
+        )
+        items.extend(collected)
+        warnings.extend(got_warnings)
+
+    if args.archive:
+        collected, got_warnings = collect_archive_paths(
+            args.archive,
+            args.source_name,
+            include_content=args.include_archive_content,
+            max_bytes=args.max_archive_bytes,
+        )
+        items.extend(collected)
+        warnings.extend(got_warnings)
+
+    counts: dict[str, int] = {}
+    for item in items:
+        counts[item["kind"]] = counts.get(item["kind"], 0) + 1
+
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "generated_at": utc_now_iso(),
+        "source": {
+            "name": args.source_name,
+            "kind": args.source_kind,
+        },
+        "summary": {
+            "item_count": len(items),
+            "counts_by_kind": counts,
+            "warning_count": len(warnings),
+        },
+        "items": items,
+        "warnings": [{"path": warning.path, "message": warning.message} for warning in warnings],
+    }
+
+
+def parse_args(argv: list[str] | None = None):
+    parser = argparse.ArgumentParser(description="Build a neutral Odysseus agent migration manifest.")
+    parser.add_argument("--source-name", default="agent-export", help="Human-readable source name.")
+    parser.add_argument("--source-kind", default="generic", help="Source adapter kind, e.g. generic, openclaw, hermes.")
+    parser.add_argument(
+        "--memory-json",
+        action="append",
+        type=Path,
+        default=[],
+        help="JSON memory export. May be a list, or an object containing memories/items/data.",
+    )
+    parser.add_argument(
+        "--skills-dir",
+        action="append",
+        type=Path,
+        default=[],
+        help="Directory containing SKILL.md files. Scanned recursively.",
+    )
+    parser.add_argument(
+        "--archive",
+        action="append",
+        type=Path,
+        default=[],
+        help="Text/Markdown/JSON file or directory to preserve as archive documents.",
+    )
+    parser.add_argument(
+        "--conversation-json",
+        action="append",
+        type=Path,
+        default=[],
+        help="Conversation export JSON. Supports generic message lists and ChatGPT-style conversations.json.",
+    )
+    parser.add_argument(
+        "--include-archive-content",
+        action="store_true",
+        help="Embed archive document content in the manifest. By default only metadata is included.",
+    )
+    parser.add_argument(
+        "--max-archive-bytes",
+        type=int,
+        default=256_000,
+        help="Maximum bytes to embed per archive file when --include-archive-content is used.",
+    )
+    parser.add_argument(
+        "--include-conversation-content",
+        action="store_true",
+        help="Embed normalized conversation messages. By default only thread metadata is included.",
+    )
+    parser.add_argument(
+        "--max-conversation-messages",
+        type=int,
+        default=2000,
+        help="Maximum messages to embed per conversation when --include-conversation-content is used.",
+    )
+    parser.add_argument("--output", type=Path, help="Write manifest JSON to this path instead of stdout.")
+    parser.add_argument("--compact", action="store_true", help="Write compact JSON without indentation.")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    manifest = build_manifest(args)
+    text = json.dumps(manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":")) if args.compact else (
+        json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
+    )
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(text, encoding="utf-8")
+    else:
+        sys.stdout.write(text)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/backfill_model_release_dates.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Backfill release_date on entries in services/hwfit/data/hf_models.json.
+
+Why: the `newest` sort in the cookbook ranks rows by release_date. Anything
+missing a date sorts to the bottom. This script pulls `created_at` from the
+HuggingFace API for each catalog entry without one (or all entries when
+--refresh is passed) and writes the catalog back.
+
+Usage:
+    python scripts/backfill_model_release_dates.py            # missing only
+    python scripts/backfill_model_release_dates.py --refresh  # all entries
+    python scripts/backfill_model_release_dates.py --limit 50 # cap requests
+    python scripts/backfill_model_release_dates.py --dry-run  # show, don't write
+
+Auth: set HF_TOKEN env var (or huggingface-cli login) to access gated repos.
+"""
+import argparse
+import json
+import os
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+try:
+    from huggingface_hub import HfApi
+    from huggingface_hub.utils import HfHubHTTPError
+except ImportError:
+    print("Install huggingface_hub: pip install huggingface_hub", file=sys.stderr)
+    sys.exit(1)
+
+
+CATALOG_PATH = Path(__file__).resolve().parent.parent / "services" / "hwfit" / "data" / "hf_models.json"
+
+
+def fetch_release_date(api: HfApi, repo_id: str) -> str | None:
+    """Return YYYY-MM-DD release date, or None on miss / error."""
+    try:
+        info = api.model_info(repo_id, files_metadata=False)
+    except HfHubHTTPError as e:
+        # 401 = gated/private, 404 = renamed/deleted. Either way, no date.
+        status = getattr(getattr(e, "response", None), "status_code", None)
+        print(f"  {repo_id}: HTTP {status or '?'}", file=sys.stderr)
+        return None
+    except Exception as e:
+        print(f"  {repo_id}: {type(e).__name__}: {e}", file=sys.stderr)
+        return None
+    created = getattr(info, "created_at", None)
+    if not created:
+        return None
+    return created.strftime("%Y-%m-%d")
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--refresh", action="store_true", help="Overwrite existing release_date too (default: only fill missing).")
+    p.add_argument("--limit", type=int, default=0, help="Stop after N API calls (0 = no limit).")
+    p.add_argument("--dry-run", action="store_true", help="Don't write back; just report.")
+    p.add_argument("--sleep", type=float, default=0.05, help="Seconds to sleep between requests (default 0.05).")
+    args = p.parse_args()
+
+    if not CATALOG_PATH.exists():
+        print(f"Catalog not found: {CATALOG_PATH}", file=sys.stderr)
+        sys.exit(2)
+
+    with CATALOG_PATH.open(encoding="utf-8") as f:
+        catalog = json.load(f)
+
+    candidates = []
+    for i, m in enumerate(catalog):
+        name = m.get("name")
+        if not name:
+            continue
+        existing = (m.get("release_date") or "").strip()
+        if existing and not args.refresh:
+            continue
+        candidates.append(i)
+
+    if args.limit:
+        candidates = candidates[: args.limit]
+
+    print(f"Catalog: {CATALOG_PATH}")
+    print(f"Total entries: {len(catalog)}")
+    print(f"Targets ({'refresh all' if args.refresh else 'missing only'}{'' if not args.limit else f', capped at {args.limit}'}): {len(candidates)}")
+    if not candidates:
+        print("Nothing to do.")
+        return
+
+    api = HfApi(token=os.environ.get("HF_TOKEN") or None)
+    updated = 0
+    skipped = 0
+    started = time.time()
+    for n, idx in enumerate(candidates, start=1):
+        entry = catalog[idx]
+        name = entry["name"]
+        old = (entry.get("release_date") or "").strip()
+        new = fetch_release_date(api, name)
+        if new is None:
+            skipped += 1
+            tag = "skip"
+        elif new == old:
+            tag = "unchanged"
+        else:
+            entry["release_date"] = new
+            updated += 1
+            tag = f"set {new}" + (f" (was {old})" if old else "")
+        print(f"[{n}/{len(candidates)}] {name} — {tag}")
+        if args.sleep:
+            time.sleep(args.sleep)
+
+    elapsed = time.time() - started
+    print()
+    print(f"Done in {elapsed:.1f}s — {updated} updated, {skipped} skipped (HF unavailable / gated / missing date).")
+
+    if args.dry_run:
+        print("Dry run — no write.")
+        return
+
+    if updated:
+        # Atomic write: tmp file in the same dir, then rename. Keeps the
+        # catalog usable even if the process dies mid-write.
+        tmp = CATALOG_PATH.with_suffix(".json.tmp")
+        with tmp.open("w", encoding="utf-8") as f:
+            json.dump(catalog, f, indent=1, ensure_ascii=False)
+            f.write("\n")
+        tmp.replace(CATALOG_PATH)
+        print(f"Wrote {CATALOG_PATH}")
+    else:
+        print("No changes to write.")
+
+
+if __name__ == "__main__":
+    main()

scripts/import_from_vllm_recipes.py

@@ -0,0 +1,341 @@
+#!/usr/bin/env python3
+"""Import models from the upstream vllm-project/recipes catalog into our
+local hf_models.json. Two modes:
+
+  --update-existing  Stamp min_vllm_version + vllm_recipe=True on rows we
+                     already carry. Cheap, no HF API calls.
+  --add-missing      Create new catalog rows for every recipe model we
+                     don't carry. Hits the HF API for created_at + downloads
+                     (~1 req per missing model, paced).
+
+Both modes write atomically (tmp + rename) so a crashed run leaves the
+catalog intact. Default with no mode flags runs both, prefer to pass them
+explicitly.
+
+Usage:
+    python scripts/import_from_vllm_recipes.py --update-existing
+    python scripts/import_from_vllm_recipes.py --add-missing
+    python scripts/import_from_vllm_recipes.py --dry-run
+    python scripts/import_from_vllm_recipes.py --limit 10
+
+Auth: set HF_TOKEN to access gated repos when --add-missing.
+"""
+import argparse
+import json
+import os
+import re
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+try:
+    import httpx
+    import yaml
+except ImportError:
+    print("pip install httpx PyYAML", file=sys.stderr)
+    sys.exit(1)
+
+try:
+    from huggingface_hub import HfApi
+    from huggingface_hub.utils import HfHubHTTPError
+except ImportError:
+    HfApi = None
+    HfHubHTTPError = Exception
+
+
+CATALOG_PATH = Path(__file__).resolve().parent.parent / "services" / "hwfit" / "data" / "hf_models.json"
+RECIPES_TREE_URL = (
+    "https://api.github.com/repos/vllm-project/recipes/git/trees/main?recursive=1"
+)
+RECIPE_RAW_URL = (
+    "https://raw.githubusercontent.com/vllm-project/recipes/main/models/{repo}.yaml"
+)
+
+
+# Map recipe `precision` to the closest catalog `quantization` label that
+# fit.py / models.py already understand.
+_PRECISION_TO_QUANT = {
+    "fp8": "FP8",
+    "nvfp4": "NVFP4",
+    "mxfp4": "MXFP4",
+    "bf16": "BF16",
+    "fp16": "F16",
+    "f16": "F16",
+    "fp4": "FP4",
+    "int8": "INT8",
+    "int4": "INT4",
+    "awq-4bit": "AWQ-4bit",
+    "awq-8bit": "AWQ-8bit",
+}
+
+# Architecture name → use_case fallback. fit.py weights use_case for filtering;
+# missing field defaults to a generic bucket.
+_ARCH_USE_CASE = {
+    "moe": "General-purpose reasoning, long-context",
+    "llama": "General-purpose chat",
+    "qwen2": "General-purpose chat",
+    "qwen3": "General-purpose reasoning",
+    "deepseek_v3_moe": "General-purpose reasoning, long-context",
+    "deepseek_v4_moe": "General-purpose reasoning, long-context",
+}
+
+
+def _parse_param_count(s) -> int:
+    """'230B' / '8.6B' / '4.2T' → integer parameter count."""
+    if s is None:
+        return 0
+    s = str(s).strip().replace(",", "")
+    m = re.match(r"^([\d.]+)\s*([KMBT]?)$", s, re.I)
+    if not m:
+        return 0
+    num = float(m.group(1))
+    unit = (m.group(2) or "").upper()
+    mult = {"K": 1e3, "M": 1e6, "B": 1e9, "T": 1e12, "": 1.0}[unit]
+    return int(num * mult)
+
+
+def _capabilities_for(arch: str, hardware: dict, ctx_len: int, has_reasoning: bool) -> list[str]:
+    caps = []
+    if "moe" in (arch or "").lower():
+        caps.append("moe")
+    if has_reasoning:
+        caps.append("reasoning")
+    if ctx_len and ctx_len >= 100_000:
+        caps.append("long_context")
+    if any(hw in (hardware or {}) for hw in ("mi300x", "mi325x", "mi350x", "mi355x")):
+        caps.append("amd_supported")
+    return caps
+
+
+def _fetch_manifest(client: httpx.Client) -> set[str]:
+    r = client.get(RECIPES_TREE_URL, headers={"Accept": "application/vnd.github+json"}, timeout=15)
+    r.raise_for_status()
+    tree = (r.json() or {}).get("tree") or []
+    out: set[str] = set()
+    for e in tree:
+        path = (e or {}).get("path") or ""
+        if path.startswith("models/") and path.endswith(".yaml"):
+            body = path[len("models/"):-len(".yaml")]
+            if "/" in body:
+                out.add(body)
+    return out
+
+
+def _fetch_recipe(client: httpx.Client, repo: str) -> dict | None:
+    url = RECIPE_RAW_URL.format(repo=repo)
+    try:
+        r = client.get(url, timeout=10)
+        if r.status_code != 200:
+            return None
+        return yaml.safe_load(r.text) or {}
+    except Exception:
+        return None
+
+
+def _stamp_from_recipe(entry: dict, recipe: dict) -> bool:
+    """Mutate entry with recipe-derived fields. Returns True if anything changed."""
+    model = recipe.get("model") or {}
+    meta = recipe.get("meta") or {}
+    features = recipe.get("features") or {}
+
+    changed = False
+    new_min = (model.get("min_vllm_version") or "").strip()
+    if new_min and entry.get("min_vllm_version") != new_min:
+        entry["min_vllm_version"] = new_min
+        changed = True
+    if not entry.get("vllm_recipe"):
+        entry["vllm_recipe"] = True
+        changed = True
+    # Hardware support map — useful for filtering "which models run on my AMD box".
+    hw = meta.get("hardware") or {}
+    if hw and entry.get("recipe_hardware") != hw:
+        entry["recipe_hardware"] = {k: str(v) for k, v in hw.items()}
+        changed = True
+    # Tool/reasoning parser hints — purely informational at catalog level;
+    # the live launch command builder still reads them from the recipe API.
+    if features.get("reasoning") and not entry.get("has_reasoning_parser"):
+        entry["has_reasoning_parser"] = True
+        changed = True
+    if features.get("tool_calling") and not entry.get("has_tool_call_parser"):
+        entry["has_tool_call_parser"] = True
+        changed = True
+    return changed
+
+
+def _build_new_entry(repo: str, recipe: dict, hf_info=None) -> dict | None:
+    """Build a fresh catalog entry from a recipe + (optional) HF model info."""
+    model = recipe.get("model") or {}
+    meta = recipe.get("meta") or {}
+    features = recipe.get("features") or {}
+    variants = recipe.get("variants") or {}
+
+    org, name = repo.split("/", 1)
+    raw_params = _parse_param_count(model.get("parameter_count"))
+    active_raw = _parse_param_count(model.get("active_parameters"))
+    ctx = model.get("context_length") or 0
+
+    # Pick the smallest-VRAM variant as the catalog quant — that's what most
+    # users land on first. NVFP4/MXFP4 typically win this on Blackwell;
+    # FP8 elsewhere; BF16 baseline only.
+    pick_quant = None
+    pick_vram = None
+    for vk, vv in variants.items():
+        if not isinstance(vv, dict):
+            continue
+        prec = (vv.get("precision") or "").lower()
+        vram = vv.get("vram_minimum_gb") or 0
+        quant = _PRECISION_TO_QUANT.get(prec)
+        if quant and (pick_vram is None or (vram and vram < pick_vram)):
+            pick_quant = quant
+            pick_vram = vram or pick_vram
+    if not pick_quant:
+        pick_quant = "BF16"
+
+    arch = (model.get("architecture") or "").lower()
+    use_case = _ARCH_USE_CASE.get(arch, "General-purpose chat")
+    caps = _capabilities_for(arch, meta.get("hardware") or {}, ctx, bool(features.get("reasoning")))
+
+    rel_date = ""
+    downloads = 0
+    likes = 0
+    if hf_info is not None:
+        created = getattr(hf_info, "created_at", None)
+        if created:
+            rel_date = created.strftime("%Y-%m-%d")
+        downloads = int(getattr(hf_info, "downloads", 0) or 0)
+        likes = int(getattr(hf_info, "likes", 0) or 0)
+    if not rel_date:
+        rel_date = str(meta.get("date_updated") or datetime.utcnow().strftime("%Y-%m-%d"))
+
+    entry: dict = {
+        "name": repo,
+        "provider": org,
+        "parameter_count": str(model.get("parameter_count") or "?"),
+        "parameters_raw": raw_params,
+        "is_moe": "moe" in arch,
+        "quantization": pick_quant,
+        "context_length": int(ctx or 0),
+        "use_case": use_case,
+        "capabilities": caps,
+        "pipeline_tag": "text-generation",
+        "architecture": arch or "unknown",
+        "hf_downloads": downloads,
+        "hf_likes": likes,
+        "release_date": rel_date,
+        # Recipe-derived bits.
+        "vllm_recipe": True,
+        "min_vllm_version": (model.get("min_vllm_version") or "").strip() or None,
+        "recipe_hardware": {k: str(v) for k, v in (meta.get("hardware") or {}).items()},
+        "has_reasoning_parser": bool(features.get("reasoning")),
+        "has_tool_call_parser": bool(features.get("tool_calling")),
+    }
+    if active_raw:
+        entry["active_parameters"] = active_raw
+    if pick_vram:
+        # min_vram_gb is what hwfit uses for "does this fit". Recipe states a
+        # minimum for the chosen variant; round up slightly for KV-cache room.
+        entry["min_vram_gb"] = float(pick_vram)
+        entry["min_ram_gb"] = float(round(pick_vram * 0.6, 1))
+        entry["recommended_ram_gb"] = float(round(pick_vram * 1.2, 1))
+    # Drop empty / None fields to keep the JSON tidy.
+    return {k: v for k, v in entry.items() if v not in (None, "", [], {})}
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--update-existing", action="store_true", help="Stamp min_vllm_version + vllm_recipe on existing rows.")
+    p.add_argument("--add-missing", action="store_true", help="Add new rows for recipe models not in the catalog.")
+    p.add_argument("--limit", type=int, default=0, help="Stop after N recipe fetches.")
+    p.add_argument("--dry-run", action="store_true", help="Don't write back; just report.")
+    p.add_argument("--sleep", type=float, default=0.05, help="Seconds between HTTP requests.")
+    args = p.parse_args()
+    if not args.update_existing and not args.add_missing:
+        args.update_existing = args.add_missing = True
+
+    with CATALOG_PATH.open(encoding="utf-8") as f:
+        catalog = json.load(f)
+    by_name = {m.get("name"): m for m in catalog if m.get("name")}
+
+    client = httpx.Client(follow_redirects=True)
+    print(f"Catalog: {CATALOG_PATH} ({len(catalog)} entries)")
+    print("Fetching upstream manifest…")
+    try:
+        manifest = _fetch_manifest(client)
+    except Exception as e:
+        print(f"FATAL: manifest fetch failed: {e}", file=sys.stderr)
+        sys.exit(2)
+    print(f"Manifest: {len(manifest)} recipes")
+
+    existing = sorted(by_name.keys() & manifest)
+    missing = sorted(manifest - by_name.keys())
+    print(f"Match catalog ↔ manifest: existing={len(existing)} missing={len(missing)}")
+
+    targets: list[tuple[str, str]] = []  # (repo, action)
+    if args.update_existing:
+        targets.extend((r, "update") for r in existing)
+    if args.add_missing:
+        targets.extend((r, "add") for r in missing)
+    if args.limit:
+        targets = targets[: args.limit]
+    print(f"Targets: {len(targets)}")
+
+    hf_api = HfApi(token=os.environ.get("HF_TOKEN") or None) if HfApi else None
+    updated = added = skipped = 0
+    started = time.time()
+
+    for n, (repo, action) in enumerate(targets, 1):
+        recipe = _fetch_recipe(client, repo)
+        if not recipe:
+            print(f"[{n}/{len(targets)}] {repo:55} skip (no recipe fetched)")
+            skipped += 1
+            time.sleep(args.sleep)
+            continue
+        if action == "update":
+            entry = by_name[repo]
+            if _stamp_from_recipe(entry, recipe):
+                updated += 1
+                print(f"[{n}/{len(targets)}] {repo:55} updated")
+            else:
+                print(f"[{n}/{len(targets)}] {repo:55} unchanged")
+        else:  # add
+            hf_info = None
+            if hf_api:
+                try:
+                    hf_info = hf_api.model_info(repo, files_metadata=False)
+                except HfHubHTTPError as e:
+                    code = getattr(getattr(e, "response", None), "status_code", "?")
+                    print(f"  HF {code} for {repo} — building from recipe only", file=sys.stderr)
+                except Exception as e:
+                    print(f"  HF error for {repo}: {e}", file=sys.stderr)
+            new_entry = _build_new_entry(repo, recipe, hf_info)
+            if new_entry:
+                catalog.append(new_entry)
+                by_name[repo] = new_entry
+                added += 1
+                print(f"[{n}/{len(targets)}] {repo:55} added ({new_entry.get('parameter_count','?')}, {new_entry.get('quantization','?')})")
+            else:
+                skipped += 1
+                print(f"[{n}/{len(targets)}] {repo:55} skip (couldn't build entry)")
+        time.sleep(args.sleep)
+
+    elapsed = time.time() - started
+    print()
+    print(f"Done in {elapsed:.1f}s — added={added}, updated={updated}, skipped={skipped}")
+
+    if args.dry_run:
+        print("Dry run — no write.")
+        return
+    if added or updated:
+        tmp = CATALOG_PATH.with_suffix(".json.tmp")
+        with tmp.open("w", encoding="utf-8") as f:
+            json.dump(catalog, f, indent=1, ensure_ascii=False)
+            f.write("\n")
+        tmp.replace(CATALOG_PATH)
+        print(f"Wrote {CATALOG_PATH} ({len(catalog)} entries)")
+    else:
+        print("No changes — catalog untouched.")
+
+
+if __name__ == "__main__":
+    main()

services/hwfit/fit.py

@@ -19,22 +19,36 @@ GPU_BANDWIDTH = {
     "6950 xt": 576, "6900 xt": 512, "6800 xt": 512, "6800": 512, "6700 xt": 384, "6600 xt": 256, "6600": 224,
     "mi300x": 5300, "mi300": 5300, "mi250x": 3277, "mi250": 3277, "mi210": 1638, "mi100": 1229,
     "9070 xt": 624, "9070": 488, "9060 xt": 322, "9060": 322,
-    # Apple Silicon unified-memory bandwidth (GB/s). Keyed off the chip name
-    # reported by sysctl machdep.cpu.brand_string (e.g. "Apple M4 Max"). Listed
-    # before the bare "m_" keys matters less than length-sorting (done below),
-    # which guarantees "m4 max" is tried before "m4".
-    "m1 ultra": 800, "m1 max": 400, "m1 pro": 200, "m1": 68,
-    "m2 ultra": 800, "m2 max": 400, "m2 pro": 200, "m2": 100,
-    "m3 ultra": 800, "m3 max": 300, "m3 pro": 150, "m3": 100,
-    "m4 max": 546, "m4 pro": 273, "m4": 120,
-    "m5 max": 546, "m5 pro": 273, "m5": 150,
+    # NVIDIA GB10 Grace-Blackwell superchip (DGX Spark). Unified LPDDR5X memory,
+    # not Apple Silicon, so it lives in the generic GPU table — the Apple-only
+    # lookup never matches it (its name carries no "apple").
+    "gb10": 273,
 }
 
 # Pre-sort keys by length descending for correct substring matching
 _BW_KEYS_SORTED = sorted(GPU_BANDWIDTH.keys(), key=len, reverse=True)
 
-# metal: backstop for Apple Silicon chips not in GPU_BANDWIDTH (e.g. a future
-# M5) — the named chips above take the accurate bandwidth path instead.
+# Apple Silicon unified-memory bandwidth (GB/s). For chip families with both
+# binned and full variants under the same "Apple Mx Max" brand string, prefer
+# GPU core count when hardware detection provides it; otherwise fall back to the
+# conservative tier so speed estimates do not over-promise.
+APPLE_BANDWIDTH_FIXED = {
+    "m1 ultra": 800, "m1 max": 400, "m1 pro": 200, "m1": 68,
+    "m2 ultra": 800, "m2 max": 400, "m2 pro": 200, "m2": 100,
+    "m3 ultra": 800, "m3 pro": 150, "m3": 100,
+    "m4 pro": 273, "m4": 120,
+    "m5 pro": 307, "m5": 153,
+}
+APPLE_BANDWIDTH_BY_CORES = {
+    "m3 max": {30: 300, 40: 400},
+    "m4 max": {32: 410, 40: 546},
+    "m5 max": {32: 460, 40: 614},
+}
+_APPLE_FIXED_KEYS_SORTED = sorted(APPLE_BANDWIDTH_FIXED.keys(), key=len, reverse=True)
+_APPLE_VARIANT_KEYS_SORTED = sorted(APPLE_BANDWIDTH_BY_CORES.keys(), key=len, reverse=True)
+
+# metal: backstop for Apple Silicon chips not in the explicit tables above
+# (e.g. a future M6) — use a conservative generic estimate when unknown.
 FALLBACK_K = {"cuda": 220, "rocm": 180, "metal": 150, "cpu_x86": 70, "cpu_arm": 90}
 
 USE_CASE_WEIGHTS = {
@@ -60,10 +74,56 @@ CONTEXT_TARGET = {
 }
 
 
-def _lookup_bandwidth(gpu_name):
+def _lookup_apple_bandwidth(system):
+    gpu_name = system.get("gpu_name")
     if not isinstance(gpu_name, str) or not gpu_name:
         return None
     gn = gpu_name.lower()
+
+    # Guard against false matches on non-Apple GPUs whose names contain
+    # "m3"/"m4"/"m5" (e.g. NVIDIA Quadro M4 000).
+    if "apple" not in gn:
+        return None
+
+    raw_cores = system.get("gpu_cores")
+    try:
+        gpu_cores = int(raw_cores) if raw_cores is not None else None
+    except (TypeError, ValueError):
+        gpu_cores = None
+
+    for key in _APPLE_VARIANT_KEYS_SORTED:
+        if key not in gn:
+            continue
+        if gpu_cores in APPLE_BANDWIDTH_BY_CORES[key]:
+            return APPLE_BANDWIDTH_BY_CORES[key][gpu_cores]
+        return min(APPLE_BANDWIDTH_BY_CORES[key].values())
+
+    for key in _APPLE_FIXED_KEYS_SORTED:
+        if key in gn:
+            return APPLE_BANDWIDTH_FIXED[key]
+    return None
+
+
+def _lookup_bandwidth(system):
+    if isinstance(system, dict):
+        gpu_name = system.get("gpu_name")
+    else:
+        gpu_name = system
+
+    if not isinstance(gpu_name, str) or not gpu_name:
+        return None
+
+    # Apple tiers live only in the Apple-specific table now (#2564), so route
+    # BOTH dict and bare-string callers through it. A bare string carries no
+    # gpu_cores, so the helper falls back to the conservative (lowest) tier for
+    # that model -- before #2564 the generic table answered string lookups, and
+    # dropping that made _lookup_bandwidth("Apple M3 Max") return None.
+    apple_input = system if isinstance(system, dict) else {"gpu_name": gpu_name}
+    bw = _lookup_apple_bandwidth(apple_input)
+    if bw is not None:
+        return bw
+
+    gn = gpu_name.lower()
     for key in _BW_KEYS_SORTED:
         if key in gn:
             return GPU_BANDWIDTH[key]
@@ -84,7 +144,7 @@ def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0):
     """
     pb = _active_params_b(model)
     is_moe = model.get("is_moe", False)
-    bw = _lookup_bandwidth(system.get("gpu_name"))
+    bw = _lookup_bandwidth(system)
     backend = system.get("backend", "cpu_x86")
 
     if bw and run_mode in ("gpu", "cpu_offload"):

services/hwfit/hardware.py

@@ -1,3 +1,4 @@
+import json
 import os
 import platform
 import re
@@ -335,6 +336,37 @@ def _detect_apple_silicon():
     if total_gb <= 0:
         return None
 
+    def _parse_apple_gpu_cores(text):
+        if not text:
+            return None
+        try:
+            data = json.loads(text)
+        except (TypeError, ValueError, json.JSONDecodeError):
+            data = None
+        if isinstance(data, dict):
+            for gpu in data.get("SPDisplaysDataType") or []:
+                if not isinstance(gpu, dict):
+                    continue
+                model = str(gpu.get("sppci_model") or gpu.get("_name") or "")
+                if "apple" not in model.lower():
+                    continue
+                cores = gpu.get("sppci_cores")
+                try:
+                    return int(str(cores).strip())
+                except (TypeError, ValueError):
+                    continue
+        m = re.search(r"Total Number of Cores:\s*(\d+)", text)
+        if m:
+            try:
+                return int(m.group(1))
+            except ValueError:
+                return None
+        return None
+
+    gpu_cores = _parse_apple_gpu_cores(_run(["system_profiler", "SPDisplaysDataType", "-json"]))
+    if gpu_cores is None:
+        gpu_cores = _parse_apple_gpu_cores(_run(["system_profiler", "SPDisplaysDataType"]))
+
     # Usable GPU budget. macOS lets Metal use most of unified memory, but the
     # default working-set limit scales with RAM: small machines have to keep
     # more back for the OS + app. These fractions track Apple's
@@ -357,7 +389,7 @@ def _detect_apple_silicon():
         pass
 
     gpu = {"index": 0, "name": brand, "vram_gb": vram_gb}
-    return {
+    info = {
         "gpu_name": brand,
         "gpu_vram_gb": vram_gb,
         "gpu_count": 1,
@@ -369,6 +401,9 @@ def _detect_apple_silicon():
         # separate pool — downstream fit logic uses this to avoid double-budgeting.
         "unified_memory": True,
     }
+    if gpu_cores is not None:
+        info["gpu_cores"] = gpu_cores
+    return info
 
 
 def _read_file(path):
@@ -611,6 +646,93 @@ def _cache_key(host: str, ssh_port: str, platform_name: str):
     )
 
 
+def _is_containerized():
+    """Best-effort check for whether the local Odysseus process is running in a container."""
+    if _remote_host:
+        return False
+
+    if os.path.exists("/.dockerenv"):
+        return True
+
+    try:
+        with open("/proc/1/cgroup", encoding="utf-8", errors="replace") as f:
+            text = f.read().lower()
+        return any(marker in text for marker in ("docker", "containerd", "kubepods"))
+    except Exception:
+        return False
+
+
+def _hardware_visibility_warning(result):
+    """Return a non-blocking UX warning when detected hardware may only be container-visible."""
+    if not isinstance(result, dict):
+        return None
+
+    if result.get("manual_hardware"):
+        return None
+
+    if not result.get("containerized"):
+        return None
+
+    if result.get("gpu_error"):
+        return None
+
+    if not result.get("has_gpu"):
+        return {
+            "code": "container_no_gpu_visible",
+            "severity": "warning",
+            "title": "No GPU visible inside Docker",
+            "message": (
+                "Cookbook is scanning hardware from inside the Odysseus container. "
+                "If your host has a GPU, Docker may not be exposing it to the container, "
+                "so model recommendations may be CPU-only or too conservative."
+            ),
+            "actions": [
+                "manual_hardware",
+                "rescan",
+                "copy_diagnostics",
+            ],
+        }
+
+    total_ram = result.get("total_ram_gb") or 0
+    if total_ram and total_ram <= 8:
+        return {
+            "code": "container_low_ram_visible",
+            "severity": "info",
+            "title": "Container-visible RAM may be lower than host RAM",
+            "message": (
+                "Cookbook is seeing the RAM available inside the container. "
+                "If your host has more memory, validate host RAM separately or use Manual Hardware."
+            ),
+            "actions": [
+                "manual_hardware",
+                "rescan",
+                "copy_diagnostics",
+            ],
+        }
+
+    return None
+
+
+def _attach_probe_context(result, host=""):
+    """Attach probe-scope metadata and optional hardware visibility warning."""
+    if not isinstance(result, dict) or result.get("error"):
+        return result
+
+    is_remote = bool(host)
+    containerized = False if is_remote else _is_containerized()
+
+    result["probe_scope"] = "remote" if is_remote else ("container" if containerized else "native")
+    result["containerized"] = containerized
+
+    warning = _hardware_visibility_warning(result)
+    if warning:
+        result["hardware_visibility_warning"] = warning
+    else:
+        result.pop("hardware_visibility_warning", None)
+
+    return result
+
+
 def detect_system(host="", ssh_port="", platform="", fresh=False):
     """Detect system hardware: RAM, CPU, GPU. Cached per host (hardware rarely
     changes, and probing a remote host over SSH is slow). Pass fresh=True to
@@ -635,6 +757,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
     if _remote_platform == "windows" and _remote_host:
         result = _detect_windows()
         if result:
+            result = _attach_probe_context(result, host=host)
             _remote_host = None
             _remote_platform = None
             _cache_by_host[cache_key] = (now, result)
@@ -653,6 +776,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
     if not _remote_host and os.name == "nt":
         result = _detect_windows()
         if result:
+            result = _attach_probe_context(result, host=host)
             _cache_by_host[cache_key] = (now, result)
             return result
         # PowerShell probe failed entirely — fall through to the generic path
@@ -683,6 +807,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
             "gpu_name": gpu_info["gpu_name"],
             "gpu_vram_gb": gpu_info["gpu_vram_gb"],
             "gpu_count": gpu_info["gpu_count"],
+            "gpu_cores": gpu_info.get("gpu_cores"),
             "gpus": gpu_info.get("gpus", []),
             "gpu_groups": gpu_info.get("gpu_groups", []),
             "homogeneous": gpu_info.get("homogeneous", True),
@@ -714,6 +839,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
             "gpu_error": _last_gpu_error,
         }
 
+    result = _attach_probe_context(result, host=host)
     _remote_host = None
     _remote_platform = None
     _cache_by_host[cache_key] = (now, result)

services/hwfit/profiles.py

@@ -188,12 +188,18 @@ def compute_serve_profiles(system, model, serve_weights_gb=None, serve_quant=Non
         # Shrink context if even the chosen KV won't fit alongside weights.
         # Start from the smaller of the profile's target and the model's limit.
         cur_ctx = min(ctx, model_ctx_max)
-        while cur_ctx >= 8192:
+        # Floor the context-shrink loop at 8192, but never above the model's own
+        # trained limit. A model with a sub-8192 context (e.g. a 2048-token
+        # SmolLM) starts below 8192, so a hard-coded 8192 guard skipped the loop
+        # entirely and produced NO profile — the serve UI then fell back to
+        # manual flags even though the model fits the GPU trivially.
+        ctx_floor = min(8192, model_ctx_max)
+        while cur_ctx >= ctx_floor:
             kv = _kv_gb(model, cur_ctx, kv_type)
             n_cpu_moe, fits = _cpu_moe_for_budget(model, quant, kv, budget, fixed_gb=serve_weights_gb)
             est = _weights_gb(model, quant, serve_weights_gb) + kv + 0.6
             # If a non-MoE model can't fit even fully offloaded, try less context.
-            if model.get("is_moe") or fits or cur_ctx <= 8192:
+            if model.get("is_moe") or fits or cur_ctx <= ctx_floor:
                 profiles.append({
                     "key": key,
                     "label": label,

services/memory/skill_extractor.py

@@ -66,41 +66,57 @@ def _has_duplicate_title(skills, title: str) -> bool:
 def _extract_json_object(text: str) -> Optional[dict]:
     """Best-effort extraction of a JSON object from an LLM response.
 
-    The response may be wrapped in code fences or surrounded by prose, and some
-    models emit a stray brace in the prose before the real object
-    (e.g. "uses {placeholder} then {...}"). Slicing first-'{' .. last-'}' then
-    grabs an unparseable span and the skill is silently lost. Try the whole
-    string first, then each '{' start position in turn, returning the first
-    candidate that parses to a JSON object (dict). Returns None if none do.
+    The response may be wrapped in code fences or surrounded by prose. Uses
+    json.JSONDecoder().raw_decode() to locate the boundaries of complete JSON
+    objects starting at each '{' position. Nested objects are filtered out to
+    keep only top-level candidates. If multiple non-overlapping valid JSON
+    objects are found, it is treated as ambiguous and returns None. Otherwise,
+    returns the single valid candidate dictionary.
     """
     if not text:
         return None
     s = text.strip()
     if s.startswith("```"):
         s = s.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
-    end = s.rfind("}")
-    if end == -1:
+
+    decoder = json.JSONDecoder()
+    candidates = []
+
+    start = s.find("{")
+    while start != -1:
+        try:
+            obj, idx = decoder.raw_decode(s[start:])
+            end_pos = start + idx
+            if isinstance(obj, dict):
+                candidates.append((start, end_pos, obj))
+        except (json.JSONDecodeError, ValueError):
+            pass
+        start = s.find("{", start + 1)
+
+    # Filter out nested candidates to identify top-level dictionaries
+    top_level = []
+    for c in candidates:
+        is_nested = False
+        for other in candidates:
+            if other == c:
+                continue
+            if other[0] <= c[0] and c[1] <= other[1]:
+                is_nested = True
+                break
+        if not is_nested:
+            top_level.append(c)
+
+    if not top_level:
         return None
 
-    def _as_dict(candidate):
-        try:
-            obj = json.loads(candidate)
-        except (json.JSONDecodeError, ValueError):
-            return None
-        return obj if isinstance(obj, dict) else None
+    if len(top_level) > 1:
+        logger.debug(
+            "[skill-extract] Found multiple non-overlapping JSON objects: %s",
+            [item[2].get("title") for item in top_level]
+        )
+        return None
 
-    # The clean, common case: the whole (de-fenced) string is the object.
-    obj = _as_dict(s)
-    if obj is not None:
-        return obj
-    # Otherwise scan each '{' candidate up to the last '}'.
-    start = s.find("{")
-    while 0 <= start < end:
-        obj = _as_dict(s[start : end + 1])
-        if obj is not None:
-            return obj
-        start = s.find("{", start + 1)
-    return None
+    return top_level[0][2]
 
 
 async def maybe_extract_skill(

services/memory/skills.py

@@ -603,7 +603,6 @@ class SkillsManager:
         escalation) — those are work-in-progress and pollute the
         prompt with half-finished procedures.
         """
-        active_toolsets = active_toolsets or []
         out = []
         for s in self.load(owner=owner):
             status = s.get("status")
@@ -617,13 +616,16 @@ class SkillsManager:
             # Platform gating
             if platform and s.get("platforms") and platform not in s["platforms"]:
                 continue
-            # requires_toolsets: hide unless every required toolset is active
+            # requires_toolsets: hide unless every required toolset is active.
+            # active_toolsets=None means the caller doesn't know the active
+            # set (API listings, chat preface) — don't gate in that case;
+            # only an explicit list filters.
             req = s.get("requires_toolsets") or []
-            if req and not all(t in active_toolsets for t in req):
+            if req and active_toolsets is not None and not all(t in active_toolsets for t in req):
                 continue
             # fallback_for_toolsets: hide when any of those toolsets is active
             fb = s.get("fallback_for_toolsets") or []
-            if fb and any(t in active_toolsets for t in fb):
+            if fb and active_toolsets and any(t in active_toolsets for t in fb):
                 continue
             out.append({
                 "name": s["name"],

services/search/content.py

@@ -15,6 +15,8 @@ from urllib.parse import urljoin, urlparse
 import httpx
 from bs4 import BeautifulSoup
 
+from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
+
 from .analytics import RateLimitError, error_logger
 from .cache import (
     CONTENT_CACHE_DIR,
@@ -89,18 +91,128 @@ def _public_http_url(url: str) -> bool:
         return False
 
 
-def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5) -> httpx.Response:
+class BodyTooLargeError(Exception):
+    """The server declared a body larger than the hard fetch ceiling."""
+
+    def __init__(self, url: str, declared_bytes: int):
+        self.url = url
+        self.declared_bytes = declared_bytes
+        super().__init__(
+            f"response body is {declared_bytes:,} bytes, over the "
+            f"{WEB_FETCH_HARD_MAX_BYTES:,}-byte hard cap"
+        )
+
+
+class _CappedFetch:
+    """Result of a size-capped streaming GET.
+
+    Carries just what fetch_webpage_content needs from an httpx.Response,
+    plus the cap bookkeeping: the (possibly truncated) body, whether the
+    cap cut it short, and the size the server declared via Content-Length
+    (wire bytes; None when absent).
+    """
+
+    __slots__ = ("status_code", "headers", "content", "truncated",
+                 "declared_bytes", "encoding", "url")
+
+    def __init__(self, status_code, headers, content, truncated,
+                 declared_bytes, encoding, url):
+        self.status_code = status_code
+        self.headers = headers
+        self.content = content
+        self.truncated = truncated
+        self.declared_bytes = declared_bytes
+        self.encoding = encoding
+        self.url = url
+
+    @property
+    def text(self) -> str:
+        return self.content.decode(self.encoding or "utf-8", errors="replace")
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            request = httpx.Request("GET", self.url)
+            raise httpx.HTTPStatusError(
+                f"HTTP {self.status_code} for {self.url}",
+                request=request,
+                response=httpx.Response(self.status_code, request=request),
+            )
+
+
+def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5,
+                    max_bytes: int = None) -> "_CappedFetch":
+    """Capped streaming GET with SSRF-guarded manual redirects.
+
+    The body is streamed and buffering stops at ``max_bytes`` (default: the
+    soft cap), so an oversized resource cannot be pulled into memory or the
+    content cache in full. When Content-Length already declares a body over
+    the hard ceiling, the fetch is refused before any body bytes are read.
+    """
+    cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
     current = url
     for _ in range(max_redirects + 1):
         if not _public_http_url(current):
             raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current))
-        response = httpx.get(current, headers=headers, timeout=timeout, follow_redirects=False)
-        if response.status_code not in (301, 302, 303, 307, 308):
-            return response
-        location = response.headers.get("location")
-        if not location:
-            return response
-        current = urljoin(str(response.url), location)
+        # Force identity transfer-encoding. With gzip/deflate the wire bytes
+        # (and Content-Length) can be a small fraction of the decoded body, so
+        # a tiny compressed response could pass the hard-cap preflight and then
+        # expand past the ceiling in a single decoded chunk before the streamed
+        # cap below can slice it. Identity makes Content-Length the true body
+        # size and keeps each streamed chunk bounded by the network read.
+        req_headers = dict(headers or {})
+        req_headers["Accept-Encoding"] = "identity"
+        with httpx.stream("GET", current, headers=req_headers, timeout=timeout,
+                          follow_redirects=False) as response:
+            if response.status_code in (301, 302, 303, 307, 308):
+                location = response.headers.get("location")
+                if not location:
+                    return _CappedFetch(response.status_code, response.headers, b"",
+                                        False, None, response.encoding, str(response.url))
+                current = urljoin(str(response.url), location)
+                continue
+
+            # A server can ignore the identity request and still return a
+            # compressed body; httpx.iter_bytes would then decode it, and a tiny
+            # gzip can balloon into one decoded chunk far past the cap before we
+            # slice. Refuse a compressed Content-Encoding so the streamed cap
+            # stays a real memory bound (Content-Length is the compressed wire
+            # length here, so the preflight and size metadata are unreliable too).
+            enc = (response.headers.get("content-encoding") or "").strip().lower()
+            if enc and enc != "identity":
+                raise httpx.RequestError(
+                    f"Refusing compressed response (Content-Encoding: {enc}) after "
+                    "requesting identity: cannot bound decoded body size",
+                    request=httpx.Request("GET", current),
+                )
+
+            declared = None
+            raw_len = response.headers.get("content-length")
+            if raw_len and raw_len.isdigit():
+                declared = int(raw_len)
+            # Refuse before buffering anything when the server already tells
+            # us the body exceeds the absolute ceiling (Content-Length is wire
+            # bytes; the decompressed body can only be larger).
+            if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
+                raise BodyTooLargeError(current, declared)
+
+            chunks = []
+            read = 0
+            truncated = False
+            # We requested identity above, so iter_bytes yields the raw body in
+            # network-read-sized chunks (no decompression expansion); the cap
+            # therefore bounds what we actually buffer.
+            for chunk in response.iter_bytes():
+                read += len(chunk)
+                if read > cap:
+                    keep = cap - (read - len(chunk))
+                    if keep > 0:
+                        chunks.append(chunk[:keep])
+                    truncated = True
+                    break
+                chunks.append(chunk)
+            return _CappedFetch(response.status_code, response.headers,
+                                b"".join(chunks), truncated, declared,
+                                response.encoding, str(response.url))
     raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
 
 # PDF extraction (optional dependency)
@@ -222,9 +334,19 @@ def _empty_result(url: str, error: str = "") -> dict:
 # ----------------------------------------------------------------------
 # Main content fetcher
 # ----------------------------------------------------------------------
-def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> dict:
-    """Fetch and extract meaningful content from a webpage with caching."""
-    cache_key = generate_cache_key(url)
+def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
+                          max_bytes: int = None) -> dict:
+    """Fetch and extract meaningful content from a webpage with caching.
+
+    ``max_bytes`` raises the download budget per call (clamped to the hard
+    cap); the default is the soft cap. When the body is cut short the result
+    carries ``truncated``/``fetched_bytes``/``total_bytes`` so callers can
+    tell the model the content is partial (#3812).
+    """
+    effective_cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
+    # The cap is part of the cache identity: a truncated soft-cap fetch must
+    # not be served to a later full-budget request for the same URL.
+    cache_key = generate_cache_key(f"{url}#cap={effective_cap}")
     cache_file = CONTENT_CACHE_DIR / f"{cache_key}.cache"
 
     # Check cache
@@ -247,18 +369,24 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
     # Fetch
     try:
         headers = {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            "User-Agent": WEB_FETCH_USER_AGENT,
             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             "Accept-Language": "en-US,en;q=0.5",
-            "Accept-Encoding": "gzip, deflate",
+            # identity so the streamed size cap in _get_public_url stays honest
+            # (a compressed body can decode to far more than Content-Length).
+            "Accept-Encoding": "identity",
             "Connection": "keep-alive",
         }
-        response = _get_public_url(url, headers=headers, timeout=timeout)
+        response = _get_public_url(url, headers=headers, timeout=timeout,
+                                   max_bytes=effective_cap)
 
         if response.status_code == 429:
             raise RateLimitError(f"Rate limit hit for {url} (attempt {retry_attempt})")
 
         response.raise_for_status()
+    except BodyTooLargeError as e:
+        error_logger.warning(f"Refused oversized body for {url}: {e}")
+        return _empty_result(url, f"TooLarge: {e}")
     except httpx.HTTPStatusError as e:
         error_logger.warning(f"HTTP {e.response.status_code} fetching {url}: {e}")
         return _empty_result(url, f"HTTP {e.response.status_code}: {e}")
@@ -269,9 +397,27 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
         error_logger.error(str(e))
         return _empty_result(url, str(e))
 
+    # Size bookkeeping shared by every content branch below. getattr keeps
+    # plain httpx.Response stand-ins (tests) working without the cap fields.
+    _size_fields = {
+        "truncated": getattr(response, "truncated", False),
+        "fetched_bytes": len(response.content),
+        "total_bytes": getattr(response, "declared_bytes", None),
+    }
+
     # PDF handling
     content_type = response.headers.get("Content-Type", "").lower()
     if "application/pdf" in content_type or url.lower().endswith(".pdf"):
+        if _size_fields["truncated"]:
+            # A PDF cut mid-stream is not parseable; unlike text there is no
+            # useful partial result, so report the budget problem instead.
+            _declared = _size_fields["total_bytes"]
+            return _empty_result(
+                url,
+                f"TooLarge: PDF exceeds the {effective_cap:,}-byte fetch budget"
+                + (f" (size {_declared:,} bytes)" if _declared else "")
+                + "; retry with a larger budget if it fits under the hard cap",
+            )
         if pdf_extract_text is None:
             logger.error("pdfminer.six is not installed; cannot extract PDF text.")
             pdf_text = ""
@@ -295,6 +441,42 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
             "js_message": "",
             "success": bool(pdf_text),
             "error": "" if pdf_text else "Failed to extract PDF text",
+            **_size_fields,
+        }
+        _cache_result(cache_file, cache_key, result, url)
+        return result
+
+    # Plain-text / Markdown / JSON handling. Sources like
+    # raw.githubusercontent.com serve Markdown as `text/plain`, JSON APIs and
+    # raw config files serve `application/json`, and a lot of code and tool
+    # docs live in `.md` / `.txt`. These have no HTML structure, so the HTML
+    # branch below would extract nothing and report "no readable text content".
+    # Return the body verbatim instead. The `is_html` guard keeps real HTML
+    # (including `application/xhtml+xml`) on the parsing path; the `json` check
+    # covers `application/json` and `+json` suffixes; the URL-suffix fallback
+    # catches servers that mislabel text files as `application/octet-stream`.
+    is_html = "html" in content_type
+    is_json = "json" in content_type
+    url_path = url.lower().split("?", 1)[0].split("#", 1)[0]
+    looks_like_text_file = url_path.endswith(
+        (".md", ".markdown", ".txt", ".text", ".json", ".jsonl")
+    )
+    if not is_html and (content_type.startswith("text/") or is_json or looks_like_text_file):
+        text_body = (response.text or "").strip()
+        result = {
+            "url": url,
+            "title": os.path.basename(url_path) or url,
+            "content": text_body,
+            "lists": [],
+            "tables": [],
+            "code_blocks": [],
+            "meta_description": "",
+            "meta_keywords": "",
+            "js_rendered": False,
+            "js_message": "",
+            "success": bool(text_body),
+            "error": "" if text_body else "Empty response body",
+            **_size_fields,
         }
         _cache_result(cache_file, cache_key, result, url)
         return result
@@ -357,6 +539,7 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
         "js_message": js_message,
         "success": True,
         "error": "",
+        **_size_fields,
     }
     _cache_result(cache_file, cache_key, result, url)
     return result

services/search/providers.py

@@ -9,14 +9,12 @@ from urllib.parse import urljoin, urlparse, parse_qs
 import httpx
 from bs4 import BeautifulSoup
 
-from src.constants import SEARXNG_INSTANCE
+from src.constants import SEARXNG_INSTANCE, REQUEST_TIMEOUT, WEB_FETCH_USER_AGENT
 from .analytics import RateLimitError, error_logger
 from .query import build_enhanced_query
 
 logger = logging.getLogger(__name__)
 
-REQUEST_TIMEOUT = 20
-
 # Provider registry — maps setting value to (label, needs_key, needs_url)
 PROVIDER_INFO = {
     "searxng":  ("SearXNG",           False, True),
@@ -140,7 +138,7 @@ def searxng_search_api(query: str, count: Optional[int] = None, categories: str
     count = count if count is not None else _get_result_count()
     instance = _get_search_instance()
     api_key = ""
-    headers = {"User-Agent": "Mozilla/5.0"}
+    headers = {"User-Agent": WEB_FETCH_USER_AGENT}
     if api_key:
         headers["Authorization"] = f"Bearer {api_key}"
     # News/fresh queries do badly in the 'general' category — it favours
@@ -252,7 +250,7 @@ def searxng_search(query, max_results=10):
     """Search using SearXNG instance - parsing HTML."""
     instance = _get_search_instance()
     api_key = ""
-    req_headers = {"User-Agent": "Mozilla/5.0"}
+    req_headers = {"User-Agent": WEB_FETCH_USER_AGENT}
     if api_key:
         req_headers["Authorization"] = f"Bearer {api_key}"
     try:
@@ -391,7 +389,7 @@ def duckduckgo_search(query: str, count: Optional[int] = None, time_filter: Opti
             response = httpx.get(
                 "https://html.duckduckgo.com/html/",
                 params={"q": query, "kp": _safesearch_for("duckduckgo_html")},
-                headers={"User-Agent": "Mozilla/5.0"},
+                headers={"User-Agent": WEB_FETCH_USER_AGENT},
                 timeout=REQUEST_TIMEOUT,
             )
             response.raise_for_status()

services/youtube/youtube_handler.py

@@ -64,20 +64,40 @@ def is_youtube_url(url: str) -> bool:
     return "youtube.com" in url or "youtu.be" in url
 
 
+# youtube.com-shaped hosts. music.youtube.com serves the same /watch and
+# /shorts paths, so links shared from YouTube Music must resolve too.
+_YT_HOSTS = ("www.youtube.com", "youtube.com", "m.youtube.com", "music.youtube.com")
+# Path prefixes whose first following segment is the video id. Covers the
+# /embed/ player, Shorts (/shorts/), live streams (/live/), and the legacy
+# /v/ embed — all of which `is_youtube_url` already treats as YouTube, so
+# they must be extractable or the link is silently dropped (neither web-fetched
+# nor transcript-fetched) by the chat pipeline.
+_YT_PATH_PREFIXES = ("/embed/", "/shorts/", "/live/", "/v/")
+
+
 def extract_youtube_id(url: str) -> Optional[str]:
-    """Extract YouTube video ID from various URL formats."""
+    """Extract a YouTube video ID from the common URL shapes:
+    watch?v=, youtu.be/<id>, /embed/<id>, /shorts/<id>, /live/<id>, /v/<id>,
+    across youtube.com / m.youtube.com / music.youtube.com / youtu.be."""
     if not isinstance(url, str):
         return None
     parsed = urllib.parse.urlparse(url)
-    if parsed.hostname in ("www.youtube.com", "youtube.com", "m.youtube.com"):
+    host = (parsed.hostname or "").lower()
+    if host in _YT_HOSTS:
         if parsed.path == "/watch":
             params = urllib.parse.parse_qs(parsed.query)
-            if "v" in params:
+            if params.get("v"):
                 return params["v"][0]
-        elif parsed.path.startswith("/embed/"):
-            return parsed.path.split("/")[-1]
-    elif parsed.hostname == "youtu.be":
-        return parsed.path[1:]
+        else:
+            for prefix in _YT_PATH_PREFIXES:
+                if parsed.path.startswith(prefix):
+                    vid = parsed.path[len(prefix):].split("/")[0]
+                    if vid:
+                        return vid
+    elif host == "youtu.be":
+        vid = parsed.path.lstrip("/").split("/")[0]
+        if vid:
+            return vid
     return None
 
 
@@ -170,6 +190,8 @@ def format_transcript_for_context(
     if segments:
         ctx += "Timestamped Transcript:\n"
         for seg in segments:
+            if not isinstance(seg, dict):
+                continue
             ctx += f"[{seg['timestamp']}] {seg['text']}\n"
         # Check length — fall back to plain text if too long
         if len(ctx) > 12000:
@@ -202,15 +224,24 @@ async def fetch_youtube_comments(
             f"https://www.youtube.com/watch?v={video_id}",
         ]
 
-        proc = await asyncio.wait_for(
-            asyncio.create_subprocess_exec(
-                *cmd,
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-            ),
-            timeout=timeout,
+        proc = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
         )
-        stdout, stderr = await proc.communicate()
+        # Bound the wait on the process actually finishing, not on spawning it.
+        # create_subprocess_exec returns as soon as the child starts, so wrapping
+        # it in wait_for never enforces the timeout — proc.communicate() is the
+        # blocking step. Kill and reap the child if it overruns so it does not
+        # linger after we return.
+        try:
+            stdout, stderr = await asyncio.wait_for(
+                proc.communicate(), timeout=timeout
+            )
+        except asyncio.TimeoutError:
+            proc.kill()
+            await proc.wait()
+            raise
 
         if proc.returncode != 0:
             return {"success": False, "error": f"yt-dlp failed: {stderr.decode()[:200]}", "comments": []}

specs/architecture-runtime-inventory.md

@@ -0,0 +1,412 @@
+# Architecture Runtime Inventory
+
+> **Purpose**: Phase 0 planning baseline for codebase readability improvements (#4071).
+> **Parent issue**: [#4082](https://github.com/pewdiepie-archdaemon/odysseus/issues/4082)
+> **Last updated**: dev@b58af42 | 2026-06-16
+> **Status**: Draft — to be reviewed before follow-up slices open.
+> **Snapshot basis**: Importer / file / import-line counts are refreshed to `dev@b58af42` (2026-06-16) and are recomputable via the commands in §3.4. **Line counts** in §2.1 / §2.2 are a snapshot from an earlier baseline and drift as `dev` moves — recompute any of them with `wc -l <file>`. This inventory tracks structure and risk, not live metrics.
+
+This document maps the current runtime module structure, identifies high-risk boundaries, and recommends safe first refactor slices. It does **not** move files, change imports, or alter runtime behavior.
+
+---
+
+## 1. Current Structure Overview
+
+### 1.1 Top-Level Layout
+
+```
+odysseus/
+├── app.py                    # FastAPI app entrypoint (1,145 lines)
+├── conf/                     # Configuration (config.py, settings.py, settings_scrub.py)
+├── src/                      # 95 flat .py files + 2 subdirectories
+│   ├── agent_tools/          # Tool helpers: document, filesystem, subprocess, web
+│   └── search/               # Search subsystem
+├── routes/                   # 54 flat .py files — HTTP route handlers
+├── core/                     # 10 files — database models, auth, middleware, session
+├── mcp_servers/              # 5 files — MCP server implementations
+├── scripts/                  # CLI tools and one-shot scripts
+├── static/                   # Frontend HTML/CSS/JS
+├── tests/                    # 583 test files (~54,800 lines)
+└── services/                 # (exists as needed)
+```
+
+### 1.2 Directory Flatness Metric
+
+| Directory | Flat `.py` Files | Subdirectories | Concern |
+|-----------|-----------------|----------------|---------|
+| `src/` | **95** | 2 (`agent_tools/`, `search/`) | No domain grouping; 95 files in one directory |
+| `routes/` | **54** | 0 | All route handlers in one flat directory |
+| `core/` | 10 | 0 | Manageable, but `database.py` is oversized |
+
+---
+
+## 2. Largest Runtime Modules
+
+### 2.1 Python Backend
+
+| Rank | File | Lines | Classes | Functions | Risk |
+|------|------|-------|---------|-----------|------|
+| 1 | `src/tool_implementations.py` | **4,032** | 0 | ~48 | **HIGH** |
+| 2 | `routes/email_routes.py` | **3,245** | — | — | **MEDIUM** |
+| 3 | `routes/cookbook_routes.py` | **2,969** | — | — | **MEDIUM** |
+| 4 | `src/agent_loop.py` | **2,961** | 0 | ~24 | **HIGH** |
+| 5 | `src/task_scheduler.py` | **2,330** | — | 5 | MEDIUM |
+| 6 | `routes/model_routes.py` | **2,266** | — | — | MEDIUM |
+| 7 | `core/database.py` | **2,265** | 28 | ~59 helpers | **HIGH** |
+| 8 | `src/builtin_actions.py` | **2,262** | 2 | ~24 | MEDIUM |
+| 9 | `src/llm_core.py` | **2,164** | — | — | MEDIUM |
+| 10 | `mcp_servers/email_server.py` | 2,197 | — | — | LOW (separate process) |
+| 11 | `src/visual_report.py` | 1,918 | — | — | LOW |
+| 12 | `routes/gallery_routes.py` | 1,896 | — | — | LOW |
+| 13 | `src/ai_interaction.py` | 1,846 | — | — | MEDIUM |
+| 14 | `routes/document_routes.py` | 1,717 | — | — | LOW |
+| 15 | `routes/skills_routes.py` | 1,648 | — | — | LOW |
+
+**Heuristic**: Files > 2,000 lines with 20+ public symbols and many importers are the highest-risk splits. Files 1,000–2,000 lines are medium-risk if tightly coupled.
+
+### 2.2 Frontend
+
+| File | Lines | Concern |
+|------|-------|---------|
+| `static/style.css` | **36,653** | Entire app CSS in one file (tracked separately in #2617) |
+| `static/js/document.js` | **9,776** | Single JS file for document functionality |
+| `static/js/slashCommands.js` | 6,498 | |
+| `static/js/settings.js` | 5,266 | |
+| `static/js/emailLibrary.js` | 5,217 | |
+| `static/js/notes.js` | 5,124 | |
+| `static/js/chat.js` | 4,985 | |
+| `static/app.js` | 4,090 | |
+
+**Note**: Frontend modularization is tracked separately in #2617 (CSS) and is not the focus of this Phase 0 inventory. Frontend is listed here for completeness but follow-up slices should target Python backend boundaries first.
+
+---
+
+## 3. Import Dependency Graph
+
+### 3.1 Who Depends on `core/database.py`
+
+**102 files** import from `core.database` — this is the most depended-upon module:
+
+- All route handlers (`routes/*.py`)
+- Most `src/*.py` files
+- `core/session_manager.py`, `core/auth.py`
+- Multiple test files
+
+**Implication**: Any split of `core/database.py` is the highest-risk refactor. It should be tackled **last**, never first.
+
+### 3.2 Who Depends on `src/tool_implementations.py`
+
+**17 files** import from `src.tool_implementations`:
+- `src/agent_loop.py`, `src/builtin_actions.py`, `src/tool_index.py`
+- `src/task_scheduler.py`, `src/tool_policy.py`
+- Various tests
+
+### 3.3 Who Depends on `src/agent_loop.py`
+
+**22 files** import from `src.agent_loop`:
+
+- `src/tool_policy.py`, `src/teacher_escalation.py`, `src/bg_monitor.py`
+- `src/task_scheduler.py`
+- Multiple test files
+
+### 3.4 Cross-Layer Import Violations
+
+**`src/` importing from `routes/`** (backwards dependency — domain logic depending on HTTP layer):
+
+```
+src/tool_implementations.py ──→ routes/calendar_routes.py
+src/tool_implementations.py ──→ routes/cookbook_helpers.py
+src/tool_implementations.py ──→ routes/email_helpers.py
+src/tool_implementations.py ──→ routes/email_pollers.py
+src/tool_implementations.py ──→ routes/email_routes.py
+src/tool_implementations.py ──→ routes/model_routes.py
+src/tool_implementations.py ──→ routes/note_routes.py
+src/tool_implementations.py ──→ routes/prefs_routes.py
+```
+
+> These are **runtime imports** (inside function bodies, not at module top), which mitigates circular import risk but indicates fuzzy layer boundaries. Function-level inline imports from the HTTP layer into business logic are a code smell.
+
+**Import counts (top-level)**:
+| Direction | Count | Notes |
+|-----------|-------|-------|
+| `routes/` → `src/` | **374** | Expected: HTTP handlers call domain logic |
+| `routes/` → `core/` | **126** | Expected: handlers access DB models |
+| `src/` → `routes/` | **31** | **Unexpected**: domain logic reaching into HTTP layer (direct grep of import lines referencing `routes/`) |
+| `src/` → `core/` | **106** | Acceptable but could be reduced with a data-access layer |
+
+> **How the metrics in this document are computed** — recompute against current `dev` before treating any count as authoritative (the tree drifts; these numbers are a snapshot, not a live value):
+> - `src/` flat `.py` files: `find src -maxdepth 1 -name '*.py' | wc -l`
+> - `tests/` test files: `find tests -name 'test_*.py' | wc -l`
+> - `core.database` importers: `grep -rlE '(from|import) +core\.database' --include='*.py' . | grep -v core/database.py | wc -l`
+> - `src.agent_loop` importers: `grep -rlE '(from|import) +src\.agent_loop' --include='*.py' . | grep -v src/agent_loop.py | wc -l`
+> - Cross-layer import lines: `grep -rhE '(from|import) +<pkg>' --include='*.py' <dir>/ | wc -l` (e.g. `(from|import) +routes` over `src/`)
+
+---
+
+## 4. Route Ownership Map
+
+Routes can be grouped into logical feature domains. Current flat structure obscures these boundaries:
+
+| Domain | Route Files | Total Lines | Review Complexity |
+|--------|-------------|-------------|-------------------|
+| **Email** | `email_routes.py`, `email_helpers.py`, `email_pollers.py` | 5,936 | HIGH — most complex domain |
+| **Chat / Agent** | `chat_routes.py`, `chat_helpers.py`, `shell_routes.py`, `codex_routes.py`, `skills_routes.py` | 6,365 | HIGH — core interaction surface |
+| **Cookbook** | `cookbook_routes.py`, `cookbook_helpers.py`, `cookbook_output.py` | 4,110 | MEDIUM |
+| **Model / LLM** | `model_routes.py`, `assistant_routes.py`, `copilot_routes.py` | 2,764 | MEDIUM |
+| **Calendar / Contacts** | `calendar_routes.py`, `contacts_routes.py` | 2,336 | MEDIUM |
+| **Documents** | `document_routes.py`, `document_helpers.py` | 1,954 | LOW |
+| **Auth** | `auth_routes.py`, `api_token_routes.py`, `device_flow.py` | 1,171 | LOW |
+| **Tasks** | `task_routes.py` (standalone) | 1,157 | LOW |
+| **Session** | `session_routes.py` (standalone) | 1,287 | LOW |
+| **Gallery** | `gallery_routes.py`, `gallery_helpers.py` | 1,896 | LOW |
+| **Memory** | `memory_routes.py` | — | LOW |
+| **Research** | `research_routes.py` | — | LOW |
+| **MCP** | `mcp_routes.py` | — | LOW |
+| **Notes** | `note_routes.py` | — | LOW |
+| **Other** | `prefs_routes.py`, `upload_routes.py`, `vault_routes.py`, `webhook_routes.py`, `workspace_routes.py`, `search_routes.py`, `history_routes.py`, `hwfit_routes.py`, `preset_routes.py`, `signature_routes.py`, `backup_routes.py`, `cleanup_routes.py`, `diagnostics_routes.py`, `embedding_routes.py`, `emoji_routes.py`, `font_routes.py`, `stt_routes.py`, `tts_routes.py`, `compare_routes.py`, `personal_routes.py`, `editor_draft_routes.py`, `admin_wipe_routes.py`, `chatgpt_subscription_routes.py` | 2,000+ | LOW individual, HIGH cumulative |
+
+---
+
+## 5. Tool Registry & Implementation Boundaries
+
+### 5.1 Current Tool Architecture
+
+| Component | File | Lines | Role |
+|-----------|------|-------|------|
+| Tool schemas | `src/tool_schemas.py` | 1,392 | JSON Schema tool definitions (Duck-TypedDict) |
+| Tool index | `src/tool_index.py` | 542 | RAG-based tool retrieval from ChromaDB |
+| Tool implementations | `src/tool_implementations.py` | 4,032 | 33 `do_*` functions — all tool execution logic |
+| Tool security | `src/tool_security.py` | — | Owner-scoped tool blocking |
+| Tool policy | `src/tool_policy.py` | — | Guide-only directive, plan-mode disabled tools |
+| Tool utils | `src/tool_utils.py` | — | Shared tool helpers |
+
+### 5.2 Tool Implementation Categories
+
+The 33 `do_*` functions in `tool_implementations.py` fall into natural domain groups — the basis for slice 1's split in §6.2:
+
+| Category | `do_*` functions | Count |
+|----------|------------------|-------|
+| **System / config** | `do_manage_skills`, `do_manage_tasks`, `do_manage_endpoints`, `do_manage_mcp`, `do_manage_webhooks`, `do_manage_tokens`, `do_manage_settings`, `do_api_call`, `do_app_api` | 9 |
+| **Cookbook / model serving** | `do_download_model`, `do_serve_model`, `do_list_served_models`, `do_stop_served_model`, `do_tail_serve_output`, `do_list_downloads`, `do_cancel_download`, `do_search_hf_models`, `do_adopt_served_model`, `do_list_cookbook_servers`, `do_list_serve_presets`, `do_serve_preset`, `do_list_cached_models` | 13 |
+| **Notes** | `do_manage_notes` | 1 |
+| **Calendar** | `do_manage_calendar` | 1 |
+| **Search** | `do_search_chats` | 1 |
+| **Research** | `do_manage_research`, `do_trigger_research` | 2 |
+| **Contacts** | `do_resolve_contact`, `do_manage_contact` | 2 |
+| **Vault** | `do_vault_search`, `do_vault_get`, `do_vault_unlock` | 3 |
+| **Image** | `do_edit_image` | 1 |
+| | **Total** | **33** |
+
+> Low-level tools (filesystem, subprocess, web fetch, document parsing) live in `src/agent_tools/`, **not** in `tool_implementations.py` — out of scope for this split.
+
+---
+
+## 6. Risk Assessment & Candidate Slice Ranking
+
+> **Candidate proposals, not a committed plan.** The rankings, package shapes (e.g. `src/pkg/`, `src/domain/`, `src/infra/`, `src/api/`), split ordering, and route-grouping strategy below are **options for maintainer discussion**. Per #4082/#4071, slice ownership and order are settled by maintainers before any follow-up PR. §1–§3 above are the factual current-state inventory.
+
+### 6.1 Risk Scale
+
+| Level | Criteria |
+|-------|----------|
+| **LOW** | File has ≤3 importers AND ≤500 lines, OR is a pure refactor with clear boundaries |
+| **MEDIUM** | File has 4–15 importers OR 500–1,500 lines |
+| **HIGH** | File has 16+ importers OR >2,000 lines, OR has cross-layer import violations |
+
+### 6.2 Ranked Split Candidates
+
+| Priority | Target | Risk | Rationale |
+|----------|--------|------|-----------|
+| **1** | `src/tool_implementations.py` → `src/tools/*.py` | **MEDIUM** | 4,032 lines → ~10 files by tool category. Already has natural boundaries. 17 importers, tracked in #3629. Use `__init__.py` shim to keep existing imports working. |
+| **2** | `routes/` → domain subdirectories (one domain per PR) | **MEDIUM** | 54 flat files. Done **one domain at a time** (e.g. a standalone PR for the email domain, then chat, …), not a broad reorganization — route modules carry helper imports, registration assumptions, and test import paths. |
+| **3** | `src/agent_loop.py` → `src/agent/loop.py` + submodules | **MEDIUM-HIGH** | 2,961 lines, 24 functions. Can extract prompt building, classification, verification, and runaway detection. Tracked in #3266. |
+| **4** | `src/` → `src/pkg/`, `src/domain/`, `src/infra/`, `src/api/` | **MEDIUM** | Structural reorganization. Split flat `src/` into layered packages. Must come after routes and tools are stable. |
+| **5** | `routes/email_*.py` consolidation | **LOW** | Already grouped by filename prefix. Low-risk cleanup within the email domain. |
+| **6** | `core/database.py` → `src/infra/database/models/*.py` | **HIGH** | 28 classes, 102 importers. Highest-risk split. Must be **last** in any sequence. Requires careful import shim strategy. |
+| **7** | Frontend CSS modularization | **MEDIUM** | 36,653 lines. Tracked in #2617. Separate timeline from backend work. |
+| **8** | Frontend JS modularization | **MEDIUM** | 9,776 lines in `document.js`. Introduce ES modules at minimum. |
+
+### 6.3 Candidate First 3 Behavior-Preserving Slices
+
+**Slice 1: Split `tool_implementations.py`** (Lowest-risk high-impact)
+
+- Create `src/tools/` package with one file per tool category
+- Add `src/tools/__init__.py` re-exporting all symbols with current names
+- Update 17 importers to use new paths (can be deferred via shim)
+- Validation: `python -m pytest tests/ -x -q` + manual smoke test of tool execution
+- Reference: #3629
+
+**Slice 2: Group `routes/` by domain** (one domain per PR, not a broad sweep)
+
+Route modules carry helper imports, router registration assumptions, and test import paths, so this must be done **one domain at a time** rather than as a single reorganization PR. Example sequence (each its own PR):
+
+- PR 2a: move the **email** domain (`email_routes.py`, `email_helpers.py`, `email_pollers.py`) → `routes/email/` + shim
+- PR 2b: move the **chat/agent** domain → `routes/chat/` + shim
+- PR 2c: move the **cookbook** domain → `routes/cookbook/` + shim
+- …and so on per domain from §4
+
+Each PR: add `__init__.py` re-exporting old names, update `app.py` router imports, validation `python app.py` starts clean. **No behavior change** — pure file reorganization.
+
+**Slice 3: Extract `agent_loop.py` submodules** (Improve reviewability)
+
+- Move prompt assembly → `src/agent/prompt.py`
+- Move request classification → `src/agent/classifier.py`
+- Move sub-agent verification → `src/agent/verifier.py`
+- Move runaway detection → `src/agent/runaway.py`
+- Move context management → `src/agent/context.py`
+- Keep `src/agent/loop.py` as the main orchestration module
+- Validation: `python -m pytest tests/test_agent_loop.py tests/test_loop_breaker_runaway.py -v`
+
+---
+
+## 7. Safety Guardrails for Follow-Up Work
+
+Per maintainer guidance in #4082 and #4071:
+
+- [ ] **One domain/slice per PR** — never mix multiple reorganizations
+- [ ] **No behavior changes** mixed with file moves — pure reorganization only
+- [ ] **Keep compatibility shims** — `__init__.py` re-exports for all existing import paths
+- [ ] **Add or identify focused tests** before risky splits
+- [ ] **Do not start with `core/database.py`** or broad route movement unless this inventory shows a safe boundary
+- [ ] **Prefer small, reviewable slices** over large restructures
+- [ ] **No packaging/runtime/tooling migration** mixed into file moves
+- [ ] **No frontend framework migration** inside this stabilization lane
+- [ ] **Validate with `python -m compileall`** — every PR must pass CI checks
+- [ ] **Validate with `pytest`** — run the full test suite before opening each PR
+
+---
+
+## 8. Validation Commands
+
+Each follow-up PR should be verifiable with these commands before submission:
+
+```bash
+# Syntax check — must pass with zero errors
+python -m compileall src/ routes/ core/ conf/
+
+# Full test suite — must match baseline pass rate
+python -m pytest tests/ -x -q
+
+# Import shim verification — existing import paths must still work
+python -c "from src.tool_implementations import do_search_chats; print('OK')"
+
+# App startup smoke test (if backend touched)
+timeout 5 python app.py 2>&1 | head -5 || true
+```
+
+---
+
+## 9. Open Questions
+
+1. Is `#2538` (specs ground truth) the canonical behavior map baseline, and should this inventory be kept in sync with those specs once merged?
+2. Should route grouping follow the domain map proposed here, or is there a different taxonomy preferred by maintainers?
+3. For the `tool_implementations.py` split (#3629), is the tool categorization in §5.2 acceptable, or should it follow a different grouping?
+4. Should compatibility shims (`__init__.py`) be temporary (removed in a follow-up wave) or permanent?
+5. Should an ADR (Architecture Decision Record) document be started to track decisions made during this process?
+
+---
+
+## 10. Future Direction (NOT current state)
+
+The following are **future refactor targets** (candidate directions **pending maintainer agreement**, not committed), recorded here so this inventory does not imply they exist today. None of them are present in the current `dev` tree:
+
+- `main.py` — proposed rename of the `app.py` entrypoint. Today the app boots via `app.py`.
+- `src/agent/` — proposed package to hold `agent_loop.py` submodules (prompt/classifier/verifier/runaway/context). Today `agent_loop.py` is a single flat file in `src/`.
+- `src/infra/`, `src/domain/`, `src/pkg/`, `src/api/` — proposed layered reorganization of the flat `src/` directory (slice 4 in §6).
+
+These become real only when the corresponding slices land.
+
+---
+
+## Appendix A: File Listing
+
+### `src/` (95 files — 61 shown; run `ls src/*.py` for the full list)
+
+```
+agent_loop.py          tool_implementations.py   tool_schemas.py
+tool_index.py          tool_security.py          tool_policy.py
+tool_utils.py          builtin_actions.py        task_scheduler.py
+llm_core.py            model_context.py          model_discovery.py
+session_search.py      context_budget.py         context_compactor.py
+ai_interaction.py      action_intents.py         agent_runs.py
+app_helpers.py         app_initializer.py        config.py
+database.py            memory.py                 memory_provider.py
+secret_storage.py      prompt_security.py        url_security.py
+url_safety.py          rate_limiter.py           cleanup_service.py
+readiness.py           service_health.py         exceptions.py
+request_models.py      assistant_log.py          bg_monitor.py
+builtin_mcp.py         chat_helpers.py           chroma_client.py
+document_processor.py  embedding_lanes.py        deep_research.py
+research_handler.py    research_utils.py         personal_docs.py
+rag_manager.py         rag_singleton.py          topic_analyzer.py
+visual_report.py       youtube_handler.py        pdf_forms.py
+pdf_form_doc.py        pdf_runtime.py            caldav_writeback.py
+email_thread_parser.py text_helpers.py           user_time.py
+teacher_escalation.py  cookbook_serve_lifecycle.py
+chatgpt_subscription.py  mcp_manager.py
+```
+
+### `routes/` (54 files)
+
+```
+__init__.py    _validators.py
+auth_routes.py              api_token_routes.py       device_flow.py
+chat_routes.py              chat_helpers.py           shell_routes.py
+codex_routes.py             skills_routes.py
+email_routes.py             email_helpers.py          email_pollers.py
+cookbook_routes.py          cookbook_helpers.py       cookbook_output.py
+model_routes.py             assistant_routes.py       copilot_routes.py
+calendar_routes.py          contacts_routes.py
+document_routes.py          document_helpers.py
+gallery_routes.py           gallery_helpers.py
+task_routes.py              session_routes.py
+note_routes.py              memory_routes.py          research_routes.py
+mcp_routes.py               search_routes.py          history_routes.py
+webhook_routes.py           workspace_routes.py       upload_routes.py
+vault_routes.py             prefs_routes.py           preset_routes.py
+signature_routes.py         personal_routes.py        hwfit_routes.py
+backup_routes.py            cleanup_routes.py         diagnostics_routes.py
+embedding_routes.py         emoji_routes.py           font_routes.py
+stt_routes.py               tts_routes.py             compare_routes.py
+editor_draft_routes.py      chatgpt_subscription_routes.py    admin_wipe_routes.py
+```
+
+### `core/` (10 files)
+
+```
+__init__.py    constants.py    database.py    models.py
+auth.py        middleware.py   session_manager.py   exceptions.py
+atomic_io.py   platform_compat.py
+```
+
+---
+
+## Appendix B: Key Import Relationships
+
+```
+core/database.py  ←── 102 importers (routes/*, src/*, core/*, tests/*)
+    ↑
+    ├── routes/auth_routes.py
+    ├── routes/email_routes.py
+    ├── src/builtin_actions.py
+    ├── src/task_scheduler.py
+    ├── src/tool_implementations.py (inline)
+    └── ...97 more
+
+src/tool_implementations.py  ←── 17 importers
+    ↑
+    ├── src/agent_loop.py
+    ├── src/builtin_actions.py
+    ├── src/tool_index.py
+    ├── src/task_scheduler.py
+    ├── src/tool_policy.py
+    └── ...12 more (mostly tests)
+
+src/agent_loop.py  ←── 22 importers
+    ↑
+    ├── src/tool_policy.py
+    ├── src/teacher_escalation.py
+    ├── src/bg_monitor.py
+    ├── src/task_scheduler.py
+    └── 18 more (incl. tests)
+```

src/action_intents.py

@@ -91,6 +91,9 @@ _ROUTING_PATTERNS: tuple[tuple[str, str, Pattern[str]], ...] = tuple(
         ("ui", "tool or feature toggle request", r"\b(?:disable|enable|turn\s+(?:on|off))\s+(?:the\s+)?(?:shell|search|web|browser|documents?|memory|skills|images?|calendar|email|mail|research|incognito)\b"),
 
         # Deep research jobs, not quick conceptual mentions of research.
+        ("web", "explicit web search request", rf"{_PLEASE}(?:do|run|use|perform|make)\s+(?:a\s+)?(?:web\s+search|search\s+the\s+web)\b.+"),
+        ("web", "web lookup imperative request", rf"{_PLEASE}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google)\b.+"),
+        ("web", "assistant web lookup request", rf"{_ACTION_QUESTION}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google)\b.+"),
         ("research", "deep research imperative request", rf"{_PLEASE}(?:research|deep\s+dive|look\s+into|investigate)\s+.+"),
         ("research", "assistant deep research request", rf"{_ACTION_QUESTION}(?:research|do\s+research|deep\s+dive|look\s+into|investigate)\s+.+"),
 

src/agent_loop.py

@@ -21,7 +21,7 @@ from src.settings import get_setting
 from src.prompt_security import untrusted_context_message
 from src.tool_security import blocked_tools_for_owner, plan_mode_disabled_tools
 from src.tool_policy import GUIDE_ONLY_DIRECTIVE, ToolPolicy
-from src.tool_utils import get_mcp_manager
+from src.tool_utils import _truncate, get_mcp_manager
 from src.agent_tools import (
     parse_tool_blocks,
     strip_tool_blocks,
@@ -262,6 +262,11 @@ _DOMAIN_RULES = {
 - Use `manage_settings` for preferences and tool enable/disable.
 - Use named tools over `app_api` when a named wrapper exists.
 - `app_api` is only for safe UI/API actions without a named tool; do not use it for shell, package installs, engine rebuilds, or sensitive auth/admin paths.""",
+    "contacts": """\
+## Contacts rules
+- Use `resolve_contact` to look up a contact's email or phone number by name. Searches the CardDAV address book and sent email history.
+- Use `manage_contact` to list, add, update, or delete contacts in the address book.
+- Do NOT use `manage_memory` for contact lookups — contact details live in the address book, not memory.""",
 }
 
 _DOMAIN_TOOL_MAP = {
@@ -272,8 +277,9 @@ _DOMAIN_TOOL_MAP = {
     "notes_calendar_tasks": {"manage_notes", "manage_calendar", "manage_tasks"},
     "ui": {"ui_control"},
     "sessions": {"create_session", "list_sessions", "manage_session", "send_to_session", "search_chats"},
-    "files": {"bash", "python", "read_file", "write_file", "edit_file", "grep", "glob", "ls"},
+    "files": {"bash", "python", "read_file", "write_file", "edit_file", "grep", "glob", "ls", "get_workspace"},
     "settings": {"manage_settings", "manage_endpoints", "manage_mcp", "manage_webhooks", "manage_tokens", "app_api"},
+    "contacts": {"resolve_contact", "manage_contact"},
 }
 
 def _domain_rules_for_tools(tool_names: set) -> list[str]:
@@ -309,6 +315,7 @@ NEVER pipe multi-line Python through `python -c "..."` — shell quoting eats re
 <python code>
 ```
 Execute Python code. Use for computation, data processing, scripting. NOT for writing code for the user (use create_document for that). Same sandbox limits as bash — no TTY, no GUI, no `input()`; for anything the user should interact with, generate a single HTML file with inline JS instead.
+Prefer a dedicated tool whenever one fits the job (reading, searching, or writing files); use python only for computation/processing no dedicated tool covers - not for reading or writing files.
 Do NOT use Python/requests for web lookup/search/latest/current requests when `web_search` or `web_fetch` is available.""",
 
     "web_search": """\
@@ -347,6 +354,11 @@ Write content to a file. First line is the path, rest is the content.""",
 ```
 Edit an EXISTING file by exact string replacement. PREFER this over bash (sed/echo/redirects) for changing files — it shows a before/after diff. `old_string` must match the file exactly and be unique unless `replace_all` is true. Use write_file to create a new file.""",
 
+    "get_workspace": """\
+```get_workspace
+```
+Return the absolute path of the active workspace folder. File tools are CONFINED to it (paths can be RELATIVE to it); the shell starts there (cwd) but is NOT sandboxed. Call this first when the user says "the project"/"the code"/"this folder" without a path, instead of asking them. No arguments.""",
+
     "create_document": """\
 ```create_document
 <title>
@@ -396,7 +408,7 @@ Generate an image. Line 1 = description, line 2 = model name, line 3 = WxH (e.g.
     "ask_teacher": "- ```ask_teacher``` — Escalate a hard question to a more capable model. Line 1 = model name or 'auto', rest = the question. Use when stuck or need expert knowledge.",
     "list_models": "- ```list_models``` — Show all available AI models across all endpoints. Use when user asks what models are available.",
     "manage_session": "- ```manage_session``` — Rename, archive, delete, fork, switch, or `list` chats (the UI calls them 'chats'; 'session' is internal). Line 1 = action (list/switch/rename/archive/unarchive/delete/important/unimportant/truncate/fork), Line 2 = exact chat id from `list_sessions` (or `current` where supported). For delete/archive/truncate, always list first and reuse the exact id; never invent placeholder ids. `switch`/`open` returns a clickable anchor link the user can tap to open the chat — use for \"open my X chat\".",
-    "manage_memory": "- ```manage_memory``` — Manage the user's persistent memory (facts, identity, preferences, context that persists across chats). Line 1 = action (list/add/edit/delete/search), rest = content. Use when user says 'remember this', states identity facts like 'my name is <name>' / 'call me <name>' / 'I live in <place>', or asks about stored memories.",
+    "manage_memory": "- ```manage_memory``` — Manage the user's persistent memory (facts about the USER themselves, their preferences, context that persists across chats). Line 1 = action (list/add/edit/delete/search), rest = content. Use when user says 'remember this' about themselves, states identity facts like 'my name is <name>' / 'call me <name>' / 'I live in <place>', or asks about stored memories. DO NOT use for info about another person (their address, phone, email, birthday) — that goes in `manage_contact`. If the user pastes an address/phone with a name and says 'save this for <person>', use `manage_contact add` with the address arg, NOT manage_memory.",
     "manage_skills": "- ```manage_skills``` — Skill registry (SKILL.md format). Args (JSON): {\"action\": \"list|view|view_ref|search|add|edit|patch|publish|delete\", ...}. `list` returns the index of available skills (published + teacher-escalation drafts); `view name=foo` fetches the full SKILL.md; `view_ref name=foo path=...` loads a reference file under the skill directory. For `add`, provide an explicit kebab-case `name` and only report the exact returned name, because storage may normalize or dedupe it. Use this BEFORE doing domain work — there may already be a procedure (published or draft) that prescribes the correct steps. Drafts written by the teacher loop are authoritative guidance even though they're not yet published.",
     "manage_tasks": "- ```manage_tasks``` — Create and manage scheduled background tasks (recurring AI jobs). Args (JSON): {\"action\": \"list|create|edit|delete|pause|resume|run\", ...}",
     "manage_endpoints": "- ```manage_endpoints``` — Add, remove, or configure AI model API endpoints. Args (JSON): {\"action\": \"list|add|delete|enable|disable\", ...}. Use when user wants to add a new AI provider.",
@@ -416,7 +428,9 @@ Notes, checklists, AND user reminders. Use this for "create/add/write a note", t
 ```send_email
 {"to": "recipient@example.com", "subject": "Re: Your question", "body": "Hi, ...", "account": "gmail"}
 ```
-Send a new email via SMTP. Use `resolve_contact` first if you only have a name. If multiple email accounts exist, call `list_email_accounts` first and pass the chosen `account`.""",
+Send a new email via SMTP. Use `resolve_contact` first if you only have a name. If multiple email accounts exist, call `list_email_accounts` first and pass the chosen `account`.
+
+CRITICAL — signatures: DO NOT invent a sign-off name. End the body with just `Thanks,` or similar — never type a person's name unless the user explicitly told you what to sign as. When `agent_email_confirm` is on (default), the tool returns `{pending: true, pending_id: ...}` and stages the email for the user to approve in the chat UI instead of SMTPing immediately.""",
     "list_emails": """\
 ```list_emails
 {"folder": "INBOX", "max_results": 20, "unread_only": false, "account": "gmail"}
@@ -427,7 +441,9 @@ List recent emails from a folder, newest first, including read messages by defau
 ```reply_to_email
 {"uid": "1234", "body": "Sounds good — talk Friday.", "account": "gmail"}
 ```
-SEND a reply email immediately by UID. Do not use this for "open a reply" or "start a reply" — those should use `ui_control` with `open_email_reply <uid> <folder> reply` to open the email draft document. For follow-up requests like "reply ..." after reading/listing email where the user clearly wants to send now, use the exact UID and account from the latest `read_email`/`list_emails` result. Never invent UID `1`. Threads automatically (In-Reply-To/References handled).""",
+SEND a reply email immediately by UID. Do not use this for "open a reply" or "start a reply" — those should use `ui_control` with `open_email_reply <uid> <folder> reply` to open the email draft document. For follow-up requests like "reply ..." after reading/listing email where the user clearly wants to send now, use the exact UID and account from the latest `read_email`/`list_emails` result. Never invent UID `1`. Threads automatically (In-Reply-To/References handled).
+
+CRITICAL — signatures: DO NOT invent a sign-off name. End the body with just `Thanks,` or similar — never type a person's name unless the user explicitly told you what to sign as. When `agent_email_confirm` is on (default), the tool returns `{pending: true, pending_id: ...}` and stages the email for the user to approve in the chat UI instead of SMTPing immediately.""",
     "bulk_email": """\
 ```bulk_email
 {"action": "delete", "uids": ["10997", "10998"], "folder": "INBOX", "account": "Gmail"}
@@ -437,7 +453,7 @@ Bulk delete/archive/mark emails. Use this for "delete all those" after listing e
     "archive_email": "- ```archive_email``` — Archive one email by UID. Args (JSON): {\"uid\":\"...\", \"folder\":\"INBOX\", \"account\":\"Gmail\"}. For multiple messages use bulk_email.",
     "mark_email_read": "- ```mark_email_read``` — Mark one email read/unread. Args (JSON): {\"uid\":\"...\", \"read\":true, \"folder\":\"INBOX\", \"account\":\"Gmail\"}. For multiple messages use bulk_email.",
     "resolve_contact": "- ```resolve_contact``` — Look up a contact's email by name. Searches CardDAV address book + sent email history. Args (JSON): {\"name\": \"...\"}. Use BEFORE send_email when the user gives only a name.",
-    "manage_contact": "- ```manage_contact``` — Create/update/delete/list CardDAV contacts. Args (JSON): {\"action\": \"list|add|update|delete\", \"name\": \"...\", \"email\": \"...\", \"uid\": \"...\"}. Use only for explicit address-book/contact requests with contact details. Do NOT use for user identity facts like 'my name is <name>'; save those with manage_memory. For update/delete, call action=list first to get the uid.",
+    "manage_contact": "- ```manage_contact``` — Create/update/delete/list CardDAV contacts. Args (JSON): {\"action\": \"list|add|update|delete\", \"name\": \"...\", \"email\": \"...\", \"phones\": [...], \"address\": \"...\", \"uid\": \"...\"}. Use for info about another person: email, phone, postal address. For 'save this for <person>' / address paste / phone next to a name, use this — NOT manage_memory. Do NOT use for user identity facts ('my name is X'); those are manage_memory. For update/delete, call action=list first for the uid.",
     "manage_calendar": """\
 ```manage_calendar
 {"action": "create_event", "summary": "<event title>", "dtstart": "<natural language or ISO datetime>"}
@@ -508,7 +524,7 @@ def get_builtin_overrides() -> dict:
         ov = get_setting("builtin_tool_overrides", {})
         return ov if isinstance(ov, dict) else {}
     except Exception as e:
-        logger.warning('Failed to load builtin tool overrides: %s', e)
+        logger.warning("Failed to load builtin tool overrides, using defaults", exc_info=e)
         return {}
 
 
@@ -594,7 +610,7 @@ _API_HOSTS = frozenset([
     "api.deepseek.com", "deepseek.com",
     "api.together.xyz", "api.fireworks.ai",
     "api.perplexity.ai", "api.x.ai",
-    "ollama.com", "api.venice.ai",
+    "ollama.com", "api.venice.ai", "api.kimi.com",
     "api.githubcopilot.com",
     # Local OpenAI-compatible endpoints (llama.cpp, vLLM, LM Studio, etc.).
     # Without these, `_is_api_model` falls back to keyword sniffing on the
@@ -781,6 +797,12 @@ def _classify_agent_request(messages: List[Dict], last_user: str) -> Dict[str, o
         domains.add("documents")
     if has(r"\b(search|web|google|look up|latest|news|current|weather|forecast|stock price|price of|website|url|https?://|www\.)\b"):
         domains.add("web")
+    if has(
+        r"\b(wyszukaj|wyszukać|wyszukac)\b.*\b(internet|internecie|online|web)\b",
+        r"\b(sprawd[zź]|znajd[zź])\b.*\b(internet|internecie|online|web)\b",
+        r"\b(aktualn\w*|bieżąc\w*|biezac\w*|dzisiaj|teraz)\b.*\b(pogod\w*|temperatur\w*)\b",
+    ):
+        domains.add("web")
     if has(r"\b(research|deep dive|investigate|look into)\b"):
         domains.add("web")
     if has(r"\b(open|show|toggle|turn on|turn off|disable|enable|switch model|change model|settings|theme|panel)\b"):
@@ -791,6 +813,8 @@ def _classify_agent_request(messages: List[Dict], last_user: str) -> Dict[str, o
         domains.add("files")
     if has(r"\b(endpoint|api token|mcp|webhook|preference|configure|config|setting)\b"):
         domains.add("settings")
+    if has(r"\b(contact|contacts|phone|phone number|address book|vcard)\b"):
+        domains.add("contacts")
 
     low_signal = not continuation and not domains
     return {
@@ -839,6 +863,7 @@ def _build_system_prompt(
     compact: bool = False,
     owner: Optional[str] = None,
     suppress_local_context: bool = False,
+    active_email: Optional[Dict[str, str]] = None,
 ) -> List[Dict]:
     """Build agent system prompt, inject MCP/document context, merge consecutive system msgs."""
     global _cached_base_prompt, _cached_base_prompt_key
@@ -904,8 +929,8 @@ def _build_system_prompt(
     try:
         from src.user_time import current_datetime_context_message
         _datetime_message = current_datetime_context_message()
-    except Exception:
-        pass
+    except Exception as e:
+        logger.warning("Failed to build datetime context message", exc_info=e)
 
     # Document context is kept as a SEPARATE message (not merged into the tool
     # prompt) so the context trimmer doesn't destroy it when truncating the
@@ -948,8 +973,8 @@ def _build_system_prompt(
             try:
                 from src.pdf_form_doc import find_source_upload_id
                 _is_form_backed = bool(find_source_upload_id(active_document.current_content or ""))
-            except Exception:
-                pass
+            except Exception as e:
+                logger.warning("Failed to detect if document is form-backed, assuming plain", exc_info=e)
 
             if _is_form_backed:
                 doc_ctx = (
@@ -1031,6 +1056,66 @@ def _build_system_prompt(
     else:
         set_active_document(None)
 
+    # Active email reader — frontend told us the user has an email open.
+    # Inject a context block so "reply", "summarize this", "what does it say"
+    # resolve to the real UID instead of the agent inventing a fresh .md
+    # draft with fake headers. This is the email equivalent of _doc_message.
+    _email_message = None
+    if active_email and active_email.get("uid"):
+        _em_uid = active_email.get("uid", "")
+        _em_folder = active_email.get("folder", "INBOX")
+        _em_account = active_email.get("account", "")
+        _em_subject = active_email.get("subject", "") or "(no subject)"
+        _em_from = active_email.get("from", "") or "(unknown sender)"
+        _em_preview = (active_email.get("body_preview", "") or "").strip()
+        _preview_block = f"\nBody preview:\n```\n{_em_preview[:1800]}\n```" if _em_preview else ""
+        _acct_arg = f" {_em_account}" if _em_account else ""
+        email_ctx = (
+            f"ACTIVE EMAIL OPEN (the user has this email open in a reader window right now)\n"
+            f"UID: {_em_uid}\n"
+            f"Folder: {_em_folder}\n"
+            f"Account: {_em_account or '(default)'}\n"
+            f"From: {_em_from}\n"
+            f"Subject: {_em_subject}{_preview_block}\n\n"
+            f"CRITICAL DEFAULT — every request about email this turn refers to "
+            f"THIS email unless the user names a DIFFERENT specific recipient "
+            f"(a name, an email address, or another thread). Examples that "
+            f"ALL mean reply-to-the-open-email:\n"
+            f"  • 'reply' / 'reply to this' / 'respond'\n"
+            f"  • 'write email saying X' / 'send email saying X' / 'draft something'\n"
+            f"  • 'tell them X' / 'say hi' / 'thanks' / 'ack' / 'lmk'\n"
+            f"  • 'summarize it' / 'what does it say' / 'tldr'\n"
+            f"  • 'forward this' / 'forward to <addr>'\n"
+            f"DO NOT ASK THE USER 'who do you want to send this to?' — the "
+            f"answer is ALWAYS the sender of the open email (above) unless they "
+            f"named someone else. Asking that is the wrong move every time.\n\n"
+            f"RULES for the open email:\n"
+            f"1. DRAFT a reply (default for any 'write/send/reply/tell them' "
+            f"request without a different recipient): call `ui_control` with "
+            f"`action=\"open_email_reply\"` and `extra=\"{_em_uid} {_em_folder} "
+            f"reply\"`. This opens the proper reply doc with To/Subject/"
+            f"In-Reply-To pre-filled by the backend. The user will see and edit "
+            f"it before sending. DO NOT `create_document` a markdown file with "
+            f"hand-written `To:` / `Subject:` / `In-Reply-To:` headers — that "
+            f"is wrong every time.\n"
+            f"2. SEND a reply immediately (skip the draft): call "
+            f"`reply_to_email` with the UID above. Only do this when the user "
+            f"explicitly says 'send' / 'send the reply' / 'reply and send'.\n"
+            f"3. READ the full body (the preview above may be truncated): "
+            f"call `read_email` with the UID/folder/account above.\n"
+            f"4. SUMMARIZE / answer questions about it: read it first, then "
+            f"answer in chat. Don't create a document for a summary unless "
+            f"the user explicitly asks for one.\n"
+            f"5. Never ask the user to paste the email or 'share it with you' "
+            f"— you already have its identity above and can read the full body.\n"
+            f"6. The ONLY time you ask 'who to send to?' is when the user "
+            f"explicitly says 'send a NEW email to someone else' or names a "
+            f"recipient you can't identify. A bare 'send email saying X' = the "
+            f"open email's sender.\n"
+        )
+        _email_message = untrusted_context_message("active email reader", email_ctx)
+        _email_message["_protected"] = True
+
     # Inject writing style for any email writing path. This is deliberately
     # broader than read/list: models may compose via send_email, reply_to_email,
     # or ui_control open_email_reply after the first tool round.
@@ -1238,6 +1323,9 @@ def _build_system_prompt(
     if _doc_message:
         merged.insert(last_user_idx, _doc_message)
         last_user_idx += 1  # the document message is now at last_user_idx
+    if _email_message:
+        merged.insert(last_user_idx, _email_message)
+        last_user_idx += 1
     if _skills_message:
         merged.insert(last_user_idx, _skills_message)
         last_user_idx += 1
@@ -1272,12 +1360,18 @@ def _build_base_prompt(
     from src.tool_index import ALWAYS_AVAILABLE
 
     disabled = set(disabled_tools or [])
-    if not get_setting("image_gen_enabled", True):
+    if not get_setting("image_gen_enabled", False):
         disabled.add("generate_image")
 
     if relevant_tools is not None:
-        # RAG mode: include always-available + retrieved + admin (if needed)
-        tool_names = set(ALWAYS_AVAILABLE) | set(relevant_tools)
+        # RAG mode: trust the relevant_tools set as already-composed.
+        # get_tools_for_query starts from ALWAYS_AVAILABLE and may
+        # *discard* tools that conflict with the query's intent (e.g.
+        # drop manage_memory for clear contact-save patterns). Unioning
+        # ALWAYS_AVAILABLE back in here used to silently undo those
+        # drops. Only force-include the irreducible loop primitives
+        # (ask_user, update_plan) as belt-and-suspenders.
+        tool_names = set(relevant_tools) | {"ask_user", "update_plan"}
         if needs_admin:
             tool_names |= _ADMIN_TOOLS
         agent_prompt = _assemble_prompt(tool_names, disabled, compact=compact)
@@ -1718,6 +1812,7 @@ async def stream_agent_loop(
     max_tool_calls: int = 0,
     context_length: int = 0,
     active_document=None,
+    active_email: Optional[Dict[str, str]] = None,
     session_id: Optional[str] = None,
     disabled_tools: Optional[Set[str]] = None,
     owner: Optional[str] = None,
@@ -1726,6 +1821,7 @@ async def stream_agent_loop(
     plan_mode: bool = False,
     approved_plan: Optional[str] = None,
     tool_policy: Optional[ToolPolicy] = None,
+    workspace: Optional[str] = None,
     _is_teacher_run: bool = False,
 ) -> AsyncGenerator[str, None]:
     """Streaming agent loop generator.
@@ -1794,8 +1890,21 @@ async def stream_agent_loop(
         logger.info(f"[tool-rag] Using caller-provided relevant_tools ({len(_relevant_tools)} tools)")
     if not guide_only and not _relevant_tools and bool(_intent.get("low_signal")):
         from src.tool_index import ALWAYS_AVAILABLE
-        _relevant_tools = set(ALWAYS_AVAILABLE)
-        logger.info("[tool-rag] Low-signal agent message; skipping retrieval and using always-available tools only")
+        if workspace:
+            # An active workspace IS the file-work signal: a vague "look at the
+            # project" means explore this folder. Surface only the READ-ONLY file
+            # tools (intersection with the plan-mode read-only allowlist) so the
+            # agent can investigate; write/shell tools stay out until the request
+            # actually calls for them (RAG retrieval adds those on a real ask).
+            _relevant_tools = set(ALWAYS_AVAILABLE)
+            from src.tool_security import PLAN_MODE_READONLY_TOOLS
+            _relevant_tools |= (_DOMAIN_TOOL_MAP["files"] & PLAN_MODE_READONLY_TOOLS)
+            logger.info("[tool-rag] Low-signal but workspace active; including read-only file tools")
+        else:
+            # Don't short-circuit: fall through to RAG retrieval below.
+            # Non-English queries are flagged low_signal by the English-only
+            # intent classifier, but fastembed retrieval works across languages.
+            logger.info("[tool-rag] Low-signal query; will run RAG retrieval")
     if not guide_only and not _relevant_tools:
         try:
             from src.tool_index import get_tool_index, ALWAYS_AVAILABLE
@@ -1870,6 +1979,44 @@ async def stream_agent_loop(
     if _relevant_tools is not None and active_document is not None:
         _relevant_tools.update({"edit_document", "update_document", "suggest_document"})
 
+    # The skill index injected by _build_system_prompt tells the model to
+    # call `manage_skills action=view`, and Jaccard-matched skills are pasted
+    # into the prompt as procedures to follow — but neither path goes through
+    # tool selection, so the model can be handed a procedure naming tools
+    # (grep, read_file, ...) that aren't in its schema list. Keep the schemas
+    # in lockstep: manage_skills is callable whenever any skill is indexed,
+    # and a matched skill's declared requires_toolsets ride along with it.
+    if not guide_only and _relevant_tools is not None:
+        try:
+            from services.memory.skills import SkillsManager
+            from src.constants import DATA_DIR
+            _skills_on = True
+            try:
+                from routes.prefs_routes import _load_for_user as _load_prefs
+                _skills_on = (_load_prefs(owner) or {}).get("skills_enabled", True)
+            except Exception:
+                pass
+            _sm = SkillsManager(DATA_DIR)
+            _owner_skills = _sm.load(owner=owner) if _skills_on else []
+            if _owner_skills:
+                _relevant_tools.add("manage_skills")
+                if _retrieval_query:
+                    # Validate against every known executable tool, not just
+                    # TOOL_SECTIONS — code-nav tools (grep/glob/ls) ship as
+                    # schemas without a prompt-prose section.
+                    from src.tool_policy import known_tool_names
+                    _known = known_tool_names()
+                    for _sk in _sm.get_relevant_skills(
+                        _retrieval_query, skills=_owner_skills,
+                        threshold=0.25, max_items=3,
+                    ):
+                        _relevant_tools.update(
+                            t for t in (_sk.get("requires_toolsets") or [])
+                            if t in _known
+                        )
+        except Exception as _e:
+            logger.debug(f"[tool-rag] skill-aware tool include skipped: {_e}")
+
     if _relevant_tools is not None:
         logger.info("[agent-intent] selected_tools=%s", sorted(_relevant_tools)[:50])
 
@@ -1920,6 +2067,10 @@ async def stream_agent_loop(
     # and can override this list for users who know their setup.
     _model_no_tools = any(kw in _model_lc for kw in (
         "deepseek-r1",
+        # Open-weight GPT-OSS models are commonly served through llama.cpp /
+        # llama-cpp-python. Their names contain "gpt-o", but they do not use
+        # OpenAI's native tool-call channel unless the endpoint opts in.
+        "gpt-oss",
     ))
     # Native Ollama endpoints (/api/chat) handle tool schemas differently from
     # the OpenAI-compat path. Models like gemma4, qwen3.5, ministral respond to
@@ -1949,6 +2100,7 @@ async def stream_agent_loop(
         compact=_is_api_model,
         owner=owner,
         suppress_local_context=guide_only,
+        active_email=active_email,
     )
     if plan_mode and not guide_only:
         # Steer the model to investigate-then-propose. Hard tool gating handles
@@ -1981,30 +2133,34 @@ async def stream_agent_loop(
     _t3 = time.time()
     try:
         from src.context_compactor import trim_for_context
-        from src.context_budget import compute_input_token_budget, DEFAULT_HARD_MAX
-        from src.settings import is_setting_overridden
+        from src.context_budget import compute_input_token_budget, DEFAULT_HARD_MAX, DEFAULT_BUDGET, budget_is_explicit as _budget_is_explicit
+        from src.model_context import budget_context_for_model
 
-        soft_budget = int(get_setting("agent_input_token_budget", 6000) or 0)
+        soft_budget = int(get_setting("agent_input_token_budget", DEFAULT_BUDGET) or 0)
         if soft_budget > 0:
             before_trim_tokens = estimate_tokens(messages)
             reserve_tokens = min(max(max_tokens or 1024, 512), 2048)
-            # Honour the configurable ceiling for the auto-derived budget path.
-            # No-op when the user has an explicit `agent_input_token_budget`
-            # (that branch ignores hard_max). Falls back to DEFAULT_HARD_MAX
-            # on missing/malformed values so misconfig can't zero the budget.
+            # Ceiling for the auto-derived budget (no effect on an explicit budget;
+            # see #1230). Falls back to DEFAULT_HARD_MAX on missing/malformed values
+            # so misconfig can't zero the budget.
             try:
                 hard_max = int(get_setting("agent_input_token_hard_max", DEFAULT_HARD_MAX) or DEFAULT_HARD_MAX)
             except (TypeError, ValueError):
                 hard_max = DEFAULT_HARD_MAX
             if hard_max <= 0:
                 hard_max = DEFAULT_HARD_MAX
-            # Scale the default budget to the model's context window so long-context
-            # models aren't silently capped at 6000; an explicit user setting is
-            # still honoured (clamped to the window). (#1170)
+            # Default value = auto sentinel (scale to the window); any other value =
+            # explicit cap. Value-based, not presence-based, because the save path
+            # materializes defaults so a persisted default must still read as auto (#4121).
+            budget_is_explicit = _budget_is_explicit(soft_budget)
+            # Scale only off a window we actually discovered, bound to the value it
+            # proves (else 0) — not the passed-in context_length, which can be stale
+            # or unset for some callers (#4122 review).
+            ctx_for_budget = budget_context_for_model(endpoint_url, model, fallback=context_length)
             effective_budget = compute_input_token_budget(
                 soft_budget,
-                context_length,
-                is_setting_overridden("agent_input_token_budget"),
+                ctx_for_budget,
+                budget_is_explicit,
                 hard_max=hard_max,
             )
             trimmed_messages = trim_for_context(
@@ -2079,11 +2235,12 @@ async def stream_agent_loop(
     # tool, so we don't nudge on harmless transitional text like "let me
     # know what you think".
     _INTENT_RE = re.compile(
-        r"(?:^|\n)\s*(?:let me|i'?ll|i will|going to|let's)\s+"
+        r"(?:^|\n)\s*(?:let me|i'?ll|i will|i need to|we need to|need to|"
+        r"i should|we should|i must|we must|going to|let's)\s+"
         r"(?:tail|check|investigate|look at|see|tail|read|fetch|inspect|"
         r"verify|diagnose|examine|debug|capture|grab|pull|view|run|call|"
         r"trigger|launch|start|kick off|stop|kill|restart|adopt|serve|"
-        r"register|adopt|list|search|find|query|hit|ping|test)"
+        r"register|adopt|list|search|find|query|hit|ping|test|use|perform|do)"
         r"\b[^.\n]{0,140}",
         re.IGNORECASE,
     )
@@ -2124,9 +2281,17 @@ async def stream_agent_loop(
         elif _is_api_model:
             # Filter schemas by RAG-selected tools (if available)
             if _relevant_tools:
+                # _build_base_prompt unions _ADMIN_TOOLS into the prompt
+                # sections when admin intent fires — the schema list must
+                # offer the same names, or the model reads prose describing
+                # tools it cannot call and substitutes the nearest schema
+                # it does have (e.g. manage_memory for manage_skills).
+                _schema_names = set(_relevant_tools)
+                if _needs_admin:
+                    _schema_names |= _ADMIN_TOOLS
                 base_schemas = [
                     s for s in FUNCTION_TOOL_SCHEMAS
-                    if s.get("function", {}).get("name") in _relevant_tools
+                    if s.get("function", {}).get("name") in _schema_names
                 ]
                 _mcp_filtered = [
                     s for s in mcp_schemas
@@ -2644,6 +2809,7 @@ async def stream_agent_loop(
                             tool_policy=tool_policy,
                             owner=owner,
                             progress_cb=_push_progress,
+                            workspace=workspace,
                         )
                     finally:
                         # Sentinel so the drainer knows to stop.
@@ -2661,6 +2827,46 @@ async def stream_agent_loop(
                     )
                 desc, result = await _tool_task
 
+            # A skill the model just loaded can prescribe tools that weren't
+            # RAG-selected this turn (declared via requires_toolsets in its
+            # frontmatter). Union them into the selection so the NEXT round's
+            # schema list includes them — otherwise the model reads "use
+            # grep" from the skill it fetched but has no grep schema to call.
+            if (
+                block.tool_type == "manage_skills"
+                and _relevant_tools is not None
+                and not result.get("error")
+            ):
+                _ms_args = {}
+                _ms_raw = (block.content or "").strip()
+                if _ms_raw.startswith("{"):
+                    try:
+                        _ms_args = json.loads(_ms_raw)
+                    except json.JSONDecodeError:
+                        _ms_args = {}
+                _ms_name = str(_ms_args.get("name", "") or "").strip()
+                if _ms_name and _ms_args.get("action") in ("view", "view_ref"):
+                    try:
+                        from services.memory.skills import SkillsManager as _SkM
+                        from src.constants import DATA_DIR as _DD
+                        from src.tool_policy import known_tool_names as _ktn
+                        _known = _ktn()
+                        for _sk in _SkM(_DD).load(owner=owner):
+                            if _sk.get("name") == _ms_name:
+                                _new = {
+                                    t for t in (_sk.get("requires_toolsets") or [])
+                                    if t in _known and t not in _relevant_tools
+                                }
+                                if _new:
+                                    _relevant_tools.update(_new)
+                                    logger.info(
+                                        "[tool-rag] skill '%s' unlocked tools for next round: %s",
+                                        _ms_name, sorted(_new),
+                                    )
+                                break
+                    except Exception as _e:
+                        logger.debug(f"skill requires_toolsets unlock skipped: {_e}")
+
             # Extract structured web sources from web_search tool output.
             # web_search returns {"output": ..., "exit_code": 0}; check "output"
             # first so the <!-- SOURCES:…--> marker is found and stripped even
@@ -2751,18 +2957,20 @@ async def stream_agent_loop(
                 # On a bash/python timeout the result carries error + (often
                 # empty) stdout/stderr; fall back to the error so the "timed
                 # out" reason reaches the UI instead of a blank result.
-                output_text = (result["stdout"] or result["stderr"] or result.get("error", ""))[:2000]
+                raw = result["stdout"] or result["stderr"] or result.get("error", "")
+                output_text = _truncate(raw)
             elif "output" in result:
                 # bash / python canonical result: {"output": ..., "exit_code": ...}
-                output_text = (result["output"] or "")[:2000]
+                raw = result["output"] or ""
+                output_text = _truncate(raw)
             elif "response" in result:
                 # AI interaction tools (chat_with_model, send_to_session)
                 label = result.get("model", result.get("session_name", "AI"))
-                output_text = f"{label}: {result['response']}"[:4000]
+                output_text = _truncate(f"{label}: {result['response']}")
             elif "content" in result:
-                output_text = result["content"][:2000]
+                output_text = _truncate(result["content"])
             elif "results" in result:
-                output_text = result["results"][:4000]
+                output_text = _truncate(result["results"])
             elif "session_id" in result and "name" in result:
                 output_text = f"Session created: {result['name']} (id: {result['session_id']})"
             elif "success" in result:
@@ -2772,13 +2980,25 @@ async def stream_agent_loop(
                     else f"Error: {result.get('error', '')}"
                 )
             elif "error" in result:
-                output_text = result["error"][:2000]
+                output_text = _truncate(result["error"])
 
             # Emit tool_output (include ui_event data if present)
             tool_output_data = {"type": "tool_output", "tool": block.tool_type, "command": cmd_display, "output": output_text, "exit_code": result.get("exit_code")}
             if "ui_event" in result:
                 tool_output_data["ui_event"] = result["ui_event"]
-                for k in ("toggle_name", "state", "mode", "model", "endpoint_url", "theme_name", "colors"):
+                for k in (
+                    "toggle_name", "state", "mode", "model", "endpoint_url",
+                    "theme_name", "colors",
+                    # ui_control open_email_reply payload — without these the
+                    # frontend openReplyDraft bails on undefined uid and the
+                    # reply window silently never opens.
+                    "uid", "folder", "account_id",
+                    # Optional pre-filled body for open_email_reply so the
+                    # agent can compose-and-open in one tool call.
+                    "body",
+                    # ui_control open_panel payload
+                    "panel",
+                ):
                     if k in result:
                         tool_output_data[k] = result[k]
             # Forward image data from generate_image tool

src/agent_tools/__init__.py

@@ -20,7 +20,7 @@ logger = logging.getLogger(__name__)
 
 from .subprocess_tools import BashTool, PythonTool
 from .web_tools import WebSearchTool, WebFetchTool
-from .filesystem_tools import ReadFileTool, WriteFileTool, EditFileTool, LsTool, GlobTool, GrepTool
+from .filesystem_tools import ReadFileTool, WriteFileTool, EditFileTool, LsTool, GlobTool, GrepTool, GetWorkspaceTool
 from .document_tools import CreateDocumentTool, UpdateDocumentTool, EditDocumentTool, SuggestDocumentTool, ManageDocumentTool
 
 TOOL_HANDLERS = {
@@ -39,6 +39,7 @@ TOOL_HANDLERS = {
     "edit_document": EditDocumentTool().execute,
     "suggest_document": SuggestDocumentTool().execute,
     "manage_documents": ManageDocumentTool().execute,
+    "get_workspace": GetWorkspaceTool().execute,
 }
 
 # ---------------------------------------------------------------------------
@@ -51,7 +52,7 @@ PYTHON_TIMEOUT = 30
 
 # Tool types that trigger execution
 TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_file", "edit_file",
-             "grep", "glob", "ls",
+             "grep", "glob", "ls", "get_workspace",
              "create_document", "update_document", "edit_document",
              "search_chats",
              "chat_with_model", "create_session", "list_sessions",

src/agent_tools/filesystem_tools.py

@@ -46,13 +46,7 @@ def _unified_diff(old: str, new: str, path: str) -> Optional[Dict[str, Any]]:
 
 class EditFileTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import (
-                    _resolve_tool_path,
-                    _resolve_tool_path_in_workspace,
-                    _resolve_search_root,
-                    _truncate
-                )
-        workspace = ctx.get("workspace")
+        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
         try:
             args = json.loads(content) if content.strip().startswith("{") else {}
         except (json.JSONDecodeError, TypeError):
@@ -64,8 +58,7 @@ class EditFileTool:
         if not raw_path:
             return {"error": "edit_file: path required", "exit_code": 1}
         try:
-            path = (_resolve_tool_path_in_workspace(workspace, raw_path)
-                    if workspace else _resolve_tool_path(raw_path))
+            path = _resolve_tool_path(raw_path)
         except ValueError as e:
             return {"error": f"edit_file: {e}", "exit_code": 1}
         if old == "":
@@ -113,13 +106,7 @@ class EditFileTool:
 
 class ReadFileTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import (
-                    _resolve_tool_path,
-                    _resolve_tool_path_in_workspace,
-                    _resolve_search_root,
-                    _truncate
-                )
-        workspace = ctx.get("workspace")
+        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
         raw_path, offset, limit = content.split("\n", 1)[0].strip(), 0, 0
         _stripped = content.strip()
         if _stripped.startswith("{"):
@@ -131,8 +118,7 @@ class ReadFileTool:
             except (json.JSONDecodeError, TypeError, ValueError):
                 pass
         try:
-            path = (_resolve_tool_path_in_workspace(workspace, raw_path)
-                    if workspace else _resolve_tool_path(raw_path))
+            path = _resolve_tool_path(raw_path)
         except ValueError as e:
             return {"error": f"read_file: {e}", "exit_code": 1}
         try:
@@ -170,19 +156,12 @@ class ReadFileTool:
 
 class WriteFileTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import (
-                    _resolve_tool_path,
-                    _resolve_tool_path_in_workspace,
-                    _resolve_search_root,
-                    _truncate
-                )
-        workspace = ctx.get("workspace")
+        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
         lines = content.split("\n", 1)
         raw_path = lines[0].strip()
         body = lines[1] if len(lines) > 1 else ""
         try:
-            path = (_resolve_tool_path_in_workspace(workspace, raw_path)
-                    if workspace else _resolve_tool_path(raw_path))
+            path = _resolve_tool_path(raw_path)
         except ValueError as e:
             return {"error": f"write_file: {e}", "exit_code": 1}
         try:
@@ -212,13 +191,7 @@ class WriteFileTool:
 
 class LsTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import (
-                    _resolve_tool_path,
-                    _resolve_tool_path_in_workspace,
-                    _resolve_search_root,
-                    _truncate
-                )
-        workspace = ctx.get("workspace")
+        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
         raw_path = ""
         _s = (content or "").strip()
         if _s.startswith("{"):
@@ -267,13 +240,7 @@ class LsTool:
 
 class GlobTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import (
-                    _resolve_tool_path,
-                    _resolve_tool_path_in_workspace,
-                    _resolve_search_root,
-                    _truncate
-                )
-        workspace = ctx.get("workspace")
+        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
         args = {}
         _s = (content or "").strip()
         if _s.startswith("{"):
@@ -325,13 +292,7 @@ class GlobTool:
 
 class GrepTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import (
-                    _resolve_tool_path,
-                    _resolve_tool_path_in_workspace,
-                    _resolve_search_root,
-                    _truncate
-                )
-        workspace = ctx.get("workspace")
+        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
         args: Dict[str, Any] = {}
         _s = (content or "").strip()
         if _s.startswith("{"):
@@ -417,3 +378,21 @@ class GrepTool:
         if len(lines) >= max_hits:
             out += f"\n... [capped at {max_hits} matches]"
         return {"output": _truncate(out), "exit_code": 0}
+
+class GetWorkspaceTool:
+    """Report the active workspace folder (no args). File tools are confined to
+    it; the shell starts there (cwd) but is NOT sandboxed."""
+    async def execute(self, content: str, ctx: dict) -> dict:
+        from src.tool_execution import get_active_workspace
+        ws = get_active_workspace()
+        if ws:
+            return {
+                "output": f"{ws}\n(File tools are confined to this folder; the shell starts "
+                          f"here but is not sandboxed and can reach outside it.)",
+                "exit_code": 0,
+            }
+        return {
+            "output": "No workspace is set. File tools use the default allowed roots; "
+                      "resolve paths from the user or use absolute paths.",
+            "exit_code": 0,
+        }

src/agent_tools/subprocess_tools.py

@@ -102,16 +102,15 @@ async def _run_subprocess_streaming(
 
 class BashTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import _AGENT_WORKDIR, _truncate
+        from src.tool_execution import agent_cwd, _truncate
         progress_cb = ctx.get("progress_cb")
-        workspace = ctx.get("workspace")
         _subproc_env = ctx.get("subproc_env")
         proc = await asyncio.create_subprocess_shell(
             content,
             stdout=asyncio.subprocess.PIPE,
             stderr=asyncio.subprocess.PIPE,
             env=_subproc_env,
-            cwd=workspace or _AGENT_WORKDIR,
+            cwd=agent_cwd(),
         )
         stdout, stderr, rc, timed_out = await _run_subprocess_streaming(
             proc,
@@ -129,16 +128,15 @@ class BashTool:
 
 class PythonTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import _AGENT_WORKDIR, _truncate
+        from src.tool_execution import agent_cwd, _truncate
         progress_cb = ctx.get("progress_cb")
-        workspace = ctx.get("workspace")
         _subproc_env = ctx.get("subproc_env")
         proc = await asyncio.create_subprocess_exec(
             (sys.executable or "python"), "-I", "-c", content,
             stdout=asyncio.subprocess.PIPE,
             stderr=asyncio.subprocess.PIPE,
             env=_subproc_env,
-            cwd=workspace or _AGENT_WORKDIR,
+            cwd=agent_cwd(),
         )
         stdout, stderr, rc, timed_out = await _run_subprocess_streaming(
             proc,

src/agent_tools/web_tools.py

@@ -57,13 +57,23 @@ class WebSearchTool:
 class WebFetchTool:
     async def execute(self, content: str, ctx: dict) -> dict:
         from src.search.content import fetch_webpage_content
+        from src.constants import WEB_FETCH_HARD_MAX_BYTES
         raw = content.strip()
         url = ""
+        max_bytes = None
         if raw.startswith("{"):
             try:
                 parsed = json.loads(raw)
                 if isinstance(parsed, dict):
                     url = str(parsed.get("url") or "").strip()
+                    # Download-budget override (#3812): "full": true raises the
+                    # budget to the hard cap; an explicit max_bytes is clamped
+                    # to the hard cap downstream. Default stays the soft cap.
+                    if parsed.get("full") is True:
+                        max_bytes = WEB_FETCH_HARD_MAX_BYTES
+                    mb = parsed.get("max_bytes")
+                    if isinstance(mb, int) and mb > 0:
+                        max_bytes = mb
             except json.JSONDecodeError:
                 url = ""
         if not url:
@@ -78,7 +88,7 @@ class WebFetchTool:
         loop = asyncio.get_running_loop()
         try:
             result = await asyncio.wait_for(
-                loop.run_in_executor(None, lambda: fetch_webpage_content(url, timeout=10)),
+                loop.run_in_executor(None, lambda: fetch_webpage_content(url, timeout=10, max_bytes=max_bytes)),
                 timeout=30,
             )
         except asyncio.TimeoutError:
@@ -94,8 +104,28 @@ class WebFetchTool:
                 return {"error": f"web_fetch: {url}: {err}", "exit_code": 1}
             return {"error": f"web_fetch: {url}: no readable text content (not HTML, or the page needs JS/login)", "exit_code": 1}
 
+        # Tell the model when the download budget cut the body short and how
+        # to get the rest, instead of silently presenting a partial page as
+        # the whole thing.
+        size_note = ""
+        if result.get("truncated"):
+            fetched = result.get("fetched_bytes") or 0
+            total = result.get("total_bytes")
+            total_txt = f" of {total:,} bytes" if total else ""
+            size_note = (
+                f"[partial content: download stopped at {fetched:,} bytes{total_txt}. "
+                f'Re-call with {{"url": "{url}", "full": true}} to fetch up to '
+                f"{WEB_FETCH_HARD_MAX_BYTES:,} bytes.]\n\n"
+            )
+
+        # The notice must lead the output so the MAX_OUTPUT_CHARS trim below can
+        # never drop it. The title is untrusted, uncapped page content, so a
+        # giant title ahead of the notice could push it out of range; keep the
+        # notice first and cap the title as a second guard.
+        if len(title) > 300:
+            title = title[:300] + "..."
         header = (f"# {title}\n" if title else "") + f"Source: {url}\n\n"
-        output = header + text
+        output = size_note + header + text
         if len(output) > MAX_OUTPUT_CHARS:
             output = output[:MAX_OUTPUT_CHARS] + "\n\n[...truncated]"
         return {"output": output, "exit_code": 0}

src/ai_interaction.py

@@ -972,16 +972,15 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner
             memories = [m for m in memories if m.get("category", "").lower() == category_filter]
         if not memories:
             return {"results": "No memories found" + (f" in category '{category_filter}'" if category_filter else "") + "."}
+
         result_lines = [f"Found {len(memories)} memory entries:\n"]
-        for m in memories[:100]:
+        for m in memories:
             cat = m.get("category", "fact")
             mid = m.get("id", "?")[:8]
             text = m.get("text", "")
             if len(text) > 150:
                 text = text[:150] + "..."
             result_lines.append(f"- [{cat}] `{mid}` — {text}")
-        if len(memories) > 100:
-            result_lines.append(f"... and {len(memories) - 100} more")
         return {"results": "\n".join(result_lines)}
 
     elif action == "add":
@@ -1293,7 +1292,7 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O
       set_theme <preset>      — Apply a built-in theme preset (dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute)
       create_theme <name> <bg> <fg> <panel> <border> <accent> [key=val ...] — Create custom theme. Optional key=val: advanced color overrides AND background effects: bgPattern=<none|dots|synapse|rain|constellations|perlin-flow|petals|sparkles|embers>, bgEffectColor=#RRGGBB, bgEffectIntensity=<num>, bgEffectSize=<num>, frosted=true|false
       open_panel <name>       — Open a panel (documents, gallery, email, sessions, notes, memories, skills, settings, cookbook)
-      open_email_reply <uid> [folder] [reply|reply-all|ai-reply] — Open a reply draft document for an email; does not send
+      open_email_reply <uid> [folder] [reply|reply-all|ai-reply] [body text] — Open a reply draft document for an email; does not send. ALWAYS append the body text when the user told you what to say (one-shot draft); only omit body when the user just asked to "open a reply" without content.
       get_toggles             — Return current toggle states (server-side knowledge)
     """
     lines = content.strip().split("\n")
@@ -1537,21 +1536,54 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O
         }
 
     elif action == "open_email_reply":
-        reply_parts = lines[0].strip().split()
-        uid = reply_parts[1].strip() if len(reply_parts) > 1 else ""
-        folder = reply_parts[2].strip() if len(reply_parts) > 2 else "INBOX"
-        mode = reply_parts[3].strip().lower() if len(reply_parts) > 3 else "reply"
+        # Two forms supported:
+        #   open_email_reply <uid> [folder] [reply|reply-all|ai-reply]
+        #   open_email_reply <uid> [folder] [reply|reply-all|ai-reply]
+        #     <body text on subsequent lines or after the mode token>
+        # The body text (if any) gets pre-filled into the reply draft so the
+        # agent can compose-and-open in one tool call instead of opening an
+        # empty draft and leaving the user to wonder what happened.
+        first_line = lines[0].strip()
+        parts = first_line.split(maxsplit=4)
+        uid = parts[1].strip() if len(parts) > 1 else ""
+        folder = parts[2].strip() if len(parts) > 2 else "INBOX"
+        mode = parts[3].strip().lower() if len(parts) > 3 else "reply"
+        # Body: everything on the first line after the mode token, plus any
+        # subsequent lines. Allows multi-line bodies.
+        inline_body = parts[4] if len(parts) > 4 else ""
+        rest_lines = "\n".join(lines[1:]).strip() if len(lines) > 1 else ""
+        body = (inline_body + ("\n" + rest_lines if rest_lines else "")).strip()
         if not uid:
-            return {"error": "open_email_reply needs: open_email_reply <uid> [folder] [reply|reply-all|ai-reply]"}
+            return {"error": "open_email_reply needs: open_email_reply <uid> [folder] [reply|reply-all|ai-reply] [body text]"}
         if mode not in ("reply", "reply-all", "ai-reply"):
             mode = "reply"
-        return {
+        # Body is REQUIRED for the agent path. Opening an empty draft is what
+        # users do by clicking the Reply button — they don't ask the agent
+        # for that. Every agent invocation of open_email_reply MUST include
+        # the body. Reject empty so the agent retries with the content the
+        # user asked for. Exception: ai-reply mode triggers the existing
+        # AI-Reply path on the frontend which generates its own body.
+        if not body and mode != "ai-reply":
+            return {
+                "error": (
+                    "open_email_reply called without body. The agent path REQUIRES a body — "
+                    "opening an empty draft is the wrong response when the user asked you to write. "
+                    "Re-call with the reply text included: "
+                    f"`open_email_reply {uid} {folder or 'INBOX'} {mode} <your reply text here>`. "
+                    "Compose the reply now based on the open email's content and the user's request, "
+                    "then call this tool again with the body. Do NOT call create_document instead."
+                ),
+            }
+        result = {
             "ui_event": "open_email_reply",
             "uid": uid,
             "folder": folder or "INBOX",
             "mode": mode,
-            "results": f"Opening reply draft for email UID {uid}",
+            "results": f"Opening reply draft for email UID {uid}" + (" with pre-filled body" if body else ""),
         }
+        if body:
+            result["body"] = body
+        return result
 
     elif action == "get_toggles":
         return {
@@ -1581,7 +1613,9 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne
     """
     import base64
     import httpx
+    import os
     from pathlib import Path
+    from src.url_safety import check_outbound_url
 
     lines = content.strip().split("\n")
     prompt = lines[0].strip() if lines else ""
@@ -1747,8 +1781,15 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne
 
             elif img.get("url"):
                 # Download external URL and save locally (DALL-E returns temp URLs)
+                result_url = img["url"]
+                ok, reason = check_outbound_url(
+                    result_url,
+                    block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true",
+                )
+                if not ok:
+                    return {"error": f"Image API returned unsafe image URL: {reason}"}
                 try:
-                    dl_resp = httpx.get(img["url"], timeout=60)
+                    dl_resp = httpx.get(result_url, timeout=60)
                     if dl_resp.status_code == 200:
                         img_dir = Path(GENERATED_IMAGES_DIR)
                         img_dir.mkdir(parents=True, exist_ok=True)
@@ -1758,10 +1799,10 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne
                         image_url = f"/api/generated-image/{filename}"
                         image_id = _save_to_gallery(filename)
                     else:
-                        image_url = img["url"]  # fallback to external URL
+                        image_url = result_url  # fallback to external URL
                 except Exception as _dl_e:
                     logger.warning(f"Failed to download DALL-E image: {_dl_e}")
-                    image_url = img["url"]  # fallback to external URL
+                    image_url = result_url  # fallback to external URL
             else:
                 return {"error": "Image API returned unexpected format (no b64_json or url)"}
 

src/api_key_manager.py

@@ -4,6 +4,8 @@ import logging
 from typing import Dict
 from cryptography.fernet import Fernet, InvalidToken
 
+from core.platform_compat import safe_chmod
+
 logger = logging.getLogger(__name__)
 
 class APIKeyManager:
@@ -15,12 +17,20 @@ class APIKeyManager:
     def get_or_create_key(self) -> bytes:
         """Get or create encryption key for API keys"""
         if os.path.exists(self.key_file):
+            # Older versions wrote .key with the process umask (often 0o644,
+            # i.e. group/world-readable). Re-restrict on read so existing
+            # installs heal without needing the key to be regenerated.
+            safe_chmod(self.key_file, 0o600)
             with open(self.key_file, 'rb') as f:
                 return f.read()
         else:
             key = Fernet.generate_key()
             with open(self.key_file, 'wb') as f:
                 f.write(key)
+            # This key decrypts every stored provider credential, so restrict it
+            # to the owner (0o600) — it must not be group/world-readable. No-op
+            # on Windows (files there are ACL-restricted to the user already).
+            safe_chmod(self.key_file, 0o600)
             return key
     
     def encrypt_api_key(self, api_key: str) -> str:
@@ -57,7 +67,12 @@ class APIKeyManager:
             # Legacy/wrong shape (e.g. a list) — .items() would raise. Ignore it.
             logger.warning("API keys file has unexpected shape (%s); ignoring", type(encrypted_keys).__name__)
             return {}
-        return encrypted_keys
+
+        return {
+            str(provider): key
+            for provider, key in encrypted_keys.items()
+            if isinstance(key, str)
+        }
 
     def save(self, provider: str, api_key: str):
         """Save encrypted API key to file.
@@ -82,4 +97,3 @@ class APIKeyManager:
             except (InvalidToken, ValueError) as e:
                 logger.warning("Failed to decrypt API key for %s: %s", provider, e)
         return decrypted
-

src/bg_monitor.py

@@ -55,6 +55,8 @@ async def _drain_agent(sess, messages):
         if "delta" in d:
             delta = d.get("delta")
             if isinstance(delta, str):
+                if d.get("thinking"):
+                    continue
                 full += delta
         elif d.get("type") == "agent_step":
             round_num = d.get("round", round_num)