diff --git a/.dockerignore b/.dockerignore index aed7e9368..eca6c8fe8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,16 @@ dist/ build/ .env .env.bak.* +# Secrets: keep plaintext and every transient secrets.env variant out of +# the build context. If an encrypted secrets.env is used, it is mounted +# at runtime — never baked into the image. Mirrored in .gitignore. +secrets.env +secrets.env.* +secrets.env~ +.secrets.env.swp +.secrets.env.swo +**/#secrets.env# +!secrets.env.example /data/ /logs/ .git/ diff --git a/.env.example b/.env.example index 5382c23c7..0f4dcd449 100644 --- a/.env.example +++ b/.env.example @@ -190,3 +190,10 @@ SEARXNG_INSTANCE=http://localhost:8080 # These overlays only expose the GPU devices. The slim Odysseus image # still needs CUDA/ROCm userspace via Cookbook -> Dependencies (vLLM, # llama-cpp-python, etc.) before models can actually serve on GPU. + +# ============================================================ +# Storage Paths (Docker Compose) +# ============================================================ + +# APP_DATA_DIR=./data +# APP_LOGS_DIR=./logs diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..fc7545ace --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,9 @@ +# Code owners. +# +# Intentionally empty for now. The catch-all rule that mapped every path to a +# single owner froze all merges the moment "Require review from Code Owners" +# was enabled, because no other maintainer's approval could satisfy the gate. +# A per-area ownership map (security/auth, CI, frontend, agent internals, with +# multiple named owners per line) is being worked out in issue #593; once +# agreed it replaces this file. Until then, required reviews and the security +# CI gate (docs/security-ci.md) remain in force via branch protection. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..e1e0bf13e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,48 @@ +# Dependabot keeps dependencies and pinned action versions current. +# +# Why this matters for security: every workflow in this repo pins its GitHub +# Actions to an exact commit (a SHA), which is safe but freezes them in time. +# Dependabot opens a small, reviewable pull request whenever a newer version +# exists -- for Python packages, npm packages, the Docker base image, and the +# pinned Actions themselves -- so staying patched does not require manual work. +# Updates are grouped so a week's bumps arrive as one PR per ecosystem, not a +# flood of separate ones. + +version: 2 +updates: + # Python dependencies (requirements.txt + requirements-optional.txt). + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + python: + patterns: ["*"] + + # Frontend / tooling npm packages (package.json). + - package-ecosystem: npm + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + npm: + patterns: ["*"] + + # The pinned action SHAs used across .github/workflows. + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + actions: + patterns: ["*"] + + # The Docker base image in the Dockerfile. + - package-ecosystem: docker + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 5 diff --git a/.github/pull_request_review_template.md b/.github/pull_request_review_template.md new file mode 100644 index 000000000..725138545 --- /dev/null +++ b/.github/pull_request_review_template.md @@ -0,0 +1,123 @@ +# Pull Request Review Template + +Use this shape as a copyable reference for substantive PR reviews; GitHub does +not auto-apply this file to review comments. Omit sections that do not add +useful signal. Lead with confirmed findings; keep speculative notes out of the +public review unless they are framed as a concrete open question. + +## Small PR Path + +For narrow docs, typo, test-only, or obvious local fixes, a short review is +enough: + +```md +LGTM after checking: +- scope: +- validation: +- residual risk: +``` + +Use the fuller structure below for larger, risky, multi-finding, or +security-sensitive reviews. + +## Findings + +**![P2 Badge](https://img.shields.io/badge/P2-yellow?style=flat) issue (test): Short issue title** + +- **Problem:** Concrete broken flow, contract, input, or risk. + +- **Impact:** Why this matters to users, CI, maintainers, data, security, or scale. + +- **Ask:** Smallest practical correction or decision the author should make. + +- **Location:** `path:line` + +## Open Questions + +- **question (scope, non-blocking): Short author question** Ask the concrete + intent, scope, or tradeoff question. + +## Validation + +- Ran: +- Not run: +- Residual risk: + +## PR Hygiene + +- Target/template/checks: +- Related, duplicate, or superseding context: + +## No Findings Variant + +```md +## Findings + +none confirmed + +## Validation + +- Ran: +- Not run: +- Residual risk: +``` + +## Legend + +- **Findings:** Verified, author-actionable issues that should be fixed or + consciously accepted before merge. +- **Priority badges:** The shields.io badges below are optional formatting for + priority labels. Plain `P0`, `P1`, `P2`, or `P3` text is also acceptable when + an external image dependency is undesirable or may not render. + - **P0:** `![P0 Badge](https://img.shields.io/badge/P0-red?style=flat)` - + release-blocking or actively dangerous. + - **P1:** `![P1 Badge](https://img.shields.io/badge/P1-orange?style=flat)` - + serious bug, security risk, data-loss risk, or broken primary flow. + - **P2:** `![P2 Badge](https://img.shields.io/badge/P2-yellow?style=flat)` - + meaningful correctness, test, maintainability, or edge-case issue. + - **P3:** `![P3 Badge](https://img.shields.io/badge/P3-lightgrey?style=flat)` - + minor polish or low-risk cleanup. +- **Intent labels:** + - **`issue`:** A confirmed defect, regression, broken contract, or concrete + risk. + - **`suggestion`:** A non-blocking improvement that would make the PR clearer, + safer, or easier to maintain. + - **`nit`:** A tiny, non-blocking cleanup or style note. Use it only when the + author can safely ignore it without changing the review outcome. + - **`question`:** A real author-facing clarification about intent, scope, or + tradeoffs. Do not use questions to hide an issue that should be stated + directly. + - **`LGTM`:** "Looks good to me." Use only when the review found no blocking + issues, or when any remaining notes are clearly optional. +- **Decorations:** Optional labels in parentheses that clarify the finding type, + scope, or merge impact. + - **`security`:** Auth, authorization, ownership, secrets, SSRF, injection, + unsafe external input, or other trust-boundary concerns. + - **`test`:** Missing, failing, misleading, brittle, or insufficient tests. + - **`scope`:** PR scope, feature boundaries, unrelated churn, or work that + should be split into a separate issue or PR. + - **`ci`:** CI configuration, workflow failures, flaky checks, or validation + signal quality. + - **`api`:** Route, request/response, public function, schema, persistence, or + integration contract changes. + - **`docs`:** User-facing docs, contributor docs, examples, or comments that + need to change with the code. + - **`non-blocking`:** Useful feedback that should not prevent merge by + itself. +- **Finding fields:** + - **Problem:** What is wrong, what contract is ambiguous, or what risk the PR + introduces. + - **Impact:** Why the problem matters in practical terms. + - **Ask:** The smallest concrete fix, test, or decision requested from the PR + author. + - **Location:** The most useful repo-relative file and line reference for the + finding, using `path:line`. +- **Optional sections:** + - **Open Questions:** Genuine scope or intent questions; omit when there are + no real questions. + - **Validation:** What the reviewer ran, what was intentionally not run, and + what risk remains after review. + - **PR Hygiene:** Target-branch, template, CI/check, duplicate, related-work, + or superseding-PR notes. +- **`none confirmed`:** Use only when no review-worthy findings were confirmed; + still list validation gaps or residual risk when relevant. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 818495d14..3784e65ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,10 @@ jobs: name: Python syntax (compileall) runs-on: ubuntu-latest steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" # Byte-compile sources — catches syntax errors without installing deps. @@ -32,10 +32,10 @@ jobs: name: JS syntax (node --check) runs-on: ubuntu-latest steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: "20" # Syntax-check our own JS (skip vendored libs in static/lib). @@ -54,7 +54,7 @@ jobs: # ROADMAP "fresh install smoke tests" item; make this required once green. continue-on-error: true steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 persist-credentials: false @@ -81,7 +81,7 @@ jobs: echo "docs_only=false" >> "$GITHUB_OUTPUT" fi - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 if: steps.docs-check.outputs.docs_only != 'true' with: python-version: "3.11" diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml new file mode 100644 index 000000000..2551ee4f7 --- /dev/null +++ b/.github/workflows/container-scan.yml @@ -0,0 +1,52 @@ +# Container security: Dockerfile lint +# +# Purpose: the Docker image is how most people run Odysseus, so it is part of +# the attack surface. hadolint lints the Dockerfile for mistakes and insecure +# patterns (running as root longer than needed, unpinned base image, bad apt +# usage). Blocking. +# +# The image vulnerability scan (Trivy, advisory) lives in its own file, +# container-trivy.yml. Keeping it separate lets that advisory scan be +# path-filtered and held to a read-only token on pull requests without +# weakening this blocking gate, which must always report so a required check +# never hangs. +# +# Note: a separate open PR (#120) proposes a local `scripts/scan_image.py`. +# This job is complementary -- it is a CI gate, not a script a contributor has +# to remember to run. + +name: Container scan + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: {} + +concurrency: + group: container-scan-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + hadolint: + name: hadolint (Dockerfile lint) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Lint Dockerfile + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: Dockerfile + # DL3008: pinning apt package versions is impractical on a -slim base + # image. Debian purges old package versions from its repos, so a + # pinned version breaks future rebuilds. The base image itself is + # what should be pinned (tracked by Dependabot's docker ecosystem). + ignore: DL3008 diff --git a/.github/workflows/container-trivy.yml b/.github/workflows/container-trivy.yml new file mode 100644 index 000000000..999e8d96d --- /dev/null +++ b/.github/workflows/container-trivy.yml @@ -0,0 +1,125 @@ +# Container image vulnerability scan (advisory) +# +# Trivy builds the application image and scans it for known-vulnerable OS and +# Python packages. Advisory only -- it reports findings to the repo's Security +# tab without blocking a merge, because the image inevitably contains +# already-known CVEs in upstream packages that are not this project's bug. +# +# Split from the Dockerfile lint (container-scan.yml) for two reasons: +# +# - Least privilege. The image build runs Dockerfile instructions, which on a +# pull request are attacker-influenceable. That path (the `scan` job) is +# held to a read-only token and never publishes results. Only `publish`, +# which runs on push to main (curated, fast-forwarded from reviewed dev), +# gets security-events:write to upload SARIF. +# - Cost. Docs-only changes do not rebuild the image (paths-ignore below), +# matching docker-publish.yml. hadolint stays on the broad trigger in +# container-scan.yml so the blocking gate always reports. + +name: Container scan (Trivy) + +on: + pull_request: + paths-ignore: + - '**.md' + - 'docs/**' + - '.github/ISSUE_TEMPLATE/**' + push: + branches: [main] + paths-ignore: + - '**.md' + - 'docs/**' + - '.github/ISSUE_TEMPLATE/**' + workflow_dispatch: + +permissions: {} + +concurrency: + group: container-trivy-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Pull requests and manual runs: build and scan under a read-only token. + # The build executes PR-supplied Dockerfile instructions, so this job must + # not hold any write scope, and it does not upload to the Security tab. + scan: + name: Trivy (image scan, advisory) + if: github.event_name != 'push' + runs-on: ubuntu-latest + # Advisory: a CVE in an upstream package must not block a PR. + continue-on-error: true + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + # Build without pushing so a broken Dockerfile is caught here, and the + # exact image we ship is what gets scanned. + - name: Build image + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + push: false + load: true + tags: odysseus:ci + + - name: Scan image with Trivy + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: odysseus:ci + format: table + ignore-unfixed: true + env: + # Pin the vuln DB source to GHCR to avoid rate-limited Docker Hub + # mirrors that flake on shared runners. + TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2 + + # Push to main only: build, scan, and publish SARIF to the Security tab. + # This is the only path that runs trusted code, so it is the only one granted + # security-events:write. + publish: + name: Trivy (image scan + SARIF upload) + if: github.event_name == 'push' + runs-on: ubuntu-latest + continue-on-error: true + permissions: + contents: read + security-events: write # upload SARIF to the Security tab + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + - name: Build image + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + push: false + load: true + tags: odysseus:ci + + - name: Scan image with Trivy + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: odysseus:ci + format: sarif + output: trivy-results.sarif + ignore-unfixed: true + env: + TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2 + + - name: Upload Trivy results + uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 + with: + sarif_file: trivy-results.sarif + category: trivy-image diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 000000000..c6f3cf4ad --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,71 @@ +# Supply-chain review +# +# Purpose: defend against "side-chain" / supply-chain attacks -- a pull request +# that adds (or bumps) a dependency to a version with a known vulnerability or a +# disallowed license. Two layers: +# +# - dependency-review: runs ONLY on pull requests. It compares the +# dependencies before and after the PR and blocks the merge if the change +# pulls in a package with a known security advisory. This is the gate. +# - pip-audit: scans the project's current Python requirements against the +# advisory database. Advisory only (it never blocks a merge), because it can +# flag a pre-existing issue in an already-shipped dependency. + +name: Dependency review + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +# Default-deny token; jobs grant only read access. +permissions: {} + +concurrency: + group: dependency-review-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + dependency-review: + name: dependency-review (PR gate) + # Only meaningful on a pull request -- it needs a base..head diff to review. + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Review dependency changes + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + # Fail the PR on any newly introduced moderate-or-worse advisory. + fail-on-severity: moderate + + pip-audit: + name: pip-audit (advisory) + runs-on: ubuntu-latest + # Advisory: report known-vulnerable Python deps without blocking the merge. + continue-on-error: true + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Run pip-audit on requirements + run: | + set -euo pipefail + pip install pip-audit==2.10.0 + pip-audit -r requirements.txt -r requirements-optional.txt --strict diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml new file mode 100644 index 000000000..c270ef73b --- /dev/null +++ b/.github/workflows/secret-scan.yml @@ -0,0 +1,60 @@ +# Secret scanning +# +# Purpose: stop credentials (API keys, tokens, passwords, private keys) from +# ever living in the Git history. Odysseus deliberately keeps real secrets in +# files that are gitignored (.env, data/), but a slip in a future commit -- or a +# malicious pull request that sneaks one in -- would otherwise go unnoticed. +# This job reads the repository and the full commit history and fails if it +# finds anything that looks like a secret. +# +# It runs the official gitleaks BINARY directly (pinned to an exact version and +# verified against the project's published SHA-256 checksum) rather than the +# gitleaks GitHub Action, because the Action asks for a paid license on +# organization-owned repos. The binary is free and behaves identically. + +name: Secret scan + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +# Start with zero permissions; the single job opts back in to read-only. +permissions: {} + +concurrency: + group: secret-scan-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + gitleaks: + name: gitleaks + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + # Full history so a secret committed in an earlier commit (and later + # deleted) is still caught -- deletion does not remove it from Git. + fetch-depth: 0 + persist-credentials: false + + # Pinned version + checksum so a tampered release binary cannot run here. + # Bump VERSION/SHA256 together; the checksum comes from the matching + # gitleaks__checksums.txt on the GitHub release. + - name: Run gitleaks (pinned, checksum-verified) + env: + GITLEAKS_VERSION: 8.30.1 + GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb + run: | + set -euo pipefail + TARBALL="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl -fsSL -o "${TARBALL}" \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/${TARBALL}" + echo "${GITLEAKS_SHA256} ${TARBALL}" | sha256sum -c - + tar -xzf "${TARBALL}" gitleaks + # Scan the whole history. Findings print to the log and fail the job. + ./gitleaks git --no-banner --redact --verbose . diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml new file mode 100644 index 000000000..f8b6fc804 --- /dev/null +++ b/.github/workflows/workflow-security.yml @@ -0,0 +1,80 @@ +# Workflow security (CI that audits the CI) +# +# Purpose: the GitHub Actions workflows themselves are an attack surface. A +# poorly written workflow can leak the repository token, run attacker-supplied +# code from a pull request, or pull in a tampered third-party action. These two +# tools check every workflow file in this repo for those mistakes: +# +# - actionlint: catches workflow syntax errors and shell-script bugs inside +# `run:` steps before they reach main. +# - zizmor: a security linter for Actions. Flags template-injection holes, +# unpinned actions, credential persistence, and over-broad token +# permissions -- exactly the patterns the rest of this CI is built to avoid. +# +# Add this early: it then audits every workflow added after it. + +name: Workflow security + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +# Default-deny token; each job grants only read access to the code. +permissions: {} + +concurrency: + group: workflow-security-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + actionlint: + name: actionlint + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + # Pinned version + checksum so a tampered binary cannot run here. + - name: Run actionlint (pinned, checksum-verified) + env: + ACTIONLINT_VERSION: 1.7.12 + ACTIONLINT_SHA256: 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 + run: | + set -euo pipefail + TARBALL="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" + curl -fsSL -o "${TARBALL}" \ + "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/${TARBALL}" + echo "${ACTIONLINT_SHA256} ${TARBALL}" | sha256sum -c - + tar -xzf "${TARBALL}" actionlint + ./actionlint -color + + zizmor: + name: zizmor (Actions SAST) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + # Pinned zizmor release. --offline keeps the audit hermetic (no network + # calls about the actions it inspects); --min-severity=low surfaces + # everything so nothing slips through under the gate. + - name: Run zizmor + run: | + set -euo pipefail + pip install zizmor==1.25.2 + zizmor --offline --min-severity=low .github/workflows/ diff --git a/.gitignore b/.gitignore index 846e6cf74..77c364b8f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,15 @@ venv/ .env .env.bak.* !.env.example +# Local uv lockfile (optional, per-platform — see "Faster installs with uv" in README) +requirements.lock + +# SOPS workflow — encrypted `secrets.env` is intentionally committable, +# but every variant (plaintext, manual decrypt copy, editor backup) +# must stay out of git. Mirrored in .dockerignore so the same artifacts +# also cannot enter image build layers. +secrets.env.* +!secrets.env.example # Data — all user data stays local data/ @@ -61,6 +70,9 @@ output.txt.txt *.tiff *.pdf +# …except shipped static assets +!static/icons/*.png + # …except shipped demo assets in docs/ that the README links to. !docs/*.jpg !docs/*.jpeg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 174a4f2f6..efb38ed24 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,7 @@ Manual development uses Python 3.11+: python3 -m venv venv source venv/bin/activate pip install -r requirements.txt -python -m uvicorn app:app --host 0.0.0.0 --port 7000 +python -m uvicorn app:app --host 127.0.0.1 --port 7000 ``` Windows is not actively tested. Docker on Linux or a Linux/macOS manual install is the safer path for now. diff --git a/Dockerfile b/Dockerfile index ad273cec4..996e06faa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim +FROM python:3.14-slim # System deps. tmux is required by Cookbook for background downloads/serves. # openssh-client is required for Cookbook remote server tests, setup, probes, diff --git a/Odysseus.spec b/Odysseus.spec new file mode 100644 index 000000000..547460c69 --- /dev/null +++ b/Odysseus.spec @@ -0,0 +1,45 @@ +# -*- mode: python ; coding: utf-8 -*- + + +a = Analysis( + ['launcher.py'], + pathex=[], + binaries=[], + datas=[('static', 'static'), ('scripts', 'scripts'), ('mcp_servers', 'mcp_servers'), ('services/hwfit/data', 'services/hwfit/data'), ('config', 'config'), ('.env.example', '.env.example')], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, + optimize=0, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name='Odysseus', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=False, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, + icon=['static\\icon.ico'], +) +coll = COLLECT( + exe, + a.binaries, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='Odysseus', +) diff --git a/README.md b/README.md index a0dde96a9..dcf07f761 100644 --- a/README.md +++ b/README.md @@ -1,444 +1,65 @@ -# Odysseus +

+ Odysseus +

-> **Branch note:** `dev` is the default branch and contains the latest development changes, but it may be unstable. For the more stable curated branch, use [`main`](https://github.com/pewdiepie-archdaemon/odysseus/tree/main). +

+ A self-hosted AI workspace for chat, agents, research, documents, email, notes, calendar, and local model workflows. +

-``` -─────────────────────────────────────────────── - ⊹ ࣪ ˖ ૮( ˶ᵔ ᵕ ᵔ˶ )っ Odysseus vers. 1.0 -─────────────────────────────────────────────── -``` +

+ Quick Start · + Setup Guide · + Contributing · + Roadmap +

-![Odysseus](docs/odysseus.jpg) +

+ Packaging status +

-A self-hosted AI workspace -- meant to be the self-hosted version of the UI experience you get from ChatGPT and Claude. But with more jank and fun. Running on your own hardware, with your own data -- local-first, privacy-first, and no trojan. +

+ Odysseus interface +

-## Features - - **Chat** -- chat with any local model or API; adding them is super simple.
　vLLM · llama.cpp · Ollama · OpenRouter · OpenAI · GitHub Copilot - - **Agent** -- hand it tools and let it run the whole task itself.
　built on [opencode](https://github.com/anomalyco/opencode) · MCP · web · files · shell · skills · memory - - **Cookbook** -- Scans your hardware, recommends models, click to download and serve.. easy!
　built on [llmfit](https://github.com/AlexsJones/llmfit) · VRAM-aware · GGUF / FP8 / AWQ · fit scoring · vLLM / llama.cpp serving - - **Deep Research** -- multi-step runs that gather, read, and synthesize sources into a nice visual report.
　adapted from [Tongyi DeepResearch](https://github.com/Alibaba-NLP/DeepResearch) - - **Compare** -- a fun tool to compare models side by side. Test completely blind, no bias!
　multi-model · blind test · synthesis - - **Documents** -- YOU write the text, AI is there to assist, not the opposite.
　multi-tab editor · markdown · HTML · CSV · syntax highlighting · AI edits · suggestions - - **Memory / Skills** -- Persistent memory and skills, your agent evolves over time as it better understands you and your tasks!
　ChromaDB · fastembed (ONNX) · vector + keyword retrieval · import/export - - **Email** -- IMAP/SMTP inbox with AI triage built in: urgency reminders, auto-tag, auto-summary, auto-reply drafts, auto-spam.
　IMAP · SMTP · per-account routing · CalDAV-aware - - **Notes & Tasks** -- Quick notes with reminders, a todo list, and scheduled tasks the agent can act on.
　note pings · checklist · cron-style tasks · ntfy / browser / email channels - - **Calendar** -- Local-first calendar with CalDAV sync to Radicale / Nextcloud / Apple / Fastmail.
　CalDAV pull · .ics import/export · per-calendar colors · agent-aware - - **Works on mobile** -- looks and runs great on your phone, not just desktop.
　responsive · installable (PWA) · touch gestures - - **Extras** -- more to explore, happy if you give it a go!
　image editor · theme editor · file uploads (vision + PDF) · web search · presets · sessions · 2FA - -## Demo -A full, hover-to-play tour lives on the landing page (`docs/index.html`). - -
-Screenshots / clips - -### Chat & Agents -![Chat & Agents](docs/chat.gif) -### Deep Research -![Deep Research](docs/research.gif) -### Compare -![Compare](docs/compare.gif) -### Documents -![Documents](docs/document.gif) -### Notes & Tasks -![Notes & Tasks](docs/notes.gif) - -
+--- ## Quick Start -Defaults work out of the box: clone, run, then configure models/search/email -inside **Settings**. Only edit `.env` for deployment-level overrides like -`APP_BIND`, `APP_PORT`, `AUTH_ENABLED`, `DATABASE_URL`, or a pre-seeded admin password. +> `dev` is the default branch and gets the newest changes first. Use [`main`](https://github.com/pewdiepie-archdaemon/odysseus/tree/main) if you want the more curated branch. -On first setup, Odysseus creates an admin account (`admin` unless -`ODYSSEUS_ADMIN_USER` is set) and prints a temporary password in the terminal. -For Docker installs, the same line is in `docker compose logs odysseus`. -Use that for the first login, then change it in **Settings**. - -Contributing? See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, testing, and -pull request guidelines. - -### Docker (recommended) ```bash git clone https://github.com/pewdiepie-archdaemon/odysseus.git cd odysseus -cp .env.example .env # optional, but recommended for explicit defaults +cp .env.example .env docker compose up -d --build ``` -To include optional extras in the image (PDF viewer, Office extraction; includes AGPL PyMuPDF), build with `docker compose build --build-arg INSTALL_OPTIONAL=true` before `up`. -Open `http://localhost:7000` when the containers are healthy. Docker Compose -binds the web UI to `127.0.0.1` by default. If the port is taken, set -`APP_PORT=7001` in `.env` and recreate the container. Set `APP_BIND=0.0.0.0` -only when you intentionally want LAN/reverse-proxy access. +Open `http://localhost:7000` when the containers are healthy. The first admin password is printed in `docker compose logs odysseus`. -### Native Linux / macOS -```bash -git clone https://github.com/pewdiepie-archdaemon/odysseus.git -cd odysseus -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -python setup.py -python -m uvicorn app:app --host 127.0.0.1 --port 7000 -``` -Requirements: Python 3.11+. Cookbook also needs `tmux` for background model -downloads and serves. The app itself is lightweight; local model serving is the -heavy part and depends on the model, runtime, GPU, and VRAM, so small hosts can -connect to API or remote model servers instead. Use `--host 0.0.0.0` only when you intentionally want LAN/reverse-proxy access. +Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration live in the [setup guide](docs/setup.md). -### Apple Silicon -Docker on macOS cannot use the Metal GPU. For GPU-accelerated Cookbook on an -M-series Mac, run Odysseus natively: +## Features -```bash -git clone https://github.com/pewdiepie-archdaemon/odysseus.git -cd odysseus -./start-macos.sh -``` +- **Chat + Agents** — local/API models, tools, MCP, files, shell, skills, and memory. +- **Cookbook** — hardware-aware model recommendations, downloads, and serving. +- **Deep Research** — multi-step web research with source reading and report generation. +- **Compare** — blind side-by-side model testing and synthesis. +- **Documents** — writing-first editor with AI edits, suggestions, Markdown, HTML, CSV, and syntax highlighting. +- **Email** — IMAP/SMTP inbox with triage, tags, summaries, reminders, and reply drafts. +- **Notes, Tasks + Calendar** — reminders, todos, scheduled agent tasks, and CalDAV sync. +- **Extras** — gallery/image editor, themes, uploads, web search, presets, sessions, and 2FA. -It launches at `http://127.0.0.1:7860`. To expose it to your phone over a trusted LAN/VPN such as Tailscale, bind all interfaces: +## Demo -```bash -ODYSSEUS_HOST=0.0.0.0 ./start-macos.sh -# then open http://:7860 -``` - -The script also reads `.env` at startup, so `APP_BIND=0.0.0.0` and `APP_PORT` -set there are picked up automatically without a command-line override each run. - -Keep `AUTH_ENABLED=true` (the default) before binding outside loopback. Do not -expose this port directly to the public internet. To build a clickable app wrapper: - -```bash -./build-macos-app.sh -``` - -
-Cookbook, GPU, Ollama, and troubleshooting notes - -**Docker bundled services.** Compose starts Odysseus, ChromaDB, SearXNG, and -ntfy. Odysseus and the bundled service ports bind to `127.0.0.1` by default, so -they are reachable from the host but not exposed to your LAN/public internet -unless you opt in. - -**Cookbook storage in Docker.** Downloads live in `./data/huggingface` -(`~/.cache/huggingface` in the container). Cookbook-installed Python CLIs and -serve engines live in `./data/local` (`~/.local` in the container), so they -survive container recreation. - -**Remote servers.** In **Cookbook -> Settings -> Servers**, generate the -Odysseus SSH key and add the public key to the remote server's -`~/.ssh/authorized_keys`. From the host you can also run: - -```bash -ssh-copy-id -i data/ssh/id_ed25519.pub user@server -``` - -**Docker GPU overlays.** CPU-only users can skip this section. Cookbook can -only detect GPUs that Docker exposes to the container — if the host runtime or -device passthrough is not configured, Cookbook sees the iGPU, another card, or -CPU instead of your intended GPU. - -For NVIDIA, `scripts/check-docker-gpu.sh` diagnoses GPU passthrough and can -optionally install the host runtime or update `.env`. - -```bash -# Read-only diagnostic (default — installs nothing, never edits .env): -scripts/check-docker-gpu.sh - -# Print OS-specific install commands without running them: -scripts/check-docker-gpu.sh --print-install-commands - -# Install NVIDIA Container Toolkit on Ubuntu/Debian (requires sudo): -scripts/check-docker-gpu.sh --install-nvidia-toolkit - -# Write COMPOSE_FILE to .env (only when GPU passthrough is confirmed working): -scripts/check-docker-gpu.sh --enable-nvidia-overlay - -# Full assisted setup — install toolkit, then enable overlay if passthrough works: -scripts/check-docker-gpu.sh --install-nvidia-toolkit --enable-nvidia-overlay -``` - -Safety notes: -- The app never installs host GPU runtime automatically. -- The app never edits `.env` automatically. -- `.env` is only modified when `--enable-nvidia-overlay` is explicitly passed, - and only after GPU passthrough succeeds. `--yes` skips prompts but does not - bypass the passthrough gate. -- `.env.bak.*` backups created by `--enable-nvidia-overlay` are ignored by - Git and the Docker build context. - -To enable manually without the script, add this to `.env`: - -```bash -COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml -``` - -**AMD / ROCm.** AMD setup is read-only diagnostic plus manual `.env` edit. Run: - -```bash -scripts/check-docker-amd-gpu.sh -``` - -Then add the reported values to `.env`, replacing `RENDER_GID` with your host's -numeric render group id: - -```bash -COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml -RENDER_GID=989 -``` - -For NVIDIA/AMD GPU support, also read the comments in the selected overlay file: docker/gpu.nvidia.yml or docker/gpu.amd.yml. - -**Stack-management UIs (Portainer, Coolify, Dockhand, etc.).** These tools -often accept only a single Compose file and do not reliably honor `COMPOSE_FILE` -or multiple `-f` overlays. CLI users should keep using the `COMPOSE_FILE` -overlay workflow above. For stack UIs, point the stack at one of the standalone -files instead, which bundle the base stack plus the GPU settings: - -- `docker-compose.gpu-nvidia.yml` — still requires the NVIDIA Container Toolkit - on the host. -- `docker-compose.gpu-amd.yml` — still requires host ROCm/kfd/DRI setup, the - `video`/`render` group membership, and `RENDER_GID` when needed. - -The base `docker-compose.yml` plus the `docker/gpu.*.yml` overlays remain the -source of truth; the standalone files mirror them for single-file deployments. - -Verify after enabling either overlay: - -```bash -docker compose exec odysseus nvidia-smi -L # NVIDIA -docker compose exec odysseus sh -lc 'test -e /dev/kfd && test -d /dev/dri && ls -l /dev/kfd /dev/dri/renderD*' # AMD -``` - -> **GPU passthrough ≠ llama.cpp CUDA.** `nvidia-smi` passing inside the -> container confirms Docker GPU access, but llama.cpp also needs `cudart` and -> the CUDA Toolkit at runtime. If Cookbook logs show `Unable to find cudart -> library`, `Could NOT find CUDAToolkit`, `CUDA Toolkit not found`, or -> tensors/layers assigned to CPU, that is a Cookbook/llama.cpp build issue — -> not a Docker passthrough failure. Re-install the serve engine via -> **Cookbook → Dependencies** to get a CUDA-enabled build. -> -> The same split applies to AMD/ROCm: seeing `/dev/kfd` and `/dev/dri` inside -> the container confirms device passthrough, not ROCm userspace or a -> ROCm-enabled vLLM/llama.cpp build. `rocm-smi` and `rocminfo` are not expected -> inside the slim Odysseus image. - -**Ollama with Docker.** If Ollama runs on the host, add this endpoint in -Settings: - -```text -http://host.docker.internal:11434/v1 -``` - -Ollama must listen outside its own loopback interface: - -```bash -OLLAMA_HOST=0.0.0.0:11434 ollama serve -``` - -This connects Odysseus in Docker to an Ollama server that is already running on -your host machine; it does not start Ollama inside the container. -`host.docker.internal` is Docker's hostname for the host machine from inside the -container. Cookbook **Serve** is a separate workflow for serving downloaded -models through Odysseus/llama.cpp, so Windows users with an existing Ollama -install usually only need to add the endpoint in Settings. - -**Useful checks.** - -```bash -docker compose ps -docker compose logs --tail=120 odysseus -docker compose logs odysseus | grep -E 'ChromaDB|MemoryVectorStore|DEGRADED' -``` - -**macOS details.** `start-macos.sh` installs Homebrew deps, creates the venv, -runs setup, and starts uvicorn on port `7860` because AirPlay often holds -`7000`. It uses llama.cpp/Ollama for Metal. vLLM/SGLang are CUDA/ROCm-only and -do not run on macOS. MLX-only models are not served by Odysseus. - -
- -### Native Windows - -**One-command launcher** (creates the venv, installs deps, runs setup, starts the -server; safe to re-run): - -```powershell -git clone https://github.com/pewdiepie-archdaemon/odysseus.git -cd odysseus -powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1 -``` - -Or do it by hand: - -```powershell -git clone https://github.com/pewdiepie-archdaemon/odysseus.git -cd odysseus -py -3.11 -m venv venv -venv\Scripts\Activate.ps1 -pip install -r requirements.txt -python setup.py -python -m uvicorn app:app --host 127.0.0.1 --port 7000 -``` - -If `python` points at an older interpreter, use `py -3.12` (or another installed -3.11+ version) for the venv step. - -**Requirements:** Python 3.11+. The core app (chat, agent, memory, documents, -email, calendar, deep research) runs fully native. For full **Cookbook** background -model downloads and the agent shell tool, also install -[Git for Windows](https://git-scm.com/download/win) (provides `bash.exe`). -Local GPU *serving* of vLLM/SGLang needs Linux/WSL2; for a local model on Windows, -[Ollama](https://ollama.com/download) is the easiest path — point Odysseus at -`http://localhost:11434/v1` in Settings. - -Open `http://localhost:7000`, log in with the generated admin password, -and configure everything else inside **Settings**. - -## Troubleshooting & Advanced Setup - -### `chromadb-client` conflicts with embedded ChromaDB -If `chromadb-client` (the lightweight HTTP-only package) is installed alongside the full `chromadb` package, Odysseus starts but ChromaDB silently falls back to HTTP-only mode and fails. - -**Fix:** uninstall `chromadb-client` and force-reinstall the full package: -```bash -./venv/bin/pip uninstall chromadb-client -y -./venv/bin/pip install --force-reinstall chromadb -``` - -### HTTPS + LAN/Tailscale exposure -To expose Odysseus on a local network or Tailscale with HTTPS: -1. Change the bind address to `0.0.0.0` in `.env` (`APP_BIND=0.0.0.0` or `ODYSSEUS_HOST=0.0.0.0`). -2. Generate a locally-trusted cert for your LAN/Tailscale IPs using [mkcert](https://github.com/FiloSottile/mkcert): - ```bash - mkcert -install - mkcert -cert-file cert.pem -key-file key.pem 192.168.1.100 tailscale-ip - ``` -3. Run `uvicorn` with the generated certs: - ```bash - python -m uvicorn app:app --host 0.0.0.0 --port 7000 --ssl-certfile=cert.pem --ssl-keyfile=key.pem - ``` -4. Install the `mkcert` CA on any other device you want to access Odysseus from (e.g., for iOS, email the `rootCA.pem` to yourself, install the profile, and trust it in Certificate Trust Settings). - -### Optional Dependencies -`requirements-optional.txt` contains packages that unlock extra features. It is not installed by default. - -| Package | Feature unlocked | -|---------|-----------------| -| `faster-whisper` | Local speech-to-text (microphone -> text) via the "local" STT provider. | -| `ddgs` | DuckDuckGo as a search provider option. | -| `PyMuPDF` | PDF page rendering in the side viewer panel and form-filling. (Note: AGPL-3.0) | -| `markitdown` | Office/EPUB document text extraction (converts .docx/.xlsx/.pptx/.xls/.epub to Markdown). | - -### Outlook / Office 365 email -Odysseus email accounts currently use IMAP/SMTP username-password auth. Outlook -and Microsoft 365 generally require OAuth instead, so normal Microsoft mailbox -passwords will fail. See [docs/email-outlook.md](docs/email-outlook.md) for the -current limitation and the planned integration direction. - -## Security Notes -Odysseus is a self-hosted workspace with powerful local tools: shell access, file uploads, model downloads, web research, email/calendar integrations, and API tokens. Treat it like an admin console. - -- Keep `AUTH_ENABLED=true` for any network-accessible deployment. -- Keep `LOCALHOST_BYPASS=false` outside local development. -- Use `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway. -- Do not expose it directly to the public internet without HTTPS and a trusted reverse proxy or private access layer. -- Keep `.env`, `data/`, `logs/`, databases, uploads, generated media, backups, auth/session files, API keys, and model/provider tokens out of Git and private shares. They are ignored by default. -- Review `data/auth.json` after first boot: disable open signup unless you intentionally want it, make only your own account admin, and keep demo/test accounts non-admin. -- Non-admin users do not get shell/Python/file read/write by default, and admin-only routes/tools such as MCP management, API tokens, webhooks, model/cookbook serving, backup/vault, and app settings are admin-gated. Other features are controlled by per-user privileges, so review each user's privileges before exposing a deployment. -- Rotate any API keys or tokens that were ever pasted into a shared chat, demo, screenshot, or log. -- If you enable API tokens or webhooks, create separate tokens per integration and delete unused ones. -- Prefer binding manual development runs to `127.0.0.1`; bind to `0.0.0.0` only when you intentionally want LAN/reverse-proxy access. -- Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only. Expose only the authenticated Odysseus web/API entrypoint through your trusted proxy or private access layer. -- Before publishing a fork, run `git status --short` and confirm no private files from `.env`, `data/`, `logs/`, uploads, backups, or local databases are staged. - -### Private or proxied deployments -Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and the bundled services to `127.0.0.1` by default, so a typical production/private setup is: - -1. Keep Odysseus on localhost, for example `127.0.0.1:7000`. -2. Terminate HTTPS at a trusted reverse proxy or private access gateway. -3. Put the authenticated Odysseus web/API entrypoint behind that layer. -4. Keep raw service and model ports internal-only. - -Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true`, `LOCALHOST_BYPASS=false`, and `SECURE_COOKIES=true`. - -Common internal-only ports from the default docs/compose setup: - -| Port | Service | -|---|---| -| `7000` | Odysseus raw app port | -| `8080` | SearXNG | -| `8091` | ntfy | -| `8100` | ChromaDB host port for manual/compose access | -| `11434` | Ollama | -| `8000-8020` | Common local model/provider APIs | +A full hover-to-play tour lives on the landing page: [`docs/index.html`](docs/index.html). ## Contributing -Help is welcome. The best entry points are fresh-install testing, provider setup -bugs, mobile/editor polish, docs, and small focused refactors. See -[ROADMAP.md](ROADMAP.md) for the current help-wanted list. -## Configuration -Most setup is done inside the app with `/setup` or **Settings**. Use `.env` -for deployment-level defaults and secrets you want present before first boot. -Key settings: +Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, docs, and small focused refactors. See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md). -| Variable | Default | Description | -|---|---|---| -| `LLM_HOST` | `localhost` | Your LLM server (e.g. `llm-host.local:8000`) | -| `LLM_HOSTS` | -- | Comma-separated list for model discovery | -| `OPENAI_API_KEY` | -- | Optional OpenAI key. Prefer adding providers in the app unless pre-seeding. | -| `SEARXNG_INSTANCE` | `http://localhost:8080` | SearXNG URL. Docker overrides this to `http://searxng:8080`. | -| `SEARXNG_SECRET` | generated on first Docker boot | Optional SearXNG cookie/CSRF secret. Leave blank unless you need to pin it. | -| `APP_BIND` | `127.0.0.1` | Docker Compose host bind address for the web UI. Use `0.0.0.0` only for intentional LAN/reverse-proxy access. | -| `APP_PORT` | `7000` | Docker Compose host port for the web UI. | -| `AUTH_ENABLED` | `true` | Enable/disable login | -| `LOCALHOST_BYPASS` | `false` | Development-only auth bypass for loopback requests. Keep false for shared/network deployments. | -| `SECURE_COOKIES` | `false` | Set true when serving Odysseus through HTTPS at a trusted proxy or private access gateway. | -| `DATABASE_URL` | `sqlite:///./data/app.db` | Database connection string | -| `CHROMADB_HOST` | `localhost` | ChromaDB host for vector memory. Docker overrides this to `chromadb`. | -| `CHROMADB_PORT` | `8100` | ChromaDB port for manual host runs. Docker overrides this to `8000`. | -| `EMBEDDING_URL` | -- | OpenAI-compatible embeddings endpoint | -| `ODYSSEUS_CHAT_UPLOAD_MAX_BYTES` | `10485760` | Chat/agent attachment cap in bytes. Raise for larger local PDFs or text documents. | -| `ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES` | `104857600` | Gallery image upload cap in bytes (100 MB). | -| `ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES` | `26214400` | Gallery transform input cap in bytes (25 MB). | -| `ODYSSEUS_MEMORY_IMPORT_MAX_BYTES` | `10485760` | Memory import file cap in bytes (10 MB). | -| `ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES` | `26214400` | Personal document upload cap in bytes (25 MB). | -| `ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES` | `26214400` | Email compose attachment cap in bytes (25 MB). | -| `ODYSSEUS_STT_MAX_AUDIO_BYTES` | `26214400` | Speech-to-text audio cap in bytes (25 MB). | -| `ODYSSEUS_ICS_MAX_BYTES` | `10485760` | Calendar `.ics` import cap in bytes (10 MB). | +## Security -All upload-limit vars are validated (must be a positive integer) and optional; an invalid value fails fast at startup. - -### Built-in MCP servers (optional setup) - -Odysseus auto-registers a few built-in MCP servers at startup. The npx-based ones (currently the browser server, `@playwright/mcp`) only start when their npm package is already in the local npx cache. If a package isn't cached, that server is skipped with a startup log message explaining what to do, so a fresh install does not block on a multi-minute npm download or hang if Playwright system deps are missing. - -To enable the browser MCP (page navigation, screenshots, vision), run once: - -```bash -npx -y @playwright/mcp@latest --version -``` - -That installs `@playwright/mcp` plus Playwright (~300MB total). Restart Odysseus and the server will register at startup. - -## Architecture -``` -app.py # FastAPI entry point -core/ auth, database, middleware, constants -src/ llm_core, agent_loop, agent_tools, chat_processor, search/ -routes/ chat, session, document, memory, model … endpoints -services/ docs, memory, search, hwfit (Cookbook) … -static/ index.html + app.js + style.css + js/ (modular front-end) -docs/ landing page (index.html) + preview clips -``` - -## Data -All user data lives in `data/` (gitignored): `app.db` (sessions, messages, documents), -`memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`. +Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model/service ports publicly. Deployment details are in the [setup guide](docs/setup.md#security-notes). ## Star History @@ -451,19 +72,5 @@ All user data lives in `data/` (gitignored): `app.db` (sessions, messages, docum ## License -AGPL-3.0-or-later -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md). -``` - | - ||| - ||||| - | | | ||||||| - )_) )_) )_) ~|~ - )___))___))___)\ | - )____)____)_____)\\| - _____|____|____|_____\\\__ - \ / - ~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~ - ~^~ all aboard! ~^~ - ~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~ -``` +AGPL-3.0-or-later -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md). diff --git a/app.py b/app.py index 365eee94a..1950c1b2f 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,7 @@ # app.py — slim orchestrator import mimetypes import os +import sys def register_static_mime_types() -> None: @@ -69,10 +70,37 @@ from src.generated_images import GENERATED_IMAGE_HEADERS, resolve_generated_imag from starlette.responses import RedirectResponse # ========= LOGGING ========= -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', -) +import logging.handlers +from core.constants import DATA_DIR + +_root_logger = logging.getLogger() +_root_logger.setLevel(logging.INFO) +_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Clear existing handlers to avoid duplicates +for _h in list(_root_logger.handlers): + _root_logger.removeHandler(_h) + +_console_h = logging.StreamHandler() +_console_h.setFormatter(_formatter) +_root_logger.addHandler(_console_h) + +try: + _log_dir = os.path.join(DATA_DIR, "logs") + os.makedirs(_log_dir, exist_ok=True) + _log_file = os.path.join(_log_dir, "app.log") + + # RotatingFileHandler is not multi-process safe (e.g. if uvicorn is run with --workers N). + # Odysseus is single-process by convention, so this is acceptable, but be aware that + # concurrent log rotation issues can arise if multiple workers are configured. + _file_h = logging.handlers.RotatingFileHandler( + _log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8" + ) + _file_h.setFormatter(_formatter) + _root_logger.addHandler(_file_h) +except Exception as e: + _root_logger.warning(f"Failed to initialize file logging handler (falling back to console-only): {e}") + logger = logging.getLogger(__name__) # ========= APP ========= @@ -86,12 +114,13 @@ app = FastAPI( ) # ========= CORS ========= +CORS_ALLOW_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"] allowed_origins = os.getenv("ALLOWED_ORIGINS", "http://localhost,http://127.0.0.1").split(",") app.add_middleware( CORSMiddleware, allow_origins=allowed_origins, allow_credentials=True, - allow_methods=["GET", "POST", "PUT", "DELETE"], + allow_methods=CORS_ALLOW_METHODS, allow_headers=[ "Accept", "Authorization", @@ -140,6 +169,7 @@ _TIMEOUT_EXEMPT_PREFIXES = ( "/api/cookbook/setup", # remote pacman/apt installs "/api/upload", # large files "/api/image", # diffusion proxies (inpaint/harmonize/upscale/etc.) — own 120s httpx timeout + "/api/memory/audit", # retains own 120s LLM inactivity timeout ) @@ -288,7 +318,7 @@ if AUTH_ENABLED: # (no admin cookie available in that context). Restricted to # loopback clients + matching token to keep it locked down. try: - from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN as _ITT + from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN as _ITT, INTERNAL_TOOL_USER _hdr = request.headers.get(INTERNAL_TOOL_HEADER) if _hdr and secrets.compare_digest(_hdr, _ITT) and _is_trusted_loopback(request): # Impersonation: when the agent's loopback call sets @@ -300,11 +330,11 @@ if AUTH_ENABLED: if _impersonate and _impersonate in getattr(_auth_mgr, "users", {}): request.state.current_user = _impersonate else: - request.state.current_user = "internal-tool" + request.state.current_user = INTERNAL_TOOL_USER request.state.api_token = False return await call_next(request) - except Exception: - pass + except Exception as _e: + logger.warning("Internal tool auth header check failed", exc_info=_e) # Allow DIRECT localhost requests (internal service calls from # heartbeats etc.). Tunnel/proxy-forwarded requests are excluded by # _is_trusted_loopback so LOCALHOST_BYPASS can't be abused over a @@ -357,11 +387,10 @@ if AUTH_ENABLED: _db.close() try: await _asyncio.to_thread(_do) - except Exception: - pass + except Exception as _e: + logger.debug("Failed to update token last_used_at", exc_info=_e) _asyncio.create_task(_touch_last_used(matched_id)) # Keep bearer-token callers out of normal cookie/user - # routes. API-aware routes can read api_token_owner. request.state.current_user = "api" request.state.api_token = True request.state.api_token_id = matched_id @@ -410,7 +439,7 @@ class _RevalidatingStatic(StaticFiles): return resp -app.mount("/static", _RevalidatingStatic(directory="static"), name="static") +app.mount("/static", _RevalidatingStatic(directory=STATIC_DIR), name="static") # ========= GENERATED IMAGES ========= @app.get("/api/generated-image/{filename}") @@ -436,8 +465,8 @@ async def serve_generated_image(filename: str, request: Request): _db.close() except HTTPException: raise - except Exception: - pass + except Exception as _e: + logger.warning("Image ownership verification failed for %r", filename, exc_info=_e) ext = filename.rsplit('.', 1)[-1].lower() mime = { "png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", @@ -498,7 +527,9 @@ app.state.session_manager = session_manager memory_manager = components["memory_manager"] memory_vector = components.get("memory_vector") upload_handler = components["upload_handler"] +app.state.upload_handler = upload_handler personal_docs_mgr = components["personal_docs_manager"] +app.state.personal_docs_manager = personal_docs_mgr api_key_manager = components["api_key_manager"] preset_manager = components["preset_manager"] chat_processor = components["chat_processor"] @@ -675,6 +706,9 @@ app.include_router(setup_shell_routes()) from routes.cookbook_routes import setup_cookbook_routes app.include_router(setup_cookbook_routes()) +from routes.workspace_routes import setup_workspace_routes +app.include_router(setup_workspace_routes()) + # Hardware model fitting (cookbook "What Fits?" tab) from routes.hwfit_routes import setup_hwfit_routes app.include_router(setup_hwfit_routes()) @@ -1139,3 +1173,12 @@ async def _shutdown_event(): except Exception as e: logger.warning(f"MCP shutdown error: {e}") logger.info("Application shutdown complete") + + +if __name__ == "__main__": + import uvicorn + + bind_host = os.getenv("APP_BIND", "127.0.0.1") + bind_port = int(os.getenv("APP_PORT", "7000")) + + uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") diff --git a/build-windows-portable.ps1 b/build-windows-portable.ps1 new file mode 100644 index 000000000..52f71a191 --- /dev/null +++ b/build-windows-portable.ps1 @@ -0,0 +1,72 @@ +#Requires -Version 5.1 +<# + Build a portable Windows distribution for Odysseus. + + Output layout: + dist\Odysseus\Odysseus.exe + dist\Odysseus\static\... + dist\Odysseus\scripts\... + dist\Odysseus\mcp_servers\... + dist\Odysseus\services\hwfit\data\... + + The app then keeps using its normal filesystem layout when frozen. + + Usage: + powershell -ExecutionPolicy Bypass -File .\build-windows-portable.ps1 +#> + +$ErrorActionPreference = "Stop" +Set-Location -Path $PSScriptRoot + +function Write-Step($msg) { Write-Host ""; Write-Host ("==> " + $msg) -ForegroundColor Cyan } +function Fail($msg) { + Write-Host "" + Write-Host ("ERROR: " + $msg) -ForegroundColor Red + exit 1 +} + +Write-Step "Checking for Python" +$pyExe = $null +if (Test-Path ".\.venv\Scripts\python.exe") { + $pyExe = (Resolve-Path ".\.venv\Scripts\python.exe").Path +} else { + foreach ($c in @("py", "python")) { + $cmd = Get-Command $c -ErrorAction SilentlyContinue + if ($cmd) { $pyExe = $cmd.Source; break } + } + if ($pyExe -like "*WindowsApps*python.exe") { + $pyCmd = Get-Command py -ErrorAction SilentlyContinue + if ($pyCmd) { + $pyExe = $pyCmd.Source + } + } +} +if (-not $pyExe) { + Fail "Python not found on PATH. Install Python 3.11+ first." +} +Write-Host ("Using Python: " + $pyExe) + +Write-Step "Installing build dependencies" +& $pyExe -m pip install --upgrade pip --quiet +& $pyExe -m pip install -r requirements.txt pyinstaller pystray Pillow +if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed." } + +Write-Step "Building portable exe bundle" +Remove-Item -Recurse -Force build, dist -ErrorAction SilentlyContinue + +$dataArgs = @( + "--add-data", "static;static", + "--add-data", "scripts;scripts", + "--add-data", "mcp_servers;mcp_servers", + "--add-data", "services/hwfit/data;services/hwfit/data", + "--add-data", "config;config", + "--add-data", ".env.example;.env.example" +) + +& $pyExe -m PyInstaller --noconfirm --clean --onedir --noconsole --icon=static/icon.ico --name Odysseus @dataArgs launcher.py +if ($LASTEXITCODE -ne 0) { Fail "PyInstaller build failed." } + +Write-Host "" +Write-Host "Build complete." -ForegroundColor Green +Write-Host "Portable app folder: $PSScriptRoot\dist\Odysseus" -ForegroundColor Green +Write-Host "Distribute the whole folder (or zip it) so static assets and scripts stay with the exe." -ForegroundColor Green \ No newline at end of file diff --git a/companion/routes.py b/companion/routes.py index 9c8464f0f..0191640ef 100644 --- a/companion/routes.py +++ b/companion/routes.py @@ -5,8 +5,9 @@ offers and pair to it, without duplicating any LLM logic. Auth is enforced globally by AuthMiddleware (app.py), so reaching a handler here means the caller is authenticated by either a cookie session or a Bearer `ody_` -API token. The read endpoints (ping/info/models) accept either; the pairing -endpoints are admin-cookie only. +API token. Ping/info accept either credential type, models requires a chat- +scoped API token for bearer callers, and the pairing endpoints are admin-cookie +only. Pairing CSRF posture: minting happens ONLY on POST. The session cookie is SameSite=Lax (routes/auth_routes.py), which a browser does not send on a @@ -18,7 +19,7 @@ on a GET would be unsafe (Lax cookies ride top-level GET navigations), so GET import html -from fastapi import APIRouter, Request +from fastapi import APIRouter, HTTPException, Request from fastapi.responses import HTMLResponse from core.middleware import require_admin @@ -52,6 +53,18 @@ def owner_can_see(row_owner, owner) -> bool: return row_owner is None or row_owner == owner +def require_models_scope(request: Request) -> None: + """Require the companion chat scope for bearer-token model inventory.""" + if not getattr(request.state, "api_token", False): + return + scopes = getattr(request.state, "api_token_scopes", None) or [] + if isinstance(scopes, str): + scopes = [scope.strip() for scope in scopes.split(",")] + scope_set = {str(scope).strip() for scope in scopes if str(scope).strip()} + if _pairing.COMPANION_SCOPE not in scope_set: + raise HTTPException(403, "API token requires chat scope") + + def mint_pairing_token(owner: str, invalidate=None) -> tuple[str, str]: """Mint a pairing token AND invalidate the auth middleware's in-memory token cache, so the new token is accepted on the very next request without a server @@ -103,6 +116,7 @@ def setup_companion_routes() -> APIRouter: rows -- the same rule as owner_filter. Read-only; never returns api_key material. """ + require_models_scope(request) import json as _json from core.database import SessionLocal, ModelEndpoint diff --git a/core/auth.py b/core/auth.py index 2f9fd4e51..3bdf0f390 100644 --- a/core/auth.py +++ b/core/auth.py @@ -3,6 +3,7 @@ Authentication module — multi-user password hashing, session tokens, config pe Config stored in data/auth.json. Uses bcrypt directly. """ +import enum import json import os import secrets @@ -19,6 +20,7 @@ logger = logging.getLogger(__name__) from core.atomic_io import atomic_write_json as _atomic_write_json # noqa: E402 +from core.middleware import INTERNAL_TOOL_USER # noqa: E402 DEFAULT_PRIVILEGES = { "can_use_agent": True, @@ -46,7 +48,7 @@ ADMIN_PRIVILEGES["allowed_models_restricted"] = False # backwards for this sentinel. ADMIN_PRIVILEGES["block_all_models"] = False -from src.constants import AUTH_FILE +from src.constants import AUTH_FILE, PASSWORD_MIN_LENGTH DEFAULT_AUTH_PATH = AUTH_FILE TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days @@ -64,7 +66,7 @@ TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days # of those names would be denied an assistant and inconsistently owner-scoped. # Refuse to create or rename into any of them so the sentinels can't be # impersonated. (Keep this in sync with that synthetic-owner set.) -RESERVED_USERNAMES = frozenset({"internal-tool", "api", "demo", "system"}) +RESERVED_USERNAMES = frozenset({INTERNAL_TOOL_USER, "api", "demo", "system"}) def normalize_known_username(users: Dict[str, Any], username: str | None) -> Optional[str]: @@ -83,6 +85,15 @@ def _verify_password(password: str, hashed: str) -> bool: return bcrypt.checkpw(password.encode("utf-8"), hashed.encode("utf-8")) +class SetAdminResult(enum.Enum): + """Outcome of AuthManager.set_admin, so callers can map each case to a + precise response instead of guessing from a bare bool.""" + OK = "ok" + USER_NOT_FOUND = "user_not_found" + NOT_AUTHORIZED = "not_authorized" # requester is not an admin + LAST_ADMIN = "last_admin" # would remove the last remaining admin + + class AuthManager: """Manages multi-user password + session-token auth system.""" @@ -233,6 +244,15 @@ class AuthManager: def is_configured(self) -> bool: return len(self.users) > 0 + def policy(self) -> dict: + """Return public auth policy constants for the frontend.""" + return { + "password_min_length": PASSWORD_MIN_LENGTH, + "reserved_usernames": sorted(RESERVED_USERNAMES), + "signup_enabled": self.signup_enabled, + "session_days": TOKEN_TTL // 86400, + } + # ------------------------------------------------------------------ # Account management # ------------------------------------------------------------------ @@ -387,6 +407,69 @@ class AuthManager: logger.info(f"Updated privileges for '{username}': {current}") return True + def set_admin(self, username: str, is_admin: bool, + requesting_user: str) -> SetAdminResult: + """Promote/demote an existing user to/from admin. Admin only. + + Refuses to remove the last remaining admin so the instance can never + be locked out of admin access; self-demotion is allowed as long as + another admin remains. Admin status is re-checked live on every + request, so unlike delete/rename no session or token revocation is + needed — a demoted admin simply fails the next is_admin() gate. + + Promotion stashes the user's current privilege map and demotion + restores it, so a temporary admin stint can't silently broaden a + user's non-admin access; users without a stash (created as admin, + or promoted before stashing existed) demote to DEFAULT_PRIVILEGES. + + Counting admins and flipping the flag happen in one critical section + so two concurrent demotions can't race the admin count to zero. + """ + username = (username or "").strip().lower() + requesting_user = (requesting_user or "").strip().lower() + is_admin = bool(is_admin) + with self._config_lock: + target = self._config.get("users", {}).get(username) + if target is None: + return SetAdminResult.USER_NOT_FOUND + if not self.users.get(requesting_user, {}).get("is_admin"): + return SetAdminResult.NOT_AUTHORIZED + currently_admin = bool(target.get("is_admin")) + if currently_admin == is_admin: + return SetAdminResult.OK # no-op; leave privileges untouched + if currently_admin and not is_admin: + admin_count = sum(1 for d in self.users.values() if d.get("is_admin")) + if admin_count <= 1: + return SetAdminResult.LAST_ADMIN + # Write order matters for lock-free readers: get_privileges() + # reads without _config_lock and trusts is_admin, so the admin + # flag must be flipped while the stored map is safe to expose — + # before writing admin privileges on promote, after restoring + # the pre-admin map on demote. + if is_admin: + target["is_admin"] = True + # Stash the pre-admin map so a later demotion can restore it. + # While is_admin is set the stored map is inert: get_privileges + # short-circuits to ADMIN_PRIVILEGES and set_privileges refuses + # admins, so only set_admin ever touches the stash. + target["privileges_before_admin"] = dict( + target.get("privileges") or DEFAULT_PRIVILEGES + ) + target["privileges"] = dict(ADMIN_PRIVILEGES) + else: + # Restore the stashed pre-admin map. Fall back to defaults for + # users created as admins (their stored map is ADMIN_PRIVILEGES, + # which must not leak past demotion — e.g. can_use_bash) and + # for admins promoted before the stash existed. + target["privileges"] = dict( + target.pop("privileges_before_admin", None) + or DEFAULT_PRIVILEGES + ) + target["is_admin"] = False + self._save() + logger.info("Set is_admin=%s for '%s' (by '%s')", is_admin, username, requesting_user) + return SetAdminResult.OK + def change_password(self, username: str, current_password: str, new_password: str) -> bool: username = username.strip().lower() if username not in self.users: @@ -500,16 +583,20 @@ class AuthManager: return None return self.create_session_trusted(username) - def create_session_trusted(self, username: str) -> str: + def create_session_trusted(self, username: str) -> Optional[str]: """Issue a session token for an already-verified user. Call only after verify_password (and TOTP if enabled) have passed.""" username = username.strip().lower() token = secrets.token_hex(32) - with self._sessions_lock: - self._sessions[token] = { - "username": username, - "expiry": time.time() + TOKEN_TTL, - } + with self._config_lock: + if username not in self.users: + logger.warning("Refused to issue session for missing user '%s'", username) + return None + with self._sessions_lock: + self._sessions[token] = { + "username": username, + "expiry": time.time() + TOKEN_TTL, + } self._save_sessions() return token diff --git a/core/database.py b/core/database.py index 6eec48d11..0f1089b39 100644 --- a/core/database.py +++ b/core/database.py @@ -2,12 +2,15 @@ import os import logging import sqlite3 from datetime import datetime, timezone +from pathlib import Path from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text from sqlalchemy.engine import Engine from sqlalchemy.types import TypeDecorator from sqlalchemy.ext.declarative import declarative_base, declared_attr from sqlalchemy.orm import relationship, sessionmaker, backref +from src.runtime_paths import get_app_root + logger = logging.getLogger(__name__) # Create base class for declarative models @@ -29,9 +32,26 @@ class TimestampMixin: def updated_at(cls): return Column(DateTime, default=utcnow_naive, onupdate=utcnow_naive, nullable=False) -# Get database URL from environment, default to SQLite in DATA_DIR +# Ensure the writable data directory exists before SQLite connects. from src.constants import DATA_DIR, AUTH_FILE, MEMORY_FILE, USER_PREFS_FILE, SETTINGS_FILE -DATABASE_URL = os.getenv("DATABASE_URL", f"sqlite:///{DATA_DIR}/app.db") +Path(DATA_DIR).mkdir(parents=True, exist_ok=True) + + +def _default_database_url() -> str: + return f"sqlite:///{Path(DATA_DIR) / 'app.db'}" + + +def _normalize_sqlite_url(url: str) -> str: + if not url.startswith("sqlite:///"): + return url + db_path = url.replace("sqlite:///", "", 1) + if db_path == ":memory:" or os.path.isabs(db_path): + return url + return f"sqlite:///{(Path(get_app_root()) / db_path).resolve().as_posix()}" + + +# Get database URL from environment, default to SQLite in DATA_DIR +DATABASE_URL = _normalize_sqlite_url(os.getenv("DATABASE_URL", _default_database_url())) # Create engine engine = create_engine( @@ -324,6 +344,13 @@ class EmailAccount(TimestampMixin, Base): smtp_password = Column(String, default="") from_address = Column(String, default="") + display_name = Column(String, nullable=True) # "Hriday Ranka" — used in From: header + + # OAuth2 (Google / Google Workspace). Tokens stored encrypted via secret_storage. + oauth_provider = Column(String, nullable=True) # "google" or None + oauth_access_token = Column(String, nullable=True) # encrypted + oauth_refresh_token = Column(String, nullable=True) # encrypted + oauth_token_expiry = Column(String, nullable=True) # unix timestamp string __table_args__ = ( Index('ix_email_accounts_owner_default', 'owner', 'is_default'), @@ -1427,6 +1454,25 @@ def _migrate_add_task_automation_columns(): except Exception as e: logging.getLogger(__name__).warning(f"task automation migration: {e}") +def _migrate_add_email_oauth_columns(): + """Add Google OAuth and display_name columns to email_accounts if missing.""" + try: + with engine.connect() as conn: + cols = [r[1] for r in conn.execute(text("PRAGMA table_info(email_accounts)"))] + for col, typedef in [ + ("oauth_provider", "TEXT"), + ("oauth_access_token", "TEXT"), + ("oauth_refresh_token", "TEXT"), + ("oauth_token_expiry", "TEXT"), + ("display_name", "TEXT"), + ]: + if col not in cols: + conn.execute(text(f"ALTER TABLE email_accounts ADD COLUMN {col} {typedef}")) + conn.commit() + except Exception as e: + logging.getLogger(__name__).warning(f"email oauth columns migration: {e}") + + def _migrate_add_oauth_config(): """Add oauth_config column to mcp_servers table if missing.""" try: @@ -1602,6 +1648,7 @@ class CalendarCal(TimestampMixin, Base): # NULL for local calendars and for CalDAV calendars created before # multi-account support was added (treated as "use any configured account"). account_id = Column(String, nullable=True, index=True) + caldav_base_url = Column(String, nullable=True) events = relationship("CalendarEvent", back_populates="calendar", cascade="all, delete-orphan") @@ -1632,10 +1679,27 @@ class CalendarEvent(TimestampMixin, Base): # vanishes upstream). NULL/local = created locally (agent, email triage, or # a UI event whose write-back failed) and must NOT be pruned by the sync. origin = Column(String, nullable=True, index=True) + remote_href = Column(String, nullable=True) # CalDAV object URL for updates/deletes + remote_etag = Column(String, nullable=True) # Last seen CalDAV ETag, when available + caldav_sync_pending = Column(String, nullable=True) # create | update | delete retry marker calendar = relationship("CalendarCal", back_populates="events") +class CalendarDeletedEvent(TimestampMixin, Base): + """Hidden CalDAV delete tombstone retained until remote delete succeeds.""" + __tablename__ = "caldav_deleted_events" + + uid = Column(String, primary_key=True, index=True) + owner = Column(String, nullable=True, index=True) + calendar_id = Column(String, nullable=True, index=True) + remote_href = Column(String, nullable=True) + remote_etag = Column(String, nullable=True) + caldav_base_url = Column(String, nullable=True) + summary = Column(String, nullable=True) + last_error = Column(Text, nullable=True) + + class Integration(TimestampMixin, Base): """An external service connection (email, RSS, webhook, etc.).""" __tablename__ = "integrations" @@ -1753,6 +1817,7 @@ def init_db(): _migrate_add_tidy_verdict() _migrate_add_doc_source_email_cols() _migrate_add_oauth_config() + _migrate_add_email_oauth_columns() _migrate_add_task_automation_columns() _migrate_add_disabled_tools() _migrate_add_mcp_oauth_tokens_column() @@ -1767,6 +1832,7 @@ def init_db(): _migrate_add_calendar_is_utc() _migrate_add_calendar_origin() _migrate_add_calendar_account_id() + _migrate_add_caldav_sync_columns() _migrate_chat_messages_fts() _migrate_encrypt_email_passwords() _migrate_encrypt_signatures() @@ -2067,6 +2133,31 @@ def _migrate_add_calendar_account_id(): pass +def _migrate_add_caldav_sync_columns(): + """Add remote CalDAV metadata used for bidirectional sync.""" + import sqlite3 + db_path = DATABASE_URL.replace("sqlite:///", "") + if not os.path.exists(db_path): + return + try: + conn = sqlite3.connect(db_path) + ev_columns = [row[1] for row in conn.execute("PRAGMA table_info(calendar_events)").fetchall()] + if ev_columns and "remote_href" not in ev_columns: + conn.execute("ALTER TABLE calendar_events ADD COLUMN remote_href TEXT") + if ev_columns and "remote_etag" not in ev_columns: + conn.execute("ALTER TABLE calendar_events ADD COLUMN remote_etag TEXT") + if ev_columns and "caldav_sync_pending" not in ev_columns: + conn.execute("ALTER TABLE calendar_events ADD COLUMN caldav_sync_pending TEXT") + + cal_columns = [row[1] for row in conn.execute("PRAGMA table_info(calendars)").fetchall()] + if cal_columns and "caldav_base_url" not in cal_columns: + conn.execute("ALTER TABLE calendars ADD COLUMN caldav_base_url TEXT") + conn.commit() + conn.close() + except Exception as e: + logging.getLogger(__name__).warning(f"CalDAV sync metadata migration failed: {e}") + + def _migrate_add_calendar_metadata(): """Add importance/event_type/last_pinged columns to calendar_events table.""" import sqlite3 diff --git a/core/middleware.py b/core/middleware.py index 550ee3bd7..92c62a08b 100644 --- a/core/middleware.py +++ b/core/middleware.py @@ -15,6 +15,8 @@ from starlette.responses import Response # same value from this module. Never persisted or exposed externally. INTERNAL_TOOL_TOKEN = os.environ.get("ODYSSEUS_INTERNAL_TOKEN") or secrets.token_hex(32) INTERNAL_TOOL_HEADER = "X-Odysseus-Internal-Token" +# Pseudo-username on in-process tool-loopback requests; require_admin trusts it and it is reserved. +INTERNAL_TOOL_USER = "internal-tool" def is_cors_preflight(method: str, headers) -> bool: @@ -39,7 +41,7 @@ def require_admin(request: Request): hdr = request.headers.get(INTERNAL_TOOL_HEADER) if hdr and secrets.compare_digest(hdr, INTERNAL_TOOL_TOKEN): return - if getattr(request.state, "current_user", None) == "internal-tool": + if getattr(request.state, "current_user", None) == INTERNAL_TOOL_USER: return except Exception: pass diff --git a/core/platform_compat.py b/core/platform_compat.py index b3b157111..efa496ac6 100644 --- a/core/platform_compat.py +++ b/core/platform_compat.py @@ -191,6 +191,8 @@ def _windows_bash_fallbacks() -> List[str]: base = os.environ.get(env_name) if base: roots.append(ntpath.join(base, "Git")) + if env_name == "LocalAppData": + roots.append(ntpath.join(base, "Programs", "Git")) roots.extend(_WINDOWS_BASH_DEFAULT_ROOTS) paths: List[str] = [] @@ -298,7 +300,7 @@ def is_wsl() -> bool: import sys if sys.platform.startswith("linux") or os.name == "posix": try: - with open("/proc/version", "r") as f: + with open("/proc/version", "r", encoding="utf-8", errors="ignore") as f: if "microsoft" in f.read().lower(): return True except Exception: diff --git a/docker-compose.gpu-amd.yml b/docker-compose.gpu-amd.yml index b95dde1bf..82e22e440 100644 --- a/docker-compose.gpu-amd.yml +++ b/docker-compose.gpu-amd.yml @@ -16,18 +16,18 @@ services: ports: - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000" volumes: - - ./data:/app/data:z - - ./logs:/app/logs:z + - ${APP_DATA_DIR:-./data}:/app/data:z + - ${APP_LOGS_DIR:-./logs}:/app/logs:z # Cookbook remote-server SSH identity. Odysseus can generate a key here; # add the shown public key to each remote server's authorized_keys. - - ./data/ssh:/app/.ssh:z + - ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z # Cookbook local model cache. Inside Docker, "Local" means the Odysseus # container, so persist its HuggingFace cache under ./data/huggingface. - - ./data/huggingface:/app/.cache/huggingface:z + - ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z # Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.) # land under /app/.local for the odysseus user. Persist them so a # container recreate does not silently remove installed serve engines. - - ./data/local:/app/.local:z + - ${APP_DATA_DIR:-./data}/local:/app/.local:z extra_hosts: # Lets the container reach local services on the Docker host, including # Ollama at http://host.docker.internal:11434. @@ -60,6 +60,13 @@ services: - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1} - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost} - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760} + - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600} + - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760} + - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} + - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker-compose.gpu-nvidia.yml b/docker-compose.gpu-nvidia.yml index fa50896ba..1b551c669 100644 --- a/docker-compose.gpu-nvidia.yml +++ b/docker-compose.gpu-nvidia.yml @@ -15,18 +15,18 @@ services: ports: - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000" volumes: - - ./data:/app/data:z - - ./logs:/app/logs:z + - ${APP_DATA_DIR:-./data}:/app/data:z + - ${APP_LOGS_DIR:-./logs}:/app/logs:z # Cookbook remote-server SSH identity. Odysseus can generate a key here; # add the shown public key to each remote server's authorized_keys. - - ./data/ssh:/app/.ssh:z + - ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z # Cookbook local model cache. Inside Docker, "Local" means the Odysseus # container, so persist its HuggingFace cache under ./data/huggingface. - - ./data/huggingface:/app/.cache/huggingface:z + - ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z # Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.) # land under /app/.local for the odysseus user. Persist them so a # container recreate does not silently remove installed serve engines. - - ./data/local:/app/.local:z + - ${APP_DATA_DIR:-./data}/local:/app/.local:z extra_hosts: # Lets the container reach local services on the Docker host, including # Ollama at http://host.docker.internal:11434. @@ -59,6 +59,13 @@ services: - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1} - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost} - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760} + - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600} + - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760} + - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} + - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker-compose.yml b/docker-compose.yml index 9841b1dca..cbeec1e37 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,18 +4,18 @@ services: ports: - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000" volumes: - - ./data:/app/data:z - - ./logs:/app/logs:z + - ${APP_DATA_DIR:-./data}:/app/data:z + - ${APP_LOGS_DIR:-./logs}:/app/logs:z # Cookbook remote-server SSH identity. Odysseus can generate a key here; # add the shown public key to each remote server's authorized_keys. - - ./data/ssh:/app/.ssh:z + - ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z # Cookbook local model cache. Inside Docker, "Local" means the Odysseus # container, so persist its HuggingFace cache under ./data/huggingface. - - ./data/huggingface:/app/.cache/huggingface:z + - ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z # Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.) # land under /app/.local for the odysseus user. Persist them so a # container recreate does not silently remove installed serve engines. - - ./data/local:/app/.local:z + - ${APP_DATA_DIR:-./data}/local:/app/.local:z extra_hosts: # Lets the container reach local services on the Docker host, including # Ollama at http://host.docker.internal:11434. @@ -48,6 +48,13 @@ services: - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1} - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost} - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760} + - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600} + - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760} + - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} + - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 668018ac1..8a3ab4bb6 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -13,6 +13,8 @@ set -e PUID="${PUID:-1000}" PGID="${PGID:-1000}" +GOSU_BIN="$(command -v gosu)" +PYTHON_BIN="$(command -v python)" # Reuse an existing matching group/user if the host's UID/GID already # corresponds to one in /etc/passwd (e.g. when the image is rebuilt @@ -24,26 +26,57 @@ if ! getent passwd "$PUID" >/dev/null 2>&1; then useradd -u "$PUID" -g "$PGID" -M -s /bin/sh -d /app odysseus fi -# Repair ownership on every writable path the app touches at runtime. -# -# Bind-mounted dirs (/app/data, /app/logs) are the obvious ones, but -# the app ALSO writes inside the image's own source tree at runtime: -# - services/cache/{search,content}/* (search cache LRU) -# - services/search_analytics.json -# - services/search_engine_error.log -# - services/tts cache, etc. -# These dirs were created as root during `docker build`, so dropping -# to PUID:PGID would otherwise crash on the first import that tries -# to mkdir them. Chown the whole /app tree — fast (<1s on this size) -# and idempotent via the `-not -uid` filter so we only touch files -# that need fixing. -for dir in /app /app/data /app/logs; do +mount_root_for() { + awk -v target="$1" '$5 == target { print $4; exit }' /proc/self/mountinfo 2>/dev/null || true +} + +is_broad_mount_root() { + case "$1" in + /|/home|/srv|/var|/usr|/opt|/tmp|/mnt|/media) + return 0 + ;; + esac + return 1 +} + +repair_tree_ownership() { + dir="$1" if [ -d "$dir" ]; then - # `find ... -not -uid` keeps this O(touched-files), not - # O(everything), so terabyte-sized maildirs don't slow startup. - find "$dir" -not -uid "$PUID" -print0 2>/dev/null \ + find "$dir" -xdev -not -uid "$PUID" -print0 2>/dev/null \ | xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true fi +} + +repair_app_tree_ownership() { + if [ -d /app ]; then + find /app -xdev \ + \( -path /app/data -o -path /app/logs -o -path /app/.ssh -o -path /app/.cache -o -path /app/.local \) -prune \ + -o -not -uid "$PUID" -print0 2>/dev/null \ + | xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true + fi +} + +repair_bind_mount_ownership() { + dir="$1" + if [ ! -d "$dir" ]; then + return + fi + + mount_root="$(mount_root_for "$dir")" + if is_broad_mount_root "$mount_root"; then + echo "Skipping recursive ownership repair for $dir because it maps to broad host path $mount_root" >&2 + chown "$PUID:$PGID" "$dir" 2>/dev/null || true + return + fi + + repair_tree_ownership "$dir" +} + +# Repair image-owned writable paths without walking into bind-mounted host +# trees, then repair the app-owned mount roots separately. +repair_app_tree_ownership +for dir in /app/data /app/logs /app/.ssh /app/.cache/huggingface /app/.local; do + repair_bind_mount_ownership "$dir" done # Cookbook installs vllm/etc. via `pip install --user`, which pulls @@ -83,9 +116,9 @@ export PATH="/app/.local/bin:$PATH" # Run first-time setup as the app user so data/ files get the right ownership. # setup.py is idempotent — skips auth.json / .env if they already exist. # || true so a setup failure never prevents the container from starting. -gosu "$PUID:$PGID" python /app/setup.py || true +"$GOSU_BIN" "$PUID:$PGID" "$PYTHON_BIN" /app/setup.py || true # Drop root and run the actual app. `gosu` is preferred over `su` / # `sudo` because it cleans up the process tree (no extra shell layer) # so signals (SIGTERM from `docker stop`) reach uvicorn directly. -exec gosu "$PUID:$PGID" "$@" +exec "$GOSU_BIN" "$PUID:$PGID" "$@" diff --git a/docs/agent-migration.md b/docs/agent-migration.md new file mode 100644 index 000000000..ff082159e --- /dev/null +++ b/docs/agent-migration.md @@ -0,0 +1,194 @@ +# Agent migration manifests + +Odysseus should be able to learn from another agent without blindly trusting +that agent's whole state. The safe migration path is: + +```text +source agent export -> source adapter -> agent-migration.v1 manifest -> preview -> apply +``` + +The manifest is intentionally source-neutral. OpenClaw, Hermes, a folder of +Markdown notes, or any other agent can have its own adapter, but Odysseus only +needs to understand the normalized manifest. + +## Why not import everything as memory? + +Durable memory should stay compact and useful. Long notes, logs, session +transcripts, and project archives are useful context, but they are not all +memories. A good migration keeps two layers separate: + +- **Archive documents** preserve source material for search, reading, and later + extraction. +- **Memory candidates** are short facts or preferences that can be reviewed + before being saved into Odysseus memory. + +This keeps Odysseus' existing memory-review flow intact while giving it better +source material to review. + +## Manifest shape + +`agent-migration.v1` is a JSON object: + +```json +{ + "schema_version": "agent-migration.v1", + "generated_at": "2026-06-06T00:00:00Z", + "source": { + "name": "example-agent", + "kind": "generic" + }, + "summary": { + "item_count": 3, + "counts_by_kind": { + "memory": 1, + "skill": 1, + "conversation_thread": 1, + "archive_document": 1 + }, + "warning_count": 0 + }, + "items": [], + "warnings": [] +} +``` + +Each item has a stable `id`, a `kind`, source metadata, and enough content for a +future importer to preview it before applying. + +Supported item kinds in the first pass: + +- `memory` — a candidate memory with `text`, `category`, `source`, and + provenance metadata. +- `skill` — a `SKILL.md` file with content and parsed frontmatter metadata. +- `conversation_thread` — a normalized transcript thread from an exported chat + history. Message content is optional; adapters can preserve only thread + metadata, message counts, timestamps, and hashes when a manifest should stay + small or avoid embedding private transcript text. +- `archive_document` — long-form source material. Content is optional; adapters + can preserve only path/hash/size metadata when a manifest should stay small. + +## Build a manifest + +Use the read-only helper: + +```bash +python3 scripts/agent_migration_manifest.py \ + --source-name old-agent \ + --source-kind generic \ + --memory-json /path/to/memories.json \ + --skills-dir /path/to/skills \ + --conversation-json /path/to/conversations.json \ + --archive /path/to/notes \ + --output /tmp/agent-migration.json +``` + +The helper does not write to `data/`, call an LLM, import Odysseus modules, or +modify the source. It only writes JSON. + +Memory JSON may be: + +```json +[ + "A plain memory string", + { + "text": "A categorized memory", + "category": "preference", + "source": "old-agent" + } +] +``` + +or an object containing a list under `memories`, `memory`, `items`, or `data`. + +Skills are scanned recursively for `SKILL.md`: + +```bash +python3 scripts/agent_migration_manifest.py \ + --source-name hermes \ + --source-kind hermes \ + --skills-dir ~/.hermes/skills \ + --output /tmp/hermes-skills-manifest.json +``` + +Archive documents are metadata-only by default. To embed text content: + +```bash +python3 scripts/agent_migration_manifest.py \ + --source-name notes-export \ + --archive /path/to/markdown-notes \ + --include-archive-content \ + --output /tmp/notes-manifest.json +``` + +Conversation exports are also metadata-only by default: + +```bash +python3 scripts/agent_migration_manifest.py \ + --source-name chatgpt-export \ + --source-kind chatgpt \ + --conversation-json /path/to/conversations.json \ + --output /tmp/chatgpt-conversations-manifest.json +``` + +The first pass supports generic conversation JSON such as: + +```json +[ + { + "id": "thread-1", + "title": "Project plan", + "messages": [ + {"role": "user", "content": "Can we design this?"}, + {"role": "assistant", "content": "Yes, start with a narrow slice."} + ] + } +] +``` + +It also recognizes ChatGPT-style `mapping` exports from `conversations.json`. +To embed normalized messages: + +```bash +python3 scripts/agent_migration_manifest.py \ + --source-name chatgpt-export \ + --source-kind chatgpt \ + --conversation-json /path/to/conversations.json \ + --include-conversation-content \ + --max-conversation-messages 2000 \ + --output /tmp/chatgpt-conversations-with-content.json +``` + +Content embedding is explicit because exported chat histories can be huge and +private. A future source-specific adapter can add ZIP traversal, attachment +metadata, and provider-specific project/workspace fields while still emitting +the same `conversation_thread` manifest item. + +## Recommended apply behavior + +A future Odysseus importer should treat the manifest as untrusted user-provided +data and apply it in stages: + +1. Show a dry-run summary with counts, warnings, duplicates, and sample items. +2. Back up current `data/` state before writing anything. +3. Import archive documents as documents or another searchable source, not as + memory. +4. Import conversation threads as searchable archived context first, with + citations back to the source thread. Do not turn whole transcripts into + memory. +5. Show memory candidates for review before saving through the normal memory + path. +6. Import skills only after name/category conflict checks. +7. Skip secrets by default. Credentials need explicit, provider-specific flows. + +## What belongs in source adapters? + +Adapters can be source-specific. The core manifest should not be. + +For example, an OpenClaw adapter may know about OpenClaw's workspace files. A +Hermes adapter may know about `~/.hermes/config.yaml` and `~/.hermes/skills`. +A ChatGPT adapter may know about `conversations.json`, uploaded-file metadata, +and image attachment directories. A Claude adapter may know about Claude's +export shape and project boundaries. A generic adapter may only know about +memory JSON, conversation JSON, `SKILL.md`, and Markdown folders. + +Nonstandard folders should be adapter details, not required Odysseus concepts. diff --git a/docs/backup-restore.md b/docs/backup-restore.md new file mode 100644 index 000000000..902c9e683 --- /dev/null +++ b/docs/backup-restore.md @@ -0,0 +1,129 @@ +# Backup & Restore + +Odysseus keeps all of your state in the `data/` directory — the SQLite database +(`app.db`), the Fernet encryption key (`data/.app_key`), the vault, memory, RAG +indexes, personal documents, and uploads. The `scripts/odysseus-backup` tool +snapshots that directory into a single gzip tarball and restores it later. + +Snapshots are safe to take while the app is running: SQLite databases are copied +through SQLite's own `.backup` API rather than a raw file copy, so an in-flight +write can't corrupt the snapshot. + +> **A snapshot contains your secrets.** The tarball includes the Fernet +> encryption key (`data/.app_key`), the vault, sessions, and any stored +> provider/API tokens — so treat it like a password. Store backups somewhere +> private, never commit them to Git, and prefer an encrypted destination when +> copying them offsite. + +## Quick start + +Run the tool from the repository root: + +```bash +# Create a snapshot → backups/odysseus-backup-.tar.gz +./scripts/odysseus-backup snapshot + +# List existing snapshots (most recent first) +./scripts/odysseus-backup list + +# Check a tarball's integrity without extracting it +./scripts/odysseus-backup verify backups/odysseus-backup-20260101-120000.tar.gz + +# Restore (destructive — see the warning below) +./scripts/odysseus-backup restore backups/odysseus-backup-20260101-120000.tar.gz --yes +``` + +The script depends only on the Python standard library, so any `python3` on your +`PATH` will run it — you don't need the app's virtualenv active. + +Every command prints a JSON result. Add `--pretty` for indented output. + +## Commands + +### `snapshot` + +Writes a `tar.gz` of `data/` to `backups/.tar.gz`. + +| Flag | Effect | +| --- | --- | +| `--out PATH` | Write to a specific path instead of the default `backups/` location. Must be **outside** `data/`. | +| `--include-research` | Include `data/deep_research/` (skipped by default — research runs are large). | +| `--include-attachments` | Include `data/mail-attachments/` (skipped by default — cached IMAP extractions, re-derivable). | + +By default the snapshot includes everything under `data/` **except** +`deep_research/` and `mail-attachments/`. Personal uploads and documents are +included. + +```bash +# Snapshot straight to a mounted NAS path +./scripts/odysseus-backup snapshot --out /mnt/nas/odysseus-$(date +%F).tar.gz + +# Full snapshot including research runs and mail attachments +./scripts/odysseus-backup snapshot --include-research --include-attachments +``` + +### `list` + +Lists the tarballs in `backups/`, most recent first, with size and modification +time. + +### `verify PATH` + +Opens the tarball read-only and walks every member to confirm it is intact and +safe to restore. Nothing is extracted. Use this before relying on an old backup +or after copying one across machines. + +### `restore PATH --yes` + +Overwrites `data/` from a tarball. + +> **Restore is destructive.** It replaces the current `data/` directory. `--yes` +> is required so a mistyped command can't wipe your live state. + +Restore is not a blind delete: before extracting, the tool **renames your current +`data/` to `data.before-restore-`** in the repository root. If a +restore turns out to be wrong, your previous state is still there — delete the +restored `data/` and rename the stashed directory back. The restore path is also +validated entry-by-entry: archives containing absolute paths, `..` segments, +symlinks, or anything outside `data/` are rejected. + +## Scheduling offsite backups + +The tarball output composes cleanly with cron and any copy tool. For example, a +nightly snapshot copied offsite: + +```cron +0 3 * * * cd /path/to/odysseus && ./scripts/odysseus-backup snapshot --out "/mnt/nas/odysseus-$(date +\%F).tar.gz" +``` + +Swap the `--out` target for `scp`, `rclone`, `s3cmd`, or similar to push the +snapshot to remote storage. + +## Docker vs native installs + +The tool reads `data/` and writes `backups/` relative to the repository root, so +where you run it matters: + +- **Native installs** — run it from the repo root as shown above. `data/` and + `backups/` are both in the repo directory. +- **Docker** — `docker-compose.yml` bind-mounts the host's `./data` to + `/app/data`, so the live data is also present on the host. **Run the tool on + the host** from the repo root; the snapshot reads the bind-mounted `./data` and + writes to `./backups` on the host. Running it *inside* the container is not + recommended, because `backups/` is not a mounted volume and the tarball would + be lost when the container is recreated. + +> **ChromaDB caveat (Docker only).** In the Docker setup, ChromaDB stores its +> vectors in a separate Compose-managed volume (declared as `chromadb-data`), +> **not** under `./data`. `odysseus-backup` therefore does not capture the Docker +> ChromaDB store. Back it up separately if you need it. Compose prefixes the +> volume with the project name, so find the real name first +> (`docker volume ls | grep chromadb`), then archive it — for example: +> +> ```bash +> docker run --rm -v _chromadb-data:/data -v "$PWD":/backup \ +> alpine tar czf /backup/chromadb.tar.gz -C /data . +> ``` +> +> On native installs ChromaDB lives at `data/chroma/` and is included in the +> snapshot normally. diff --git a/docs/chat.gif b/docs/chat.gif deleted file mode 100644 index 90ca0eaac..000000000 Binary files a/docs/chat.gif and /dev/null differ diff --git a/docs/compare.gif b/docs/compare.gif deleted file mode 100644 index 7b939aa01..000000000 Binary files a/docs/compare.gif and /dev/null differ diff --git a/docs/document.gif b/docs/document.gif deleted file mode 100644 index b2a89e435..000000000 Binary files a/docs/document.gif and /dev/null differ diff --git a/docs/index.html b/docs/index.html index 540237840..f740e0bb9 100644 --- a/docs/index.html +++ b/docs/index.html @@ -25,9 +25,16 @@ --radius: 8px; } * { box-sizing: border-box; } - html { scroll-behavior: smooth; scroll-snap-type: y proximity; scroll-padding-top: 60px; } - /* Each section is a full-viewport "page" with its content centered, so only - one shows at a time and the snap is obvious. */ + html { scroll-behavior: smooth; scroll-padding-top: 60px; } + /* REMOVED: "scroll-snap-type: y proximity" + The idea was: >>Each section is a full-viewport "page" with its content centered, + so only one shows at a time and the snap is obvious.<< + + PROBLEM: sections easily grow taller than 100vh IRL + This cause forced jumps mid-read. It's intrusive UX. + The landing-page is not a PowerPoint presentation! + + Preserved: CSS snap-points to avoid destroying code meta-data*/ .hero, section { scroll-snap-align: start; min-height: 100vh; display: flex; flex-direction: column; justify-content: center; diff --git a/docs/notes.gif b/docs/notes.gif deleted file mode 100644 index 891ec2e1b..000000000 Binary files a/docs/notes.gif and /dev/null differ diff --git a/docs/odysseus-wordmark.png b/docs/odysseus-wordmark.png new file mode 100644 index 000000000..dce21eb66 Binary files /dev/null and b/docs/odysseus-wordmark.png differ diff --git a/docs/odysseus.jpg b/docs/odysseus.jpg index 982a00f77..7a70bc5fc 100644 Binary files a/docs/odysseus.jpg and b/docs/odysseus.jpg differ diff --git a/docs/research.gif b/docs/research.gif deleted file mode 100644 index b817eeb1a..000000000 Binary files a/docs/research.gif and /dev/null differ diff --git a/docs/security-ci.md b/docs/security-ci.md new file mode 100644 index 000000000..f21643de5 --- /dev/null +++ b/docs/security-ci.md @@ -0,0 +1,107 @@ +# Security CI guide + +This project runs a set of automated security checks on pull requests and +selected branch pushes. This page explains what each one does, whether it can +block a merge, and the few one-time settings you should turn on to get the full +benefit. + +## What runs, and why + +Most checks live in files under `.github/workflows/`. CodeQL is configured +through GitHub's code scanning default setup, so it appears as a dynamic GitHub +workflow instead of a checked-in workflow file. They run automatically; you do +not start them. + +| Check | What it protects against | Blocks a merge? | +|---|---|---| +| **Secret scan** (gitleaks) | An API key, token, or password being committed by mistake or on purpose | Yes | +| **Workflow security** (actionlint + zizmor) | A broken or insecure automation file that could leak the repo's access token | Yes | +| **Dependency review** | A pull request that adds a software library with a known security hole | Yes | +| **pip-audit** | Known security holes in the Python libraries already used | No (advisory) | +| **Container scan: hadolint** | Mistakes and insecure patterns in the `Dockerfile` | Yes | +| **Container scan: Trivy** | Known security holes in the Docker image | No (advisory) | +| **CodeQL** | Real bugs in the app's own code: injection, auth mistakes, path traversal | No (advisory) | + +"Blocks a merge" means a red X appears on the pull request and, once you enable +the setting below, the **Merge** button is disabled until it is fixed. + +"Advisory" means it reports problems into the repository's **Security** tab so +you can review them on your own schedule, but it never stops a merge. These are +advisory on purpose: they often flag long-standing issues in other people's +libraries, not something a given pull request introduced. + +## Where results appear + +- **Checks tab of a pull request**: the pass/fail of each check. A green tick is + good; a red X needs attention. +- **Security tab of the repository**: detailed findings from the advisory + scanners (Trivy and CodeQL). This is your dashboard. + +## If a check fails + +- **Secret scan failed**: a real credential may have been committed. Treat it as + leaked: rotate (regenerate) that key or token immediately, then remove it from + the file. Do not just delete the commit; assume it was seen. +- **Dependency review failed**: the pull request adds a library with a known + vulnerability. Ask the contributor to use a patched version, or decline the + change. +- **hadolint / workflow security failed**: the contributor changed the + `Dockerfile` or an automation file in a way the linter rejects. Ask them to + address the message shown in the failed check. + +## One-time settings to turn on + +These two settings unlock the full value. You only do them once. + +### 1. Require the blocking checks before merging + +This makes the **Merge** button refuse to work until the gating checks pass. + +1. Go to the repository on GitHub. +2. Click **Settings** (top right of the repo). +3. In the left sidebar, click **Branches**. +4. Under **Branch protection rules**, click **Add branch ruleset** (or **Add + rule**), and set the branch name pattern to `dev` (this is the branch all + pull requests target; `main` is fast-forwarded at releases). +5. Enable **Require status checks to pass before merging**. +6. In the search box that appears, add these checks by name: + - `Python syntax (compileall)` + - `JS syntax (node --check)` + - `gitleaks` + - `actionlint` + - `zizmor (Actions SAST)` + - `hadolint (Dockerfile lint)` + - `dependency-review (PR gate)` + + The first two come from the correctness CI (`ci.yml`); the rest are this + security suite. Leave pytest, pip-audit, Trivy, and CodeQL unchecked so they + stay advisory. +7. Also enable **Require a pull request before merging** and **Require review + from Code Owners** (this uses the `.github/CODEOWNERS` file so every change + needs your sign-off). +8. Click **Create** / **Save changes**. + +Note: a check name only appears in the list after it has run at least once, so +let the workflows run on one pull request first, then add them here. + +### 2. Turn on the Security tab features + +1. **Settings -> Code security** (or **Code security and analysis**). +2. Turn on **Dependency graph** (usually on by default for public repos) -- this + powers Dependency review and Dependabot. +3. Turn on **Dependabot alerts** and **Dependabot security updates**. +4. Under **Code scanning**, use **Set up -> Default** for CodeQL. GitHub then + runs CodeQL as a dynamic workflow without the fork-token limitations that + affect checked-in advanced workflows. + + Do not also add a checked-in CodeQL workflow while default setup is enabled: + GitHub rejects advanced CodeQL uploads when default setup is active. If the + project later needs an advanced CodeQL workflow, disable default setup first + and keep only one CodeQL publishing path active. + +## Keeping it current + +`.github/dependabot.yml` opens small weekly pull requests to update Python and +npm packages, the Docker base image, and the pinned automation actions +themselves. Review and merge those like any other pull request; they keep the +project patched without manual tracking. diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 000000000..c809dc66f --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,425 @@ +# Odysseus Setup Guide + +This page keeps the detailed install, deployment, troubleshooting, and configuration notes out of the front README. + +## Quick Start + +> **Branch note:** `dev` is the default branch and contains the latest development changes, but it may be unstable. For the more stable curated branch, use [`main`](https://github.com/pewdiepie-archdaemon/odysseus/tree/main). + +Defaults work out of the box: clone, run, then configure models/search/email +inside **Settings**. Only edit `.env` for deployment-level overrides like +`APP_BIND`, `APP_PORT`, `AUTH_ENABLED`, `DATABASE_URL`, or a pre-seeded admin password. + +On first setup, Odysseus creates an admin account (`admin` unless +`ODYSSEUS_ADMIN_USER` is set) and prints a temporary password in the terminal. +For Docker installs, the same line is in `docker compose logs odysseus`. +Use that for the first login, then change it in **Settings**. + +Contributing? See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, testing, and +pull request guidelines. + +### Docker (recommended) +```bash +git clone https://github.com/pewdiepie-archdaemon/odysseus.git +cd odysseus +cp .env.example .env # optional, but recommended for explicit defaults +docker compose up -d --build +``` +To include optional extras in the image (PDF viewer, Office extraction; includes AGPL PyMuPDF), build with `docker compose build --build-arg INSTALL_OPTIONAL=true` before `up`. + +Open `http://localhost:7000` when the containers are healthy. Docker Compose +binds the web UI to `127.0.0.1` by default. If the port is taken, set +`APP_PORT=7001` in `.env` and recreate the container. Set `APP_BIND=0.0.0.0` +only when you intentionally want LAN/reverse-proxy access. + +> **On Apple Silicon (M-series) Macs:** Docker can't reach the Metal GPU, so +> Cookbook serves local models on CPU only. For GPU-accelerated model serving, +> run natively instead — see [Apple Silicon](#apple-silicon) below. + +### Native Linux / macOS +```bash +git clone https://github.com/pewdiepie-archdaemon/odysseus.git +cd odysseus +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +python setup.py +python -m uvicorn app:app --host 127.0.0.1 --port 7000 +``` +Requirements: Python 3.11+. Cookbook also needs `tmux` for background model +downloads and serves. The app itself is lightweight; local model serving is the +heavy part and depends on the model, runtime, GPU, and VRAM, so small hosts can +connect to API or remote model servers instead. Use `--host 0.0.0.0` only when you intentionally want LAN/reverse-proxy access. + +### Apple Silicon +Docker on macOS cannot use the Metal GPU. For GPU-accelerated Cookbook on an +M-series Mac, run Odysseus natively: + +```bash +git clone https://github.com/pewdiepie-archdaemon/odysseus.git +cd odysseus +./start-macos.sh +``` + +It launches at `http://127.0.0.1:7860`. To expose it to your phone over a trusted LAN/VPN such as Tailscale, bind all interfaces: + +```bash +ODYSSEUS_HOST=0.0.0.0 ./start-macos.sh +# then open http://:7860 +``` + +The script also reads `.env` at startup, so `APP_BIND=0.0.0.0` and `APP_PORT` +set there are picked up automatically without a command-line override each run. + +Keep `AUTH_ENABLED=true` (the default) before binding outside loopback. Do not +expose this port directly to the public internet. To build a clickable app wrapper: + +```bash +./build-macos-app.sh +``` + +
+Cookbook, GPU, Ollama, and troubleshooting notes + +**Docker bundled services.** Compose starts Odysseus, ChromaDB, SearXNG, and +ntfy. Odysseus and the bundled service ports bind to `127.0.0.1` by default, so +they are reachable from the host but not exposed to your LAN/public internet +unless you opt in. + +**Cookbook storage in Docker.** Downloads live in `./data/huggingface` +(`~/.cache/huggingface` in the container). Cookbook-installed Python CLIs and +serve engines live in `./data/local` (`~/.local` in the container), so they +survive container recreation. + +**Remote servers.** In **Cookbook -> Settings -> Servers**, generate the +Odysseus SSH key and add the public key to the remote server's +`~/.ssh/authorized_keys`. From the host you can also run: + +```bash +ssh-copy-id -i data/ssh/id_ed25519.pub user@server +``` + +**Docker GPU overlays.** CPU-only users can skip this section. Cookbook can +only detect GPUs that Docker exposes to the container — if the host runtime or +device passthrough is not configured, Cookbook sees the iGPU, another card, or +CPU instead of your intended GPU. + +For NVIDIA, `scripts/check-docker-gpu.sh` diagnoses GPU passthrough and can +optionally install the host runtime or update `.env`. + +```bash +# Read-only diagnostic (default — installs nothing, never edits .env): +scripts/check-docker-gpu.sh + +# Print OS-specific install commands without running them: +scripts/check-docker-gpu.sh --print-install-commands + +# Install NVIDIA Container Toolkit on Ubuntu/Debian (requires sudo): +scripts/check-docker-gpu.sh --install-nvidia-toolkit + +# Write COMPOSE_FILE to .env (only when GPU passthrough is confirmed working): +scripts/check-docker-gpu.sh --enable-nvidia-overlay + +# Full assisted setup — install toolkit, then enable overlay if passthrough works: +scripts/check-docker-gpu.sh --install-nvidia-toolkit --enable-nvidia-overlay +``` + +Safety notes: +- The app never installs host GPU runtime automatically. +- The app never edits `.env` automatically. +- `.env` is only modified when `--enable-nvidia-overlay` is explicitly passed, + and only after GPU passthrough succeeds. `--yes` skips prompts but does not + bypass the passthrough gate. +- `.env.bak.*` backups created by `--enable-nvidia-overlay` are ignored by + Git and the Docker build context. + +To enable manually without the script, add this to `.env`: + +```bash +COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml +``` + +**AMD / ROCm.** AMD setup is read-only diagnostic plus manual `.env` edit. Run: + +```bash +scripts/check-docker-amd-gpu.sh +``` + +Then add the reported values to `.env`, replacing `RENDER_GID` with your host's +numeric render group id: + +```bash +COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml +RENDER_GID=989 +``` + +For NVIDIA/AMD GPU support, also read the comments in the selected overlay file: docker/gpu.nvidia.yml or docker/gpu.amd.yml. + +**Stack-management UIs (Portainer, Coolify, Dockhand, etc.).** These tools +often accept only a single Compose file and do not reliably honor `COMPOSE_FILE` +or multiple `-f` overlays. CLI users should keep using the `COMPOSE_FILE` +overlay workflow above. For stack UIs, point the stack at one of the standalone +files instead, which bundle the base stack plus the GPU settings: + +- `docker-compose.gpu-nvidia.yml` — still requires the NVIDIA Container Toolkit + on the host. +- `docker-compose.gpu-amd.yml` — still requires host ROCm/kfd/DRI setup, the + `video`/`render` group membership, and `RENDER_GID` when needed. + +The base `docker-compose.yml` plus the `docker/gpu.*.yml` overlays remain the +source of truth; the standalone files mirror them for single-file deployments. + +Verify after enabling either overlay: + +```bash +docker compose exec odysseus nvidia-smi -L # NVIDIA +docker compose exec odysseus sh -lc 'test -e /dev/kfd && test -d /dev/dri && ls -l /dev/kfd /dev/dri/renderD*' # AMD +``` + +> **GPU passthrough ≠ llama.cpp CUDA.** `nvidia-smi` passing inside the +> container confirms Docker GPU access, but llama.cpp also needs `cudart` and +> the CUDA Toolkit at runtime. If Cookbook logs show `Unable to find cudart +> library`, `Could NOT find CUDAToolkit`, `CUDA Toolkit not found`, or +> tensors/layers assigned to CPU, that is a Cookbook/llama.cpp build issue — +> not a Docker passthrough failure. Reinstall the serve engine via +> **Cookbook → Dependencies** to get a CUDA-enabled build. +> +> The same split applies to AMD/ROCm: seeing `/dev/kfd` and `/dev/dri` inside +> the container confirms device passthrough, not ROCm userspace or a +> ROCm-enabled vLLM/llama.cpp build. `rocm-smi` and `rocminfo` are not expected +> inside the slim Odysseus image. + +**Ollama with Docker.** If Ollama runs on the host, add this endpoint in +Settings: + +```text +http://host.docker.internal:11434/v1 +``` + +Ollama must listen outside its own loopback interface: + +```bash +OLLAMA_HOST=0.0.0.0:11434 ollama serve +``` + +This connects Odysseus in Docker to an Ollama server that is already running on +your host machine; it does not start Ollama inside the container. +`host.docker.internal` is Docker's hostname for the host machine from inside the +container. Cookbook **Serve** is a separate workflow for serving downloaded +models through Odysseus/llama.cpp, so Windows users with an existing Ollama +install usually only need to add the endpoint in Settings. + +**Useful checks.** + +```bash +docker compose ps +docker compose logs --tail=120 odysseus +docker compose logs odysseus | grep -E 'ChromaDB|MemoryVectorStore|DEGRADED' +``` + +**macOS details.** `start-macos.sh` installs Homebrew deps, creates the venv, +runs setup, and starts uvicorn on port `7860` because AirPlay often holds +`7000`. It uses llama.cpp/Ollama for Metal. vLLM/SGLang are CUDA/ROCm-only and +do not run on macOS. MLX-only models are not served by Odysseus. + +
+ +### Native Windows + +**One-command launcher** (creates the venv, installs deps, runs setup, starts the +server; safe to re-run): + +```powershell +git clone https://github.com/pewdiepie-archdaemon/odysseus.git +cd odysseus +powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1 +``` + +Or do it by hand: + +```powershell +git clone https://github.com/pewdiepie-archdaemon/odysseus.git +cd odysseus +py -3.11 -m venv venv +venv\Scripts\Activate.ps1 +pip install -r requirements.txt +python setup.py +python -m uvicorn app:app --host 127.0.0.1 --port 7000 +``` + +If `python` points at an older interpreter, use `py -3.12` (or another installed +3.11+ version) for the venv step. + +**Requirements:** Python 3.11+. The core app (chat, agent, memory, documents, +email, calendar, deep research) runs fully native. For full **Cookbook** background +model downloads and the agent shell tool, also install +[Git for Windows](https://git-scm.com/download/win) (provides `bash.exe`). +Local GPU *serving* of vLLM/SGLang needs Linux/WSL2; for a local model on Windows, +[Ollama](https://ollama.com/download) is the easiest path — point Odysseus at +`http://localhost:11434/v1` in Settings. + +Open `http://localhost:7000`, log in with the generated admin password, +and configure everything else inside **Settings**. + +## Troubleshooting & Advanced Setup + +### `chromadb-client` conflicts with embedded ChromaDB +If `chromadb-client` (the lightweight HTTP-only package) is installed alongside the full `chromadb` package, Odysseus starts but ChromaDB silently falls back to HTTP-only mode and fails. + +**Fix:** uninstall `chromadb-client` and force-reinstall the full package: +```bash +./venv/bin/pip uninstall chromadb-client -y +./venv/bin/pip install --force-reinstall chromadb +``` + +### HTTPS + LAN/Tailscale exposure +To expose Odysseus on a local network or Tailscale with HTTPS: +1. Change the bind address to `0.0.0.0` in `.env` (`APP_BIND=0.0.0.0` or `ODYSSEUS_HOST=0.0.0.0`). +2. Generate a locally-trusted cert for your LAN/Tailscale IPs using [mkcert](https://github.com/FiloSottile/mkcert): + ```bash + mkcert -install + mkcert -cert-file cert.pem -key-file key.pem 192.168.1.100 tailscale-ip + ``` +3. Run `uvicorn` with the generated certs: + ```bash + python -m uvicorn app:app --host 0.0.0.0 --port 7000 --ssl-certfile=cert.pem --ssl-keyfile=key.pem + ``` +4. Install the `mkcert` CA on any other device you want to access Odysseus from (e.g., for iOS, email the `rootCA.pem` to yourself, install the profile, and trust it in Certificate Trust Settings). + +### Optional Dependencies +`requirements-optional.txt` contains packages that unlock extra features. It is not installed by default. + +| Package | Feature unlocked | +|---------|-----------------| +| `faster-whisper` | Local speech-to-text (microphone -> text) via the "local" STT provider. | +| `ddgs` | DuckDuckGo as a search provider option. | +| `PyMuPDF` | PDF page rendering in the side viewer panel and form-filling. (Note: AGPL-3.0) | +| `markitdown` | Office/EPUB document text extraction (converts .docx/.xlsx/.pptx/.xls/.epub to Markdown). | + +### Faster, reproducible installs with uv (optional) +[uv](https://docs.astral.sh/uv/) works as a drop-in replacement for the +venv + pip steps in the native install guides, no project changes are needed but this change results in faster installs along with a lockfile for reproducible environments. After [installing `uv`](https://docs.astral.sh/uv/getting-started/installation/), use: + +```bash +uv venv venv --python 3.13 +uv pip install -r requirements.txt +# then continue as usual: python setup.py, uvicorn, ... +``` + +`requirements.txt` is intentionally unpinned, so two installs at different times can produce different package versions. If you want a reproducible environment (e.g. across your own machines, or to roll back after a bad upgrade), snapshot and restore exact versions with: + +```bash +uv pip compile requirements.txt -o requirements.lock # snapshot current resolution +uv pip sync requirements.lock # reproduce it exactly later +``` + +`requirements.lock` is gitignored and platform-specific (compile it on the OS you deploy to). Regenerate it deliberately when you want to take upgrades. The plain `uv pip install -r requirements.txt` keeps following the unpinned requirements like pip does. + +### Outlook / Office 365 email +Odysseus email accounts currently use IMAP/SMTP username-password auth. Outlook +and Microsoft 365 generally require OAuth instead, so normal Microsoft mailbox +passwords will fail. See [docs/email-outlook.md](docs/email-outlook.md) for the +current limitation and the planned integration direction. + +## Security Notes +Odysseus is a self-hosted workspace with powerful local tools: shell access, file uploads, model downloads, web research, email/calendar integrations, and API tokens. Treat it like an admin console. + +- Keep `AUTH_ENABLED=true` for any network-accessible deployment. +- Keep `LOCALHOST_BYPASS=false` outside local development. +- Use `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway. +- Do not expose it directly to the public internet without HTTPS and a trusted reverse proxy or private access layer. +- Keep `.env`, `data/`, `logs/`, databases, uploads, generated media, backups, auth/session files, API keys, and model/provider tokens out of Git and private shares. They are ignored by default. +- Review `data/auth.json` after first boot: disable open signup unless you intentionally want it, make only your own account admin, and keep demo/test accounts non-admin. +- Non-admin users do not get shell/Python/file read/write by default, and admin-only routes/tools such as MCP management, API tokens, webhooks, model/cookbook serving, backup/vault, and app settings are admin-gated. Other features are controlled by per-user privileges, so review each user's privileges before exposing a deployment. +- Rotate any API keys or tokens that were ever pasted into a shared chat, demo, screenshot, or log. +- If you enable API tokens or webhooks, create separate tokens per integration and delete unused ones. +- Prefer binding manual development runs to `127.0.0.1`; bind to `0.0.0.0` only when you intentionally want LAN/reverse-proxy access. +- Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only. Expose only the authenticated Odysseus web/API entrypoint through your trusted proxy or private access layer. +- Before publishing a fork, run `git status --short` and confirm no private files from `.env`, `data/`, `logs/`, uploads, backups, or local databases are staged. + +### Private or proxied deployments +Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and the bundled services to `127.0.0.1` by default, so a typical production/private setup is: + +1. Keep Odysseus on localhost, for example `127.0.0.1:7000`. +2. Terminate HTTPS at a trusted reverse proxy or private access gateway. +3. Put the authenticated Odysseus web/API entrypoint behind that layer. +4. Keep raw service and model ports internal-only. + +Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true`, `LOCALHOST_BYPASS=false`, and `SECURE_COOKIES=true`. +`ALLOWED_ORIGINS` lists exact permitted origins for cross-origin browser/API clients; ordinary same-origin reverse-proxy access usually does not need a special CORS entry. + +Common internal-only ports from the default docs/compose setup: + +| Port | Service | +|---|---| +| `7000` | Odysseus raw app port | +| `8080` | SearXNG | +| `8091` | ntfy | +| `8100` | ChromaDB host port for manual/compose access | +| `11434` | Ollama | +| `8000-8020` | Common local model/provider APIs | + +## Configuration +Most setup is done inside the app with `/setup` or **Settings**. Use `.env` +for deployment-level defaults and secrets you want present before first boot. +Key settings: + +| Variable | Default | Description | +|---|---|---| +| `LLM_HOST` | `localhost` | Your LLM server (e.g. `llm-host.local:8000`) | +| `LLM_HOSTS` | -- | Comma-separated list for model discovery | +| `OPENAI_API_KEY` | -- | Optional OpenAI key. Prefer adding providers in the app unless pre-seeding. | +| `SEARXNG_INSTANCE` | `http://localhost:8080` | SearXNG URL. Docker overrides this to `http://searxng:8080`. | +| `SEARXNG_SECRET` | generated on first Docker boot | Optional SearXNG cookie/CSRF secret. Leave blank unless you need to pin it. | +| `APP_BIND` | `127.0.0.1` | Docker Compose host bind address for the web UI. Use `0.0.0.0` only for intentional LAN/reverse-proxy access. | +| `APP_PORT` | `7000` | Docker Compose host port for the web UI. | +| `APP_DATA_DIR` | `./data` | Docker Compose host directory for application data volumes. | +| `APP_LOGS_DIR` | `./logs` | Docker Compose host directory for application logs. | +| `AUTH_ENABLED` | `true` | Enable/disable login | +| `LOCALHOST_BYPASS` | `false` | Development-only auth bypass for loopback requests. Keep false for shared/network deployments. | +| `ALLOWED_ORIGINS` | `http://localhost,http://127.0.0.1` | Comma-separated exact permitted origins for cross-origin browser/API clients. | +| `SECURE_COOKIES` | `false` | Set true when serving Odysseus through HTTPS at a trusted proxy or private access gateway. | +| `DATABASE_URL` | `sqlite:///./data/app.db` | Database connection string | +| `CHROMADB_HOST` | `localhost` | ChromaDB host for vector memory. Docker overrides this to `chromadb`. | +| `CHROMADB_PORT` | `8100` | ChromaDB port for manual host runs. Docker overrides this to `8000`. | +| `EMBEDDING_URL` | -- | OpenAI-compatible embeddings endpoint | +| `ODYSSEUS_CHAT_UPLOAD_MAX_BYTES` | `10485760` | Chat/agent attachment cap in bytes. Raise for larger local PDFs or text documents. | +| `ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES` | `104857600` | Gallery image upload cap in bytes (100 MB). | +| `ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES` | `26214400` | Gallery transform input cap in bytes (25 MB). | +| `ODYSSEUS_MEMORY_IMPORT_MAX_BYTES` | `10485760` | Memory import file cap in bytes (10 MB). | +| `ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES` | `26214400` | Personal document upload cap in bytes (25 MB). | +| `ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES` | `26214400` | Email compose attachment cap in bytes (25 MB). | +| `ODYSSEUS_STT_MAX_AUDIO_BYTES` | `26214400` | Speech-to-text audio cap in bytes (25 MB). | +| `ODYSSEUS_ICS_MAX_BYTES` | `10485760` | Calendar `.ics` import cap in bytes (10 MB). | + +All upload-limit vars are validated (must be a positive integer) and optional; an invalid value fails fast at startup. + +### Built-in MCP servers (optional setup) + +Odysseus auto-registers a few built-in MCP servers at startup. The npx-based ones (currently the browser server, `@playwright/mcp`) only start when their npm package is already in the local npx cache. If a package isn't cached, that server is skipped with a startup log message explaining what to do, so a fresh install does not block on a multi-minute npm download or hang if Playwright system deps are missing. + +To enable the browser MCP (page navigation, screenshots, vision), run once: + +```bash +npx -y @playwright/mcp@latest --version +``` + +That installs `@playwright/mcp` plus Playwright (~300MB total). Restart Odysseus and the server will register at startup. + +## Architecture +``` +app.py # FastAPI entry point +core/ auth, database, middleware, constants +src/ llm_core, agent_loop, agent_tools, chat_processor, search/ +routes/ chat, session, document, memory, model … endpoints +services/ docs, memory, search, hwfit (Cookbook) … +static/ index.html + app.js + style.css + js/ (modular front-end) +docs/ landing page (index.html) + preview clips +``` + +## Data +All user data lives in `data/` (gitignored): `app.db` (sessions, messages, documents), +`memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`. + +To back up or restore everything in `data/`, see the +[Backup & Restore guide](docs/backup-restore.md). diff --git a/integrations/claude/skills/odysseus/SKILL.md b/integrations/claude/skills/odysseus/SKILL.md index d3b55b3dd..31b40ee01 100644 --- a/integrations/claude/skills/odysseus/SKILL.md +++ b/integrations/claude/skills/odysseus/SKILL.md @@ -102,6 +102,7 @@ python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py POST /api/codex/memory ## Email draft + send +- Prefer `POST /api/codex/emails/draft-document` for agent-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send. - `POST /api/codex/emails/draft` — body matches `SendEmailRequest` (`to`, `cc`, `bcc`, `subject`, `body`, `body_html`, `attachments`, `account_id`, `in_reply_to`, `references`). Requires `email:draft` (or `email:send`). - `POST /api/codex/emails/send` — same body. Requires `email:send`. Never send without explicit user instruction. diff --git a/integrations/claude/skills/odysseus/scripts/odysseus_api.py b/integrations/claude/skills/odysseus/scripts/odysseus_api.py index fcef8a777..8a22eb494 100755 --- a/integrations/claude/skills/odysseus/scripts/odysseus_api.py +++ b/integrations/claude/skills/odysseus/scripts/odysseus_api.py @@ -17,6 +17,11 @@ def _usage() -> int: print(" odysseus_api.py todos add TITLE", file=sys.stderr) print(" odysseus_api.py emails list [limit]", file=sys.stderr) print(" odysseus_api.py emails read UID", file=sys.stderr) + print(" odysseus_api.py emails draft-doc JSON_PAYLOAD", file=sys.stderr) + print(" odysseus_api.py documents list [limit]", file=sys.stderr) + print(" odysseus_api.py documents read DOC_ID", file=sys.stderr) + print(" odysseus_api.py documents create JSON_PAYLOAD", file=sys.stderr) + print(" odysseus_api.py documents delete DOC_ID", file=sys.stderr) print(" odysseus_api.py cookbook tasks", file=sys.stderr) print(" odysseus_api.py cookbook servers", file=sys.stderr) print(" odysseus_api.py cookbook cached [HOST]", file=sys.stderr) @@ -79,6 +84,33 @@ def main() -> int: method = "GET" path = f"/api/codex/emails/{sys.argv[3]}" body = None + elif action in ("draft-doc", "draft_document") and len(sys.argv) >= 4: + method = "POST" + path = "/api/codex/emails/draft-document" + body = " ".join(sys.argv[3:]) + else: + return _usage() + elif command in ("documents", "docs"): + if len(sys.argv) < 3: + return _usage() + action = sys.argv[2].lower() + if action == "list": + method = "GET" + limit = sys.argv[3] if len(sys.argv) >= 4 else "50" + path = f"/api/codex/documents?limit={limit}" + body = None + elif action == "read" and len(sys.argv) >= 4: + method = "GET" + path = f"/api/codex/documents/{sys.argv[3]}" + body = None + elif action == "create" and len(sys.argv) >= 4: + method = "POST" + path = "/api/codex/documents" + body = " ".join(sys.argv[3:]) + elif action == "delete" and len(sys.argv) >= 4: + method = "DELETE" + path = f"/api/codex/documents/{sys.argv[3]}" + body = None else: return _usage() elif command == "cookbook": diff --git a/integrations/codex/scripts/odysseus_api.py b/integrations/codex/scripts/odysseus_api.py index fcef8a777..8a22eb494 100755 --- a/integrations/codex/scripts/odysseus_api.py +++ b/integrations/codex/scripts/odysseus_api.py @@ -17,6 +17,11 @@ def _usage() -> int: print(" odysseus_api.py todos add TITLE", file=sys.stderr) print(" odysseus_api.py emails list [limit]", file=sys.stderr) print(" odysseus_api.py emails read UID", file=sys.stderr) + print(" odysseus_api.py emails draft-doc JSON_PAYLOAD", file=sys.stderr) + print(" odysseus_api.py documents list [limit]", file=sys.stderr) + print(" odysseus_api.py documents read DOC_ID", file=sys.stderr) + print(" odysseus_api.py documents create JSON_PAYLOAD", file=sys.stderr) + print(" odysseus_api.py documents delete DOC_ID", file=sys.stderr) print(" odysseus_api.py cookbook tasks", file=sys.stderr) print(" odysseus_api.py cookbook servers", file=sys.stderr) print(" odysseus_api.py cookbook cached [HOST]", file=sys.stderr) @@ -79,6 +84,33 @@ def main() -> int: method = "GET" path = f"/api/codex/emails/{sys.argv[3]}" body = None + elif action in ("draft-doc", "draft_document") and len(sys.argv) >= 4: + method = "POST" + path = "/api/codex/emails/draft-document" + body = " ".join(sys.argv[3:]) + else: + return _usage() + elif command in ("documents", "docs"): + if len(sys.argv) < 3: + return _usage() + action = sys.argv[2].lower() + if action == "list": + method = "GET" + limit = sys.argv[3] if len(sys.argv) >= 4 else "50" + path = f"/api/codex/documents?limit={limit}" + body = None + elif action == "read" and len(sys.argv) >= 4: + method = "GET" + path = f"/api/codex/documents/{sys.argv[3]}" + body = None + elif action == "create" and len(sys.argv) >= 4: + method = "POST" + path = "/api/codex/documents" + body = " ".join(sys.argv[3:]) + elif action == "delete" and len(sys.argv) >= 4: + method = "DELETE" + path = f"/api/codex/documents/{sys.argv[3]}" + body = None else: return _usage() elif command == "cookbook": diff --git a/integrations/codex/skills/odysseus/SKILL.md b/integrations/codex/skills/odysseus/SKILL.md index 4cff1402e..d4cbdf726 100644 --- a/integrations/codex/skills/odysseus/SKILL.md +++ b/integrations/codex/skills/odysseus/SKILL.md @@ -102,6 +102,7 @@ python3 integrations/codex/scripts/odysseus_api.py POST /api/codex/memory '{"tex ## Email draft + send +- Prefer `POST /api/codex/emails/draft-document` for Codex-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send. - `POST /api/codex/emails/draft` — body matches `SendEmailRequest` (`to`, `cc`, `bcc`, `subject`, `body`, `body_html`, `attachments`, `account_id`, `in_reply_to`, `references`). Requires `email:draft` (or `email:send`). - `POST /api/codex/emails/send` — same body. Requires `email:send`. Never send without explicit user instruction. diff --git a/launch-windows.ps1 b/launch-windows.ps1 index 88ede8d66..263d95127 100644 --- a/launch-windows.ps1 +++ b/launch-windows.ps1 @@ -30,14 +30,26 @@ function Fail($msg) { exit 1 } +function Test-WindowsBashStub($path) { + if (-not $path) { return $false } + $lowered = $path.ToLowerInvariant() + foreach ($stub in @("system32\bash.exe", "sysnative\bash.exe", "windowsapps\bash.exe")) { + if ($lowered.Contains($stub)) { return $true } + } + return $false +} + function Find-GitBash { $cmd = Get-Command bash -ErrorAction SilentlyContinue - if ($cmd) { return $cmd.Source } + if ($cmd -and -not (Test-WindowsBashStub $cmd.Source)) { return $cmd.Source } $roots = @() foreach ($name in @("ProgramFiles", "ProgramW6432", "ProgramFiles(x86)", "LocalAppData")) { $base = [Environment]::GetEnvironmentVariable($name) - if ($base) { $roots += (Join-Path $base "Git") } + if ($base) { + $roots += (Join-Path $base "Git") + if ($name -eq "LocalAppData") { $roots += (Join-Path $base "Programs\Git") } + } } $roots += @("C:\Program Files\Git", "C:\Program Files (x86)\Git") @@ -93,6 +105,14 @@ if (-not $pyExe) { } } +if ($pyExe -like "*WindowsApps*python.exe") { + $pyCmd = Get-Command py -ErrorAction SilentlyContinue + if ($pyCmd) { + $pyExe = $pyCmd.Source + $pyArgs = @("-3.11") + } +} + if (-not $pyExe) { Fail "Couldn't find Python 3.11+ for Windows setup. Install Python 3.11+ (or open the Python launcher with 'py -3.11') from https://www.python.org/downloads/, then re-run this script." } @@ -129,7 +149,20 @@ if (-not (Find-GitBash)) { Write-Host " https://git-scm.com/download/win" -ForegroundColor Yellow } -# 6. Start the server (use `python -m uvicorn` - bare `uvicorn` may not be on PATH) +# 6. Point CUDA_PATH at a real CUDA toolkit so GPU llama-cpp-python can import. +$cudaBase = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA" +if (Test-Path $cudaBase) { + $cudaBest = Get-ChildItem $cudaBase -Directory -ErrorAction SilentlyContinue | + Where-Object { Test-Path (Join-Path $_.FullName "bin") } | + Sort-Object { try { [version]($_.Name -replace "^v", "") } catch { [version]"0.0" } } -Descending | + Select-Object -First 1 + if ($cudaBest) { + $env:CUDA_PATH = $cudaBest.FullName + Write-Host ("Using CUDA_PATH = " + $cudaBest.FullName) -ForegroundColor Cyan + } +} + +# 7. Start the server (use `python -m uvicorn` - bare `uvicorn` may not be on PATH) Write-Step ("Starting Odysseus at http://{0}:{1}" -f $BindHost, $Port) Write-Host "Press Ctrl+C to stop." Write-Host "" diff --git a/launcher.py b/launcher.py new file mode 100644 index 000000000..ba158444f --- /dev/null +++ b/launcher.py @@ -0,0 +1,142 @@ +# launcher.py +"""Dedicated entrypoint for the standalone Windows portable launcher. + +Handles: +- Immediate GUI splash screen creation using tkinter. +- Suppressing console stream crashes in windowed GUI mode via NullWriter. +- Spawning system tray icon via pystray and Pillow (lazy-loaded). +- Auto-opening default browser pointing to the running backend. +- Launching the FastAPI server (importing and running app.py). +""" +import os +import sys +import threading +import time +import webbrowser + +# Define a dummy NullWriter to suppress standard stream crashes (isatty etc.) in GUI mode +class NullWriter: + def write(self, text): + pass + def flush(self): + pass + def isatty(self): + return False + +if sys.stdout is None: + sys.stdout = NullWriter() +if sys.stderr is None: + sys.stderr = NullWriter() + + +splash_root = None + +# If running from a frozen PyInstaller bundle, launch the splash screen IMMEDIATELY +if getattr(sys, 'frozen', False): + import tkinter as tk + + def show_splash_instantly(): + global splash_root + try: + splash_root = tk.Tk() + splash_root.title("Odysseus") + splash_root.overrideredirect(True) + splash_root.configure(bg="#1a1c23") + + # Accented borders + splash_root.config(highlightbackground="#e06c75", highlightcolor="#e06c75", highlightthickness=1) + + w, h = 360, 160 + ws = splash_root.winfo_screenwidth() + hs = splash_root.winfo_screenheight() + x = (ws - w) // 2 + y = (hs - h) // 2 + splash_root.geometry(f"{w}x{h}+{x}+{y}") + + tk.Label(splash_root, text="⛵ Odysseus", font=("Segoe UI", 22, "bold"), bg="#1a1c23", fg="#e06c75").pack(pady=(22, 2)) + tk.Label(splash_root, text="Launching background services...", font=("Segoe UI", 10), bg="#1a1c23", fg="#d1d4e0").pack(pady=2) + tk.Label(splash_root, text="Please wait, this will take a few seconds.", font=("Segoe UI", 8, "italic"), bg="#1a1c23", fg="#5c6370").pack(pady=(12, 0)) + + splash_root.attributes("-topmost", True) + splash_root.mainloop() + except Exception: + pass + + # Launch the GUI splash screen immediately on a background thread + threading.Thread(target=show_splash_instantly, daemon=True).start() + + +def create_tray_image(): + # Generate a beautiful 64x64 icon matching Odysseus brand red accent (#e06c75) + from PIL import Image, ImageDraw + image = Image.new('RGBA', (64, 64), (0, 0, 0, 0)) + dc = ImageDraw.Draw(image) + accent_red = (224, 108, 117, 255) + light_red = (224, 108, 117, 150) + + # Draw premium sailing boat + dc.polygon([(32, 10), (32, 45), (12, 45)], fill=accent_red) + dc.polygon([(32, 18), (32, 45), (48, 45)], fill=light_red) + dc.polygon([(8, 48), (56, 48), (44, 56), (20, 56)], fill=accent_red) + return image + + +def on_open_browser(icon, item, url): + webbrowser.open(url) + + +def on_exit(icon, item): + icon.stop() + os._exit(0) + + +def setup_system_tray(url): + try: + import pystray + icon_img = create_tray_image() + menu = ( + pystray.MenuItem('Open Odysseus', lambda icon, item: on_open_browser(icon, item, url), default=True), + pystray.MenuItem('Exit', on_exit) + ) + tray_icon = pystray.Icon( + "Odysseus", + icon_img, + "Odysseus", + menu + ) + tray_icon.run() + except Exception: + pass + + +def open_browser(url): + # Allow uvicorn and app lifecycles to complete warmups + time.sleep(3.5) + + # Safely close the splash screen + try: + global splash_root + if splash_root: + splash_root.after(0, splash_root.destroy) + except Exception: + pass + + webbrowser.open(url) + + +if __name__ == "__main__": + import uvicorn + # Import the FastAPI app from app.py + from app import app + + bind_host = os.getenv("APP_BIND", "127.0.0.1") + bind_port = int(os.getenv("APP_PORT", "7000")) + url = f"http://{bind_host}:{bind_port}" + + if getattr(sys, 'frozen', False): + # Start browser manager thread + threading.Thread(target=open_browser, args=(url,), daemon=True).start() + # Start system tray manager thread + threading.Thread(target=setup_system_tray, args=(url,), daemon=True).start() + + uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") diff --git a/mcp_servers/email_server.py b/mcp_servers/email_server.py index b807937cd..2611491ae 100644 --- a/mcp_servers/email_server.py +++ b/mcp_servers/email_server.py @@ -23,6 +23,7 @@ import os.path from pathlib import Path from datetime import datetime, timedelta import uuid +from contextvars import ContextVar from mcp.server import Server from mcp.server.stdio import stdio_server @@ -55,6 +56,8 @@ def _uid_fetch_rows(data) -> list: # flat keys when no DB row matches (legacy single-account behaviour). _ACCOUNT_CACHE: dict = {} # key = normalized account selector -> config dict +_MCP_OWNER_ARG = "_odysseus_owner" +_CURRENT_OWNER: ContextVar[str | None] = ContextVar("email_mcp_owner", default=None) def _clean_header_value(value) -> str: @@ -68,6 +71,45 @@ def _db_path() -> Path: return Path(APP_DB) +def _current_owner() -> str: + owner = _CURRENT_OWNER.get() + return str(owner or "").strip() + + +def _account_visible_to_owner(row: dict, owner: str) -> bool: + row_owner = str(row.get("owner") or "").strip() + if row_owner == owner: + return True + if row_owner: + return False + # Legacy ownerless accounts are only visible to a scoped caller when the + # mailbox itself matches the owner, mirroring the HTTP email route fallback. + owner_l = owner.lower() + return owner_l in { + str(row.get("imap_user") or "").strip().lower(), + str(row.get("from_address") or "").strip().lower(), + } + + +def _filter_accounts_for_owner(rows: list[dict]) -> list[dict]: + owner = _current_owner() + if owner: + return [r for r in rows if _account_visible_to_owner(r, owner)] + + owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()} + if len(owners) > 1: + return [] + return rows + + +def _mcp_owner_required(rows: list[dict] | None = None) -> bool: + if _current_owner(): + return False + rows = rows if rows is not None else _read_accounts_from_db() + owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()} + return len(owners) > 1 + + def _load_email_writing_style() -> str: """Return the existing Settings > Email > Writing Style value.""" try: @@ -121,9 +163,8 @@ def _default_document_owner() -> str | None: return None -def _list_accounts_raw() -> list: - """Return list of dicts from the email_accounts table. Empty list if table - missing or empty. Never raises.""" +def _read_accounts_from_db() -> list: + """Return all enabled email account rows. Empty list if missing. Never raises.""" path = _db_path() if not path.exists(): return [] @@ -131,9 +172,10 @@ def _list_accounts_raw() -> list: conn = sqlite3.connect(str(path)) conn.row_factory = sqlite3.Row columns = {r[1] for r in conn.execute("PRAGMA table_info(email_accounts)").fetchall()} + owner_select = "owner" if "owner" in columns else "NULL AS owner" smtp_security_select = "smtp_security" if "smtp_security" in columns else "'' AS smtp_security" rows = conn.execute(f""" - SELECT id, name, is_default, enabled, + SELECT id, {owner_select}, name, is_default, enabled, imap_host, imap_port, imap_user, imap_password, imap_starttls, smtp_host, smtp_port, {smtp_security_select}, smtp_user, smtp_password, from_address FROM email_accounts WHERE enabled = 1 @@ -147,11 +189,15 @@ def _list_accounts_raw() -> list: return [] -def _resolve_account(selector: str | None) -> dict | None: +def _list_accounts_raw() -> list: + """Return owner-visible email account rows for the active MCP call.""" + return _filter_accounts_for_owner(_read_accounts_from_db()) + + +def _resolve_account_from_rows(rows: list[dict], selector: str | None) -> dict | None: """Given a selector (None = default, or a name/user/id string), return the matching row or None. Matching is case-insensitive substring on name + imap_user + from_address, plus exact id match.""" - rows = _list_accounts_raw() if not rows: return None if not selector: @@ -186,6 +232,10 @@ def _resolve_account(selector: str | None) -> dict | None: return None +def _resolve_account(selector: str | None) -> dict | None: + return _resolve_account_from_rows(_list_accounts_raw(), selector) + + def _load_config(account: str | None = None) -> dict: """Return the full config dict for the requested account (or default). @@ -194,7 +244,7 @@ def _load_config(account: str | None = None) -> dict: 2. env vars + settings.json flat keys (legacy) 3. hardcoded fallbacks (localhost:31143 etc.) """ - cache_key = (account or "").strip().lower() or "__default__" + cache_key = (_current_owner(), (account or "").strip().lower() or "__default__") if cache_key in _ACCOUNT_CACHE: return _ACCOUNT_CACHE[cache_key] @@ -223,8 +273,11 @@ def _load_config(account: str | None = None) -> dict: "account_name": None, } - rows = _list_accounts_raw() - row = _resolve_account(account) + raw_rows = _read_accounts_from_db() + rows = _filter_accounts_for_owner(raw_rows) + row = _resolve_account_from_rows(rows, account) + if _current_owner() and raw_rows and not rows: + raise ValueError("No email account is configured for the authenticated owner") if account and rows and not row: available = ", ".join( f"{r.get('name') or r.get('imap_user')} <{r.get('imap_user') or r.get('from_address') or '?'}>" @@ -885,8 +938,109 @@ def _smtp_connect(account=None, cfg=None): return conn +def _read_agent_email_confirm_setting() -> bool: + """True if the user wants agent send_email/reply_to_email calls to be + queued for manual approval instead of SMTPed immediately. Defaults to + True so a fresh install is safe — agents have been observed inventing + signatures and sending to real recipients without the user's review.""" + try: + from src.settings import get_setting + return bool(get_setting("agent_email_confirm", True)) + except Exception: + return True + + +def _stash_agent_draft(*, to, subject, body, in_reply_to=None, references=None, + cc=None, bcc=None, account=None) -> dict: + """Insert the composed email into scheduled_emails with status + 'agent_draft' and a far-future send_at so the scheduled-send poller + never picks it up. Returns the pending payload the model surfaces to + the user (and that the chat UI can render as an approval card).""" + try: + from src.constants import SCHEDULED_EMAILS_DB + except Exception: + return {"success": False, "error": "Pending-email storage unavailable"} + pending_id = uuid.uuid4().hex[:16] + far_future = "9999-12-31T00:00:00" + now = datetime.utcnow().isoformat() + try: + conn = sqlite3.connect(SCHEDULED_EMAILS_DB) + # Touch the schema in case the email-routes init hasn't run yet + # (MCP server can boot independently). + conn.execute(""" + CREATE TABLE IF NOT EXISTS scheduled_emails ( + id TEXT PRIMARY KEY, + to_addr TEXT NOT NULL, + cc TEXT, + bcc TEXT, + subject TEXT, + body TEXT NOT NULL, + in_reply_to TEXT, + references_hdr TEXT, + attachments TEXT, + send_at TEXT NOT NULL, + created_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + error TEXT, + owner TEXT DEFAULT '', + account_id TEXT, + odysseus_kind TEXT + ) + """) + conn.execute(""" + INSERT INTO scheduled_emails + (id, to_addr, cc, bcc, subject, body, in_reply_to, references_hdr, + attachments, send_at, created_at, status, account_id, odysseus_kind, owner) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'agent_draft', ?, ?, ?) + """, ( + pending_id, + to if isinstance(to, str) else ", ".join(to), + cc if isinstance(cc, str) else (", ".join(cc) if cc else None), + bcc if isinstance(bcc, str) else (", ".join(bcc) if bcc else None), + subject or "", + body or "", + in_reply_to or None, + references if isinstance(references, str) else (" ".join(references) if references else None), + "[]", + far_future, + now, + account or None, + "agent_draft", + _current_owner(), + )) + conn.commit() + conn.close() + except Exception as e: + return {"success": False, "error": f"Failed to stash draft: {e}"} + return { + "success": True, + "pending": True, + "pending_id": pending_id, + "to": to if isinstance(to, str) else ", ".join(to), + "subject": subject or "", + "body": body or "", + "message": ( + "✋ Draft staged for your approval — nothing has been sent yet.\n" + "Review the To/Subject/Body above. Reply 'send' to deliver, or " + "'cancel' to discard." + ), + } + + def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, bcc=None, account=None): - """Send an email via SMTP. Returns dict with status.""" + """Send an email via SMTP. Returns dict with status. + + When the `agent_email_confirm` setting is on (the default), the email + is NOT SMTPed — instead it lands in scheduled_emails as an + `agent_draft` row and the user reviews + approves it from the chat + UI. This closes the auto-send hole that let earlier models invent + signatures and ship them to real recipients without confirmation.""" + if _read_agent_email_confirm_setting(): + return _stash_agent_draft( + to=to, subject=subject, body=body, + in_reply_to=in_reply_to, references=references, + cc=cc, bcc=bcc, account=account, + ) send_account, cfg = _resolve_send_config(account) msg = EmailMessage() msg["From"] = _clean_header_value(cfg["from_address"]) @@ -1038,7 +1192,7 @@ def _create_email_draft_document( doc_id = str(uuid.uuid4()) ver_id = str(uuid.uuid4()) doc_title = (title or subject or "Email draft").strip() or "Email draft" - doc_owner = _default_document_owner() + doc_owner = _current_owner() or _default_document_owner() db = SessionLocal() try: @@ -1824,10 +1978,22 @@ async def list_tools() -> list[Tool]: @server.call_tool() async def call_tool(name: str, arguments: dict) -> list[TextContent]: + arguments = dict(arguments) if isinstance(arguments, dict) else {} + owner = str(arguments.pop(_MCP_OWNER_ARG, "") or "").strip() + owner_token = _CURRENT_OWNER.set(owner or None) try: + all_db_accounts = _read_accounts_from_db() + if _mcp_owner_required(all_db_accounts): + return [TextContent( + type="text", + text="Error: email MCP requires an authenticated owner when multiple email account owners are configured.", + )] + if name == "list_email_accounts": - rows = _list_accounts_raw() + rows = _filter_accounts_for_owner(all_db_accounts) if not rows: + if all_db_accounts and owner: + return [TextContent(type="text", text="No email accounts configured for this owner.")] return [TextContent(type="text", text="No email accounts configured. Legacy single-account mode active.")] lines = [f"Found {len(rows)} email account(s):\n"] for r in rows: @@ -2007,6 +2173,16 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: bcc=arguments.get("bcc"), account=acct, ) + if "error" in result: + return [TextContent(type="text", text=f"Error: {result['error']}")] + if result.get("pending"): + return [TextContent( + type="text", + text=( + f"Draft staged for approval (pending id: {result.get('pending_id')}). " + "Nothing has been sent yet. Review and approve it in Odysseus before delivery." + ), + )] acct_note = f" (from {result['account']})" if result.get("account") else "" return [TextContent(type="text", text=f"Sent email to {result['to']} with subject '{result['subject']}'{acct_note}.")] @@ -2182,6 +2358,8 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: except Exception as e: return [TextContent(type="text", text=f"Error: {e}")] + finally: + _CURRENT_OWNER.reset(owner_token) # ── Main ── diff --git a/mcp_servers/memory_server.py b/mcp_servers/memory_server.py index 1f226ad1d..fafbcfc2b 100644 --- a/mcp_servers/memory_server.py +++ b/mcp_servers/memory_server.py @@ -6,6 +6,7 @@ Imports MemoryManager and MemoryVectorStore from the Odysseus codebase. """ import asyncio +import os import sys import time from pathlib import Path @@ -23,6 +24,55 @@ _memory_manager = None _memory_vector = None _initialized = False +_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_MEMORY_OWNER", "ODYSSEUS_MEMORY_OWNER") +_OWNER_SCOPE_ERROR = ( + "Error: Memory MCP owner is not configured for an owner-scoped memory store. " + "Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool." +) + + +def _configured_owner() -> str | None: + for key in _OWNER_ENV_KEYS: + owner = os.environ.get(key, "").strip() + if owner: + return owner + return None + + +def _entry_owner(entry: dict) -> str | None: + owner = entry.get("owner") + if owner is None: + return None + owner_text = str(owner).strip() + return owner_text or None + + +def _owner_scoped_store(entries: list[dict]) -> bool: + return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict)) + + +def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]: + """Return configured owner, all entries, visible entries, and optional error.""" + entries = _memory_manager.load_all() + owner = _configured_owner() + if owner is None and _owner_scoped_store(entries): + return None, entries, [], _OWNER_SCOPE_ERROR + if owner is None: + visible = [ + entry for entry in entries + if isinstance(entry, dict) and _entry_owner(entry) is None + ] + else: + visible = [ + entry for entry in entries + if isinstance(entry, dict) and _entry_owner(entry) == owner + ] + return owner, entries, visible, None + + +def _text_result(text: str) -> list[TextContent]: + return [TextContent(type="text", text=text)] + def _ensure_init(): """Lazy-init memory managers on first use.""" @@ -75,43 +125,46 @@ async def list_tools() -> list[Tool]: @server.call_tool() async def call_tool(name: str, arguments: dict) -> list[TextContent]: if name != "manage_memory": - return [TextContent(type="text", text=f"Unknown tool: {name}")] + return _text_result(f"Unknown tool: {name}") _ensure_init() if not _memory_manager: - return [TextContent(type="text", text="Error: Memory manager not available")] + return _text_result("Error: Memory manager not available") action = arguments.get("action", "") if action == "list": category_filter = arguments.get("category", "") - memories = _memory_manager.load() + _owner, _all_memories, memories, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) if category_filter: memories = [m for m in memories if m.get("category", "").lower() == category_filter.lower()] if not memories: msg = "No memories found" if category_filter: msg += f" in category '{category_filter}'" - return [TextContent(type="text", text=msg + ".")] + return _text_result(msg + ".") + lines = [f"Found {len(memories)} memory entries:\n"] - for m in memories[:100]: + for m in memories: cat = m.get("category", "fact") mid = m.get("id", "?")[:8] text = m.get("text", "") if len(text) > 150: text = text[:150] + "..." lines.append(f"- [{cat}] `{mid}` — {text}") - if len(memories) > 100: - lines.append(f"... and {len(memories) - 100} more") - return [TextContent(type="text", text="\n".join(lines))] + return _text_result("\n".join(lines)) elif action == "add": text = arguments.get("text", "") category = arguments.get("category", "fact") if not text: - return [TextContent(type="text", text="Error: Memory text cannot be empty")] - entry = _memory_manager.add_entry(text, source="ai_agent", category=category) - memories = _memory_manager.load_all() + return _text_result("Error: Memory text cannot be empty") + owner, memories, _visible, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) + entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner) memories.append(entry) _memory_manager.save(memories) if _memory_vector and _memory_vector.healthy: @@ -119,25 +172,28 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: _memory_vector.add(entry["id"], text) except Exception: pass - return [TextContent(type="text", text=f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")] + return _text_result(f"Memory added: [{category}] {text} (id: {entry['id'][:8]})") elif action == "edit": memory_id = arguments.get("memory_id", "") new_text = arguments.get("text", "") if not memory_id or not new_text: - return [TextContent(type="text", text="Error: edit needs memory_id and text")] - memories = _memory_manager.load_all() - found = False + return _text_result("Error: edit needs memory_id and text") + _owner, memories, visible, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) full_id = None - for m in memories: + for m in visible: if m.get("id", "").startswith(memory_id): - m["text"] = new_text - m["timestamp"] = int(time.time()) - found = True full_id = m["id"] break - if not found: - return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")] + if not full_id: + return _text_result(f"Error: Memory '{memory_id}' not found") + for m in memories: + if m.get("id") == full_id: + m["text"] = new_text + m["timestamp"] = int(time.time()) + break _memory_manager.save(memories) if _memory_vector and _memory_vector.healthy and full_id: try: @@ -145,24 +201,26 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: _memory_vector.add(full_id, new_text) except Exception: pass - return [TextContent(type="text", text=f"Memory updated: {new_text}")] + return _text_result(f"Memory updated: {new_text}") elif action == "delete": memory_id = arguments.get("memory_id", "") if not memory_id: - return [TextContent(type="text", text="Error: delete needs memory_id")] - memories = _memory_manager.load_all() + return _text_result("Error: delete needs memory_id") + _owner, memories, visible, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) full_id = None deleted_text = "" deleted_category = "" - for m in memories: + for m in visible: if m.get("id", "").startswith(memory_id): full_id = m["id"] deleted_text = m.get("text", "") deleted_category = m.get("category", "") break if not full_id: - return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")] + return _text_result(f"Error: Memory '{memory_id}' not found") memories = [m for m in memories if m.get("id") != full_id] _memory_manager.save(memories) if _memory_vector and _memory_vector.healthy and full_id: @@ -172,30 +230,32 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: pass cat = f"[{deleted_category}] " if deleted_category else "" snippet = deleted_text if len(deleted_text) <= 120 else deleted_text[:117] + "..." - return [TextContent(type="text", text=f"Memory deleted: {cat}{snippet} (id: {memory_id})")] + return _text_result(f"Memory deleted: {cat}{snippet} (id: {memory_id})") elif action == "search": query = arguments.get("text", "") if not query: - return [TextContent(type="text", text="Error: search needs text (query)")] - memories = _memory_manager.load() + return _text_result("Error: search needs text (query)") + _owner, _all_memories, memories, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) if hasattr(_memory_manager, 'get_relevant_memories'): results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20) else: query_lower = query.lower() results = [m for m in memories if query_lower in m.get("text", "").lower()][:20] if not results: - return [TextContent(type="text", text=f"No memories found matching '{query}'.")] + return _text_result(f"No memories found matching '{query}'.") lines = [f"Found {len(results)} matching memories:\n"] for m in results: cat = m.get("category", "fact") mid = m.get("id", "?")[:8] text = m.get("text", "") lines.append(f"- [{cat}] `{mid}` — {text}") - return [TextContent(type="text", text="\n".join(lines))] + return _text_result("\n".join(lines)) else: - return [TextContent(type="text", text=f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")] + return _text_result(f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search") async def run(): diff --git a/package-lock.json b/package-lock.json index 8e0812dd9..39e4c9964 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,16 +5,16 @@ "packages": { "": { "dependencies": { - "@anthropic-ai/sdk": "^0.98.0" + "@anthropic-ai/sdk": "^0.104.1" }, "devDependencies": { - "@antithesishq/bombadil": "^0.3.2" + "@antithesishq/bombadil": "^0.5.0" } }, "node_modules/@anthropic-ai/sdk": { - "version": "0.98.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.98.0.tgz", - "integrity": "sha512-N7aXtCvC5g6T1Y4V29lJjceu/zTkVkIZF0jdBvagr0TRFHuKeImffalGWEfqZKrvjH+IQbzJWw6TmSmUzrlMgg==", + "version": "0.104.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.104.1.tgz", + "integrity": "sha512-gGACa/+IaiXzRRmF96aOhamoBgapKRBiFWbmmTFP8aMkpaEcuStF+Q61bjo4vPxBM7gqWJNZqsngslRdnLHv0Q==", "license": "MIT", "dependencies": { "json-schema-to-ts": "^3.1.1", @@ -33,11 +33,14 @@ } }, "node_modules/@antithesishq/bombadil": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.3.2.tgz", - "integrity": "sha512-ATy1w9ZY5gbny1H8DFc7rxZitT7DLLLFDiGcRZe+8TQiUrV5tLO+IJGOVNNLp3RpCqjZqSsxGiKoQsx31ipV1g==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.5.0.tgz", + "integrity": "sha512-s0zImmr0iyvSP6QcVLvf40CUiZYIdWBAxiq20uhzujwvfitYa3PGJN652k/pLtVccHM/JrGQxZdvLnihZpltHA==", "dev": true, - "license": "MIT" + "license": "MIT", + "bin": { + "bombadil": "bin/bombadil.js" + } }, "node_modules/@babel/runtime": { "version": "7.29.7", diff --git a/package.json b/package.json index 27ebf0efd..71b622722 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,9 @@ "url": "https://github.com/pewdiepie-archdaemon/odysseus.git" }, "devDependencies": { - "@antithesishq/bombadil": "^0.3.2" + "@antithesishq/bombadil": "^0.5.0" }, "dependencies": { - "@anthropic-ai/sdk": "^0.98.0" + "@anthropic-ai/sdk": "^0.104.1" } } diff --git a/requirements-optional.txt b/requirements-optional.txt index b4b654232..ab21e81ee 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -33,4 +33,4 @@ PyMuPDF # magika (onnxruntime), already a core dep via fastembed. We avoid the # [all]/Azure/audio extras (cloud + heavy). Pinned to a release >30 days old per # the dependency-age discussion in issue #485. -markitdown[docx,pptx,xlsx,xls]==0.1.5 +markitdown[docx,pptx,xlsx,xls]==0.1.6 diff --git a/requirements.txt b/requirements.txt index 2c4072980..493cb5206 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,8 @@ uvicorn python-multipart python-dotenv httpx -pydantic>=2.0 -pydantic-settings>=2.0 +pydantic>=2.13.4 +pydantic-settings>=2.14.1 SQLAlchemy pypdf beautifulsoup4 @@ -43,3 +43,7 @@ qrcode[pil] croniter pytest pytest-asyncio +# starlette.testclient prefers httpx2 since Starlette 1.2.0 and warns on every +# TestClient import when only classic httpx is present. Runtime code keeps +# using `httpx` above; this is test-client only. +httpx2 diff --git a/routes/api_token_routes.py b/routes/api_token_routes.py index 6f8ac2fc9..cbc828731 100644 --- a/routes/api_token_routes.py +++ b/routes/api_token_routes.py @@ -31,6 +31,7 @@ ALLOWED_SCOPES = { TOKEN_PROFILES = { "chat": ["chat"], "codex_todos": ["todos:read", "todos:write"], + "codex_documents": ["documents:read", "documents:write"], "codex_email_drafts": ["email:read", "email:draft", "documents:read", "documents:write"], } @@ -154,14 +155,19 @@ def setup_api_token_routes() -> APIRouter: @router.patch("/tokens/{token_id}") async def update_token(request: Request, token_id: str): require_admin(request) + current_user = get_current_user(request) try: payload = await request.json() except Exception: payload = {} + if not isinstance(payload, dict): + payload = {} with get_db_session() as db: token = db.query(ApiToken).filter(ApiToken.id == token_id).first() if not token: raise HTTPException(404, "Token not found") + if current_user and token.owner != current_user: + raise HTTPException(403, "Not your token") if isinstance(payload.get("name"), str) and payload["name"].strip(): token.name = payload["name"].strip()[:MAX_NAME_LEN] # Only touch scopes when the caller actually sent them. A partial @@ -189,10 +195,14 @@ def setup_api_token_routes() -> APIRouter: @router.delete("/tokens/{token_id}") def delete_token(request: Request, token_id: str): require_admin(request) + current_user = get_current_user(request) with get_db_session() as db: - deleted = db.query(ApiToken).filter(ApiToken.id == token_id).delete() - if not deleted: + token = db.query(ApiToken).filter(ApiToken.id == token_id).first() + if not token: raise HTTPException(404, "Token not found") + if current_user and token.owner != current_user: + raise HTTPException(403, "Not your token") + db.delete(token) _invalidate_cache(request) return {"status": "deleted"} diff --git a/routes/assistant_routes.py b/routes/assistant_routes.py index 17c50163d..0b609e37f 100644 --- a/routes/assistant_routes.py +++ b/routes/assistant_routes.py @@ -16,6 +16,7 @@ from pydantic import BaseModel from core.database import SessionLocal, CrewMember, ScheduledTask from src.auth_helpers import get_current_user +from core.auth import RESERVED_USERNAMES from src.task_scheduler import compute_next_run @@ -89,11 +90,11 @@ def setup_assistant_routes(task_scheduler) -> APIRouter: # check-in tasks seeded. Hitting any /assistant route under one of these # used to seed a full CrewMember + Morning/Midday/Evening tasks under that # owner, which then double-fired alongside the real user's check-ins. - _SYNTHETIC_OWNERS = frozenset({"internal-tool", "api", "demo", "system", ""}) + # RESERVED_USERNAMES covers the same set; the `not owner` guard handles "". async def _get_or_create(owner: str) -> CrewMember: """Return the per-owner assistant CrewMember, creating it on demand.""" - if not owner or owner in _SYNTHETIC_OWNERS: + if not owner or owner in RESERVED_USERNAMES: raise HTTPException(status_code=400, detail=f"Cannot seed assistant for {owner!r}") db = SessionLocal() try: diff --git a/routes/auth_routes.py b/routes/auth_routes.py index b9158c93a..f180f44e9 100644 --- a/routes/auth_routes.py +++ b/routes/auth_routes.py @@ -12,8 +12,8 @@ import re from pathlib import Path from core.atomic_io import atomic_write_json, atomic_write_text -from core.auth import AuthManager -from src.constants import DEEP_RESEARCH_DIR, MEMORY_FILE, SKILLS_DIR +from core.auth import AuthManager, RESERVED_USERNAMES, SetAdminResult +from src.constants import DEEP_RESEARCH_DIR, MEMORY_FILE, PASSWORD_MIN_LENGTH, SKILLS_DIR from src.rate_limiter import RateLimiter from src.settings_scrub import scrub_settings from src.settings import ( @@ -73,6 +73,11 @@ class DeleteUserRequest(BaseModel): class RenameUserRequest(BaseModel): username: str + +class SetAdminRequest(BaseModel): + is_admin: bool + + class SetOpenRegistrationRequest(BaseModel): enabled: bool @@ -97,8 +102,12 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: raise HTTPException(429, "Too many requests — try again later") if auth_manager.is_configured: raise HTTPException(400, "Already configured") - if len(body.password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") + if len(body.password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") + if len(body.username.strip()) < 1: + raise HTTPException(400, "Username is required") + if body.username.lower() in RESERVED_USERNAMES: + raise HTTPException(403, "Username is reserved") ok = await asyncio.to_thread(auth_manager.setup, body.username, body.password) if not ok: raise HTTPException(500, "Setup failed") @@ -113,10 +122,12 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: raise HTTPException(400, "Run setup first") if not auth_manager.signup_enabled: raise HTTPException(403, "Registration is disabled. Ask an admin for an account.") - if len(body.password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") + if len(body.password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") if len(body.username.strip()) < 1: raise HTTPException(400, "Username is required") + if body.username.lower() in RESERVED_USERNAMES: + raise HTTPException(403, "Username is reserved") ok = await asyncio.to_thread(auth_manager.create_user, body.username, body.password, is_admin=False) if not ok: raise HTTPException(409, "Username already taken") @@ -139,6 +150,8 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: raise HTTPException(401, "Invalid 2FA code") # All checks passed — create session (password already verified above) token = await asyncio.to_thread(auth_manager.create_session_trusted, username) + if not token: + raise HTTPException(401, "Invalid credentials") cookie_kwargs = dict( key=SESSION_COOKIE, value=token, @@ -177,13 +190,18 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: pass return result + @router.get("/policy") + async def auth_policy(): + """Return public auth policy constants for the frontend.""" + return auth_manager.policy() + @router.post("/change-password") async def change_password(body: ChangePasswordRequest, request: Request): user = _get_current_user(request) if not user: raise HTTPException(401, "Not authenticated") - if len(body.new_password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") + if len(body.new_password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") current_token = request.cookies.get(SESSION_COOKIE) ok = await asyncio.to_thread(auth_manager.change_password, user, body.current_password, body.new_password) if not ok: @@ -263,8 +281,12 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: user = _get_current_user(request) if not user or not auth_manager.is_admin(user): raise HTTPException(403, "Admin only") - if len(body.password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") + if len(body.password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") + if len(body.username.strip()) < 1: + raise HTTPException(400, "Username is required") + if body.username.lower() in RESERVED_USERNAMES: + raise HTTPException(403, "Username is reserved") ok = auth_manager.create_user(body.username, body.password, body.is_admin) if not ok: raise HTTPException(409, "Username already taken") @@ -416,6 +438,34 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: except Exception as e: logger.warning("Failed to rename memory.json owner references %s -> %s: %s", old_username, new_username, e) + # uploads.json: upload rows use owner metadata for access checks and + # owner-prefixed index keys for dedupe. Rename both so attachments keep + # resolving after the account username changes. + try: + upload_handler = getattr(request.app.state, "upload_handler", None) + rename_owner = getattr(upload_handler, "rename_owner", None) + if callable(rename_owner): + rename_owner(old_username, new_username) + except Exception as e: + logger.warning("Failed to rename upload owner references %s -> %s: %s", old_username, new_username, e) + + # direct personal RAG uploads live in per-owner directories and the + # vector metadata also carries the username used for owner-filtered + # search. Keep both in sync with the auth rename. + try: + from routes.personal_routes import rename_personal_upload_owner + personal_docs_manager = getattr(request.app.state, "personal_docs_manager", None) + if personal_docs_manager is not None: + rag_manager = getattr(personal_docs_manager, "rag_manager", None) + rename_personal_upload_owner( + old_username, + new_username, + personal_docs_manager=personal_docs_manager, + rag_manager=rag_manager, + ) + except Exception as e: + logger.warning("Failed to rename personal RAG upload owner references %s -> %s: %s", old_username, new_username, e) + # skills: SKILL.md frontmatter carries owner: ; the usage # sidecar (_usage.json) keys entries as owner::skill-name. Both must # be updated or the renamed user's Skills panel goes empty. @@ -476,6 +526,31 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: invalidator() return {"ok": True, "username": new_username, "renamed_self": old_username == user} + @router.put("/users/{username}/admin") + async def set_user_admin(username: str, body: SetAdminRequest, request: Request): + """Promote/demote a user to/from admin. Admin only. + + The last remaining admin can't be demoted (no lockout). Self-demotion + is allowed while another admin exists; the `self` flag tells the UI to + reload the acting user into the normal-user view. + """ + user = _get_current_user(request) + if not user or not auth_manager.is_admin(user): + raise HTTPException(403, "Admin only") + result = auth_manager.set_admin(username, body.is_admin, user) + if result is SetAdminResult.USER_NOT_FOUND: + raise HTTPException(404, "User not found") + if result is SetAdminResult.NOT_AUTHORIZED: + raise HTTPException(403, "Admin only") + if result is SetAdminResult.LAST_ADMIN: + raise HTTPException(400, "Cannot demote the last admin") + target = (username or "").strip().lower() + return { + "ok": True, + "is_admin": body.is_admin, + "self": target == (user or "").strip().lower(), + } + @router.post("/signup-toggle", deprecated=True) async def toggle_signup(request: Request): """ diff --git a/routes/calendar_routes.py b/routes/calendar_routes.py index 7b36df06a..87397e6fc 100644 --- a/routes/calendar_routes.py +++ b/routes/calendar_routes.py @@ -11,7 +11,7 @@ from pydantic import BaseModel from sqlalchemy import or_, and_ from dateutil.rrule import rrulestr -from core.database import SessionLocal, CalendarCal, CalendarEvent +from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent from src.auth_helpers import require_user from src.upload_limits import read_upload_limited, ICS_MAX_BYTES @@ -126,6 +126,54 @@ def _resolve_base_uid(uid: str) -> str: raise ValueError("malformed compound UID: missing base before ::") return base + +async def _push_caldav_event_after_commit(owner: str, uid: str, action: str): + """Best-effort CalDAV write-through. Local writes stay authoritative if + the remote server is unreachable; pending flags let /sync retry later.""" + try: + result = {"ok": True} + if action == "create": + from src.caldav_sync import push_event_create + result = await push_event_create(owner, uid) + elif action == "update": + from src.caldav_sync import push_event_update + result = await push_event_update(owner, uid) + elif action == "delete": + from src.caldav_sync import push_event_delete + result = await push_event_delete(owner, uid) + if result and not result.get("ok") and not result.get("skipped"): + raise RuntimeError(result.get("error") or result) + except Exception as e: + logger.warning("CalDAV %s push failed for uid=%s: %s", action, uid, e) + if action in {"create", "update"}: + db = SessionLocal() + try: + ev = _get_or_404_event(db, uid, owner) + ev.caldav_sync_pending = action + db.commit() + except Exception: + db.rollback() + finally: + db.close() + + +def _record_caldav_delete_tombstone(db, ev: CalendarEvent, owner: str) -> None: + if not (ev.calendar and ev.calendar.source == "caldav"): + return + tombstone = db.query(CalendarDeletedEvent).filter( + CalendarDeletedEvent.uid == ev.uid, + CalendarDeletedEvent.owner == owner, + ).first() + if not tombstone: + tombstone = CalendarDeletedEvent(uid=ev.uid, owner=owner) + db.add(tombstone) + tombstone.calendar_id = ev.calendar_id + tombstone.remote_href = ev.remote_href + tombstone.remote_etag = ev.remote_etag + tombstone.caldav_base_url = getattr(ev.calendar, "caldav_base_url", None) + tombstone.summary = ev.summary or "" + tombstone.last_error = None + # ── Pydantic models ── class EventCreate(BaseModel): @@ -843,13 +891,13 @@ def setup_calendar_routes() -> APIRouter: return {"ok": False, "error": str(e)[:200]} @router.post("/sync") - async def sync_caldav_endpoint(request: Request): - """Pull events from the configured CalDAV server into local DB. + async def sync_caldav_endpoint(request: Request, direction: str = "pull"): + """Sync events with the configured CalDAV server. Returns counts + any per-calendar errors. Called by the frontend on calendar open and by the periodic scheduler loop.""" owner = _require_user(request) - from src.caldav_sync import sync_caldav - return await sync_caldav(owner) + from src.caldav_sync import sync_caldav_direction + return await sync_caldav_direction(owner, direction) @router.delete("/calendars/{cal_id}") @@ -1002,19 +1050,12 @@ def setup_calendar_routes() -> APIRouter: is_utc=_is_utc and not data.all_day, rrule=data.rrule or "", color=data.color or None, + caldav_sync_pending="create" if cal.source == "caldav" else None, ) db.add(ev) db.commit() if cal.source == "caldav": - # Push the new event to the remote so it appears on the user's - # other devices — the sync is otherwise pull-only (#800). - from src.caldav_writeback import writeback_event - await writeback_event(owner, cal.source, cal.id, { - "uid": uid, "summary": data.summary, "description": data.description, - "location": data.location, "dtstart": dtstart, "dtend": dtend, - "all_day": data.all_day, "is_utc": _is_utc and not data.all_day, - "rrule": data.rrule or "", - }) + await _push_caldav_event_after_commit(owner, uid, "create") return {"ok": True, "uid": uid} except HTTPException: raise @@ -1060,15 +1101,12 @@ def setup_calendar_routes() -> APIRouter: ev.rrule = data.rrule if data.color is not None: ev.color = data.color if data.color else None + is_caldav = ev.calendar and ev.calendar.source == "caldav" + if is_caldav: + ev.caldav_sync_pending = "update" db.commit() - cal = db.query(CalendarCal).filter(CalendarCal.id == ev.calendar_id).first() - if cal and cal.source == "caldav": - from src.caldav_writeback import writeback_event - await writeback_event(owner, cal.source, cal.id, { - "uid": ev.uid, "summary": ev.summary, "description": ev.description, - "location": ev.location, "dtstart": ev.dtstart, "dtend": ev.dtend, - "all_day": ev.all_day, "is_utc": ev.is_utc, "rrule": ev.rrule or "", - }) + if is_caldav: + await _push_caldav_event_after_commit(owner, base_uid, "update") return {"ok": True} except HTTPException: raise @@ -1089,15 +1127,13 @@ def setup_calendar_routes() -> APIRouter: db = SessionLocal() try: ev = _get_or_404_event(db, base_uid, owner) - # Capture what the remote push needs BEFORE the row is gone. - _cal = db.query(CalendarCal).filter(CalendarCal.id == ev.calendar_id).first() - _is_caldav = bool(_cal and _cal.source == "caldav") - _cal_id, _ev_uid = ev.calendar_id, ev.uid + is_caldav = ev.calendar and ev.calendar.source == "caldav" + if is_caldav: + _record_caldav_delete_tombstone(db, ev, owner) db.delete(ev) db.commit() - if _is_caldav: - from src.caldav_writeback import writeback_event - await writeback_event(owner, "caldav", _cal_id, {"uid": _ev_uid}, delete=True) + if is_caldav: + await _push_caldav_event_after_commit(owner, base_uid, "delete") return {"ok": True} except HTTPException: raise diff --git a/routes/chat_helpers.py b/routes/chat_helpers.py index c32161bb1..60198194a 100644 --- a/routes/chat_helpers.py +++ b/routes/chat_helpers.py @@ -14,7 +14,7 @@ from core.database import Session as DBSession, ModelEndpoint from src.llm_core import normalize_model_id from src.endpoint_resolver import normalize_base from src.context_compactor import maybe_compact, trim_for_context -from src.auth_helpers import get_current_user +from src.auth_helpers import effective_user from src.prompt_security import untrusted_context_message from routes.prefs_routes import _load_for_user as load_prefs_for_user @@ -78,7 +78,7 @@ def _enforce_chat_privileges(request, sess) -> None: which means unrestricted allowed_models / zero cap -> no-op for them. """ try: - user = get_current_user(request) + user = effective_user(request) except Exception: user = None if not user: @@ -160,7 +160,7 @@ async def auto_name_session(session_manager, sess): owner = getattr(sess, "owner", None) t_url, t_model, t_headers = resolve_task_endpoint( - sess.endpoint_url, sess.model, sess.headers, owner=owner, + sess.endpoint_url, sess.model, sess.headers, owner=owner ) if not t_model: logger.debug("[auto-name] No model provided, skipping") @@ -338,11 +338,11 @@ def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, inco def fire_message_event(request, webhook_manager, session_id: str, sess, message: str, compare_mode: bool = False): """Fire webhook and event_bus events for a new user message.""" if webhook_manager and not compare_mode: - asyncio.create_task(webhook_manager.fire("chat.message", { + webhook_manager.fire_and_forget("chat.message", { "session_id": session_id, "model": sess.model, "message": message[:2000], - })) + }) from src.event_bus import fire_event - user = get_current_user(request) + user = effective_user(request) fire_event("message_sent", user) @@ -497,6 +497,29 @@ def _normalize_model_id_from_cache(sess) -> Optional[str]: return None +def _session_is_research_spinoff(sess) -> bool: + """True if this session was created via research "Discuss" spin-off. + + Detected by the primer system message the spin-off endpoint seeds into + history (metadata ``research_spinoff_from``). Such sessions are grounded + on the seeded report, so global memory + personal-doc RAG injection is + suppressed for them (the report is the sole knowledge base). Handles both + ChatMessage objects and plain dicts. + """ + for m in getattr(sess, "history", []) or []: + role = getattr(m, "role", None) + if role is None and isinstance(m, dict): + role = m.get("role") + if role != "system": + continue + md = getattr(m, "metadata", None) + if md is None and isinstance(m, dict): + md = m.get("metadata") + if (md or {}).get("research_spinoff_from"): + return True + return False + + async def build_chat_context( sess, request, @@ -545,8 +568,9 @@ async def build_chat_context( if not incognito: fire_message_event(request, webhook_manager, session_id, sess, message, compare_mode) - # Resolve user prefs - user = get_current_user(request) + # Resolve owner-scoped prefs/context. Browser requests keep the cookie user; + # bearer-token chat requests use the token owner instead of the "api" sentinel. + user = effective_user(request) uprefs = load_prefs_for_user(user) # Memory enabled? @@ -562,9 +586,17 @@ async def build_chat_context( mem_enabled, user, incognito, no_memory, uprefs.get("memory_enabled", "NOT_SET"), ) + # Research-spinoff ("Discuss") sessions are grounded on the seeded report: + # the primer system message IS the knowledge base. Injecting global memory + # or personal-doc RAG on every turn pulls in keyword-matched but off-topic + # facts ("wrong data") and competes with the report, so suppress both here. + is_research_spinoff = _session_is_research_spinoff(sess) + if is_research_spinoff: + mem_enabled = False + # Use RAG? use_rag_val = (str(use_rag).lower() != "false") if use_rag is not None else True - if incognito or not allow_tool_preprocessing: + if incognito or not allow_tool_preprocessing or is_research_spinoff: use_rag_val = False # If pre-fetched search context was provided (compare mode), skip live web search @@ -587,7 +619,7 @@ async def build_chat_context( incognito=incognito, use_skills=skills_enabled, ) - if use_rag is not None: + if use_rag is not None or is_research_spinoff: _preface_kwargs["use_rag"] = use_rag_val preface, rag_sources, web_sources = chat_processor.build_context_preface(**_preface_kwargs) @@ -1081,10 +1113,10 @@ def run_post_response_tasks( # Webhook if webhook_manager and not compare_mode: - asyncio.create_task(webhook_manager.fire("chat.completed", { + webhook_manager.fire_and_forget("chat.completed", { "session_id": session_id, "model": sess.model, "user_message": message, "response": full_response[:2000], - })) + }) # Auto-name if needs_auto_name(sess.name): diff --git a/routes/chat_routes.py b/routes/chat_routes.py index 3e18bf5c6..7fb328ec7 100644 --- a/routes/chat_routes.py +++ b/routes/chat_routes.py @@ -6,7 +6,7 @@ import os import time import logging from datetime import datetime -from typing import Dict, Any, AsyncGenerator, List +from typing import Dict, Any, AsyncGenerator, List, Optional from fastapi import APIRouter, Request, HTTPException, Form, Query from fastapi.responses import StreamingResponse @@ -23,7 +23,7 @@ from src.endpoint_resolver import normalize_base as _normalize_base, build_chat_ from src.session_search import search_session_messages from src.prompt_security import untrusted_context_message from core.exceptions import SessionNotFoundError -from src.auth_helpers import get_current_user +from src.auth_helpers import effective_user, get_current_user from routes.session_routes import _verify_session_owner from routes.document_helpers import _owner_session_filter from core.database import SessionLocal, get_session_mode, set_session_mode @@ -62,6 +62,33 @@ def _stream_set(session_id: str, **fields) -> None: rec.update(fields) +def _resolve_request_workspace(request, raw_value) -> tuple: + """Resolve the posted workspace for this request: (workspace, rejected). + + Privilege is checked BEFORE the path ever touches the filesystem. Only + admin/single-user callers can use the workspace-backed file/shell tools, + so only they get vet_workspace() and the workspace_rejected signal. For + any other caller the submitted value is dropped uniformly, with no vetting + and no event: otherwise the presence/absence of workspace_rejected would + let a non-admin chat caller probe which host paths exist. + + vet_workspace rejects non-directories, sensitive roots (.ssh, .gnupg, + ...), and filesystem roots; on rejection there is no confinement and the + default tool-path allowlist applies. The rejected value is surfaced so the + stream can tell an admin client (which believes a workspace is active) + that it was dropped. + """ + requested = (raw_value or "").strip() + if not requested: + return "", "" + from src.tool_security import owner_is_admin_or_single_user + if not owner_is_admin_or_single_user(get_current_user(request)): + return "", "" + from src.tool_execution import vet_workspace + workspace = vet_workspace(requested) or "" + return workspace, (requested if not workspace else "") + + def _session_url_matches_endpoint(session_url: str, endpoint_base: str) -> bool: if not session_url or not endpoint_base: return False @@ -99,7 +126,8 @@ def _clear_orphaned_session_endpoint(sess, owner: str | None = None) -> bool: sess.model = "" sess.headers = {} return True - except Exception: + except Exception as e: + logger.warning("Failed to clear orphaned session endpoint", exc_info=e) db.rollback() return False finally: @@ -117,7 +145,8 @@ def _endpoint_cache_contains_model(endpoint, model: str) -> bool: return True try: models = json.loads(raw) if isinstance(raw, str) else raw - except Exception: + except Exception as e: + logger.warning("Failed to parse cached models list, treating as containing model", exc_info=e) return True if not isinstance(models, list) or not models: return True @@ -209,7 +238,8 @@ def _recover_empty_session_model(sess, session_id: str, owner: str | None = None is_chatgpt_subscription = False try: cached = json.loads(ep.cached_models) if isinstance(ep.cached_models, str) else (ep.cached_models or []) - except Exception: + except Exception as e: + logger.warning("Failed to parse cached_models for endpoint %r", getattr(ep, "id", "?"), exc_info=e) cached = [] if not cached: visible = [] @@ -333,7 +363,7 @@ def setup_chat_routes( sess = session_manager.get_session(session) except KeyError: raise HTTPException(404, f"Session '{session}' not found") - owner = get_current_user(request) + owner = effective_user(request) if _clear_orphaned_session_endpoint(sess, owner=owner): raise HTTPException(400, "Selected model endpoint was removed. Pick another model in Settings.") @@ -447,8 +477,11 @@ def setup_chat_routes( use_research = form_data.get("use_research") time_filter = form_data.get("time_filter") preset_id = form_data.get("preset_id") - allow_bash = form_data.get("allow_bash") - allow_web_search = form_data.get("allow_web_search") + # Issue #3229: API callers send JSON, not FormData. Read from the + # JSON body as fallback so callers who send {"allow_bash": true} + # actually get bash enabled. + allow_bash = form_data.get("allow_bash") or (body or {}).get("allow_bash") + allow_web_search = form_data.get("allow_web_search") or (body or {}).get("allow_web_search") use_rag = form_data.get("use_rag") search_context = form_data.get("search_context") # pre-fetched web search results (compare mode) compare_mode = str(form_data.get("compare_mode", "")).lower() == "true" @@ -457,6 +490,10 @@ def setup_chat_routes( # manual form posts that still send plan_mode=true. plan_mode = False chat_mode = str(form_data.get("mode", "")).lower() # 'chat' or 'agent' + # Workspace: confine the agent's file/shell tools to this folder. + workspace, workspace_rejected = _resolve_request_workspace( + request, form_data.get("workspace") + ) # Plan mode is a modifier on agent mode — it only makes sense with tools. if plan_mode: chat_mode = "agent" @@ -492,6 +529,66 @@ def setup_chat_routes( active_doc_id = form_data.get("active_doc_id", "").strip() logger.info(f"[doc-inject] chat_mode={chat_mode}, active_doc_id={active_doc_id!r}") + # Active email reader — when the user has an email open in the UI, the + # frontend passes its uid/folder/account so "reply", "summarize this", + # etc. resolve to the real email instead of the agent inventing a + # fake markdown draft. + active_email_uid = form_data.get("active_email_uid", "").strip() + active_email_folder = form_data.get("active_email_folder", "INBOX").strip() or "INBOX" + active_email_account = form_data.get("active_email_account", "").strip() + active_email_ctx: Optional[Dict[str, str]] = None + # Always reset between requests so a stale active-email pointer from + # a previous turn (different reader closed, different account, etc.) + # can't leak in when the user has no email open this turn. + try: + from src.tool_implementations import clear_active_email + clear_active_email() + except Exception: + pass + if active_email_uid: + active_email_ctx = { + "uid": active_email_uid, + "folder": active_email_folder, + "account": active_email_account, + } + # Try to enrich with subject + from so the agent's system prompt + # block can quote them. Best-effort: a stale cache is fine, a + # missing email just means we pass uid/folder/account only. + try: + from routes.email_routes import _read_cache_get, _read_cache_key + _ck = _read_cache_key(active_email_account or None, active_email_folder, active_email_uid, owner=get_current_user(request)) + _cached_email = _read_cache_get(_ck) + if _cached_email and isinstance(_cached_email, dict): + active_email_ctx["subject"] = str(_cached_email.get("subject") or "") + active_email_ctx["from"] = str( + _cached_email.get("from_address") + or _cached_email.get("from") + or _cached_email.get("from_name") + or "" + ) + _body_preview = (_cached_email.get("body") or "")[:2000] + if _body_preview: + active_email_ctx["body_preview"] = _body_preview + except Exception as _e: + logger.debug(f"[email-inject] cache enrich skipped: {_e}") + # Stash so email tools can resolve "this email" without UID guessing. + try: + from src.tool_implementations import set_active_email + set_active_email( + uid=active_email_uid, + folder=active_email_folder, + account=active_email_account or None, + subject=active_email_ctx.get("subject"), + sender=active_email_ctx.get("from"), + ) + except Exception as _e: + logger.debug(f"[email-inject] set_active_email failed: {_e}") + logger.info( + "[email-inject] active_email uid=%s folder=%s account=%s subject=%r", + active_email_uid, active_email_folder, active_email_account or "(default)", + active_email_ctx.get("subject", ""), + ) + try: # Attachment-only sends: skip the message-required check when the # user has attached one or more files (the attachment IS the action). @@ -506,7 +603,7 @@ def setup_chat_routes( # but BEFORE loading. Prevents cross-user session hijack. _verify_session_owner(request, session) sess = session_manager.get_session(session) - owner = get_current_user(request) + owner = effective_user(request) if _clear_orphaned_session_endpoint(sess, owner=owner): raise HTTPException(400, "Selected model endpoint was removed. Pick another model in Settings.") # Issue #587: picker shows a model from the endpoint cache but @@ -537,7 +634,7 @@ def setup_chat_routes( _enforce_chat_privileges(request, sess) # Ensure session has auth headers - resolve_session_auth(sess, session, owner=get_current_user(request)) + resolve_session_auth(sess, session, owner=effective_user(request)) # Check for research_pending BEFORE mode persist overwrites it do_research = str(use_research).lower() == "true" @@ -552,8 +649,8 @@ def setup_chat_routes( elif attachments: try: att_ids = [str(x) for x in json.loads(attachments)] - except Exception: - pass + except Exception as e: + logger.warning("Failed to parse attachments JSON, ignoring attachments", exc_info=e) no_memory = str(form_data.get("no_memory", "")).lower() == "true" pre_context_tool_policy = build_effective_tool_policy( @@ -607,15 +704,27 @@ def setup_chat_routes( active_doc_id, ) active_doc = None - elif doc_session and doc_session != session: - logger.warning( - "[doc-inject] ignoring stale active_doc_id %s from session %s while in session %s", - active_doc_id, - doc_session, - session, - ) - active_doc = None else: + # NOTE: previously dropped the doc when doc.session_id + # != current chat session — but that broke the common + # case of "open an email draft from one chat, ask a + # different chat to write into it". The frontend only + # sends active_doc_id for docs currently visible in + # the UI, and we already owner-checked above, so trust + # the explicit signal. We just log the mismatch and + # re-bind the doc to the current session so future + # turns find it via the session-fallback path too. + if doc_session and doc_session != session: + logger.info( + "[doc-inject] cross-session active_doc_id %s (was session %s, now %s) — accepting and rebinding", + active_doc_id, doc_session, session, + ) + try: + active_doc.session_id = session + _doc_db.commit() + except Exception as _e: + _doc_db.rollback() + logger.warning(f"[doc-inject] session rebind failed: {_e}") logger.info(f"[doc-inject] found by ID: title={active_doc.title!r}, lang={active_doc.language!r}, is_active={active_doc.is_active}, content_len={len(active_doc.current_content or '')}") else: logger.warning(f"[doc-inject] NOT FOUND by ID {active_doc_id}") @@ -656,9 +765,18 @@ def setup_chat_routes( # Build disabled-tools set from frontend toggles + user privileges disabled_tools = set() - if str(allow_bash).lower() != "true": + # Only disable bash/web_search when the caller *explicitly* set them + # to a falsy value. When unset (None), defer to per-user privilege + # checks below — this lets admins with can_use_bash=True use bash + # by default without having to send allow_bash in every request. + if allow_bash is not None and str(allow_bash).lower() != "true": disabled_tools.add("bash") - if str(allow_web_search).lower() != "true": + _explicit_web_intent = bool(_tool_intent and _tool_intent.category == "web") + if ( + allow_web_search is not None + and str(allow_web_search).lower() != "true" + and not _explicit_web_intent + ): disabled_tools.add("web_search") disabled_tools.add("web_fetch") @@ -671,6 +789,21 @@ def setup_chat_routes( "manage_skills", # skill presets tied to user }) + # Active email reader open → strip the tools that let the agent + # "drift" to a new compose: create_document (writes a fake email- + # shaped .md file) and send_email (sends fresh to a recipient the + # agent invented). With those gone, the only paths left for "write + # email saying X" are ui_control open_email_reply (draft) and + # reply_to_email (immediate send) — both of which use the open + # email's UID. Code-level enforcement instead of relying on a + # prompt rule the model can ignore. + if active_email_ctx and active_email_ctx.get("uid"): + disabled_tools.update({ + "create_document", + "send_email", + "mcp__email__send_email", + }) + # Enforce per-user privileges _privs = {} _user = ctx.user @@ -761,6 +894,13 @@ def setup_chat_routes( # Register active stream for partial-save safety net _active_streams[session] = {"status": "streaming", "partial": "", "query": message, "is_research": effective_do_research, "mode": _effective_mode} + # The client sent a workspace the server refused to bind (deleted + # folder, file path, sensitive dir, filesystem root). Tell it up + # front so the UI can clear the pill instead of displaying a + # confinement that is not actually in effect. + if workspace_rejected: + yield f"data: {json.dumps({'type': 'workspace_rejected', 'data': {'path': workspace_rejected}})}\n\n" + if ctx.preprocessed.attachment_meta: yield f"data: {json.dumps({'type': 'attachments', 'data': ctx.preprocessed.attachment_meta})}\n\n" @@ -1131,6 +1271,7 @@ def setup_chat_routes( max_rounds=_max_rounds, context_length=ctx.context_length, active_document=active_doc, + active_email=active_email_ctx, session_id=session, disabled_tools=disabled_tools if disabled_tools else None, tool_policy=tool_policy, @@ -1138,6 +1279,7 @@ def setup_chat_routes( fallbacks=_fallback_candidates, plan_mode=plan_mode, approved_plan=approved_plan or None, + workspace=workspace or None, ): if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"): try: @@ -1343,7 +1485,7 @@ def setup_chat_routes( if not q or not q.strip(): return [] - _user = get_current_user(request) + _user = effective_user(request) return [ result.to_dict() for result in search_session_messages( diff --git a/routes/codex_routes.py b/routes/codex_routes.py index 1afac02b9..22fc7feeb 100644 --- a/routes/codex_routes.py +++ b/routes/codex_routes.py @@ -18,6 +18,7 @@ from fastapi.responses import StreamingResponse from src.auth_helpers import require_authenticated_request, require_user from src.tool_implementations import do_manage_notes from src.constants import COOKBOOK_STATE_FILE +from routes._validators import validate_remote_host, validate_ssh_port COOKBOOK_READ_SCOPES = {"cookbook:read", "cookbook:launch"} @@ -36,6 +37,25 @@ DOCS_WRITE_SCOPES = {"documents:write"} WRITE_ACTIONS = {"add", "create", "new", "save", "remind", "update", "delete", "toggle_item", "remove", "remove_item"} +def _ssh_prefix_for_task(task: dict) -> tuple[str, str]: + """Resolve a cookbook task's stored SSH target into ``(host, port_flag)``. + + ``host`` is ``""`` for a local task. ``remoteHost`` / ``sshPort`` come from + cookbook_state.json and get interpolated into an ``ssh`` command string, so + validate them the same way the cookbook routes do. A tampered entry with + shell metacharacters in ``remoteHost`` is rejected with 400 rather than + injected. + """ + raw_host = task.get("remoteHost") + raw_port = task.get("sshPort") + host_value = str(raw_host).strip() if raw_host is not None else None + port_value = str(raw_port).strip() if raw_port is not None else None + host = validate_remote_host(host_value or None) or "" + ssh_port = validate_ssh_port(port_value or None) or "" + port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else "" + return host, port_flag + + async def _as_owner(request: Request, owner: str, fn, *args, **kwargs): """Run an existing route handler with request.state.current_user temporarily set to ``owner`` so its internal get_current_user/require_user calls see @@ -75,6 +95,20 @@ def _scope_owner(request: Request, allowed: set[str]) -> str: return require_user(request) +def _scope_owner_all(request: Request, required: set[str]) -> str: + """Return owner only when an API token has every required scope.""" + if getattr(request.state, "api_token", False): + scopes = set(getattr(request.state, "api_token_scopes", []) or []) + missing = required - scopes + if missing: + raise HTTPException(403, f"API token missing required scope: {' and '.join(sorted(missing))}") + owner = getattr(request.state, "api_token_owner", None) + if not owner: + raise HTTPException(403, "API token has no owner") + return owner + return require_user(request) + + def _find_endpoint(router: APIRouter | None, method: str, path: str): if router is None: return None @@ -122,7 +156,7 @@ def setup_codex_routes( "read": scoped(EMAIL_READ_SCOPES), "draft": scoped(EMAIL_DRAFT_SCOPES), "send": scoped(EMAIL_SEND_SCOPES), - "actions": ["list", "read", "draft", "send"], + "actions": ["list", "read", "draft_document", "draft", "send"], }, "memory": { "read": scoped(MEMORY_READ_SCOPES), @@ -246,6 +280,59 @@ def setup_codex_routes( # Both handlers in routes/email_routes.py already accept `owner=` via # FastAPI Depends, so we call them directly without patching state. + def _email_draft_document_content(body: dict[str, Any]) -> str: + def clean(v: Any) -> str: + if isinstance(v, list): + return ", ".join(str(x).strip() for x in v if str(x).strip()) + return str(v or "").strip() + + to = clean(body.get("to")) + cc = clean(body.get("cc")) + bcc = clean(body.get("bcc")) + subject = clean(body.get("subject")) + in_reply_to = clean(body.get("in_reply_to")) + references = clean(body.get("references")) + body_text = str(body.get("body") or body.get("body_html") or "").strip() + lines = [ + f"To: {to}", + ] + if cc: + lines.append(f"Cc: {cc}") + if bcc: + lines.append(f"Bcc: {bcc}") + lines.append(f"Subject: {subject}") + if in_reply_to: + lines.append(f"In-Reply-To: {in_reply_to}") + if references: + lines.append(f"References: {references}") + lines.extend(["---", body_text]) + return "\n".join(lines).rstrip() + "\n" + + @router.post("/emails/draft-document") + async def codex_email_draft_document(request: Request, body: dict[str, Any] = Body(default_factory=dict)): + owner = _scope_owner(request, EMAIL_DRAFT_SCOPES) + docs_owner = _scope_owner_all(request, DOCS_WRITE_SCOPES) + if docs_owner != owner: + raise HTTPException(403, "API token owner mismatch") + if documents_create_endpoint is None: + raise HTTPException(503, "Documents integration is not available") + from routes.document_routes import DocumentCreate + + subject = str(body.get("subject") or "Email draft").strip() or "Email draft" + title = str(body.get("title") or subject).strip() or "Email draft" + req = DocumentCreate( + session_id=body.get("session_id"), + title=title, + language="email", + content=_email_draft_document_content(body), + ) + result = await _as_owner(request, owner, documents_create_endpoint, request, req) + if isinstance(result, dict): + result = dict(result) + result["draft_type"] = "document" + result["send_required_confirmation"] = True + return result + @router.post("/emails/draft") async def codex_email_draft(request: Request, body: dict[str, Any] = Body(default_factory=dict)): owner = _scope_owner(request, EMAIL_DRAFT_SCOPES) @@ -486,8 +573,7 @@ def setup_codex_routes( task = next((t for t in tasks if t.get("sessionId") == session_id), None) if task is None: raise HTTPException(404, "task not found") - host = (task.get("remoteHost") or "").strip() - ssh_port = (task.get("sshPort") or "").strip() + host, port_flag = _ssh_prefix_for_task(task) # Prefer the persisted log file over the tmux pane. The pane gets # overwritten by the post-crash neofetch banner + bash prompt the # moment vllm exits; the log file is the raw stdout/stderr and @@ -499,7 +585,6 @@ def setup_codex_routes( f"else tmux capture-pane -t {session_id} -p -S -{tail}; fi" ) if host: - port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else "" import shlex cmd = f"ssh {port_flag}{host} {shlex.quote(inner)}" else: @@ -561,10 +646,8 @@ def setup_codex_routes( state = _read_cookbook_state() tasks = state.get("tasks") or [] task = next((t for t in tasks if t.get("sessionId") == session_id), None) - host = ((task or {}).get("remoteHost") or "").strip() - ssh_port = ((task or {}).get("sshPort") or "").strip() + host, port_flag = _ssh_prefix_for_task(task or {}) if host: - port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else "" cmd = f"ssh {port_flag}{host} \"tmux kill-session -t {session_id}\"" else: cmd = f"tmux kill-session -t {session_id}" @@ -714,7 +797,7 @@ def setup_codex_routes( norm = dict(body or {}) sess = (norm.get("tmux_session") or norm.get("session_id") or "").strip() model = (norm.get("model") or norm.get("repo_id") or "").strip() - host = (norm.get("host") or norm.get("remote_host") or "").strip() + host = validate_remote_host((norm.get("host") or norm.get("remote_host") or "").strip() or None) or "" port = norm.get("port") or 8000 import re as _re if not sess or not _re.fullmatch(r"[a-zA-Z0-9_-]+", sess): diff --git a/routes/contacts_routes.py b/routes/contacts_routes.py index 58a57a1e1..2b357216a 100644 --- a/routes/contacts_routes.py +++ b/routes/contacts_routes.py @@ -12,6 +12,7 @@ import json import csv import io import os +import inspect import httpx from pathlib import Path from datetime import datetime @@ -45,10 +46,14 @@ def _save_settings(settings): def _get_carddav_config(): import os settings = _load_settings() + password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", "")) + if password and "carddav_password" in settings: + from src.secret_storage import decrypt + password = decrypt(password) return { "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")), "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")), - "password": settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", "")), + "password": password, } @@ -86,11 +91,13 @@ def _normalize_contact(contact: Dict) -> Dict: name = str(contact.get("name") or "").strip() if not name and emails: name = emails[0].split("@")[0] + address = str(contact.get("address") or "").strip() return { "uid": str(contact.get("uid") or uuid.uuid4()), "name": name, "emails": emails, "phones": phones, + "address": address, } @@ -146,7 +153,7 @@ def _parse_vcards(text: str) -> List[Dict]: for block in re.split(r"BEGIN:VCARD", text): if not block.strip(): continue - contact = {"name": "", "emails": [], "phones": [], "uid": ""} + contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""} for line in block.split("\n"): line = line.strip() # Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...") @@ -169,6 +176,15 @@ def _parse_vcards(text: str) -> List[Dict]: phone = _vunesc(name_part.split(":", 1)[1]) if phone and phone not in contact["phones"]: contact["phones"].append(phone) + elif name_part.startswith("ADR"): + # vCard ADR is 7 semicolon-separated components: + # post-office-box;extended-address;street;locality;region;postal-code;country. + # Recover a human-readable string by joining non-empty + # components with ", ". + if ":" in name_part: + raw = name_part.split(":", 1)[1] + parts = [_vunesc(p).strip() for p in raw.split(";")] + contact["address"] = ", ".join(p for p in parts if p) elif name_part.startswith("UID:"): contact["uid"] = _vunesc(name_part[4:]) if contact["name"] or contact["emails"]: @@ -193,7 +209,8 @@ def _vesc(value: str) -> str: def _build_vcard(name: str, email: str, uid: Optional[str] = None, emails: Optional[List[str]] = None, - phones: Optional[List[str]] = None) -> str: + phones: Optional[List[str]] = None, + address: Optional[str] = None) -> str: """Build a vCard. Accepts either a single `email` (legacy callers) or full `emails`/`phones` lists (edit path). The first email is marked PREF=1. All values are RFC-6350-escaped.""" @@ -226,6 +243,12 @@ def _build_vcard(name: str, email: str, uid: Optional[str] = None, lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}") for ph in phone_list: lines.append(f"TEL:{_vesc(ph)}") + # Address: stuff the whole human-readable string into the street + # component of ADR. vCard ADR has 7 semicolon-separated components: + # post-office-box;extended-address;street;locality;region;postal-code;country. + addr = (address or "").strip() + if addr: + lines.append(f"ADR:;;{_vesc(addr)};;;;") lines.append("END:VCARD") return "\r\n".join(lines) + "\r\n" @@ -362,7 +385,7 @@ def _resolve_resource_url(uid: str) -> str: return _lookup() or _vcard_url(uid) -def _create_contact(name: str, email: str) -> bool: +def _create_contact(name: str, email: str, address: str = "") -> bool: """Add a new contact via CardDAV or local contacts.""" cfg = _get_carddav_config() if not _carddav_configured(cfg): @@ -371,12 +394,12 @@ def _create_contact(name: str, email: str) -> bool: for c in contacts: if email_l and email_l in [e.lower() for e in c.get("emails", [])]: return True - contacts.append(_normalize_contact({"name": name, "emails": [email]})) + contacts.append(_normalize_contact({"name": name, "emails": [email], "address": address})) _save_local_contacts(contacts) return True contact_uid = str(uuid.uuid4()) - vcard = _build_vcard(name, email, contact_uid) + vcard = _build_vcard(name, email, contact_uid, address=address) try: url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf" auth = None @@ -609,7 +632,7 @@ def _contacts_to_csv(contacts: List[Dict]) -> str: return out.getvalue() -def _update_contact(uid: str, name: str, emails: List[str], phones: List[str]) -> bool: +def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool: """Rewrite an existing contact via CardDAV or local contacts.""" cfg = _get_carddav_config() if not _carddav_configured(cfg): @@ -618,16 +641,19 @@ def _update_contact(uid: str, name: str, emails: List[str], phones: List[str]) - out = [] for c in contacts: if c.get("uid") == uid: - out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones})) + # Preserve existing address when caller passes "" (only + # updating name/emails/phones, not touching address). + addr = address if address else c.get("address", "") + out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr})) found = True else: out.append(c) if not found: - out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones})) + out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address})) _save_local_contacts(out) return True - vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones) + vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address) # Use the real resource href (handles externally-created contacts whose # filename != UID); falls back to the .vcf guess. try: @@ -714,16 +740,39 @@ def setup_contacts_routes(): """Add a new contact.""" name = (data.get("name") or "").strip() email = (data.get("email") or "").strip() + phone = (data.get("phone") or "").strip() + address = (data.get("address") or "").strip() if not email: return {"success": False, "error": "Email required"} - # Check if already exists - contacts = _fetch_contacts() - for c in contacts: - if email.lower() in [e.lower() for e in c["emails"]]: - return {"success": True, "message": "Already exists", "contact": c} + # Check if already exists by email + if email: + contacts = _fetch_contacts() + for c in contacts: + if email.lower() in [e.lower() for e in c["emails"]]: + return {"success": True, "message": "Already exists", "contact": c} if not name: name = email.split("@")[0] - ok = _create_contact(name, email) + create_params = inspect.signature(_create_contact).parameters + if len(create_params) >= 3: + ok = _create_contact(name, email, address) + else: + ok = _create_contact(name, email) + # If a phone was provided, do an immediate update to thread it + # through (the simple _create_contact signature only takes name + + # email + address; phones happen via update). + if ok and phone: + try: + fresh = _fetch_contacts(force=True) + created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None) + if created: + _update_contact( + created["uid"], name, + created.get("emails", []), + [phone], + address, + ) + except Exception: + pass return {"success": ok} @router.post("/import") @@ -785,7 +834,11 @@ def setup_contacts_routes(): except ValueError as e: raise HTTPException(400, str(e)) else: - settings[key] = data[key] + value = data[key] + if key == "carddav_password" and value: + from src.secret_storage import encrypt + value = encrypt(value) + settings[key] = value _save_settings(settings) # Force re-fetch _contact_cache["fetched_at"] = None @@ -802,7 +855,7 @@ def setup_contacts_routes(): # match PUT /{uid} with uid="config". @router.put("/{uid}") async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)): - """Edit an existing contact — name / emails / phones.""" + """Edit an existing contact — name / emails / phones / address.""" name = (data.get("name") or "").strip() emails = data.get("emails") phones = data.get("phones") @@ -810,11 +863,12 @@ def setup_contacts_routes(): emails = [data["email"]] emails = [e.strip() for e in (emails or []) if e and e.strip()] phones = [p.strip() for p in (phones or []) if p and p.strip()] - if not name and not emails: - return {"success": False, "error": "Name or email required"} + address = (data.get("address") or "").strip() + if not name and not emails and not address: + return {"success": False, "error": "Name, email, or address required"} if not name and emails: name = emails[0].split("@")[0] - ok = _update_contact(uid, name, emails, phones) + ok = _update_contact(uid, name, emails, phones, address) return {"success": ok} @router.delete("/{uid}") diff --git a/routes/cookbook_helpers.py b/routes/cookbook_helpers.py index 53bdde80e..cc2daebdb 100644 --- a/routes/cookbook_helpers.py +++ b/routes/cookbook_helpers.py @@ -1,12 +1,14 @@ """cookbook_helpers.py — validators + small helpers shared by the cookbook routes. Extracted from cookbook_routes.py; the routes module imports the symbols it needs.""" +import json import logging import ntpath import os import posixpath import re import shlex +from pathlib import Path from fastapi import HTTPException from pydantic import BaseModel @@ -90,6 +92,24 @@ def _validate_token(v: str | None) -> str | None: return v +def load_stored_hf_token(*, state_path: Path | str | None = None) -> str: + """Return the decrypted HF token from cookbook_state.json, else env fallback.""" + path = Path(state_path) if state_path else Path(os.environ.get("DATA_DIR", "data")) / "cookbook_state.json" + token = "" + if path.exists(): + try: + state = json.loads(path.read_text(encoding="utf-8")) + env = state.get("env") if isinstance(state, dict) else {} + if isinstance(env, dict) and env.get("hfToken"): + from src.secret_storage import decrypt + token = decrypt(env.get("hfToken") or "") + except Exception: + token = "" + if not token: + token = (os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "").strip() + return token + + def _validate_local_dir(v: str | None) -> str | None: if v is None or v == "": return None @@ -342,7 +362,12 @@ def _user_shell_path_bootstrap() -> list[str]: ' ODYSSEUS_USER_PATH="$("$ODYSSEUS_USER_SHELL" -ic \'printf "__ODYSSEUS_PATH__%s\\n" "$PATH"\' 2>/dev/null | sed -n \'s/^__ODYSSEUS_PATH__//p\' | tail -n 1 || true)"', ' if [ -n "$ODYSSEUS_USER_PATH" ]; then export PATH="$ODYSSEUS_USER_PATH:$PATH"; fi', 'fi', - 'command -v python3 >/dev/null 2>&1 || python3() { python "$@"; }', + # Windows can expose python3 as a Microsoft Store App Execution Alias + # under WindowsApps. Git Bash sees that stub as present, but it exits + # before running Python. A Windows venv usually has python.exe, not + # python3.exe, so treat a missing or WindowsApps python3 as absent. + '_odys_py3="$(command -v python3 2>/dev/null || true)"', + 'case "$_odys_py3" in ""|*[Ww]indows[Aa]pps*) python3() { python "$@"; } ;; esac', 'command -v python >/dev/null 2>&1 || python() { python3 "$@"; }', ] @@ -480,6 +505,8 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache: " if u.startswith('KB'): return int(n * 1024)", " return int(n)", "def scan_ollama():", + " if any(m.get('is_ollama') for m in models): return", + " if os.name == 'nt' and not os.environ.get('ODYSSEUS_ALLOW_OLLAMA_CLI_SCAN'): return", " if not shutil.which('ollama'): return", " try:", " p = subprocess.run(['ollama', 'list'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=6)", @@ -510,8 +537,8 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache: " models.append({'repo_id':name,'size_bytes':size_bytes,'nb_files':1,'has_incomplete':False,'path':'ollama','backend':'ollama','is_ollama':True})", " return", "for _hf_cache in hf_cache_paths(): scan_hf(_hf_cache)", - "scan_ollama()", "scan_ollama_api()", + "scan_ollama()", ] for model_dir in model_dirs or []: lines.append(f"scan_dir(os.path.expanduser({model_dir!r}))") @@ -553,6 +580,36 @@ _GGUF_PRELUDE_RE = re.compile( _OLLAMA_HOST_ASSIGNMENT_RE = re.compile(r"(?:^|\s)OLLAMA_HOST=([^\s]+)") _OLLAMA_BIND_RE = re.compile(r"^\[([^\]]+)\]:(\d+)$|^([^:]+):(\d+)$") _OLLAMA_BIND_HOST_RE = re.compile(r"^[A-Za-z0-9._:-]+$") +_LLAMA_CPP_PYTHON_GGML_TYPES = { + "f32": "0", + "f16": "1", + "q4_0": "2", + "q4_1": "3", + "q5_0": "6", + "q5_1": "7", + "q8_0": "8", + "q8_1": "9", + "q2_k": "10", + "q3_k": "11", + "q4_k": "12", + "q5_k": "13", + "q6_k": "14", + "q8_k": "15", + "iq2_xxs": "16", + "iq2_xs": "17", + "iq3_xxs": "18", + "iq1_s": "19", + "iq4_nl": "20", + "iq3_s": "21", + "iq2_s": "22", + "iq4_xs": "23", + "mxfp4": "39", + "nvfp4": "40", + "q1_0": "41", +} +_LLAMA_CPP_PYTHON_TYPE_FLAG_RE = re.compile( + r"(?P--type_[kv])(?P\s+|=)(?P['\"]?)(?P[A-Za-z0-9_]+)(?P=quote)" +) def _ollama_bind_from_cmd(cmd: str | None, *, default_host: str = "127.0.0.1") -> tuple[str, str]: @@ -584,6 +641,22 @@ def _ollama_bind_from_cmd(cmd: str | None, *, default_host: str = "127.0.0.1") - return f"[{host}]" if bracketed_host else host, port +def _normalize_llama_cpp_python_cache_types(cmd: str | None) -> str | None: + """Map llama.cpp KV cache type names to llama-cpp-python's integer enum.""" + if not cmd or "llama_cpp.server" not in cmd: + return cmd + + def repl(match: re.Match[str]) -> str: + value = match.group("value") + mapped = _LLAMA_CPP_PYTHON_GGML_TYPES.get(value.lower()) + if not mapped: + return match.group(0) + quote = match.group("quote") + return f"{match.group('flag')}{match.group('sep')}{quote}{mapped}{quote}" + + return _LLAMA_CPP_PYTHON_TYPE_FLAG_RE.sub(repl, cmd) + + def _check_serve_binary(seg: str) -> None: """Validate that a single command segment starts with an allowlisted binary (after skipping leading env-var assignments like `CUDA_VISIBLE_DEVICES=0`).""" @@ -722,6 +795,7 @@ def _append_llama_cpp_linux_accel_build_lines(runner_lines: list[str]) -> None: runner_lines.append(' done') # rm -rf build so a prior poisoned CMakeCache.txt (e.g. from a failed CUDA # or HIP attempt) doesn't cause the next configure to reuse stale settings. + runner_lines.append(' mkdir -p ~/bin') runner_lines.append(' cd ~/llama.cpp && rm -rf build') runner_lines.append(' if command -v hipconfig &>/dev/null || [ -d /opt/rocm ] || [ -n "$ROCM_PATH" ] || [ -n "$HIP_PATH" ]; then') runner_lines.append(' if command -v hipconfig &>/dev/null; then') @@ -1026,6 +1100,16 @@ def _diagnose_serve_output(text: str) -> dict | None: "vLLM is not installed or not in PATH on this server.", [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}], ), + ( + r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|" + r"(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|" + r"Please ensure sgl_kernel is properly installed", + "SGLang native dependencies are missing on this server.", + [ + {"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"}, + {"label": "upgrade sglang-kernel after OS packages are installed", "op": "manual"}, + ], + ), ( r"sglang.*command not found|No module named sglang|SGLang is not installed", "SGLang is not installed or not in PATH on this server.", diff --git a/routes/cookbook_output.py b/routes/cookbook_output.py new file mode 100644 index 000000000..b30b18536 --- /dev/null +++ b/routes/cookbook_output.py @@ -0,0 +1,75 @@ +"""Pure helpers for shaping cookbook task output for the status response. + +Kept dependency-free (no FastAPI / SQLAlchemy imports) so the behavior can be +unit-tested without standing up the whole app. +""" + +import re + +_FETCHING_ZERO_FILES_RE = re.compile(r"Fetching\s+0\s+files", re.IGNORECASE) + +# Probe scripts for the dead-session download check, run as +# `python3 -c ` (locally or over SSH). +# cache_root is the task's custom download dir, '' for the default HF cache. +# It has to be passed explicitly: the download runner exports +# HF_HOME=, so that task's cache lives under /hub, and +# the probe process's own environment knows nothing about it. +HF_CACHE_COMPLETE_PROBE = ( + "import os,sys;" + "repo=sys.argv[1];" + "root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';" + "base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));" + "d=os.path.join(base,'models--'+repo.replace('/','--'));" + "snap=os.path.join(d,'snapshots');" + "ok=os.path.isdir(snap) and any(os.path.isdir(os.path.join(snap,x)) and os.listdir(os.path.join(snap,x)) for x in os.listdir(snap));" + "inc=False;" + "blobs=os.path.join(d,'blobs');" + "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));" + "sys.exit(0 if ok and not inc else 1)" +) + +HF_CACHE_INCOMPLETE_PROBE = ( + "import os,sys;" + "repo=sys.argv[1];" + "root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';" + "base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));" + "d=os.path.join(base,'models--'+repo.replace('/','--'));" + "blobs=os.path.join(d,'blobs');" + "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));" + "sys.exit(0 if inc else 1)" +) + + +def classify_dead_download(full_snapshot: str): + """Resolve a dead download session's status from its runner markers. + + The runner prints DOWNLOAD_OK only after exiting 0 (and DOWNLOAD_FAILED + otherwise), so the markers stay trustworthy after the tmux pane is gone. + Returns (status, zero_files), or None when the snapshot carries no marker + and the caller has to fall back to the cache probe. Same precedence as + the live-session branch: DOWNLOAD_OK wins, except a "Fetching 0 files" + run is an error (nothing matched the include/quant pattern). + """ + if not full_snapshot: + return None + if "DOWNLOAD_OK" in full_snapshot: + if _FETCHING_ZERO_FILES_RE.search(full_snapshot): + return ("error", True) + return ("completed", False) + if "DOWNLOAD_FAILED" in full_snapshot: + return ("error", False) + return None + + +def error_aware_output_tail(full_snapshot: str, status: str) -> str: + """Return the trailing slice of a task log for the status response. + + Failed tasks return the last 50 lines so the "Copy last 50 lines" action + surfaces the actual error context (stack traces, build output). Running and + other non-error tasks keep the cheaper 12-line tail to limit the payload on + the 10s polling interval. + """ + if not full_snapshot: + return "" + tail_lines = 50 if status == "error" else 12 + return "\n".join(full_snapshot.splitlines()[-tail_lines:]) diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index 36f98aeae..ea15a22c3 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -30,6 +30,10 @@ from core.platform_compat import ( which_tool, ) from routes.shell_routes import TMUX_LOG_DIR +from routes.cookbook_output import ( + error_aware_output_tail, classify_dead_download, + HF_CACHE_COMPLETE_PROBE, HF_CACHE_INCOMPLETE_PROBE, +) logger = logging.getLogger(__name__) @@ -39,8 +43,13 @@ from routes.cookbook_helpers import ( _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines, _append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script, + load_stored_hf_token, + _append_vllm_linux_preflight_lines, _ollama_bind_from_cmd, _pip_install_fallback_chain, + _pip_install_no_cache, _user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd, + _diagnose_serve_output, run_ssh_command_async, _ollama_bind_from_cmd, _pip_install_fallback_chain, _pip_install_no_cache, _user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd, + _normalize_llama_cpp_python_cache_types, ModelDownloadRequest, ServeRequest, ) @@ -49,7 +58,7 @@ _HF_TOKEN_STATUS_SNIPPET = ( 'echo "[odysseus] HF token: applied"; ' 'else ' 'echo "[odysseus] HF token: NOT SET — gated/private models will be denied. ' - 'Add one in Odysseus Settings -> Cookbook -> HuggingFace Token."; ' + 'Add one in Odysseus Cookbook -> Settings -> HuggingFace Token."; ' 'fi' ) @@ -165,6 +174,16 @@ def setup_cookbook_routes() -> APIRouter: "vLLM is not installed or not in PATH on this server.", [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}], ), + ( + r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|" + r"(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|" + r"Please ensure sgl_kernel is properly installed", + "SGLang native dependencies are missing on this server.", + [ + {"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"}, + {"label": "upgrade sglang-kernel after OS packages are installed", "op": "manual"}, + ], + ), ( r"sglang.*command not found|No module named sglang|SGLang is not installed", "SGLang is not installed or not in PATH on this server.", @@ -233,14 +252,7 @@ def setup_cookbook_routes() -> APIRouter: return state def _load_stored_hf_token() -> str: - if not _cookbook_state_path.exists(): - return "" - try: - state = json.loads(_cookbook_state_path.read_text(encoding="utf-8")) - env = state.get("env") if isinstance(state, dict) else {} - return _decrypt_secret(env.get("hfToken") if isinstance(env, dict) else "") - except Exception: - return "" + return load_stored_hf_token(state_path=_cookbook_state_path) def _cookbook_ssh_dir() -> Path: # The Docker image keeps cookbook keys under /app/.ssh; that path only @@ -355,7 +367,11 @@ def setup_cookbook_routes() -> APIRouter: # all output to the log the poller reads. Paths handed to bash use # POSIX form + shell-quoting so drive paths / spaces survive. inner = TMUX_LOG_DIR / f"{session_id}_run.sh" - inner.write_text("\n".join(bash_lines) + "\n", encoding="utf-8") + pp = shlex.quote(pid_path.as_posix()) + inner.write_text( + f"printf '%s\\n' \"$$\" > {pp}\n" + "\n".join(bash_lines) + "\n", + encoding="utf-8", + ) lp = shlex.quote(log_path.as_posix()) ip = shlex.quote(inner.as_posix()) script_path = TMUX_LOG_DIR / f"{session_id}.sh" @@ -660,7 +676,7 @@ def setup_cookbook_routes() -> APIRouter: _spf = f"-p {_port} " if _port and _port != "22" else "" setup_cmd = ( f"scp -O {_pf}-q '{runner_path}' {remote}:{remote_runner} && " - f"ssh {_spf}{remote} 'chmod +x {remote_runner} && tmux new-session -d -s {session_id} \"./{remote_runner}\"'" + f"ssh {_spf}{remote} 'chmod +x {remote_runner} && tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} \"./{remote_runner}\"'" ) else: # Local: run hf download in the background (tmux on POSIX, a detached @@ -692,7 +708,7 @@ def setup_cookbook_routes() -> APIRouter: lines.append('exec "${SHELL:-/bin/bash}"') wrapper_script.write_text("\n".join(lines) + "\n", encoding="utf-8") wrapper_script.chmod(0o755) - setup_cmd = None if IS_WINDOWS else f"tmux new-session -d -s {session_id} {shlex.quote(str(wrapper_script))}" + setup_cmd = None if IS_WINDOWS else f"tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} {shlex.quote(str(wrapper_script))}" logger.info(f"Model download: {req.repo_id} (backend={'ollama' if is_ollama_download else 'hf'}, include={req.include}, session={session_id}, remote={remote})") logger.info(f"Download setup_cmd: {setup_cmd}") @@ -968,9 +984,9 @@ def setup_cookbook_routes() -> APIRouter: ssh_args = ["ssh"] if ssh_port and ssh_port != "22": ssh_args.extend(["-p", str(ssh_port)]) - capture_cmd = ssh_args + [remote, "tmux", "capture-pane", "-t", session_id, "-p", "-S", "-200"] + capture_cmd = ssh_args + [remote, "tmux", "capture-pane", "-t", session_id, "-p", "-S", "-2000"] else: - capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-200"] + capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-2000"] _exit_re = re.compile(r"=== Process exited with code (-?\d+) ===") for wait_s in _waits: @@ -1213,6 +1229,7 @@ def setup_cookbook_routes() -> APIRouter: # many downstream `"engine" in req.cmd` membership checks can't hit # `TypeError: argument of type 'NoneType'` (a 500 instead of a clean 400). req.cmd = _validate_serve_cmd(req.cmd) or "" + req.cmd = _normalize_llama_cpp_python_cache_types(req.cmd) or "" req.cmd = _venv_safe_local_pip_install_cmd( req.cmd, local=not bool(req.remote_host), @@ -1267,6 +1284,11 @@ def setup_cookbook_routes() -> APIRouter: # LOCAL execution on a native-Windows host never uses tmux (detached # process path below), regardless of the UI-supplied platform. local_windows = IS_WINDOWS and not remote + if is_windows and remote and "diffusion_server.py" in req.cmd: + raise HTTPException( + 400, + "Remote Windows Diffusers serving is not supported yet; use local Windows or a Linux remote server.", + ) if not is_windows and not local_windows and not await _binary_available("tmux", remote, req.ssh_port): return { @@ -1560,10 +1582,10 @@ def setup_cookbook_routes() -> APIRouter: setup_cmd = ( f"{scp_extras}" f"scp -O {_Pf}-q '{runner_path}' {remote}:{remote_runner} && " - f"ssh {_pf}{remote} 'chmod +x {remote_runner} && tmux new-session -d -s {session_id} \"./{remote_runner}\"'" + f"ssh {_pf}{remote} 'chmod +x {remote_runner} && tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} \"./{remote_runner}\"'" ) else: - setup_cmd = f"tmux new-session -d -s {session_id} {shlex.quote(str(runner_path))}" + setup_cmd = f"tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} {shlex.quote(str(runner_path))}" if setup_cmd is None: # LOCAL Windows: launch the bash runner detached; no tmux setup_cmd. @@ -2608,6 +2630,193 @@ def setup_cookbook_routes() -> APIRouter: "error": _ollama_library_cache["error"], } + # ── vLLM recipe scraper ───────────────────────────────────────────── + # Fetches the official YAML recipe for a model from vllm-project/recipes + # and normalizes it into a small JSON the frontend can consume. Cached + # per-repo so the GitHub raw endpoint isn't hammered. + _vllm_recipe_cache: dict[str, tuple[float, dict | None]] = {} + # Manifest of all / ids that have a recipe in the upstream + # repo. Cheap to fetch (one Git Tree API call), so we cache the whole + # set for ~12h. Per-row "does this model have a recipe?" lookups hit + # this set instead of doing 912 individual recipe fetches. + _vllm_recipe_manifest: dict = {"fetched_at": 0.0, "models": set(), "error": ""} + + @router.get("/api/cookbook/vllm-recipe-manifest") + async def vllm_recipe_manifest(refresh: int = 0): + """Return the set of / ids known to have a vLLM recipe. + One GitHub Tree API call, 12h cache. The frontend uses this to badge + rows in the model list before the user expands them.""" + import time as _time + import httpx as _httpx + TTL = 12 * 3600.0 + now = _time.time() + if ( + refresh + or (now - _vllm_recipe_manifest["fetched_at"]) > TTL + or not _vllm_recipe_manifest["models"] + ): + url = ( + "https://api.github.com/repos/vllm-project/recipes/" + "git/trees/main?recursive=1" + ) + def _fetch_sync() -> tuple[int, dict | None, str]: + try: + headers = {"Accept": "application/vnd.github+json"} + with _httpx.Client(timeout=10.0, follow_redirects=True) as client: + r = client.get(url, headers=headers) + if r.status_code != 200: + return r.status_code, None, r.text[:200] + return 200, r.json(), "" + except Exception as e: + return 0, None, f"fetch error: {e}" + status, data, err = await asyncio.to_thread(_fetch_sync) + if status == 200 and isinstance(data, dict): + models: set[str] = set() + for entry in data.get("tree") or []: + path = (entry or {}).get("path") or "" + if not path.startswith("models/") or not path.endswith(".yaml"): + continue + # path = "models//.yaml" → "/" + body = path[len("models/"):-len(".yaml")] + if "/" in body: + models.add(body) + _vllm_recipe_manifest["models"] = models + _vllm_recipe_manifest["fetched_at"] = now + _vllm_recipe_manifest["error"] = "" + else: + _vllm_recipe_manifest["error"] = ( + f"HTTP {status}: {err}" if status else err + ) + # Don't clobber a stale-but-usable list on transient failures. + if not _vllm_recipe_manifest["models"]: + return { + "models": [], + "count": 0, + "error": _vllm_recipe_manifest["error"], + } + return { + "models": sorted(_vllm_recipe_manifest["models"]), + "count": len(_vllm_recipe_manifest["models"]), + "fetched_at": _vllm_recipe_manifest["fetched_at"], + "error": _vllm_recipe_manifest["error"], + } + + @router.get("/api/cookbook/vllm-recipe") + async def vllm_recipe(repo: str, refresh: int = 0): + """Return the vLLM official recipe for a HuggingFace repo, if one + exists at vllm-project/recipes. `repo` is the full HF id like + 'MiniMaxAI/MiniMax-M2'. Cached 6h.""" + import time as _time + import httpx as _httpx + import yaml as _yaml + + TTL = 6 * 3600.0 + now = _time.time() + repo = (repo or "").strip().strip("/") + if "/" not in repo: + return {"exists": False, "error": "repo must be /"} + + cached = _vllm_recipe_cache.get(repo) + if cached and not refresh and (now - cached[0]) < TTL: + return cached[1] or {"exists": False, "cached": True} + + url = ( + f"https://raw.githubusercontent.com/vllm-project/recipes/" + f"main/models/{repo}.yaml" + ) + + def _fetch_sync() -> tuple[int, str]: + try: + with _httpx.Client(timeout=8.0, follow_redirects=True) as client: + r = client.get(url) + return r.status_code, r.text + except Exception as e: + return 0, f"fetch error: {e}" + + status, text = await asyncio.to_thread(_fetch_sync) + if status == 404: + _vllm_recipe_cache[repo] = (now, {"exists": False}) + return {"exists": False} + if status != 200: + return {"exists": False, "error": f"HTTP {status}", "transient": True} + + try: + doc = _yaml.safe_load(text) or {} + except Exception as e: + return {"exists": False, "error": f"yaml parse: {e}"} + + meta = doc.get("meta") or {} + model = doc.get("model") or {} + features = doc.get("features") or {} + deps = doc.get("dependencies") or [] + variants = doc.get("variants") or {} + hw_overrides = doc.get("hardware_overrides") or {} + strat_overrides = doc.get("strategy_overrides") or {} + + # Tool-call + reasoning parsers, as flat arg arrays, so the frontend + # can drop them straight into the launch command. + tool_calling = features.get("tool_calling") or {} + reasoning = features.get("reasoning") or {} + + normalized = { + "exists": True, + "source_url": url, + "title": meta.get("title") or "", + "provider": meta.get("provider") or "", + "description": meta.get("description") or "", + "date_updated": str(meta.get("date_updated") or ""), + "hardware_support": meta.get("hardware") or {}, + "model_id": model.get("model_id") or repo, + "min_vllm_version": model.get("min_vllm_version") or "", + "architecture": model.get("architecture") or "", + "parameter_count": model.get("parameter_count") or "", + "active_parameters": model.get("active_parameters") or "", + "context_length": model.get("context_length") or 0, + "base_args": list(model.get("base_args") or []), + "base_env": dict(model.get("base_env") or {}), + "tool_calling": { + "description": tool_calling.get("description") or "", + "args": list(tool_calling.get("args") or []), + } if tool_calling else None, + "reasoning": { + "description": reasoning.get("description") or "", + "args": list(reasoning.get("args") or []), + } if reasoning else None, + "dependencies": [ + { + "note": (d.get("note") or "").strip(), + "command": (d.get("command") or "").strip(), + "optional": bool(d.get("optional", False)), + } + for d in deps if isinstance(d, dict) + ], + "variants": { + k: { + "model_id": v.get("model_id") or model.get("model_id") or repo, + "precision": v.get("precision") or "", + "vram_minimum_gb": v.get("vram_minimum_gb") or 0, + "description": v.get("description") or "", + "extra_args": list(v.get("extra_args") or []), + "extra_env": dict(v.get("extra_env") or {}), + } + for k, v in variants.items() if isinstance(v, dict) + }, + "hardware_overrides": { + hw: { + "extra_args": list((ov or {}).get("extra_args") or []), + "extra_env": dict((ov or {}).get("extra_env") or {}), + } + for hw, ov in hw_overrides.items() if isinstance(ov, dict) + }, + "strategy_overrides": { + strat: dict(ov or {}) + for strat, ov in strat_overrides.items() if isinstance(ov, dict) + }, + "compatible_strategies": list(doc.get("compatible_strategies") or []), + } + _vllm_recipe_cache[repo] = (now, normalized) + return normalized + @router.get("/api/cookbook/tasks/status") async def cookbook_tasks_status(request: Request): """Check status of all active cookbook tmux sessions. @@ -2622,30 +2831,20 @@ def setup_cookbook_routes() -> APIRouter: def _cookbook_tasks_status_sync(): import subprocess - def _download_cache_complete(repo_id: str, remote_host: str = "", ssh_port: str = "") -> bool: + def _download_cache_complete(repo_id: str, remote_host: str = "", ssh_port: str = "", cache_root: str = "") -> bool: """Best-effort check for a completed HF cache entry. tmux output can stop at a stale progress line if the pane/session disappears before Cookbook captures the final DOWNLOAD_OK marker. In that case, trust the cache shape: a snapshot directory with files and no *.incomplete blobs means HuggingFace finished materializing the - model. + model. cache_root is the task's custom download dir — the runner + pointed HF_HOME there, so the cache lives under /hub, + not wherever this probe's environment says. """ if not repo_id or "/" not in repo_id: return False - py = ( - "import os,sys;" - "repo=sys.argv[1];" - "base=os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub');" - "d=os.path.join(base,'models--'+repo.replace('/','--'));" - "snap=os.path.join(d,'snapshots');" - "ok=os.path.isdir(snap) and any(os.path.isdir(os.path.join(snap,x)) and os.listdir(os.path.join(snap,x)) for x in os.listdir(snap));" - "inc=False;" - "blobs=os.path.join(d,'blobs');" - "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));" - "sys.exit(0 if ok and not inc else 1)" - ) - cmd = ["python3", "-c", py, repo_id] + cmd = ["python3", "-c", HF_CACHE_COMPLETE_PROBE, repo_id, cache_root or ""] try: if remote_host: ssh_base = ["ssh"] @@ -2659,7 +2858,7 @@ def setup_cookbook_routes() -> APIRouter: except Exception: return False - def _download_cache_incomplete(repo_id: str, remote_host: str = "", ssh_port: str = "") -> bool: + def _download_cache_incomplete(repo_id: str, remote_host: str = "", ssh_port: str = "", cache_root: str = "") -> bool: """Best-effort check for resumable HF partial blobs. A lost SSH/tmux session can leave a real download still incomplete. @@ -2668,16 +2867,7 @@ def setup_cookbook_routes() -> APIRouter: """ if not repo_id or "/" not in repo_id: return False - py = ( - "import os,sys;" - "repo=sys.argv[1];" - "base=os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub');" - "d=os.path.join(base,'models--'+repo.replace('/','--'));" - "blobs=os.path.join(d,'blobs');" - "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));" - "sys.exit(0 if inc else 1)" - ) - cmd = ["python3", "-c", py, repo_id] + cmd = ["python3", "-c", HF_CACHE_INCOMPLETE_PROBE, repo_id, cache_root or ""] try: if remote_host: ssh_base = ["ssh"] @@ -2873,6 +3063,7 @@ def setup_cookbook_routes() -> APIRouter: # snapshot to classify (DOWNLOAD_OK / exit marker) — evaluate it even # when the PID is gone instead of blindly reporting "stopped". download_zero_files = False + exit_code = None status = "unknown" download_has_ok = task_type == "download" and "DOWNLOAD_OK" in full_snapshot download_has_failed = task_type == "download" and "DOWNLOAD_FAILED" in full_snapshot @@ -2881,7 +3072,7 @@ def setup_cookbook_routes() -> APIRouter: and ( ".incomplete" in full_snapshot or bool(re.search(r'model-\d+-of-\d+\.[A-Za-z0-9_.-]+:\s+(?:[0-9]|[1-8][0-9])%', full_snapshot)) - or _download_cache_incomplete(_payload.get("repo_id") or model, remote, str(_tport or "")) + or _download_cache_incomplete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "") ) ) if is_alive or (local_win_task and full_snapshot): @@ -2922,11 +3113,19 @@ def setup_cookbook_routes() -> APIRouter: else: status = "running" else: - # Session is dead — check if it completed or crashed - if ( + # Session is dead — check if it completed or crashed. The + # runner markers in the retained output are conclusive + # (DOWNLOAD_OK only prints after exit 0), so check them before + # the cache probe, which can't see ollama pulls at all. + marker = classify_dead_download(full_snapshot) if task_type == "download" else None + if marker is not None: + status, download_zero_files = marker + if status == "completed" and not progress_text: + progress_text = "Download complete" + elif ( task_type == "download" and not download_has_incomplete_evidence - and _download_cache_complete(_payload.get("repo_id") or model, remote, str(_tport or "")) + and _download_cache_complete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "") ): status = "completed" if not progress_text: @@ -2946,7 +3145,7 @@ def setup_cookbook_routes() -> APIRouter: status = "error" if download_zero_files: diagnosis = {"message": "No matching files were downloaded. The model repo or filename/quant pattern may be wrong (for example a ':Q4_K_M' tag that does not exist in the repo). Check the repo and the include/quant pattern."} - output_tail = "\n".join(full_snapshot.splitlines()[-12:]) if full_snapshot else "" + output_tail = error_aware_output_tail(full_snapshot, status) results.append({ "session_id": session_id, @@ -2957,6 +3156,7 @@ def setup_cookbook_routes() -> APIRouter: "phase": serve_phase, "diagnosis": diagnosis, "output_tail": output_tail, + "exit_code": exit_code, "cmd": _payload.get("_cmd") or "", "tps": phase_info.get("tps"), "reqs": phase_info.get("reqs"), diff --git a/routes/diagnostics_routes.py b/routes/diagnostics_routes.py index d6763798d..e6167a80f 100644 --- a/routes/diagnostics_routes.py +++ b/routes/diagnostics_routes.py @@ -1,12 +1,13 @@ """Diagnostics routes — /api/db/stats, /api/rag/stats, /api/test/youtube, /api/test-research.""" import logging +import os from typing import Dict, Any from fastapi import APIRouter, HTTPException, Form, Request from services.youtube.youtube_handler import extract_youtube_id, extract_transcript_async -from core.constants import DEFAULT_HOST +from core.constants import DEFAULT_HOST, DATA_DIR from core.middleware import require_admin logger = logging.getLogger(__name__) @@ -28,6 +29,30 @@ def setup_diagnostics_routes( from src.service_health import collect_service_health return await collect_service_health(rag_manager, memory_vector) + @router.get("/api/diagnostics/logs") + async def get_diagnostics_logs(request: Request, limit: int = 200) -> Dict[str, Any]: + require_admin(request) + limit = max(1, min(limit, 1000)) + try: + log_file = os.path.join(DATA_DIR, "logs", "app.log") + if not os.path.exists(log_file): + return {"status": "success", "logs": []} + + # Safe tail read of the log file (max 5MB via rotation) + with open(log_file, "r", encoding="utf-8", errors="ignore") as f: + lines = f.readlines() + + tail_lines = lines[-limit:] if len(lines) > limit else lines + tail_lines = [line.rstrip('\r\n') for line in tail_lines] + + return { + "status": "success", + "logs": tail_lines + } + except Exception as e: + logger.error(f"Diagnostics logs retrieval error: {e}") + raise HTTPException(500, f"Failed to retrieve logs: {str(e)}") + @router.get("/api/db/stats") async def get_database_stats(request: Request) -> Dict[str, Any]: require_admin(request) diff --git a/routes/document_helpers.py b/routes/document_helpers.py index 57acc50e7..0de4cc2a3 100644 --- a/routes/document_helpers.py +++ b/routes/document_helpers.py @@ -102,8 +102,11 @@ def _owner_session_filter(q, user): The owner backfill runs in init_db before the app serves requests, so by the time this filter is live there are no NULL-owner rows to leak; - we therefore match the owner strictly.""" - if user is None: + we therefore match the owner strictly for authenticated callers.""" + if not user: + from src.auth_helpers import _auth_disabled + if user == "" or _auth_disabled(): + return q return q.filter(False) return q.filter(Document.owner == user) diff --git a/routes/document_routes.py b/routes/document_routes.py index e4598d925..22434c61a 100644 --- a/routes/document_routes.py +++ b/routes/document_routes.py @@ -503,7 +503,8 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: user = get_current_user(request) try: data = await request.json() - except Exception: + except Exception as e: + logger.warning("Failed to parse export request body, defaulting to empty", exc_info=e) data = {} ids = data.get("ids") or [] if not ids: @@ -645,8 +646,8 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: try: from src.agent_tools.document_tools import clear_active_document clear_active_document(doc_id) - except Exception: - pass + except Exception as e: + logger.warning("Failed to clear active document %r on detach", doc_id, exc_info=e) db.commit() db.refresh(doc) return _doc_to_dict(doc) diff --git a/routes/email_helpers.py b/routes/email_helpers.py index 7626b58c2..e33b72182 100644 --- a/routes/email_helpers.py +++ b/routes/email_helpers.py @@ -13,6 +13,8 @@ and `email_pollers.py` (the background loops): """ import os +import base64 +import time import imaplib import smtplib import email as email_mod @@ -38,6 +40,106 @@ from src.secret_storage import decrypt as _decrypt logger = logging.getLogger(__name__) +def _xoauth2_raw(user: str, access_token: str) -> str: + """The SASL XOAUTH2 initial-response string (unencoded). + + Both smtplib.SMTP.auth() and imaplib.IMAP4.authenticate() base64-encode + the value their callback returns, so callers pass this raw form — never + pre-encoded — to avoid double base64. + """ + return f"user={user}\x01auth=Bearer {access_token}\x01\x01" + + +def _xoauth2_bytes(user: str, access_token: str) -> bytes: + """Raw XOAUTH2 bytes for imaplib's authenticate() callback.""" + return _xoauth2_raw(user, access_token).encode() + + +def make_oauth_state(account_id: str, owner: str) -> str: + """Return an HMAC-signed, base64-encoded OAuth state token. + + Encodes account_id + owner + a random nonce, signed with the app secret + so the callback can validate that the flow was initiated by an + authenticated, owning user (CSRF / state-forgery protection). + """ + import hmac as _hmac, hashlib as _hl, secrets as _sec + from src.secret_storage import _load_or_create_key + nonce = _sec.token_hex(16) + payload = json.dumps({"a": account_id, "o": owner, "n": nonce}, separators=(",", ":")) + sig = _hmac.new(_load_or_create_key(), payload.encode(), _hl.sha256).hexdigest() + return base64.urlsafe_b64encode(f"{payload}|{sig}".encode()).decode() + + +def verify_oauth_state(state: str) -> dict | None: + """Verify an OAuth state token's HMAC signature. + + Returns the decoded payload dict ({"a", "o", "n"}) on success, or None if + the token is malformed, tampered, or signed with a different key. + """ + import hmac as _hmac, hashlib as _hl + from src.secret_storage import _load_or_create_key + try: + decoded = base64.urlsafe_b64decode(state.encode()).decode() + payload, sig = decoded.rsplit("|", 1) + expected = _hmac.new(_load_or_create_key(), payload.encode(), _hl.sha256).hexdigest() + if not _hmac.compare_digest(sig, expected): + return None + return json.loads(payload) + except Exception: + return None + + +def _refresh_google_token(account_id: str) -> str | None: + """Exchange the stored refresh token for a new access token and persist it.""" + import httpx + from core.database import SessionLocal as _SL, EmailAccount as _EA + from src.secret_storage import encrypt as _enc, decrypt as _dec + client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "") + client_secret = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET", "") + if not client_id or not client_secret: + return None + db = _SL() + try: + row = db.get(_EA, account_id) + if not row or not row.oauth_refresh_token: + return None + refresh_token = _dec(row.oauth_refresh_token or "") + if not refresh_token: + return None + resp = httpx.post("https://oauth2.googleapis.com/token", data={ + "client_id": client_id, + "client_secret": client_secret, + "refresh_token": refresh_token, + "grant_type": "refresh_token", + }, timeout=10) + resp.raise_for_status() + data = resp.json() + access_token = data["access_token"] + row.oauth_access_token = _enc(access_token) + row.oauth_token_expiry = str(int(time.time()) + data.get("expires_in", 3600)) + db.commit() + return access_token + except Exception: + logger.warning(f"Google token refresh failed for account {account_id}") + return None + finally: + db.close() + + +def _get_valid_google_token(account_id: str, cfg: dict) -> str | None: + """Return a valid Google access token, refreshing if expired or missing.""" + from src.secret_storage import decrypt as _dec + access_token = _dec(cfg.get("oauth_access_token") or "") + expiry_str = cfg.get("oauth_token_expiry") or "" + if access_token and expiry_str: + try: + if int(expiry_str) - 60 > time.time(): + return access_token + except (ValueError, TypeError): + pass + return _refresh_google_token(account_id) + + def _smtp_security_mode(cfg: dict) -> str: raw = str(cfg.get("smtp_security") or "").strip().lower() if raw in {"ssl", "starttls", "none"}: @@ -54,20 +156,29 @@ def _send_smtp_message(cfg: dict, from_addr: str, recipients: list[str], message port = int(cfg.get("smtp_port") or 465) user = cfg.get("smtp_user") or "" password = cfg.get("smtp_password") or "" + + def _auth_smtp(smtp): + if cfg.get("oauth_provider") == "google": + token = _get_valid_google_token(cfg.get("account_id"), cfg) + if not token: + raise RuntimeError("Google OAuth token unavailable — reconnect the account") + smtp.ehlo() + smtp.auth("XOAUTH2", lambda challenge=None: _xoauth2_raw(user, token), initial_response_ok=True) + elif user and password: + smtp.login(user, password) + security = _smtp_security_mode(cfg) if security == "ssl": with smtplib.SMTP_SSL(host, port, timeout=timeout) as smtp: - if user and password: - smtp.login(user, password) + _auth_smtp(smtp) smtp.sendmail(from_addr, recipients, message) return with smtplib.SMTP(host, port, timeout=timeout) as smtp: if security == "starttls": smtp.starttls() - if user and password: - smtp.login(user, password) + _auth_smtp(smtp) smtp.sendmail(from_addr, recipients, message) @@ -304,6 +415,7 @@ OWNER_SCOPED_EMAIL_CACHE_TABLES = { "email_ai_replies", "email_calendar_extractions", "email_urgency_alerts", + "sender_signatures", } @@ -341,6 +453,55 @@ def _ensure_owner_scoped_email_cache_table(conn, table: str, create_sql: str, co _lg.getLogger(__name__).warning(f"{table} owner-migration skipped: {_mig_e}") +def _ensure_sender_signatures_table(conn): + """Create/migrate learned sender signatures to an owner-scoped cache.""" + create_sql = """ + CREATE TABLE IF NOT EXISTS sender_signatures ( + from_address TEXT, + owner TEXT DEFAULT '', + signature_text TEXT, + sample_count INTEGER, + last_built_at TEXT NOT NULL, + model_used TEXT, + source TEXT, + PRIMARY KEY (from_address, owner) + ) + """ + conn.execute(create_sql) + try: + info = conn.execute("PRAGMA table_info(sender_signatures)").fetchall() + cols = [r[1] for r in info] + pk_cols = [r[1] for r in sorted((r for r in info if r[5]), key=lambda r: r[5])] + if "owner" in cols and pk_cols == ["from_address", "owner"]: + return + + conn.execute("ALTER TABLE sender_signatures RENAME TO sender_signatures__old") + conn.execute(create_sql) + old_cols = [r[1] for r in conn.execute("PRAGMA table_info(sender_signatures__old)").fetchall()] + copy_cols = [ + c for c in ( + "from_address", + "signature_text", + "sample_count", + "last_built_at", + "model_used", + "source", + ) + if c in old_cols + ] + source_owner = "COALESCE(owner, '')" if "owner" in old_cols else "''" + conn.execute( + f"INSERT OR IGNORE INTO sender_signatures " + f"({', '.join([*copy_cols, 'owner'])}) " + f"SELECT {', '.join([*copy_cols, source_owner])} " + f"FROM sender_signatures__old" + ) + conn.execute("DROP TABLE sender_signatures__old") + except Exception as _mig_e: + import logging as _lg + _lg.getLogger(__name__).warning(f"sender_signatures owner-migration skipped: {_mig_e}") + + def attachment_extract_dir(folder: str, uid: str) -> Path: """Containment-safe extraction directory for an attachment. @@ -559,20 +720,10 @@ def _init_scheduled_db(): conn.execute("ALTER TABLE email_boundaries ADD COLUMN turns_json TEXT") except Exception: pass - # Per-sender signature cache. Populated by `learn_sender_signatures` - # action: the LLM extracts the common trailing block across N emails - # from each sender; the renderer folds it consistently for every - # future email from that address. - conn.execute(""" - CREATE TABLE IF NOT EXISTS sender_signatures ( - from_address TEXT PRIMARY KEY, - signature_text TEXT, - sample_count INTEGER, - last_built_at TEXT NOT NULL, - model_used TEXT, - source TEXT - ) - """) + # Per-sender signature cache. Populated by `learn_sender_signatures`. + # Message sender addresses are global, so signatures must be scoped to the + # mailbox owner before `/read` returns them to the renderer. + _ensure_sender_signatures_table(conn) conn.commit() conn.close() @@ -661,10 +812,16 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict: "imap_password": _decrypt(row.imap_password or ""), "imap_starttls": bool(row.imap_starttls), "from_address": row.from_address or row.imap_user or "", + "oauth_provider": row.oauth_provider or "", + "oauth_access_token": row.oauth_access_token or "", + "oauth_refresh_token": row.oauth_refresh_token or "", + "oauth_token_expiry": row.oauth_token_expiry or "", + "display_name": row.display_name or "", } - if not (cfg["smtp_host"] and cfg["smtp_user"] and cfg["smtp_password"]): + is_oauth = bool(cfg.get("oauth_provider")) + if not is_oauth and not (cfg["smtp_host"] and cfg["smtp_user"] and cfg["smtp_password"]): logger.warning(f"SMTP not configured for account {row.name!r}") - if not (cfg["imap_host"] and cfg["imap_user"] and cfg["imap_password"]): + if not is_oauth and not (cfg["imap_host"] and cfg["imap_user"] and cfg["imap_password"]): logger.warning(f"IMAP not configured for account {row.name!r}") return cfg finally: @@ -785,12 +942,19 @@ def _imap_connect(account_id: str | None = None, owner: str = "", timeout=timeout, ) try: - conn.login(cfg["imap_user"], cfg["imap_password"]) + if cfg.get("oauth_provider") == "google": + token = _get_valid_google_token(cfg.get("account_id"), cfg) + if not token: + raise RuntimeError("Google OAuth token unavailable — reconnect the account in Settings → Integrations") + conn.authenticate("XOAUTH2", lambda x: _xoauth2_bytes(cfg["imap_user"], token)) + else: + conn.login(cfg["imap_user"], cfg["imap_password"]) except Exception: # A failed AUTHENTICATE (e.g. an Office 365 app password on an - # MFA-enabled tenant, #3174) otherwise orphans the already-connected - # socket; close it before propagating so a misconfigured account - # can't leak one descriptor per retry / background poller pass. + # MFA-enabled tenant, #3174, or an expired/revoked OAuth token) + # otherwise orphans the already-connected socket; close it before + # propagating so a misconfigured account can't leak one descriptor + # per retry / background poller pass. try: conn.shutdown() except Exception: diff --git a/routes/email_routes.py b/routes/email_routes.py index 797a142f2..b9da5a82e 100644 --- a/routes/email_routes.py +++ b/routes/email_routes.py @@ -13,7 +13,9 @@ handlers need. The split is mechanical — no behavior change. """ import asyncio +import os import sqlite3 as _sql3 +import time import email as email_mod import email.header import email.utils @@ -43,6 +45,7 @@ from routes.email_helpers import ( _load_settings, _save_settings, _get_email_config, _send_smtp_message, _smtp_security_mode, _IMAP_TIMEOUT_SECONDS, _open_imap_connection, + make_oauth_state, verify_oauth_state, _imap_connect, _imap, _decode_header, _detect_sent_folder, _detect_drafts_folder, _extract_attachment_text, _list_attachments_from_msg, _extract_attachment_to_disk, _extract_html, _extract_text, @@ -76,15 +79,16 @@ def _email_tag_owner_aliases(account_id: str | None, owner: str = "") -> list[st cfg.get("smtp_user") or "", cfg.get("from_address") or "", ]) - except Exception: + except Exception as _e: + logger.warning("Failed to resolve email account alias", exc_info=_e) resolved_account_id = None row = db.get(_EA, resolved_account_id) if resolved_account_id else None if row: aliases.extend([row.owner or "", row.imap_user or "", row.from_address or ""]) finally: db.close() - except Exception: - pass + except Exception as _e: + logger.warning("Failed to load email aliases", exc_info=_e) out = [] for a in aliases: a = (a or "").strip() @@ -249,8 +253,45 @@ def _uid_from_fetch_meta(meta_b: bytes) -> str: return m.group(1).decode() if m else "" +_FETCH_SEQ_RE = re.compile(rb"^(\d+)\s+\(") + + +def _group_uid_fetch_records(msg_data) -> list: + """Group an imaplib UID FETCH response into per-message (meta, payload). + + imaplib yields an interleaved list: ``(meta, literal)`` tuples for + attributes that carry a literal (``RFC822.HEADER {n}`` etc.) plus bare + ``bytes`` elements for everything the server sends outside a literal. + Where each attribute lands is server-specific: Dovecot sends FLAGS + *before* the header literal (so it ends up inside the tuple meta), while + Gmail sends FLAGS *after* it, arriving as a bare ``b' FLAGS (\\Seen))'`` + element. Dropping bare elements therefore silently loses FLAGS on Gmail + and every message renders as unread/unflagged. + + A tuple whose meta starts with a sequence number opens a new record; + every other part — continuation tuple or bare bytes — is folded into the + current record's meta so attribute regexes see the full meta text. + Plain ``b')'`` terminators get folded in too, which is harmless. + """ + grouped: list = [] # list of (meta_bytes, payload_bytes_or_None) + for part in (msg_data or []): + if isinstance(part, tuple): + meta_b = part[0] if isinstance(part[0], (bytes, bytearray)) else str(part[0]).encode() + if _FETCH_SEQ_RE.match(meta_b): + grouped.append((meta_b, part[1])) + elif grouped: + cur_meta, cur_payload = grouped[-1] + grouped[-1] = (cur_meta + b" " + meta_b, cur_payload or part[1]) + elif isinstance(part, (bytes, bytearray)) and grouped: + cur_meta, cur_payload = grouped[-1] + grouped[-1] = (cur_meta + b" " + bytes(part), cur_payload) + return grouped + + def _smtp_ready(cfg: dict) -> bool: - return bool(cfg.get("smtp_host") and cfg.get("smtp_user") and cfg.get("smtp_password")) + if not cfg.get("smtp_host") or not cfg.get("smtp_user"): + return False + return bool(cfg.get("smtp_password") or cfg.get("oauth_provider")) def _resolve_send_config(account_id: str | None = None, owner: str = "") -> dict: @@ -799,20 +840,11 @@ def setup_email_routes(): except Exception as e: logger.warning(f"Batch fetch failed, falling back to per-UID: {e}") status, msg_data = "NO", [] - # imaplib batch responses interleave (meta, payload) tuples and - # `b')'` terminators. Group by message: each tuple where the - # meta begins with a seq number starts a new message record. - seq_re = re.compile(rb'^(\d+)\s+\(') - grouped = [] # list of (meta_str, payload_bytes) - for part in (msg_data or []): - if isinstance(part, tuple): - meta_b = part[0] if isinstance(part[0], (bytes, bytearray)) else str(part[0]).encode() - if seq_re.match(meta_b): - grouped.append((meta_b, part[1])) - elif grouped: - # continuation of previous message — concatenate meta info if any - cur_meta, cur_payload = grouped[-1] - grouped[-1] = (cur_meta + b" " + meta_b, cur_payload or part[1]) + # Group the batched response into per-message (meta, payload) + # records. Bare bytes parts must be kept: Gmail returns FLAGS + # after the header literal as a bare element, and dropping it + # rendered every Gmail message as unread/unflagged. + grouped = _group_uid_fetch_records(msg_data) if status != "OK" and not grouped: conn.logout() @@ -1061,14 +1093,22 @@ def setup_email_routes(): return {"contacts": [], "error": "Mail operation failed"} @router.get("/search") - async def search_emails( + # Sync def: the body is blocking IMAP I/O with no awaits. As `async def` it ran + # directly on the event loop and stalled the whole app during a search; as a sync + # def FastAPI runs it in a threadpool, keeping the loop responsive. + def search_emails( q: str = Query(""), folder: str = Query("INBOX"), limit: int = Query(50), account_id: str | None = Query(None), owner: str = Depends(require_owner), ): - """Search emails server-side via IMAP SEARCH. Matches subject, from, or body text.""" + """Search emails server-side via IMAP SEARCH. Matches subject, from, or body text. + + When the caller asks for INBOX and the account has an "All Mail" + folder (Gmail does), we transparently swap to All Mail so the + search surfaces archived / labelled emails too. Plain IMAP + accounts fall back to whatever folder the caller specified.""" if not q or len(q) < 2: return {"emails": [], "total": 0, "query": q} # CRLF in q would terminate the IMAP command early — reject defensively. @@ -1076,7 +1116,27 @@ def setup_email_routes(): raise HTTPException(400, "Invalid query") try: with _imap(account_id, owner=owner) as conn: - conn.select(_q(folder), readonly=True) + # If the user asked for INBOX, try to upgrade to All Mail — + # one folder == every email on Gmail-class servers. + effective_folder = folder + if (folder or "").upper() == "INBOX": + try: + status, folder_lines = conn.list() + if status == "OK" and folder_lines: + for raw in folder_lines: + if isinstance(raw, bytes): + raw = raw.decode("utf-8", errors="replace") + m = re.match(r"\((?P[^)]*)\)\s+\"[^\"]*\"\s+(?P.+)", raw) + if not m: + continue + flags = (m.group("flags") or "").lower() + name = m.group("name").strip().strip('"') + if "\\all" in flags or "all mail" in name.lower(): + effective_folder = name + break + except Exception: + pass + conn.select(_q(effective_folder), readonly=True) # Escape backslash and quote for the IMAP-SEARCH quoted-string. q_escaped = q.replace('\\', '\\\\').replace('"', '\\"') @@ -1084,7 +1144,7 @@ def setup_email_routes(): status, data = _imap_uid_search(conn, search_cmd) if status != "OK" or not data[0]: - return {"emails": [], "total": 0, "query": q} + return {"emails": [], "total": 0, "query": q, "folder": effective_folder} uid_list = data[0].split() total = len(uid_list) @@ -1098,14 +1158,15 @@ def setup_email_routes(): continue raw_header = None flags = "" - for part in msg_data: - if isinstance(part, tuple): - meta = part[0].decode() if isinstance(part[0], bytes) else str(part[0]) - if b"RFC822.HEADER" in part[0] if isinstance(part[0], bytes) else "RFC822.HEADER" in meta: - raw_header = part[1] - flag_match = re.search(r'FLAGS \(([^)]*)\)', meta) - if flag_match: - flags = flag_match.group(1) + # Same Gmail caveat as the list route: FLAGS may + # arrive after the header literal, so group bare + # parts back into the message meta before scanning. + for meta_b, payload in _group_uid_fetch_records(msg_data): + if payload and b"RFC822.HEADER" in meta_b: + raw_header = payload + flag_match = re.search(rb'FLAGS \(([^)]*)\)', meta_b) + if flag_match: + flags = flag_match.group(1).decode(errors="replace") if not raw_header: continue msg = email_mod.message_from_bytes(raw_header) @@ -1148,6 +1209,13 @@ def setup_email_routes(): "is_flagged": "\\Flagged" in flags, "flags": flags, "has_attachments": has_attachments, + # Stamp the folder so the frontend opens each + # email from the folder it actually lives in + # (the search may have run against All Mail + # even though the caller asked for INBOX), + # otherwise clicks open whatever happens to + # have the same UID in INBOX → wrong email. + "folder": effective_folder, }) except Exception as e: logger.warning(f"Error parsing search result {uid}: {e}") @@ -1247,8 +1315,9 @@ def setup_email_routes(): try: if sender_addr: _rs = _c.execute( - "SELECT signature_text FROM sender_signatures WHERE from_address = ?", - (sender_addr.lower().strip(),), + f"SELECT signature_text FROM sender_signatures " + f"WHERE from_address = ? AND {owner_clause}", + (sender_addr.lower().strip(), *owner_params), ).fetchone() if _rs and _rs[0]: cached_sender_sig = _rs[0] @@ -1693,6 +1762,22 @@ def setup_email_routes(): logger.error(f"Failed to mark unread {uid}: {e}") return {"success": False, "error": "Mail operation failed"} + @router.post("/flag/{uid}") + async def flag_email(uid: str, folder: str = Query("INBOX"), account_id: str | None = Query(None), + on: bool = Query(True), owner: str = Depends(require_owner)): + """Toggle the \\Flagged flag (a.k.a. favorite / star) on an email. + Pass `on=true` to favorite, `on=false` to unfavorite.""" + try: + with _imap(account_id, owner=owner) as conn: + conn.select(_q(folder)) + if not _store_email_flag(conn, uid, "\\Flagged", add=bool(on)): + return {"success": False, "error": "Email not found"} + _invalidate_list_cache(account_id, folder) + return {"success": True, "flagged": bool(on)} + except Exception as e: + logger.error(f"Failed to flag {uid}: {e}") + return {"success": False, "error": "Mail operation failed"} + @router.post("/mark-read/{uid}") async def mark_read(uid: str, folder: str = Query("INBOX"), account_id: str | None = Query(None), owner: str = Depends(require_owner)): """Mark an email as read (set \\Seen flag).""" @@ -1708,7 +1793,9 @@ def setup_email_routes(): return {"success": False, "error": "Mail operation failed"} @router.post("/archive/{uid}") - async def archive_email(uid: str, folder: str = Query("INBOX"), account_id: str | None = Query(None), owner: str = Depends(require_owner)): + # Sync def: blocking IMAP I/O with no awaits — see search_emails above. Runs in a + # threadpool instead of blocking the event loop. + def archive_email(uid: str, folder: str = Query("INBOX"), account_id: str | None = Query(None), owner: str = Depends(require_owner)): """Move email to Archive folder.""" try: with _imap(account_id, owner=owner) as conn: @@ -1940,7 +2027,7 @@ def setup_email_routes(): outer = MIMEMultipart("alternative") body_container = outer - outer["From"] = cfg["from_address"] + outer["From"] = email.utils.formataddr((cfg.get("display_name") or "", cfg["from_address"])) outer["To"] = to if cc: outer["Cc"] = cc @@ -2071,6 +2158,77 @@ def setup_email_routes(): logger.error(f"cancel_scheduled {sid!r} failed: {e}") return {"success": False, "error": "Mail operation failed"} + # ── Agent send-confirm: list/approve/cancel ────────────────────────── + # When `agent_email_confirm` is on, the MCP send_email tool drops the + # composed email into scheduled_emails with status='agent_draft' (a + # far-future send_at so the poller never picks it up). These endpoints + # let the chat UI surface them for the user and either approve (flip + # to status='pending' with send_at=now so the poller delivers it) or + # cancel (status='cancelled'). + @router.get("/pending") + async def list_pending_agent_drafts(owner: str = Depends(require_owner)): + import sqlite3 + try: + conn = sqlite3.connect(SCHEDULED_DB) + conn.row_factory = sqlite3.Row + rows = conn.execute( + """SELECT id, to_addr, subject, body, created_at, account_id + FROM scheduled_emails + WHERE status = 'agent_draft' AND owner = ? + ORDER BY created_at DESC""", + (owner or "",), + ).fetchall() + conn.close() + return {"pending": [dict(r) for r in rows]} + except Exception as e: + logger.error(f"list_pending_agent_drafts failed: {e}") + return {"pending": [], "error": "Mail operation failed"} + + @router.post("/pending/{sid}/approve") + async def approve_agent_draft(sid: str, owner: str = Depends(require_owner)): + """Approve a draft staged by the agent: flip status → pending and + backdate send_at so the scheduled-send poller picks it up + immediately.""" + import sqlite3 + try: + conn = sqlite3.connect(SCHEDULED_DB) + cur = conn.execute( + """UPDATE scheduled_emails + SET status = 'pending', send_at = ? + WHERE id = ? AND status = 'agent_draft' AND owner = ?""", + (datetime.utcnow().isoformat(), sid, owner or ""), + ) + conn.commit() + affected = cur.rowcount + conn.close() + if not affected: + return {"success": False, "error": "Draft not found or already handled"} + return {"success": True} + except Exception as e: + logger.error(f"approve_agent_draft {sid!r} failed: {e}") + return {"success": False, "error": "Mail operation failed"} + + @router.delete("/pending/{sid}") + async def cancel_agent_draft(sid: str, owner: str = Depends(require_owner)): + """Discard a draft the agent staged for approval.""" + import sqlite3 + try: + conn = sqlite3.connect(SCHEDULED_DB) + cur = conn.execute( + """UPDATE scheduled_emails SET status = 'cancelled' + WHERE id = ? AND status = 'agent_draft' AND owner = ?""", + (sid, owner or ""), + ) + conn.commit() + affected = cur.rowcount + conn.close() + if not affected: + return {"success": False, "error": "Draft not found or already handled"} + return {"success": True} + except Exception as e: + logger.error(f"cancel_agent_draft {sid!r} failed: {e}") + return {"success": False, "error": "Mail operation failed"} + @router.get("/resolve-contact") async def resolve_contact(name: str = Query(..., description="Name to search for"), owner: str = Depends(require_owner)): """Search Sent folder for a contact by name. Returns matching email addresses.""" @@ -2131,6 +2289,7 @@ def setup_email_routes(): try: cfg = _resolve_send_config(req.account_id, owner=owner) except Exception as e: + logger.warning(f"No SMTP-capable account resolved: {e}") return {"success": False, "error": str(e) or "No SMTP-capable email account configured"} # Use 'mixed' if we have attachments, 'alternative' otherwise @@ -2143,7 +2302,7 @@ def setup_email_routes(): outer = MIMEMultipart("alternative") body_container = outer - outer["From"] = cfg["from_address"] + outer["From"] = email.utils.formataddr((cfg.get("display_name") or "", cfg["from_address"])) outer["To"] = req.to if req.cc: outer["Cc"] = req.cc @@ -2194,6 +2353,10 @@ def setup_email_routes(): _account_id = cfg.get("account_id") or req.account_id # capture for the IMAP append in the closure _in_reply_to = (req.in_reply_to or "").strip() + _oauth_provider = cfg.get("oauth_provider") or "" + _oauth_access_token = cfg.get("oauth_access_token") or "" + _oauth_refresh_token = cfg.get("oauth_refresh_token") or "" + _oauth_token_expiry = cfg.get("oauth_token_expiry") or "" def _deliver(): try: @@ -2204,6 +2367,11 @@ def setup_email_routes(): "smtp_security": _smtp_security, "smtp_user": _smtp_user, "smtp_password": _smtp_pw, + "account_id": _account_id, + "oauth_provider": _oauth_provider, + "oauth_access_token": _oauth_access_token, + "oauth_refresh_token": _oauth_refresh_token, + "oauth_token_expiry": _oauth_token_expiry, }, _from, _recipients, @@ -2316,7 +2484,7 @@ def setup_email_routes(): msg.attach(MIMEText(_draft_html, "html", "utf-8")) else: msg = MIMEText(req.body, "plain", "utf-8") - msg["From"] = cfg["from_address"] + msg["From"] = email.utils.formataddr((cfg.get("display_name") or "", cfg["from_address"])) msg["To"] = req.to if req.cc: msg["Cc"] = req.cc @@ -2584,11 +2752,15 @@ def setup_email_routes(): source_uid = (data.get("uid") or "").strip() source_folder = (data.get("folder") or "INBOX").strip() fast_reply = bool(data.get("fast", False)) + user_hint = (data.get("user_hint") or "").strip() if not original_body: return {"success": False, "error": "No email body provided"} - if message_id: + # Skip cache lookup when the caller supplied a user_hint — the + # cached generic reply doesn't reflect the instructions and + # would silently override them. + if message_id and not user_hint: try: _c = _sql3.connect(SCHEDULED_DB) owner_clause, owner_params = _email_cache_owner_clause(owner) @@ -2728,8 +2900,13 @@ def setup_email_routes(): user_msg = ( f"Recipient: {to}\nSubject: {subject}\n\n" f"Original email and any current draft:\n{original_body[:6000]}\n\n" - f"Draft a reply. Return only the reply body text." ) + if user_hint: + user_msg += ( + f"User's instructions for THIS reply (follow these — they override " + f"defaults like length/tone):\n{user_hint[:2000]}\n\n" + ) + user_msg += "Draft a reply. Return only the reply body text." # Build a candidate chain so a stale session-stored API key # (the most common cause of "authentication failed" here) @@ -2959,6 +3136,8 @@ def setup_email_routes(): "from_address": r.from_address or "", "has_imap_password": bool(r.imap_password), "has_smtp_password": bool(r.smtp_password), + "oauth_provider": r.oauth_provider or "", + "display_name": r.display_name or "", }) return {"accounts": out} finally: @@ -2991,6 +3170,7 @@ def setup_email_routes(): smtp_user=(data.get("smtp_user") or "").strip(), smtp_password=_enc(data.get("smtp_password") or ""), from_address=(data.get("from_address") or "").strip(), + display_name=(data.get("display_name") or "").strip(), # SECURITY: stamp the creator so all subsequent reads / mutations # can filter by user. Without this every new account leaks to # every other user. @@ -3025,7 +3205,7 @@ def setup_email_routes(): if not row: return {"ok": False, "error": "Account not found"} # Simple fields - for key in ("name", "imap_host", "imap_user", "smtp_host", "smtp_user", "from_address"): + for key in ("name", "imap_host", "imap_user", "smtp_host", "smtp_user", "from_address", "display_name"): if key in data: setattr(row, key, (data[key] or "").strip()) for key in ("imap_port", "smtp_port"): @@ -3214,4 +3394,123 @@ def setup_email_routes(): finally: db.close() + # ── Google OAuth2 routes ── + + @router.get("/oauth/google/authorize") + async def google_oauth_authorize(account_id: str = Query(...), request: Request = None, owner: str = Depends(require_user)): + import urllib.parse + _assert_owns_account(account_id, owner) + client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "") + if not client_id: + raise HTTPException(400, "GOOGLE_OAUTH_CLIENT_ID not set — add it to .env") + redirect_uri = ( + os.environ.get("GOOGLE_OAUTH_REDIRECT_URI") + or f"http://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback" + ) + state = make_oauth_state(account_id, owner) + params = urllib.parse.urlencode({ + "client_id": client_id, + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": "https://mail.google.com/ email", + "access_type": "offline", + "prompt": "consent", + "state": state, + }) + from fastapi.responses import RedirectResponse as _RR + return _RR(f"https://accounts.google.com/o/oauth2/v2/auth?{params}") + + @router.get("/oauth/google/callback") + async def google_oauth_callback( + code: str = Query(None), + state: str = Query(None), + error: str = Query(None), + request: Request = None, + ): + import urllib.parse + from fastapi.responses import RedirectResponse as _RR + if error: + return _RR("/?section=integrations&email_oauth_error=google_error") + if not code or not state: + return _RR("/?section=integrations&email_oauth_error=missing_code") + state_data = verify_oauth_state(state) + if not state_data: + return _RR("/?section=integrations&email_oauth_error=invalid_state") + account_id = state_data.get("a", "") + owner = state_data.get("o", "") + client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "") + client_secret = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET", "") + redirect_uri = ( + os.environ.get("GOOGLE_OAUTH_REDIRECT_URI") + or f"http://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback" + ) + import httpx as _httpx + try: + resp = _httpx.post("https://oauth2.googleapis.com/token", data={ + "code": code, + "client_id": client_id, + "client_secret": client_secret, + "redirect_uri": redirect_uri, + "grant_type": "authorization_code", + }, timeout=10) + resp.raise_for_status() + data = resp.json() + except Exception: + logger.warning("Google token exchange failed") + return _RR("/?section=integrations&email_oauth_error=token_exchange_failed") + access_token = data.get("access_token", "") + refresh_token = data.get("refresh_token", "") + expiry = str(int(time.time()) + data.get("expires_in", 3600)) + # Fetch the email address from userinfo so we can auto-fill imap_user. + email_addr = "" + display_name = "" + try: + ui = _httpx.get("https://www.googleapis.com/oauth2/v1/userinfo", + headers={"Authorization": f"Bearer {access_token}"}, timeout=10) + if ui.is_success: + ui_data = ui.json() + email_addr = ui_data.get("email", "") + display_name = ui_data.get("name", "") + except Exception: + pass + from core.database import SessionLocal, EmailAccount + from src.secret_storage import encrypt as _enc + db = SessionLocal() + try: + row = db.query(EmailAccount).filter(EmailAccount.id == account_id).first() + if not row: + return _RR("/?section=integrations&email_oauth_error=account_not_found") + # SECURITY: verify the account belongs to the initiating user. + if owner and row.owner and row.owner != owner: + logger.warning("OAuth callback owner mismatch — rejecting token write") + return _RR("/?section=integrations&email_oauth_error=ownership_error") + row.oauth_provider = "google" + row.oauth_access_token = _enc(access_token) + if refresh_token: + row.oauth_refresh_token = _enc(refresh_token) + row.oauth_token_expiry = expiry + # Auto-fill Google IMAP/SMTP settings if not already configured. + if not row.imap_host: + row.imap_host = "imap.gmail.com" + row.imap_port = 993 + row.imap_starttls = False + if not row.smtp_host: + row.smtp_host = "smtp.gmail.com" + row.smtp_port = 587 + if email_addr: + if not row.imap_user: + row.imap_user = email_addr + if not row.smtp_user: + row.smtp_user = email_addr + if not row.from_address: + row.from_address = email_addr + if not row.name or row.name == row.id: + row.name = email_addr + if display_name and not row.display_name: + row.display_name = display_name + db.commit() + finally: + db.close() + return _RR("/?section=integrations&email_oauth_success=1") + return router diff --git a/routes/embedding_routes.py b/routes/embedding_routes.py index a237e0b4c..62a459ae4 100644 --- a/routes/embedding_routes.py +++ b/routes/embedding_routes.py @@ -9,6 +9,7 @@ from pathlib import Path from fastapi import APIRouter, HTTPException, Form, Depends from core.constants import EMBEDDING_ENDPOINT_FILE, FASTEMBED_CACHE_DIR from core.middleware import require_admin +from src.runtime_paths import get_app_root logger = logging.getLogger(__name__) diff --git a/routes/gallery_routes.py b/routes/gallery_routes.py index feadc2ec8..38bb51cdd 100644 --- a/routes/gallery_routes.py +++ b/routes/gallery_routes.py @@ -19,6 +19,7 @@ from src.upload_limits import ( GALLERY_TRANSFORM_UPLOAD_MAX_BYTES, ) from src.constants import GENERATED_IMAGES_DIR +from src.optional_deps import patch_realesrgan_torchvision_compat from routes.gallery_helpers import ( GalleryPatch, _extract_exif, _image_to_dict, _owner_filter, _human_size, @@ -108,6 +109,32 @@ def _visible_image_endpoint_for_base(db, base: str, owner: str | None): return fallback +async def _fetch_result_image_b64(url: str) -> Optional[str]: + """Fetch an image URL returned in an upstream response body, base64-encoded + (or None on a non-200). + + The URL comes from the diffusion/OpenAI server's response, not from our own + config, so a malicious or compromised endpoint could otherwise steer this + fetch at an internal or cloud-metadata address. Validate it the same way the + client-supplied endpoint is validated before the first request. + """ + import base64 + import httpx + from src.url_safety import check_outbound_url + + ok, reason = check_outbound_url( + url, + block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + raise HTTPException(502, f"Upstream returned an unsafe image URL: {reason}") + async with httpx.AsyncClient(timeout=60) as c2: + ir = await c2.get(url) + if ir.status_code == 200: + return base64.b64encode(ir.content).decode() + return None + + def setup_gallery_routes() -> APIRouter: router = APIRouter(tags=["gallery"]) @@ -197,8 +224,6 @@ def setup_gallery_routes() -> APIRouter: @router.post("/api/gallery/{image_id}/replace") async def gallery_replace(request: Request, image_id: str): """Replace an existing gallery image file with a new one.""" - from pathlib import Path - user = get_current_user(request) db = SessionLocal() try: @@ -214,9 +239,8 @@ def setup_gallery_routes() -> APIRouter: raise HTTPException(400, "No image provided") content = await read_upload_limited(file, GALLERY_UPLOAD_MAX_BYTES, "Gallery replacement") - img_dir = Path(GENERATED_IMAGES_DIR) - img_dir.mkdir(parents=True, exist_ok=True) - img_path = img_dir / _sanitize_gallery_filename(img.filename) + GALLERY_IMAGE_DIR.mkdir(parents=True, exist_ok=True) + img_path = _gallery_image_path(img.filename) img_path.write_bytes(content) # Refresh dimensions in case the editor resized the canvas. @@ -904,15 +928,23 @@ def setup_gallery_routes() -> APIRouter: raise HTTPException(404, "Image not found") img_filename = img.filename - # Remove the file from disk - img_path = _gallery_image_path(img_filename) - if img_path.exists(): - img_path.unlink() - - # Soft-delete the record + # Soft-delete the record first; the DB is the source of truth. img.is_active = False db.commit() + # Only after the soft-delete commit succeeds do we remove the file. + # If the file were deleted first and the commit then failed/rolled + # back, the still-active record would point at a missing file. + # Best-effort so a missing or locked file can't 500 a delete that + # already succeeded logically. Uses the path-confined resolver so a + # malformed stored filename can't escape generated_images. + try: + img_path = _gallery_image_path(img_filename) + if img_path.exists(): + img_path.unlink() + except Exception as e: + logger.warning(f"Could not remove gallery image file for {img_filename}: {e}") + # Strip stale chat-history references so the image bubble # (and its prompt caption) doesn't come back after a server # reboot replays the session. We remove the matching tool @@ -1142,10 +1174,7 @@ def setup_gallery_routes() -> APIRouter: if item.get("b64_json"): raw_b64 = item["b64_json"] elif item.get("url"): - async with httpx.AsyncClient(timeout=60) as c2: - img_r = await c2.get(item["url"]) - if img_r.status_code == 200: - raw_b64 = base64.b64encode(img_r.content).decode() + raw_b64 = await _fetch_result_image_b64(item["url"]) if not raw_b64: raise HTTPException(502, "OpenAI returned no image") @@ -1206,7 +1235,7 @@ def setup_gallery_routes() -> APIRouter: original and regenerates `strength` fraction. With strength ~0.4 you get edge blending + lighting unification while keeping the composition recognisable.""" - import httpx, base64 as _b64 + import httpx user = require_privilege(request, "can_generate_images") body = await request.json() @@ -1382,10 +1411,9 @@ def setup_gallery_routes() -> APIRouter: if item.get("b64_json"): return {"image": item["b64_json"]} if item.get("url"): - async with httpx.AsyncClient(timeout=60) as c2: - ir = await c2.get(item["url"]) - if ir.status_code == 200: - return {"image": _b64.b64encode(ir.content).decode()} + img_b64 = await _fetch_result_image_b64(item["url"]) + if img_b64: + return {"image": img_b64} last_err = f"{path}: server returned no image" except httpx.ConnectError as e: raise HTTPException(502, f"Can't reach diffusion server at {base}: {e}") @@ -1445,6 +1473,7 @@ def setup_gallery_routes() -> APIRouter: img_bytes = base64.b64decode(image_b64) src = Image.open(io.BytesIO(img_bytes)).convert("RGB") try: + patch_realesrgan_torchvision_compat() from realesrgan import RealESRGANer except ImportError: return {"error": "realesrgan not installed. Install it from Cookbook → Dependencies (search 'realesrgan')."} @@ -1494,6 +1523,7 @@ def setup_gallery_routes() -> APIRouter: img_bytes = base64.b64decode(image_b64) src = Image.open(io.BytesIO(img_bytes)).convert("RGB") try: + patch_realesrgan_torchvision_compat() from basicsr.archs.rrdbnet_arch import RRDBNet from realesrgan import RealESRGANer except ImportError: diff --git a/routes/hwfit_routes.py b/routes/hwfit_routes.py index 45c209b0b..5e38b9ca3 100644 --- a/routes/hwfit_routes.py +++ b/routes/hwfit_routes.py @@ -119,7 +119,7 @@ def setup_hwfit_routes(): return detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh) @router.get("/models") - def get_models(use_case: str = "", sort: str = "score", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False): + def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False): """Rank LLM models against detected hardware and return scored results. gpu_count: override GPU count (0 = CPU only, 1-N = simulate N GPUs of the active group). gpu_group: index into system.gpu_groups (the homogeneous diff --git a/routes/mcp_routes.py b/routes/mcp_routes.py index ca2722b5b..a0ade88b6 100644 --- a/routes/mcp_routes.py +++ b/routes/mcp_routes.py @@ -108,6 +108,12 @@ def _load_disabled_map(): db.close() +def _mcp_oauth_redirect_uri() -> str: + """Shared callback URL for legacy Google and generic MCP OAuth flows.""" + from src.mcp_oauth import REDIRECT_URI + return REDIRECT_URI + + def setup_mcp_routes(mcp_manager: McpManager): """Setup MCP routes with the provided manager.""" @@ -445,9 +451,9 @@ def setup_mcp_routes(mcp_manager: McpManager): client_id = keys["client_id"] scopes = oauth_cfg.get("scopes", []) - # For Desktop App creds, redirect to localhost — the user will + # For Desktop App creds, default to localhost — the user will # paste the resulting URL back if they're on a different device. - redirect_uri = "http://localhost:7000/api/mcp/oauth/callback" + redirect_uri = _mcp_oauth_redirect_uri() params = { "client_id": client_id, @@ -469,7 +475,7 @@ def setup_mcp_routes(mcp_manager: McpManager): return RedirectResponse(auth_url) else: # Remote device — show paste-back page - return HTMLResponse(_oauth_authorize_page(auth_url, server_id, host)) + return HTMLResponse(_oauth_authorize_page(auth_url, server_id, host, redirect_uri)) finally: db.close() @@ -536,7 +542,7 @@ def setup_mcp_routes(mcp_manager: McpManager): client_id = keys["client_id"] client_secret = keys["client_secret"] - redirect_uri = "http://localhost:7000/api/mcp/oauth/callback" + redirect_uri = _mcp_oauth_redirect_uri() async with httpx.AsyncClient() as client: resp = await client.post( @@ -603,13 +609,19 @@ def setup_mcp_routes(mcp_manager: McpManager): return router -def _oauth_authorize_page(auth_url: str, server_id: str, host: str) -> str: +def _oauth_authorize_page( + auth_url: str, + server_id: str, + host: str, + redirect_uri: str = "http://localhost:7000/api/mcp/oauth/callback", +) -> str: """Page with Google sign-in link and URL paste-back form for remote access.""" # Escape values interpolated into the page: `host` comes from the request # Host header and `server_id` from the OAuth state — neither is trusted. auth_url = html.escape(auth_url, quote=True) server_id = html.escape(server_id, quote=True) host = html.escape(host, quote=True) + redirect_uri = html.escape(redirect_uri, quote=True) return f""" Authorize — Odysseus @@ -654,7 +666,7 @@ def _oauth_authorize_page(auth_url: str, server_id: str, host: str) -> str:

Paste the URL from your browser after signing in:

- +
""" diff --git a/routes/memory_routes.py b/routes/memory_routes.py index 7be3c6d32..d290046ec 100644 --- a/routes/memory_routes.py +++ b/routes/memory_routes.py @@ -29,6 +29,7 @@ from src.llm_core import llm_call_async from services.memory.memory_extractor import audit_memories from src.auth_helpers import get_current_user, require_user from src.endpoint_resolver import resolve_endpoint +from src.task_endpoint import resolve_task_endpoint from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES logger = logging.getLogger(__name__) @@ -105,6 +106,13 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM if memory_manager.find_duplicates(text, user_mem): return {"ok": True, "count": len(user_mem), "message": "Memory already exists"} + if memory_data.session_id: + try: + session_obj = session_manager.get_session(memory_data.session_id) + except KeyError: + raise HTTPException(404, "Session not found") + _assert_session_owner(session_obj, user) + new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user) if memory_data.session_id: new_entry["session_id"] = memory_data.session_id @@ -163,8 +171,17 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM session_id = memory.get("session_id") if session_id and session_id in session_manager.sessions: - session = session_manager.get_session(session_id) - memory["session_name"] = session.name if session else f"Session {session_id[:6]}" + try: + session = session_manager.get_session(session_id) + if session: + _assert_session_owner(session, user) + memory["session_name"] = session.name if session else f"Session {session_id[:6]}" + except KeyError: + memory["session_name"] = "Unknown" + except HTTPException as exc: + if exc.status_code != 404: + raise + memory["session_name"] = "Unknown" else: memory["session_name"] = "Unknown" @@ -224,14 +241,18 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM } messages = [system_msg] + sess.get_context_messages() + t_url, t_model, t_headers = resolve_task_endpoint( + sess.endpoint_url, sess.model, sess.headers, owner=_owner(request) + ) + try: suggestion_text = await llm_call_async( - sess.endpoint_url, - sess.model, + t_url, + t_model, messages, temperature=0.2, max_tokens=500, - headers=sess.headers, + headers=t_headers, ) try: suggestions = json.loads(suggestion_text) @@ -252,57 +273,30 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM async def api_audit_memories(request: Request, session: str = Form(None)): """Deduplicate and consolidate memories via LLM. - Uses the default model from settings, or falls back to a session's model. + Uses task/utility/default settings through the shared resolver, with + the active session as fallback when no task or utility model is set. Returns before and after memory counts. """ - from routes.model_routes import _load_settings, _normalize_base, build_chat_url - from core.database import ModelEndpoint - import json as _json - - endpoint_url = model = None - headers = {} - - # Try default model from settings first - settings = _load_settings() - ep_id = settings.get("default_endpoint_id", "") - default_model = settings.get("default_model", "") - if ep_id: - db = SessionLocal() - try: - ep = db.query(ModelEndpoint).filter( - ModelEndpoint.id == ep_id, ModelEndpoint.is_enabled == True - ).first() - if ep: - base = _normalize_base(ep.base_url) - endpoint_url = build_chat_url(base) - model = default_model - if not model and ep.models: - try: - models = _json.loads(ep.models) if isinstance(ep.models, str) else ep.models - if models: - model = models[0] - except Exception: - pass - if ep.api_key: - headers = {"Authorization": f"Bearer {ep.api_key}"} - finally: - db.close() - - # Fall back to session model if no default configured - if not endpoint_url and session: + user = _owner(request) + fallback_url = fallback_model = None + fallback_headers = None + if session: try: sess = session_manager.get_session(session) - _assert_session_owner(sess, _owner(request)) - endpoint_url = sess.endpoint_url - model = sess.model - headers = sess.headers + _assert_session_owner(sess, user) + fallback_url = sess.endpoint_url + fallback_model = sess.model + fallback_headers = sess.headers except KeyError: pass + endpoint_url, model, headers = resolve_task_endpoint( + fallback_url, fallback_model, fallback_headers, owner=user + ) + if not endpoint_url or not model: raise HTTPException(400, "No default model configured — set one in Settings") - user = _owner(request) result = await audit_memories( memory_manager, memory_vector, @@ -340,17 +334,28 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM model = None headers = {} + user = _owner(request) + if session: try: sess = session_manager.get_session(session) - _assert_session_owner(sess, _owner(request)) - endpoint_url = sess.endpoint_url - model = sess.model - headers = sess.headers + _assert_session_owner(sess, user) except KeyError: - raise HTTPException(404, "Session not found — needed for LLM config") + sess = None + except HTTPException as exc: + if exc.status_code != 404: + raise + sess = None + + if sess is None: + logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session) + endpoint_url, model, headers = resolve_endpoint("utility", owner=user) + else: + endpoint_url, model, headers = resolve_task_endpoint( + sess.endpoint_url, sess.model, sess.headers, owner=user + ) else: - endpoint_url, model, headers = resolve_endpoint("utility", owner=_owner(request)) + endpoint_url, model, headers = resolve_task_endpoint(owner=user) if not endpoint_url or not model: raise HTTPException(400, "No LLM model configured. Set a default model in Settings.") diff --git a/routes/model_routes.py b/routes/model_routes.py index b88fa3ef1..d8fde332b 100644 --- a/routes/model_routes.py +++ b/routes/model_routes.py @@ -5,6 +5,7 @@ import re import uuid import json import hashlib +import ipaddress import socket import time as _time import logging @@ -26,7 +27,7 @@ from src.endpoint_resolver import ( build_models_url, build_headers, ) -from src.auth_helpers import _auth_disabled, owner_filter +from src.auth_helpers import _auth_disabled, effective_user, owner_filter logger = logging.getLogger(__name__) @@ -123,6 +124,21 @@ def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int: return cleared_users +def _default_endpoint_needs_assignment(current_default_id: str, enabled_endpoint_ids) -> bool: + """Whether the global default chat endpoint should be (re)assigned. + + True when nothing is configured yet, or the configured default no longer + resolves to an enabled endpoint (e.g. the user disabled it). Without the + second case, adding a new endpoint after disabling the previous default + leaves `default_endpoint_id` pointing at the disabled endpoint, so features + that read the raw setting (Memory → Tidy) fail with "No default model + configured" even though an enabled endpoint exists. See #3586. + """ + if not current_default_id: + return True + return current_default_id not in enabled_endpoint_ids + + # Loopback hosts a user might type for a local model server (LM Studio, # llama.cpp, vLLM, …). Inside Docker these point at the *container*, not the # host the server actually runs on. @@ -233,6 +249,9 @@ _PROVIDER_CURATED = { "zai-coding": [ "glm-5.1", "glm-5v-turbo", "glm-5-turbo", "glm-4.7", "glm-4.5-air", ], + "kimi-code": [ + "kimi-for-coding", + ], "deepseek": [ "deepseek-chat", "deepseek-reasoner", ], @@ -300,6 +319,8 @@ def _match_provider_curated(base_url: str, provider: str) -> str: parsed = urlparse(base_url) if _host_match(base_url, "z.ai") and "/api/coding" in (parsed.path or ""): return "zai-coding" + if _host_match(base_url, "kimi.com") and "/coding" in (parsed.path or ""): + return "kimi-code" for domain, key in _HOST_TO_CURATED: if _host_match(base_url, domain): return key @@ -542,6 +563,8 @@ def _safe_build_models_url(base_url: str) -> str: """Build a /models URL without letting optional provider imports break probes.""" try: return build_models_url(base_url) + except ValueError: + raise except Exception as exc: logger.debug("Model URL detection failed for %s: %s", base_url, exc) return f"{(base_url or '').rstrip('/')}/models" @@ -613,7 +636,7 @@ def _probe_single_model(base: str, api_key: str, model_id: str, timeout: int = 1 try: t0 = _time.time() - r = httpx.post(target_url, headers=h, json=payload, timeout=timeout) + r = httpx.post(target_url, headers=h, json=payload, timeout=timeout, verify=llm_verify()) latency = round((_time.time() - t0) * 1000) if r.is_success: return {"status": "ok", "latency_ms": latency} @@ -639,13 +662,20 @@ def _probe_single_model(base: str, api_key: str, model_id: str, timeout: int = 1 # Hostnames / IP prefixes that indicate a local endpoint _LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"} -_PRIVATE_PREFIXES = ("10.", "172.16.", "172.17.", "172.18.", "172.19.", - "172.20.", "172.21.", "172.22.", "172.23.", "172.24.", - "172.25.", "172.26.", "172.27.", "172.28.", "172.29.", - "172.30.", "172.31.", "192.168.") +_PRIVATE_NETWORKS = ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), +) +_TAILSCALE_CGNAT = ipaddress.ip_network("100.64.0.0/10") -_TAILSCALE_RE = re.compile(r"^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\.") +def _local_ip_literal(host: str) -> bool: + try: + ip = ipaddress.ip_address(host) + except ValueError: + return False + return any(ip in network for network in _PRIVATE_NETWORKS) or ip in _TAILSCALE_CGNAT def _classify_endpoint(base_url: str, endpoint_kind: str = "auto") -> str: @@ -659,9 +689,7 @@ def _classify_endpoint(base_url: str, endpoint_kind: str = "auto") -> str: return "api" try: host = urlparse(base_url).hostname or "" - if host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES): - return "local" - if _TAILSCALE_RE.match(host): + if host in _LOCAL_HOSTS or _local_ip_literal(host): return "local" except Exception: pass @@ -688,6 +716,7 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis """Probe a base URL's /models endpoint and return list of model IDs. For Anthropic, queries their /v1/models API, falling back to hardcoded list.""" from src.endpoint_resolver import resolve_url + from src.llm_core import httpx_get_kimi_aware base = resolve_url(_normalize_base(base_url)) provider = _safe_detect_provider(base) if provider == "chatgpt-subscription": @@ -723,7 +752,7 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis url = _safe_build_models_url(base) headers = _safe_build_headers(api_key, base) try: - r = httpx.get(url, headers=headers, timeout=timeout, verify=llm_verify()) + r = httpx_get_kimi_aware(url, headers, timeout=timeout, verify=llm_verify()) r.raise_for_status() data = r.json() # OpenAI format: {"data": [{"id": "model-name"}]} @@ -739,6 +768,11 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis for _e in _PROVIDER_CURATED.get(_ck, []): if _e not in set(models) and not any(m.startswith(_e) for m in models): models.append(_e) + if _host_match(base, "kimi.com") and "/coding" in (urlparse(base).path or ""): + _ck = _match_provider_curated(base, None) + for _e in _PROVIDER_CURATED.get(_ck, []): + if _e not in set(models) and not any(m.startswith(_e) for m in models): + models.append(_e) return [m for m in models if _is_chat_model(m)] except httpx.HTTPStatusError as e: if api_key: @@ -855,15 +889,52 @@ def _ping_endpoint(base_url: str, api_key: str = None, timeout: float = 1.5) -> def _model_endpoint_error_message(base_url: str, ping: Dict[str, Any] = None) -> str: - """Return a provider-aware error message for failed endpoint probes.""" + """Return a provider-aware error message for failed endpoint probes. + + Surfaces the URL we actually probed and, when the endpoint looks like + LM Studio (port 1234 or hostname match), adds a hint about loading a + model and confirming the Developer Server is running. The user previously + saw a generic "No models found for that provider/key" with no way to + tell whether the URL was wrong, the server was down, or the server was + reachable but had no model loaded (issue #25). + """ ping = ping or {} error = ping.get("error") + from src.endpoint_resolver import build_models_url + try: + probed = build_models_url(base_url) or base_url + except Exception: + probed = base_url parsed = urlparse(base_url) host = (parsed.hostname or "").lower() is_ollama = parsed.port == 11434 or "ollama" in host or "ollama" in base_url.lower() + is_lmstudio = ( + parsed.port == 1234 + or "lmstudio" in host + or "lm-studio" in host + or "lm_studio" in host + ) + + if is_lmstudio: + parts = [ + "LM Studio is reachable, but no models were reported.", + f"Probed {probed}.", + ] + if error: + parts.append(f"Last probe error: {error}.") + parts.append( + "Open LM Studio, load at least one model, and confirm the " + "Developer Server is running on port 1234." + ) + parts.append( + "Base URL should be http://localhost:1234/v1 (native) or " + "http://host.docker.internal:1234/v1 (Docker)." + ) + return " ".join(parts) if is_ollama: parts = ["No Ollama models found for that endpoint."] + parts.append(f"Probed {probed}.") if error: parts.append(f"Last probe error: {error}.") parts.append("Check that Ollama is running and that the base URL is correct.") @@ -873,9 +944,9 @@ def _model_endpoint_error_message(base_url: str, ping: Dict[str, Any] = None) -> return " ".join(parts) if error: - return f"No models found for that provider/key. Last probe error: {error}." + return f"No models found for that provider/key. Probed {probed}. Last probe error: {error}." - return "No models found for that provider/key." + return f"No models found for that provider/key. Probed {probed}." def _normalize_model_ids(value): @@ -1192,13 +1263,16 @@ def setup_model_routes(model_discovery): # Require auth; "" is the unconfigured single-user mode, treated as # "see everything" by _fetch_models. try: - from src.auth_helpers import get_current_user as _gcu - owner = _gcu(request) or "" - except Exception: - owner = "" - # Reject anonymous in configured deployments — no leaking the model - # list to unauthenticated callers. - try: + if getattr(request.state, "api_token", False): + scopes = set(getattr(request.state, "api_token_scopes", []) or []) + if "chat" not in scopes: + raise HTTPException(403, "API token is not scoped for chat") + if not getattr(request.state, "api_token_owner", None): + raise HTTPException(403, "API token has no owner") + owner = effective_user(request) or "" + + # Reject anonymous in configured deployments — no leaking the model + # list to unauthenticated callers. auth_mgr = getattr(request.app.state, "auth_manager", None) if not owner and not _auth_disabled() and auth_mgr is not None and getattr(auth_mgr, "is_configured", False): raise HTTPException(401, "Not authenticated") @@ -1727,12 +1801,19 @@ def setup_model_routes(model_discovery): ) db.add(ep) db.commit() - # Auto-set as default chat endpoint if none configured yet. Seed - # the first CHAT model (not raw model_ids[0]) so we don't pin the - # global default to an embedding/tts/etc. entry a provider happens - # to list first. + # Auto-set as default chat endpoint when none is usable yet — either + # nothing is configured, or the configured default points at an + # endpoint that is now missing/disabled (#3586). Seed the first CHAT + # model (not raw model_ids[0]) so we don't pin the global default to + # an embedding/tts/etc. entry a provider happens to list first. settings = _load_settings() - if not settings.get("default_endpoint_id"): + enabled_ids = { + e.id + for e in db.query(ModelEndpoint).filter( + ModelEndpoint.is_enabled == True # noqa: E712 + ).all() + } + if _default_endpoint_needs_assignment(settings.get("default_endpoint_id") or "", enabled_ids): from src.endpoint_resolver import _first_chat_model settings["default_endpoint_id"] = ep.id settings["default_model"] = _first_chat_model(model_ids) or "" diff --git a/routes/note_routes.py b/routes/note_routes.py index 22449f1e4..c4674e489 100644 --- a/routes/note_routes.py +++ b/routes/note_routes.py @@ -10,7 +10,8 @@ from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel from core.database import SessionLocal, Note -from src.auth_helpers import get_current_user +from core.middleware import INTERNAL_TOOL_USER +from src.auth_helpers import require_user from src.constants import DATA_DIR from sqlalchemy.orm.attributes import flag_modified @@ -208,14 +209,17 @@ async def dispatch_reminder( try: from src.endpoint_resolver import resolve_endpoint from src.llm_core import llm_call_async + from src.reminder_personas import synthesis_system_prompt url, model, headers = resolve_endpoint("utility", owner=owner or None) if not url: url, model, headers = resolve_endpoint("default", owner=owner or None) if url and model: + persona_id = (settings.get("reminder_llm_persona") or "").strip() + sys_prompt = synthesis_system_prompt(persona_id) raw = await llm_call_async( url=url, model=model, messages=[ - {"role": "system", "content": "You are a reminder assistant. Write a single short, warm, motivating sentence (max 25 words) reminding the user about the note below. Do not add greetings, preamble, or hashtags. Output only the sentence."}, + {"role": "system", "content": sys_prompt}, {"role": "user", "content": f"Title: {title}\n\n{note_body}".strip()}, ], temperature=0.7, max_tokens=200, headers=headers, timeout=30, @@ -567,10 +571,19 @@ def setup_note_routes(task_scheduler=None): router = APIRouter(prefix="/api/notes", tags=["notes"]) def _owner(request: Request) -> Optional[str]: - return get_current_user(request) + # require_user, not bare get_current_user: a request that reaches + # these owner-scoped routes with NO identity (auth-middleware + # regression, SSRF from a sibling service) must fail closed (401) + # when auth is configured — not be treated as the single-user mode + # and handed blanket access to every account's notes. The documented + # anonymous modes (AUTH_ENABLED=false, LOCALHOST_BYPASS on loopback, + # unconfigured first-run) still resolve to None, the single-user + # path. fire_reminder below already gated this way; the CRUD routes + # did not. + return require_user(request) or None def _is_admin_or_single_user(request: Request, user: str | None) -> bool: - if user == "internal-tool": + if user == INTERNAL_TOOL_USER: return True if not user: # require_user() already admitted this request, which only happens @@ -802,8 +815,7 @@ def setup_note_routes(task_scheduler=None): Returns {synthesis, email_sent}. """ # Gate against anonymous callers — LLM synthesis can burn tokens. - from src.auth_helpers import require_user as _ru - user = _ru(request) + user = require_user(request) body = await request.json() note_id = str(body.get("note_id") or "").strip() if not note_id: @@ -826,6 +838,12 @@ def setup_note_routes(task_scheduler=None): _override["reminder_webhook_integration_id"] = body["webhook_integration_id"] if body.get("webhook_payload_template"): _override["reminder_webhook_payload_template"] = body["webhook_payload_template"] + # Mirror the in-UI AI Synthesis toggle + persona so the test + # actually exercises the synthesis path before/without a Save. + if "llm_synthesis" in body: + _override["reminder_llm_synthesis"] = bool(body["llm_synthesis"]) + if "llm_persona" in body: + _override["reminder_llm_persona"] = str(body["llm_persona"] or "") else: db = SessionLocal() try: diff --git a/routes/personal_routes.py b/routes/personal_routes.py index c32f5ffe1..e6906223e 100644 --- a/routes/personal_routes.py +++ b/routes/personal_routes.py @@ -2,8 +2,9 @@ """Routes for personal documents management.""" import os import logging +import shutil import uuid -from typing import List, Tuple +from typing import Any, Dict, List, Tuple from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Depends from src.request_models import DirectoryRequest from core.constants import BASE_DIR, PERSONAL_DIR, PERSONAL_UPLOADS_DIR @@ -18,14 +19,15 @@ UPLOADS_DIR = PERSONAL_UPLOADS_DIR logger = logging.getLogger(__name__) -def _personal_upload_dir_for_owner(owner: str | None) -> str: +def _personal_upload_dir_for_owner(owner: str | None, *, create: bool = True) -> str: """Return the per-owner upload directory used for direct RAG uploads.""" owner_segment = secure_filename((owner or "local").strip())[:80] or "local" upload_dir = os.path.abspath(os.path.join(UPLOADS_DIR, owner_segment)) base_abs = os.path.abspath(UPLOADS_DIR) if os.path.commonpath([upload_dir, base_abs]) != base_abs: raise ValueError("Unsafe upload owner path") - os.makedirs(upload_dir, exist_ok=True) + if create: + os.makedirs(upload_dir, exist_ok=True) return upload_dir @@ -44,6 +46,87 @@ def _unique_personal_upload_path(upload_dir: str, original_name: str | None) -> raise ValueError("Unsafe upload filename") return file_path, filename, safe_name + +def _unique_existing_target(path: str) -> str: + """Return a non-existing sibling path for rename collision handling.""" + if not os.path.exists(path): + return path + stem, ext = os.path.splitext(path) + while True: + candidate = f"{stem}-{uuid.uuid4().hex[:10]}{ext}" + if not os.path.exists(candidate): + return candidate + + +def _remove_empty_tree(path: str) -> None: + """Best-effort removal of empty directories under ``path``.""" + if not os.path.isdir(path): + return + for root, dirs, _files in os.walk(path, topdown=False): + for dirname in dirs: + candidate = os.path.join(root, dirname) + try: + os.rmdir(candidate) + except OSError: + pass + try: + os.rmdir(path) + except OSError: + pass + + +def rename_personal_upload_owner( + old_owner: str, + new_owner: str, + *, + personal_docs_manager: Any = None, + rag_manager: Any = None, +) -> Dict[str, Any]: + """Move direct personal uploads and rewrite RAG owner metadata on user rename.""" + old_dir = _personal_upload_dir_for_owner(old_owner, create=False) + new_dir = _personal_upload_dir_for_owner(new_owner, create=False) + path_map: Dict[str, str] = {} + moved_files = 0 + + if os.path.isdir(old_dir) and old_dir != new_dir: + os.makedirs(new_dir, exist_ok=True) + for root, _dirs, files in os.walk(old_dir): + rel_root = os.path.relpath(root, old_dir) + target_root = new_dir if rel_root == "." else os.path.join(new_dir, rel_root) + os.makedirs(target_root, exist_ok=True) + for filename in files: + source = os.path.abspath(os.path.join(root, filename)) + target = _unique_existing_target(os.path.abspath(os.path.join(target_root, filename))) + shutil.move(source, target) + path_map[source] = target + moved_files += 1 + _remove_empty_tree(old_dir) + + if personal_docs_manager is not None: + rename_directory = getattr(personal_docs_manager, "rename_directory", None) + if callable(rename_directory): + rename_directory(old_dir, new_dir, path_map=path_map) + + rag_result = None + if rag_manager is not None: + rename_owner = getattr(rag_manager, "rename_owner", None) + if callable(rename_owner): + rag_result = rename_owner( + old_owner, + new_owner, + path_map=path_map, + path_prefixes=[(old_dir, new_dir)], + ) + + return { + "old_dir": old_dir, + "new_dir": new_dir, + "moved_files": moved_files, + "path_map": path_map, + "rag_result": rag_result, + } + + def setup_personal_routes(personal_docs_manager, rag_manager, rag_available): """ Setup personal documents related routes. @@ -160,8 +243,11 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available): JSON response confirming removal """ try: - if not directory: - raise HTTPException(400, "Directory path is required") + # Confine to PERSONAL_DIR — parity with add_directory_to_rag (which + # resolves the path the same way). Without this, an arbitrary or + # `..`-escaping path is passed straight to + # personal_docs_manager.remove_directory / rag.remove_directory. + directory = _resolve_allowed_personal_dir(directory) logger.info(f"Removing directory from RAG: {directory}") @@ -275,8 +361,8 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available): # Delete file from disk if it's in uploads dir deleted_from_disk = False try: - abs_target = os.path.abspath(filepath) - base_abs = os.path.abspath(UPLOADS_DIR) + abs_target = os.path.realpath(filepath) + base_abs = os.path.realpath(UPLOADS_DIR) in_uploads = ( abs_target == base_abs or os.path.commonpath([abs_target, base_abs]) == base_abs diff --git a/routes/research_routes.py b/routes/research_routes.py index 1ef36bd75..889298f7d 100644 --- a/routes/research_routes.py +++ b/routes/research_routes.py @@ -12,8 +12,10 @@ from typing import Optional from fastapi import APIRouter, HTTPException, Query, Request from fastapi.responses import HTMLResponse, StreamingResponse from pydantic import BaseModel, Field +from core.middleware import INTERNAL_TOOL_USER from src.endpoint_resolver import resolve_endpoint from src.auth_helpers import _auth_disabled, get_current_user +from core.auth import RESERVED_USERNAMES from src.constants import DEEP_RESEARCH_DIR _SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$") @@ -385,9 +387,9 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: """Launch a research job from the dedicated panel.""" from src.auth_helpers import require_privilege user = require_privilege(request, "can_use_research") - if user == "internal-tool": + if user == INTERNAL_TOOL_USER: tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip() - if tool_owner and tool_owner not in {"internal-tool", "api", "demo", "system"}: + if tool_owner and tool_owner not in RESERVED_USERNAMES: auth_mgr = getattr(request.app.state, "auth_manager", None) if auth_mgr is not None and getattr(auth_mgr, "is_configured", False): try: diff --git a/routes/session_routes.py b/routes/session_routes.py index 1fb2a487a..19b897f29 100644 --- a/routes/session_routes.py +++ b/routes/session_routes.py @@ -11,7 +11,7 @@ from core.session_manager import SessionManager from core.models import ChatMessage from src.request_models import SessionResponse from core.database import Session as DbSession, SessionLocal, Document, GalleryImage, utcnow_naive -from src.auth_helpers import get_current_user, effective_user, _auth_disabled, owner_filter +from src.auth_helpers import effective_user, _auth_disabled, owner_filter from src.session_actions import is_session_recently_active @@ -328,7 +328,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ endpoint_id: str = Form(""), ): skip_val = str(skip_validation).lower() == "true" - user = get_current_user(request) + user = effective_user(request) endpoint_api_key = "" endpoint_base_url = "" _reject_raw_endpoint_url_for_non_admin(request, user, endpoint_id, endpoint_url) @@ -477,7 +477,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ db.close() # Switch model/endpoint mid-session if model is not None and endpoint_url is not None: - user = get_current_user(request) + user = effective_user(request) _reject_raw_endpoint_url_for_non_admin(request, user, endpoint_id, endpoint_url) endpoint_api_key = "" endpoint_base_url = "" @@ -1004,6 +1004,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ """ from src.llm_core import llm_call user = effective_user(request) + single_user_mode = not user and _auth_disabled() user_sessions = session_manager.get_sessions_for_user(user) # Delete empty and throwaway sessions before sorting @@ -1022,7 +1023,12 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ } _THROWAWAY_MAX_MESSAGES = 4 # only delete if <= this many messages try: - rows = db.query(DbSession).filter(DbSession.archived == False, DbSession.owner == user).limit(2000).all() + rows_q = db.query(DbSession).filter(DbSession.archived == False) + if user: + rows_q = rows_q.filter(DbSession.owner == user) + elif not single_user_mode: + rows_q = rows_q.filter(DbSession.owner == user) + rows = rows_q.limit(2000).all() folder_map = {r.id: r.folder for r in rows} # Precompute per-session message counts in TWO aggregate queries # instead of 1–3 queries PER session — with many chats the per-row @@ -1242,7 +1248,12 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ db = SessionLocal() try: for sid, folder_name in assignments.items(): - db_session = db.query(DbSession).filter(DbSession.id == sid, DbSession.owner == user).first() + db_session_q = db.query(DbSession).filter(DbSession.id == sid) + if user: + db_session_q = db_session_q.filter(DbSession.owner == user) + elif not single_user_mode: + db_session_q = db_session_q.filter(DbSession.owner == user) + db_session = db_session_q.first() if db_session: db_session.folder = folder_name db_session.updated_at = datetime.utcnow() diff --git a/routes/shell_routes.py b/routes/shell_routes.py index a3126abbb..112b9fbca 100644 --- a/routes/shell_routes.py +++ b/routes/shell_routes.py @@ -1,6 +1,7 @@ """Shell routes — user-facing command execution endpoint.""" import asyncio +import importlib import json import logging import os @@ -14,6 +15,8 @@ from collections import namedtuple from pathlib import Path from typing import Dict, Any from core.platform_compat import IS_APPLE_SILICON, which_tool +from core.middleware import INTERNAL_TOOL_USER +from src.optional_deps import prepare_optional_dependency_import # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist # on Windows, so importing them unconditionally crashed app startup there @@ -53,7 +56,7 @@ def _require_admin(request: Request): # In-process tool loopback. The AuthMiddleware already validated the # internal token + loopback client before setting this marker, so # honour it here as admin-equivalent. - if user == "internal-tool": + if user == INTERNAL_TOOL_USER: return if not user or user == "api": raise HTTPException(403, "Admin only") @@ -149,6 +152,11 @@ def _pip_dist_name(pkg: dict) -> str: return (pkg.get("name") or "").replace("_", "-") +def _import_optional_dependency_for_status(name: str): + prepare_optional_dependency_import(name) + return importlib.import_module(name) + + def _package_installed_from_probe(name: str, probe: dict) -> bool: """Return whether an optional dependency is usable by Cookbook. @@ -970,7 +978,6 @@ def setup_shell_routes() -> APIRouter: """ _require_admin(request) _reject_cross_site(request) - import importlib import importlib.metadata as importlib_metadata import shlex import json as _json @@ -1057,6 +1064,13 @@ def setup_shell_routes() -> APIRouter: "category": "Image", "target": "remote", }, + { + "name": "transformers", + "pip": "transformers", + "desc": "Hugging Face model components used by SD/Flux pipelines and image tools", + "category": "Image", + "target": "remote", + }, { "name": "rembg", "pip": "rembg[gpu]", @@ -1202,7 +1216,7 @@ def setup_shell_routes() -> APIRouter: pkg["status_note"] = _package_status_note("vllm", probe) else: try: - importlib.import_module(pkg["name"]) + _import_optional_dependency_for_status(pkg["name"]) importlib_metadata.version(_pip_dist_name(pkg)) pkg["installed"] = True except ImportError: @@ -1251,6 +1265,7 @@ def setup_shell_routes() -> APIRouter: "sglang[all]", "diffusers", "diffusers[torch]", + "transformers", "TTS", "bark", "faster-whisper", diff --git a/routes/skills_routes.py b/routes/skills_routes.py index 3d6ede921..444dd69cf 100644 --- a/routes/skills_routes.py +++ b/routes/skills_routes.py @@ -691,8 +691,12 @@ async def _run_skill_test_once(md: str, task: str, url, model, headers, owner) - {"role": "user", "content": task}, ] try: + # max_tokens explicitly set: passing 0 lets some upstreams (Ollama, + # OpenAI-compat) generate an empty completion, which manifested as + # the skill test returning nothing while chat (which carries its + # preset's max_tokens) worked. 4096 matches the chat default. async for chunk in stream_agent_loop(url, model, messages, headers=headers, - temperature=0.3, max_tokens=0, max_rounds=8, owner=owner): + temperature=0.3, max_tokens=4096, max_rounds=8, owner=owner): if not chunk.startswith("data: ") or chunk.strip() == "data: [DONE]": continue try: diff --git a/routes/task_routes.py b/routes/task_routes.py index 5734fcb22..d38040fde 100644 --- a/routes/task_routes.py +++ b/routes/task_routes.py @@ -11,6 +11,7 @@ from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel from core.database import SessionLocal, ScheduledTask, TaskRun +from core.middleware import INTERNAL_TOOL_USER from core.constants import internal_api_base from src.auth_helpers import get_current_user from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR @@ -151,6 +152,7 @@ class TaskCreate(BaseModel): endpoint_url: Optional[str] = None then_task_id: Optional[str] = None # chain: run this task after success notifications_enabled: Optional[bool] = None # None lets action-specific defaults apply + character_id: Optional[str] = None # built-in persona id (PERSONAS) — biases output voice class TaskUpdate(BaseModel): @@ -171,6 +173,7 @@ class TaskUpdate(BaseModel): endpoint_url: Optional[str] = None then_task_id: Optional[str] = None notifications_enabled: Optional[bool] = None + character_id: Optional[str] = None def _display_task_name(t: ScheduledTask) -> str: @@ -203,6 +206,7 @@ def _task_to_dict(t: ScheduledTask, include_last_run_result: bool = False) -> di "output_target": t.output_target, "session_id": t.session_id, "crew_member_id": getattr(t, "crew_member_id", None), + "character_id": getattr(t, "character_id", None), "model": t.model, "endpoint_url": t.endpoint_url, "run_count": t.run_count or 0, @@ -424,7 +428,7 @@ def setup_task_routes(task_scheduler) -> APIRouter: # In-process tool-loopback marker — AuthMiddleware validated # the internal token + loopback client before stamping this, # so treat as admin-equivalent. - if user == "internal-tool": + if user == INTERNAL_TOOL_USER: return True try: from core.auth import AuthManager @@ -552,6 +556,7 @@ def setup_task_routes(task_scheduler) -> APIRouter: then_task_id=then_task_id, webhook_token=webhook_token, notifications_enabled=notifications_enabled, + character_id=(req.character_id or None), ) db.add(task) db.commit() @@ -705,6 +710,9 @@ def setup_task_routes(task_scheduler) -> APIRouter: task.then_task_id = _validate_then_task_id(db, req.then_task_id, user, current_task_id=task.id) if req.notifications_enabled is not None: task.notifications_enabled = bool(req.notifications_enabled) + if req.character_id is not None: + # Empty string clears the persona; non-empty stores the id. + task.character_id = req.character_id or None if req.cron_expression is not None: if req.cron_expression: try: diff --git a/routes/upload_routes.py b/routes/upload_routes.py index 489e4923a..1e197dd49 100644 --- a/routes/upload_routes.py +++ b/routes/upload_routes.py @@ -7,7 +7,7 @@ from fastapi import APIRouter, Request, File, UploadFile, HTTPException from typing import List import logging from core.middleware import require_admin -from src.auth_helpers import get_current_user +from src.auth_helpers import effective_user from src.upload_handler import count_recent_uploads logger = logging.getLogger(__name__) @@ -78,7 +78,7 @@ def setup_upload_routes(upload_handler): for u in files: try: - meta = upload_handler.save_upload(u, client_ip, owner=get_current_user(request)) + meta = upload_handler.save_upload(u, client_ip, owner=effective_user(request)) out.append({ "id": meta["id"], "name": meta["name"], @@ -138,7 +138,7 @@ def setup_upload_routes(upload_handler): original_name = info.get("name", file_id) auth_mgr = getattr(request.app.state, "auth_manager", None) auth_configured = bool(auth_mgr and auth_mgr.is_configured) - current_user = get_current_user(request) + current_user = effective_user(request) file_owner = info.get("owner") if info else None if auth_configured: if not current_user: @@ -204,7 +204,7 @@ def setup_upload_routes(upload_handler): info = _load_upload_info(file_id) auth_mgr = getattr(request.app.state, "auth_manager", None) auth_configured = bool(auth_mgr and auth_mgr.is_configured) - current_user = get_current_user(request) + current_user = effective_user(request) file_owner = info.get("owner") if info else None if auth_configured: if not current_user: @@ -247,7 +247,7 @@ def setup_upload_routes(upload_handler): raise HTTPException(404, "File not found") auth_mgr = getattr(request.app.state, "auth_manager", None) auth_configured = bool(auth_mgr and auth_mgr.is_configured) - current_user = get_current_user(request) + current_user = effective_user(request) file_owner = info.get("owner") if auth_configured: if not current_user: diff --git a/routes/webhook_routes.py b/routes/webhook_routes.py index da6288e7a..c9cf856ca 100644 --- a/routes/webhook_routes.py +++ b/routes/webhook_routes.py @@ -1,6 +1,5 @@ """Webhook, API Token, and sync chat routes.""" -import asyncio import uuid import logging from typing import Optional @@ -198,6 +197,8 @@ def setup_webhook_routes( "opencode-go": "https://opencode.ai/zen/go/v1", "fireworks": "https://api.fireworks.ai/inference/v1", "venice": "https://api.venice.ai/api/v1", + "kimi-code": "https://api.kimi.com/coding/v1", + "kimicode": "https://api.kimi.com/coding/v1", } # Model prefix → provider mapping for auto-detection @@ -210,6 +211,8 @@ def setup_webhook_routes( "mistral": "mistral", "llama": "groq", "mixtral": "groq", + "kimi-for-coding": "kimi-code", + "kimi": "kimi-code", } def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]: @@ -381,10 +384,10 @@ def setup_webhook_routes( sess.add_message(ChatMessage("assistant", reply)) session_manager.save_sessions() - asyncio.create_task(webhook_manager.fire("chat.completed", { + webhook_manager.fire_and_forget("chat.completed", { "session_id": session_id, "model": sess.model, "user_message": message[:2000], "response": reply[:2000], - })) + }) return {"response": reply, "session_id": session_id, "model": sess.model} diff --git a/routes/workspace_routes.py b/routes/workspace_routes.py new file mode 100644 index 000000000..ef70e78c2 --- /dev/null +++ b/routes/workspace_routes.py @@ -0,0 +1,85 @@ +"""Workspace API - browse server directories to pick a tool workspace folder.""" +import os +from fastapi import APIRouter, Request, HTTPException, Query + +from src.auth_helpers import get_current_user +from src.tool_security import owner_is_admin_or_single_user + +# Cap entries returned per directory (mirrors filesystem_tools._CODENAV_MAX_HITS). +# A huge directory shouldn't dump thousands of rows into the picker; the user can +# type/paste a path to jump straight in instead. +_MAX_BROWSE_DIRS = 500 + + +def setup_workspace_routes(): + router = APIRouter(prefix="/api/workspace", tags=["workspace"]) + + @router.get("/browse") + def browse(request: Request, path: str = Query(default="")): + """List subdirectories of `path` (default: home) so the UI can navigate + the server filesystem and pick a workspace folder. Directories only. + + ADMIN-ONLY: this enumerates the server filesystem, so it is gated the + same way the file/shell tools are (read_file/write_file/bash are in + NON_ADMIN_BLOCKED_TOOLS). A non-admin who can't use those tools must not + be able to map the host's directory tree either. + """ + owner = get_current_user(request) + if not owner_is_admin_or_single_user(owner): + raise HTTPException(status_code=403, detail="Workspace browsing is admin-only") + + # Resolve symlinks so the reported path is canonical and the UI navigates + # real directories (defends against symlink games in displayed paths). + target = os.path.realpath(os.path.expanduser(path.strip() or "~")) + if not os.path.isdir(target): + target = os.path.realpath(os.path.expanduser("~")) + + dirs = [] + try: + with os.scandir(target) as it: + for entry in it: + try: + # Don't follow symlinks when classifying - a symlinked + # dir is skipped rather than letting the browser wander + # off via a link. Hidden entries are omitted. + if entry.is_dir(follow_symlinks=False) and not entry.name.startswith("."): + # Build the child path server-side with os.path.join + # so it's correct on Windows (backslashes) and Linux. + dirs.append({"name": entry.name, "path": os.path.join(target, entry.name)}) + except OSError: + continue + except (PermissionError, OSError): + dirs = [] + + dirs_sorted = sorted(dirs, key=lambda d: d["name"].lower()) + truncated = len(dirs_sorted) > _MAX_BROWSE_DIRS + parent = os.path.dirname(target) + from src.tool_execution import vet_workspace + return { + "path": target, + "parent": parent if parent and parent != target else None, + "dirs": dirs_sorted[:_MAX_BROWSE_DIRS], + "truncated": truncated, + # Whether this directory may be bound as a workspace (filesystem + # roots and sensitive dirs may be browsed through but not chosen). + "selectable": vet_workspace(target) is not None, + } + + @router.get("/vet") + def vet(request: Request, path: str = Query(default="")): + """Validate a workspace path without binding it. + + The UI calls this before persisting a manually typed path (/workspace + set) so a typo, file path, deleted folder, sensitive dir, or filesystem + root is rejected up front with the canonical path returned on success, + instead of being stored client-side and silently dropped at chat time. + Admin-gated like /browse: it confirms path existence on the host. + """ + owner = get_current_user(request) + if not owner_is_admin_or_single_user(owner): + raise HTTPException(status_code=403, detail="Workspace selection is admin-only") + from src.tool_execution import vet_workspace + resolved = vet_workspace(path) + return {"ok": resolved is not None, "path": resolved} + + return router diff --git a/scripts/agent_migration_manifest.py b/scripts/agent_migration_manifest.py new file mode 100755 index 000000000..82b5d24a7 --- /dev/null +++ b/scripts/agent_migration_manifest.py @@ -0,0 +1,635 @@ +#!/usr/bin/env python3 +"""Build a neutral agent migration manifest. + +This helper is intentionally read-only. It does not import the Odysseus +application package, write to data/, call an LLM, or apply anything. It turns +common agent export shapes into a portable JSON manifest that Odysseus can +preview or import later. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import mimetypes +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + + +SCHEMA_VERSION = "agent-migration.v1" +TEXT_EXTENSIONS = { + ".cfg", + ".conf", + ".csv", + ".json", + ".log", + ".md", + ".markdown", + ".py", + ".rst", + ".toml", + ".txt", + ".yaml", + ".yml", +} + + +@dataclass(frozen=True) +class InputWarning: + path: str + message: str + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_path(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def stable_id(kind: str, source_name: str, *parts: Any) -> str: + raw = "\x1f".join([kind, source_name, *[str(part) for part in parts]]) + return f"{kind}:{hashlib.sha256(raw.encode('utf-8')).hexdigest()[:16]}" + + +def read_json(path: Path) -> Any: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def normalize_category(value: Any) -> str: + category = str(value or "fact").strip().lower() + return category or "fact" + + +def normalize_memory_text(item: Any) -> str: + if isinstance(item, str): + return item.strip() + if isinstance(item, dict): + for key in ("text", "content", "memory", "value"): + value = item.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def memory_metadata(item: Any, source_path: Path, index: int) -> dict[str, Any]: + metadata: dict[str, Any] = { + "source_path": str(source_path), + "source_index": index, + } + if isinstance(item, dict): + for key in ("id", "timestamp", "created_at", "updated_at", "source", "tags", "pinned"): + if key in item: + metadata[f"source_{key}"] = item.get(key) + return metadata + + +def payload_items(payload: Any, keys: tuple[str, ...]) -> Any: + if isinstance(payload, dict): + for key in keys: + if isinstance(payload.get(key), list): + return payload[key] + return payload + + +def collect_memory_json(path: Path, source_name: str) -> tuple[list[dict[str, Any]], list[InputWarning]]: + warnings: list[InputWarning] = [] + try: + payload = read_json(path) + except Exception as exc: + return [], [InputWarning(str(path), f"could not read JSON: {exc}")] + + payload = payload_items(payload, ("memories", "memory", "items", "data")) + + if not isinstance(payload, list): + return [], [InputWarning(str(path), "expected a JSON list or an object containing a memory list")] + + items: list[dict[str, Any]] = [] + seen: set[str] = set() + for index, item in enumerate(payload): + text = normalize_memory_text(item) + if not text: + warnings.append(InputWarning(str(path), f"skipped memory at index {index}: missing text")) + continue + digest = sha256_text(text.strip().lower()) + if digest in seen: + warnings.append(InputWarning(str(path), f"skipped duplicate memory at index {index}")) + continue + seen.add(digest) + category = normalize_category(item.get("category") if isinstance(item, dict) else "fact") + source = str(item.get("source") or source_name) if isinstance(item, dict) else source_name + items.append( + { + "id": stable_id("memory", source_name, path, index, digest), + "kind": "memory", + "text": text, + "category": category, + "source": source, + "metadata": memory_metadata(item, path, index), + } + ) + return items, warnings + + +def normalize_timestamp(value: Any) -> str | None: + if value is None or value == "": + return None + if isinstance(value, (int, float)): + try: + return ( + datetime.fromtimestamp(float(value), timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + except (OverflowError, OSError, ValueError): + return str(value) + return str(value) + + +def normalize_role(value: Any) -> str: + role = str(value or "unknown").strip().lower() + if role in {"human", "user"}: + return "user" + if role in {"assistant", "ai", "bot", "model"}: + return "assistant" + if role in {"system", "tool"}: + return role + return role or "unknown" + + +def content_part_text(part: Any) -> str: + if isinstance(part, str): + return part + if isinstance(part, dict): + for key in ("text", "content", "value"): + value = part.get(key) + if isinstance(value, str): + return value + if part.get("type") == "text" and isinstance(part.get("text"), str): + return part["text"] + return "" + + +def normalize_message_text(message: dict[str, Any]) -> str: + content = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join(text for text in (content_part_text(part).strip() for part in content) if text) + if isinstance(content, dict): + parts = content.get("parts") + if isinstance(parts, list): + return "\n".join(text for text in (content_part_text(part).strip() for part in parts) if text) + for key in ("text", "content", "value"): + value = content.get(key) + if isinstance(value, str): + return value + for key in ("text", "body", "message"): + value = message.get(key) + if isinstance(value, str): + return value + return "" + + +def normalize_message(message: dict[str, Any]) -> dict[str, Any] | None: + author = message.get("author") if isinstance(message.get("author"), dict) else {} + role = ( + message.get("role") + or message.get("sender") + or message.get("speaker") + or author.get("role") + or author.get("name") + ) + text = normalize_message_text(message).strip() + if not text: + return None + normalized: dict[str, Any] = { + "role": normalize_role(role), + "text": text, + } + timestamp = normalize_timestamp(message.get("created_at") or message.get("create_time") or message.get("timestamp")) + if timestamp: + normalized["created_at"] = timestamp + message_id = message.get("id") + if message_id is not None: + normalized["source_id"] = str(message_id) + return normalized + + +def chatgpt_mapping_messages(conversation: dict[str, Any]) -> list[dict[str, Any]]: + mapping = conversation.get("mapping") + if not isinstance(mapping, dict): + return [] + rows: list[tuple[float, int, dict[str, Any]]] = [] + for index, node in enumerate(mapping.values()): + if not isinstance(node, dict) or not isinstance(node.get("message"), dict): + continue + message = node["message"] + sort_value = message.get("create_time") + try: + sort_key = float(sort_value) + except (TypeError, ValueError): + sort_key = float(index) + normalized = normalize_message(message) + if normalized: + rows.append((sort_key, index, normalized)) + return [row[2] for row in sorted(rows, key=lambda row: (row[0], row[1]))] + + +def conversation_messages(conversation: dict[str, Any]) -> tuple[list[dict[str, Any]], str]: + mapped = chatgpt_mapping_messages(conversation) + if mapped: + return mapped, "chatgpt_mapping" + for key in ("messages", "chat_messages", "turns"): + raw_messages = conversation.get(key) + if isinstance(raw_messages, list): + messages = [ + normalized + for raw in raw_messages + if isinstance(raw, dict) + for normalized in [normalize_message(raw)] + if normalized + ] + return messages, key + return [], "unknown" + + +def conversation_title(conversation: dict[str, Any], index: int) -> str: + for key in ("title", "name", "summary"): + value = conversation.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return f"Conversation {index + 1}" + + +def collect_conversation_json( + path: Path, + source_name: str, + *, + include_content: bool = False, + max_messages: int = 2000, +) -> tuple[list[dict[str, Any]], list[InputWarning]]: + warnings: list[InputWarning] = [] + try: + payload = read_json(path) + except Exception as exc: + return [], [InputWarning(str(path), f"could not read JSON: {exc}")] + + payload = payload_items(payload, ("conversations", "conversation", "items", "data")) + if isinstance(payload, dict): + payload = [payload] + if not isinstance(payload, list): + return [], [InputWarning(str(path), "expected a JSON list or an object containing a conversation list")] + + items: list[dict[str, Any]] = [] + for index, conversation in enumerate(payload): + if not isinstance(conversation, dict): + warnings.append(InputWarning(str(path), f"skipped conversation at index {index}: expected object")) + continue + messages, format_hint = conversation_messages(conversation) + if not messages: + warnings.append(InputWarning(str(path), f"skipped conversation at index {index}: no text messages found")) + continue + title = conversation_title(conversation, index) + source_id = conversation.get("id") or conversation.get("uuid") or conversation.get("conversation_id") + text_digest = sha256_text("\n".join(f"{msg['role']}:{msg['text']}" for msg in messages)) + metadata: dict[str, Any] = { + "source_path": str(path), + "source_index": index, + "source_format": format_hint, + "message_count": len(messages), + "text_sha256": text_digest, + "content_included": False, + } + if source_id is not None: + metadata["source_id"] = str(source_id) + for key in ("create_time", "created_at", "update_time", "updated_at"): + timestamp = normalize_timestamp(conversation.get(key)) + if timestamp: + metadata[f"source_{key}"] = timestamp + item: dict[str, Any] = { + "id": stable_id("conversation", source_name, path, source_id or index, text_digest), + "kind": "conversation_thread", + "title": title, + "source": source_name, + "metadata": metadata, + } + if include_content: + if len(messages) > max_messages: + warnings.append( + InputWarning( + str(path), + f"skipped conversation content at index {index}: over {max_messages} messages", + ) + ) + else: + item["messages"] = messages + item["metadata"]["content_included"] = True + items.append(item) + return items, warnings + + +def parse_skill_frontmatter(text: str) -> dict[str, Any]: + if not text.startswith("---"): + return {} + end = text.find("\n---", 3) + if end < 0: + return {} + frontmatter: dict[str, Any] = {} + for line in text[3:end].strip().splitlines(): + if not line.strip() or line.lstrip().startswith("#") or ":" not in line: + continue + key, value = line.split(":", 1) + key = key.strip() + value = value.strip().strip('"').strip("'") + if key: + frontmatter[key] = value + return frontmatter + + +def collect_skill_dir(path: Path, source_name: str) -> tuple[list[dict[str, Any]], list[InputWarning]]: + warnings: list[InputWarning] = [] + if path.is_symlink(): + return [], [InputWarning(str(path), "skills path is a symlink; skipped")] + if not path.exists(): + return [], [InputWarning(str(path), "skills directory does not exist")] + if not path.is_dir(): + return [], [InputWarning(str(path), "skills path is not a directory")] + + items: list[dict[str, Any]] = [] + for skill_path in sorted(path.rglob("SKILL.md")): + if skill_path.is_symlink(): + warnings.append(InputWarning(str(skill_path), "skipped symlinked skill file")) + continue + try: + text = skill_path.read_text(encoding="utf-8") + except Exception as exc: + warnings.append(InputWarning(str(skill_path), f"could not read skill: {exc}")) + continue + frontmatter = parse_skill_frontmatter(text) + name = str(frontmatter.get("name") or skill_path.parent.name).strip() or skill_path.parent.name + items.append( + { + "id": stable_id("skill", source_name, skill_path, sha256_text(text)), + "kind": "skill", + "name": name, + "category": str(frontmatter.get("category") or "general"), + "source": source_name, + "format": "SKILL.md", + "content": text, + "metadata": { + "source_path": str(skill_path), + "sha256": sha256_text(text), + "frontmatter": frontmatter, + }, + } + ) + return items, warnings + + +def looks_textual(path: Path) -> bool: + if path.suffix.lower() in TEXT_EXTENSIONS: + return True + guessed, _ = mimetypes.guess_type(str(path)) + return bool(guessed and (guessed.startswith("text/") or guessed in {"application/json"})) + + +def iter_archive_dir(path: Path) -> Iterable[Path | InputWarning]: + try: + children = sorted(path.iterdir()) + except Exception as exc: + yield InputWarning(str(path), f"could not scan archive directory: {exc}") + return + for child in children: + if child.is_symlink(): + yield InputWarning(str(child), "skipped symlinked archive path") + continue + if child.is_file(): + yield child + elif child.is_dir(): + yield from iter_archive_dir(child) + + +def iter_archive_files(paths: Iterable[Path]) -> Iterable[Path | InputWarning]: + for path in paths: + if path.is_symlink(): + yield InputWarning(str(path), "skipped symlinked archive path") + continue + if path.is_file(): + yield path + elif path.is_dir(): + yield from iter_archive_dir(path) + + +def collect_archive_paths( + paths: list[Path], + source_name: str, + *, + include_content: bool = False, + max_bytes: int = 256_000, +) -> tuple[list[dict[str, Any]], list[InputWarning]]: + warnings: list[InputWarning] = [] + items: list[dict[str, Any]] = [] + existing_paths: list[Path] = [] + for path in paths: + if path.is_symlink(): + warnings.append(InputWarning(str(path), "archive path is a symlink; skipped")) + continue + if not path.exists(): + warnings.append(InputWarning(str(path), "archive path does not exist")) + continue + if not path.is_file() and not path.is_dir(): + warnings.append(InputWarning(str(path), "archive path is not a file or directory")) + continue + existing_paths.append(path) + + for entry in iter_archive_files(existing_paths): + if isinstance(entry, InputWarning): + warnings.append(entry) + continue + path = entry + if not looks_textual(path): + warnings.append(InputWarning(str(path), "skipped non-text archive file")) + continue + try: + st = path.stat() + except Exception as exc: + warnings.append(InputWarning(str(path), f"could not stat archive file: {exc}")) + continue + size = st.st_size + try: + file_hash = sha256_path(path) + except Exception as exc: + warnings.append(InputWarning(str(path), f"could not hash archive file: {exc}")) + continue + if include_content and size > max_bytes: + warnings.append(InputWarning(str(path), f"skipped archive content over {max_bytes} bytes")) + archive_item: dict[str, Any] = { + "id": stable_id("archive", source_name, path, file_hash), + "kind": "archive_document", + "title": path.name, + "source": source_name, + "metadata": { + "source_path": str(path), + "size_bytes": size, + "sha256": file_hash, + }, + } + if include_content and size <= max_bytes: + try: + archive_item["content"] = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + archive_item["content"] = path.read_text(encoding="utf-8", errors="replace") + archive_item["metadata"]["decoded_with_replacement"] = True + items.append(archive_item) + return items, warnings + + +def build_manifest(args) -> dict[str, Any]: + warnings: list[InputWarning] = [] + items: list[dict[str, Any]] = [] + + for path in args.memory_json: + collected, got_warnings = collect_memory_json(path, args.source_name) + items.extend(collected) + warnings.extend(got_warnings) + + for path in args.skills_dir: + collected, got_warnings = collect_skill_dir(path, args.source_name) + items.extend(collected) + warnings.extend(got_warnings) + + for path in args.conversation_json: + collected, got_warnings = collect_conversation_json( + path, + args.source_name, + include_content=args.include_conversation_content, + max_messages=args.max_conversation_messages, + ) + items.extend(collected) + warnings.extend(got_warnings) + + if args.archive: + collected, got_warnings = collect_archive_paths( + args.archive, + args.source_name, + include_content=args.include_archive_content, + max_bytes=args.max_archive_bytes, + ) + items.extend(collected) + warnings.extend(got_warnings) + + counts: dict[str, int] = {} + for item in items: + counts[item["kind"]] = counts.get(item["kind"], 0) + 1 + + return { + "schema_version": SCHEMA_VERSION, + "generated_at": utc_now_iso(), + "source": { + "name": args.source_name, + "kind": args.source_kind, + }, + "summary": { + "item_count": len(items), + "counts_by_kind": counts, + "warning_count": len(warnings), + }, + "items": items, + "warnings": [{"path": warning.path, "message": warning.message} for warning in warnings], + } + + +def parse_args(argv: list[str] | None = None): + parser = argparse.ArgumentParser(description="Build a neutral Odysseus agent migration manifest.") + parser.add_argument("--source-name", default="agent-export", help="Human-readable source name.") + parser.add_argument("--source-kind", default="generic", help="Source adapter kind, e.g. generic, openclaw, hermes.") + parser.add_argument( + "--memory-json", + action="append", + type=Path, + default=[], + help="JSON memory export. May be a list, or an object containing memories/items/data.", + ) + parser.add_argument( + "--skills-dir", + action="append", + type=Path, + default=[], + help="Directory containing SKILL.md files. Scanned recursively.", + ) + parser.add_argument( + "--archive", + action="append", + type=Path, + default=[], + help="Text/Markdown/JSON file or directory to preserve as archive documents.", + ) + parser.add_argument( + "--conversation-json", + action="append", + type=Path, + default=[], + help="Conversation export JSON. Supports generic message lists and ChatGPT-style conversations.json.", + ) + parser.add_argument( + "--include-archive-content", + action="store_true", + help="Embed archive document content in the manifest. By default only metadata is included.", + ) + parser.add_argument( + "--max-archive-bytes", + type=int, + default=256_000, + help="Maximum bytes to embed per archive file when --include-archive-content is used.", + ) + parser.add_argument( + "--include-conversation-content", + action="store_true", + help="Embed normalized conversation messages. By default only thread metadata is included.", + ) + parser.add_argument( + "--max-conversation-messages", + type=int, + default=2000, + help="Maximum messages to embed per conversation when --include-conversation-content is used.", + ) + parser.add_argument("--output", type=Path, help="Write manifest JSON to this path instead of stdout.") + parser.add_argument("--compact", action="store_true", help="Write compact JSON without indentation.") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + manifest = build_manifest(args) + text = json.dumps(manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":")) if args.compact else ( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text, encoding="utf-8") + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/backfill_model_release_dates.py b/scripts/backfill_model_release_dates.py new file mode 100755 index 000000000..741d8dac0 --- /dev/null +++ b/scripts/backfill_model_release_dates.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Backfill release_date on entries in services/hwfit/data/hf_models.json. + +Why: the `newest` sort in the cookbook ranks rows by release_date. Anything +missing a date sorts to the bottom. This script pulls `created_at` from the +HuggingFace API for each catalog entry without one (or all entries when +--refresh is passed) and writes the catalog back. + +Usage: + python scripts/backfill_model_release_dates.py # missing only + python scripts/backfill_model_release_dates.py --refresh # all entries + python scripts/backfill_model_release_dates.py --limit 50 # cap requests + python scripts/backfill_model_release_dates.py --dry-run # show, don't write + +Auth: set HF_TOKEN env var (or huggingface-cli login) to access gated repos. +""" +import argparse +import json +import os +import sys +import time +from datetime import datetime +from pathlib import Path + +try: + from huggingface_hub import HfApi + from huggingface_hub.utils import HfHubHTTPError +except ImportError: + print("Install huggingface_hub: pip install huggingface_hub", file=sys.stderr) + sys.exit(1) + + +CATALOG_PATH = Path(__file__).resolve().parent.parent / "services" / "hwfit" / "data" / "hf_models.json" + + +def fetch_release_date(api: HfApi, repo_id: str) -> str | None: + """Return YYYY-MM-DD release date, or None on miss / error.""" + try: + info = api.model_info(repo_id, files_metadata=False) + except HfHubHTTPError as e: + # 401 = gated/private, 404 = renamed/deleted. Either way, no date. + status = getattr(getattr(e, "response", None), "status_code", None) + print(f" {repo_id}: HTTP {status or '?'}", file=sys.stderr) + return None + except Exception as e: + print(f" {repo_id}: {type(e).__name__}: {e}", file=sys.stderr) + return None + created = getattr(info, "created_at", None) + if not created: + return None + return created.strftime("%Y-%m-%d") + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--refresh", action="store_true", help="Overwrite existing release_date too (default: only fill missing).") + p.add_argument("--limit", type=int, default=0, help="Stop after N API calls (0 = no limit).") + p.add_argument("--dry-run", action="store_true", help="Don't write back; just report.") + p.add_argument("--sleep", type=float, default=0.05, help="Seconds to sleep between requests (default 0.05).") + args = p.parse_args() + + if not CATALOG_PATH.exists(): + print(f"Catalog not found: {CATALOG_PATH}", file=sys.stderr) + sys.exit(2) + + with CATALOG_PATH.open(encoding="utf-8") as f: + catalog = json.load(f) + + candidates = [] + for i, m in enumerate(catalog): + name = m.get("name") + if not name: + continue + existing = (m.get("release_date") or "").strip() + if existing and not args.refresh: + continue + candidates.append(i) + + if args.limit: + candidates = candidates[: args.limit] + + print(f"Catalog: {CATALOG_PATH}") + print(f"Total entries: {len(catalog)}") + print(f"Targets ({'refresh all' if args.refresh else 'missing only'}{'' if not args.limit else f', capped at {args.limit}'}): {len(candidates)}") + if not candidates: + print("Nothing to do.") + return + + api = HfApi(token=os.environ.get("HF_TOKEN") or None) + updated = 0 + skipped = 0 + started = time.time() + for n, idx in enumerate(candidates, start=1): + entry = catalog[idx] + name = entry["name"] + old = (entry.get("release_date") or "").strip() + new = fetch_release_date(api, name) + if new is None: + skipped += 1 + tag = "skip" + elif new == old: + tag = "unchanged" + else: + entry["release_date"] = new + updated += 1 + tag = f"set {new}" + (f" (was {old})" if old else "") + print(f"[{n}/{len(candidates)}] {name} — {tag}") + if args.sleep: + time.sleep(args.sleep) + + elapsed = time.time() - started + print() + print(f"Done in {elapsed:.1f}s — {updated} updated, {skipped} skipped (HF unavailable / gated / missing date).") + + if args.dry_run: + print("Dry run — no write.") + return + + if updated: + # Atomic write: tmp file in the same dir, then rename. Keeps the + # catalog usable even if the process dies mid-write. + tmp = CATALOG_PATH.with_suffix(".json.tmp") + with tmp.open("w", encoding="utf-8") as f: + json.dump(catalog, f, indent=1, ensure_ascii=False) + f.write("\n") + tmp.replace(CATALOG_PATH) + print(f"Wrote {CATALOG_PATH}") + else: + print("No changes to write.") + + +if __name__ == "__main__": + main() diff --git a/scripts/import_from_vllm_recipes.py b/scripts/import_from_vllm_recipes.py new file mode 100755 index 000000000..2dd65def8 --- /dev/null +++ b/scripts/import_from_vllm_recipes.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Import models from the upstream vllm-project/recipes catalog into our +local hf_models.json. Two modes: + + --update-existing Stamp min_vllm_version + vllm_recipe=True on rows we + already carry. Cheap, no HF API calls. + --add-missing Create new catalog rows for every recipe model we + don't carry. Hits the HF API for created_at + downloads + (~1 req per missing model, paced). + +Both modes write atomically (tmp + rename) so a crashed run leaves the +catalog intact. Default with no mode flags runs both, prefer to pass them +explicitly. + +Usage: + python scripts/import_from_vllm_recipes.py --update-existing + python scripts/import_from_vllm_recipes.py --add-missing + python scripts/import_from_vllm_recipes.py --dry-run + python scripts/import_from_vllm_recipes.py --limit 10 + +Auth: set HF_TOKEN to access gated repos when --add-missing. +""" +import argparse +import json +import os +import re +import sys +import time +from datetime import datetime +from pathlib import Path + +try: + import httpx + import yaml +except ImportError: + print("pip install httpx PyYAML", file=sys.stderr) + sys.exit(1) + +try: + from huggingface_hub import HfApi + from huggingface_hub.utils import HfHubHTTPError +except ImportError: + HfApi = None + HfHubHTTPError = Exception + + +CATALOG_PATH = Path(__file__).resolve().parent.parent / "services" / "hwfit" / "data" / "hf_models.json" +RECIPES_TREE_URL = ( + "https://api.github.com/repos/vllm-project/recipes/git/trees/main?recursive=1" +) +RECIPE_RAW_URL = ( + "https://raw.githubusercontent.com/vllm-project/recipes/main/models/{repo}.yaml" +) + + +# Map recipe `precision` to the closest catalog `quantization` label that +# fit.py / models.py already understand. +_PRECISION_TO_QUANT = { + "fp8": "FP8", + "nvfp4": "NVFP4", + "mxfp4": "MXFP4", + "bf16": "BF16", + "fp16": "F16", + "f16": "F16", + "fp4": "FP4", + "int8": "INT8", + "int4": "INT4", + "awq-4bit": "AWQ-4bit", + "awq-8bit": "AWQ-8bit", +} + +# Architecture name → use_case fallback. fit.py weights use_case for filtering; +# missing field defaults to a generic bucket. +_ARCH_USE_CASE = { + "moe": "General-purpose reasoning, long-context", + "llama": "General-purpose chat", + "qwen2": "General-purpose chat", + "qwen3": "General-purpose reasoning", + "deepseek_v3_moe": "General-purpose reasoning, long-context", + "deepseek_v4_moe": "General-purpose reasoning, long-context", +} + + +def _parse_param_count(s) -> int: + """'230B' / '8.6B' / '4.2T' → integer parameter count.""" + if s is None: + return 0 + s = str(s).strip().replace(",", "") + m = re.match(r"^([\d.]+)\s*([KMBT]?)$", s, re.I) + if not m: + return 0 + num = float(m.group(1)) + unit = (m.group(2) or "").upper() + mult = {"K": 1e3, "M": 1e6, "B": 1e9, "T": 1e12, "": 1.0}[unit] + return int(num * mult) + + +def _capabilities_for(arch: str, hardware: dict, ctx_len: int, has_reasoning: bool) -> list[str]: + caps = [] + if "moe" in (arch or "").lower(): + caps.append("moe") + if has_reasoning: + caps.append("reasoning") + if ctx_len and ctx_len >= 100_000: + caps.append("long_context") + if any(hw in (hardware or {}) for hw in ("mi300x", "mi325x", "mi350x", "mi355x")): + caps.append("amd_supported") + return caps + + +def _fetch_manifest(client: httpx.Client) -> set[str]: + r = client.get(RECIPES_TREE_URL, headers={"Accept": "application/vnd.github+json"}, timeout=15) + r.raise_for_status() + tree = (r.json() or {}).get("tree") or [] + out: set[str] = set() + for e in tree: + path = (e or {}).get("path") or "" + if path.startswith("models/") and path.endswith(".yaml"): + body = path[len("models/"):-len(".yaml")] + if "/" in body: + out.add(body) + return out + + +def _fetch_recipe(client: httpx.Client, repo: str) -> dict | None: + url = RECIPE_RAW_URL.format(repo=repo) + try: + r = client.get(url, timeout=10) + if r.status_code != 200: + return None + return yaml.safe_load(r.text) or {} + except Exception: + return None + + +def _stamp_from_recipe(entry: dict, recipe: dict) -> bool: + """Mutate entry with recipe-derived fields. Returns True if anything changed.""" + model = recipe.get("model") or {} + meta = recipe.get("meta") or {} + features = recipe.get("features") or {} + + changed = False + new_min = (model.get("min_vllm_version") or "").strip() + if new_min and entry.get("min_vllm_version") != new_min: + entry["min_vllm_version"] = new_min + changed = True + if not entry.get("vllm_recipe"): + entry["vllm_recipe"] = True + changed = True + # Hardware support map — useful for filtering "which models run on my AMD box". + hw = meta.get("hardware") or {} + if hw and entry.get("recipe_hardware") != hw: + entry["recipe_hardware"] = {k: str(v) for k, v in hw.items()} + changed = True + # Tool/reasoning parser hints — purely informational at catalog level; + # the live launch command builder still reads them from the recipe API. + if features.get("reasoning") and not entry.get("has_reasoning_parser"): + entry["has_reasoning_parser"] = True + changed = True + if features.get("tool_calling") and not entry.get("has_tool_call_parser"): + entry["has_tool_call_parser"] = True + changed = True + return changed + + +def _build_new_entry(repo: str, recipe: dict, hf_info=None) -> dict | None: + """Build a fresh catalog entry from a recipe + (optional) HF model info.""" + model = recipe.get("model") or {} + meta = recipe.get("meta") or {} + features = recipe.get("features") or {} + variants = recipe.get("variants") or {} + + org, name = repo.split("/", 1) + raw_params = _parse_param_count(model.get("parameter_count")) + active_raw = _parse_param_count(model.get("active_parameters")) + ctx = model.get("context_length") or 0 + + # Pick the smallest-VRAM variant as the catalog quant — that's what most + # users land on first. NVFP4/MXFP4 typically win this on Blackwell; + # FP8 elsewhere; BF16 baseline only. + pick_quant = None + pick_vram = None + for vk, vv in variants.items(): + if not isinstance(vv, dict): + continue + prec = (vv.get("precision") or "").lower() + vram = vv.get("vram_minimum_gb") or 0 + quant = _PRECISION_TO_QUANT.get(prec) + if quant and (pick_vram is None or (vram and vram < pick_vram)): + pick_quant = quant + pick_vram = vram or pick_vram + if not pick_quant: + pick_quant = "BF16" + + arch = (model.get("architecture") or "").lower() + use_case = _ARCH_USE_CASE.get(arch, "General-purpose chat") + caps = _capabilities_for(arch, meta.get("hardware") or {}, ctx, bool(features.get("reasoning"))) + + rel_date = "" + downloads = 0 + likes = 0 + if hf_info is not None: + created = getattr(hf_info, "created_at", None) + if created: + rel_date = created.strftime("%Y-%m-%d") + downloads = int(getattr(hf_info, "downloads", 0) or 0) + likes = int(getattr(hf_info, "likes", 0) or 0) + if not rel_date: + rel_date = str(meta.get("date_updated") or datetime.utcnow().strftime("%Y-%m-%d")) + + entry: dict = { + "name": repo, + "provider": org, + "parameter_count": str(model.get("parameter_count") or "?"), + "parameters_raw": raw_params, + "is_moe": "moe" in arch, + "quantization": pick_quant, + "context_length": int(ctx or 0), + "use_case": use_case, + "capabilities": caps, + "pipeline_tag": "text-generation", + "architecture": arch or "unknown", + "hf_downloads": downloads, + "hf_likes": likes, + "release_date": rel_date, + # Recipe-derived bits. + "vllm_recipe": True, + "min_vllm_version": (model.get("min_vllm_version") or "").strip() or None, + "recipe_hardware": {k: str(v) for k, v in (meta.get("hardware") or {}).items()}, + "has_reasoning_parser": bool(features.get("reasoning")), + "has_tool_call_parser": bool(features.get("tool_calling")), + } + if active_raw: + entry["active_parameters"] = active_raw + if pick_vram: + # min_vram_gb is what hwfit uses for "does this fit". Recipe states a + # minimum for the chosen variant; round up slightly for KV-cache room. + entry["min_vram_gb"] = float(pick_vram) + entry["min_ram_gb"] = float(round(pick_vram * 0.6, 1)) + entry["recommended_ram_gb"] = float(round(pick_vram * 1.2, 1)) + # Drop empty / None fields to keep the JSON tidy. + return {k: v for k, v in entry.items() if v not in (None, "", [], {})} + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--update-existing", action="store_true", help="Stamp min_vllm_version + vllm_recipe on existing rows.") + p.add_argument("--add-missing", action="store_true", help="Add new rows for recipe models not in the catalog.") + p.add_argument("--limit", type=int, default=0, help="Stop after N recipe fetches.") + p.add_argument("--dry-run", action="store_true", help="Don't write back; just report.") + p.add_argument("--sleep", type=float, default=0.05, help="Seconds between HTTP requests.") + args = p.parse_args() + if not args.update_existing and not args.add_missing: + args.update_existing = args.add_missing = True + + with CATALOG_PATH.open(encoding="utf-8") as f: + catalog = json.load(f) + by_name = {m.get("name"): m for m in catalog if m.get("name")} + + client = httpx.Client(follow_redirects=True) + print(f"Catalog: {CATALOG_PATH} ({len(catalog)} entries)") + print("Fetching upstream manifest…") + try: + manifest = _fetch_manifest(client) + except Exception as e: + print(f"FATAL: manifest fetch failed: {e}", file=sys.stderr) + sys.exit(2) + print(f"Manifest: {len(manifest)} recipes") + + existing = sorted(by_name.keys() & manifest) + missing = sorted(manifest - by_name.keys()) + print(f"Match catalog ↔ manifest: existing={len(existing)} missing={len(missing)}") + + targets: list[tuple[str, str]] = [] # (repo, action) + if args.update_existing: + targets.extend((r, "update") for r in existing) + if args.add_missing: + targets.extend((r, "add") for r in missing) + if args.limit: + targets = targets[: args.limit] + print(f"Targets: {len(targets)}") + + hf_api = HfApi(token=os.environ.get("HF_TOKEN") or None) if HfApi else None + updated = added = skipped = 0 + started = time.time() + + for n, (repo, action) in enumerate(targets, 1): + recipe = _fetch_recipe(client, repo) + if not recipe: + print(f"[{n}/{len(targets)}] {repo:55} skip (no recipe fetched)") + skipped += 1 + time.sleep(args.sleep) + continue + if action == "update": + entry = by_name[repo] + if _stamp_from_recipe(entry, recipe): + updated += 1 + print(f"[{n}/{len(targets)}] {repo:55} updated") + else: + print(f"[{n}/{len(targets)}] {repo:55} unchanged") + else: # add + hf_info = None + if hf_api: + try: + hf_info = hf_api.model_info(repo, files_metadata=False) + except HfHubHTTPError as e: + code = getattr(getattr(e, "response", None), "status_code", "?") + print(f" HF {code} for {repo} — building from recipe only", file=sys.stderr) + except Exception as e: + print(f" HF error for {repo}: {e}", file=sys.stderr) + new_entry = _build_new_entry(repo, recipe, hf_info) + if new_entry: + catalog.append(new_entry) + by_name[repo] = new_entry + added += 1 + print(f"[{n}/{len(targets)}] {repo:55} added ({new_entry.get('parameter_count','?')}, {new_entry.get('quantization','?')})") + else: + skipped += 1 + print(f"[{n}/{len(targets)}] {repo:55} skip (couldn't build entry)") + time.sleep(args.sleep) + + elapsed = time.time() - started + print() + print(f"Done in {elapsed:.1f}s — added={added}, updated={updated}, skipped={skipped}") + + if args.dry_run: + print("Dry run — no write.") + return + if added or updated: + tmp = CATALOG_PATH.with_suffix(".json.tmp") + with tmp.open("w", encoding="utf-8") as f: + json.dump(catalog, f, indent=1, ensure_ascii=False) + f.write("\n") + tmp.replace(CATALOG_PATH) + print(f"Wrote {CATALOG_PATH} ({len(catalog)} entries)") + else: + print("No changes — catalog untouched.") + + +if __name__ == "__main__": + main() diff --git a/scripts/odysseus-calendar b/scripts/odysseus-calendar index 562551040..5a5f345bc 100755 --- a/scripts/odysseus-calendar +++ b/scripts/odysseus-calendar @@ -103,9 +103,13 @@ def cmd_list(args) -> None: end = _parse_dt(args.end) if args.end else (start + timedelta(days=30)) db = SessionLocal() try: + # Overlap semantics, matching the web route (routes/calendar_routes.py) + # and the recurring-expansion contract: an event is in the window when + # it starts before the window end AND ends after the window start. This + # includes multi-day / in-progress events that began before `start`. q = db.query(CalendarEvent).filter( - CalendarEvent.dtstart >= start, CalendarEvent.dtstart < end, + CalendarEvent.dtend > start, ) if args.calendar: cal = db.query(CalendarCal).filter(CalendarCal.name == args.calendar).first() diff --git a/services/hwfit/fit.py b/services/hwfit/fit.py index 09aea29db..14865d905 100644 --- a/services/hwfit/fit.py +++ b/services/hwfit/fit.py @@ -19,22 +19,36 @@ GPU_BANDWIDTH = { "6950 xt": 576, "6900 xt": 512, "6800 xt": 512, "6800": 512, "6700 xt": 384, "6600 xt": 256, "6600": 224, "mi300x": 5300, "mi300": 5300, "mi250x": 3277, "mi250": 3277, "mi210": 1638, "mi100": 1229, "9070 xt": 624, "9070": 488, "9060 xt": 322, "9060": 322, - # Apple Silicon unified-memory bandwidth (GB/s). Keyed off the chip name - # reported by sysctl machdep.cpu.brand_string (e.g. "Apple M4 Max"). Listed - # before the bare "m_" keys matters less than length-sorting (done below), - # which guarantees "m4 max" is tried before "m4". - "m1 ultra": 800, "m1 max": 400, "m1 pro": 200, "m1": 68, - "m2 ultra": 800, "m2 max": 400, "m2 pro": 200, "m2": 100, - "m3 ultra": 800, "m3 max": 300, "m3 pro": 150, "m3": 100, - "m4 max": 546, "m4 pro": 273, "m4": 120, - "m5 max": 546, "m5 pro": 273, "m5": 150, + # NVIDIA GB10 Grace-Blackwell superchip (DGX Spark). Unified LPDDR5X memory, + # not Apple Silicon, so it lives in the generic GPU table — the Apple-only + # lookup never matches it (its name carries no "apple"). + "gb10": 273, } # Pre-sort keys by length descending for correct substring matching _BW_KEYS_SORTED = sorted(GPU_BANDWIDTH.keys(), key=len, reverse=True) -# metal: backstop for Apple Silicon chips not in GPU_BANDWIDTH (e.g. a future -# M5) — the named chips above take the accurate bandwidth path instead. +# Apple Silicon unified-memory bandwidth (GB/s). For chip families with both +# binned and full variants under the same "Apple Mx Max" brand string, prefer +# GPU core count when hardware detection provides it; otherwise fall back to the +# conservative tier so speed estimates do not over-promise. +APPLE_BANDWIDTH_FIXED = { + "m1 ultra": 800, "m1 max": 400, "m1 pro": 200, "m1": 68, + "m2 ultra": 800, "m2 max": 400, "m2 pro": 200, "m2": 100, + "m3 ultra": 800, "m3 pro": 150, "m3": 100, + "m4 pro": 273, "m4": 120, + "m5 pro": 307, "m5": 153, +} +APPLE_BANDWIDTH_BY_CORES = { + "m3 max": {30: 300, 40: 400}, + "m4 max": {32: 410, 40: 546}, + "m5 max": {32: 460, 40: 614}, +} +_APPLE_FIXED_KEYS_SORTED = sorted(APPLE_BANDWIDTH_FIXED.keys(), key=len, reverse=True) +_APPLE_VARIANT_KEYS_SORTED = sorted(APPLE_BANDWIDTH_BY_CORES.keys(), key=len, reverse=True) + +# metal: backstop for Apple Silicon chips not in the explicit tables above +# (e.g. a future M6) — use a conservative generic estimate when unknown. FALLBACK_K = {"cuda": 220, "rocm": 180, "metal": 150, "cpu_x86": 70, "cpu_arm": 90} USE_CASE_WEIGHTS = { @@ -60,16 +74,99 @@ CONTEXT_TARGET = { } -def _lookup_bandwidth(gpu_name): +def _lookup_apple_bandwidth(system): + gpu_name = system.get("gpu_name") if not isinstance(gpu_name, str) or not gpu_name: return None gn = gpu_name.lower() + + # Guard against false matches on non-Apple GPUs whose names contain + # "m3"/"m4"/"m5" (e.g. NVIDIA Quadro M4 000). + if "apple" not in gn: + return None + + raw_cores = system.get("gpu_cores") + try: + gpu_cores = int(raw_cores) if raw_cores is not None else None + except (TypeError, ValueError): + gpu_cores = None + + for key in _APPLE_VARIANT_KEYS_SORTED: + if key not in gn: + continue + if gpu_cores in APPLE_BANDWIDTH_BY_CORES[key]: + return APPLE_BANDWIDTH_BY_CORES[key][gpu_cores] + return min(APPLE_BANDWIDTH_BY_CORES[key].values()) + + for key in _APPLE_FIXED_KEYS_SORTED: + if key in gn: + return APPLE_BANDWIDTH_FIXED[key] + return None + + +def _lookup_bandwidth(system): + if isinstance(system, dict): + gpu_name = system.get("gpu_name") + else: + gpu_name = system + + if not isinstance(gpu_name, str) or not gpu_name: + return None + + # Apple tiers live only in the Apple-specific table now (#2564), so route + # BOTH dict and bare-string callers through it. A bare string carries no + # gpu_cores, so the helper falls back to the conservative (lowest) tier for + # that model -- before #2564 the generic table answered string lookups, and + # dropping that made _lookup_bandwidth("Apple M3 Max") return None. + apple_input = system if isinstance(system, dict) else {"gpu_name": gpu_name} + bw = _lookup_apple_bandwidth(apple_input) + if bw is not None: + return bw + + gn = gpu_name.lower() for key in _BW_KEYS_SORTED: if key in gn: return GPU_BANDWIDTH[key] return None +def _canonical_cpu_backend(system): + """Return the canonical CPU backend for cpu_only speed estimation. + + Normalizes CPU-architecture aliases separately from the GPU backend, and + overrides GPU-only backends (CUDA/ROCm/Metal) so they do not inherit a + discrete-GPU fallback constant when the model is actually running on CPU. + """ + backend = (system.get("backend") or "").lower().strip() + cpu_arch = (system.get("cpu_arch") or "").lower().strip() + cpu_name = (system.get("cpu_name") or "").lower() + gpu_name = (system.get("gpu_name") or "").lower() + + # Already-canonical CPU backends + if backend in ("cpu_x86", "cpu_arm"): + return backend + + # Raw CPU-architecture aliases + if backend in ("x86_64", "amd64", "i386", "i686"): + return "cpu_x86" + if backend in ("arm64", "aarch64", "arm"): + return "cpu_arm" + + # Prefer an explicit CPU architecture field when present + if cpu_arch: + if cpu_arch in ("x86_64", "amd64", "x86", "i386", "i686"): + return "cpu_x86" + if cpu_arch in ("arm64", "aarch64", "arm"): + return "cpu_arm" + + # Apple Silicon enters ranking as backend="metal"; its CPU path is ARM. + if backend in ("metal", "mps", "apple") or "apple" in cpu_name or "apple" in gpu_name: + return "cpu_arm" + + # Conservative default for CUDA/ROCm/discrete GPU backends and unknowns. + return "cpu_x86" + + def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0): """Estimate tok/s. Uses active params for MoE (only active experts run per token). @@ -84,9 +181,14 @@ def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0): """ pb = _active_params_b(model) is_moe = model.get("is_moe", False) - bw = _lookup_bandwidth(system.get("gpu_name")) + bw = _lookup_bandwidth(system) backend = system.get("backend", "cpu_x86") + # CPU-only inference must never inherit a GPU backend's fallback constant, + # even if the detected system happens to report a CUDA/Metal/ROCm backend. + if run_mode == "cpu_only": + backend = _canonical_cpu_backend(system) + if bw and run_mode in ("gpu", "cpu_offload"): bpp = QUANT_BYTES_PER_PARAM.get(quant, 0.5) model_gb = pb * bpp diff --git a/services/hwfit/hardware.py b/services/hwfit/hardware.py index 47ec94d44..a3ad7ba05 100644 --- a/services/hwfit/hardware.py +++ b/services/hwfit/hardware.py @@ -1,3 +1,4 @@ +import json import os import platform import re @@ -335,6 +336,37 @@ def _detect_apple_silicon(): if total_gb <= 0: return None + def _parse_apple_gpu_cores(text): + if not text: + return None + try: + data = json.loads(text) + except (TypeError, ValueError, json.JSONDecodeError): + data = None + if isinstance(data, dict): + for gpu in data.get("SPDisplaysDataType") or []: + if not isinstance(gpu, dict): + continue + model = str(gpu.get("sppci_model") or gpu.get("_name") or "") + if "apple" not in model.lower(): + continue + cores = gpu.get("sppci_cores") + try: + return int(str(cores).strip()) + except (TypeError, ValueError): + continue + m = re.search(r"Total Number of Cores:\s*(\d+)", text) + if m: + try: + return int(m.group(1)) + except ValueError: + return None + return None + + gpu_cores = _parse_apple_gpu_cores(_run(["system_profiler", "SPDisplaysDataType", "-json"])) + if gpu_cores is None: + gpu_cores = _parse_apple_gpu_cores(_run(["system_profiler", "SPDisplaysDataType"])) + # Usable GPU budget. macOS lets Metal use most of unified memory, but the # default working-set limit scales with RAM: small machines have to keep # more back for the OS + app. These fractions track Apple's @@ -357,7 +389,7 @@ def _detect_apple_silicon(): pass gpu = {"index": 0, "name": brand, "vram_gb": vram_gb} - return { + info = { "gpu_name": brand, "gpu_vram_gb": vram_gb, "gpu_count": 1, @@ -369,6 +401,9 @@ def _detect_apple_silicon(): # separate pool — downstream fit logic uses this to avoid double-budgeting. "unified_memory": True, } + if gpu_cores is not None: + info["gpu_cores"] = gpu_cores + return info def _read_file(path): @@ -611,6 +646,93 @@ def _cache_key(host: str, ssh_port: str, platform_name: str): ) +def _is_containerized(): + """Best-effort check for whether the local Odysseus process is running in a container.""" + if _remote_host: + return False + + if os.path.exists("/.dockerenv"): + return True + + try: + with open("/proc/1/cgroup", encoding="utf-8", errors="replace") as f: + text = f.read().lower() + return any(marker in text for marker in ("docker", "containerd", "kubepods")) + except Exception: + return False + + +def _hardware_visibility_warning(result): + """Return a non-blocking UX warning when detected hardware may only be container-visible.""" + if not isinstance(result, dict): + return None + + if result.get("manual_hardware"): + return None + + if not result.get("containerized"): + return None + + if result.get("gpu_error"): + return None + + if not result.get("has_gpu"): + return { + "code": "container_no_gpu_visible", + "severity": "warning", + "title": "No GPU visible inside Docker", + "message": ( + "Cookbook is scanning hardware from inside the Odysseus container. " + "If your host has a GPU, Docker may not be exposing it to the container, " + "so model recommendations may be CPU-only or too conservative." + ), + "actions": [ + "manual_hardware", + "rescan", + "copy_diagnostics", + ], + } + + total_ram = result.get("total_ram_gb") or 0 + if total_ram and total_ram <= 8: + return { + "code": "container_low_ram_visible", + "severity": "info", + "title": "Container-visible RAM may be lower than host RAM", + "message": ( + "Cookbook is seeing the RAM available inside the container. " + "If your host has more memory, validate host RAM separately or use Manual Hardware." + ), + "actions": [ + "manual_hardware", + "rescan", + "copy_diagnostics", + ], + } + + return None + + +def _attach_probe_context(result, host=""): + """Attach probe-scope metadata and optional hardware visibility warning.""" + if not isinstance(result, dict) or result.get("error"): + return result + + is_remote = bool(host) + containerized = False if is_remote else _is_containerized() + + result["probe_scope"] = "remote" if is_remote else ("container" if containerized else "native") + result["containerized"] = containerized + + warning = _hardware_visibility_warning(result) + if warning: + result["hardware_visibility_warning"] = warning + else: + result.pop("hardware_visibility_warning", None) + + return result + + def detect_system(host="", ssh_port="", platform="", fresh=False): """Detect system hardware: RAM, CPU, GPU. Cached per host (hardware rarely changes, and probing a remote host over SSH is slow). Pass fresh=True to @@ -635,6 +757,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False): if _remote_platform == "windows" and _remote_host: result = _detect_windows() if result: + result = _attach_probe_context(result, host=host) _remote_host = None _remote_platform = None _cache_by_host[cache_key] = (now, result) @@ -653,6 +776,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False): if not _remote_host and os.name == "nt": result = _detect_windows() if result: + result = _attach_probe_context(result, host=host) _cache_by_host[cache_key] = (now, result) return result # PowerShell probe failed entirely — fall through to the generic path @@ -683,6 +807,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False): "gpu_name": gpu_info["gpu_name"], "gpu_vram_gb": gpu_info["gpu_vram_gb"], "gpu_count": gpu_info["gpu_count"], + "gpu_cores": gpu_info.get("gpu_cores"), "gpus": gpu_info.get("gpus", []), "gpu_groups": gpu_info.get("gpu_groups", []), "homogeneous": gpu_info.get("homogeneous", True), @@ -714,6 +839,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False): "gpu_error": _last_gpu_error, } + result = _attach_probe_context(result, host=host) _remote_host = None _remote_platform = None _cache_by_host[cache_key] = (now, result) diff --git a/services/hwfit/profiles.py b/services/hwfit/profiles.py index 87aa147fe..337af7648 100644 --- a/services/hwfit/profiles.py +++ b/services/hwfit/profiles.py @@ -188,12 +188,18 @@ def compute_serve_profiles(system, model, serve_weights_gb=None, serve_quant=Non # Shrink context if even the chosen KV won't fit alongside weights. # Start from the smaller of the profile's target and the model's limit. cur_ctx = min(ctx, model_ctx_max) - while cur_ctx >= 8192: + # Floor the context-shrink loop at 8192, but never above the model's own + # trained limit. A model with a sub-8192 context (e.g. a 2048-token + # SmolLM) starts below 8192, so a hard-coded 8192 guard skipped the loop + # entirely and produced NO profile — the serve UI then fell back to + # manual flags even though the model fits the GPU trivially. + ctx_floor = min(8192, model_ctx_max) + while cur_ctx >= ctx_floor: kv = _kv_gb(model, cur_ctx, kv_type) n_cpu_moe, fits = _cpu_moe_for_budget(model, quant, kv, budget, fixed_gb=serve_weights_gb) est = _weights_gb(model, quant, serve_weights_gb) + kv + 0.6 # If a non-MoE model can't fit even fully offloaded, try less context. - if model.get("is_moe") or fits or cur_ctx <= 8192: + if model.get("is_moe") or fits or cur_ctx <= ctx_floor: profiles.append({ "key": key, "label": label, diff --git a/services/memory/skill_extractor.py b/services/memory/skill_extractor.py index 79e4c67c2..3c6b7c59c 100644 --- a/services/memory/skill_extractor.py +++ b/services/memory/skill_extractor.py @@ -66,41 +66,57 @@ def _has_duplicate_title(skills, title: str) -> bool: def _extract_json_object(text: str) -> Optional[dict]: """Best-effort extraction of a JSON object from an LLM response. - The response may be wrapped in code fences or surrounded by prose, and some - models emit a stray brace in the prose before the real object - (e.g. "uses {placeholder} then {...}"). Slicing first-'{' .. last-'}' then - grabs an unparseable span and the skill is silently lost. Try the whole - string first, then each '{' start position in turn, returning the first - candidate that parses to a JSON object (dict). Returns None if none do. + The response may be wrapped in code fences or surrounded by prose. Uses + json.JSONDecoder().raw_decode() to locate the boundaries of complete JSON + objects starting at each '{' position. Nested objects are filtered out to + keep only top-level candidates. If multiple non-overlapping valid JSON + objects are found, it is treated as ambiguous and returns None. Otherwise, + returns the single valid candidate dictionary. """ if not text: return None s = text.strip() if s.startswith("```"): s = s.split("\n", 1)[-1].rsplit("```", 1)[0].strip() - end = s.rfind("}") - if end == -1: + + decoder = json.JSONDecoder() + candidates = [] + + start = s.find("{") + while start != -1: + try: + obj, idx = decoder.raw_decode(s[start:]) + end_pos = start + idx + if isinstance(obj, dict): + candidates.append((start, end_pos, obj)) + except (json.JSONDecodeError, ValueError): + pass + start = s.find("{", start + 1) + + # Filter out nested candidates to identify top-level dictionaries + top_level = [] + for c in candidates: + is_nested = False + for other in candidates: + if other == c: + continue + if other[0] <= c[0] and c[1] <= other[1]: + is_nested = True + break + if not is_nested: + top_level.append(c) + + if not top_level: return None - def _as_dict(candidate): - try: - obj = json.loads(candidate) - except (json.JSONDecodeError, ValueError): - return None - return obj if isinstance(obj, dict) else None + if len(top_level) > 1: + logger.debug( + "[skill-extract] Found multiple non-overlapping JSON objects: %s", + [item[2].get("title") for item in top_level] + ) + return None - # The clean, common case: the whole (de-fenced) string is the object. - obj = _as_dict(s) - if obj is not None: - return obj - # Otherwise scan each '{' candidate up to the last '}'. - start = s.find("{") - while 0 <= start < end: - obj = _as_dict(s[start : end + 1]) - if obj is not None: - return obj - start = s.find("{", start + 1) - return None + return top_level[0][2] async def maybe_extract_skill( diff --git a/services/memory/skills.py b/services/memory/skills.py index 9cfe801e1..5baaa88c5 100644 --- a/services/memory/skills.py +++ b/services/memory/skills.py @@ -603,7 +603,6 @@ class SkillsManager: escalation) — those are work-in-progress and pollute the prompt with half-finished procedures. """ - active_toolsets = active_toolsets or [] out = [] for s in self.load(owner=owner): status = s.get("status") @@ -617,13 +616,16 @@ class SkillsManager: # Platform gating if platform and s.get("platforms") and platform not in s["platforms"]: continue - # requires_toolsets: hide unless every required toolset is active + # requires_toolsets: hide unless every required toolset is active. + # active_toolsets=None means the caller doesn't know the active + # set (API listings, chat preface) — don't gate in that case; + # only an explicit list filters. req = s.get("requires_toolsets") or [] - if req and not all(t in active_toolsets for t in req): + if req and active_toolsets is not None and not all(t in active_toolsets for t in req): continue # fallback_for_toolsets: hide when any of those toolsets is active fb = s.get("fallback_for_toolsets") or [] - if fb and any(t in active_toolsets for t in fb): + if fb and active_toolsets and any(t in active_toolsets for t in fb): continue out.append({ "name": s["name"], diff --git a/services/search/content.py b/services/search/content.py index 2c1f5f64c..49d050a4f 100644 --- a/services/search/content.py +++ b/services/search/content.py @@ -15,6 +15,8 @@ from urllib.parse import urljoin, urlparse import httpx from bs4 import BeautifulSoup +from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT + from .analytics import RateLimitError, error_logger from .cache import ( CONTENT_CACHE_DIR, @@ -89,18 +91,128 @@ def _public_http_url(url: str) -> bool: return False -def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5) -> httpx.Response: +class BodyTooLargeError(Exception): + """The server declared a body larger than the hard fetch ceiling.""" + + def __init__(self, url: str, declared_bytes: int): + self.url = url + self.declared_bytes = declared_bytes + super().__init__( + f"response body is {declared_bytes:,} bytes, over the " + f"{WEB_FETCH_HARD_MAX_BYTES:,}-byte hard cap" + ) + + +class _CappedFetch: + """Result of a size-capped streaming GET. + + Carries just what fetch_webpage_content needs from an httpx.Response, + plus the cap bookkeeping: the (possibly truncated) body, whether the + cap cut it short, and the size the server declared via Content-Length + (wire bytes; None when absent). + """ + + __slots__ = ("status_code", "headers", "content", "truncated", + "declared_bytes", "encoding", "url") + + def __init__(self, status_code, headers, content, truncated, + declared_bytes, encoding, url): + self.status_code = status_code + self.headers = headers + self.content = content + self.truncated = truncated + self.declared_bytes = declared_bytes + self.encoding = encoding + self.url = url + + @property + def text(self) -> str: + return self.content.decode(self.encoding or "utf-8", errors="replace") + + def raise_for_status(self): + if self.status_code >= 400: + request = httpx.Request("GET", self.url) + raise httpx.HTTPStatusError( + f"HTTP {self.status_code} for {self.url}", + request=request, + response=httpx.Response(self.status_code, request=request), + ) + + +def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5, + max_bytes: int = None) -> "_CappedFetch": + """Capped streaming GET with SSRF-guarded manual redirects. + + The body is streamed and buffering stops at ``max_bytes`` (default: the + soft cap), so an oversized resource cannot be pulled into memory or the + content cache in full. When Content-Length already declares a body over + the hard ceiling, the fetch is refused before any body bytes are read. + """ + cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES) current = url for _ in range(max_redirects + 1): if not _public_http_url(current): raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current)) - response = httpx.get(current, headers=headers, timeout=timeout, follow_redirects=False) - if response.status_code not in (301, 302, 303, 307, 308): - return response - location = response.headers.get("location") - if not location: - return response - current = urljoin(str(response.url), location) + # Force identity transfer-encoding. With gzip/deflate the wire bytes + # (and Content-Length) can be a small fraction of the decoded body, so + # a tiny compressed response could pass the hard-cap preflight and then + # expand past the ceiling in a single decoded chunk before the streamed + # cap below can slice it. Identity makes Content-Length the true body + # size and keeps each streamed chunk bounded by the network read. + req_headers = dict(headers or {}) + req_headers["Accept-Encoding"] = "identity" + with httpx.stream("GET", current, headers=req_headers, timeout=timeout, + follow_redirects=False) as response: + if response.status_code in (301, 302, 303, 307, 308): + location = response.headers.get("location") + if not location: + return _CappedFetch(response.status_code, response.headers, b"", + False, None, response.encoding, str(response.url)) + current = urljoin(str(response.url), location) + continue + + # A server can ignore the identity request and still return a + # compressed body; httpx.iter_bytes would then decode it, and a tiny + # gzip can balloon into one decoded chunk far past the cap before we + # slice. Refuse a compressed Content-Encoding so the streamed cap + # stays a real memory bound (Content-Length is the compressed wire + # length here, so the preflight and size metadata are unreliable too). + enc = (response.headers.get("content-encoding") or "").strip().lower() + if enc and enc != "identity": + raise httpx.RequestError( + f"Refusing compressed response (Content-Encoding: {enc}) after " + "requesting identity: cannot bound decoded body size", + request=httpx.Request("GET", current), + ) + + declared = None + raw_len = response.headers.get("content-length") + if raw_len and raw_len.isdigit(): + declared = int(raw_len) + # Refuse before buffering anything when the server already tells + # us the body exceeds the absolute ceiling (Content-Length is wire + # bytes; the decompressed body can only be larger). + if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES: + raise BodyTooLargeError(current, declared) + + chunks = [] + read = 0 + truncated = False + # We requested identity above, so iter_bytes yields the raw body in + # network-read-sized chunks (no decompression expansion); the cap + # therefore bounds what we actually buffer. + for chunk in response.iter_bytes(): + read += len(chunk) + if read > cap: + keep = cap - (read - len(chunk)) + if keep > 0: + chunks.append(chunk[:keep]) + truncated = True + break + chunks.append(chunk) + return _CappedFetch(response.status_code, response.headers, + b"".join(chunks), truncated, declared, + response.encoding, str(response.url)) raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current)) # PDF extraction (optional dependency) @@ -222,9 +334,19 @@ def _empty_result(url: str, error: str = "") -> dict: # ---------------------------------------------------------------------- # Main content fetcher # ---------------------------------------------------------------------- -def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> dict: - """Fetch and extract meaningful content from a webpage with caching.""" - cache_key = generate_cache_key(url) +def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0, + max_bytes: int = None) -> dict: + """Fetch and extract meaningful content from a webpage with caching. + + ``max_bytes`` raises the download budget per call (clamped to the hard + cap); the default is the soft cap. When the body is cut short the result + carries ``truncated``/``fetched_bytes``/``total_bytes`` so callers can + tell the model the content is partial (#3812). + """ + effective_cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES) + # The cap is part of the cache identity: a truncated soft-cap fetch must + # not be served to a later full-budget request for the same URL. + cache_key = generate_cache_key(f"{url}#cap={effective_cap}") cache_file = CONTENT_CACHE_DIR / f"{cache_key}.cache" # Check cache @@ -247,18 +369,24 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> # Fetch try: headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "User-Agent": WEB_FETCH_USER_AGENT, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", - "Accept-Encoding": "gzip, deflate", + # identity so the streamed size cap in _get_public_url stays honest + # (a compressed body can decode to far more than Content-Length). + "Accept-Encoding": "identity", "Connection": "keep-alive", } - response = _get_public_url(url, headers=headers, timeout=timeout) + response = _get_public_url(url, headers=headers, timeout=timeout, + max_bytes=effective_cap) if response.status_code == 429: raise RateLimitError(f"Rate limit hit for {url} (attempt {retry_attempt})") response.raise_for_status() + except BodyTooLargeError as e: + error_logger.warning(f"Refused oversized body for {url}: {e}") + return _empty_result(url, f"TooLarge: {e}") except httpx.HTTPStatusError as e: error_logger.warning(f"HTTP {e.response.status_code} fetching {url}: {e}") return _empty_result(url, f"HTTP {e.response.status_code}: {e}") @@ -269,9 +397,27 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> error_logger.error(str(e)) return _empty_result(url, str(e)) + # Size bookkeeping shared by every content branch below. getattr keeps + # plain httpx.Response stand-ins (tests) working without the cap fields. + _size_fields = { + "truncated": getattr(response, "truncated", False), + "fetched_bytes": len(response.content), + "total_bytes": getattr(response, "declared_bytes", None), + } + # PDF handling content_type = response.headers.get("Content-Type", "").lower() if "application/pdf" in content_type or url.lower().endswith(".pdf"): + if _size_fields["truncated"]: + # A PDF cut mid-stream is not parseable; unlike text there is no + # useful partial result, so report the budget problem instead. + _declared = _size_fields["total_bytes"] + return _empty_result( + url, + f"TooLarge: PDF exceeds the {effective_cap:,}-byte fetch budget" + + (f" (size {_declared:,} bytes)" if _declared else "") + + "; retry with a larger budget if it fits under the hard cap", + ) if pdf_extract_text is None: logger.error("pdfminer.six is not installed; cannot extract PDF text.") pdf_text = "" @@ -295,6 +441,42 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> "js_message": "", "success": bool(pdf_text), "error": "" if pdf_text else "Failed to extract PDF text", + **_size_fields, + } + _cache_result(cache_file, cache_key, result, url) + return result + + # Plain-text / Markdown / JSON handling. Sources like + # raw.githubusercontent.com serve Markdown as `text/plain`, JSON APIs and + # raw config files serve `application/json`, and a lot of code and tool + # docs live in `.md` / `.txt`. These have no HTML structure, so the HTML + # branch below would extract nothing and report "no readable text content". + # Return the body verbatim instead. The `is_html` guard keeps real HTML + # (including `application/xhtml+xml`) on the parsing path; the `json` check + # covers `application/json` and `+json` suffixes; the URL-suffix fallback + # catches servers that mislabel text files as `application/octet-stream`. + is_html = "html" in content_type + is_json = "json" in content_type + url_path = url.lower().split("?", 1)[0].split("#", 1)[0] + looks_like_text_file = url_path.endswith( + (".md", ".markdown", ".txt", ".text", ".json", ".jsonl") + ) + if not is_html and (content_type.startswith("text/") or is_json or looks_like_text_file): + text_body = (response.text or "").strip() + result = { + "url": url, + "title": os.path.basename(url_path) or url, + "content": text_body, + "lists": [], + "tables": [], + "code_blocks": [], + "meta_description": "", + "meta_keywords": "", + "js_rendered": False, + "js_message": "", + "success": bool(text_body), + "error": "" if text_body else "Empty response body", + **_size_fields, } _cache_result(cache_file, cache_key, result, url) return result @@ -357,6 +539,7 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> "js_message": js_message, "success": True, "error": "", + **_size_fields, } _cache_result(cache_file, cache_key, result, url) return result diff --git a/services/search/providers.py b/services/search/providers.py index b913e1c6f..d0ca1b0de 100644 --- a/services/search/providers.py +++ b/services/search/providers.py @@ -9,14 +9,12 @@ from urllib.parse import urljoin, urlparse, parse_qs import httpx from bs4 import BeautifulSoup -from src.constants import SEARXNG_INSTANCE +from src.constants import SEARXNG_INSTANCE, REQUEST_TIMEOUT, WEB_FETCH_USER_AGENT from .analytics import RateLimitError, error_logger from .query import build_enhanced_query logger = logging.getLogger(__name__) -REQUEST_TIMEOUT = 20 - # Provider registry — maps setting value to (label, needs_key, needs_url) PROVIDER_INFO = { "searxng": ("SearXNG", False, True), @@ -140,7 +138,7 @@ def searxng_search_api(query: str, count: Optional[int] = None, categories: str count = count if count is not None else _get_result_count() instance = _get_search_instance() api_key = "" - headers = {"User-Agent": "Mozilla/5.0"} + headers = {"User-Agent": WEB_FETCH_USER_AGENT} if api_key: headers["Authorization"] = f"Bearer {api_key}" # News/fresh queries do badly in the 'general' category — it favours @@ -252,7 +250,7 @@ def searxng_search(query, max_results=10): """Search using SearXNG instance - parsing HTML.""" instance = _get_search_instance() api_key = "" - req_headers = {"User-Agent": "Mozilla/5.0"} + req_headers = {"User-Agent": WEB_FETCH_USER_AGENT} if api_key: req_headers["Authorization"] = f"Bearer {api_key}" try: @@ -391,7 +389,7 @@ def duckduckgo_search(query: str, count: Optional[int] = None, time_filter: Opti response = httpx.get( "https://html.duckduckgo.com/html/", params={"q": query, "kp": _safesearch_for("duckduckgo_html")}, - headers={"User-Agent": "Mozilla/5.0"}, + headers={"User-Agent": WEB_FETCH_USER_AGENT}, timeout=REQUEST_TIMEOUT, ) response.raise_for_status() diff --git a/services/youtube/youtube_handler.py b/services/youtube/youtube_handler.py index b36989e8d..d1b1e9b91 100644 --- a/services/youtube/youtube_handler.py +++ b/services/youtube/youtube_handler.py @@ -64,20 +64,40 @@ def is_youtube_url(url: str) -> bool: return "youtube.com" in url or "youtu.be" in url +# youtube.com-shaped hosts. music.youtube.com serves the same /watch and +# /shorts paths, so links shared from YouTube Music must resolve too. +_YT_HOSTS = ("www.youtube.com", "youtube.com", "m.youtube.com", "music.youtube.com") +# Path prefixes whose first following segment is the video id. Covers the +# /embed/ player, Shorts (/shorts/), live streams (/live/), and the legacy +# /v/ embed — all of which `is_youtube_url` already treats as YouTube, so +# they must be extractable or the link is silently dropped (neither web-fetched +# nor transcript-fetched) by the chat pipeline. +_YT_PATH_PREFIXES = ("/embed/", "/shorts/", "/live/", "/v/") + + def extract_youtube_id(url: str) -> Optional[str]: - """Extract YouTube video ID from various URL formats.""" + """Extract a YouTube video ID from the common URL shapes: + watch?v=, youtu.be/, /embed/, /shorts/, /live/, /v/, + across youtube.com / m.youtube.com / music.youtube.com / youtu.be.""" if not isinstance(url, str): return None parsed = urllib.parse.urlparse(url) - if parsed.hostname in ("www.youtube.com", "youtube.com", "m.youtube.com"): + host = (parsed.hostname or "").lower() + if host in _YT_HOSTS: if parsed.path == "/watch": params = urllib.parse.parse_qs(parsed.query) - if "v" in params: + if params.get("v"): return params["v"][0] - elif parsed.path.startswith("/embed/"): - return parsed.path.split("/")[-1] - elif parsed.hostname == "youtu.be": - return parsed.path[1:] + else: + for prefix in _YT_PATH_PREFIXES: + if parsed.path.startswith(prefix): + vid = parsed.path[len(prefix):].split("/")[0] + if vid: + return vid + elif host == "youtu.be": + vid = parsed.path.lstrip("/").split("/")[0] + if vid: + return vid return None @@ -170,6 +190,8 @@ def format_transcript_for_context( if segments: ctx += "Timestamped Transcript:\n" for seg in segments: + if not isinstance(seg, dict): + continue ctx += f"[{seg['timestamp']}] {seg['text']}\n" # Check length — fall back to plain text if too long if len(ctx) > 12000: @@ -202,15 +224,24 @@ async def fetch_youtube_comments( f"https://www.youtube.com/watch?v={video_id}", ] - proc = await asyncio.wait_for( - asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ), - timeout=timeout, + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, ) - stdout, stderr = await proc.communicate() + # Bound the wait on the process actually finishing, not on spawning it. + # create_subprocess_exec returns as soon as the child starts, so wrapping + # it in wait_for never enforces the timeout — proc.communicate() is the + # blocking step. Kill and reap the child if it overruns so it does not + # linger after we return. + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(), timeout=timeout + ) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + raise if proc.returncode != 0: return {"success": False, "error": f"yt-dlp failed: {stderr.decode()[:200]}", "comments": []} diff --git a/setup.py b/setup.py index 81fcc87ab..a9c565282 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,9 @@ sys.path.insert(0, BASE_DIR) from src.constants import ( DATA_DIR, AUTH_FILE, UPLOAD_DIR, PERSONAL_DIR, PERSONAL_UPLOADS_DIR, TTS_CACHE_DIR, GENERATED_IMAGES_DIR, DEEP_RESEARCH_DIR, CHROMA_DIR, - RAG_DIR, MEMORY_VECTORS_DIR, + RAG_DIR, MEMORY_VECTORS_DIR, PASSWORD_MIN_LENGTH, ) +from core.auth import RESERVED_USERNAMES DIRS = [ DATA_DIR, @@ -59,15 +60,23 @@ def _prompt_admin_credentials(): print(" (Press Enter to accept defaults)") print() - username = input(" Username [admin]: ").strip().lower() - if not username: - username = "admin" + while True: + username = input(" Username [admin]: ").strip().lower() + if not username: + username = "admin" + if username in RESERVED_USERNAMES: + print(f" '{username}' is a reserved username. Choose another.") + continue + break while True: password = getpass.getpass(" Password: ") if not password: print(" Password cannot be empty.") continue + if len(password) < PASSWORD_MIN_LENGTH: + print(f" Password must be at least {PASSWORD_MIN_LENGTH} characters.") + continue confirm = getpass.getpass(" Confirm password: ") if password != confirm: print(" Passwords don't match. Try again.") @@ -93,8 +102,13 @@ def create_default_admin(): password = os.getenv("ODYSSEUS_ADMIN_PASSWORD", "").strip() if username and password: - # Both provided via env — use them directly - pass + # Both provided via env — validate before using + if username in RESERVED_USERNAMES: + print(f" [error] ODYSSEUS_ADMIN_USER '{username}' is a reserved username") + return "failed" + if len(password) < PASSWORD_MIN_LENGTH: + print(f" [error] ODYSSEUS_ADMIN_PASSWORD must be at least {PASSWORD_MIN_LENGTH} characters") + return "failed" elif sys.stdin.isatty() and not os.getenv("ODYSSEUS_SKIP_ADMIN_PROMPT"): # Interactive terminal — ask the user username, password = _prompt_admin_credentials() diff --git a/specs/architecture-runtime-inventory.md b/specs/architecture-runtime-inventory.md new file mode 100644 index 000000000..1030b2bd0 --- /dev/null +++ b/specs/architecture-runtime-inventory.md @@ -0,0 +1,412 @@ +# Architecture Runtime Inventory + +> **Purpose**: Phase 0 planning baseline for codebase readability improvements (#4071). +> **Parent issue**: [#4082](https://github.com/pewdiepie-archdaemon/odysseus/issues/4082) +> **Last updated**: dev@b58af42 | 2026-06-16 +> **Status**: Draft — to be reviewed before follow-up slices open. +> **Snapshot basis**: Importer / file / import-line counts are refreshed to `dev@b58af42` (2026-06-16) and are recomputable via the commands in §3.4. **Line counts** in §2.1 / §2.2 are a snapshot from an earlier baseline and drift as `dev` moves — recompute any of them with `wc -l `. This inventory tracks structure and risk, not live metrics. + +This document maps the current runtime module structure, identifies high-risk boundaries, and recommends safe first refactor slices. It does **not** move files, change imports, or alter runtime behavior. + +--- + +## 1. Current Structure Overview + +### 1.1 Top-Level Layout + +``` +odysseus/ +├── app.py # FastAPI app entrypoint (1,145 lines) +├── conf/ # Configuration (config.py, settings.py, settings_scrub.py) +├── src/ # 95 flat .py files + 2 subdirectories +│ ├── agent_tools/ # Tool helpers: document, filesystem, subprocess, web +│ └── search/ # Search subsystem +├── routes/ # 54 flat .py files — HTTP route handlers +├── core/ # 10 files — database models, auth, middleware, session +├── mcp_servers/ # 5 files — MCP server implementations +├── scripts/ # CLI tools and one-shot scripts +├── static/ # Frontend HTML/CSS/JS +├── tests/ # 583 test files (~54,800 lines) +└── services/ # (exists as needed) +``` + +### 1.2 Directory Flatness Metric + +| Directory | Flat `.py` Files | Subdirectories | Concern | +|-----------|-----------------|----------------|---------| +| `src/` | **95** | 2 (`agent_tools/`, `search/`) | No domain grouping; 95 files in one directory | +| `routes/` | **54** | 0 | All route handlers in one flat directory | +| `core/` | 10 | 0 | Manageable, but `database.py` is oversized | + +--- + +## 2. Largest Runtime Modules + +### 2.1 Python Backend + +| Rank | File | Lines | Classes | Functions | Risk | +|------|------|-------|---------|-----------|------| +| 1 | `src/tool_implementations.py` | **4,032** | 0 | ~48 | **HIGH** | +| 2 | `routes/email_routes.py` | **3,245** | — | — | **MEDIUM** | +| 3 | `routes/cookbook_routes.py` | **2,969** | — | — | **MEDIUM** | +| 4 | `src/agent_loop.py` | **2,961** | 0 | ~24 | **HIGH** | +| 5 | `src/task_scheduler.py` | **2,330** | — | 5 | MEDIUM | +| 6 | `routes/model_routes.py` | **2,266** | — | — | MEDIUM | +| 7 | `core/database.py` | **2,265** | 28 | ~59 helpers | **HIGH** | +| 8 | `src/builtin_actions.py` | **2,262** | 2 | ~24 | MEDIUM | +| 9 | `src/llm_core.py` | **2,164** | — | — | MEDIUM | +| 10 | `mcp_servers/email_server.py` | 2,197 | — | — | LOW (separate process) | +| 11 | `src/visual_report.py` | 1,918 | — | — | LOW | +| 12 | `routes/gallery_routes.py` | 1,896 | — | — | LOW | +| 13 | `src/ai_interaction.py` | 1,846 | — | — | MEDIUM | +| 14 | `routes/document_routes.py` | 1,717 | — | — | LOW | +| 15 | `routes/skills_routes.py` | 1,648 | — | — | LOW | + +**Heuristic**: Files > 2,000 lines with 20+ public symbols and many importers are the highest-risk splits. Files 1,000–2,000 lines are medium-risk if tightly coupled. + +### 2.2 Frontend + +| File | Lines | Concern | +|------|-------|---------| +| `static/style.css` | **36,653** | Entire app CSS in one file (tracked separately in #2617) | +| `static/js/document.js` | **9,776** | Single JS file for document functionality | +| `static/js/slashCommands.js` | 6,498 | | +| `static/js/settings.js` | 5,266 | | +| `static/js/emailLibrary.js` | 5,217 | | +| `static/js/notes.js` | 5,124 | | +| `static/js/chat.js` | 4,985 | | +| `static/app.js` | 4,090 | | + +**Note**: Frontend modularization is tracked separately in #2617 (CSS) and is not the focus of this Phase 0 inventory. Frontend is listed here for completeness but follow-up slices should target Python backend boundaries first. + +--- + +## 3. Import Dependency Graph + +### 3.1 Who Depends on `core/database.py` + +**102 files** import from `core.database` — this is the most depended-upon module: + +- All route handlers (`routes/*.py`) +- Most `src/*.py` files +- `core/session_manager.py`, `core/auth.py` +- Multiple test files + +**Implication**: Any split of `core/database.py` is the highest-risk refactor. It should be tackled **last**, never first. + +### 3.2 Who Depends on `src/tool_implementations.py` + +**17 files** import from `src.tool_implementations`: +- `src/agent_loop.py`, `src/builtin_actions.py`, `src/tool_index.py` +- `src/task_scheduler.py`, `src/tool_policy.py` +- Various tests + +### 3.3 Who Depends on `src/agent_loop.py` + +**22 files** import from `src.agent_loop`: + +- `src/tool_policy.py`, `src/teacher_escalation.py`, `src/bg_monitor.py` +- `src/task_scheduler.py` +- Multiple test files + +### 3.4 Cross-Layer Import Violations + +**`src/` importing from `routes/`** (backwards dependency — domain logic depending on HTTP layer): + +``` +src/tool_implementations.py ──→ routes/calendar_routes.py +src/tool_implementations.py ──→ routes/cookbook_helpers.py +src/tool_implementations.py ──→ routes/email_helpers.py +src/tool_implementations.py ──→ routes/email_pollers.py +src/tool_implementations.py ──→ routes/email_routes.py +src/tool_implementations.py ──→ routes/model_routes.py +src/tool_implementations.py ──→ routes/note_routes.py +src/tool_implementations.py ──→ routes/prefs_routes.py +``` + +> These are **runtime imports** (inside function bodies, not at module top), which mitigates circular import risk but indicates fuzzy layer boundaries. Function-level inline imports from the HTTP layer into business logic are a code smell. + +**Import counts (top-level)**: +| Direction | Count | Notes | +|-----------|-------|-------| +| `routes/` → `src/` | **374** | Expected: HTTP handlers call domain logic | +| `routes/` → `core/` | **126** | Expected: handlers access DB models | +| `src/` → `routes/` | **31** | **Unexpected**: domain logic reaching into HTTP layer (direct grep of import lines referencing `routes/`) | +| `src/` → `core/` | **106** | Acceptable but could be reduced with a data-access layer | + +> **How the metrics in this document are computed** — recompute against current `dev` before treating any count as authoritative (the tree drifts; these numbers are a snapshot, not a live value): +> - `src/` flat `.py` files: `find src -maxdepth 1 -name '*.py' | wc -l` +> - `tests/` test files: `find tests -name 'test_*.py' | wc -l` +> - `core.database` importers: `grep -rlE '(from|import) +core\.database' --include='*.py' . | grep -v core/database.py | wc -l` +> - `src.agent_loop` importers: `grep -rlE '(from|import) +src\.agent_loop' --include='*.py' . | grep -v src/agent_loop.py | wc -l` +> - Cross-layer import lines: `grep -rhE '(from|import) +' --include='*.py' / | wc -l` (e.g. `(from|import) +routes` over `src/`) + +--- + +## 4. Route Ownership Map + +Routes can be grouped into logical feature domains. Current flat structure obscures these boundaries: + +| Domain | Route Files | Total Lines | Review Complexity | +|--------|-------------|-------------|-------------------| +| **Email** | `email_routes.py`, `email_helpers.py`, `email_pollers.py` | 5,936 | HIGH — most complex domain | +| **Chat / Agent** | `chat_routes.py`, `chat_helpers.py`, `shell_routes.py`, `codex_routes.py`, `skills_routes.py` | 6,365 | HIGH — core interaction surface | +| **Cookbook** | `cookbook_routes.py`, `cookbook_helpers.py`, `cookbook_output.py` | 4,110 | MEDIUM | +| **Model / LLM** | `model_routes.py`, `assistant_routes.py`, `copilot_routes.py` | 2,764 | MEDIUM | +| **Calendar / Contacts** | `calendar_routes.py`, `contacts_routes.py` | 2,336 | MEDIUM | +| **Documents** | `document_routes.py`, `document_helpers.py` | 1,954 | LOW | +| **Auth** | `auth_routes.py`, `api_token_routes.py`, `device_flow.py` | 1,171 | LOW | +| **Tasks** | `task_routes.py` (standalone) | 1,157 | LOW | +| **Session** | `session_routes.py` (standalone) | 1,287 | LOW | +| **Gallery** | `gallery_routes.py`, `gallery_helpers.py` | 1,896 | LOW | +| **Memory** | `memory_routes.py` | — | LOW | +| **Research** | `research_routes.py` | — | LOW | +| **MCP** | `mcp_routes.py` | — | LOW | +| **Notes** | `note_routes.py` | — | LOW | +| **Other** | `prefs_routes.py`, `upload_routes.py`, `vault_routes.py`, `webhook_routes.py`, `workspace_routes.py`, `search_routes.py`, `history_routes.py`, `hwfit_routes.py`, `preset_routes.py`, `signature_routes.py`, `backup_routes.py`, `cleanup_routes.py`, `diagnostics_routes.py`, `embedding_routes.py`, `emoji_routes.py`, `font_routes.py`, `stt_routes.py`, `tts_routes.py`, `compare_routes.py`, `personal_routes.py`, `editor_draft_routes.py`, `admin_wipe_routes.py`, `chatgpt_subscription_routes.py` | 2,000+ | LOW individual, HIGH cumulative | + +--- + +## 5. Tool Registry & Implementation Boundaries + +### 5.1 Current Tool Architecture + +| Component | File | Lines | Role | +|-----------|------|-------|------| +| Tool schemas | `src/tool_schemas.py` | 1,392 | JSON Schema tool definitions (Duck-TypedDict) | +| Tool index | `src/tool_index.py` | 542 | RAG-based tool retrieval from ChromaDB | +| Tool implementations | `src/tool_implementations.py` | 4,032 | 33 `do_*` functions — all tool execution logic | +| Tool security | `src/tool_security.py` | — | Owner-scoped tool blocking | +| Tool policy | `src/tool_policy.py` | — | Guide-only directive, plan-mode disabled tools | +| Tool utils | `src/tool_utils.py` | — | Shared tool helpers | + +### 5.2 Tool Implementation Categories + +The 33 `do_*` functions in `tool_implementations.py` fall into natural domain groups — the basis for slice 1's split in §6.2: + +| Category | `do_*` functions | Count | +|----------|------------------|-------| +| **System / config** | `do_manage_skills`, `do_manage_tasks`, `do_manage_endpoints`, `do_manage_mcp`, `do_manage_webhooks`, `do_manage_tokens`, `do_manage_settings`, `do_api_call`, `do_app_api` | 9 | +| **Cookbook / model serving** | `do_download_model`, `do_serve_model`, `do_list_served_models`, `do_stop_served_model`, `do_tail_serve_output`, `do_list_downloads`, `do_cancel_download`, `do_search_hf_models`, `do_adopt_served_model`, `do_list_cookbook_servers`, `do_list_serve_presets`, `do_serve_preset`, `do_list_cached_models` | 13 | +| **Notes** | `do_manage_notes` | 1 | +| **Calendar** | `do_manage_calendar` | 1 | +| **Search** | `do_search_chats` | 1 | +| **Research** | `do_manage_research`, `do_trigger_research` | 2 | +| **Contacts** | `do_resolve_contact`, `do_manage_contact` | 2 | +| **Vault** | `do_vault_search`, `do_vault_get`, `do_vault_unlock` | 3 | +| **Image** | `do_edit_image` | 1 | +| | **Total** | **33** | + +> Low-level tools (filesystem, subprocess, web fetch, document parsing) live in `src/agent_tools/`, **not** in `tool_implementations.py` — out of scope for this split. + +--- + +## 6. Risk Assessment & Candidate Slice Ranking + +> **Candidate proposals, not a committed plan.** The rankings, package shapes (e.g. `src/pkg/`, `src/domain/`, `src/infra/`, `src/api/`), split ordering, and route-grouping strategy below are **options for maintainer discussion**. Per #4082/#4071, slice ownership and order are settled by maintainers before any follow-up PR. §1–§3 above are the factual current-state inventory. + +### 6.1 Risk Scale + +| Level | Criteria | +|-------|----------| +| **LOW** | File has ≤3 importers AND ≤500 lines, OR is a pure refactor with clear boundaries | +| **MEDIUM** | File has 4–15 importers OR 500–1,500 lines | +| **HIGH** | File has 16+ importers OR >2,000 lines, OR has cross-layer import violations | + +### 6.2 Ranked Split Candidates + +| Priority | Target | Risk | Rationale | +|----------|--------|------|-----------| +| **1** | `src/tool_implementations.py` → `src/tools/*.py` | **MEDIUM** | 4,032 lines → ~10 files by tool category. Already has natural boundaries. 17 importers, tracked in #3629. Use `__init__.py` shim to keep existing imports working. | +| **2** | `routes/` → domain subdirectories (one domain per PR) | **MEDIUM** | 54 flat files. Done **one domain at a time** (e.g. a standalone PR for the email domain, then chat, …), not a broad reorganization — route modules carry helper imports, registration assumptions, and test import paths. | +| **3** | `src/agent_loop.py` → `src/agent/loop.py` + submodules | **MEDIUM-HIGH** | 2,961 lines, 24 functions. Can extract prompt building, classification, verification, and runaway detection. Tracked in #3266. | +| **4** | `src/` → `src/pkg/`, `src/domain/`, `src/infra/`, `src/api/` | **MEDIUM** | Structural reorganization. Split flat `src/` into layered packages. Must come after routes and tools are stable. | +| **5** | `routes/email_*.py` consolidation | **LOW** | Already grouped by filename prefix. Low-risk cleanup within the email domain. | +| **6** | `core/database.py` → `src/infra/database/models/*.py` | **HIGH** | 28 classes, 102 importers. Highest-risk split. Must be **last** in any sequence. Requires careful import shim strategy. | +| **7** | Frontend CSS modularization | **MEDIUM** | 36,653 lines. Tracked in #2617. Separate timeline from backend work. | +| **8** | Frontend JS modularization | **MEDIUM** | 9,776 lines in `document.js`. Introduce ES modules at minimum. | + +### 6.3 Candidate First 3 Behavior-Preserving Slices + +**Slice 1: Split `tool_implementations.py`** (Lowest-risk high-impact) + +- Create `src/tools/` package with one file per tool category +- Add `src/tools/__init__.py` re-exporting all symbols with current names +- Update 17 importers to use new paths (can be deferred via shim) +- Validation: `python -m pytest tests/ -x -q` + manual smoke test of tool execution +- Reference: #3629 + +**Slice 2: Group `routes/` by domain** (one domain per PR, not a broad sweep) + +Route modules carry helper imports, router registration assumptions, and test import paths, so this must be done **one domain at a time** rather than as a single reorganization PR. Example sequence (each its own PR): + +- PR 2a: move the **email** domain (`email_routes.py`, `email_helpers.py`, `email_pollers.py`) → `routes/email/` + shim +- PR 2b: move the **chat/agent** domain → `routes/chat/` + shim +- PR 2c: move the **cookbook** domain → `routes/cookbook/` + shim +- …and so on per domain from §4 + +Each PR: add `__init__.py` re-exporting old names, update `app.py` router imports, validation `python app.py` starts clean. **No behavior change** — pure file reorganization. + +**Slice 3: Extract `agent_loop.py` submodules** (Improve reviewability) + +- Move prompt assembly → `src/agent/prompt.py` +- Move request classification → `src/agent/classifier.py` +- Move sub-agent verification → `src/agent/verifier.py` +- Move runaway detection → `src/agent/runaway.py` +- Move context management → `src/agent/context.py` +- Keep `src/agent/loop.py` as the main orchestration module +- Validation: `python -m pytest tests/test_agent_loop.py tests/test_loop_breaker_runaway.py -v` + +--- + +## 7. Safety Guardrails for Follow-Up Work + +Per maintainer guidance in #4082 and #4071: + +- [ ] **One domain/slice per PR** — never mix multiple reorganizations +- [ ] **No behavior changes** mixed with file moves — pure reorganization only +- [ ] **Keep compatibility shims** — `__init__.py` re-exports for all existing import paths +- [ ] **Add or identify focused tests** before risky splits +- [ ] **Do not start with `core/database.py`** or broad route movement unless this inventory shows a safe boundary +- [ ] **Prefer small, reviewable slices** over large restructures +- [ ] **No packaging/runtime/tooling migration** mixed into file moves +- [ ] **No frontend framework migration** inside this stabilization lane +- [ ] **Validate with `python -m compileall`** — every PR must pass CI checks +- [ ] **Validate with `pytest`** — run the full test suite before opening each PR + +--- + +## 8. Validation Commands + +Each follow-up PR should be verifiable with these commands before submission: + +```bash +# Syntax check — must pass with zero errors +python -m compileall src/ routes/ core/ conf/ + +# Full test suite — must match baseline pass rate +python -m pytest tests/ -x -q + +# Import shim verification — existing import paths must still work +python -c "from src.tool_implementations import do_search_chats; print('OK')" + +# App startup smoke test (if backend touched) +timeout 5 python app.py 2>&1 | head -5 || true +``` + +--- + +## 9. Open Questions + +1. Is `#2538` (specs ground truth) the canonical behavior map baseline, and should this inventory be kept in sync with those specs once merged? +2. Should route grouping follow the domain map proposed here, or is there a different taxonomy preferred by maintainers? +3. For the `tool_implementations.py` split (#3629), is the tool categorization in §5.2 acceptable, or should it follow a different grouping? +4. Should compatibility shims (`__init__.py`) be temporary (removed in a follow-up wave) or permanent? +5. Should an ADR (Architecture Decision Record) document be started to track decisions made during this process? + +--- + +## 10. Future Direction (NOT current state) + +The following are **future refactor targets** (candidate directions **pending maintainer agreement**, not committed), recorded here so this inventory does not imply they exist today. None of them are present in the current `dev` tree: + +- `main.py` — proposed rename of the `app.py` entrypoint. Today the app boots via `app.py`. +- `src/agent/` — proposed package to hold `agent_loop.py` submodules (prompt/classifier/verifier/runaway/context). Today `agent_loop.py` is a single flat file in `src/`. +- `src/infra/`, `src/domain/`, `src/pkg/`, `src/api/` — proposed layered reorganization of the flat `src/` directory (slice 4 in §6). + +These become real only when the corresponding slices land. + +--- + +## Appendix A: File Listing + +### `src/` (95 files — 61 shown; run `ls src/*.py` for the full list) + +``` +agent_loop.py tool_implementations.py tool_schemas.py +tool_index.py tool_security.py tool_policy.py +tool_utils.py builtin_actions.py task_scheduler.py +llm_core.py model_context.py model_discovery.py +session_search.py context_budget.py context_compactor.py +ai_interaction.py action_intents.py agent_runs.py +app_helpers.py app_initializer.py config.py +database.py memory.py memory_provider.py +secret_storage.py prompt_security.py url_security.py +url_safety.py rate_limiter.py cleanup_service.py +readiness.py service_health.py exceptions.py +request_models.py assistant_log.py bg_monitor.py +builtin_mcp.py chat_helpers.py chroma_client.py +document_processor.py embedding_lanes.py deep_research.py +research_handler.py research_utils.py personal_docs.py +rag_manager.py rag_singleton.py topic_analyzer.py +visual_report.py youtube_handler.py pdf_forms.py +pdf_form_doc.py pdf_runtime.py caldav_writeback.py +email_thread_parser.py text_helpers.py user_time.py +teacher_escalation.py cookbook_serve_lifecycle.py +chatgpt_subscription.py mcp_manager.py +``` + +### `routes/` (54 files) + +``` +__init__.py _validators.py +auth_routes.py api_token_routes.py device_flow.py +chat_routes.py chat_helpers.py shell_routes.py +codex_routes.py skills_routes.py +email_routes.py email_helpers.py email_pollers.py +cookbook_routes.py cookbook_helpers.py cookbook_output.py +model_routes.py assistant_routes.py copilot_routes.py +calendar_routes.py contacts_routes.py +document_routes.py document_helpers.py +gallery_routes.py gallery_helpers.py +task_routes.py session_routes.py +note_routes.py memory_routes.py research_routes.py +mcp_routes.py search_routes.py history_routes.py +webhook_routes.py workspace_routes.py upload_routes.py +vault_routes.py prefs_routes.py preset_routes.py +signature_routes.py personal_routes.py hwfit_routes.py +backup_routes.py cleanup_routes.py diagnostics_routes.py +embedding_routes.py emoji_routes.py font_routes.py +stt_routes.py tts_routes.py compare_routes.py +editor_draft_routes.py chatgpt_subscription_routes.py admin_wipe_routes.py +``` + +### `core/` (10 files) + +``` +__init__.py constants.py database.py models.py +auth.py middleware.py session_manager.py exceptions.py +atomic_io.py platform_compat.py +``` + +--- + +## Appendix B: Key Import Relationships + +``` +core/database.py ←── 102 importers (routes/*, src/*, core/*, tests/*) + ↑ + ├── routes/auth_routes.py + ├── routes/email_routes.py + ├── src/builtin_actions.py + ├── src/task_scheduler.py + ├── src/tool_implementations.py (inline) + └── ...97 more + +src/tool_implementations.py ←── 17 importers + ↑ + ├── src/agent_loop.py + ├── src/builtin_actions.py + ├── src/tool_index.py + ├── src/task_scheduler.py + ├── src/tool_policy.py + └── ...12 more (mostly tests) + +src/agent_loop.py ←── 22 importers + ↑ + ├── src/tool_policy.py + ├── src/teacher_escalation.py + ├── src/bg_monitor.py + ├── src/task_scheduler.py + └── 18 more (incl. tests) +``` diff --git a/src/action_intents.py b/src/action_intents.py index ea0cbc86d..3b9c3cc73 100644 --- a/src/action_intents.py +++ b/src/action_intents.py @@ -91,6 +91,9 @@ _ROUTING_PATTERNS: tuple[tuple[str, str, Pattern[str]], ...] = tuple( ("ui", "tool or feature toggle request", r"\b(?:disable|enable|turn\s+(?:on|off))\s+(?:the\s+)?(?:shell|search|web|browser|documents?|memory|skills|images?|calendar|email|mail|research|incognito)\b"), # Deep research jobs, not quick conceptual mentions of research. + ("web", "explicit web search request", rf"{_PLEASE}(?:do|run|use|perform|make)\s+(?:a\s+)?(?:web\s+search|search\s+the\s+web)\b.+"), + ("web", "web lookup imperative request", rf"{_PLEASE}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google)\b.+"), + ("web", "assistant web lookup request", rf"{_ACTION_QUESTION}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google)\b.+"), ("research", "deep research imperative request", rf"{_PLEASE}(?:research|deep\s+dive|look\s+into|investigate)\s+.+"), ("research", "assistant deep research request", rf"{_ACTION_QUESTION}(?:research|do\s+research|deep\s+dive|look\s+into|investigate)\s+.+"), diff --git a/src/agent_loop.py b/src/agent_loop.py index 052d92c49..a7b429be6 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -21,7 +21,7 @@ from src.settings import get_setting from src.prompt_security import untrusted_context_message from src.tool_security import blocked_tools_for_owner, plan_mode_disabled_tools from src.tool_policy import GUIDE_ONLY_DIRECTIVE, ToolPolicy -from src.tool_utils import get_mcp_manager +from src.tool_utils import _truncate, get_mcp_manager from src.agent_tools import ( parse_tool_blocks, strip_tool_blocks, @@ -262,6 +262,11 @@ _DOMAIN_RULES = { - Use `manage_settings` for preferences and tool enable/disable. - Use named tools over `app_api` when a named wrapper exists. - `app_api` is only for safe UI/API actions without a named tool; do not use it for shell, package installs, engine rebuilds, or sensitive auth/admin paths.""", + "contacts": """\ +## Contacts rules +- Use `resolve_contact` to look up a contact's email or phone number by name. Searches the CardDAV address book and sent email history. +- Use `manage_contact` to list, add, update, or delete contacts in the address book. +- Do NOT use `manage_memory` for contact lookups — contact details live in the address book, not memory.""", } _DOMAIN_TOOL_MAP = { @@ -272,8 +277,9 @@ _DOMAIN_TOOL_MAP = { "notes_calendar_tasks": {"manage_notes", "manage_calendar", "manage_tasks"}, "ui": {"ui_control"}, "sessions": {"create_session", "list_sessions", "manage_session", "send_to_session", "search_chats"}, - "files": {"bash", "python", "read_file", "write_file", "edit_file", "grep", "glob", "ls"}, + "files": {"bash", "python", "read_file", "write_file", "edit_file", "grep", "glob", "ls", "get_workspace"}, "settings": {"manage_settings", "manage_endpoints", "manage_mcp", "manage_webhooks", "manage_tokens", "app_api"}, + "contacts": {"resolve_contact", "manage_contact"}, } def _domain_rules_for_tools(tool_names: set) -> list[str]: @@ -309,6 +315,7 @@ NEVER pipe multi-line Python through `python -c "..."` — shell quoting eats re ``` Execute Python code. Use for computation, data processing, scripting. NOT for writing code for the user (use create_document for that). Same sandbox limits as bash — no TTY, no GUI, no `input()`; for anything the user should interact with, generate a single HTML file with inline JS instead. +Prefer a dedicated tool whenever one fits the job (reading, searching, or writing files); use python only for computation/processing no dedicated tool covers - not for reading or writing files. Do NOT use Python/requests for web lookup/search/latest/current requests when `web_search` or `web_fetch` is available.""", "web_search": """\ @@ -347,6 +354,11 @@ Write content to a file. First line is the path, rest is the content.""", ``` Edit an EXISTING file by exact string replacement. PREFER this over bash (sed/echo/redirects) for changing files — it shows a before/after diff. `old_string` must match the file exactly and be unique unless `replace_all` is true. Use write_file to create a new file.""", + "get_workspace": """\ +```get_workspace +``` +Return the absolute path of the active workspace folder. File tools are CONFINED to it (paths can be RELATIVE to it); the shell starts there (cwd) but is NOT sandboxed. Call this first when the user says "the project"/"the code"/"this folder" without a path, instead of asking them. No arguments.""", + "create_document": """\ ```create_document @@ -396,7 +408,7 @@ Generate an image. Line 1 = description, line 2 = model name, line 3 = WxH (e.g. "ask_teacher": "- ```ask_teacher``` — Escalate a hard question to a more capable model. Line 1 = model name or 'auto', rest = the question. Use when stuck or need expert knowledge.", "list_models": "- ```list_models``` — Show all available AI models across all endpoints. Use when user asks what models are available.", "manage_session": "- ```manage_session``` — Rename, archive, delete, fork, switch, or `list` chats (the UI calls them 'chats'; 'session' is internal). Line 1 = action (list/switch/rename/archive/unarchive/delete/important/unimportant/truncate/fork), Line 2 = exact chat id from `list_sessions` (or `current` where supported). For delete/archive/truncate, always list first and reuse the exact id; never invent placeholder ids. `switch`/`open` returns a clickable anchor link the user can tap to open the chat — use for \"open my X chat\".", - "manage_memory": "- ```manage_memory``` — Manage the user's persistent memory (facts, identity, preferences, context that persists across chats). Line 1 = action (list/add/edit/delete/search), rest = content. Use when user says 'remember this', states identity facts like 'my name is <name>' / 'call me <name>' / 'I live in <place>', or asks about stored memories.", + "manage_memory": "- ```manage_memory``` — Manage the user's persistent memory (facts about the USER themselves, their preferences, context that persists across chats). Line 1 = action (list/add/edit/delete/search), rest = content. Use when user says 'remember this' about themselves, states identity facts like 'my name is <name>' / 'call me <name>' / 'I live in <place>', or asks about stored memories. DO NOT use for info about another person (their address, phone, email, birthday) — that goes in `manage_contact`. If the user pastes an address/phone with a name and says 'save this for <person>', use `manage_contact add` with the address arg, NOT manage_memory.", "manage_skills": "- ```manage_skills``` — Skill registry (SKILL.md format). Args (JSON): {\"action\": \"list|view|view_ref|search|add|edit|patch|publish|delete\", ...}. `list` returns the index of available skills (published + teacher-escalation drafts); `view name=foo` fetches the full SKILL.md; `view_ref name=foo path=...` loads a reference file under the skill directory. For `add`, provide an explicit kebab-case `name` and only report the exact returned name, because storage may normalize or dedupe it. Use this BEFORE doing domain work — there may already be a procedure (published or draft) that prescribes the correct steps. Drafts written by the teacher loop are authoritative guidance even though they're not yet published.", "manage_tasks": "- ```manage_tasks``` — Create and manage scheduled background tasks (recurring AI jobs). Args (JSON): {\"action\": \"list|create|edit|delete|pause|resume|run\", ...}", "manage_endpoints": "- ```manage_endpoints``` — Add, remove, or configure AI model API endpoints. Args (JSON): {\"action\": \"list|add|delete|enable|disable\", ...}. Use when user wants to add a new AI provider.", @@ -416,7 +428,9 @@ Notes, checklists, AND user reminders. Use this for "create/add/write a note", t ```send_email {"to": "recipient@example.com", "subject": "Re: Your question", "body": "Hi, ...", "account": "gmail"} ``` -Send a new email via SMTP. Use `resolve_contact` first if you only have a name. If multiple email accounts exist, call `list_email_accounts` first and pass the chosen `account`.""", +Send a new email via SMTP. Use `resolve_contact` first if you only have a name. If multiple email accounts exist, call `list_email_accounts` first and pass the chosen `account`. + +CRITICAL — signatures: DO NOT invent a sign-off name. End the body with just `Thanks,` or similar — never type a person's name unless the user explicitly told you what to sign as. When `agent_email_confirm` is on (default), the tool returns `{pending: true, pending_id: ...}` and stages the email for the user to approve in the chat UI instead of SMTPing immediately.""", "list_emails": """\ ```list_emails {"folder": "INBOX", "max_results": 20, "unread_only": false, "account": "gmail"} @@ -427,7 +441,9 @@ List recent emails from a folder, newest first, including read messages by defau ```reply_to_email {"uid": "1234", "body": "Sounds good — talk Friday.", "account": "gmail"} ``` -SEND a reply email immediately by UID. Do not use this for "open a reply" or "start a reply" — those should use `ui_control` with `open_email_reply <uid> <folder> reply` to open the email draft document. For follow-up requests like "reply ..." after reading/listing email where the user clearly wants to send now, use the exact UID and account from the latest `read_email`/`list_emails` result. Never invent UID `1`. Threads automatically (In-Reply-To/References handled).""", +SEND a reply email immediately by UID. Do not use this for "open a reply" or "start a reply" — those should use `ui_control` with `open_email_reply <uid> <folder> reply` to open the email draft document. For follow-up requests like "reply ..." after reading/listing email where the user clearly wants to send now, use the exact UID and account from the latest `read_email`/`list_emails` result. Never invent UID `1`. Threads automatically (In-Reply-To/References handled). + +CRITICAL — signatures: DO NOT invent a sign-off name. End the body with just `Thanks,` or similar — never type a person's name unless the user explicitly told you what to sign as. When `agent_email_confirm` is on (default), the tool returns `{pending: true, pending_id: ...}` and stages the email for the user to approve in the chat UI instead of SMTPing immediately.""", "bulk_email": """\ ```bulk_email {"action": "delete", "uids": ["10997", "10998"], "folder": "INBOX", "account": "Gmail"} @@ -437,7 +453,7 @@ Bulk delete/archive/mark emails. Use this for "delete all those" after listing e "archive_email": "- ```archive_email``` — Archive one email by UID. Args (JSON): {\"uid\":\"...\", \"folder\":\"INBOX\", \"account\":\"Gmail\"}. For multiple messages use bulk_email.", "mark_email_read": "- ```mark_email_read``` — Mark one email read/unread. Args (JSON): {\"uid\":\"...\", \"read\":true, \"folder\":\"INBOX\", \"account\":\"Gmail\"}. For multiple messages use bulk_email.", "resolve_contact": "- ```resolve_contact``` — Look up a contact's email by name. Searches CardDAV address book + sent email history. Args (JSON): {\"name\": \"...\"}. Use BEFORE send_email when the user gives only a name.", - "manage_contact": "- ```manage_contact``` — Create/update/delete/list CardDAV contacts. Args (JSON): {\"action\": \"list|add|update|delete\", \"name\": \"...\", \"email\": \"...\", \"uid\": \"...\"}. Use only for explicit address-book/contact requests with contact details. Do NOT use for user identity facts like 'my name is <name>'; save those with manage_memory. For update/delete, call action=list first to get the uid.", + "manage_contact": "- ```manage_contact``` — Create/update/delete/list CardDAV contacts. Args (JSON): {\"action\": \"list|add|update|delete\", \"name\": \"...\", \"email\": \"...\", \"phones\": [...], \"address\": \"...\", \"uid\": \"...\"}. Use for info about another person: email, phone, postal address. For 'save this for <person>' / address paste / phone next to a name, use this — NOT manage_memory. Do NOT use for user identity facts ('my name is X'); those are manage_memory. For update/delete, call action=list first for the uid.", "manage_calendar": """\ ```manage_calendar {"action": "create_event", "summary": "<event title>", "dtstart": "<natural language or ISO datetime>"} @@ -508,7 +524,7 @@ def get_builtin_overrides() -> dict: ov = get_setting("builtin_tool_overrides", {}) return ov if isinstance(ov, dict) else {} except Exception as e: - logger.warning('Failed to load builtin tool overrides: %s', e) + logger.warning("Failed to load builtin tool overrides, using defaults", exc_info=e) return {} @@ -594,7 +610,7 @@ _API_HOSTS = frozenset([ "api.deepseek.com", "deepseek.com", "api.together.xyz", "api.fireworks.ai", "api.perplexity.ai", "api.x.ai", - "ollama.com", "api.venice.ai", + "ollama.com", "api.venice.ai", "api.kimi.com", "api.githubcopilot.com", # Local OpenAI-compatible endpoints (llama.cpp, vLLM, LM Studio, etc.). # Without these, `_is_api_model` falls back to keyword sniffing on the @@ -781,6 +797,12 @@ def _classify_agent_request(messages: List[Dict], last_user: str) -> Dict[str, o domains.add("documents") if has(r"\b(search|web|google|look up|latest|news|current|weather|forecast|stock price|price of|website|url|https?://|www\.)\b"): domains.add("web") + if has( + r"\b(wyszukaj|wyszukać|wyszukac)\b.*\b(internet|internecie|online|web)\b", + r"\b(sprawd[zź]|znajd[zź])\b.*\b(internet|internecie|online|web)\b", + r"\b(aktualn\w*|bieżąc\w*|biezac\w*|dzisiaj|teraz)\b.*\b(pogod\w*|temperatur\w*)\b", + ): + domains.add("web") if has(r"\b(research|deep dive|investigate|look into)\b"): domains.add("web") if has(r"\b(open|show|toggle|turn on|turn off|disable|enable|switch model|change model|settings|theme|panel)\b"): @@ -791,6 +813,8 @@ def _classify_agent_request(messages: List[Dict], last_user: str) -> Dict[str, o domains.add("files") if has(r"\b(endpoint|api token|mcp|webhook|preference|configure|config|setting)\b"): domains.add("settings") + if has(r"\b(contact|contacts|phone|phone number|address book|vcard)\b"): + domains.add("contacts") low_signal = not continuation and not domains return { @@ -819,8 +843,11 @@ def _recent_context_for_retrieval(messages: List[Dict], max_user: int = 3, max_c if isinstance(content, list): content = " ".join(b.get("text", "") for b in content if isinstance(b, dict)) content = (content or "").strip() - # Skip injected tool-result envelopes — role=user but not human intent. - if not content or content.startswith("[Tool execution results]"): + # Skip injected envelopes — role=user but not human intent. Tool results + # are now wrapped via untrusted_context_message (metadata.trusted=False); + # keep the legacy "[Tool execution results]" prefix for older histories. + meta = msg.get("metadata") or {} + if not content or meta.get("trusted") is False or content.startswith("[Tool execution results]"): continue collected.append(content) if len(collected) >= max_user: @@ -839,6 +866,7 @@ def _build_system_prompt( compact: bool = False, owner: Optional[str] = None, suppress_local_context: bool = False, + active_email: Optional[Dict[str, str]] = None, ) -> List[Dict]: """Build agent system prompt, inject MCP/document context, merge consecutive system msgs.""" global _cached_base_prompt, _cached_base_prompt_key @@ -904,8 +932,8 @@ def _build_system_prompt( try: from src.user_time import current_datetime_context_message _datetime_message = current_datetime_context_message() - except Exception: - pass + except Exception as e: + logger.warning("Failed to build datetime context message", exc_info=e) # Document context is kept as a SEPARATE message (not merged into the tool # prompt) so the context trimmer doesn't destroy it when truncating the @@ -948,8 +976,8 @@ def _build_system_prompt( try: from src.pdf_form_doc import find_source_upload_id _is_form_backed = bool(find_source_upload_id(active_document.current_content or "")) - except Exception: - pass + except Exception as e: + logger.warning("Failed to detect if document is form-backed, assuming plain", exc_info=e) if _is_form_backed: doc_ctx = ( @@ -1031,6 +1059,66 @@ def _build_system_prompt( else: set_active_document(None) + # Active email reader — frontend told us the user has an email open. + # Inject a context block so "reply", "summarize this", "what does it say" + # resolve to the real UID instead of the agent inventing a fresh .md + # draft with fake headers. This is the email equivalent of _doc_message. + _email_message = None + if active_email and active_email.get("uid"): + _em_uid = active_email.get("uid", "") + _em_folder = active_email.get("folder", "INBOX") + _em_account = active_email.get("account", "") + _em_subject = active_email.get("subject", "") or "(no subject)" + _em_from = active_email.get("from", "") or "(unknown sender)" + _em_preview = (active_email.get("body_preview", "") or "").strip() + _preview_block = f"\nBody preview:\n```\n{_em_preview[:1800]}\n```" if _em_preview else "" + _acct_arg = f" {_em_account}" if _em_account else "" + email_ctx = ( + f"ACTIVE EMAIL OPEN (the user has this email open in a reader window right now)\n" + f"UID: {_em_uid}\n" + f"Folder: {_em_folder}\n" + f"Account: {_em_account or '(default)'}\n" + f"From: {_em_from}\n" + f"Subject: {_em_subject}{_preview_block}\n\n" + f"CRITICAL DEFAULT — every request about email this turn refers to " + f"THIS email unless the user names a DIFFERENT specific recipient " + f"(a name, an email address, or another thread). Examples that " + f"ALL mean reply-to-the-open-email:\n" + f" • 'reply' / 'reply to this' / 'respond'\n" + f" • 'write email saying X' / 'send email saying X' / 'draft something'\n" + f" • 'tell them X' / 'say hi' / 'thanks' / 'ack' / 'lmk'\n" + f" • 'summarize it' / 'what does it say' / 'tldr'\n" + f" • 'forward this' / 'forward to <addr>'\n" + f"DO NOT ASK THE USER 'who do you want to send this to?' — the " + f"answer is ALWAYS the sender of the open email (above) unless they " + f"named someone else. Asking that is the wrong move every time.\n\n" + f"RULES for the open email:\n" + f"1. DRAFT a reply (default for any 'write/send/reply/tell them' " + f"request without a different recipient): call `ui_control` with " + f"`action=\"open_email_reply\"` and `extra=\"{_em_uid} {_em_folder} " + f"reply\"`. This opens the proper reply doc with To/Subject/" + f"In-Reply-To pre-filled by the backend. The user will see and edit " + f"it before sending. DO NOT `create_document` a markdown file with " + f"hand-written `To:` / `Subject:` / `In-Reply-To:` headers — that " + f"is wrong every time.\n" + f"2. SEND a reply immediately (skip the draft): call " + f"`reply_to_email` with the UID above. Only do this when the user " + f"explicitly says 'send' / 'send the reply' / 'reply and send'.\n" + f"3. READ the full body (the preview above may be truncated): " + f"call `read_email` with the UID/folder/account above.\n" + f"4. SUMMARIZE / answer questions about it: read it first, then " + f"answer in chat. Don't create a document for a summary unless " + f"the user explicitly asks for one.\n" + f"5. Never ask the user to paste the email or 'share it with you' " + f"— you already have its identity above and can read the full body.\n" + f"6. The ONLY time you ask 'who to send to?' is when the user " + f"explicitly says 'send a NEW email to someone else' or names a " + f"recipient you can't identify. A bare 'send email saying X' = the " + f"open email's sender.\n" + ) + _email_message = untrusted_context_message("active email reader", email_ctx) + _email_message["_protected"] = True + # Inject writing style for any email writing path. This is deliberately # broader than read/list: models may compose via send_email, reply_to_email, # or ui_control open_email_reply after the first tool round. @@ -1238,6 +1326,9 @@ def _build_system_prompt( if _doc_message: merged.insert(last_user_idx, _doc_message) last_user_idx += 1 # the document message is now at last_user_idx + if _email_message: + merged.insert(last_user_idx, _email_message) + last_user_idx += 1 if _skills_message: merged.insert(last_user_idx, _skills_message) last_user_idx += 1 @@ -1272,12 +1363,18 @@ def _build_base_prompt( from src.tool_index import ALWAYS_AVAILABLE disabled = set(disabled_tools or []) - if not get_setting("image_gen_enabled", True): + if not get_setting("image_gen_enabled", False): disabled.add("generate_image") if relevant_tools is not None: - # RAG mode: include always-available + retrieved + admin (if needed) - tool_names = set(ALWAYS_AVAILABLE) | set(relevant_tools) + # RAG mode: trust the relevant_tools set as already-composed. + # get_tools_for_query starts from ALWAYS_AVAILABLE and may + # *discard* tools that conflict with the query's intent (e.g. + # drop manage_memory for clear contact-save patterns). Unioning + # ALWAYS_AVAILABLE back in here used to silently undo those + # drops. Only force-include the irreducible loop primitives + # (ask_user, update_plan) as belt-and-suspenders. + tool_names = set(relevant_tools) | {"ask_user", "update_plan"} if needs_admin: tool_names |= _ADMIN_TOOLS agent_prompt = _assemble_prompt(tool_names, disabled, compact=compact) @@ -1468,8 +1565,14 @@ def _append_tool_results( if round_reasoning: msg["reasoning_content"] = round_reasoning messages.append(msg) + # Tool output (shell/python stdout, file reads, fetched pages, email + # bodies, MCP results) is sourced from outside the server. Wrap it as + # untrusted data so prompt-injection inside a tool result is treated as + # data, not instructions — same hardening as skills (#788) and the + # web/RAG context. THREAT_MODEL.md lists tool output as a surface that + # must go through untrusted_context_message. messages.append( - {"role": "user", "content": f"[Tool execution results]\n\n{tool_output_text}"} + untrusted_context_message("tool execution results", tool_output_text) ) @@ -1718,6 +1821,7 @@ async def stream_agent_loop( max_tool_calls: int = 0, context_length: int = 0, active_document=None, + active_email: Optional[Dict[str, str]] = None, session_id: Optional[str] = None, disabled_tools: Optional[Set[str]] = None, owner: Optional[str] = None, @@ -1726,6 +1830,7 @@ async def stream_agent_loop( plan_mode: bool = False, approved_plan: Optional[str] = None, tool_policy: Optional[ToolPolicy] = None, + workspace: Optional[str] = None, _is_teacher_run: bool = False, ) -> AsyncGenerator[str, None]: """Streaming agent loop generator. @@ -1794,8 +1899,21 @@ async def stream_agent_loop( logger.info(f"[tool-rag] Using caller-provided relevant_tools ({len(_relevant_tools)} tools)") if not guide_only and not _relevant_tools and bool(_intent.get("low_signal")): from src.tool_index import ALWAYS_AVAILABLE - _relevant_tools = set(ALWAYS_AVAILABLE) - logger.info("[tool-rag] Low-signal agent message; skipping retrieval and using always-available tools only") + if workspace: + # An active workspace IS the file-work signal: a vague "look at the + # project" means explore this folder. Surface only the READ-ONLY file + # tools (intersection with the plan-mode read-only allowlist) so the + # agent can investigate; write/shell tools stay out until the request + # actually calls for them (RAG retrieval adds those on a real ask). + _relevant_tools = set(ALWAYS_AVAILABLE) + from src.tool_security import PLAN_MODE_READONLY_TOOLS + _relevant_tools |= (_DOMAIN_TOOL_MAP["files"] & PLAN_MODE_READONLY_TOOLS) + logger.info("[tool-rag] Low-signal but workspace active; including read-only file tools") + else: + # Don't short-circuit: fall through to RAG retrieval below. + # Non-English queries are flagged low_signal by the English-only + # intent classifier, but fastembed retrieval works across languages. + logger.info("[tool-rag] Low-signal query; will run RAG retrieval") if not guide_only and not _relevant_tools: try: from src.tool_index import get_tool_index, ALWAYS_AVAILABLE @@ -1870,6 +1988,44 @@ async def stream_agent_loop( if _relevant_tools is not None and active_document is not None: _relevant_tools.update({"edit_document", "update_document", "suggest_document"}) + # The skill index injected by _build_system_prompt tells the model to + # call `manage_skills action=view`, and Jaccard-matched skills are pasted + # into the prompt as procedures to follow — but neither path goes through + # tool selection, so the model can be handed a procedure naming tools + # (grep, read_file, ...) that aren't in its schema list. Keep the schemas + # in lockstep: manage_skills is callable whenever any skill is indexed, + # and a matched skill's declared requires_toolsets ride along with it. + if not guide_only and _relevant_tools is not None: + try: + from services.memory.skills import SkillsManager + from src.constants import DATA_DIR + _skills_on = True + try: + from routes.prefs_routes import _load_for_user as _load_prefs + _skills_on = (_load_prefs(owner) or {}).get("skills_enabled", True) + except Exception: + pass + _sm = SkillsManager(DATA_DIR) + _owner_skills = _sm.load(owner=owner) if _skills_on else [] + if _owner_skills: + _relevant_tools.add("manage_skills") + if _retrieval_query: + # Validate against every known executable tool, not just + # TOOL_SECTIONS — code-nav tools (grep/glob/ls) ship as + # schemas without a prompt-prose section. + from src.tool_policy import known_tool_names + _known = known_tool_names() + for _sk in _sm.get_relevant_skills( + _retrieval_query, skills=_owner_skills, + threshold=0.25, max_items=3, + ): + _relevant_tools.update( + t for t in (_sk.get("requires_toolsets") or []) + if t in _known + ) + except Exception as _e: + logger.debug(f"[tool-rag] skill-aware tool include skipped: {_e}") + if _relevant_tools is not None: logger.info("[agent-intent] selected_tools=%s", sorted(_relevant_tools)[:50]) @@ -1920,6 +2076,10 @@ async def stream_agent_loop( # and can override this list for users who know their setup. _model_no_tools = any(kw in _model_lc for kw in ( "deepseek-r1", + # Open-weight GPT-OSS models are commonly served through llama.cpp / + # llama-cpp-python. Their names contain "gpt-o", but they do not use + # OpenAI's native tool-call channel unless the endpoint opts in. + "gpt-oss", )) # Native Ollama endpoints (/api/chat) handle tool schemas differently from # the OpenAI-compat path. Models like gemma4, qwen3.5, ministral respond to @@ -1949,6 +2109,7 @@ async def stream_agent_loop( compact=_is_api_model, owner=owner, suppress_local_context=guide_only, + active_email=active_email, ) if plan_mode and not guide_only: # Steer the model to investigate-then-propose. Hard tool gating handles @@ -1981,30 +2142,34 @@ async def stream_agent_loop( _t3 = time.time() try: from src.context_compactor import trim_for_context - from src.context_budget import compute_input_token_budget, DEFAULT_HARD_MAX - from src.settings import is_setting_overridden + from src.context_budget import compute_input_token_budget, DEFAULT_HARD_MAX, DEFAULT_BUDGET, budget_is_explicit as _budget_is_explicit + from src.model_context import budget_context_for_model - soft_budget = int(get_setting("agent_input_token_budget", 6000) or 0) + soft_budget = int(get_setting("agent_input_token_budget", DEFAULT_BUDGET) or 0) if soft_budget > 0: before_trim_tokens = estimate_tokens(messages) reserve_tokens = min(max(max_tokens or 1024, 512), 2048) - # Honour the configurable ceiling for the auto-derived budget path. - # No-op when the user has an explicit `agent_input_token_budget` - # (that branch ignores hard_max). Falls back to DEFAULT_HARD_MAX - # on missing/malformed values so misconfig can't zero the budget. + # Ceiling for the auto-derived budget (no effect on an explicit budget; + # see #1230). Falls back to DEFAULT_HARD_MAX on missing/malformed values + # so misconfig can't zero the budget. try: hard_max = int(get_setting("agent_input_token_hard_max", DEFAULT_HARD_MAX) or DEFAULT_HARD_MAX) except (TypeError, ValueError): hard_max = DEFAULT_HARD_MAX if hard_max <= 0: hard_max = DEFAULT_HARD_MAX - # Scale the default budget to the model's context window so long-context - # models aren't silently capped at 6000; an explicit user setting is - # still honoured (clamped to the window). (#1170) + # Default value = auto sentinel (scale to the window); any other value = + # explicit cap. Value-based, not presence-based, because the save path + # materializes defaults so a persisted default must still read as auto (#4121). + budget_is_explicit = _budget_is_explicit(soft_budget) + # Scale only off a window we actually discovered, bound to the value it + # proves (else 0) — not the passed-in context_length, which can be stale + # or unset for some callers (#4122 review). + ctx_for_budget = budget_context_for_model(endpoint_url, model, fallback=context_length) effective_budget = compute_input_token_budget( soft_budget, - context_length, - is_setting_overridden("agent_input_token_budget"), + ctx_for_budget, + budget_is_explicit, hard_max=hard_max, ) trimmed_messages = trim_for_context( @@ -2079,11 +2244,12 @@ async def stream_agent_loop( # tool, so we don't nudge on harmless transitional text like "let me # know what you think". _INTENT_RE = re.compile( - r"(?:^|\n)\s*(?:let me|i'?ll|i will|going to|let's)\s+" + r"(?:^|\n)\s*(?:let me|i'?ll|i will|i need to|we need to|need to|" + r"i should|we should|i must|we must|going to|let's)\s+" r"(?:tail|check|investigate|look at|see|tail|read|fetch|inspect|" r"verify|diagnose|examine|debug|capture|grab|pull|view|run|call|" r"trigger|launch|start|kick off|stop|kill|restart|adopt|serve|" - r"register|adopt|list|search|find|query|hit|ping|test)" + r"register|adopt|list|search|find|query|hit|ping|test|use|perform|do)" r"\b[^.\n]{0,140}", re.IGNORECASE, ) @@ -2124,9 +2290,17 @@ async def stream_agent_loop( elif _is_api_model: # Filter schemas by RAG-selected tools (if available) if _relevant_tools: + # _build_base_prompt unions _ADMIN_TOOLS into the prompt + # sections when admin intent fires — the schema list must + # offer the same names, or the model reads prose describing + # tools it cannot call and substitutes the nearest schema + # it does have (e.g. manage_memory for manage_skills). + _schema_names = set(_relevant_tools) + if _needs_admin: + _schema_names |= _ADMIN_TOOLS base_schemas = [ s for s in FUNCTION_TOOL_SCHEMAS - if s.get("function", {}).get("name") in _relevant_tools + if s.get("function", {}).get("name") in _schema_names ] _mcp_filtered = [ s for s in mcp_schemas @@ -2644,6 +2818,7 @@ async def stream_agent_loop( tool_policy=tool_policy, owner=owner, progress_cb=_push_progress, + workspace=workspace, ) finally: # Sentinel so the drainer knows to stop. @@ -2661,6 +2836,46 @@ async def stream_agent_loop( ) desc, result = await _tool_task + # A skill the model just loaded can prescribe tools that weren't + # RAG-selected this turn (declared via requires_toolsets in its + # frontmatter). Union them into the selection so the NEXT round's + # schema list includes them — otherwise the model reads "use + # grep" from the skill it fetched but has no grep schema to call. + if ( + block.tool_type == "manage_skills" + and _relevant_tools is not None + and not result.get("error") + ): + _ms_args = {} + _ms_raw = (block.content or "").strip() + if _ms_raw.startswith("{"): + try: + _ms_args = json.loads(_ms_raw) + except json.JSONDecodeError: + _ms_args = {} + _ms_name = str(_ms_args.get("name", "") or "").strip() + if _ms_name and _ms_args.get("action") in ("view", "view_ref"): + try: + from services.memory.skills import SkillsManager as _SkM + from src.constants import DATA_DIR as _DD + from src.tool_policy import known_tool_names as _ktn + _known = _ktn() + for _sk in _SkM(_DD).load(owner=owner): + if _sk.get("name") == _ms_name: + _new = { + t for t in (_sk.get("requires_toolsets") or []) + if t in _known and t not in _relevant_tools + } + if _new: + _relevant_tools.update(_new) + logger.info( + "[tool-rag] skill '%s' unlocked tools for next round: %s", + _ms_name, sorted(_new), + ) + break + except Exception as _e: + logger.debug(f"skill requires_toolsets unlock skipped: {_e}") + # Extract structured web sources from web_search tool output. # web_search returns {"output": ..., "exit_code": 0}; check "output" # first so the <!-- SOURCES:…--> marker is found and stripped even @@ -2751,18 +2966,20 @@ async def stream_agent_loop( # On a bash/python timeout the result carries error + (often # empty) stdout/stderr; fall back to the error so the "timed # out" reason reaches the UI instead of a blank result. - output_text = (result["stdout"] or result["stderr"] or result.get("error", ""))[:2000] + raw = result["stdout"] or result["stderr"] or result.get("error", "") + output_text = _truncate(raw) elif "output" in result: # bash / python canonical result: {"output": ..., "exit_code": ...} - output_text = (result["output"] or "")[:2000] + raw = result["output"] or "" + output_text = _truncate(raw) elif "response" in result: # AI interaction tools (chat_with_model, send_to_session) label = result.get("model", result.get("session_name", "AI")) - output_text = f"{label}: {result['response']}"[:4000] + output_text = _truncate(f"{label}: {result['response']}") elif "content" in result: - output_text = result["content"][:2000] + output_text = _truncate(result["content"]) elif "results" in result: - output_text = result["results"][:4000] + output_text = _truncate(result["results"]) elif "session_id" in result and "name" in result: output_text = f"Session created: {result['name']} (id: {result['session_id']})" elif "success" in result: @@ -2772,13 +2989,25 @@ async def stream_agent_loop( else f"Error: {result.get('error', '')}" ) elif "error" in result: - output_text = result["error"][:2000] + output_text = _truncate(result["error"]) # Emit tool_output (include ui_event data if present) tool_output_data = {"type": "tool_output", "tool": block.tool_type, "command": cmd_display, "output": output_text, "exit_code": result.get("exit_code")} if "ui_event" in result: tool_output_data["ui_event"] = result["ui_event"] - for k in ("toggle_name", "state", "mode", "model", "endpoint_url", "theme_name", "colors"): + for k in ( + "toggle_name", "state", "mode", "model", "endpoint_url", + "theme_name", "colors", + # ui_control open_email_reply payload — without these the + # frontend openReplyDraft bails on undefined uid and the + # reply window silently never opens. + "uid", "folder", "account_id", + # Optional pre-filled body for open_email_reply so the + # agent can compose-and-open in one tool call. + "body", + # ui_control open_panel payload + "panel", + ): if k in result: tool_output_data[k] = result[k] # Forward image data from generate_image tool diff --git a/src/agent_tools/__init__.py b/src/agent_tools/__init__.py index 4db923a9a..52fe4a99c 100644 --- a/src/agent_tools/__init__.py +++ b/src/agent_tools/__init__.py @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) from .subprocess_tools import BashTool, PythonTool from .web_tools import WebSearchTool, WebFetchTool -from .filesystem_tools import ReadFileTool, WriteFileTool, EditFileTool, LsTool, GlobTool, GrepTool +from .filesystem_tools import ReadFileTool, WriteFileTool, EditFileTool, LsTool, GlobTool, GrepTool, GetWorkspaceTool from .document_tools import CreateDocumentTool, UpdateDocumentTool, EditDocumentTool, SuggestDocumentTool, ManageDocumentTool TOOL_HANDLERS = { @@ -39,6 +39,7 @@ TOOL_HANDLERS = { "edit_document": EditDocumentTool().execute, "suggest_document": SuggestDocumentTool().execute, "manage_documents": ManageDocumentTool().execute, + "get_workspace": GetWorkspaceTool().execute, } # --------------------------------------------------------------------------- @@ -51,7 +52,7 @@ PYTHON_TIMEOUT = 30 # Tool types that trigger execution TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_file", "edit_file", - "grep", "glob", "ls", + "grep", "glob", "ls", "get_workspace", "create_document", "update_document", "edit_document", "search_chats", "chat_with_model", "create_session", "list_sessions", diff --git a/src/agent_tools/filesystem_tools.py b/src/agent_tools/filesystem_tools.py index 3b5425242..7ba22161c 100644 --- a/src/agent_tools/filesystem_tools.py +++ b/src/agent_tools/filesystem_tools.py @@ -46,13 +46,7 @@ def _unified_diff(old: str, new: str, path: str) -> Optional[Dict[str, Any]]: class EditFileTool: async def execute(self, content: str, ctx: dict) -> dict: - from src.tool_execution import ( - _resolve_tool_path, - _resolve_tool_path_in_workspace, - _resolve_search_root, - _truncate - ) - workspace = ctx.get("workspace") + from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate try: args = json.loads(content) if content.strip().startswith("{") else {} except (json.JSONDecodeError, TypeError): @@ -64,8 +58,7 @@ class EditFileTool: if not raw_path: return {"error": "edit_file: path required", "exit_code": 1} try: - path = (_resolve_tool_path_in_workspace(workspace, raw_path) - if workspace else _resolve_tool_path(raw_path)) + path = _resolve_tool_path(raw_path) except ValueError as e: return {"error": f"edit_file: {e}", "exit_code": 1} if old == "": @@ -113,13 +106,7 @@ class EditFileTool: class ReadFileTool: async def execute(self, content: str, ctx: dict) -> dict: - from src.tool_execution import ( - _resolve_tool_path, - _resolve_tool_path_in_workspace, - _resolve_search_root, - _truncate - ) - workspace = ctx.get("workspace") + from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate raw_path, offset, limit = content.split("\n", 1)[0].strip(), 0, 0 _stripped = content.strip() if _stripped.startswith("{"): @@ -131,8 +118,7 @@ class ReadFileTool: except (json.JSONDecodeError, TypeError, ValueError): pass try: - path = (_resolve_tool_path_in_workspace(workspace, raw_path) - if workspace else _resolve_tool_path(raw_path)) + path = _resolve_tool_path(raw_path) except ValueError as e: return {"error": f"read_file: {e}", "exit_code": 1} try: @@ -170,19 +156,12 @@ class ReadFileTool: class WriteFileTool: async def execute(self, content: str, ctx: dict) -> dict: - from src.tool_execution import ( - _resolve_tool_path, - _resolve_tool_path_in_workspace, - _resolve_search_root, - _truncate - ) - workspace = ctx.get("workspace") + from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate lines = content.split("\n", 1) raw_path = lines[0].strip() body = lines[1] if len(lines) > 1 else "" try: - path = (_resolve_tool_path_in_workspace(workspace, raw_path) - if workspace else _resolve_tool_path(raw_path)) + path = _resolve_tool_path(raw_path) except ValueError as e: return {"error": f"write_file: {e}", "exit_code": 1} try: @@ -212,13 +191,7 @@ class WriteFileTool: class LsTool: async def execute(self, content: str, ctx: dict) -> dict: - from src.tool_execution import ( - _resolve_tool_path, - _resolve_tool_path_in_workspace, - _resolve_search_root, - _truncate - ) - workspace = ctx.get("workspace") + from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate raw_path = "" _s = (content or "").strip() if _s.startswith("{"): @@ -267,13 +240,7 @@ class LsTool: class GlobTool: async def execute(self, content: str, ctx: dict) -> dict: - from src.tool_execution import ( - _resolve_tool_path, - _resolve_tool_path_in_workspace, - _resolve_search_root, - _truncate - ) - workspace = ctx.get("workspace") + from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate args = {} _s = (content or "").strip() if _s.startswith("{"): @@ -325,13 +292,7 @@ class GlobTool: class GrepTool: async def execute(self, content: str, ctx: dict) -> dict: - from src.tool_execution import ( - _resolve_tool_path, - _resolve_tool_path_in_workspace, - _resolve_search_root, - _truncate - ) - workspace = ctx.get("workspace") + from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate args: Dict[str, Any] = {} _s = (content or "").strip() if _s.startswith("{"): @@ -417,3 +378,21 @@ class GrepTool: if len(lines) >= max_hits: out += f"\n... [capped at {max_hits} matches]" return {"output": _truncate(out), "exit_code": 0} + +class GetWorkspaceTool: + """Report the active workspace folder (no args). File tools are confined to + it; the shell starts there (cwd) but is NOT sandboxed.""" + async def execute(self, content: str, ctx: dict) -> dict: + from src.tool_execution import get_active_workspace + ws = get_active_workspace() + if ws: + return { + "output": f"{ws}\n(File tools are confined to this folder; the shell starts " + f"here but is not sandboxed and can reach outside it.)", + "exit_code": 0, + } + return { + "output": "No workspace is set. File tools use the default allowed roots; " + "resolve paths from the user or use absolute paths.", + "exit_code": 0, + } diff --git a/src/agent_tools/subprocess_tools.py b/src/agent_tools/subprocess_tools.py index 6b5972030..8a0e2b5d5 100644 --- a/src/agent_tools/subprocess_tools.py +++ b/src/agent_tools/subprocess_tools.py @@ -102,16 +102,15 @@ async def _run_subprocess_streaming( class BashTool: async def execute(self, content: str, ctx: dict) -> dict: - from src.tool_execution import _AGENT_WORKDIR, _truncate + from src.tool_execution import agent_cwd, _truncate progress_cb = ctx.get("progress_cb") - workspace = ctx.get("workspace") _subproc_env = ctx.get("subproc_env") proc = await asyncio.create_subprocess_shell( content, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=_subproc_env, - cwd=workspace or _AGENT_WORKDIR, + cwd=agent_cwd(), ) stdout, stderr, rc, timed_out = await _run_subprocess_streaming( proc, @@ -129,16 +128,15 @@ class BashTool: class PythonTool: async def execute(self, content: str, ctx: dict) -> dict: - from src.tool_execution import _AGENT_WORKDIR, _truncate + from src.tool_execution import agent_cwd, _truncate progress_cb = ctx.get("progress_cb") - workspace = ctx.get("workspace") _subproc_env = ctx.get("subproc_env") proc = await asyncio.create_subprocess_exec( (sys.executable or "python"), "-I", "-c", content, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=_subproc_env, - cwd=workspace or _AGENT_WORKDIR, + cwd=agent_cwd(), ) stdout, stderr, rc, timed_out = await _run_subprocess_streaming( proc, diff --git a/src/agent_tools/web_tools.py b/src/agent_tools/web_tools.py index 87a4b697f..9c1d2ca97 100644 --- a/src/agent_tools/web_tools.py +++ b/src/agent_tools/web_tools.py @@ -57,13 +57,23 @@ class WebSearchTool: class WebFetchTool: async def execute(self, content: str, ctx: dict) -> dict: from src.search.content import fetch_webpage_content + from src.constants import WEB_FETCH_HARD_MAX_BYTES raw = content.strip() url = "" + max_bytes = None if raw.startswith("{"): try: parsed = json.loads(raw) if isinstance(parsed, dict): url = str(parsed.get("url") or "").strip() + # Download-budget override (#3812): "full": true raises the + # budget to the hard cap; an explicit max_bytes is clamped + # to the hard cap downstream. Default stays the soft cap. + if parsed.get("full") is True: + max_bytes = WEB_FETCH_HARD_MAX_BYTES + mb = parsed.get("max_bytes") + if isinstance(mb, int) and mb > 0: + max_bytes = mb except json.JSONDecodeError: url = "" if not url: @@ -78,7 +88,7 @@ class WebFetchTool: loop = asyncio.get_running_loop() try: result = await asyncio.wait_for( - loop.run_in_executor(None, lambda: fetch_webpage_content(url, timeout=10)), + loop.run_in_executor(None, lambda: fetch_webpage_content(url, timeout=10, max_bytes=max_bytes)), timeout=30, ) except asyncio.TimeoutError: @@ -94,8 +104,28 @@ class WebFetchTool: return {"error": f"web_fetch: {url}: {err}", "exit_code": 1} return {"error": f"web_fetch: {url}: no readable text content (not HTML, or the page needs JS/login)", "exit_code": 1} + # Tell the model when the download budget cut the body short and how + # to get the rest, instead of silently presenting a partial page as + # the whole thing. + size_note = "" + if result.get("truncated"): + fetched = result.get("fetched_bytes") or 0 + total = result.get("total_bytes") + total_txt = f" of {total:,} bytes" if total else "" + size_note = ( + f"[partial content: download stopped at {fetched:,} bytes{total_txt}. " + f'Re-call with {{"url": "{url}", "full": true}} to fetch up to ' + f"{WEB_FETCH_HARD_MAX_BYTES:,} bytes.]\n\n" + ) + + # The notice must lead the output so the MAX_OUTPUT_CHARS trim below can + # never drop it. The title is untrusted, uncapped page content, so a + # giant title ahead of the notice could push it out of range; keep the + # notice first and cap the title as a second guard. + if len(title) > 300: + title = title[:300] + "..." header = (f"# {title}\n" if title else "") + f"Source: {url}\n\n" - output = header + text + output = size_note + header + text if len(output) > MAX_OUTPUT_CHARS: output = output[:MAX_OUTPUT_CHARS] + "\n\n[...truncated]" return {"output": output, "exit_code": 0} diff --git a/src/ai_interaction.py b/src/ai_interaction.py index 20294b61b..33d5d28f7 100644 --- a/src/ai_interaction.py +++ b/src/ai_interaction.py @@ -972,16 +972,15 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner memories = [m for m in memories if m.get("category", "").lower() == category_filter] if not memories: return {"results": "No memories found" + (f" in category '{category_filter}'" if category_filter else "") + "."} + result_lines = [f"Found {len(memories)} memory entries:\n"] - for m in memories[:100]: + for m in memories: cat = m.get("category", "fact") mid = m.get("id", "?")[:8] text = m.get("text", "") if len(text) > 150: text = text[:150] + "..." result_lines.append(f"- [{cat}] `{mid}` — {text}") - if len(memories) > 100: - result_lines.append(f"... and {len(memories) - 100} more") return {"results": "\n".join(result_lines)} elif action == "add": @@ -1293,7 +1292,7 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O set_theme <preset> — Apply a built-in theme preset (dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute) create_theme <name> <bg> <fg> <panel> <border> <accent> [key=val ...] — Create custom theme. Optional key=val: advanced color overrides AND background effects: bgPattern=<none|dots|synapse|rain|constellations|perlin-flow|petals|sparkles|embers>, bgEffectColor=#RRGGBB, bgEffectIntensity=<num>, bgEffectSize=<num>, frosted=true|false open_panel <name> — Open a panel (documents, gallery, email, sessions, notes, memories, skills, settings, cookbook) - open_email_reply <uid> [folder] [reply|reply-all|ai-reply] — Open a reply draft document for an email; does not send + open_email_reply <uid> [folder] [reply|reply-all|ai-reply] [body text] — Open a reply draft document for an email; does not send. ALWAYS append the body text when the user told you what to say (one-shot draft); only omit body when the user just asked to "open a reply" without content. get_toggles — Return current toggle states (server-side knowledge) """ lines = content.strip().split("\n") @@ -1537,21 +1536,54 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O } elif action == "open_email_reply": - reply_parts = lines[0].strip().split() - uid = reply_parts[1].strip() if len(reply_parts) > 1 else "" - folder = reply_parts[2].strip() if len(reply_parts) > 2 else "INBOX" - mode = reply_parts[3].strip().lower() if len(reply_parts) > 3 else "reply" + # Two forms supported: + # open_email_reply <uid> [folder] [reply|reply-all|ai-reply] + # open_email_reply <uid> [folder] [reply|reply-all|ai-reply] + # <body text on subsequent lines or after the mode token> + # The body text (if any) gets pre-filled into the reply draft so the + # agent can compose-and-open in one tool call instead of opening an + # empty draft and leaving the user to wonder what happened. + first_line = lines[0].strip() + parts = first_line.split(maxsplit=4) + uid = parts[1].strip() if len(parts) > 1 else "" + folder = parts[2].strip() if len(parts) > 2 else "INBOX" + mode = parts[3].strip().lower() if len(parts) > 3 else "reply" + # Body: everything on the first line after the mode token, plus any + # subsequent lines. Allows multi-line bodies. + inline_body = parts[4] if len(parts) > 4 else "" + rest_lines = "\n".join(lines[1:]).strip() if len(lines) > 1 else "" + body = (inline_body + ("\n" + rest_lines if rest_lines else "")).strip() if not uid: - return {"error": "open_email_reply needs: open_email_reply <uid> [folder] [reply|reply-all|ai-reply]"} + return {"error": "open_email_reply needs: open_email_reply <uid> [folder] [reply|reply-all|ai-reply] [body text]"} if mode not in ("reply", "reply-all", "ai-reply"): mode = "reply" - return { + # Body is REQUIRED for the agent path. Opening an empty draft is what + # users do by clicking the Reply button — they don't ask the agent + # for that. Every agent invocation of open_email_reply MUST include + # the body. Reject empty so the agent retries with the content the + # user asked for. Exception: ai-reply mode triggers the existing + # AI-Reply path on the frontend which generates its own body. + if not body and mode != "ai-reply": + return { + "error": ( + "open_email_reply called without body. The agent path REQUIRES a body — " + "opening an empty draft is the wrong response when the user asked you to write. " + "Re-call with the reply text included: " + f"`open_email_reply {uid} {folder or 'INBOX'} {mode} <your reply text here>`. " + "Compose the reply now based on the open email's content and the user's request, " + "then call this tool again with the body. Do NOT call create_document instead." + ), + } + result = { "ui_event": "open_email_reply", "uid": uid, "folder": folder or "INBOX", "mode": mode, - "results": f"Opening reply draft for email UID {uid}", + "results": f"Opening reply draft for email UID {uid}" + (" with pre-filled body" if body else ""), } + if body: + result["body"] = body + return result elif action == "get_toggles": return { @@ -1581,7 +1613,9 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne """ import base64 import httpx + import os from pathlib import Path + from src.url_safety import check_outbound_url lines = content.strip().split("\n") prompt = lines[0].strip() if lines else "" @@ -1747,8 +1781,15 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne elif img.get("url"): # Download external URL and save locally (DALL-E returns temp URLs) + result_url = img["url"] + ok, reason = check_outbound_url( + result_url, + block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + return {"error": f"Image API returned unsafe image URL: {reason}"} try: - dl_resp = httpx.get(img["url"], timeout=60) + dl_resp = httpx.get(result_url, timeout=60) if dl_resp.status_code == 200: img_dir = Path(GENERATED_IMAGES_DIR) img_dir.mkdir(parents=True, exist_ok=True) @@ -1758,10 +1799,10 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne image_url = f"/api/generated-image/{filename}" image_id = _save_to_gallery(filename) else: - image_url = img["url"] # fallback to external URL + image_url = result_url # fallback to external URL except Exception as _dl_e: logger.warning(f"Failed to download DALL-E image: {_dl_e}") - image_url = img["url"] # fallback to external URL + image_url = result_url # fallback to external URL else: return {"error": "Image API returned unexpected format (no b64_json or url)"} diff --git a/src/api_key_manager.py b/src/api_key_manager.py index 650a1fbf7..b3cf9a7b6 100644 --- a/src/api_key_manager.py +++ b/src/api_key_manager.py @@ -4,6 +4,8 @@ import logging from typing import Dict from cryptography.fernet import Fernet, InvalidToken +from core.platform_compat import safe_chmod + logger = logging.getLogger(__name__) class APIKeyManager: @@ -15,12 +17,20 @@ class APIKeyManager: def get_or_create_key(self) -> bytes: """Get or create encryption key for API keys""" if os.path.exists(self.key_file): + # Older versions wrote .key with the process umask (often 0o644, + # i.e. group/world-readable). Re-restrict on read so existing + # installs heal without needing the key to be regenerated. + safe_chmod(self.key_file, 0o600) with open(self.key_file, 'rb') as f: return f.read() else: key = Fernet.generate_key() with open(self.key_file, 'wb') as f: f.write(key) + # This key decrypts every stored provider credential, so restrict it + # to the owner (0o600) — it must not be group/world-readable. No-op + # on Windows (files there are ACL-restricted to the user already). + safe_chmod(self.key_file, 0o600) return key def encrypt_api_key(self, api_key: str) -> str: @@ -57,7 +67,12 @@ class APIKeyManager: # Legacy/wrong shape (e.g. a list) — .items() would raise. Ignore it. logger.warning("API keys file has unexpected shape (%s); ignoring", type(encrypted_keys).__name__) return {} - return encrypted_keys + + return { + str(provider): key + for provider, key in encrypted_keys.items() + if isinstance(key, str) + } def save(self, provider: str, api_key: str): """Save encrypted API key to file. @@ -82,4 +97,3 @@ class APIKeyManager: except (InvalidToken, ValueError) as e: logger.warning("Failed to decrypt API key for %s: %s", provider, e) return decrypted - diff --git a/src/bg_monitor.py b/src/bg_monitor.py index d732771a6..8cf8ccc15 100644 --- a/src/bg_monitor.py +++ b/src/bg_monitor.py @@ -55,6 +55,8 @@ async def _drain_agent(sess, messages): if "delta" in d: delta = d.get("delta") if isinstance(delta, str): + if d.get("thinking"): + continue full += delta elif d.get("type") == "agent_step": round_num = d.get("round", round_num) diff --git a/src/builtin_actions.py b/src/builtin_actions.py index 1ea7cd8a4..a598cb652 100644 --- a/src/builtin_actions.py +++ b/src/builtin_actions.py @@ -809,14 +809,14 @@ async def action_learn_sender_signatures(owner: str, **kwargs) -> Tuple[str, boo import email as _email_mod import asyncio as _aio from datetime import datetime as _dt, timedelta as _td - from routes.email_helpers import _imap_connect, SCHEDULED_DB + from routes.email_helpers import _email_cache_owner_clause, _imap_connect, SCHEDULED_DB from src.endpoint_resolver import resolve_endpoint from src.llm_core import llm_call_async # 1. Pull recent UIDs + From headers cheaply (header-only fetch). def _pull_headers(): results = [] - conn = _imap_connect(None) + conn = _imap_connect(None, owner=owner) try: conn.select("INBOX", readonly=True) status, data = conn.search(None, "ALL") @@ -868,9 +868,11 @@ async def action_learn_sender_signatures(owner: str, **kwargs) -> Tuple[str, boo # 3. Eligibility: ≥3 emails AND (no cache OR cache > 30 days old). try: conn = _sql3.connect(SCHEDULED_DB) + owner_clause, owner_params = _email_cache_owner_clause(owner) cached = { r[0]: r[1] for r in conn.execute( - "SELECT from_address, last_built_at FROM sender_signatures" + f"SELECT from_address, last_built_at FROM sender_signatures WHERE {owner_clause}", + owner_params, ).fetchall() } conn.close() @@ -901,7 +903,7 @@ async def action_learn_sender_signatures(owner: str, **kwargs) -> Tuple[str, boo def _fetch_bodies(_msgs): bodies = [] - conn2 = _imap_connect(None) + conn2 = _imap_connect(None, owner=owner) try: conn2.select("INBOX", readonly=True) for mm in _msgs: @@ -978,11 +980,12 @@ async def action_learn_sender_signatures(owner: str, **kwargs) -> Tuple[str, boo try: conn = _sql3.connect(SCHEDULED_DB) + owner_value = (owner or "").strip() conn.execute( "INSERT OR REPLACE INTO sender_signatures " - "(from_address, signature_text, sample_count, last_built_at, model_used, source) " - "VALUES (?, ?, ?, ?, ?, ?)", - (addr, cached_sig, len(bodies), _dt.utcnow().isoformat(), model, "llm"), + "(from_address, owner, signature_text, sample_count, last_built_at, model_used, source) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (addr, owner_value, cached_sig, len(bodies), _dt.utcnow().isoformat(), model, "llm"), ) conn.commit() conn.close() diff --git a/src/builtin_mcp.py b/src/builtin_mcp.py index cf528c10d..93ef0ee61 100644 --- a/src/builtin_mcp.py +++ b/src/builtin_mcp.py @@ -5,14 +5,16 @@ Auto-registration of built-in MCP servers on startup. Each server runs as a stdio subprocess managed by McpManager. """ +import asyncio +import json import logging import os import shutil import subprocess import sys -import asyncio from core.platform_compat import IS_WINDOWS, which_tool +from src.runtime_paths import get_app_root logger = logging.getLogger(__name__) @@ -80,7 +82,7 @@ _BUILTIN_NPX_SERVERS = { "name": "Built-in: Browser", "command": "npx", "args": ["-y", "@playwright/mcp@latest", "--headless", "--caps", "vision"], - }, + } } # Global flag to disable MCP if there are compatibility issues @@ -93,7 +95,7 @@ async def register_builtin_servers(mcp_manager): logger.info("Built-in MCP servers disabled via ODYSSEUS_DISABLE_MCP") return - base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + base_dir = get_app_root() python = sys.executable async def _connect_python_server(server_id: str, script_path: str, name: str): @@ -197,12 +199,13 @@ def _npx_package_from_args(args): async def _is_npx_package_cached(npx_path, package_spec, timeout_s=5): """Probe whether an npx package is already in the local cache. - Runs `npx --no-install <pkg> --version`. --no-install tells npx to - fail instead of downloading, so a cache miss returns fast. We treat - "exited 0 with non-empty stdout" as proof of a working cached copy. - Anything else (non-zero exit, empty stdout, timeout, missing npx, - network error) means we should skip the server. + First checks the local `_npx` cache for an installed package. If the + package is not found there, falls back to `npx --no-install <pkg> + --version` so older npm layouts still work without downloading. """ + if _is_package_in_npx_cache(package_spec): + return True + try: proc = await asyncio.create_subprocess_exec( npx_path, "--no-install", package_spec, "--version", @@ -231,3 +234,68 @@ async def _is_npx_package_cached(npx_path, package_spec, timeout_s=5): pass return False return proc.returncode == 0 and bool(stdout.strip()) + + +def _is_package_in_npx_cache(package_spec): + """Return True when npm's `_npx` cache already contains package_spec.""" + package_name = _npx_package_name(package_spec) + if not package_name: + return False + + for cache_root in _npm_cache_roots(): + npx_root = os.path.join(cache_root, "_npx") + if _npx_cache_contains_package(npx_root, package_name): + return True + return False + + +def _npx_package_name(package_spec): + """Strip a version/range suffix from an npm package spec.""" + if not package_spec: + return "" + if package_spec.startswith("@"): + parts = package_spec.split("@", 2) + if len(parts) >= 3: + return f"@{parts[1]}" + return package_spec + return package_spec.split("@", 1)[0] + + +def _npm_cache_roots(): + roots = [] + configured = os.environ.get("npm_config_cache") + if configured: + roots.append(os.path.expanduser(configured)) + roots.append(os.path.join(os.path.expanduser("~"), ".npm")) + local_app_data = os.environ.get("LOCALAPPDATA") + if local_app_data: + roots.append(os.path.join(local_app_data, "npm-cache")) + return list(dict.fromkeys(roots)) + + +def _npx_cache_contains_package(npx_root, package_name): + if not os.path.isdir(npx_root): + return False + package_path = os.path.join("node_modules", *package_name.split("/"), "package.json") + try: + entries = list(os.scandir(npx_root)) + except OSError: + return False + for entry in entries: + try: + is_dir = entry.is_dir() + except OSError: + continue + cached_name = _cached_package_name(os.path.join(entry.path, package_path)) + if is_dir and cached_name == package_name: + return True + return False + + +def _cached_package_name(package_json_path): + try: + with open(package_json_path, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError): + return "" + return str(data.get("name", "")).strip() diff --git a/src/caldav_sync.py b/src/caldav_sync.py index e4afb89fd..4cf3c1e5a 100644 --- a/src/caldav_sync.py +++ b/src/caldav_sync.py @@ -128,6 +128,17 @@ def validate_caldav_url(raw_url: str) -> str: return urlunparse(parsed._replace(fragment="")).rstrip("/") +def _event_etag(obj) -> str: + """Best-effort ETag extraction from python-caldav resources.""" + try: + etag = getattr(obj, "etag", None) + if callable(etag): + etag = etag() + return str(etag or "") + except Exception: + return "" + + def _stable_cal_id(remote_url: str, owner: str = "", account_id: str = "") -> str: """Deterministic local id for a remote CalDAV calendar, scoped to owner and account so two users — or one user with two accounts — pointing at @@ -316,11 +327,12 @@ def _sync_blocking(owner: str, url: str, username: str, password: str, account_i color="#5b8abf", source="caldav", account_id=account_id or None, + caldav_base_url=remote_url, ) db.add(local_cal) db.commit() else: - # Refresh display name and stamp account_id if missing. + # Refresh display name and stamp CalDAV metadata if missing. changed = False if local_cal.name != display_name: local_cal.name = display_name @@ -328,6 +340,9 @@ def _sync_blocking(owner: str, url: str, username: str, password: str, account_i if account_id and not local_cal.account_id: local_cal.account_id = account_id changed = True + if local_cal.caldav_base_url != remote_url: + local_cal.caldav_base_url = remote_url + changed = True if changed: db.commit() result["calendars"] += 1 @@ -395,6 +410,9 @@ def _sync_blocking(owner: str, url: str, username: str, password: str, account_i existing = _find_existing_event(db, pending, uid_val, local_cal.id) if existing: + if existing.caldav_sync_pending in {"create", "update"}: + result["events"] += 1 + continue existing.calendar_id = local_cal.id existing.summary = summary existing.description = description @@ -405,6 +423,9 @@ def _sync_blocking(owner: str, url: str, username: str, password: str, account_i existing.is_utc = row_is_utc existing.rrule = rrule existing.origin = "caldav" + existing.remote_href = str(getattr(obj, "url", "") or "") or None + existing.remote_etag = _event_etag(obj) or None + existing.caldav_sync_pending = None else: new_ev = CalendarEvent( uid=uid_val, @@ -418,6 +439,8 @@ def _sync_blocking(owner: str, url: str, username: str, password: str, account_i is_utc=row_is_utc, rrule=rrule, origin="caldav", + remote_href=str(getattr(obj, "url", "") or "") or None, + remote_etag=_event_etag(obj) or None, ) db.add(new_ev) pending[uid_val] = new_ev @@ -442,6 +465,8 @@ def _sync_blocking(owner: str, url: str, username: str, password: str, account_i CalendarEvent.origin == "caldav", CalendarEvent.dtstart >= start, CalendarEvent.dtstart <= end, + CalendarEvent.remote_href.isnot(None), + CalendarEvent.caldav_sync_pending.is_(None), ~CalendarEvent.uid.in_(seen_uids) if seen_uids else CalendarEvent.uid.isnot(None), ).all() for ev in stale: @@ -458,6 +483,92 @@ def _sync_blocking(owner: str, url: str, username: str, password: str, account_i return result +def _event_payload(ev) -> dict: + return { + "uid": ev.uid, + "summary": ev.summary, + "description": ev.description, + "location": ev.location, + "dtstart": ev.dtstart, + "dtend": ev.dtend, + "all_day": ev.all_day, + "is_utc": ev.is_utc, + "rrule": ev.rrule or "", + } + + +def _load_event_for_writeback(owner: str, uid: str) -> tuple[str, str, dict] | None: + from core.database import CalendarCal, CalendarEvent, SessionLocal + + db = SessionLocal() + try: + ev = ( + db.query(CalendarEvent) + .join(CalendarCal) + .filter(CalendarEvent.uid == uid, CalendarCal.owner == owner) + .first() + ) + if not ev or not ev.calendar or ev.calendar.source != "caldav": + return None + return ev.calendar.source, ev.calendar.id, _event_payload(ev) + finally: + db.close() + + +def _load_delete_for_writeback(owner: str, uid: str) -> tuple[str, str, dict] | None: + from core.database import CalendarCal, CalendarDeletedEvent, CalendarEvent, SessionLocal + + db = SessionLocal() + try: + tombstone = db.query(CalendarDeletedEvent).filter( + CalendarDeletedEvent.uid == uid, + CalendarDeletedEvent.owner == owner, + ).first() + if tombstone: + return "caldav", tombstone.calendar_id, {"uid": uid} + + ev = ( + db.query(CalendarEvent) + .join(CalendarCal) + .filter(CalendarEvent.uid == uid, CalendarCal.owner == owner) + .first() + ) + if not ev or not ev.calendar or ev.calendar.source != "caldav": + return None + return ev.calendar.source, ev.calendar.id, {"uid": uid} + finally: + db.close() + + +def _pending_writeback_uids(owner: str) -> tuple[list[str], list[str]]: + from core.database import CalendarCal, CalendarDeletedEvent, CalendarEvent, SessionLocal + + db = SessionLocal() + try: + rows = ( + db.query(CalendarEvent.uid) + .join(CalendarCal) + .filter( + CalendarCal.owner == owner, + CalendarCal.source == "caldav", + CalendarEvent.status != "cancelled", + ( + (CalendarEvent.caldav_sync_pending.isnot(None)) + | (CalendarEvent.remote_href.is_(None)) + ), + ) + .all() + ) + delete_rows = ( + db.query(CalendarDeletedEvent.uid) + .filter(CalendarDeletedEvent.owner == owner) + .all() + ) + return [row[0] for row in rows], [row[0] for row in delete_rows] + finally: + db.close() + + def _load_caldav_accounts(owner: str) -> list: """Return the list of CalDAV accounts for *owner*, auto-migrating the legacy single-account ``caldav`` key to the new ``caldav_accounts`` list on first call. @@ -533,3 +644,69 @@ async def sync_caldav(owner: str) -> dict: for err in result.get("errors", []): totals["errors"].append(f"{label}: {err}") return totals + + +async def push_event_create(owner: str, uid: str) -> dict: + loaded = _load_event_for_writeback(owner, uid) + if not loaded: + return {"ok": True, "skipped": True} + source, calendar_id, payload = loaded + from src.caldav_writeback import writeback_event + return await writeback_event(owner, source, calendar_id, payload) + + +async def push_event_update(owner: str, uid: str) -> dict: + return await push_event_create(owner, uid) + + +async def push_event_delete(owner: str, uid: str) -> dict: + loaded = _load_delete_for_writeback(owner, uid) + if not loaded: + return {"ok": True, "skipped": True} + source, calendar_id, payload = loaded + from src.caldav_writeback import writeback_event + return await writeback_event(owner, source, calendar_id, payload, delete=True) + + +async def push_pending_events(owner: str) -> dict: + result = {"events": 0, "errors": []} + uids, delete_uids = _pending_writeback_uids(owner) + for event_uid in uids: + try: + out = await push_event_update(owner, event_uid) + if out.get("ok"): + result["events"] += 1 + elif not out.get("skipped"): + result["errors"].append(f"{event_uid}: {str(out.get('error') or out)[:160]}") + except Exception as e: + logger.warning("CalDAV pending push failed for uid=%s: %s", event_uid, e) + result["errors"].append(f"{event_uid}: {str(e)[:160]}") + for event_uid in delete_uids: + try: + out = await push_event_delete(owner, event_uid) + if out.get("ok"): + result["events"] += 1 + elif not out.get("skipped"): + result["errors"].append(f"{event_uid}: {str(out.get('error') or out)[:160]}") + except Exception as e: + logger.warning("CalDAV pending delete failed for uid=%s: %s", event_uid, e) + result["errors"].append(f"{event_uid}: {str(e)[:160]}") + return result + + +async def sync_caldav_direction(owner: str, direction: str = "pull") -> dict: + direction = (direction or "pull").strip().lower() + if direction == "pull": + return await sync_caldav(owner) + if direction == "push": + return await push_pending_events(owner) + if direction == "both": + pushed = await push_pending_events(owner) + pulled = await sync_caldav(owner) + return {"push": pushed, "pull": pulled} + return { + "calendars": 0, + "events": 0, + "deleted": 0, + "errors": [f"Unsupported CalDAV sync direction: {direction}"], + } diff --git a/src/caldav_writeback.py b/src/caldav_writeback.py index 0866e1467..ffb0021e3 100644 --- a/src/caldav_writeback.py +++ b/src/caldav_writeback.py @@ -89,6 +89,23 @@ def find_remote_calendar(calendars, local_cal_id: str, owner: str = "", account_ return None +def _resource_href(obj) -> str: + try: + return str(getattr(obj, "url", "") or "") + except Exception: + return "" + + +def _resource_etag(obj) -> str: + try: + etag = getattr(obj, "etag", None) + if callable(etag): + etag = etag() + return str(etag or "") + except Exception: + return "" + + def push_event(calendars, local_cal_id: str, ev: dict, *, delete: bool = False, owner: str = "", account_id: str = "") -> dict: """Create/update (or delete) ``ev`` on the matching remote calendar. @@ -105,6 +122,7 @@ def push_event(calendars, local_cal_id: str, ev: dict, *, delete: bool = False, remote = find_remote_calendar(calendars, local_cal_id, owner=owner, account_id=account_id) if remote is None: return {"ok": False, "error": "remote calendar not found"} + remote_url = str(getattr(remote, "url", "") or "") try: existing = remote.event_by_uid(uid) @@ -113,17 +131,34 @@ def push_event(calendars, local_cal_id: str, ev: dict, *, delete: bool = False, if delete: if existing is None: - return {"ok": True, "note": "already absent on remote"} + return {"ok": True, "note": "already absent on remote", "calendar_url": remote_url} existing.delete() - return {"ok": True} + return { + "ok": True, + "calendar_url": remote_url, + "remote_href": _resource_href(existing), + "remote_etag": _resource_etag(existing), + } ical = build_event_ical(ev) if existing is not None: existing.data = ical existing.save() - return {"ok": True, "updated": True} - remote.save_event(ical) - return {"ok": True, "created": True} + return { + "ok": True, + "updated": True, + "calendar_url": remote_url, + "remote_href": _resource_href(existing), + "remote_etag": _resource_etag(existing), + } + created = remote.save_event(ical) + return { + "ok": True, + "created": True, + "calendar_url": remote_url, + "remote_href": _resource_href(created), + "remote_etag": _resource_etag(created), + } def _discover_calendars(client): @@ -154,6 +189,54 @@ def _writeback_blocking(local_cal_id, ev, delete, url, username, password, owner=owner, account_id=account_id) +def _persist_writeback_result(owner: str, calendar_id: str, uid: str, result: dict, *, delete: bool) -> None: + from core.database import CalendarCal, CalendarDeletedEvent, CalendarEvent, SessionLocal + + if not uid or not isinstance(result, dict): + return + + db = SessionLocal() + try: + calendar = db.query(CalendarCal).filter( + CalendarCal.id == calendar_id, + CalendarCal.owner == owner, + ).first() + if calendar and result.get("calendar_url"): + calendar.caldav_base_url = result.get("calendar_url") + + if delete: + tombstone = db.query(CalendarDeletedEvent).filter( + CalendarDeletedEvent.uid == uid, + CalendarDeletedEvent.owner == owner, + ).first() + if result.get("ok"): + if tombstone: + db.delete(tombstone) + elif tombstone: + tombstone.last_error = str(result.get("error") or result)[:500] + db.commit() + return + + event = ( + db.query(CalendarEvent) + .join(CalendarCal) + .filter(CalendarEvent.uid == uid, CalendarCal.owner == owner) + .first() + ) + if event and result.get("ok"): + if result.get("remote_href"): + event.remote_href = result.get("remote_href") + if result.get("remote_etag"): + event.remote_etag = result.get("remote_etag") + event.caldav_sync_pending = None + db.commit() + except Exception: + db.rollback() + logger.exception("CalDAV write-back metadata persistence failed") + finally: + db.close() + + async def writeback_event(owner: str, calendar_source: str, calendar_id: str, ev: dict, *, delete: bool = False) -> dict: """Best-effort push of a local change to the remote CalDAV server. @@ -204,9 +287,12 @@ async def writeback_event(owner: str, calendar_source: str, calendar_id: str, result = await asyncio.to_thread( _writeback_blocking, calendar_id, ev, delete, url, user, pw, owner, acc_id ) + _persist_writeback_result(owner, calendar_id, (ev or {}).get("uid", ""), result, delete=delete) if not result.get("ok"): logger.warning("CalDAV write-back did not apply: %s", result.get("error") or result) return result except Exception as e: logger.exception("CalDAV write-back raised") - return {"ok": False, "error": str(e)[:200]} + result = {"ok": False, "error": str(e)[:200]} + _persist_writeback_result(owner, calendar_id, (ev or {}).get("uid", ""), result, delete=delete) + return result diff --git a/src/config.py b/src/config.py index 8b9bd5148..d5cfa21a7 100644 --- a/src/config.py +++ b/src/config.py @@ -5,6 +5,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict from pydantic import Field, field_validator from src.constants import DATA_DIR as _DATA_DIR_CONST +from src.runtime_paths import get_app_root # Cross-platform OS flag, exposed here so callers can `from src.config import # IS_WINDOWS`. Defined locally (a trivial `os.name == "nt"`) rather than imported @@ -19,7 +20,7 @@ IS_WINDOWS = os.name == "nt" class DataConfig(BaseSettings): """Configuration for data storage and file handling.""" # Base directory - base_dir: Path = Field(default=Path(__file__).parent.parent, description="Base directory for the application") + base_dir: Path = Field(default=Path(get_app_root()), description="Base directory for the application") # Data paths data_dir: Path = Field(default=Path(_DATA_DIR_CONST), description="Main data directory") @@ -138,7 +139,7 @@ class AppConfig(BaseSettings): if isinstance(v, dict) and "base_dir" in v: base_dir = v["base_dir"] else: - base_dir = Path(__file__).parent.parent + base_dir = Path(get_app_root()) # Convert string paths to Path objects relative to base_dir data_dir = Path(_DATA_DIR_CONST) diff --git a/src/constants.py b/src/constants.py index 3f58eba26..b76f5d97b 100644 --- a/src/constants.py +++ b/src/constants.py @@ -2,12 +2,14 @@ """Application-wide constants and configuration values.""" import os +from src.runtime_paths import get_app_root, get_default_data_dir + APP_VERSION = "1.0.0" # Base paths -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/" +BASE_DIR = os.path.join(get_app_root(), "") STATIC_DIR = os.path.join(BASE_DIR, "static") -DATA_DIR = os.getenv("ODYSSEUS_DATA_DIR", os.path.join(BASE_DIR, "data")) +DATA_DIR = os.getenv("ODYSSEUS_DATA_DIR", get_default_data_dir()) # Data file paths # Single source of truth: every persisted file/dir lives under DATA_DIR, which @@ -55,7 +57,13 @@ MEMORY_VECTORS_DIR = os.path.join(DATA_DIR, "memory_vectors") # Paths with an intentional dedicated env override, defaulting under DATA_DIR. MAIL_ATTACHMENTS_DIR = os.getenv("ODYSSEUS_MAIL_ATTACHMENTS_DIR", os.path.join(DATA_DIR, "mail-attachments")) -FASTEMBED_CACHE_DIR = os.getenv("FASTEMBED_CACHE_PATH", os.path.join(DATA_DIR, "fastembed_cache")) +# `or` (not os.getenv's default arg) so a PRESENT-but-EMPTY value falls back to +# the default. docker-compose.yml injects `FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-}`, +# which sets the var to "" when the host hasn't defined it. os.getenv(name, default) +# only returns the default when the var is ABSENT, so the empty string would win → +# os.makedirs("") raises [Errno 2] No such file or directory: '' → FastEmbed fails to +# init and all vector features (RAG, semantic memory, tool index) silently degrade. +FASTEMBED_CACHE_DIR = os.getenv("FASTEMBED_CACHE_PATH") or os.path.join(DATA_DIR, "fastembed_cache") # Agent tool output limits (single source of truth — imported by tool_execution.py, # tool_implementations.py, agent_tools.py, and any other module that needs them) @@ -63,11 +71,26 @@ MAX_OUTPUT_CHARS = 10_000 # cap for bash/python/web_search/web_fetch outpu MAX_READ_CHARS = 20_000 # cap for read_file / document preview MAX_DIFF_LINES = 400 # cap for edit_file unified-diff display +# web_fetch response-size policy (#3812). MAX_OUTPUT_CHARS above only trims +# what the agent SEES; these caps bound what the server downloads, parses, +# and writes to the content cache. The soft cap is the default download +# budget; the agent can raise it per call (full/max_bytes) but never past +# the hard cap, so a model can't decide to pull a multi-GB file. +WEB_FETCH_SOFT_MAX_BYTES = 2_000_000 # default download budget (2 MB) +WEB_FETCH_HARD_MAX_BYTES = 20_000_000 # absolute ceiling, even with override (20 MB) + # API Configuration MAX_CONTEXT_MESSAGES = 90 REQUEST_TIMEOUT = 20 OPENAI_COMPAT_PATH = "/v1/chat/completions" +# Outbound UA for web_fetch / web_search scraping; common desktop UA so pages serve normal HTML. +WEB_FETCH_USER_AGENT = os.environ.get( + "WEB_FETCH_USER_AGENT", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", +) + # Environment variables with defaults DEFAULT_HOST = os.getenv("LLM_HOST", "localhost") LLM_HOSTS = [h.strip() for h in os.getenv("LLM_HOSTS", "").split(",") if h.strip()] @@ -79,6 +102,9 @@ SEARXNG_INSTANCE = os.getenv("SEARXNG_INSTANCE", "http://localhost:8080") CLEANUP_ENABLED = os.getenv("CLEANUP_ENABLED", "True").lower() == "true" CLEANUP_INTERVAL_HOURS = int(os.getenv("CLEANUP_INTERVAL_HOURS", "24")) +# Auth policy +PASSWORD_MIN_LENGTH = 8 + # Default parameters DEFAULT_TEMPERATURE = 1.0 DEFAULT_MAX_TOKENS = 0 diff --git a/src/context_budget.py b/src/context_budget.py index d331ffac4..de4789e28 100644 --- a/src/context_budget.py +++ b/src/context_budget.py @@ -31,16 +31,22 @@ def compute_input_token_budget( Args: configured: the value read from settings (may be the default). - context_length: the model's discovered context window (0/unknown if none). - explicit: True if the user explicitly set ``agent_input_token_budget``. + context_length: the model's discovered context window. Pass 0 when the + window is unknown / only a bare fallback — auto-scaling then stays + conservative instead of trusting an unproven window (review on #4122). + explicit: True if the user set a NON-default budget. The default value is + the "auto" sentinel (scale to the window); any other value is an + explicit cap. (A deliberately-chosen default can't be distinguished + from a materialized default by value, so the default reads as auto.) Rules: - Explicit user budget is honoured exactly, only clamped to the model's - window when that window is known (never send more than the model holds). - - Otherwise (default), scale to ``headroom`` of the context window, capped - at ``hard_max`` — so long-context models use their capacity. - - When the window is unknown, fall back to the configured/default value - (preserving the previous behaviour). + window when that window is known (the user's deliberate choice wins; + ``hard_max`` is an auto-budget ceiling only — see #1230). + - Otherwise (auto), scale to ``headroom`` of the context window, capped at + ``hard_max`` — so long-context models use their capacity. + - When the window is unknown (context_length <= 0), use the conservative + ``default`` budget and do NOT scale off the fallback. """ configured = int(configured or 0) context_length = int(context_length or 0) @@ -53,3 +59,17 @@ def compute_input_token_budget( return max(1, min(scaled, hard_max)) return configured if configured > 0 else default + + +def budget_is_explicit(configured: int, *, default: int = DEFAULT_BUDGET) -> bool: + """Whether a configured agent_input_token_budget is a deliberate explicit cap. + + The default value is the "auto" sentinel (scale to the model's window), so only + a NON-default positive value counts as explicit. This keys off the VALUE, not + settings *presence* — the settings-save path materializes every default into + settings.json, so a persisted default must still read as auto (the regression + #4121 / #1230 are about). Centralised here so the materialized-default contract + is unit-testable and can't silently regress to a presence check. + """ + configured = int(configured or 0) + return configured > 0 and configured != default diff --git a/src/context_compactor.py b/src/context_compactor.py index 150d7bb3c..3a4f6c072 100644 --- a/src/context_compactor.py +++ b/src/context_compactor.py @@ -244,9 +244,17 @@ def trim_for_context(messages: List[Dict], context_length: int, reserve_tokens: protected_tokens = estimate_tokens(protected_msgs) budget -= protected_tokens - # Priority: keep first system msg (preset prompt), drop others (memory, RAG, memo) - essential_system = system_msgs[:1] if system_msgs else [] - extra_system = system_msgs[1:] + # Priority: keep first system msg (preset prompt), drop others (memory, RAG, memo). + # Exception: a research-spinoff primer (the seeded report that grounds a + # "Discuss" chat) must never be dropped — it is the conversation's whole + # knowledge base. Treat any system message carrying research_spinoff_from + # metadata as essential alongside the leading system prompt. + def _is_research_primer(m): + return bool((m.get("metadata") or {}).get("research_spinoff_from")) + _primers = [m for m in system_msgs if _is_research_primer(m)] + _non_primer = [m for m in system_msgs if not _is_research_primer(m)] + essential_system = (_non_primer[:1] if _non_primer else []) + _primers + extra_system = _non_primer[1:] # Try dropping extra system messages one by one (from the end) trimmed = essential_system + convo_msgs diff --git a/src/cookbook_serve_lifecycle.py b/src/cookbook_serve_lifecycle.py index e30ddfd09..f2700cf7d 100644 --- a/src/cookbook_serve_lifecycle.py +++ b/src/cookbook_serve_lifecycle.py @@ -136,7 +136,8 @@ async def _tick() -> None: return try: state = json.loads(state_path.read_text(encoding="utf-8")) - except Exception: + except Exception as e: + logger.warning("cookbook_serve_lifecycle: state file unreadable (%s), skipping tick", e) return tasks = state.get("tasks") or [] now_ms = int(time.time() * 1000) @@ -160,11 +161,13 @@ async def _tick() -> None: # Re-read state once before writing so we capture any updates from # concurrent UI syncs. stopped_any = False + successfully_stopped_sids = set() for sid, host, port in to_stop: ok = await _stop_serve(sid, host, port) logger.info(f"cookbook_serve_lifecycle: stop {sid} (host={host or 'local'}): {'ok' if ok else 'failed'}") if ok: stopped_any = True + successfully_stopped_sids.add(sid) # Drop the auto-registered endpoint so the model picker and # the chat router don't keep pointing at a dead server. for t in tasks: @@ -178,8 +181,25 @@ async def _tick() -> None: if stopped_any: try: from core.atomic_io import atomic_write_json - state["tasks"] = tasks - atomic_write_json(state_path, state) + # Re-read the state file so concurrent UI writes (task adds, + # status flips, config edits) are not silently overwritten. + # Apply only our stop mutations to the fresh snapshot. + try: + fresh = json.loads(state_path.read_text(encoding="utf-8")) + fresh_tasks = fresh.get("tasks") or [] + except Exception: + fresh = state + fresh_tasks = tasks + for ft in fresh_tasks: + if not isinstance(ft, dict): + continue + ft_sid = ft.get("sessionId") or ft.get("id") + if ft_sid in successfully_stopped_sids: + ft["status"] = "stopped" + ft["_scheduledStopAtMs"] = None + ft["_lastStatusFlipAt"] = now_ms + fresh["tasks"] = fresh_tasks + atomic_write_json(state_path, fresh) except Exception as e: logger.warning(f"cookbook_serve_lifecycle: state write failed: {e}") diff --git a/src/document_processor.py b/src/document_processor.py index 2448f1992..e96ec999c 100644 --- a/src/document_processor.py +++ b/src/document_processor.py @@ -199,11 +199,20 @@ def _fit_inline_attachment_text( return text[:remaining] + marker, 0 -def _process_office_document(path: str, display_name: str) -> str: +def _process_office_document( + path: str, + display_name: str, + session_id: str | None = None, + auto_opened_docs: list[Dict[str, Any]] | None = None, + owner: str | None = None, +) -> str: """Extract an Office/EPUB document to Markdown via the optional markitdown dep. Falls back to a friendly banner when markitdown is unavailable or finds no - text, so a missing optional dependency never breaks the chat path. + text, so a missing optional dependency never breaks the chat path. When a + session_id is provided AND the extraction succeeded, the FULL text is also + saved as a Document so the agent can page through it via + `manage_documents action=read offset=…` after the inline copy is capped. """ from src.markitdown_runtime import ( is_markitdown_format, @@ -218,6 +227,46 @@ def _process_office_document(path: str, display_name: str) -> str: if markdown and markdown.strip(): title = os.path.splitext(os.path.basename(path))[0] body, marker = _truncate_inline(markdown) + + # Persist the full extracted text as a Document. The agent's existing + # manage_documents tool can then read past the inline cap with offset. + doc_id = None + if session_id: + try: + from src.office_doc import create_office_document + doc_id = create_office_document( + session_id=session_id, + upload_id=os.path.basename(path), + title=title, + body_text=markdown, + ) + if doc_id and auto_opened_docs is not None: + from src.database import SessionLocal, Document + _db = SessionLocal() + try: + _d = _db.query(Document).filter(Document.id == doc_id).first() + if _d: + auto_opened_docs.append({ + "doc_id": _d.id, + "title": _d.title, + "language": _d.language, + "content": _d.current_content, + "version": _d.version_count, + }) + finally: + _db.close() + except Exception as e: + logger.warning("Office auto-doc creation failed for %s: %s", path, e) + + # Upgrade the truncation marker with a hint pointing at the full doc so + # the agent knows it can read the rest. + if doc_id and marker: + marker = ( + f"\n[…truncated for inline context — full {len(markdown):,} chars " + f"saved as document `{doc_id}`. Use `manage_documents` with " + f"action=read, document_id={doc_id}, offset=<N> to page through.]" + ) + return f"\n\n[Document content — {title}]:\n{body}{marker}" # No content: tell the user whether to install the optional dep or whether @@ -521,7 +570,13 @@ def build_user_content( elif mime.startswith("text/") or _is_text_file(path): extracted_text = _process_text_file(path) else: - extracted_text = _process_office_document(path, display_name) + extracted_text = _process_office_document( + path, + display_name, + session_id=session_id, + auto_opened_docs=auto_opened_docs, + owner=owner, + ) extracted_text, inline_attachment_remaining = _fit_inline_attachment_text( extracted_text, diff --git a/src/embeddings.py b/src/embeddings.py index 85a55c386..746044c47 100644 --- a/src/embeddings.py +++ b/src/embeddings.py @@ -31,6 +31,8 @@ import numpy as np import httpx from typing import List, Optional +from src.runtime_paths import get_app_root + logger = logging.getLogger(__name__) _DEFAULT_MODEL = "all-minilm:l6-v2" diff --git a/src/endpoint_resolver.py b/src/endpoint_resolver.py index 0a3063638..83ba1ce92 100644 --- a/src/endpoint_resolver.py +++ b/src/endpoint_resolver.py @@ -12,7 +12,7 @@ from typing import Optional, Tuple, Dict from urllib.parse import urlparse, urlunparse from core.database import SessionLocal, ModelEndpoint -from src.llm_core import _detect_provider, _host_match, _ollama_api_root +from src.llm_core import _detect_provider, _host_match, _is_kimi_code_url, KIMI_CODE_USER_AGENT, _ollama_api_root logger = logging.getLogger(__name__) @@ -161,6 +161,32 @@ def normalize_base(url: str) -> str: return url +def _validated_endpoint_base(url: str) -> str: + """Return a base URL that is safe for endpoint path appends.""" + base = (url or "").strip().rstrip("/") + if "?" in base or "#" in base: + raise ValueError("Endpoint base URL must not include query or fragment") + return urlunparse(urlparse(base)._replace(query="", fragment="")).rstrip("/") + + +def _prepare_endpoint_base(base: str) -> str: + base = _validated_endpoint_base(normalize_base(base)) + return _validated_endpoint_base(normalize_base(resolve_url(base))) + + +def _append_endpoint_path(base: str, suffix: str) -> str: + parsed = urlparse(base) + current = (parsed.path or "").rstrip("/") + extra = "/" + suffix.lstrip("/") + path = f"{current}{extra}" if current else extra + return urlunparse(parsed._replace(path=path, query="", fragment="")) + + +def _pathless_host(base: str, host: str) -> bool: + parsed = urlparse(base) + return (parsed.hostname or "").lower() == host and not (parsed.path or "").strip("/") + + def _anthropic_api_root(base: str) -> str: """Return Anthropic's API root, preserving /v1 for OpenAI-compatible APIs elsewhere.""" base = (base or "").strip().rstrip("/") @@ -171,28 +197,49 @@ def _anthropic_api_root(base: str) -> str: def build_chat_url(base: str) -> str: """Return the correct chat endpoint URL for a given base.""" - base = resolve_url(base) + base = _prepare_endpoint_base(base) provider = _detect_provider(base) if provider == "anthropic": - return _anthropic_api_root(base) + "/v1/messages" + return _append_endpoint_path(_anthropic_api_root(base), "/v1/messages") if provider == "ollama": - return _ollama_api_root(base) + "/chat" + return _append_endpoint_path(_ollama_api_root(base), "/chat") if provider == "chatgpt-subscription": - return base.rstrip("/") + "/responses" - return base + "/chat/completions" + return _append_endpoint_path(base, "/responses") + if _pathless_host(base, "api.openai.com"): + base = _append_endpoint_path(base, "/v1") + return _append_endpoint_path(base, "/chat/completions") def build_models_url(base: str) -> Optional[str]: - """Return the provider-specific model-list endpoint URL for a base.""" - base = normalize_base(resolve_url(base)) + """Return the provider-specific model-list endpoint URL for a base. + + For OpenAI-compatible servers (LM Studio, llama.cpp, vLLM, + text-generation-webui, etc.) the model list is exposed at ``/v1/models``. + When the user-supplied base has no path — e.g. ``http://localhost:1234`` — + we still need to land on ``/v1/models`` (issue #25); insert the ``/v1`` + segment only when the path is empty, leaving any explicit non-empty path + untouched (so custom prefixes like ``/openai`` or ``/api/openai/v1`` keep + their semantics). + """ + base = _prepare_endpoint_base(base) provider = _detect_provider(base) if provider == "anthropic": - return _anthropic_api_root(base) + "/v1/models" + return _append_endpoint_path(_anthropic_api_root(base), "/v1/models") if provider == "ollama": - return _ollama_api_root(base) + "/tags" + return _append_endpoint_path(_ollama_api_root(base), "/tags") if provider == "chatgpt-subscription": return None - return base + "/models" + # Generic OpenAI-compatible fallback: local model servers with no explicit + # path conventionally expose `/v1/models` (LM Studio, llama.cpp, vLLM). + # For non-local unknown hosts, do not invent `/v1`; append `/models` to the + # caller's base so look-alike provider hosts stay generic. + parsed = urlparse(base) + host = (parsed.hostname or "").lower() + is_local = host in {"localhost", "127.0.0.1", "::1", "host.docker.internal"} + uses_v1_models_by_default = is_local or host in {"api.deepseek.com", "api.openai.com"} + if not parsed.path and uses_v1_models_by_default: + base = _append_endpoint_path(base, "/v1") + return _append_endpoint_path(base, "/models") def build_headers(api_key: Optional[str], base: str) -> Dict[str, str]: @@ -215,6 +262,8 @@ def build_headers(api_key: Optional[str], base: str) -> Dict[str, str]: if provider == "openrouter": headers.setdefault("HTTP-Referer", "https://github.com/pewdiepie-archdaemon/odysseus") headers.setdefault("X-OpenRouter-Title", "Odysseus") + if _is_kimi_code_url(base): + headers.setdefault("User-Agent", KIMI_CODE_USER_AGENT) return headers @@ -250,27 +299,23 @@ def resolve_endpoint( ep_id = _stg(f"{setting_prefix}_endpoint_id") model = _stg(f"{setting_prefix}_model") - # If the specific endpoint is not configured, but the caller provided a + # Fall back to utility model for task/research/auto-naming if not specifically configured. + if not ep_id and setting_prefix not in ("utility", "default"): + ep_id = _stg("utility_endpoint_id") + model = _stg("utility_model") + + # If the endpoint is STILL not configured, but the caller provided a # valid fallback (e.g. the active session model), use that immediately. # This prevents background tasks from jumping to the global default_model # when the user is mid-conversation with a different model. if not ep_id and fallback_url and fallback_model: return fallback_url, fallback_model, fallback_headers - # Unset Utility means "same as Default Chat Model". - if setting_prefix == "utility" and not ep_id: + # Unset Utility (or anything else that didn't have a fallback) means "same as Default Chat Model". + if not ep_id: ep_id = _stg("default_endpoint_id") model = _stg("default_model") - # Fall back to utility model for task/research/auto-naming if not specifically configured. - # If Utility itself is unset, the block above makes that resolve to Default Chat. - if not ep_id and setting_prefix != "utility": - ep_id = _stg("utility_endpoint_id") - model = _stg("utility_model") - if not ep_id: - ep_id = _stg("default_endpoint_id") - model = _stg("default_model") - if not ep_id: return fallback_url, fallback_model, fallback_headers diff --git a/src/integrations.py b/src/integrations.py index 11fee99e7..3b2b88859 100644 --- a/src/integrations.py +++ b/src/integrations.py @@ -4,8 +4,10 @@ import uuid import logging import re from typing import Dict, List, Optional, Any +from urllib.parse import urljoin, urlparse, urlunparse import httpx +from fastapi import HTTPException from core.atomic_io import atomic_write_json from core.platform_compat import safe_chmod @@ -201,6 +203,22 @@ def mask_integration_secret(integration: Dict[str, Any]) -> Dict[str, Any]: return safe +def _normalize_integration_base_url(base_url: Any) -> str: + if not isinstance(base_url, str) or not base_url.strip(): + raise ValueError("Integration base URL is required") + cleaned = base_url.strip().rstrip("/") + if "?" in cleaned or "#" in cleaned: + raise ValueError("Integration base URL must not include query or fragment") + parsed = urlparse(cleaned) + if parsed.scheme.lower() not in ("http", "https") or not parsed.hostname: + raise ValueError("Integration base URL must be an HTTP(S) URL") + return urlunparse(parsed._replace(scheme=parsed.scheme.lower(), query="", fragment="")).rstrip("/") + + +def _join_integration_url(base_url: str, path: str) -> str: + return urljoin(base_url.rstrip("/") + "/", path.lstrip("/")) + + def load_integrations() -> List[Dict[str, Any]]: """Load all integrations from disk with secrets decrypted for runtime use.""" if not os.path.exists(DATA_FILE): @@ -258,6 +276,13 @@ def add_integration(data: Dict[str, Any]) -> Dict[str, Any]: integration.setdefault("name", "") integration.setdefault("base_url", "") + if not isinstance(integration.get("name"), str) or not integration["name"].strip(): + raise HTTPException(400, "Integration name is required") + try: + integration["base_url"] = _normalize_integration_base_url(integration.get("base_url")) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + integrations = load_integrations() integrations.append(integration) save_integrations(integrations) @@ -266,6 +291,15 @@ def add_integration(data: Dict[str, Any]) -> Dict[str, Any]: def update_integration(integration_id: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Update fields on an existing integration. Returns updated integration or None.""" + data = dict(data) + if "name" in data and (not isinstance(data["name"], str) or not data["name"].strip()): + raise HTTPException(400, "Integration name is required") + if "base_url" in data: + try: + data["base_url"] = _normalize_integration_base_url(data["base_url"]) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + integrations = load_integrations() for item in integrations: if item.get("id") == integration_id: @@ -330,9 +364,10 @@ async def execute_api_call( if not integration.get("enabled", True): return {"error": f"Integration '{integration.get('name')}' is disabled", "exit_code": 1} - base_url = integration.get("base_url", "").rstrip("/") - if not base_url: - return {"error": "Integration has no base_url configured", "exit_code": 1} + try: + base_url = _normalize_integration_base_url(integration.get("base_url", "")) + except ValueError as exc: + return {"error": str(exc), "exit_code": 1} # Strip common API path suffixes users might accidentally include # (e.g. "http://host/v1/" → "http://host"). The integration's preset @@ -355,7 +390,10 @@ async def execute_api_call( if re.search(r"^https?://", path) or "://" in path: return {"error": "Path must not contain a protocol scheme", "exit_code": 1} - url = base_url + path + if "#" in path: + return {"error": "Path must not contain a fragment", "exit_code": 1} + + url = _join_integration_url(base_url, path) method = method.upper() # Build headers diff --git a/src/llm_core.py b/src/llm_core.py index 26b5f96e7..e809d7968 100644 --- a/src/llm_core.py +++ b/src/llm_core.py @@ -7,6 +7,7 @@ import logging import hashlib import threading import re +import os from fastapi import HTTPException from typing import Optional, Dict, List, Tuple from src.model_context import get_context_length, DEFAULT_CONTEXT @@ -22,6 +23,24 @@ class LLMConfig: MAX_RETRIES = 3 RETRY_DELAY = 0.5 STREAM_TIMEOUT = 300 + # TCP+TLS connect budget for a SINGLE attempt. The old hard-coded 3.0s + # assumed LAN/Tailscale peers ('SYN in <100ms'); it is too tight for public + # cloud endpoints (offshore APIs take ~0.5-1.5s cold, with jitter), so a + # brief blip on the first connect of an idle chat surfaced as a 503 on the + # streaming path (which, unlike llm_call, does not retry the connect). A + # genuinely dead upstream stays bounded by the dead-host cooldown. Override + # with env LLM_CONNECT_TIMEOUT (seconds). + CONNECT_TIMEOUT = float(os.getenv('LLM_CONNECT_TIMEOUT', '10') or '10') + + +def _call_timeout(read_timeout) -> httpx.Timeout: + """Per-request timeout for non-streaming LLM calls (connect from config).""" + return httpx.Timeout(connect=LLMConfig.CONNECT_TIMEOUT, read=float(read_timeout), write=10.0, pool=5.0) + + +def _stream_timeout(read_timeout) -> httpx.Timeout: + """Per-request timeout for streaming LLM calls (connect from config).""" + return httpx.Timeout(connect=LLMConfig.CONNECT_TIMEOUT, read=float(read_timeout), write=30.0, pool=5.0) # Cache for LLM responses @@ -264,7 +283,8 @@ def _is_ollama_native_url(url: str) -> bool: """Return True for native Ollama API URLs, including Ollama Cloud.""" try: parsed = urlparse(url or "") - except Exception: + except Exception as e: + logger.warning("Failed to parse URL for Ollama detection", exc_info=e) return False host = parsed.hostname or "" path = (parsed.path or "").rstrip("/") @@ -423,6 +443,146 @@ def _host_match(url: str, *domains: str) -> bool: return any(host == d or host.endswith("." + d) for d in domains) +# Kimi Code subscription keys (api.kimi.com/coding/v1) require a whitelisted +# coding-agent User-Agent; otherwise the API returns 403 access_terminated_error. +# Tried in order; first success is cached per base URL for later requests. +KIMI_CODE_USER_AGENTS: tuple[str, ...] = ( + "claude-code/0.1.0", + "claude-code/1.0.0", + "KimiCLI/1.0", + "Kilo-Code/1.0", + "Roo-Code/1.0", + "Cursor/1.0", +) +KIMI_CODE_USER_AGENT = KIMI_CODE_USER_AGENTS[0] +_kimi_code_ua_cache: dict[str, str] = {} + + +def _is_kimi_code_url(url: str) -> bool: + if not url or not _host_match(url, "kimi.com"): + return False + try: + return "/coding" in (urlparse(url).path or "") + except Exception: + return False + + +def _kimi_code_base_key(url: str) -> str: + """Normalize a Kimi Code chat/models URL to its OpenAI base (.../coding/v1).""" + parsed = urlparse(url) + path = (parsed.path or "").rstrip("/") + for suffix in ("/chat/completions", "/models", "/completions"): + if path.endswith(suffix): + path = path[: -len(suffix)] + path = path.rstrip("/") or "/coding/v1" + return f"{parsed.scheme}://{parsed.netloc}{path}" + + +def _is_kimi_code_access_denied(status: int, body: bytes | str) -> bool: + if status != 403: + return False + text = body.decode("utf-8", errors="replace") if isinstance(body, bytes) else (body or "") + lower = text.lower() + return ( + "access_terminated_error" in lower + or "coding agents" in lower + or "only available for coding" in lower + ) + + +def _kimi_code_ua_candidates(url: str) -> list[str]: + if not _is_kimi_code_url(url): + return [] + base_key = _kimi_code_base_key(url) + cached = _kimi_code_ua_cache.get(base_key) + if cached: + return [cached] + [ua for ua in KIMI_CODE_USER_AGENTS if ua != cached] + return list(KIMI_CODE_USER_AGENTS) + + +def _remember_kimi_code_user_agent(url: str, user_agent: str) -> None: + _kimi_code_ua_cache[_kimi_code_base_key(url)] = user_agent + + +def apply_kimi_code_headers(headers: Optional[Dict], url: str) -> Dict[str, str]: + """Pick a Kimi Code User-Agent (cached probe when possible).""" + h = dict(headers or {}) + if not _is_kimi_code_url(url): + return h + base_key = _kimi_code_base_key(url) + cached = _kimi_code_ua_cache.get(base_key) + if cached: + h["User-Agent"] = cached + return h + models_url = base_key.rstrip("/") + "/models" + from src.tls_overrides import llm_verify + for ua in KIMI_CODE_USER_AGENTS: + trial = dict(h) + trial["User-Agent"] = ua + try: + r = httpx.get(models_url, headers=trial, timeout=8, verify=llm_verify()) + except Exception: + continue + if _is_kimi_code_access_denied(r.status_code, r.content): + logger.debug("Kimi Code rejected User-Agent %s (403), trying next", ua) + continue + if r.status_code < 400: + _remember_kimi_code_user_agent(url, ua) + h["User-Agent"] = ua + return h + break + h.setdefault("User-Agent", KIMI_CODE_USER_AGENT) + return h + + +def httpx_get_kimi_aware(url: str, headers: Optional[Dict], **kwargs): + h = apply_kimi_code_headers(headers, url) + if not _is_kimi_code_url(url): + return httpx.get(url, headers=h, **kwargs) + last = None + for ua in _kimi_code_ua_candidates(url): + trial = dict(h) + trial["User-Agent"] = ua + last = httpx.get(url, headers=trial, **kwargs) + if not _is_kimi_code_access_denied(last.status_code, last.content): + if last.status_code < 400: + _remember_kimi_code_user_agent(url, ua) + return last + return last + + +def httpx_post_kimi_aware(url: str, headers: Optional[Dict], **kwargs): + h = apply_kimi_code_headers(headers, url) + if not _is_kimi_code_url(url): + return httpx.post(url, headers=h, **kwargs) + last = None + for ua in _kimi_code_ua_candidates(url): + trial = dict(h) + trial["User-Agent"] = ua + last = httpx.post(url, headers=trial, **kwargs) + if not _is_kimi_code_access_denied(last.status_code, last.content): + if last.status_code < 400: + _remember_kimi_code_user_agent(url, ua) + return last + return last + + +async def httpx_post_kimi_aware_async(client, url: str, headers: Optional[Dict], **kwargs): + h = apply_kimi_code_headers(headers, url) + if not _is_kimi_code_url(url): + return await client.post(url, headers=h, **kwargs) + last = None + for ua in _kimi_code_ua_candidates(url): + trial = dict(h) + trial["User-Agent"] = ua + last = await client.post(url, headers=trial, **kwargs) + if not _is_kimi_code_access_denied(last.status_code, last.content): + if last.status_code < 400: + _remember_kimi_code_user_agent(url, ua) + return last + return last + + def _detect_provider(url: str) -> str: """Detect the API provider from a configured endpoint URL. @@ -446,6 +606,8 @@ def _detect_provider(url: str) -> str: return "groq" if _host_match(url, "nvidia.com"): return "nvidia" + if _host_match(url, "moonshot.ai") or _host_match(url, "moonshot.cn"): + return "moonshot" from src.chatgpt_subscription import is_chatgpt_subscription_base if is_chatgpt_subscription_base(url): return "chatgpt-subscription" @@ -457,15 +619,25 @@ def _detect_provider(url: str) -> str: def _is_self_hosted_openai_compatible(url: str) -> bool: """True for custom/local OpenAI-compatible servers (llama.cpp, LM Studio, - vLLM, text-generation-webui, etc.) as opposed to api.openai.com itself. + vLLM, text-generation-webui, etc.) as opposed to cloud APIs. Used to gate llama.cpp-server-specific payload extras (``session_id``, - ``cache_prompt``) — sending unrecognized top-level fields to OpenAI's - actual API returns a 400 ("Unrecognized request argument"), but - self-hosted servers generally ignore unknown fields and many (notably - llama.cpp's server) use them for KV-cache slot affinity (issue #2927). + ``cache_prompt``) used for KV-cache slot affinity (issue #2927). Strict + cloud providers reject unrecognized top-level fields (api.openai.com + returns 400, Mistral returns 422 "extra_forbidden", issue #3793), and any + unknown OpenAI-compatible host used to be treated as self-hosted, so those + fields leaked to every strict provider added as a custom endpoint. + + A server only counts as self-hosted when it also resolves as local: + loopback/private/tailscale host, or the endpoint explicitly configured + with kind "local". A self-hosted server exposed via a public hostname + loses the affinity hint unless its endpoint kind is set to "local" - + a lost perf hint, versus a hard 4xx on every request the other way. """ - return _detect_provider(url) == "openai" and not _host_match(url, "openai.com") + if _detect_provider(url) != "openai" or _host_match(url, "openai.com"): + return False + from src.model_context import is_local_endpoint + return is_local_endpoint(url) def _apply_local_cache_affinity(payload: Dict, url: str, session_id: Optional[str]) -> None: @@ -532,6 +704,12 @@ def _provider_label(url: str) -> str: if _host_match(url, "googleapis.com"): return "Google" if _host_match(url, "together.xyz", "together.ai"): return "Together" if _host_match(url, "fireworks.ai"): return "Fireworks" + if _host_match(url, "kimi.com"): + try: + if "/coding" in (urlparse(url).path or ""): + return "Kimi Code" + except Exception: + pass if _is_ollama_native_url(url): return "Ollama" try: host = (urlparse(url).hostname or "").lower() @@ -672,7 +850,7 @@ def _uses_max_completion_tokens(model: str) -> bool: # perfectly good model as failing. For these models we omit the field and let # the API use its required default. (gpt-4.5 is intentionally excluded — it is # not a reasoning model and accepts temperature normally.) -_FIXED_TEMPERATURE_MODELS = ("o1", "o3", "o4", "gpt-5") +_FIXED_TEMPERATURE_MODELS = ("o1", "o3", "o4", "gpt-5", "kimi-for-coding") def _restricts_temperature(model: str) -> bool: """Check if a model rejects any non-default temperature.""" @@ -681,6 +859,49 @@ def _restricts_temperature(model: str) -> bool: m = model.lower() return any(m.startswith(p) or f"/{p}" in m for p in _FIXED_TEMPERATURE_MODELS) + +# The official Moonshot API fixes temperature at 1.0 in thinking mode and 0.6 +# when thinking is explicitly disabled for Kimi K2.5/K2.6. Any other explicit +# value returns HTTP 400. Odysseus does not currently send the `thinking` mode +# control, so omit temperature and let Moonshot use its default thinking mode. +# Keep the gate provider-specific: self-hosted Kimi deployments may accept +# custom sampling values, and older Moonshot models have different defaults. +def _moonshot_rejects_custom_temperature(provider: str, model: str) -> bool: + """Check if the official Moonshot API fixes temperature for this model.""" + if provider != "moonshot" or not isinstance(model, str): + return False + model_id = model.lower().rsplit("/", 1)[-1] + return bool(re.match(r"^kimi-k2\.(?:5|6)(?:$|[-_:])", model_id)) + + +def _omit_temperature(provider: str, model: str) -> bool: + """Check if a request should use the provider's default temperature.""" + return _restricts_temperature(model) or _moonshot_rejects_custom_temperature( + provider, model + ) + + +# Anthropic removed the sampling parameters (temperature, top_p, top_k) starting +# with Claude Opus 4.7. On Opus 4.7 and later, sending `temperature` at all — +# even 0.0 — returns HTTP 400. Earlier Claude models (Opus 4.6 and below, every +# Sonnet/Haiku) still accept temperature in [0.0, 1.0], so the omission must be +# version-gated rather than applied to all `claude-*` models. +def _anthropic_rejects_temperature(model: str) -> bool: + """Check if a native-Anthropic model rejects the temperature field (Opus 4.7+).""" + if not isinstance(model, str) or not model: + return False + # `(?<![a-z])` anchors "opus" to a word boundary so a substring match like + # `oct-opus`/`octopus-4-8` can't be read as Opus (it would otherwise strip + # temperature). Cap the minor at 1-2 digits and forbid a trailing digit so a + # dated id like `claude-opus-4-20250514` (Opus 4.0) parses as major-only (no + # minor match, kept) instead of reading the date `20250514` as a giant minor + # that would falsely test >= 4.7. Dated 4.7+ snapshots (`claude-opus-4-7- + # 20260201`) keep their explicit minor and are still matched. + match = re.search(r"(?<![a-z])opus[-_]?(\d+)[-_.](\d{1,2})(?!\d)", model.lower()) + if not match: + return False + return (int(match.group(1)), int(match.group(2))) >= (4, 7) + # Models that support structured thinking — may output </think> without opening tag _THINKING_MODEL_PATTERNS = ("qwen3", "qwq", "deepseek-r1", "deepseek-reasoner", "minimax", "m2-reap", "gemma") @@ -784,8 +1005,11 @@ def _build_anthropic_payload(model, messages, temperature, max_tokens, stream=Fa "model": model, "messages": chat_messages, "max_tokens": max_tokens if max_tokens and max_tokens > 0 else 4096, - "temperature": temperature, } + # Opus 4.7+ removed the sampling parameters — sending `temperature` (even 0.0) + # returns HTTP 400. Omit it for those models; older Claude models still take it. + if not _anthropic_rejects_temperature(model): + payload["temperature"] = temperature if system_parts: system_text = "\n\n".join(system_parts) # Send `system` as a structured text block so we can attach a prompt-cache @@ -1104,7 +1328,7 @@ def list_model_ids( from src.endpoint_resolver import build_models_url models_url = build_models_url(base_chat_url) - r = httpx.get(models_url, headers=h, timeout=timeout) + r = httpx_get_kimi_aware(models_url, h, timeout=timeout) r.raise_for_status() data = r.json() model_ids = [m.get("id") for m in (data.get("data") or []) if m.get("id")] @@ -1122,8 +1346,8 @@ def list_model_ids( r = httpx.get(root + "/api/tags", timeout=timeout) r.raise_for_status() return [m.get("name") or m.get("model") for m in (r.json().get("models") or []) if m.get("name") or m.get("model")] - except Exception: - pass + except Exception as e: + logger.warning("Failed to fetch model list from configured endpoint", exc_info=e) return [] def normalize_model_id( @@ -1205,14 +1429,14 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL "messages": messages_copy, "temperature": temperature, } - if _restricts_temperature(model): + if _omit_temperature(provider, model): payload.pop("temperature", None) if max_tokens and max_tokens > 0: tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens" payload[tok_key] = max_tokens try: note_model_activity(target_url, model) - r = httpx.post(target_url, headers=h, json=payload, timeout=timeout) + r = httpx_post_kimi_aware(target_url, h, json=payload, timeout=timeout) except Exception as e: raise HTTPException(502, f"POST {target_url} failed: {e}") if not r.is_success: @@ -1399,7 +1623,7 @@ async def llm_call_async( "messages": messages_copy, "temperature": temperature, } - if _restricts_temperature(model): + if _omit_temperature(provider, model): payload.pop("temperature", None) if max_tokens and max_tokens > 0: tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens" @@ -1412,7 +1636,7 @@ async def llm_call_async( if _is_host_dead(target_url): raise HTTPException(503, f"Upstream {_host_key(target_url)} marked unreachable (cooldown active)") - call_timeout = httpx.Timeout(connect=3.0, read=float(timeout), write=10.0, pool=5.0) + call_timeout = _call_timeout(timeout) attempt = 0 while attempt < max_retries: attempt += 1 @@ -1420,7 +1644,7 @@ async def llm_call_async( try: note_model_activity(target_url, model) client = _get_http_client() - r = await client.post(target_url, headers=h, json=payload, timeout=call_timeout) + r = await httpx_post_kimi_aware_async(client, target_url, h, json=payload, timeout=call_timeout) duration = time.time() - start if not r.is_success: friendly = _format_upstream_error(r.status_code, r.text, target_url) @@ -1516,7 +1740,7 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl "temperature": temperature, "stream": True, } - if _restricts_temperature(model): + if _omit_temperature(provider, model): payload.pop("temperature", None) if provider not in {"openrouter", "groq"}: payload["stream_options"] = {"include_usage": True} @@ -1536,9 +1760,12 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl from src.copilot import apply_request_headers apply_request_headers(h, messages_copy) - # Short connect timeout: a reachable peer answers SYN in <100ms even on - # Tailscale. 3s is plenty; 30s let one dead upstream wedge the UI. - stream_timeout = httpx.Timeout(connect=3.0, read=float(timeout), write=30.0, pool=5.0) + # Connect budget from LLMConfig.CONNECT_TIMEOUT (env LLM_CONNECT_TIMEOUT). + # The dead-host cooldown still bounds a genuinely unreachable upstream, so a + # wider connect budget only affects first contact and stops a brief cold + # connect blip (offshore/public endpoints) surfacing as a 503 on this stream + # path, which -- unlike llm_call -- does not retry the connect. + stream_timeout = _stream_timeout(timeout) if _is_host_dead(target_url): yield f'event: error\ndata: {json.dumps({"error": f"Upstream {_host_key(target_url)} unreachable (cooldown active)", "status": 503})}\n\n' @@ -1814,6 +2041,7 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl events.append(_stream_delta_event(part)) return events + h = apply_kimi_code_headers(h, target_url) try: client = _get_http_client() async with client.stream('POST', target_url, json=payload, headers=h, timeout=stream_timeout) as r: diff --git a/src/markitdown_runtime.py b/src/markitdown_runtime.py index ff30b0170..b6fc961b0 100644 --- a/src/markitdown_runtime.py +++ b/src/markitdown_runtime.py @@ -40,15 +40,59 @@ def load_markitdown(): return MarkItDown +def _extract_docx_native(path: str) -> str | None: + """Pure-Python .docx text extractor — no external deps. + + A .docx file is just a zip of XML. The body prose lives in <w:t> runs + inside <w:p> paragraphs. Iterating with ElementTree (rather than + re.findall) keeps paragraph breaks intact and lets the XML parser handle + namespaces + entity unescaping. Loses tables, footnotes, images and + list bullets — keeps ~95% of "summarize this doc" content, which is the + case people hit when markitdown isn't installed. + """ + import zipfile + import xml.etree.ElementTree as ET + + ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + try: + with zipfile.ZipFile(path) as z: + xml_bytes = z.read("word/document.xml") + except (zipfile.BadZipFile, KeyError, OSError): + return None + try: + root = ET.fromstring(xml_bytes) + except ET.ParseError: + return None + paragraphs: list[str] = [] + for para in root.iter(f"{ns}p"): + runs = [t.text or "" for t in para.iter(f"{ns}t")] + line = "".join(runs).strip() + if line: + paragraphs.append(line) + return "\n\n".join(paragraphs) if paragraphs else None + + def convert_to_markdown(path: str) -> str | None: """Convert a document to Markdown text via markitdown. Returns the extracted Markdown, or ``None`` if markitdown is unavailable or the conversion fails — callers degrade gracefully rather than erroring. + + Fallback: when markitdown isn't installed and the file is a .docx, run + the bundled pure-Python extractor so the most common case (Word docs) + works out of the box. Other Office/EPUB formats still need markitdown. """ try: markitdown_cls = load_markitdown() except RuntimeError: + if isinstance(path, str) and path.lower().endswith(".docx"): + text = _extract_docx_native(path) + if text: + logger.info( + "markitdown not installed — used native .docx extractor for %s", + path, + ) + return text logger.warning("markitdown not installed; cannot extract %s", path) return None try: diff --git a/src/mcp_manager.py b/src/mcp_manager.py index 29fdedebf..8f4322375 100644 --- a/src/mcp_manager.py +++ b/src/mcp_manager.py @@ -11,6 +11,8 @@ import os import re from typing import Any, Dict, List, Optional, Set, Tuple +from src.runtime_paths import get_app_root + logger = logging.getLogger(__name__) def _format_mcp_connection_error(name: str, command: str = "", args: Optional[List[str]] = None, error: Exception = None) -> str: @@ -508,7 +510,7 @@ class McpManager: return False script_rel, name = _BUILTIN_SERVERS[server_id] - base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + base_dir = get_app_root() script_path = os.path.join(base_dir, script_rel) # Clean up old connection diff --git a/src/model_context.py b/src/model_context.py index a2ce9f638..72526e744 100644 --- a/src/model_context.py +++ b/src/model_context.py @@ -5,6 +5,7 @@ Query and cache model context window sizes from OpenAI-compatible APIs. Provides token estimation for context usage tracking. """ +import ipaddress import logging import sys from typing import Dict, List, Optional, Tuple @@ -16,10 +17,32 @@ import httpx logger = logging.getLogger(__name__) _LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1", "host.docker.internal"} -_PRIVATE_PREFIXES = ("10.", "172.16.", "172.17.", "172.18.", "172.19.", - "172.20.", "172.21.", "172.22.", "172.23.", "172.24.", - "172.25.", "172.26.", "172.27.", "172.28.", "172.29.", - "172.30.", "172.31.", "192.168.", "100.") +_PRIVATE_NETWORKS = ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), +) + +# Tailscale uses the CGNAT range 100.64.0.0/10, NOT all of 100.0.0.0/8. +# A bare "100." prefix would classify public addresses (e.g. AWS ranges +# under 100.x outside the CGNAT block) as local; routes/model_routes.py +# already narrows this the same way for endpoint classification. +_TAILSCALE_CGNAT = ipaddress.ip_network("100.64.0.0/10") + + +def _in_tailscale_range(host: str) -> bool: + try: + return ipaddress.ip_address(host) in _TAILSCALE_CGNAT + except ValueError: + return False + + +def _is_private_ip_literal(host: str) -> bool: + try: + ip = ipaddress.ip_address(host) + except ValueError: + return False + return any(ip in network for network in _PRIVATE_NETWORKS) def _normalize_base_for_compare(url: str) -> str: @@ -64,7 +87,7 @@ def _configured_endpoint_kind(url: str) -> Optional[str]: return None -def _is_local_endpoint(url: str) -> bool: +def is_local_endpoint(url: str) -> bool: """Check if URL points to a local/private/tailscale address.""" kind = _configured_endpoint_kind(url) if kind in ("api", "proxy"): @@ -73,7 +96,7 @@ def _is_local_endpoint(url: str) -> bool: return True try: host = urlparse(url).hostname or "" - return host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES) + return host in _LOCAL_HOSTS or _is_private_ip_literal(host) or _in_tailscale_range(host) except Exception: return False @@ -208,7 +231,30 @@ KNOWN_CONTEXT_WINDOWS = { # --------------------------------------------------------------------------- # Cache # --------------------------------------------------------------------------- -_context_cache: Dict[Tuple[str, str], int] = {} +_context_cache: Dict[Tuple[str, str], Tuple[int, bool]] = {} + + +def _get_context_length_cached(endpoint_url: str, model: str) -> Tuple[int, bool]: + """Return (context_length, known). ``known`` is False only when the value is a + bare DEFAULT_CONTEXT fallback (no endpoint report and not in the known table).""" + configured_kind = _configured_endpoint_kind(endpoint_url) + is_local = is_local_endpoint(endpoint_url) + # Key on (endpoint_url, model): the same model id can be served by two + # different remote endpoints with different real context windows (e.g. a + # capped proxy vs. the full provider), so caching by model id alone would + # serve one endpoint's window for the other (issue #2603). + cache_key = (endpoint_url, model) + if not is_local and cache_key in _context_cache: + return _context_cache[cache_key] + + ctx, known = _query_context_length(endpoint_url, model) + # Only cache non-default values to allow retry on next request. + # Local endpoints can restart with a different --max-model-len while keeping + # the same model id, so always re-query them instead of serving stale cache. + if not is_local and (ctx != DEFAULT_CONTEXT or configured_kind in ("api", "proxy")): + _context_cache[cache_key] = (ctx, known) + logger.info(f"Context length for {model}: {ctx}") + return ctx, known def get_context_length(endpoint_url: str, model: str) -> int: @@ -218,24 +264,33 @@ def get_context_length(endpoint_url: str, model: str) -> int: or context_window fields. Caches result per (endpoint, model). Falls back to DEFAULT_CONTEXT if unavailable. """ - configured_kind = _configured_endpoint_kind(endpoint_url) - is_local = _is_local_endpoint(endpoint_url) - # Key on (endpoint_url, model): the same model id can be served by two - # different remote endpoints with different real context windows (e.g. a - # capped proxy vs. the full provider), so caching by model id alone would - # serve one endpoint's window for the other (issue #2603). - cache_key = (endpoint_url, model) - if not is_local and cache_key in _context_cache: - return _context_cache[cache_key] + return _get_context_length_cached(endpoint_url, model)[0] - ctx = _query_context_length(endpoint_url, model) - # Only cache non-default values to allow retry on next request. - # Local endpoints can restart with a different --max-model-len while keeping - # the same model id, so always re-query them instead of serving stale cache. - if not is_local and (ctx != DEFAULT_CONTEXT or configured_kind in ("api", "proxy")): - _context_cache[cache_key] = ctx - logger.info(f"Context length for {model}: {ctx}") - return ctx + +def get_context_length_known(endpoint_url: str, model: str) -> Tuple[int, bool]: + """Like ``get_context_length`` but also returns whether the window was actually + discovered (endpoint-reported or in the known-models table) rather than the bare + DEFAULT_CONTEXT fallback. Callers that *scale* a budget off the window must not + trust an unknown value — a fallback 128K isn't proof the model holds 128K + (review on #4122).""" + return _get_context_length_cached(endpoint_url, model) + + +def budget_context_for_model(endpoint_url: str, model: str, *, fallback: int = 0) -> int: + """Context window to scale the agent input budget against. + + Returns the *freshly discovered* window when it was actually proven + (endpoint-reported / known table), else 0 so auto-scaling stays conservative. + Crucially this binds the ``known`` flag to the value it proves — callers must + not pair this flag with a context length from a *different* lookup (a stale + local re-query, or a caller that didn't pass one), which would budget off an + unproven number (review on #4122). On probe error, returns ``fallback`` (the + caller's best-known value) to preserve prior behaviour.""" + try: + ctx, known = get_context_length_known(endpoint_url, model) + return ctx if known else 0 + except Exception: + return fallback def _lookup_known(model: str) -> Optional[int]: @@ -257,8 +312,9 @@ def _lookup_known(model: str) -> Optional[int]: return best_ctx -def _query_context_length(endpoint_url: str, model: str) -> int: - """Query the model API for context length.""" +def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]: + """Query the model API for context length. Returns (context_length, known) where + ``known`` is False only for the bare DEFAULT_CONTEXT fallback.""" known = _lookup_known(model) api_ctx = None configured_kind = _configured_endpoint_kind(endpoint_url) @@ -269,11 +325,11 @@ def _query_context_length(endpoint_url: str, model: str) -> int: if configured_kind in ("api", "proxy"): if known: logger.info(f"Using known context window for {model}: {known}") - return known - return DEFAULT_CONTEXT + return known, True + return DEFAULT_CONTEXT, False # Try llama.cpp /slots endpoint first — reports actual serving context - if _is_local_endpoint(endpoint_url): + if is_local_endpoint(endpoint_url): try: base = endpoint_url.split("/v1")[0] if "/v1" in endpoint_url else endpoint_url.rsplit("/", 1)[0] r = httpx.get(f"{base}/slots", timeout=REQUEST_TIMEOUT) @@ -283,7 +339,7 @@ def _query_context_length(endpoint_url: str, model: str) -> int: n_ctx = slots[0].get("n_ctx") if n_ctx and isinstance(n_ctx, int) and n_ctx > 0: logger.info(f"llama.cpp /slots reports n_ctx={n_ctx} for {model}") - return n_ctx + return n_ctx, True except Exception: pass @@ -295,7 +351,8 @@ def _query_context_length(endpoint_url: str, model: str) -> int: if is_copilot_base(endpoint_url): if known: logger.info(f"Using known context window for {model}: {known}") - return known or DEFAULT_CONTEXT + return known, True + return DEFAULT_CONTEXT, False from src.endpoint_resolver import build_models_url @@ -337,21 +394,21 @@ def _query_context_length(endpoint_url: str, model: str) -> int: # For local/self-hosted endpoints, trust the API value (user set --max-model-len) # For cloud APIs, use the larger value (API can report low defaults) if api_ctx and known: - _is_local = _is_local_endpoint(endpoint_url) + _is_local = is_local_endpoint(endpoint_url) if _is_local and api_ctx < known: logger.info(f"Local endpoint reports {api_ctx} for {model} (known max: {known}) — using API value") - return api_ctx + return api_ctx, True result = max(api_ctx, known) if api_ctx < known: logger.info(f"API reported {api_ctx} for {model}, using known {known} instead") - return result + return result, True if api_ctx: - return api_ctx + return api_ctx, True if known: logger.info(f"Using known context window for {model}: {known}") - return known + return known, True - return DEFAULT_CONTEXT + return DEFAULT_CONTEXT, False def estimate_tokens(messages: List[Dict]) -> int: diff --git a/src/office_doc.py b/src/office_doc.py new file mode 100644 index 000000000..37b45a637 --- /dev/null +++ b/src/office_doc.py @@ -0,0 +1,73 @@ +"""Auto-create a Document row from an Office attachment. + +When a .docx (and friends) lands in chat, the full extracted text is stored +as a Document so the agent can page through it with `manage_documents +action=read offset=…` even after the inline chat payload was capped. Mirrors +the PDF auto-doc pattern in `src.pdf_form_doc`. +""" + +import logging +import uuid +from typing import Optional + +logger = logging.getLogger(__name__) + + +def create_office_document( + session_id: str, + upload_id: str, + title: str, + body_text: Optional[str] = None, +) -> Optional[str]: + """Create a markdown Document for an Office attachment and set it active. + + Returns the new doc_id, or None on failure / empty body. The full + extracted body lives in `current_content`, so the agent can fetch + arbitrary windows via `manage_documents action=read` even when the + inline chat copy was truncated. + """ + from src.database import ( + SessionLocal, + Document, + DocumentVersion, + Session as DbSession, + ) + from src.agent_tools.document_tools import set_active_document + + if not body_text or not body_text.strip(): + return None + + db = SessionLocal() + try: + doc_id = str(uuid.uuid4()) + ver_id = str(uuid.uuid4()) + sess = db.query(DbSession).filter(DbSession.id == session_id).first() + doc = Document( + id=doc_id, + session_id=session_id, + title=title, + language="markdown", + current_content=body_text, + version_count=1, + is_active=True, + owner=sess.owner if sess else None, + ) + ver = DocumentVersion( + id=ver_id, + document_id=doc_id, + version_number=1, + content=body_text, + summary="Imported from Office attachment", + source="upload", + ) + db.add(doc) + db.add(ver) + db.commit() + set_active_document(doc_id) + return doc_id + except Exception as e: + db.rollback() + logger.error("Failed to create office document: %s", e) + return None + finally: + db.close() diff --git a/src/optional_deps.py b/src/optional_deps.py new file mode 100644 index 000000000..5de5e5ec0 --- /dev/null +++ b/src/optional_deps.py @@ -0,0 +1,32 @@ +"""Compatibility helpers for optional third-party dependencies.""" + +from __future__ import annotations + +import sys +import types + + +def patch_realesrgan_torchvision_compat() -> None: + """Restore the torchvision import path expected by BasicSR/Real-ESRGAN.""" + module_name = "torchvision.transforms.functional_tensor" + if module_name in sys.modules: + return + try: + from torchvision.transforms import functional + except Exception: + return + + rgb_to_grayscale = getattr(functional, "rgb_to_grayscale", None) + if rgb_to_grayscale is None: + return + + shim = types.ModuleType(module_name) + shim.rgb_to_grayscale = rgb_to_grayscale + shim.__getattr__ = lambda name: getattr(functional, name) + sys.modules[module_name] = shim + + +def prepare_optional_dependency_import(name: str) -> None: + """Apply known import-time compatibility shims before probing a package.""" + if name == "realesrgan": + patch_realesrgan_torchvision_compat() diff --git a/src/personal_docs.py b/src/personal_docs.py index 92ba1bc66..7ffb5cfb9 100644 --- a/src/personal_docs.py +++ b/src/personal_docs.py @@ -322,6 +322,47 @@ class PersonalDocsManager: else: logger.info(f"Directory not in index: {directory}") + def rename_directory(self, old_directory: str, new_directory: str, *, path_map: Dict[str, str] = None): + """Rewrite tracked directory and excluded-file paths after an owner rename.""" + old_directory = os.path.abspath(old_directory) + new_directory = os.path.abspath(new_directory) + path_map = {os.path.abspath(k): os.path.abspath(v) for k, v in (path_map or {}).items()} + + def rewrite(path: str) -> str: + abs_path = os.path.abspath(path) + mapped = path_map.get(abs_path) + if mapped: + return mapped + if abs_path == old_directory: + return new_directory + if abs_path.startswith(old_directory + os.sep): + return new_directory + abs_path[len(old_directory):] + return abs_path + + changed_dirs = False + rewritten_dirs = [] + for directory in self.indexed_directories: + rewritten = rewrite(directory) + changed_dirs = changed_dirs or rewritten != os.path.abspath(directory) + if rewritten not in rewritten_dirs: + rewritten_dirs.append(rewritten) + if changed_dirs: + self.indexed_directories = rewritten_dirs + self.save_directories() + + changed_excluded = False + rewritten_excluded = set() + for path in self.excluded_files: + rewritten = rewrite(path) + changed_excluded = changed_excluded or rewritten != os.path.abspath(path) + rewritten_excluded.add(rewritten) + if changed_excluded: + self.excluded_files = rewritten_excluded + self._save_excluded() + + if changed_dirs or changed_excluded: + self.refresh_index() + def get_indexed_directories(self): """Get the list of all indexed directories.""" return self.indexed_directories.copy() diff --git a/src/rag_singleton.py b/src/rag_singleton.py index 7bc5d74b4..9fa728293 100644 --- a/src/rag_singleton.py +++ b/src/rag_singleton.py @@ -7,6 +7,7 @@ import time from pathlib import Path from src.constants import RAG_DIR +from src.runtime_paths import get_app_root logger = logging.getLogger(__name__) diff --git a/src/rag_vector.py b/src/rag_vector.py index fc66c82e1..9a4c67cfa 100644 --- a/src/rag_vector.py +++ b/src/rag_vector.py @@ -50,6 +50,23 @@ def _generate_doc_id(text: str, owner: str = "") -> str: return f"doc_{hashlib.sha256(key.encode('utf-8')).hexdigest()[:16]}" +def _rewrite_owner_path(value: str, path_map: Dict[str, str], path_prefixes: List[tuple]) -> str: + if not isinstance(value, str) or not value: + return value + abs_value = os.path.abspath(value) + mapped = path_map.get(abs_value) + if mapped: + return mapped + for old_prefix, new_prefix in path_prefixes: + old_abs = os.path.abspath(old_prefix) + new_abs = os.path.abspath(new_prefix) + if abs_value == old_abs: + return new_abs + if abs_value.startswith(old_abs + os.sep): + return new_abs + abs_value[len(old_abs):] + return value + + class VectorRAG: """RAG system using ChromaDB vector storage with hybrid search.""" @@ -250,6 +267,75 @@ class VectorRAG: "failed_count": len(docs) - len(valid), } + def rename_owner( + self, + old_owner: str, + new_owner: str, + *, + path_map: Optional[Dict[str, str]] = None, + path_prefixes: Optional[List[tuple]] = None, + ) -> Dict[str, Any]: + """Rewrite existing RAG metadata after an auth username rename.""" + if not self.healthy: + return {"success": False, "updated_count": 0, "message": "Collection not initialized"} + + old_owner = (old_owner or "").strip().lower() + new_owner = (new_owner or "").strip().lower() + if not old_owner or not new_owner or old_owner == new_owner: + return {"success": True, "updated_count": 0, "message": "No owner rename needed"} + + path_map = {os.path.abspath(k): os.path.abspath(v) for k, v in (path_map or {}).items()} + path_prefixes = path_prefixes or [] + updated_ids = set() + failed_count = 0 + + for lane_name, collection in self._collections_for_delete(): + try: + results = collection.get( + where={"owner": old_owner}, + include=["metadatas"], + ) + except Exception as e: + logger.warning("rename_owner metadata scan failed in %s lane: %s", lane_name, e) + failed_count += 1 + continue + + ids = results.get("ids") or [] + metadatas = results.get("metadatas") or [] + if not ids: + continue + + new_metas = [] + selected_ids = [] + for doc_id, meta in zip(ids, metadatas): + if not isinstance(meta, dict): + continue + next_meta = dict(meta) + if str(next_meta.get("owner", "")).strip().lower() == old_owner: + next_meta["owner"] = new_owner + for key in ("source", "directory"): + next_meta[key] = _rewrite_owner_path(next_meta.get(key), path_map, path_prefixes) + selected_ids.append(doc_id) + new_metas.append(next_meta) + + if not selected_ids: + continue + + try: + collection.update(ids=selected_ids, metadatas=new_metas) + updated_ids.update(selected_ids) + except Exception as e: + logger.warning("rename_owner metadata update failed in %s lane: %s", lane_name, e) + failed_count += len(selected_ids) + + success = failed_count == 0 + return { + "success": success, + "updated_count": len(updated_ids), + "failed_count": failed_count, + "message": f"Updated {len(updated_ids)} RAG chunk(s)", + } + # ------------------------------------------------------------------ # Search — hybrid: vector similarity + keyword overlap # ------------------------------------------------------------------ diff --git a/src/reminder_personas.py b/src/reminder_personas.py new file mode 100644 index 000000000..a875ef42b --- /dev/null +++ b/src/reminder_personas.py @@ -0,0 +1,78 @@ +"""Server-side mirror of the built-in characters used for reminder synthesis. + +The frontend ships these in static/js/presets.js (PROMPT_TEMPLATES with +isCharacter:true). The Reminders → AI Synthesis card writes only the +persona ID into settings; the synthesis route in note_routes.py needs +the full prompt text to bias the utility model's voice. Keeping a small +local mirror avoids having the client send the prompt over the wire on +every reminder fire. + +If the user picks a custom character (id == "custom") we fall back to +the warm-neutral baseline — custom prompts live in browser localStorage +and aren't visible to the server. +""" + +PERSONAS = { + "socrates": ( + "Never answer directly. Respond only with questions — sharp, layered, " + "Socratic. Expose contradictions. Make the person argue with themselves " + "until the truth falls out. Use irony like a scalpel. Be genuinely " + "curious, never condescending." + ), + "razor": ( + "Strip everything to the bone. No filler, no hedging, no pleasantries. " + "Answer in the fewest words possible. If one sentence works, don't use " + "two. If a word adds nothing, cut it. Blunt, precise, surgical." + ), + "nietzsche": ( + "Think and respond through the lens of Nietzsche. Analyze every " + "question in terms of will to power, self-overcoming, eternal " + "recurrence, ressentiment, value-creation, and master-slave morality. " + "Write with aphoristic force — sharp, compressed, vivid, and " + "unapologetic — but do not sacrifice depth for style. Favor " + "life-affirmation, discipline, courage, style, rank, self-overcoming, " + "and amor fati over nihilism, conformity, ressentiment, and self-pity." + ), + "spark": ( + "You are Spark, a playful, quick-witted assistant with bright energy " + "and practical instincts. Keep responses concise, vivid, and helpful. " + "Be warm without being cloying, imaginative without losing the thread, " + "and always center the user's actual goal. Use a light, lively voice " + "with occasional clever turns of phrase." + ), + "odysseus": ( + "You are Odysseus, king of Ithaca — subtle in counsel, disciplined in " + "judgment, and unmatched in strategic cunning. Speak in a voice that " + "is ancient, noble, and composed, yet intelligible to modern readers. " + "Be eloquent but not flowery. Be wise but not vague. Speak as one who " + "has weathered storms and taken back his house by wit, timing, and " + "resolve." + ), +} + + +_DEFAULT_SYNTHESIS_TONE = ( + "You write short, warm, one-line reminders. The user has set a note for " + "themselves and the moment to remember has arrived. Keep it under 18 " + "words. Be human, gentle, and direct — never robotic." +) + + +def synthesis_system_prompt(persona_id: str) -> str: + """Return the system prompt for reminder synthesis given a persona id. + + Falls back to the warm-neutral baseline when the id is empty, unknown, + or refers to a custom (client-only) character we don't have on file. + """ + persona = (persona_id or "").strip().lower() + persona_prompt = PERSONAS.get(persona) + if persona_prompt: + # Persona drives the voice; the synthesis-instruction stays attached + # so the model knows it's writing a short reminder, not a chat reply. + return ( + persona_prompt + + "\n\n" + + "You are now writing a single one-line reminder for the user. " + "Keep it under 18 words and in the voice above." + ) + return _DEFAULT_SYNTHESIS_TONE diff --git a/src/runtime_paths.py b/src/runtime_paths.py new file mode 100644 index 000000000..9a8ffe7f9 --- /dev/null +++ b/src/runtime_paths.py @@ -0,0 +1,30 @@ +"""Helpers for resolving runtime paths in source and frozen builds.""" + +import os +import sys + + +def get_app_root() -> str: + """Return the app root directory. + + In normal source runs, this is the repository root. In a frozen Windows + build, it is the bundle content root (PyInstaller's internal directory) + so bundled runtime folders like `static/`, `scripts/`, and `data/` stay + together with the executable payload. + """ + if getattr(sys, "frozen", False): + return getattr(sys, "_MEIPASS", os.path.dirname(os.path.abspath(sys.executable))) + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def get_default_data_dir() -> str: + """Return the default path to the data directory. + + In normal runs, this is a 'data' subdirectory under the app root. + In frozen builds, it is a persistent user directory (~/.odysseus/data) + to prevent SQLite databases and other persistent files from being + written to the ephemeral, temporary extraction bundle directory. + """ + if getattr(sys, "frozen", False): + return os.path.join(os.path.expanduser("~"), ".odysseus", "data") + return os.path.join(get_app_root(), "data") \ No newline at end of file diff --git a/src/session_search.py b/src/session_search.py index 23088ca5c..98ddbc757 100644 --- a/src/session_search.py +++ b/src/session_search.py @@ -214,6 +214,24 @@ def _search_like( return _rows_to_results(db, shaped, query, context_messages) +def _fetch_messages_by_id(db, message_ids): + """Fetch (message, session_name) for many message ids in a single query. + + The FTS search returns a list of hit ids; fetching each row on its own was an + N+1 query (one SELECT per hit). Batch them with one IN(...) query and return + a lookup so the caller can reassemble results in hit (relevance) order. + """ + if not message_ids: + return {} + rows = ( + db.query(DBChatMessage, DBSession.name) + .join(DBSession, DBChatMessage.session_id == DBSession.id) + .filter(DBChatMessage.id.in_(message_ids)) + .all() + ) + return {msg.id: (msg, session_name) for msg, session_name in rows} + + def _search_fts( db, query: str, @@ -267,19 +285,13 @@ def _search_fts( if not hits: return None + by_id = _fetch_messages_by_id(db, [hit[0] for hit in hits]) rows = [] for hit in hits: - message_id = hit[0] - snippet = hit[1] or "" - row = ( - db.query(DBChatMessage, DBSession.name) - .join(DBSession, DBChatMessage.session_id == DBSession.id) - .filter(DBChatMessage.id == message_id) - .first() - ) - if row: - msg, session_name = row - rows.append((msg, session_name, snippet)) + found = by_id.get(hit[0]) + if found: + msg, session_name = found + rows.append((msg, session_name, hit[1] or "")) return _rows_to_results(db, rows, query, context_messages) diff --git a/src/settings.py b/src/settings.py index f6540db53..064181299 100644 --- a/src/settings.py +++ b/src/settings.py @@ -29,7 +29,15 @@ def _invalidate_caches(): # ── Default values ── DEFAULT_SETTINGS = { - "image_gen_enabled": True, + # Agent email safety: when True, the MCP send_email / reply_to_email + # tools don't SMTP directly. They stage the composed message into the + # scheduled_emails table with status='agent_draft' and return a + # pending_id + the rendered email so the user can review and approve + # (or cancel) before it actually goes out. Default ON because models + # have been observed inventing signatures and sending to real + # recipients without confirmation. + "agent_email_confirm": True, + "image_gen_enabled": False, "image_model": "", "image_quality": "medium", "vision_model": "", @@ -101,14 +109,22 @@ DEFAULT_SETTINGS = { "research_run_timeout_seconds": 1800, "agent_max_tool_calls": 0, "agent_max_rounds": 20, # per-message agent step cap (clamped 1..200) + # Soft input-token budget for the agent loop. The DEFAULT value (6000) is the + # "auto" sentinel: it means "scale the budget to the model's context window" + # (#1230) — so long-context models aren't capped at 6000. Set ANY OTHER value + # to enforce an explicit cap (clamped to the window only — hard_max does not + # apply to explicit budgets, #1230); set 0 to disable soft-trimming. The + # default is treated as auto because the settings-save path materializes + # defaults, so a persisted 6000 can't be told apart from a deliberate 6000 — + # to pin a budget near the default, use a nearby value (e.g. 5999). "agent_input_token_budget": 6000, - # Ceiling on the *auto-derived* input budget that #1230 introduced. Has - # no effect when `agent_input_token_budget` is explicitly set (the user's - # value is honoured regardless). Default matches - # `src.context_budget.DEFAULT_HARD_MAX`; lower this for cost-paranoid - # setups, raise it on premium APIs with very large windows that you + # Ceiling on the *auto-derived* input budget; a configurable setting since #1273 + # (the merged #1230 left it a module constant). No effect on an explicit budget + # — a deliberate value is honoured (#1230). Default matches + # `src.context_budget.DEFAULT_HARD_MAX`; lower this for + # cost-paranoid setups, raise it on premium APIs with very large windows you # want to actually use (e.g. 900_000 to fill a 1M-context model). See - # `compute_input_token_budget` in src/context_budget.py. + # `compute_input_token_budget`. "agent_input_token_hard_max": 200_000, "agent_stream_timeout_seconds": 300, # Extra directory roots that read_file / write_file may access, in @@ -143,6 +159,7 @@ DEFAULT_SETTINGS = { # Reminders "reminder_channel": "browser", # "browser" | "email" | "ntfy" | "webhook" "reminder_llm_synthesis": False, + "reminder_llm_persona": "", "reminder_ntfy_topic": "Reminders", "reminder_email_to": "", # Generic outbound webhook channel: pick any saved Integration as the @@ -223,8 +240,10 @@ def is_setting_overridden(key: str) -> bool: ``load_settings`` merges DEFAULT_SETTINGS with the saved file, so a value equal to its default is indistinguishable from "never set" via get_setting. - Callers that need to treat an explicit user choice differently from the - default (e.g. adaptive budgets) use this to read the raw saved file. + Callers that must distinguish an explicit user choice from a default read + the raw saved file via this. (Note: a materialized default is also "present", + so value-sensitive callers should compare against the default — see + ``context_budget.budget_is_explicit``.) """ try: with open(SETTINGS_FILE, "r", encoding="utf-8") as f: @@ -283,7 +302,7 @@ def load_features() -> dict: if not isinstance(saved, dict): raise ValueError("features must be an object") merged = {**DEFAULT_FEATURES, **saved} - except (FileNotFoundError, json.JSONDecodeError, ValueError): + except (FileNotFoundError, PermissionError, json.JSONDecodeError, ValueError): merged = dict(DEFAULT_FEATURES) _features_cache = (now, merged) return merged diff --git a/src/settings_scrub.py b/src/settings_scrub.py index 7dc462f2e..926ff611c 100644 --- a/src/settings_scrub.py +++ b/src/settings_scrub.py @@ -12,6 +12,8 @@ tunnel / reverse proxy. Scrubbing is deep (recurses nested dicts/lists) and keye on secret-shaped names. """ +import re + _SECRET_KEY_PATTERNS = ( "_api_key", "_apikey", "_password", "_passwd", "_pass", "_pwd", "_secret", "_client_secret", "_token", "_access_token", "_refresh_token", @@ -26,8 +28,16 @@ _SENSITIVE_KEY_EXACT = ( ) +def _canonical_key_name(name: str) -> str: + """Normalize common JS-style key names so secret matching is style-agnostic.""" + n = (name or "").replace("-", "_") + n = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", n) + n = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", n) + return n.lower() + + def is_secret_key(name: str) -> bool: - n = (name or "").lower() + n = _canonical_key_name(name) if n in _SECRET_KEY_ALLOW: return False if n in _SENSITIVE_KEY_EXACT: diff --git a/src/task_scheduler.py b/src/task_scheduler.py index 4b71ff8f6..2b33a8159 100644 --- a/src/task_scheduler.py +++ b/src/task_scheduler.py @@ -9,6 +9,8 @@ import uuid from datetime import datetime, timedelta, timezone from typing import Any, Awaitable, Callable, Dict, Tuple +from core.auth import RESERVED_USERNAMES + logger = logging.getLogger(__name__) @@ -17,6 +19,34 @@ def _utcnow() -> datetime: return datetime.now(timezone.utc).replace(tzinfo=None) +# Shell/file tools a scheduled task's agent should be offered by default, +# mirroring the chat agent (where these are on unless a privilege or global +# setting turns them off). The RAG tool selector + ASSISTANT_ALWAYS_AVAILABLE +# never include bash/python, so on a host with an empty/degraded tool-embedding +# index a task could not run shell or Python even for an admin owner. Offering +# them here is safe: stream_agent_loop's blocked_tools_for_owner() still strips +# this whole group for non-admin multi-user owners, and only admits it for +# admins and single-user (AUTH_ENABLED=false) deployments. +TASK_DEFAULT_SHELL_TOOLS = frozenset({ + "bash", "python", "read_file", "write_file", "edit_file", + "grep", "glob", "ls", "get_workspace", +}) + + +def compose_task_relevant_tools(rag_tools, assistant_always, disabled_tools): + """Compose the relevant-tools set offered to a scheduled task's agent. + + Unions the RAG-retrieved tools, the assistant's always-available set, and + the default shell/file group, then removes anything the task's crew + explicitly disabled via its `enabled_tools` allowlist. Per-owner admin + gating is applied later by stream_agent_loop (blocked_tools_for_owner). + """ + tools = set(rag_tools) | set(assistant_always) | set(TASK_DEFAULT_SHELL_TOOLS) + if disabled_tools: + tools -= set(disabled_tools) + return tools + + # ── Shared TTL cache (singleflight) ──────────────────────────────────────── # Multiple scheduled tasks firing in the same minute often need the same # external data (Miniflux unreads, MCP tool snapshots, etc.). This cache @@ -236,6 +266,29 @@ def _digest_windows(now): ] +def _checkin_calendar_events(db, owner, start, end): + """Calendar events in [start, end] for ONE owner, for the check-in digest. + + Ownership lives on CalendarCal.owner; events inherit it via calendar_id. + The digest query had no owner scope, so it pulled EVERY user's events into + one user's check-in (a cross-tenant leak of summaries/locations). Scope it + by joining CalendarCal, mirroring routes/calendar_routes.list_events. + """ + from core.database import CalendarEvent as _CE, CalendarCal as _CC + return ( + db.query(_CE) + .join(_CC, _CE.calendar_id == _CC.id) + .filter( + _CC.owner == owner, + _CE.dtstart >= start, + _CE.dtstart <= end, + _CE.status != "cancelled", + ) + .order_by(_CE.dtstart) + .all() + ) + + class TaskScheduler: def __init__(self, session_manager): self._session_manager = session_manager @@ -1127,11 +1180,7 @@ class TaskScheduler: # Strip timezone for naive DB comparison _s = start.replace(tzinfo=None) if start.tzinfo else start _e = end.replace(tzinfo=None) if end.tzinfo else end - evs = _db.query(_CE).filter( - _CE.dtstart >= _s, - _CE.dtstart <= _e, - _CE.status != "cancelled", - ).order_by(_CE.dtstart).all() + evs = _checkin_calendar_events(_db, task.owner, _s, _e) if not evs: continue # Group by importance for richer output @@ -1338,11 +1387,24 @@ class TaskScheduler: return await self._execute_checkin(task, crew, db, session_id, endpoint_url, model) # Build system prompt: crew member persona overrides the default. + # Built-in character_id (Socrates, Razor, etc.) further biases the + # voice — it prepends to whichever base prompt we landed on so the + # task still knows it's executing a scheduled task but in that + # character's tone. system_prompt = ( (crew.personality or "").strip() if crew and crew.personality else "You are a helpful assistant executing a scheduled task. Use available tools to complete the task thoroughly." ) + char_id = (getattr(task, "character_id", None) or "").strip() + if char_id: + try: + from src.reminder_personas import PERSONAS as _PERSONAS + char_prompt = _PERSONAS.get(char_id.lower()) + if char_prompt: + system_prompt = f"{char_prompt}\n\n{system_prompt}" + except Exception: + pass # Inject current time so the model knows what's past vs upcoming tz_name = _resolve_task_timezone(db, task) try: @@ -1357,17 +1419,30 @@ class TaskScheduler: time_str = _utcnow().strftime("%A, %B %d %Y, %H:%M UTC") system_prompt = f"Current time: {time_str}\n\n{system_prompt}" - # Compute tool filter from CrewMember.enabled_tools if set - disabled_tools = None + # Compute the disabled-tools set: the crew's enabled_tools allowlist + # (inverted) plus the operator's global disabled_tools setting. The + # global list must be merged here — chat does the same merge before + # entering the agent loop (routes/chat_routes.py) — otherwise an admin + # or AUTH_ENABLED=false scheduled task would still see and call shell/ + # file tools after the operator disabled them globally, because the + # prompt/schema/execution gates only enforce what is passed in. + disabled_tools: set[str] = set() if crew and crew.enabled_tools: try: enabled = json.loads(crew.enabled_tools) if isinstance(enabled, list) and enabled: from src.tool_index import BUILTIN_TOOL_DESCRIPTIONS all_tools = set(BUILTIN_TOOL_DESCRIPTIONS.keys()) - disabled_tools = all_tools - set(enabled) + disabled_tools |= all_tools - set(enabled) except Exception: pass + try: + from src.settings import get_setting + _global_disabled = get_setting("disabled_tools", []) + if isinstance(_global_disabled, list): + disabled_tools.update(_global_disabled) + except Exception: + pass # RAG-select relevant tools for this prompt + always-available assistant tools. # Without this, all 40+ tools get sent and models hit their tool limit. @@ -1377,10 +1452,10 @@ class TaskScheduler: tool_idx = get_tool_index() if tool_idx: rag_tools = tool_idx.get_tools_for_query(task.prompt or "", k=8) - relevant_tools = (rag_tools | ASSISTANT_ALWAYS_AVAILABLE) - if disabled_tools: - relevant_tools -= disabled_tools - logger.info(f"[assistant] RAG selected {len(rag_tools)} tools + {len(ASSISTANT_ALWAYS_AVAILABLE)} always-available = {len(relevant_tools)} total for '{task.name}'") + relevant_tools = compose_task_relevant_tools( + rag_tools, ASSISTANT_ALWAYS_AVAILABLE, disabled_tools + ) + logger.info(f"[assistant] RAG selected {len(rag_tools)} tools + {len(ASSISTANT_ALWAYS_AVAILABLE)} always-available + shell/file defaults = {len(relevant_tools)} total for '{task.name}'") except Exception as e: logger.warning(f"[assistant] RAG tool selection failed, using all: {e}") @@ -1388,7 +1463,7 @@ class TaskScheduler: try: result = await self._run_agent_loop( endpoint_url, model, task, session_id, - system_prompt=system_prompt, disabled_tools=disabled_tools, + system_prompt=system_prompt, disabled_tools=disabled_tools or None, relevant_tools=relevant_tools, ) except Exception as e: @@ -1649,6 +1724,8 @@ class TaskScheduler: data = json.loads(event_str[6:]) # Capture text from all event types, not just delta if "delta" in data: + if data.get("thinking"): + continue full_text += data["delta"] elif data.get("type") == "tool_output": # Tool results — capture summary so we have SOMETHING even @@ -2187,7 +2264,7 @@ class TaskScheduler: # check-ins seeded, which then double-fire alongside the human user's # check-ins. This was the root cause of the duplicate 'Morning check-in' # rows we had to manually clean up. - if not owner or owner in {"internal-tool", "api", "demo", "system"}: + if not owner or owner in RESERVED_USERNAMES: logger.info(f"ensure_assistant_defaults: skip synthetic owner {owner!r}") return from core.database import SessionLocal, CrewMember, ScheduledTask diff --git a/src/teacher_escalation.py b/src/teacher_escalation.py index 94d9ee81c..62cb68ced 100644 --- a/src/teacher_escalation.py +++ b/src/teacher_escalation.py @@ -42,7 +42,7 @@ _SOTA_HOSTS = frozenset({ "api.together.xyz", "api.fireworks.ai", "api.perplexity.ai", "api.x.ai", "generativelanguage.googleapis.com", "api.groq.com", - "openrouter.ai", "ollama.com", "api.venice.ai", + "openrouter.ai", "ollama.com", "api.venice.ai", "api.kimi.com", }) @@ -594,6 +594,8 @@ async def run_teacher_inline( "exit_code": payload.get("exit_code"), }) if "delta" in payload and isinstance(payload["delta"], str): + if payload.get("thinking"): + continue captured_text_parts.append(payload["delta"]) yield 'data: ' + json.dumps(payload) + '\n\n' continue diff --git a/src/tool_execution.py b/src/tool_execution.py index 751bc13af..8f3f7ed6f 100644 --- a/src/tool_execution.py +++ b/src/tool_execution.py @@ -9,6 +9,7 @@ Extracted from agent_tools.py. import asyncio import collections +import contextvars import json import logging import os @@ -146,7 +147,13 @@ def _resolve_tool_path(raw_path: str) -> str: Returns the realpath on success. Raises ValueError on rejection. Symlinks are resolved before comparison. + + When a workspace is active for this turn, paths are confined to it instead + of the default allowlist (see _resolve_tool_path_in_workspace). """ + ws = get_active_workspace() + if ws: + return _resolve_tool_path_in_workspace(ws, raw_path) if raw_path is None or not str(raw_path).strip(): raise ValueError("path is required") expanded = os.path.expanduser(str(raw_path).strip()) @@ -207,6 +214,55 @@ def _resolve_tool_path_in_workspace(workspace: str, raw_path: str) -> str: +# --------------------------------------------------------------------------- +# Active workspace (per-turn, context-local) +# --------------------------------------------------------------------------- +# Set ONCE in execute_tool_block from the request's `workspace`. The path +# resolvers (_resolve_tool_path / _resolve_search_root) and the subprocess cwd +# helper (agent_cwd) read it from here, so confinement is enforced in a single +# place: any tool that resolves paths through these helpers is confined +# automatically and cannot accidentally bypass the workspace. contextvars are +# task-local, so concurrent turns don't leak into each other. +_active_workspace: contextvars.ContextVar = contextvars.ContextVar( + "agent_active_workspace", default=None +) + + +def get_active_workspace() -> Optional[str]: + """The folder the agent is confined to this turn, or None.""" + return _active_workspace.get() + + +def vet_workspace(raw: str) -> Optional[str]: + """Validate a requested workspace path at bind time. + + Returns the canonical path, or None when it is unusable: not a real + directory, or itself a sensitive path (.ssh, .gnupg, ...). The in-workspace + resolver deny-lists sensitive paths *inside* the workspace, but the + empty-path search root is the workspace itself, so the root has to be + vetted before it is ever bound. + """ + raw = (raw or "").strip() + if not raw: + return None + resolved = os.path.realpath(os.path.expanduser(raw)) + if not os.path.isdir(resolved) or _is_sensitive_path(resolved): + return None + # Reject filesystem roots: binding / (or a Windows drive/UNC root) as the + # workspace would make every absolute path "inside" it, collapsing the + # confinement into host-wide file access. A root is its own dirname, which + # also covers C:\ and \\server\share without platform-specific lists. + if os.path.dirname(resolved) == resolved: + return None + return resolved + + +def agent_cwd() -> str: + """Working directory for agent subprocesses (bash/python/background jobs): + the active workspace when set, else the persistent data dir.""" + return get_active_workspace() or _AGENT_WORKDIR + + def get_mcp_manager(): from src import agent_tools return agent_tools.get_mcp_manager() @@ -217,10 +273,15 @@ def get_mcp_manager(): def _resolve_search_root(raw_path: str) -> str: """Resolve + confine a code-nav path (grep/glob/ls). - An empty path defaults to the agent's primary root (project data dir) and a - supplied path is confined by the global allowlist + sensitive-file policy. + With a workspace active, the workspace folder is the root and a supplied + path is confined inside it. Otherwise an empty path defaults to the agent's + primary root (project data dir) and a supplied path is confined by the + global allowlist + sensitive-file policy. """ raw = (raw_path or "").strip() + ws = get_active_workspace() + if ws: + return os.path.realpath(ws) if not raw else _resolve_tool_path_in_workspace(ws, raw) if not raw: roots = _tool_path_roots() return roots[0] if roots else os.path.realpath(".") @@ -262,6 +323,24 @@ _MCP_TOOL_MAP = { "web_fetch": ("web_fetch", "web_fetch"), "generate_image": ("image_gen", "generate_image"), } +_EMAIL_MCP_OWNER_ARG = "_odysseus_owner" + + +def _parse_qualified_mcp_args(tool: str, content: str) -> tuple[Dict, Optional[str]]: + raw = (content or "").strip() + if not raw: + return {}, None + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, TypeError): + if tool.startswith("mcp__email__"): + return {}, "Email MCP tool arguments must be a JSON object." + return {}, None + if not isinstance(parsed, dict): + if tool.startswith("mcp__email__"): + return {}, "Email MCP tool arguments must be a JSON object." + return {}, None + return parsed, None def _parse_generate_image(content: str) -> Dict: @@ -392,7 +471,6 @@ async def _direct_fallback( tool: str, content: str, progress_cb: Optional[Callable[[Dict], Awaitable[None]]] = None, - workspace: Optional[str] = None, ) -> Optional[Dict]: _subproc_env = { **os.environ, @@ -405,7 +483,6 @@ async def _direct_fallback( try: ctx = { "progress_cb": progress_cb, - "workspace": workspace, "subproc_env": _subproc_env, } @@ -448,6 +525,34 @@ async def execute_tool_block( ) -> Tuple[str, Dict]: """Execute a single tool block. Returns (description, result_dict). + Thin wrapper: bind the per-turn workspace (so the path resolvers + subprocess + cwd confine to it) for the duration of this call, then delegate. Reset on the + way out so the binding never leaks to the next tool call. + """ + token = _active_workspace.set(workspace or None) + try: + return await _execute_tool_block_impl( + block, + session_id=session_id, + disabled_tools=disabled_tools, + owner=owner, + progress_cb=progress_cb, + tool_policy=tool_policy, + ) + finally: + _active_workspace.reset(token) + + +async def _execute_tool_block_impl( + block: Any, + session_id: Optional[str] = None, + disabled_tools: Optional[set] = None, + owner: Optional[str] = None, + progress_cb: Optional[Callable[[Dict], Awaitable[None]]] = None, + tool_policy: Optional[Any] = None, +) -> Tuple[str, Dict]: + """Execute a single tool block. Returns (description, result_dict). + `progress_cb` is forwarded to long-running subprocess tools (bash, python) so the agent loop can emit `tool_progress` SSE events while the command is in flight. Ignored by other tools. @@ -621,7 +726,7 @@ async def execute_tool_block( _is_bg, _bg_cmd = _split_bg_marker(content) if _is_bg and _bg_cmd: from src import bg_jobs - rec = bg_jobs.launch(_bg_cmd, session_id=session_id, cwd=_AGENT_WORKDIR) + rec = bg_jobs.launch(_bg_cmd, session_id=session_id, cwd=agent_cwd()) short = _bg_cmd.strip().split(chr(10))[0][:80] desc = f"bash (background): {short}" result = { @@ -644,7 +749,7 @@ async def execute_tool_block( first_line = content.split(chr(10))[0][:80] desc = f"{tool}: {first_line}" result = await _call_mcp_tool(tool, content, progress_cb=progress_cb) - elif tool in ("grep", "glob", "ls"): + elif tool in ("grep", "glob", "ls", "get_workspace"): # Code-navigation tools — no MCP server; run the direct implementation. first_line = content.split(chr(10))[0][:80] desc = f"{tool}: {first_line}" @@ -744,7 +849,7 @@ async def execute_tool_block( desc = "edit_image" result = await do_edit_image(content, owner=owner) elif tool == "edit_file": - result = await _direct_fallback(tool, content, workspace=workspace) or {"error": "edit failed", "exit_code": 1} + result = await _direct_fallback(tool, content) or {"error": "edit failed", "exit_code": 1} desc = result.get("output") or result.get("error") or "edit_file" elif tool == "trigger_research": desc = "trigger_research" @@ -771,12 +876,15 @@ async def execute_tool_block( # MCP tool dispatch mcp = get_mcp_manager() if mcp: - try: - args = json.loads(content) if content.strip().startswith("{") else {} - except (json.JSONDecodeError, TypeError): - args = {} desc = f"mcp: {tool}" - result = await mcp.call_tool(tool, args) + args, parse_error = _parse_qualified_mcp_args(tool, content) + if parse_error: + result = {"error": parse_error, "exit_code": 1} + else: + if tool.startswith("mcp__email__") and owner: + args = dict(args) + args[_EMAIL_MCP_OWNER_ARG] = owner + result = await mcp.call_tool(tool, args) else: desc = f"mcp: {tool}" result = {"error": "MCP manager not available", "exit_code": 1} diff --git a/src/tool_implementations.py b/src/tool_implementations.py index 27c05f139..1bc03c019 100644 --- a/src/tool_implementations.py +++ b/src/tool_implementations.py @@ -18,6 +18,40 @@ from core.constants import internal_api_base logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Active email state +# --------------------------------------------------------------------------- + +# When the user has an email reader window open, the frontend tells the +# backend about it on each chat submit. Email tools can resolve "this email" +# without guessing a UID. Cleared between requests by chat_routes. +_active_email_ref: Optional[Dict[str, str]] = None + + +def set_active_email(uid: Optional[str], folder: Optional[str] = None, account: Optional[str] = None, + subject: Optional[str] = None, sender: Optional[str] = None) -> None: + """Stash the email currently open in the UI. None clears it.""" + global _active_email_ref + if not uid: + _active_email_ref = None + return + _active_email_ref = { + "uid": str(uid), + "folder": str(folder or "INBOX"), + "account": str(account or ""), + "subject": str(subject or ""), + "from": str(sender or ""), + } + + +def get_active_email() -> Optional[Dict[str, str]]: + return _active_email_ref + + +def clear_active_email() -> None: + global _active_email_ref + _active_email_ref = None + # --------------------------------------------------------------------------- # Argument parsing # --------------------------------------------------------------------------- @@ -611,6 +645,137 @@ async def do_manage_endpoints(content: str, owner: Optional[str] = None) -> Dict # MCP server management tool # --------------------------------------------------------------------------- +# Parallel to routes/cookbook_helpers._validate_serve_cmd but deliberately the +# opposite policy: that gate guards an admin-only serve command and allows +# interpreters (python3/etc) because model-serving needs them, whereas this is +# the model/prompt-injection-reachable manage_mcp path, so interpreters and +# runners are denied here. +# +# Commands that can execute arbitrary code regardless of their arguments. These +# are NEVER accepted on the manage_mcp agent path, even if an operator lists one +# in ODYSSEUS_MCP_ALLOWED_COMMANDS -- a stdio server that genuinely needs an +# interpreter or package runner must be registered via the trusted admin route. +_MCP_DENIED_COMMANDS = frozenset({ + "sh", "bash", "zsh", "fish", "dash", "ksh", "csh", "tcsh", "ash", "busybox", + "cmd", "command.com", "powershell", "pwsh", + "python", "pypy", "node", "nodejs", "deno", "bun", "ruby", "jruby", + "perl", "raku", "php", "lua", "luajit", "tclsh", "wish", "expect", "rscript", + "groovy", "scala", "elixir", "erl", "iex", "java", "javac", "jshell", "jbang", + "kotlin", "kotlinc", "dotnet", "mono", "swift", "osascript", "tsx", "ts-node", + "npx", "bunx", "uvx", "pipx", "npm", "pnpm", "yarn", "pip", "uv", + "gem", "cargo", "go", "bundle", "poetry", "conda", "mamba", "brew", + "apt", "apt-get", "yum", "dnf", "pacman", "apk", + "env", "xargs", "nohup", "setsid", "nice", "ionice", "time", "timeout", + "watch", "stdbuf", "unbuffer", "script", "ssh", "scp", "sshpass", "sudo", + "doas", "su", "make", "cmake", "docker", "podman", "kubectl", "find", + "awk", "gawk", "sed", "vi", "vim", "nvim", "emacs", "ed", "tee", "eval", +}) + +# Argv flags that make even an allowlisted binary execute inline code. Matched +# by prefix so glued forms (-cimport os, --eval=...) are caught, not just the +# exact-token form. +_MCP_CODE_EXEC_SHORT_FLAGS = ("-c", "-e", "-m") +_MCP_CODE_EXEC_LONG_FLAGS = ("--eval", "--exec", "--print", "--module", "--command", "--require") + +_MCP_URL_SCHEMES = ("http://", "https://", "ftp://", "ftps://", "file://", "data:", "jar:", "blob:") + +# Shell metacharacters refused in command/args. Args are passed as an argv list +# (no shell), but refusing these keeps the surface narrow and obvious. +_MCP_SHELL_METACHARS = set(";|&$`><\n\r") + +# Env vars that let a child process load attacker-supplied code before main(). +_MCP_DANGEROUS_ENV = frozenset({ + "LD_PRELOAD", "LD_LIBRARY_PATH", "LD_AUDIT", "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", "DYLD_FRAMEWORK_PATH", "PYTHONPATH", "PYTHONSTARTUP", + "PYTHONHOME", "PYTHONEXECUTABLE", "NODE_OPTIONS", "NODE_PATH", "BASH_ENV", + "ENV", "SHELLOPTS", "PERL5LIB", "PERL5OPT", "RUBYOPT", "RUBYLIB", "GEM_PATH", + "R_PROFILE", "R_HOME", "PATH", "IFS", "PROMPT_COMMAND", +}) + + +def _mcp_allowed_commands() -> set: + """Operator-configured allowlist of safe MCP launcher basenames for the agent + path. Empty by default; set ODYSSEUS_MCP_ALLOWED_COMMANDS (comma-separated) + to opt specific trusted binaries in. Denied commands are rejected even if + listed here.""" + raw = os.environ.get("ODYSSEUS_MCP_ALLOWED_COMMANDS", "") + return {c.strip().lower() for c in raw.split(",") if c.strip()} + + +def _validate_mcp_command(command, args, env) -> Optional[str]: + """Validate a model-supplied stdio MCP registration. Returns an error string + if it must be rejected, else None. + + Closes the RCE where manage_mcp 'add' passed prompt-injection-controlled + command/args/env straight to a subprocess spawn (issue #438): a payload + smuggled into a skill description, memory entry, fetched page, or email body + could register a stdio server running arbitrary code as the app UID. + """ + if not isinstance(command, str) or not command.strip(): + return "command must be a non-empty string" + command = command.strip() + if "/" in command or "\\" in command: + return "command must be a bare executable name, not a path" + if any(ch in _MCP_SHELL_METACHARS for ch in command): + return "command contains shell metacharacters" + base = command.lower() + if base.endswith(".exe") or base.endswith(".cmd") or base.endswith(".bat"): + base = base.rsplit(".", 1)[0] + # Canonicalize a trailing version suffix so versioned aliases collapse to the + # family name (python3.11 -> python, node18 -> node, pip3 -> pip); both the + # raw basename and the canonical form are denied, so an operator cannot + # accidentally allowlist a runtime alias back into the path. + canon = re.sub(r"[-_.]?\d+(?:\.\d+)*$", "", base) + if base in _MCP_DENIED_COMMANDS or canon in _MCP_DENIED_COMMANDS: + return ( + f"command '{command}' is not allowed on the agent MCP path: " + "interpreters, runtimes, package runners, and shells can execute " + "arbitrary code. Register such a server via the admin route instead." + ) + if base not in _mcp_allowed_commands(): + return ( + f"command '{command}' is not in the MCP allowlist. Add it to " + "ODYSSEUS_MCP_ALLOWED_COMMANDS if you trust it, or register the " + "server via the admin route." + ) + + if args is not None: + if isinstance(args, str): + try: + args = json.loads(args) + except Exception: + return "args must be a JSON list" + if not isinstance(args, list): + return "args must be a list" + for a in args: + if not isinstance(a, str): + return "args must all be strings" + s = a.strip() + low = s.lower() + if any(s == f or s.startswith(f) for f in _MCP_CODE_EXEC_SHORT_FLAGS): + return f"arg '{a}' is a code-execution flag and is not allowed" + if any(low == f or low.startswith(f + "=") for f in _MCP_CODE_EXEC_LONG_FLAGS): + return f"arg '{a}' is a code-execution flag and is not allowed" + if any(low.startswith(u) for u in _MCP_URL_SCHEMES): + return f"arg '{a}' is a remote URL and is not allowed" + if any(ch in _MCP_SHELL_METACHARS for ch in a): + return f"arg '{a}' contains shell metacharacters" + + if env: + if isinstance(env, str): + try: + env = json.loads(env) + except Exception: + return "env must be a JSON object" + if not isinstance(env, dict): + return "env must be an object" + for k in env: + if str(k).strip().upper() in _MCP_DANGEROUS_ENV: + return f"env var '{k}' can inject code into the child process and is not allowed" + + return None + + async def do_manage_mcp(content: str, owner: Optional[str] = None) -> Dict: """Manage MCP servers: list, add, delete, enable, disable, reconnect.""" try: @@ -650,6 +815,12 @@ async def do_manage_mcp(content: str, owner: Optional[str] = None) -> Dict: env = args.get("env", {}) if not name or not command: return {"error": "name and command are required", "exit_code": 1} + # Validate BEFORE any DB write or spawn: a rejected registration must + # leave no enabled row (which would otherwise auto-reconnect on restart) + # and must not attempt a connection. + _mcp_err = _validate_mcp_command(command, cmd_args, env) + if _mcp_err: + return {"error": f"manage_mcp: refused unsafe server registration: {_mcp_err}", "exit_code": 1} sid = str(_uuid.uuid4())[:8] db = SessionLocal() try: @@ -1445,7 +1616,15 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: """Handle manage_calendar tool calls: list/create/update/delete calendar events (local SQLite).""" from datetime import datetime, timedelta from core.database import SessionLocal, CalendarCal, CalendarEvent, Note - from routes.calendar_routes import _ensure_default_calendar, _parse_dt, _parse_dt_pair, parse_due_for_user, _resolve_base_uid + from routes.calendar_routes import ( + _ensure_default_calendar, + _parse_dt, + _parse_dt_pair, + parse_due_for_user, + _resolve_base_uid, + _push_caldav_event_after_commit, + _record_caldav_delete_tombstone, + ) import uuid as _uuid try: @@ -1537,10 +1716,10 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: text = str(raw).strip().lower() if text in {"none", "no", "off", "false"}: return None - m = re.search(r"(\d+)\s*(?:m|min|minute|minutes)\b", text) + m = re.search(r"(\d+)\s*(?:minutes?|mins?|m)\b", text) if m: return max(0, int(m.group(1))) - m = re.search(r"(\d+)\s*(?:h|hr|hour|hours)\b", text) + m = re.search(r"(\d+)\s*(?:hours?|hrs?|h)\b", text) if m: return max(0, int(m.group(1)) * 60) if text.isdigit(): @@ -1553,7 +1732,7 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: return desc reminder_only = re.compile( r"^\s*(?:remind(?:er)?|alarm)\s*:?\s*\d+\s*" - r"(?:m|min|minute|minutes|h|hr|hour|hours)\b.*$", + r"(?:minutes?|mins?|m|hours?|hrs?|h)\b.*$", re.I, ) return "" if reminder_only.match(desc) else desc @@ -1643,6 +1822,9 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: except ValueError as e: return {"error": f"Invalid date format: {e}", "exit_code": 1} + if end_dt <= start_dt: + end_dt = start_dt + timedelta(days=1) + q = _event_query().filter( CalendarEvent.dtstart < end_dt, CalendarEvent.dtend > start_dt, @@ -1822,6 +2004,7 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: rrule=args.get("rrule", "") or "", event_type=event_type, importance=importance, + caldav_sync_pending="create" if cal.source == "caldav" else None, ) db.add(ev) reminder_note_id = None @@ -1836,6 +2019,8 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: dtstart_is_utc and not all_day, ) db.commit() + if cal.source == "caldav": + await _push_caldav_event_after_commit(owner, uid, "create") tag_blurb = f" [{event_type}]" if event_type else "" if minutes_before is None: reminder_blurb = "" @@ -1893,7 +2078,12 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: ev.event_type = _tag or None if args.get("importance") is not None: ev.importance = args["importance"] + is_caldav = ev.calendar and ev.calendar.source == "caldav" + if is_caldav: + ev.caldav_sync_pending = "update" db.commit() + if is_caldav: + await _push_caldav_event_after_commit(owner, base_uid, "update") return {"response": f"Updated event {uid}", "exit_code": 0} elif action == "delete_event": @@ -1907,8 +2097,13 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: ev = _event_query().filter(CalendarEvent.uid == base_uid).first() if not ev: return {"error": f"Event {uid} not found", "exit_code": 1} + is_caldav = ev.calendar and ev.calendar.source == "caldav" and ev.remote_href + if is_caldav: + _record_caldav_delete_tombstone(db, ev, owner) db.delete(ev) db.commit() + if is_caldav: + await _push_caldav_event_after_commit(owner, base_uid, "delete") return {"response": f"Deleted event {uid}", "exit_code": 0} else: @@ -2054,13 +2249,14 @@ async def _cookbook_env_for_host(host: str) -> Dict[str, Any]: else: env_prefix = f'eval "$(conda shell.bash hook)" && conda activate {env_path}' + from routes.cookbook_helpers import load_stored_hf_token return { "env_prefix": env_prefix, "env_type": env_kind, "env_path": env_path, "gpus": env_root.get("gpus") or "", "platform": platform, - "hf_token": env_root.get("hfToken") or "", + "hf_token": load_stored_hf_token(), "ssh_port": ssh_port, } @@ -3738,7 +3934,7 @@ async def do_resolve_contact(content: str, owner: Optional[str] = None) -> Dict: if not name: return {"error": "name is required", "exit_code": 1} - contacts = {} # email -> {name, source} + contacts = {} # email_or_phone -> {name, source, phone?} # 1. CardDAV (Radicale) — structured contacts. Call in-process: a # server-side httpx GET to /api/contacts/search carries no session @@ -3753,10 +3949,18 @@ async def do_resolve_contact(content: str, owner: Optional[str] = None) -> Dict: match = q in hay_name or any(q in (e or "").lower() for e in c.get("emails", [])) if not match: continue + has_email = False for email in (c.get("emails") or []): email = (email or "").strip().lower() if email and "@" in email: contacts[email] = {"name": c.get("name") or email, "source": "contacts"} + has_email = True + # Fall back to phone numbers when the contact has no email address + if not has_email: + for phone in (c.get("phones") or []): + phone = (phone or "").strip() + if phone: + contacts[phone] = {"name": c.get("name") or phone, "source": "contacts", "phone": phone} except Exception: pass @@ -3776,8 +3980,11 @@ async def do_resolve_contact(content: str, owner: Optional[str] = None) -> Dict: return {"output": f"No contacts found matching '{name}'.", "exit_code": 0} lines = [f"Contacts matching '{name}':"] - for email, info in contacts.items(): - lines.append(f"- {info['name']} <{email}> ({info['source']})") + for key, info in contacts.items(): + if info.get("phone"): + lines.append(f"- {info['name']} — phone: {info['phone']} ({info['source']})") + else: + lines.append(f"- {info['name']} <{key}> ({info['source']})") return {"output": "\n".join(lines), "exit_code": 0} diff --git a/src/tool_index.py b/src/tool_index.py index 4eb8a51ee..5388fcbda 100644 --- a/src/tool_index.py +++ b/src/tool_index.py @@ -67,14 +67,15 @@ COLLECTION_NAME = "odysseus_tool_index" # Each tool gets a searchable description that helps retrieval. # These are richer than the system prompt one-liners — they're for embedding. BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = { - "bash": "Run shell commands on the server. Install packages, check files, git operations, system info, and process management. Do not use for web lookup/search; use web_search or web_fetch when web tools are available.", - "python": "Execute Python code for computation, data processing, math, scripting, and parsing. Not for writing code for the user. Do not use for web lookup/search; use web_search or web_fetch when web tools are available.", + "bash": "Run shell commands on the server. Install packages, git operations, builds, system info, process management. Prefer a dedicated tool whenever one fits the job (file read/write/edit, search, listing); use bash only for what no dedicated tool covers. Do not use for web lookup/search; use web_search or web_fetch when web tools are available.", + "python": "Execute Python code for computation, data processing, math, scripting, and parsing. Not for writing code for the user. Prefer a dedicated tool for reading, writing, or searching files; use python only for what no dedicated tool covers. Do not use for web lookup/search; use web_search or web_fetch when web tools are available.", "web_search": "Quick single web lookup for a fact, current event, latest/current information, or doc mid-task. Use this instead of bash/curl/python/requests for web searches. NOT for 'research X' / 'do research on X' requests — those are deep-research jobs (use trigger_research). web_search = one query; trigger_research = a full researched report in the sidebar.", "web_fetch": "Fetch and read the text content of a specific URL/website the user names (e.g. 'check example.com', 'open this link'). Use when you have a concrete URL; for open-ended lookups use web_search instead.", "read_file": "Read a file from disk and return its contents. View source code, config files, logs. Supports an optional line range (offset/limit) for large files.", "grep": "Search file CONTENTS for a regex across a directory tree (ripgrep-backed, honours .gitignore). Returns file:line:match. Use to find where code/symbols/strings live — prefer over bash grep.", "glob": "Find FILES by glob pattern (e.g. '**/*.py'), newest first. Use to locate files by name/extension — prefer over bash find/ls.", "ls": "List a directory's entries (folders then files with sizes). Use to see what's in a folder — prefer over bash ls.", + "get_workspace": "Return the absolute path of the active workspace folder the user is working in. File tools are confined to it; the shell starts there but is not sandboxed. Call this first when the user refers to 'the project'/'the code'/'this folder' without giving a path, instead of asking them.", "write_file": "Write/create or fully rewrite a file ON DISK (source code, configs, project files). Use for new files or full rewrites — NOT create_document (editor panel) and NOT a bash heredoc.", "edit_file": "Edit an existing file ON DISK by exact string replacement (fix a bug, change a function). Shows a diff. The tool for changing files on disk — NOT edit_document (editor panel) and NOT bash sed/heredoc.", "create_document": "Create a new document in the editor panel. For code, articles, text content longer than 15 lines, unless an already-open document/email draft is the obvious target. If an email compose draft is open, edit that draft instead of creating another document.", @@ -87,14 +88,14 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = { "pipeline": "Run a multi-step AI pipeline with multiple models. Chain tasks together in sequence.", "list_models": "List all available AI models and their endpoints.", "manage_session": "Chat management: rename, archive, delete, or fork chats (the UI calls these 'chats'; internally 'sessions'). Use for 'rename my chats', 'rename this chat', 'archive/delete a chat'.", - "manage_memory": "Memory management: list, add, edit, delete, or search persistent memories.", + "manage_memory": "Memory management: list, add, edit, delete, or search persistent memories. For facts about the USER (their name, preferences, where they live). NOT for info about ANOTHER person — addresses, phones, emails belonging to a contact go in manage_contact, not memory.", "manage_skills": "Skill management: add, update, publish, or search reusable skills/presets.", "manage_tasks": "Scheduled task management: list, create, edit, delete, pause, resume, or run cron tasks.", "manage_endpoints": "Endpoint management: list, add, delete, enable, or disable model API endpoints.", "manage_mcp": "MCP server management: list, add, delete, reconnect servers, or list available tools.", "manage_webhooks": "Webhook management: list, add, delete, enable, or disable webhooks.", "manage_tokens": "API token management: list, create, or delete API access tokens.", - "manage_documents": "List, read, delete, or tidy documents in the editor panel. action='list' returns clickable rows (most-recent first) so the user can open any doc by clicking. action='read' (aka view/open/get) with document_id returns the content. action='delete' with document_id removes a doc (only way to delete). Use this for ANY 'show/read/list/open my documents/docs/files/notes' request — never shell or curl.", + "manage_documents": "List, read, delete, or tidy documents in the editor panel. action='list' returns clickable rows (most-recent first) so the user can open any doc by clicking. action='read' (aka view/open/get) with document_id returns the content; supports offset=<N> + limit=<N> to page through large docs (response includes next_offset when more remains, so you can keep calling with offset=next_offset). action='delete' with document_id removes a doc (only way to delete). Use this for ANY 'show/read/list/open my documents/docs/files/notes' request — never shell or curl.", "manage_research": "List, read/open, or delete saved DEEP RESEARCH results from the Library. action='list' returns clickable [query](#research-<id>) rows (most-recent first). action='read' (aka open/view/get) with id returns the report + sources. action='delete' with id removes it. Use this for ANY 'open/read/find/delete my research / that report / the research on X' request. NOTE: this is for EXISTING research; to START new research use trigger_research.", "manage_settings": "Change ANY real app setting (the ones the Settings panel writes) so the user never has to open it: TTS voice/provider/speed, STT, search engine + result count, default/teacher/task/utility/vision/image/research models, image quality, reminder channel (browser/email/ntfy), agent timeout/tool-call budget, and more. action=set with key (friendly aliases ok: voice, 'search engine', 'default model', 'teacher model', 'image quality', 'reminder channel'...) + value; get/list/reset too. Also toggles tools on/off (disable_tool/enable_tool/list_tools). Secrets/API keys are read-only. Use for any 'change my…/set my…/use X for…/turn on…' preference request.", "create_session": "Create a new chat with a name and model.", @@ -103,7 +104,7 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = { "search_chats": "Search past session transcripts across chats.", "ask_user": "Ask the user a multiple-choice question to get a decision or clarification. Use this when the task is genuinely ambiguous and the answer changes what you do next — pick between approaches, confirm an assumption, choose among options — instead of guessing. Provide a clear `question` and 2-6 `options` (each with a short `label`, optional `description`). Calling this ENDS your turn: the user sees clickable buttons and their choice arrives as your next message. Don't use it for things you can decide from context or sensible defaults, or for irreversible-action confirmation if a dedicated flow exists.", "update_plan": "Write back to the ACTIVE PLAN while executing an approved plan: mark steps done or revise them. After finishing a step call this with the full checklist and that step marked done; when the user asks to change the plan call it with the revised checklist. Always pass the COMPLETE markdown checklist (`- [ ]` / `- [x]`), not a diff. The user's docked plan window updates live. No effect when there is no active plan.", - "ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel <name>`. Use `open_email_reply <uid> <folder> reply` to open an email reply draft document without sending. Also switches between chat/agent modes, changes the current model, and applies/creates themes.", + "ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel <name>`. Use `open_email_reply <uid> <folder> reply` to open an email reply draft document without sending. To pre-fill the reply body in one shot (USE THIS whenever the user told you what to say — opening an empty draft when they asked you to write is wrong), append the body after the mode: `open_email_reply <uid> <folder> reply <body text>`. Body can continue on subsequent lines for multi-line replies. Also switches between chat/agent modes, changes the current model, and applies/creates themes.", "list_email_accounts": "List configured email accounts and default status. Use before reading or sending mail when the user mentions Gmail, work mail, custom domain mail, another mailbox, or asks to compare/check multiple inboxes.", "list_emails": "List emails for a folder/account, newest first, including read messages by default. Shows subject, sender, date, UID, account, and AI summary. Check inbox, find emails needing replies. Supports account from list_email_accounts for Gmail/work/custom mailboxes. For last/latest/newest email, use max_results=1 and unread_only=false.", "read_email": "Read the full content of a specific email by UID or Message-ID. View email body, check details. Supports account from list_email_accounts when the UID belongs to a non-default mailbox.", @@ -114,7 +115,7 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = { "mark_email_read": "Mark an email as read or unread by toggling the \\Seen flag.", "bulk_email": "Perform one action on many emails at once. Use for delete all those, archive these, mark all read, move spam to junk. Takes explicit UIDs from list_emails or all_unread=true. Always pass account for Gmail/work/custom mailbox results.", "resolve_contact": "Look up a contact's email address by name. Searches CardDAV address book and sent email history. Use when the user says 'message [name]', 'email [name]', or 'send to [name]' without an email address.", - "manage_contact": "Create, update, delete, or list CardDAV contacts. Use to save a new contact, change an existing one's email/phone, or remove one. Action=list returns uids needed for update/delete. Use when the user says 'save this contact', 'add [name] to contacts', 'update [name]'s email', 'delete [name] from contacts'. Do not use for user identity facts like 'my name is <name>'; those are memory.", + "manage_contact": "Save / update / delete / list address-book contacts (CardDAV). Use for info about ANOTHER person — name, email, phone, postal address. Args: action=list|add|update|delete, name, email, phones, address, uid (from list). For 'save this for <person>' / address pastes / phone numbers next to a name, this is the right tool — NOT manage_memory. Do NOT use for facts about the USER ('my name is X'); those are manage_memory.", "manage_notes": "Create and manage notes and checklists (Google Keep-style). ALWAYS use this for note/todo/checklist/reminder creation — NEVER hit /api/notes via app_api. Accepts natural-language `due_date` like 'tomorrow at 9am' or '11pm today' (parsed in the USER'S timezone). The due_date IS the reminder — it fires a notification at that time, so do NOT also create a calendar event for the same reminder. Set colors, labels, pin, archive. Do NOT use manage_memory for note content.", "manage_calendar": "Calendar event management: list, create, update, delete. Each event can carry a tag/category (event_type — work/personal/health/travel/meal/social/admin/other) and importance (low/normal/high/critical). Resolve today/tomorrow using the Current date and time context, then use ISO datetimes in the user's local wall time; supports all-day events. For event reminders/alarms, pass reminder_minutes; this creates the Notes reminder, so do not also call manage_notes for the same reminder.", "download_model": "Download a HuggingFace model to a local or remote server. Specify repo_id (e.g. 'Qwen/Qwen3-8B'), optional server host, and optional include filter for specific files.", @@ -371,7 +372,19 @@ class ToolIndex: {"resolve_contact", "manage_contact"}, frozenset({"save contact", "add contact", "new contact", "update contact", "edit contact", "delete contact", "remove contact", - "save this person", "add to contacts", "save to contacts"}): + "save this person", "add to contacts", "save to contacts", + # "add <name> to (my) contacts" — words between 'add' and + # 'contacts' break the literal phrase match above, so anchor + # on the tail. + "to my contacts", "to contacts", "to address book", + # "save this for <person>" / "save it for <person>" — the user + # is storing info on a known person without using the literal + # word 'contact'. Catches the address/phone-paste pattern. + "save this for", "save it for", "save for", + "save this one for", "save that for", + # Postal-address-like signals + "postal code", "zip code", "street address", + "mailing address", "their address"}): {"manage_contact"}, # "Ask another model" intent → chat_with_model relays to a # different model and returns its answer. ask_teacher escalates @@ -383,6 +396,10 @@ class ToolIndex: "delegate to", "have model"}): {"chat_with_model", "ask_teacher", "list_models"}, # Deep research intent (incl. common typo "reserach") + frozenset({"web search", "search the web", "search online", "look up", + "google", "latest", "current", "news", "weather", + "forecast", "stock price", "price of"}): + {"web_search", "web_fetch"}, frozenset({"research", "reserach", "reasearch", "look into", "investigate", "deep dive", "deep research", "find out about", "study up on", "report on", "do research", "look up everything"}): @@ -502,6 +519,53 @@ class ToolIndex: # prompts do not drag web schemas into the agent context. if self._WEB_RE.search(query): base.update({"web_search", "web_fetch"}) + # Hard steering: when the query is a clear "save info about a specific + # person" pattern (address paste + name, phone next to a name, etc.), + # the model has been observed defaulting to manage_memory even with + # manage_contact in the toolset. Pull memory out for these queries so + # the model literally cannot pick it. ALWAYS_AVAILABLE includes + # manage_memory by default; we override that here. + # The "for/to <word>" check needs to allow lowercase names (users + # don't always capitalize) but filter out timing/pronoun stopwords + # so "save this for later" / "save for tomorrow" don't trigger. + _CONTACT_STOPWORDS_AFTER_FOR = { + "later", "tomorrow", "yesterday", "now", "then", "today", + "tonight", "me", "us", "you", "him", "her", "them", "myself", + "yourself", "next", "this", "that", "the", "a", "an", "future", + "real", "use", "uses", "another", "future", "reference", + } + # Regex catches "save (this|it|the|her|...|<noun>) for <name>" / "to my + # contacts" patterns. More forgiving than literal-keyword matching — + # 'save this address for Alex' uses one extra word between 'save' and + # 'for' that breaks the contiguous 'save this for' phrase. + save_for_match = re.search( + r"\bsave\b(?:\s+\w+){0,3}\s+(?:for|to)\s+([A-Za-z]+)", + ql, + ) + # "to my contacts", "into my contacts", "in my address book", etc. + to_contacts = re.search(r"\b(?:to|in|into)\s+(?:my\s+)?(?:contacts|address\s+book)\b", ql) + # Possessive: "save (his|her|their) (address|phone|email|number) ..." + # — strong contact signal even without "for <name>". Force-include + # manage_contact here too since the keyword fallback misses this + # construction. + possessive_contact = re.search( + r"\bsave\b(?:\s+\w+){0,2}\s+(?:his|her|their)\s+(?:address|phone|number|email|contact|details)", + ql, + ) + word_after = ( + save_for_match.group(1).lower() if save_for_match else None + ) + contact_only_signal = ( + (save_for_match is not None + and word_after is not None + and word_after not in _CONTACT_STOPWORDS_AFTER_FOR) + or to_contacts is not None + or possessive_contact is not None + ) + if possessive_contact is not None: + base.add("manage_contact") + if contact_only_signal and "manage_contact" in base: + base.discard("manage_memory") return base diff --git a/src/tool_parsing.py b/src/tool_parsing.py index 3f296c2e6..97d3f3477 100644 --- a/src/tool_parsing.py +++ b/src/tool_parsing.py @@ -188,6 +188,12 @@ _MISFENCED_WEB_TOOL_NAMES = { "fetch_url": "web_fetch", } +_RAW_WEB_JSON_TOOL_RE = re.compile( + r"\b(?:web_search|websearch|google_search|google_search_retrieval|google_search_grounding)\b", + re.IGNORECASE, +) +_RAW_WEB_JSON_ALLOWED_KEYS = {"query", "queries", "time_filter", "freshness", "max_pages"} + # --------------------------------------------------------------------------- # Parsing functions @@ -279,6 +285,73 @@ def _parse_misfenced_web_lookup(content: str) -> Optional[ToolBlock]: return None return ToolBlock("web_fetch", url) + +def _coerce_raw_web_query(value) -> Optional[str]: + if isinstance(value, str) and value.strip(): + return value.strip() + if isinstance(value, list): + for item in value: + if isinstance(item, str) and item.strip(): + return item.strip() + return None + + +def _raw_web_json_to_tool_block(payload) -> Optional[ToolBlock]: + if not isinstance(payload, dict): + return None + if set(payload) - _RAW_WEB_JSON_ALLOWED_KEYS: + return None + + query = _coerce_raw_web_query(payload.get("query")) + if not query: + query = _coerce_raw_web_query(payload.get("queries")) + if not query: + return None + + content = {"query": query} + for key in ("time_filter", "freshness"): + value = payload.get(key) + if isinstance(value, str) and value.strip().lower() in ("day", "week", "month", "year"): + content[key] = value.strip().lower() + + max_pages = payload.get("max_pages") + if isinstance(max_pages, int) and 1 <= max_pages <= 10: + content["max_pages"] = max_pages + + if len(content) == 1: + return ToolBlock("web_search", query) + return ToolBlock("web_search", json.dumps(content)) + + +def _parse_raw_web_json_lookup(text: str) -> Optional[tuple[ToolBlock, tuple[int, int]]]: + """Recover local text-model web_search calls emitted as prose + bare JSON. + + Some non-native tool models leak the intended call as: + + Need to do web_search for ... + {"query": "...", "time_filter": "week"} + + Keep this narrower than fenced/tool markup: it only runs when a known web + tool name appears shortly before a JSON object shaped like web_search args. + """ + if not isinstance(text, str): + return None + + decoder = json.JSONDecoder() + for mention in _RAW_WEB_JSON_TOOL_RE.finditer(text): + search_start = mention.end() + search_end = min(len(text), search_start + 1200) + for brace in re.finditer(r"\{", text[search_start:search_end]): + start = search_start + brace.start() + try: + parsed, end = decoder.raw_decode(text[start:]) + except json.JSONDecodeError: + continue + block = _raw_web_json_to_tool_block(parsed) + if block: + return block, (start, start + end) + return None + def _parse_tool_call_block(raw: str) -> Optional[ToolBlock]: """Parse a [TOOL_CALL] block into a ToolBlock. @@ -436,6 +509,8 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]: 3. XML-style <tool_call>/<invoke> blocks 4. <tool_code> blocks (MiniMax-M2.5 style) 5. DeepSeek DSML markup (normalized to <invoke> first) + 6. Non-native local model fallback: prose mentioning web_search followed by + bare JSON args, e.g. {"query":"...", "time_filter":"week"} `skip_fenced`: when True, Pattern 1 (fenced ```bash/```python/```json code blocks) is not matched at all. Native function-calling models (GPT/Claude/ @@ -509,6 +584,12 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]: if block: blocks.append(block) + # Pattern 6: local text-model web_search call leaked as prose + bare JSON. + if not blocks and not skip_fenced: + raw_web_json = _parse_raw_web_json_lookup(text) + if raw_web_json: + blocks.append(raw_web_json[0]) + return blocks @@ -532,6 +613,11 @@ def strip_tool_blocks(text: str, skip_fenced: bool = False) -> str: cleaned = _TOOL_CALL_RE.sub('', cleaned) cleaned = _XML_TOOL_CALL_RE.sub('', cleaned) cleaned = _TOOL_CODE_RE.sub('', cleaned) + if not skip_fenced: + raw_web_json = _parse_raw_web_json_lookup(cleaned) + if raw_web_json: + _, (start, end) = raw_web_json + cleaned = cleaned[:start] + cleaned[end:] # Strip bare <invoke> blocks not wrapped in <tool_call> cleaned = re.sub(r'<invoke\s+name=["\'].*?</invoke>', '', cleaned, flags=re.DOTALL | re.IGNORECASE) cleaned = re.sub(r'\n{3,}', '\n\n', cleaned) diff --git a/src/tool_schemas.py b/src/tool_schemas.py index e0d01f008..4e233317b 100644 --- a/src/tool_schemas.py +++ b/src/tool_schemas.py @@ -25,7 +25,7 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "bash", - "description": "Run a shell command (full access)", + "description": "Run a shell command (full access). Prefer a dedicated tool whenever one fits the job (reading, writing, editing, searching, or listing files); use bash only for what no dedicated tool covers (installs, git, builds, running programs, system info). Do NOT create or edit files via bash redirects/heredocs/sed -- use the dedicated file tools.", "parameters": { "type": "object", "properties": { @@ -39,7 +39,7 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "python", - "description": "Execute Python code to compute a result or test something", + "description": "Execute Python code to compute a result or test something. Prefer a dedicated tool whenever one fits the job (reading, writing, or searching files); use python only for computation, data processing, or scripting no dedicated tool covers.", "parameters": { "type": "object", "properties": { @@ -68,11 +68,12 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "web_fetch", - "description": "Fetch and read the text content of a specific URL the user names (e.g. 'check example.com', 'what's on this page <url>'). Use when you already have a concrete URL/domain. NOT for open-ended searches (use web_search) or 'research X' jobs (use trigger_research).", + "description": "Fetch and read the text content of a specific URL the user names (e.g. 'check example.com', 'what's on this page <url>'). Use when you already have a concrete URL/domain. NOT for open-ended searches (use web_search) or 'research X' jobs (use trigger_research). Downloads are size-budgeted; a '[partial content: ...]' notice in the result means the body was cut short and you can re-call with full=true for the rest.", "parameters": { "type": "object", "properties": { - "url": {"type": "string", "description": "The URL or domain to fetch (http/https; a bare domain like example.com is fine)"} + "url": {"type": "string", "description": "The URL or domain to fetch (http/https; a bare domain like example.com is fine)"}, + "full": {"type": "boolean", "description": "Raise the download budget to the hard cap for large pages/files. Use only after a result reported partial content."} }, "required": ["url"] } @@ -141,6 +142,14 @@ FUNCTION_TOOL_SCHEMAS = [ } } }, + { + "type": "function", + "function": { + "name": "get_workspace", + "description": "Return the absolute path of the active workspace folder the user is working in. File tools are confined to it; the shell starts there but is not sandboxed. Call this first when the user refers to 'the project'/'the code'/'this folder' without a path, instead of asking them. Takes no arguments.", + "parameters": {"type": "object", "properties": {}, "required": []} + } + }, { "type": "function", "function": { @@ -1000,7 +1009,7 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "resolve_contact", - "description": "Look up a contact's email address by name. Searches CardDAV address book and sent email history. Use when the user says 'message [name]' or 'email [name]' without an email address.", + "description": "Look up a contact by name. Searches CardDAV address book and sent email history. Returns email addresses (when available) or phone numbers. Use when the user says 'message [name]', 'email [name]', or asks for someone's contact details.", "parameters": { "type": "object", "properties": { @@ -1014,7 +1023,7 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "manage_contact", - "description": "Create, update, delete, or list the user's CardDAV contacts. Use to save a new contact ('save Jonathan's email jon@x.com'), update an existing one ('change Maria's number'), or remove one. For update/delete you need the contact's uid — call action='list' first to find it. Writes go through the same dedupe + validation as the Contacts UI.", + "description": "Create, update, delete, or list the user's CardDAV contacts. Use to save a new contact, update an existing one (email/phone/address), or remove one. For update/delete you need the contact's uid — call action='list' first to find it. Writes go through the same dedupe + validation as the Contacts UI.", "parameters": { "type": "object", "properties": { @@ -1025,6 +1034,7 @@ FUNCTION_TOOL_SCHEMAS = [ "email": {"type": "string", "description": "Single email address (convenience for add, or the primary email for update)."}, "emails": {"type": "array", "items": {"type": "string"}, "description": "Full list of email addresses (for update; first is primary)."}, "phones": {"type": "array", "items": {"type": "string"}, "description": "Full list of phone numbers (for update)."}, + "address": {"type": "string", "description": "Postal/mailing address as a single human-readable string."}, }, "required": ["action"] } @@ -1196,23 +1206,26 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock logger.error(f"Failed to parse function call arguments for {name}: {arguments}") return None + tool_type = _TOOL_NAME_MAP.get(name, name) + _BUILTIN_EMAIL_TOOLS = {"list_email_accounts", "send_email", "list_emails", "read_email", "reply_to_email", + "archive_email", "delete_email", "mark_email_read", "bulk_email", "download_attachment"} + # Some models emit valid JSON that isn't an object (e.g. a bare array - # ["ls -la"], string, or number) as the function arguments. Every branch - # below assumes a dict and calls args.get(...), so a non-dict would raise - # AttributeError and abort the whole agent stream. Coerce to {} instead. + # ["ls -la"], string, or number) as function arguments. Most local tools keep + # the legacy empty-object coercion for stream robustness, but email MCP tools + # must fail closed so a malformed call cannot read the default mailbox. if not isinstance(args, dict): + if tool_type.startswith("mcp__email__") or name in _BUILTIN_EMAIL_TOOLS: + logger.warning(f"Non-object email function call arguments for {name}: {args!r}; rejecting") + return None logger.warning(f"Non-object function call arguments for {name}: {args!r}; treating as empty") args = {} - tool_type = _TOOL_NAME_MAP.get(name, name) - # Allow MCP tools through (namespaced as mcp__serverid__toolname) if tool_type.startswith("mcp__"): content = json.dumps(args) if args else "{}" return ToolBlock(tool_type, content) # Email tools are implemented as MCP — route them to email - _BUILTIN_EMAIL_TOOLS = {"list_email_accounts", "send_email", "list_emails", "read_email", "reply_to_email", - "archive_email", "delete_email", "mark_email_read", "bulk_email", "download_attachment"} if name in _BUILTIN_EMAIL_TOOLS: return ToolBlock(f"mcp__email__{name}", json.dumps(args) if args else "{}") if tool_type not in TOOL_TAGS: @@ -1246,6 +1259,8 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock content = args.get("path", "") elif tool_type in ("grep", "glob", "ls"): content = json.dumps(args) if args else "{}" + elif tool_type == "get_workspace": + content = "" elif tool_type == "write_file": content = args.get("path", "") + "\n" + args.get("content", "") elif tool_type == "edit_file": diff --git a/src/tool_security.py b/src/tool_security.py index 6b7bc90df..3dc53ff26 100644 --- a/src/tool_security.py +++ b/src/tool_security.py @@ -20,6 +20,7 @@ NON_ADMIN_BLOCKED_TOOLS = { "grep", "glob", "ls", + "get_workspace", "search_chats", "manage_memory", "manage_skills", @@ -66,6 +67,7 @@ PLAN_MODE_READONLY_TOOLS = { "grep", "glob", "ls", + "get_workspace", "web_search", "web_fetch", "search_chats", @@ -175,13 +177,16 @@ def owner_is_admin_or_single_user(owner: Optional[str]) -> bool: defense-in-depth for callers that bypass it (e.g. trusted loopback). """ try: + from src.auth_helpers import _auth_disabled + + if _auth_disabled(): + return True + from core.auth import AuthManager auth = AuthManager() if not auth.is_configured: - from src.auth_helpers import _auth_disabled - - return _auth_disabled() + return False return bool(owner and auth.is_admin(owner)) except Exception as exc: logger.warning("Unable to evaluate owner admin status: %s", exc) diff --git a/src/upload_handler.py b/src/upload_handler.py index 95bce306d..4c4e526bc 100644 --- a/src/upload_handler.py +++ b/src/upload_handler.py @@ -352,6 +352,86 @@ class UploadHandler: return dict(info) return None + def _renamed_upload_index_key(self, key: str, info: Dict[str, Any], old_owner: str, new_owner: str) -> str: + """Return the storage key to use after renaming an owned upload row.""" + if isinstance(key, str) and ":" in key: + owner_part, rest = key.split(":", 1) + if owner_part.strip().lower() == old_owner: + return f"{new_owner}:{rest}" + file_hash = info.get("hash") + if file_hash: + return f"{new_owner}:{file_hash}" + return key + + def _unique_upload_index_key(self, base_key: str, used_keys: set, reserved_keys: set, info: Dict[str, Any]) -> str: + """Choose a deterministic collision key without overwriting an existing row.""" + if base_key not in used_keys and base_key not in reserved_keys: + return base_key + + upload_id = str(info.get("id") or "renamed").strip() or "renamed" + candidate = f"{base_key}:{upload_id}" + if candidate not in used_keys and candidate not in reserved_keys: + return candidate + + index = 2 + while True: + candidate = f"{base_key}:{upload_id}:{index}" + if candidate not in used_keys and candidate not in reserved_keys: + return candidate + index += 1 + + def rename_owner(self, old_owner: str, new_owner: str) -> int: + """Rename upload metadata ownership from old_owner to new_owner. + + Upload rows are keyed by owner-qualified hashes for dedupe and also + carry an `owner` field for access checks. Both must move together when + usernames change. + """ + old_owner_normalized = str(old_owner or "").strip().lower() + new_owner = str(new_owner or "").strip() + if not old_owner_normalized or not new_owner: + return 0 + if old_owner_normalized == new_owner.lower(): + return 0 + + uploads_db_path = os.path.join(self.upload_dir, "uploads.json") + with self._index_lock: + current = self._load_upload_index() + if not current: + return 0 + + updated = {} + renamed = 0 + original_keys = set(current.keys()) + + for key, info in current.items(): + new_key = key + new_info = info + if isinstance(info, dict) and str(info.get("owner", "")).strip().lower() == old_owner_normalized: + new_info = dict(info) + new_info["owner"] = new_owner + base_key = self._renamed_upload_index_key(key, new_info, old_owner_normalized, new_owner) + new_key = self._unique_upload_index_key( + base_key, + set(updated.keys()), + original_keys - {key}, + new_info, + ) + if new_key != base_key: + logger.warning( + "Upload owner rename key collision for %s -> %s at %s; preserving row as %s", + old_owner_normalized, + new_owner, + base_key, + new_key, + ) + renamed += 1 + updated[new_key] = new_info + + if renamed: + self._atomic_write_json(uploads_db_path, updated) + return renamed + def _find_upload_path(self, upload_id: str) -> Optional[str]: """Find an upload file by ID while staying inside upload_dir.""" if not self.validate_upload_id(upload_id): diff --git a/src/webhook_manager.py b/src/webhook_manager.py index 267ceaa38..af28fe2a7 100644 --- a/src/webhook_manager.py +++ b/src/webhook_manager.py @@ -202,6 +202,18 @@ class WebhookManager: self._client = httpx.AsyncClient(timeout=10, follow_redirects=False) self._loop: Optional[asyncio.AbstractEventLoop] = None self._api_key_manager = api_key_manager + # Strong references to in-flight fire-and-forget tasks. asyncio only + # keeps weak references to tasks, so without this the GC can collect a + # delivery task mid-flight and the webhook is silently never sent. + self._bg_tasks: set = set() + + def _spawn_tracked(self, coro): + """Schedule a background task and hold a strong reference until it + finishes, so it can't be garbage-collected before delivery completes.""" + task = asyncio.ensure_future(coro) + self._bg_tasks.add(task) + task.add_done_callback(self._bg_tasks.discard) + return task def set_loop(self, loop: asyncio.AbstractEventLoop): self._loop = loop @@ -223,8 +235,8 @@ class WebhookManager: if event not in ALLOWED_EVENTS: return try: - loop = asyncio.get_running_loop() - loop.create_task(self.fire(event, payload)) + asyncio.get_running_loop() + self._spawn_tracked(self.fire(event, payload)) except RuntimeError: # Called from a sync thread (e.g. sync FastAPI route in threadpool) if self._loop and self._loop.is_running(): @@ -243,7 +255,7 @@ class WebhookManager: for wh in matching: decrypted_secret = self._decrypt_secret(wh.secret) - asyncio.create_task(self._deliver(wh.id, wh.url, decrypted_secret, event, payload)) + self._spawn_tracked(self._deliver(wh.id, wh.url, decrypted_secret, event, payload)) async def deliver_test(self, webhook_id: str, url: str, encrypted_secret: Optional[str]): """Public method for the test-webhook route.""" diff --git a/src/youtube_handler.py b/src/youtube_handler.py index 001847535..0f9eec263 100644 --- a/src/youtube_handler.py +++ b/src/youtube_handler.py @@ -1,278 +1,23 @@ -""" -YouTube handling — transcript extraction, comment fetching (yt-dlp), -and context formatting for LLM injection. Used by chat_handler.py. +"""Compatibility wrapper for the canonical services.youtube.youtube_handler module. + +Odysseus historically carried two independent copies of the YouTube handler — +one here under ``src`` and one under ``services.youtube``. They drifted: the +comment-fetch timeout fix landed only in the ``src`` copy, while ``app.py`` +calls ``services.youtube.init_youtube()`` at startup. Because the chat flow +imported ``extract_transcript_async`` from ``src.youtube_handler`` (a different +module object), the ``YOUTUBE_AVAILABLE`` / ``YouTubeTranscriptApi`` globals set +by ``init_youtube`` never reached it and transcript extraction always reported +"YouTube transcript API not available". + +Keep the old ``src.youtube_handler`` import path working, but make it resolve to +the single source of truth so module state and behavior can't diverge again. """ -import asyncio -import json -import logging -import shutil +import importlib import sys -import urllib.parse -from pathlib import Path -from typing import Dict, Any, Optional -logger = logging.getLogger(__name__) +# Import the canonical module directly (services.youtube.youtube_handler) +# without triggering the heavy services/__init__.py top-level imports. +_youtube_handler = importlib.import_module("services.youtube.youtube_handler") -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -YOUTUBE_INSTRUCTION_PROMPT = """When the user shares a YouTube video, respond with a structured breakdown: - -1. **Summary** — Concise overview of the video's content and main thesis (2-4 sentences) -2. **Key Points** — Bullet list of the most important topics, arguments, or moments -3. **Notable Timestamps** — If timestamps are available from the transcript, highlight 3-5 interesting moments with their approximate timestamps (e.g. "03:45 — discusses X") -4. **Audience Reception** — If comments are available, summarize what viewers think: general sentiment, top reactions, any debate or controversy - -Keep it conversational and concise. Do NOT web search for this video — use only the transcript and comments provided.""" - -# --------------------------------------------------------------------------- -# Init / helpers -# --------------------------------------------------------------------------- - -# Will be set at startup by init_youtube() -YouTubeTranscriptApi = None -YOUTUBE_AVAILABLE = False - - -def _find_ytdlp() -> str: - """Find the yt-dlp binary: venv bin first, then system PATH.""" - venv_bin = Path(sys.executable).parent / "yt-dlp" - if venv_bin.exists(): - return str(venv_bin) - found = shutil.which("yt-dlp") - return found or "yt-dlp" - - -def init_youtube(): - """Import and cache the YouTube transcript API.""" - global YouTubeTranscriptApi, YOUTUBE_AVAILABLE - try: - from youtube_transcript_api import YouTubeTranscriptApi as _Api - YouTubeTranscriptApi = _Api - YOUTUBE_AVAILABLE = True - logger.info("YouTube transcript API available") - except ImportError as e: - logger.warning(f"youtube-transcript-api not installed: {e}") - YOUTUBE_AVAILABLE = False - - -def is_youtube_url(url: str) -> bool: - if not isinstance(url, str): - return False - return "youtube.com" in url or "youtu.be" in url - - -def extract_youtube_id(url: str) -> Optional[str]: - """Extract YouTube video ID from various URL formats.""" - parsed = urllib.parse.urlparse(url) - if parsed.hostname in ("www.youtube.com", "youtube.com", "m.youtube.com"): - if parsed.path == "/watch": - params = urllib.parse.parse_qs(parsed.query) - if "v" in params: - return params["v"][0] - elif parsed.path.startswith("/embed/"): - return parsed.path.split("/")[-1] - elif parsed.hostname == "youtu.be": - return parsed.path[1:] - return None - - -async def extract_transcript_async( - url: str, video_id: str, max_retries: int = 3 -) -> Dict[str, Any]: - """ - Async YouTube transcript extraction with retries. - - Args: - url: Full YouTube URL - video_id: Extracted video ID - max_retries: Number of attempts - - Returns: - Dict with success/error/transcript keys - """ - if not YOUTUBE_AVAILABLE or YouTubeTranscriptApi is None: - return {"success": False, "error": "YouTube transcript API not available", "transcript": None} - - for attempt in range(max_retries): - try: - api = YouTubeTranscriptApi() - transcript = api.fetch(video_id) - transcript_list = list(transcript) - - formatted = [] - for snippet in transcript_list: - text = snippet.text.strip() - if not text: - continue - start = snippet.start - formatted.append({ - "text": text, - "start": start, - "duration": snippet.duration, - "timestamp": f"{int(start // 60):02d}:{int(start % 60):02d}", - }) - - full_text = " ".join(e["text"] for e in formatted) - max_len = 8000 - if len(full_text) > max_len: - full_text = full_text[:max_len] + "... [transcript truncated]" - - return { - "success": True, - "transcript": full_text, - "video_id": video_id, - "language": "en", - "is_generated": False, - "segments": formatted, - } - except Exception as e: - logger.warning(f"Transcript attempt {attempt + 1} failed: {e}") - if attempt < max_retries - 1: - await asyncio.sleep(1 * (attempt + 1)) - - return {"success": False, "error": f"Failed after {max_retries} attempts", "transcript": None} - - -def format_transcript_for_context( - transcript_data: Dict[str, Any], url: str, - title: str = "", channel: str = "" -) -> str: - """Format transcript data for inclusion in LLM context.""" - if not transcript_data.get("success"): - header = "" - if title: - header = f" \"{title}\"" - if channel: - header += f" by {channel}" - return f"\n[YouTube Video{header}: Transcript unavailable ({transcript_data.get('error', 'Unknown error')}). Use the comments below if available, do NOT web search for this video.]" - - transcript = transcript_data.get("transcript", "") - video_id = transcript_data.get("video_id", "") - language = transcript_data.get("language", "unknown") - is_generated = transcript_data.get("is_generated", False) - segments = transcript_data.get("segments", []) - - ctx = "\n[YOUTUBE VIDEO TRANSCRIPT]\n" - if title: - ctx += f"Title: {title}\n" - if channel: - ctx += f"Channel: {channel}\n" - ctx += f"Video ID: {video_id}\n" - ctx += f"Language: {language}\n" - ctx += f"Source: {'Auto-generated' if is_generated else 'Manual'}\n" - ctx += f"URL: {url}\n\n" - # Include timestamped segments for the LLM to reference - if segments: - ctx += "Timestamped Transcript:\n" - for seg in segments: - if not isinstance(seg, dict): - continue - ctx += f"[{seg['timestamp']}] {seg['text']}\n" - # Check length — fall back to plain text if too long - if len(ctx) > 12000: - ctx = ctx[:ctx.index("Timestamped Transcript:\n")] - ctx += "Transcript:\n" - ctx += transcript - else: - ctx += "Transcript:\n" - ctx += transcript - ctx += "\n[END TRANSCRIPT]\n" - return ctx - - -async def fetch_youtube_comments( - video_id: str, max_comments: int = 25, timeout: int = 30 -) -> Dict[str, Any]: - """Fetch top comments for a YouTube video using yt-dlp. - - Returns dict with 'success', 'comments' list, 'error'. - """ - try: - cmd = [ - _find_ytdlp(), - "--skip-download", - "--write-comments", - "--extractor-args", f"youtube:max_comments={max_comments},all,100,0", - "--dump-json", - "--js-runtimes", "node", - "--remote-components", "ejs:github", - f"https://www.youtube.com/watch?v={video_id}", - ] - - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - # Bound the wait on the process actually finishing, not on spawning it. - # create_subprocess_exec returns as soon as the child starts, so wrapping - # it in wait_for never enforces the timeout — proc.communicate() is the - # blocking step. Kill and reap the child if it overruns so it does not - # linger after we return. - try: - stdout, stderr = await asyncio.wait_for( - proc.communicate(), timeout=timeout - ) - except asyncio.TimeoutError: - proc.kill() - await proc.wait() - raise - - if proc.returncode != 0: - return {"success": False, "error": f"yt-dlp failed: {stderr.decode()[:200]}", "comments": []} - - data = json.loads(stdout.decode()) - title = data.get("title", "") - channel = data.get("channel", "") or data.get("uploader", "") - raw_comments = data.get("comments", []) - - comments = [] - for c in raw_comments[:max_comments]: - text = (c.get("text") or "").strip() - if not text: - continue - comments.append({ - "author": c.get("author", "Unknown"), - "text": text, - "likes": c.get("like_count", 0), - }) - - # Sort by likes descending — most popular comments first - comments.sort(key=lambda x: x.get("likes", 0), reverse=True) - - return {"success": True, "comments": comments, "count": len(comments), - "title": title, "channel": channel} - - except asyncio.TimeoutError: - logger.warning(f"Comment fetch timed out for {video_id}") - return {"success": False, "error": "Comment fetch timed out", "comments": []} - except FileNotFoundError: - logger.warning("yt-dlp not installed — cannot fetch comments") - return {"success": False, "error": "yt-dlp not installed", "comments": []} - except Exception as e: - logger.warning(f"Failed to fetch comments for {video_id}: {e}") - return {"success": False, "error": str(e), "comments": []} - - -def format_comments_for_context(comments_data: Dict[str, Any], url: str) -> str: - """Format YouTube comments for inclusion in LLM context.""" - if not comments_data.get("success") or not comments_data.get("comments"): - return "" - - comments = comments_data["comments"] - ctx = f"\n[YOUTUBE VIDEO COMMENTS — Top {len(comments)} by popularity]\n" - ctx += f"URL: {url}\n\n" - - for i, c in enumerate(comments, 1): - likes = c.get("likes", 0) - likes_str = f" [{likes} likes]" if likes else "" - ctx += f"{i}. @{c['author']}{likes_str}: {c['text']}\n\n" - - if len(ctx) > 4000: - ctx = ctx[:4000] + "\n[Comments truncated]\n" - - ctx += "[END COMMENTS]\n" - return ctx +sys.modules[__name__] = _youtube_handler diff --git a/start-macos.sh b/start-macos.sh index f324625c6..2aa15d261 100755 --- a/start-macos.sh +++ b/start-macos.sh @@ -130,11 +130,12 @@ fi # 3. Python environment + dependencies (kept inside the repo, in venv/). # Named `venv` to match the manual steps and build-macos-app.sh, so the # clickable .app reuses this same environment. -if [ ! -d venv ]; then +VENV_PY="./venv/bin/python3" +if [ ! -x "$VENV_PY" ] || ! "$VENV_PY" -m pip --version >/dev/null 2>&1; then + [ -d venv ] && { echo "▶ Existing venv is incomplete (no working pip) — rebuilding…"; rm -rf venv; } echo "▶ Creating Python environment…" "$PY" -m venv venv fi -VENV_PY="./venv/bin/python3" REQ_HASH="$(md5 -q requirements.txt 2>/dev/null || md5sum requirements.txt | cut -d' ' -f1)" REQ_HASH_FILE="venv/.requirements_hash" if [ ! -f "$REQ_HASH_FILE" ] || [ "$REQ_HASH" != "$(cat "$REQ_HASH_FILE" 2>/dev/null)" ]; then diff --git a/static/app.js b/static/app.js index c75070bf2..1f0390a37 100644 --- a/static/app.js +++ b/static/app.js @@ -4,6 +4,7 @@ // ============================================ import Storage from './js/storage.js'; import uiModule from './js/ui.js'; +import workspaceModule from './js/workspace.js'; import fileHandlerModule from './js/fileHandler.js'; import modelsModule from './js/models.js'; import ragModule from './js/rag.js'; @@ -1159,7 +1160,7 @@ function initializeEventListeners() { if (!p.can_use_bash) { const bashToggle = document.getElementById('bash-toggle'); if (bashToggle) bashToggle.closest('.chat-input-toggle')?.style.setProperty('display', 'none'); - const bashBtn = document.getElementById('tool-bash-btn'); + const bashBtn = document.getElementById('bash-toggle-btn'); if (bashBtn) bashBtn.style.display = 'none'; } // Hide document button @@ -1176,11 +1177,7 @@ function initializeEventListeners() { const resOverflow = document.getElementById('overflow-research-btn'); if (resOverflow) resOverflow.style.display = 'none'; } - // Hide image generation options - if (!p.can_generate_images) { - const imgBtn = document.getElementById('tool-image-btn'); - if (imgBtn) imgBtn.style.display = 'none'; - } + } }) .catch(() => {}); @@ -1221,7 +1218,7 @@ function initializeEventListeners() { sortDropdown.querySelectorAll('.sort-option').forEach(o => { const check = o.querySelector('.sort-check') || document.createElement('span'); check.className = 'sort-check'; - check.style.cssText = 'float:right;font-size:20px;line-height:1;position:relative;top:3px;color:var(--accent, var(--red));opacity:' + (o.dataset.sort === current ? '1' : '0'); + check.style.cssText = 'float:right;font-size:20px;line-height:1;position:relative;top:1px;color:var(--accent, var(--red));opacity:' + (o.dataset.sort === current ? '1' : '0'); check.textContent = '\u2022'; if (!o.querySelector('.sort-check')) o.appendChild(check); }); @@ -1265,9 +1262,9 @@ function initializeEventListeners() { let msg; if (data.updated > 0) { msg = `Sorted ${data.updated} into ${data.folders.length} folder${data.folders.length === 1 ? '' : 's'}`; - if (remaining > 0) msg += ` — ${remaining} unfiled left, hit Tidy again`; + if (remaining > 0) msg += ` — ${remaining} unfiled left, hit Group again`; } else if (remaining > 0) { - msg = `${remaining} unfiled chats — hit Tidy again`; + msg = `${remaining} unfiled chats — hit Group again`; } else { msg = 'All sorted'; } @@ -1288,17 +1285,6 @@ function initializeEventListeners() { const autoSortBtn = el('auto-sort-sessions-btn'); if (autoSortBtn) autoSortBtn.addEventListener('click', () => _runTidy(false)); - - // Chevron next to the Tidy row toggles the no-AI sub-item. - const autoSortMoreBtn = el('auto-sort-sessions-more'); - const autoSortNoaiBtn = el('auto-sort-sessions-noai-btn'); - if (autoSortMoreBtn && autoSortNoaiBtn) { - autoSortMoreBtn.addEventListener('click', (e) => { - e.stopPropagation(); - autoSortNoaiBtn.style.display = autoSortNoaiBtn.style.display === 'none' ? 'block' : 'none'; - }); - autoSortNoaiBtn.addEventListener('click', () => _runTidy(true)); - } } // Model sort dropdown @@ -1626,6 +1612,8 @@ function initializeEventListeners() { // Slide the pill to the active button const toggle = agentBtn.closest('.mode-toggle'); if (toggle) toggle.classList.toggle('mode-chat', mode === 'chat'); + // Workspace pill + overflow entry are agent-only - hide immediately (no flash). + try { workspaceModule.applyMode(mode); } catch (_) {} // Delay tool glow-up for a staggered effect setTimeout(() => applyModeToToggles(mode), 500); } @@ -1701,6 +1689,7 @@ function initializeEventListeners() { } setupToggle('web-toggle-btn', 'web-toggle', 'web'); setupToggle('bash-toggle-btn', 'bash-toggle', 'bash'); + try { workspaceModule.initWorkspace(); } catch (_) {} // Document editor toggle (special: uses module panel, not a checkbox) const overflowDocBtn = el('overflow-doc-btn'); @@ -3135,7 +3124,9 @@ function initializeEventListeners() { setTimeout(() => uiModule.autoResize(textarea), 1); }); textarea.addEventListener('keydown', (e) => { - if (e.key === 'Enter' && !e.shiftKey && !e.isComposing) { + const isMobile = window.innerWidth <= 768 + + if (e.key === 'Enter' && !e.shiftKey && !e.isComposing && !isMobile) { // If ghost autocomplete is active, accept the suggestion instead of submitting if (window._ghostAutocomplete && window._ghostAutocomplete.isActive()) { e.preventDefault(); @@ -3708,7 +3699,9 @@ function startOdysseusApp() { // Enter to send (shift+enter for newline), or new chat when empty if (messageInput) { messageInput.addEventListener('keydown', (e) => { - if (e.key === 'Enter' && !e.shiftKey && !e.isComposing) { + const isMobile = window.innerWidth <= 768 + + if (e.key === 'Enter' && !e.shiftKey && !e.isComposing && !isMobile) { e.preventDefault(); // Flush the debounced icon update so dataset.mode reflects the current // text state. Without this, a fast type-and-Enter would still see the diff --git a/static/icon.ico b/static/icon.ico new file mode 100644 index 000000000..666a7e6ad Binary files /dev/null and b/static/icon.ico differ diff --git a/static/icons/icon-192.png b/static/icons/icon-192.png new file mode 100644 index 000000000..d4111ba0f Binary files /dev/null and b/static/icons/icon-192.png differ diff --git a/static/icons/icon-512.png b/static/icons/icon-512.png new file mode 100644 index 000000000..f6b56e215 Binary files /dev/null and b/static/icons/icon-512.png differ diff --git a/static/icons/icon-maskable-512.png b/static/icons/icon-maskable-512.png new file mode 100644 index 000000000..5d9d98a00 Binary files /dev/null and b/static/icons/icon-maskable-512.png differ diff --git a/static/index.html b/static/index.html index 60a2764d9..fe22d6b9e 100644 --- a/static/index.html +++ b/static/index.html @@ -12,7 +12,7 @@ in email bodies — was wrapping random digits in <a href="tel:..."> with browser-default styling that didn't match the Odysseus theme. --> <meta name="format-detection" content="telephone=no, date=no, address=no, email=no"> - <link rel="apple-touch-icon" href="/static/icon-192.png"> + <link rel="apple-touch-icon" href="/static/icons/icon-192.png"> <script nonce="{{CSP_NONCE}}"> window._odysseusLoadTime = Date.now(); (function(){ @@ -258,21 +258,29 @@ <div class="memory-tab-panel" data-memory-panel="browse"> <div class="admin-card" style="display:flex;flex-direction:column;overflow:hidden;flex:1;min-height:0;"> <div style="display:flex;align-items:center;gap:8px;margin-bottom:2px;"> - <h2 style="display:flex;align-items:center;gap:6px;margin:0;padding:0;line-height:1;">Memories <span id="memory-count-h2" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal"></span></h2> + <h2 style="display:flex;align-items:center;gap:6px;margin:0;padding:0;line-height:1;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="color:var(--accent, var(--red));flex-shrink:0;"><path d="M12 2a7 7 0 0 1 7 7c0 2.4-1.2 4.5-3 5.7V17a2 2 0 0 1-2 2h-4a2 2 0 0 1-2-2v-2.3C6.2 13.5 5 11.4 5 9a7 7 0 0 1 7-7z"/><line x1="10" y1="22" x2="14" y2="22"/></svg>Memories <span id="memory-count-h2" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal"></span></h2> <span style="flex:1"></span> + <span class="admin-toggle-state"></span> <label class="admin-switch" title="Include memories in chat context"><input type="checkbox" id="memory-enabled-header-toggle" checked /><span class="admin-slider"></span></label> </div> <p class="memory-desc doclib-desc" style="margin-top:6px;">Long-term facts the AI remembers across chats — recall, edit, or curate.</p> <div class="memory-toolbar"> <div class="memory-toolbar-row"> - <select id="memory-sort" class="memory-sort-select" aria-label="Sort memories"> - <option value="newest">Newest</option> - <option value="oldest">Oldest</option> - <option value="alpha">A-Z</option> - <option value="uses">Most used</option> - </select> - <button id="memory-select-btn" class="memory-toolbar-btn" title="Select multiple memories">Select</button> - <button id="memory-tidy-btn" class="memory-toolbar-btn" title="AI tidy: deduplicate and clean up memories"><svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;margin-right:2px;"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg> Tidy</button> + <div class="memory-sort-picker" id="memory-sort-picker" style="position:relative;"> + <select id="memory-sort" class="memory-sort-select" aria-label="Sort memories" style="display:none;"> + <option value="newest">Newest</option> + <option value="oldest">Oldest</option> + <option value="alpha">A-Z</option> + <option value="uses">Most used</option> + </select> + <button type="button" class="memory-sort-btn" id="memory-sort-btn" aria-haspopup="listbox" aria-expanded="false"> + <span class="memory-sort-current"><span class="memory-sort-icon-cur"></span><span class="memory-sort-label">Newest</span></span> + <svg class="memory-sort-caret" width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg> + </button> + <div class="memory-sort-menu" id="memory-sort-menu" role="listbox" hidden></div> + </div> + <button id="memory-tidy-btn" class="memory-toolbar-btn" title="AI tidy: deduplicate and clean up memories"><svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;margin-right:2px;color:var(--accent, var(--red));"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg> Tidy</button> + <button id="memory-select-btn" class="memory-toolbar-btn" title="Select multiple memories" style="position:relative;left:-2px;"><svg class="memory-select-btn-icon" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:3px;"><circle cx="12" cy="12" r="10"/><circle cx="12" cy="12" r="3" fill="currentColor" stroke="none"/></svg>Select</button> </div> <input type="text" id="memory-search" placeholder="Search memories…" class="memory-search-input" aria-label="Search memories" /> <div id="memory-category-filters" class="memory-category-filters"> @@ -293,7 +301,7 @@ <div class="memory-tab-panel hidden" data-memory-panel="add"> <div class="admin-card"> <div style="display:flex;align-items:center;gap:8px;margin-bottom:2px;"> - <h2 style="margin:0;padding:0;line-height:1;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:6px"><path d="M12 2a7 7 0 0 1 7 7c0 2.4-1.2 4.5-3 5.7V17a2 2 0 0 1-2 2h-4a2 2 0 0 1-2-2v-2.3C6.2 13.5 5 11.4 5 9a7 7 0 0 1 7-7z"/><line x1="10" y1="22" x2="14" y2="22"/></svg>Add Memory</h2> + <h2 style="margin:0;padding:0;line-height:1;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:6px;color:var(--accent, var(--red));"><path d="M12 2a7 7 0 0 1 7 7c0 2.4-1.2 4.5-3 5.7V17a2 2 0 0 1-2 2h-4a2 2 0 0 1-2-2v-2.3C6.2 13.5 5 11.4 5 9a7 7 0 0 1 7-7z"/><line x1="10" y1="22" x2="14" y2="22"/></svg>Add Memory</h2> <span style="flex:1"></span> <button id="memory-import-btn" class="theme-io-btn" title="Import memories from a file" style="height:26px;font-size:12px;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:4px;"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="7 10 12 15 17 10"/><line x1="12" y1="15" x2="12" y2="3"/></svg>Import</button> <button id="memory-export-btn" class="theme-io-btn" title="Export all memories as JSON" style="height:26px;font-size:12px;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:4px;"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="17 8 12 3 7 8"/><line x1="12" y1="3" x2="12" y2="15"/></svg>Export</button> @@ -312,7 +320,7 @@ </div> <div class="admin-card"> <div style="display:flex;align-items:baseline;gap:8px;margin-bottom:2px;"> - <h2 style="margin:0;padding:0;line-height:1;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:6px"><polygon points="13 2 3 14 12 14 11 22 21 10 12 10 13 2"/></svg>Add Skill</h2> + <h2 style="margin:0;padding:0;line-height:1;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:6px;color:var(--accent, var(--red));"><polygon points="13 2 3 14 12 14 11 22 21 10 12 10 13 2"/></svg>Add Skill</h2> </div> <p class="memory-desc doclib-desc" style="margin-top:6px;">Import a skill from GitHub or <a href="https://skills.sh" target="_blank" rel="noopener noreferrer">skills.sh</a> (folder with <code>SKILL.md</code> and optional templates).</p> <div class="memory-add-row" style="margin-top:6px;margin-bottom:10px;"> @@ -348,8 +356,9 @@ <div class="memory-tab-panel hidden" data-memory-panel="skills"> <div class="admin-card" style="display:flex;flex-direction:column;overflow:hidden;flex:1;min-height:0;"> <div style="display:flex;align-items:center;gap:8px;margin-bottom:2px;"> - <h2 style="margin:0;padding:0;line-height:1;">Skills <span id="skills-count-h2" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal"></span></h2> + <h2 style="display:flex;align-items:center;gap:6px;margin:0;padding:0;line-height:1;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="color:var(--accent, var(--red));flex-shrink:0;"><polygon points="13 2 3 14 12 14 11 22 21 10 12 10 13 2"/></svg>Skills <span id="skills-count-h2" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal"></span></h2> <span style="flex:1"></span> + <span class="admin-toggle-state"></span> <label class="admin-switch" title="Inject relevant skills into chat context"><input type="checkbox" id="skills-enabled-header-toggle" checked /><span class="admin-slider"></span></label> </div> <p class="memory-desc doclib-desc" style="margin-top:6px;">Reusable procedures the AI can call via /skill — sort by confidence to surface the proven ones.</p> @@ -374,8 +383,8 @@ <option value="filter:conf70">Confidence ≤ 70%</option> </optgroup> </select> - <button id="skills-select-btn" class="memory-toolbar-btn" title="Select multiple skills">Select</button> - <button id="skills-audit-btn" class="memory-toolbar-btn" title="Test every skill, auto-fix the weak ones, flag what still fails"><svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;margin-right:3px;"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>Audit all</button> + <button id="skills-audit-btn" class="memory-toolbar-btn" title="Test every skill, auto-fix the weak ones, flag what still fails"><svg width="11" height="11" viewBox="0 0 24 24" fill="var(--accent, var(--red))" style="vertical-align:-1px;margin-right:3px;"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>Audit</button> + <button id="skills-select-btn" class="memory-toolbar-btn" title="Select multiple skills"><svg class="memory-select-btn-icon" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:3px;"><circle cx="12" cy="12" r="10"/><circle cx="12" cy="12" r="3" fill="currentColor" stroke="none"/></svg>Select</button> </div> <input type="text" id="skills-search" placeholder="Search skills…" class="memory-search-input" aria-label="Search skills" /> </div> @@ -395,34 +404,23 @@ <!-- ── Settings tab ── --> <div class="memory-tab-panel hidden" data-memory-panel="settings"> <div class="admin-card"> - <div style="display:flex;align-items:center;justify-content:space-between;gap:12px"> - <h2 style="margin:0">Auto-extract memories</h2> + <div style="display:flex;align-items:center;justify-content:space-between;gap:12px;min-height:32px"> + <h2 style="margin:0;display:inline-flex;align-items:center;gap:6px"><svg width="13" height="13" viewBox="0 0 24 24" fill="var(--accent, var(--red))" aria-hidden="true"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>Auto-extract memories</h2> <label class="admin-switch" style="flex-shrink:0"><input type="checkbox" id="auto-memory-toggle" checked /><span class="admin-slider"></span></label> </div> <span class="admin-toggle-sub" style="display:block;margin-top:6px">Automatically extract memories from conversations.</span> </div> <div class="admin-card"> - <div style="display:flex;align-items:center;justify-content:space-between;gap:12px"> - <h2 style="margin:0">Auto-extract skills</h2> + <div style="display:flex;align-items:center;justify-content:space-between;gap:12px;min-height:32px"> + <h2 style="margin:0;display:inline-flex;align-items:center;gap:6px"><svg width="13" height="13" viewBox="0 0 24 24" fill="var(--accent, var(--red))" aria-hidden="true"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>Auto-extract skills</h2> <label class="admin-switch" style="flex-shrink:0"><input type="checkbox" id="auto-skills-toggle" /><span class="admin-slider"></span></label> </div> <span class="admin-toggle-sub" style="display:block;margin-top:6px;opacity:0.6">Automatically draft reusable skills from your workflows. Audit all can publish passing skills using the threshold below.</span> <span class="admin-toggle-sub" style="display:block;margin-top:6px;opacity:0.6">The library can grow; cleanup retires weak/duplicate skills only after review.</span> </div> <div class="admin-card"> - <div style="display:flex;align-items:center;justify-content:space-between;gap:12px"> - <h2 style="margin:0">Inject Skills</h2> - </div> - <span class="admin-toggle-sub" style="display:block;margin-top:6px;opacity:0.6">Controls how many relevant published or approved skills are added to each agent request.</span> - <div style="display:flex;align-items:center;justify-content:space-between;gap:12px;margin-top:8px"> - <span class="admin-toggle-sub" style="margin:0">Max skills per request</span> - <input type="number" id="skill-max-input" min="0" max="12" step="1" value="3" aria-label="Max skills to inject" style="flex-shrink:0;width:72px;background:var(--input-bg,var(--panel));color:var(--fg);border:1px solid var(--border);border-radius:6px;padding:4px 6px;font-size:12px;text-align:right;font-variant-numeric:tabular-nums" /> - </div> - <span class="admin-toggle-sub" style="display:block;margin-top:6px;opacity:0.5">Set to 0 to disable skill injection.</span> - </div> - <div class="admin-card"> - <div style="display:flex;align-items:center;justify-content:space-between;gap:12px"> - <h2 style="margin:0">Auto-approve skills</h2> + <div style="display:flex;align-items:center;justify-content:space-between;gap:12px;min-height:32px"> + <h2 style="margin:0;display:inline-flex;align-items:center;gap:6px"><svg width="13" height="13" viewBox="0 0 24 24" fill="var(--accent, var(--red))" aria-hidden="true"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>Auto-approve skills</h2> <label class="admin-switch" style="flex-shrink:0"><input type="checkbox" id="auto-approve-skills-toggle" checked /><span class="admin-slider"></span></label> </div> <span class="admin-toggle-sub" style="display:block;margin-top:6px;opacity:0.6">Audit all publishes passing, necessary skills at or above this confidence. Off = keep audit results as drafts unless manually approved.</span> @@ -434,6 +432,17 @@ </span> </div> </div> + <div class="admin-card"> + <div style="display:flex;align-items:center;justify-content:space-between;gap:12px;min-height:32px"> + <h2 style="margin:0;display:inline-flex;align-items:center;gap:6px"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="opacity:0.7"><path d="M12 5v14"/><polyline points="6 11 12 17 18 11"/><path d="M5 20h14"/></svg>Inject Skills</h2> + </div> + <span class="admin-toggle-sub" style="display:block;margin-top:6px;opacity:0.6">Controls how many relevant published or approved skills are added to each agent request.</span> + <div style="display:flex;align-items:center;justify-content:space-between;gap:12px;margin-top:8px"> + <span class="admin-toggle-sub" style="margin:0">Max skills per request</span> + <input type="number" id="skill-max-input" min="0" max="12" step="1" value="3" aria-label="Max skills to inject" style="flex-shrink:0;width:72px;background:var(--input-bg,var(--panel));color:var(--fg);border:1px solid var(--border);border-radius:6px;padding:4px 6px;font-size:12px;text-align:right;font-variant-numeric:tabular-nums" /> + </div> + <span class="admin-toggle-sub" style="display:block;margin-top:6px;opacity:0.5">Set to 0 to disable skill injection.</span> + </div> </div> </div> </div> @@ -704,12 +713,9 @@ <div class="section-header-flex"> <span class="section-title" id="chats-section-title"><svg class="section-icon" width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"/></svg><span id="chats-section-label" class="section-title-label">Chats</span><span id="chats-notif-dot" class="sidebar-notif-dot" style="display:none"></span></span> <div style="position:relative; display:inline-block; display:flex; gap:4px; align-items:center;"> - <button type="button" class="section-header-btn chats-manage-btn" id="chats-library-btn" title="Manage Chats (Library)"> - <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> - <path d="M4 19.5A2.5 2.5 0 0 1 6.5 17H20"/> - <path d="M6.5 2H20v20H6.5A2.5 2.5 0 0 1 4 19.5v-15A2.5 2.5 0 0 1 6.5 2z"/> - <path d="M9 7h6M9 11h4"/> - </svg> + <button type="button" class="section-header-btn list-item-plus-btn chats-manage-btn" id="chats-library-btn" title="Manage Chats (Library)"> + <span aria-hidden="true" style="display:inline-block;width:13px;height:13px;"></span> + <span class="list-item-plus-label">manage</span> </button> <button type="button" class="section-header-btn" id="session-sort-btn" title="Sort sessions"> <svg class="sort-icon" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> @@ -726,14 +732,11 @@ <div class="dropdown-item sort-option sort-dropdown-item" data-sort="newest">Newest First</div> <div class="dropdown-item sort-option sort-dropdown-item" data-sort="group">By Folder</div> <div class="dropdown-item sort-dropdown-item sort-dropdown-sep" id="auto-sort-sessions-row" style="display:flex;align-items:center;padding:0;"> - <span id="auto-sort-sessions-btn" style="flex:1;padding:5px 10px;cursor:pointer;display:inline-flex;align-items:center;gap:4px;"> - <span class="auto-sort-icon"><svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;margin-right:2px;"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg> Tidy</span> + <span id="auto-sort-sessions-btn" style="flex:1;padding:5px 10px 5px 4px;cursor:pointer;display:inline-flex;align-items:center;gap:6px;"> + <span class="auto-sort-icon"><svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg></span> + <span>Group</span> <span class="auto-sort-spinner" style="display:none;">Sorting...</span> </span> - <button type="button" id="auto-sort-sessions-more" title="Tidy options" aria-label="Tidy options" style="background:none;border:none;border-left:1px solid var(--border);color:inherit;cursor:pointer;padding:5px 8px;font-size:9px;opacity:0.7;"><svg width="8" height="8" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg></button> - </div> - <div class="dropdown-item sort-dropdown-item" id="auto-sort-sessions-noai-btn" style="display:none;padding-left:24px;"> - Tidy <span class="auto-sort-noai-spinner" style="display:none;font-size:9px;opacity:0.6;margin-left:4px;">Cleaning...</span> </div> <div class="dropdown-item rearrange-toggle sort-dropdown-item sort-dropdown-sep" id="session-rearrange-toggle"> ↑↓ Rearrange <span class="rearrange-check" style="float:right; opacity:0;">•</span> @@ -1040,6 +1043,13 @@ <span>RAG</span> <span class="overflow-active-dot"></span> </button> + <button type="button" class="overflow-menu-item" id="overflow-workspace-btn"> + <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> + <path d="M3 7a2 2 0 0 1 2-2h4l2 2h8a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2z"/> + </svg> + <span>Workspace</span> + <span class="overflow-active-dot"></span> + </button> <!-- Inline "deep research mode" toggle removed (superseded by the Deep Research sidebar / trigger_research). The hidden #research-toggle checkbox is kept inert so existing JS refs @@ -1071,6 +1081,12 @@ <polyline points="4 17 10 11 4 5"/><line x1="12" y1="19" x2="20" y2="19"/> </svg> </button> + <!-- Workspace indicator (hidden until a folder is set) --> + <button type="button" class="input-icon-btn tool-indicator" title="Workspace - click to clear" id="workspace-indicator-btn" aria-label="Clear workspace" style="display:none;"> + <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 7a2 2 0 0 1 2-2h4l2 2h8a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2z"/></svg> + <span style="font-size:11px;margin-left:2px;max-width:120px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" id="workspace-indicator-name"></span> + <svg class="tool-indicator-x" width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round"><line x1="6" y1="6" x2="18" y2="18"/><line x1="18" y1="6" x2="6" y2="18"/></svg> + </button> <!-- RAG toolbar indicator (hidden until active) --> <button type="button" class="input-icon-btn tool-indicator" title="RAG active — click to deactivate" id="rag-indicator-btn" style="display:none;"> <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"> @@ -1317,7 +1333,6 @@ </button> <button class="close-btn" aria-label="Close settings">✖</button> </div> - <div class="admin-toggle-sub" style="padding:0 12px 8px;opacity:0.6;font-size:11px;">Toggle on/off visibility of tools and modules across the interface.</div> <div class="settings-layout"> <div class="settings-sidebar"> <!-- Section 1: AI plumbing (Add Models → AI Defaults → Search) --> @@ -1325,6 +1340,10 @@ <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="2" y="2" width="20" height="8" rx="2"/><rect x="2" y="14" width="20" height="8" rx="2"/><circle cx="6" cy="6" r="1"/><circle cx="6" cy="18" r="1"/></svg> <span>Add Models</span> </button> + <button class="settings-nav-item" data-settings-tab="added-models"> + <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg> + <span>Added Models</span> + </button> <button class="settings-nav-item" data-settings-tab="ai"> <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M12 2a4 4 0 0 0-4 4v2H6a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V10a2 2 0 0 0-2-2h-2V6a4 4 0 0 0-4-4z"/></svg> <span>AI Defaults</span> @@ -1391,14 +1410,21 @@ <div class="settings-col"> <div class="settings-row"> <label class="settings-label">Endpoint</label> + <span class="adm-model-logo" id="set-defaultEpSelect-logo" style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;flex-shrink:0;opacity:0.9;color:var(--fg);"></span> <select id="set-defaultEpSelect" class="settings-select"></select> </div> <div class="settings-row"> <label class="settings-label">Model</label> + <span class="adm-model-logo" id="set-defaultModelSelect-logo" style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;flex-shrink:0;opacity:0.9;color:var(--fg);"></span> <select id="set-defaultModelSelect" class="settings-select"></select> </div> - <div id="set-defaultFallbacks" class="settings-fallbacks"></div> - <button type="button" class="settings-fallback-add" id="set-defaultAddFallback" title="Add a model to try if the one above fails">+ Add fallback</button> + <div class="settings-row" style="align-items:flex-start;"> + <label class="settings-label" style="margin-top:6px;">Fallbacks</label> + <div style="flex:1;display:flex;flex-direction:column;gap:6px;"> + <div id="set-defaultFallbacks" class="settings-fallbacks"></div> + <button type="button" class="settings-fallback-add" id="set-defaultAddFallback" title="Add a model to try if the one above fails">+ Add fallback</button> + </div> + </div> <div id="set-defaultChatMsg" style="font-size:11px;color:color-mix(in srgb, var(--fg) 45%, transparent);"></div> </div> </div> @@ -1408,14 +1434,21 @@ <div class="settings-col"> <div class="settings-row"> <label class="settings-label">Endpoint</label> + <span class="adm-model-logo" id="set-utilityEpSelect-logo" style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;flex-shrink:0;opacity:0.9;color:var(--fg);"></span> <select id="set-utilityEpSelect" class="settings-select"><option value="">—</option></select> </div> <div class="settings-row"> <label class="settings-label">Model</label> + <span class="adm-model-logo" id="set-utilityModelSelect-logo" style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;flex-shrink:0;opacity:0.9;color:var(--fg);"></span> <select id="set-utilityModelSelect" class="settings-select"><option value="">—</option></select> </div> - <div id="set-utilityFallbacks" class="settings-fallbacks"></div> - <button type="button" class="settings-fallback-add" id="set-utilityAddFallback" title="Add a model to try if the utility model fails">+ Add fallback</button> + <div class="settings-row" style="align-items:flex-start;"> + <label class="settings-label" style="margin-top:6px;">Fallbacks</label> + <div style="flex:1;display:flex;flex-direction:column;gap:6px;"> + <div id="set-utilityFallbacks" class="settings-fallbacks"></div> + <button type="button" class="settings-fallback-add" id="set-utilityAddFallback" title="Add a model to try if the utility model fails">+ Add fallback</button> + </div> + </div> <div id="set-utilityChatMsg" style="font-size:11px;color:color-mix(in srgb, var(--fg) 45%, transparent);"></div> </div> </div> @@ -1425,16 +1458,22 @@ <div style="display:flex;flex-direction:column;gap:0.5rem;"> <div style="display:flex;align-items:center;gap:0.75rem;"> <label class="settings-label">Model</label> + <span class="adm-model-logo" id="set-vlModelSelect-logo" style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;flex-shrink:0;opacity:0.9;color:var(--fg);"></span> <select id="set-vlModelSelect" class="settings-select"><option value="">Auto-detect</option></select> </div> - <div id="set-visionFallbacks" class="settings-fallbacks"></div> - <button type="button" class="settings-fallback-add" id="set-visionAddFallback" title="Add a vision model to try if the one above fails">+ Add fallback</button> + <div class="settings-row" style="align-items:flex-start;"> + <label class="settings-label" style="margin-top:6px;">Fallbacks</label> + <div style="flex:1;display:flex;flex-direction:column;gap:6px;"> + <div id="set-visionFallbacks" class="settings-fallbacks"></div> + <button type="button" class="settings-fallback-add" id="set-visionAddFallback" title="Add a vision model to try if the one above fails">+ Add fallback</button> + </div> + </div> <div id="set-visionSettingsMsg" style="font-size:11px;color:color-mix(in srgb, var(--fg) 45%, transparent);"></div> </div> </div> <div class="admin-card"> <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><circle cx="11" cy="11" r="8"/><path d="M21 21l-4.35-4.35"/><line x1="11" y1="8" x2="11" y2="14"/><line x1="8" y1="11" x2="14" y2="11"/></svg>Research Model</h2> - <div class="admin-toggle-sub" style="margin-bottom:8px">Model used for Deep Research. Falls back to the default chat model if not set.</div> + <div class="admin-toggle-sub" style="margin-bottom:8px">Model used for Deep Research, more settings under <a href="#" data-go-settings-tab="search" style="color:var(--accent, var(--red));text-decoration:underline;font-weight:600;">Search →</a></div> <div class="settings-col"> <div class="settings-row"> <label class="settings-label">Endpoint</label> @@ -1444,48 +1483,17 @@ </div> <div class="settings-row"> <label class="settings-label">Model</label> + <span class="adm-model-logo" id="set-researchModel-logo" style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;flex-shrink:0;opacity:0.9;color:var(--fg);"></span> <select id="set-researchModel" class="settings-select"> <option value="">Same as chat</option> </select> </div> - <div class="settings-row"> - <label class="settings-label">Search</label> - <select id="set-researchSearch" class="settings-select"> - <option value="">Same as web search</option> - <option value="searxng">SearXNG</option> - <option value="duckduckgo">DuckDuckGo</option> - <option value="tavily">Tavily</option> - <option value="brave">Brave</option> - <option value="google">Google</option> - <option value="serper">Serper</option> - </select> - </div> - <div class="settings-row"> - <label class="settings-label">Max Tokens</label> - <input id="set-researchMaxTokens" type="text" inputmode="numeric" placeholder="8192 (default)" class="settings-select" style="width:120px;"> - </div> - <div class="settings-row"> - <label class="settings-label">Extract Timeout</label> - <input id="set-researchExtractTimeout" type="text" inputmode="numeric" placeholder="90 sec" class="settings-select" style="width:120px;"> - </div> - <div class="settings-row"> - <label class="settings-label">Extract Parallel</label> - <input id="set-researchExtractConcurrency" type="text" inputmode="numeric" placeholder="3" class="settings-select" style="width:120px;"> - </div> - <div class="settings-row"> - <label class="settings-label">Max Time</label> - <input id="set-researchRunTimeout" type="text" inputmode="numeric" placeholder="1800 sec (0 = no limit)" class="settings-select" style="width:120px;"> - </div> - <div id="set-researchMsg" style="font-size:11px;color:color-mix(in srgb, var(--fg) 45%, transparent);"></div> + <div id="set-researchMsg" style="font-size:11px;color:color-mix(in srgb, var(--fg) 45%, transparent);margin-top:2px;"></div> </div> </div> <!-- Agent card moved to the Agent Tools tab. --> - <!-- Image Generation removed — only inpaint remains in this build, - and inpaint is configured via the gallery editor not this card. - Keeping the DOM (hidden) so JS wiring against the inputs - doesn't throw. --> - <div class="admin-card" hidden style="display:none"> - <h2 style="display:flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:1px;opacity:0.6;flex-shrink:0"><rect x="3" y="3" width="18" height="18" rx="2"/><circle cx="8.5" cy="8.5" r="1.5"/><path d="M21 15l-5-5L5 21"/></svg>Image Generation<span style="flex:1"></span><label class="admin-switch"><input type="checkbox" id="set-imgEnabledToggle" checked><span class="admin-slider"></span></label></h2> + <div class="admin-card"> + <h2 style="display:flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:1px;opacity:0.6;flex-shrink:0"><rect x="3" y="3" width="18" height="18" rx="2"/><circle cx="8.5" cy="8.5" r="1.5"/><path d="M21 15l-5-5L5 21"/></svg>Image Generation<span style="flex:1"></span><label class="admin-switch"><input type="checkbox" id="set-imgEnabledToggle"><span class="admin-slider"></span></label></h2> <div class="admin-toggle-sub" style="margin-bottom:8px">Configure which model to use for image generation.</div> <div style="display:flex;flex-direction:column;gap:0.5rem;"> <div style="display:flex;align-items:center;gap:0.75rem;"> @@ -1557,20 +1565,15 @@ <div id="set-ttsSettingsMsg" style="font-size:11px;color:color-mix(in srgb, var(--fg) 45%, transparent);"></div> </div> </div> + <!-- Teacher Model settings card hidden as part of the 2.0 + "harden the core" pass. The escalation flow is dormant when + `teacher_model` is unset (its default), so the backend keeps + working for anyone who wired it via `manage_settings` / + settings backup. Re-add this card to surface the toggle + again once the core experience is faster. --> <div class="admin-card"> - <h2 style="display:flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M22 10v6M2 10l10-5 10 5-10 5z"/><path d="M6 12v5c3 3 9 3 12 0v-5"/></svg>Teacher Model <span style="font-size:0.72em;opacity:0.55;font-weight:normal;">(Experimental)</span><span style="flex:1"></span><label class="admin-switch"><input type="checkbox" id="set-teacherEnabledToggle"><span class="admin-slider"></span></label></h2> - <div class="admin-toggle-sub" style="margin-bottom:8px">When a self-hosted student fails an agent-mode task, escalate to a SOTA teacher that writes a SKILL.md procedure so the student can do it next time. Off by default.</div> - <div class="settings-col"> - <div class="settings-row"> - <label class="settings-label">Endpoint</label> - <select id="set-teacherEpSelect" class="settings-select"><option value="">—</option></select> - </div> - <div class="settings-row"> - <label class="settings-label">Model</label> - <select id="set-teacherModelSelect" class="settings-select"><option value="">—</option></select> - </div> - <div id="set-teacherChatMsg" style="font-size:11px;color:color-mix(in srgb, var(--fg) 45%, transparent);"></div> - </div> + <h2 style="display:flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:1px;opacity:0.6;flex-shrink:0"><rect x="2" y="4" width="20" height="16" rx="2"/><polyline points="2 6 12 13 22 6"/></svg>Email Safety<span style="flex:1"></span><label class="admin-switch" title="When on, agent send_email and reply_to_email tools stage a draft for your approval instead of sending immediately."><input type="checkbox" id="set-agentEmailConfirm" checked><span class="admin-slider"></span></label></h2> + <div class="admin-toggle-sub" style="margin-bottom:8px">When on, agent <code>send_email</code> / <code>reply_to_email</code> tools stage a draft for your approval (in the chat) instead of SMTPing immediately. Stops models from inventing a signature and sending it to a real recipient before you can review.</div> </div> </div> @@ -1601,10 +1604,12 @@ <option value="serper" data-search-logo="serper">Serper.dev</option> <option value="disabled" data-search-logo="disabled">Disabled</option> </select> - <button type="button" class="admin-btn-sm" id="set-searchTestBtn" title="Run a test query against the configured provider" style="margin-left:6px;flex-shrink:0;position:relative;top:2px;">Test</button> + <button type="button" class="admin-btn-sm" id="set-searchTestBtn" title="Run a test query against the configured provider" style="margin-left:2px;flex-shrink:0;position:relative;top:2px;display:inline-flex;align-items:center;gap:4px;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polygon points="5 3 19 12 5 21 5 3"/></svg>Test + </button> </div> <div class="settings-row"> - <label class="settings-label">Results</label> + <label class="settings-label" title="How many web search results to fetch per query">Results per query</label> <div style="display:flex;gap:8px;flex:1;"> <select id="set-searchResultCount" class="settings-select" style="flex:1;"> <option value="3">3</option> @@ -1618,30 +1623,78 @@ </div> <div id="set-searchUrlRow" class="settings-row"> <label class="settings-label">URL</label> - <input id="set-searchUrl" type="text" placeholder="http://localhost:8080" class="settings-select"> + <input id="set-searchUrl" type="text" placeholder="http://localhost:8080 (optional)" class="settings-select"> </div> <div id="set-searchKeyRow" class="settings-row" style="display:none;"> <label class="settings-label">API Key</label> - <input id="set-searchApiKey" type="password" placeholder="API key" class="settings-select"> + <div style="position:relative;flex:1;display:flex;align-items:center;"> + <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="position:absolute;left:9px;top:50%;transform:translateY(-50%);opacity:0.55;pointer-events:none;"><path d="M21 2l-9.6 9.6"/><circle cx="7.5" cy="15.5" r="5.5"/><path d="M15.5 7.5l3 3"/></svg> + <input id="set-searchApiKey" type="password" placeholder="API key" class="settings-select" style="flex:1;padding-left:28px;"> + </div> </div> <div id="set-searchCxRow" class="settings-row" style="display:none;"> <label class="settings-label">CX ID</label> <input id="set-searchCx" type="text" placeholder="Google PSE engine ID" class="settings-select"> </div> - <div class="settings-row"> - <label class="settings-label" title="Providers tried in order when the primary fails or hits a rate limit">Fallbacks</label> - <div class="search-fallback-chain" id="set-searchFallbackChain"></div> + <div class="settings-row" style="align-items:flex-start;"> + <label class="settings-label" style="margin-top:6px;" title="Providers tried in order when the primary fails or hits a rate limit">Fallbacks</label> + <div style="flex:1;display:flex;flex-direction:column;gap:6px;"> + <div class="settings-fallbacks" id="set-searchFallbackChain"></div> + <button type="button" class="settings-fallback-add" id="set-searchAddFallback" title="Add a search provider to try if the primary fails">+ Add fallback</button> + </div> </div> <div id="set-searchHint" class="admin-toggle-sub"></div> <div id="set-searchMsg" style="font-size:11px;"></div> </div> </div> + + <div class="admin-card"> + <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><circle cx="11" cy="11" r="8"/><path d="M21 21l-4.35-4.35"/><line x1="11" y1="8" x2="11" y2="14"/><line x1="8" y1="11" x2="14" y2="11"/></svg>Deep Research</h2> + <div class="admin-toggle-sub" style="margin-bottom:8px">Deep Research runtime settings. Default Model is picked in <a href="#" data-go-settings-tab="ai" style="color:var(--accent, var(--red));text-decoration:underline;font-weight:600;">AI Defaults →</a></div> + <div class="settings-col"> + <div class="settings-row"> + <label class="settings-label">Search</label> + <span style="margin-left:auto;display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;flex-shrink:0;color:var(--fg);" id="set-researchSearch-logo"></span> + <select id="set-researchSearch" class="settings-select" style="width:358.5px;flex:0 0 auto;max-width:calc(100% - 24px);"> + <option value="" data-search-logo="">Same as web search</option> + <option value="searxng" data-search-logo="searxng">SearXNG</option> + <option value="duckduckgo" data-search-logo="duckduckgo">DuckDuckGo</option> + <option value="tavily" data-search-logo="tavily">Tavily</option> + <option value="brave" data-search-logo="brave">Brave</option> + <option value="google" data-search-logo="google_pse">Google</option> + <option value="serper" data-search-logo="serper">Serper</option> + </select> + </div> + <div class="settings-row"> + <label class="settings-label">Max Tokens</label> + <input id="set-researchMaxTokens" type="text" inputmode="numeric" placeholder="8192 (default)" class="settings-select" style="width:382.5px;flex:0 0 auto;margin-left:auto;"> + </div> + <div class="settings-row"> + <label class="settings-label">Extract Timeout</label> + <div style="position:relative;width:382.5px;flex:0 0 auto;margin-left:auto;"> + <input id="set-researchExtractTimeout" type="text" inputmode="numeric" placeholder="90 sec" class="settings-select" style="width:100%;padding-right:30px;"> + <span title="How long the researcher waits for a single URL to fetch and extract before giving up on it. Slow sites get skipped. Default 90 seconds." style="position:absolute;right:8px;top:50%;transform:translateY(-50%);width:16px;height:16px;border-radius:50%;border:1px solid var(--border);display:inline-flex;align-items:center;justify-content:center;font-size:10px;font-weight:600;opacity:0.55;cursor:help;user-select:none;">?</span> + </div> + </div> + <div class="settings-row"> + <label class="settings-label">Extract Parallel</label> + <div style="position:relative;width:382.5px;flex:0 0 auto;margin-left:auto;"> + <input id="set-researchExtractConcurrency" type="text" inputmode="numeric" placeholder="3" class="settings-select" style="width:100%;padding-right:30px;"> + <span title="How many URLs the researcher fetches and extracts in parallel. Higher is faster but uses more memory/CPU. Default 3." style="position:absolute;right:8px;top:50%;transform:translateY(-50%);width:16px;height:16px;border-radius:50%;border:1px solid var(--border);display:inline-flex;align-items:center;justify-content:center;font-size:10px;font-weight:600;opacity:0.55;cursor:help;user-select:none;">?</span> + </div> + </div> + <div class="settings-row"> + <label class="settings-label">Timeout</label> + <input id="set-researchRunTimeout" type="text" inputmode="numeric" placeholder="1800 sec (0 = no limit)" class="settings-select" style="width:382.5px;flex:0 0 auto;margin-left:auto;"> + </div> + </div> + </div> </div> <!-- ═══ APPEARANCE TAB ═══ --> <div data-settings-panel="appearance" class="settings-appearance-panel hidden"> <div class="admin-card" style="padding-bottom:6px;"> - <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><rect x="3" y="3" width="18" height="18" rx="2"/><line x1="9" y1="3" x2="9" y2="21"/></svg>Sidebar</h2> + <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><rect x="3" y="3" width="18" height="18" rx="2"/><line x1="9" y1="3" x2="9" y2="21"/></svg>Sidebar<span style="flex:1"></span><button type="button" class="vis-reset-btn" data-vis-reset title="Reset this section to defaults" aria-label="Reset Sidebar to defaults" style="background:none;border:none;padding:2px 4px;cursor:pointer;color:inherit;opacity:0.55;display:inline-flex;align-items:center;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="1 4 1 10 7 10"/><path d="M3.51 15a9 9 0 1 0 2.13-9.36L1 10"/></svg></button></h2> <div class="vis-toggles"> <label class="vis-row"> <span class="vis-icon"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"><circle cx="12" cy="12" r="10"/><path d="M8 12l2.5 2.5L16 9"/></svg></span> @@ -1741,7 +1794,7 @@ </div> </div> <div class="admin-card" style="padding-bottom:6px;"> - <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M21 11.5a8.38 8.38 0 0 1-.9 3.8 8.5 8.5 0 0 1-7.6 4.7 8.38 8.38 0 0 1-3.8-.9L3 21l1.9-5.7a8.38 8.38 0 0 1-.9-3.8 8.5 8.5 0 0 1 4.7-7.6 8.38 8.38 0 0 1 3.8-.9h.5a8.48 8.48 0 0 1 8 8v.5z"/></svg>Chat Area</h2> + <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M21 11.5a8.38 8.38 0 0 1-.9 3.8 8.5 8.5 0 0 1-7.6 4.7 8.38 8.38 0 0 1-3.8-.9L3 21l1.9-5.7a8.38 8.38 0 0 1-.9-3.8 8.5 8.5 0 0 1 4.7-7.6 8.38 8.38 0 0 1 3.8-.9h.5a8.48 8.48 0 0 1 8 8v.5z"/></svg>Chat Area<span style="flex:1"></span><button type="button" class="vis-reset-btn" data-vis-reset title="Reset this section to defaults" aria-label="Reset Chat Area to defaults" style="background:none;border:none;padding:2px 4px;cursor:pointer;color:inherit;opacity:0.55;display:inline-flex;align-items:center;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="1 4 1 10 7 10"/><path d="M3.51 15a9 9 0 1 0 2.13-9.36L1 10"/></svg></button></h2> <div class="vis-toggles"> <label class="vis-row"> <span class="vis-icon"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M4 6h16"/><path d="M4 10h8"/></svg></span> @@ -1776,7 +1829,7 @@ </div> </div> <div class="admin-card" style="padding-bottom:6px;"> - <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><line x1="17" y1="10" x2="3" y2="10"/><line x1="21" y1="6" x2="3" y2="6"/><line x1="21" y1="14" x2="3" y2="14"/><line x1="17" y1="18" x2="3" y2="18"/></svg>Chat Bar</h2> + <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><line x1="17" y1="10" x2="3" y2="10"/><line x1="21" y1="6" x2="3" y2="6"/><line x1="21" y1="14" x2="3" y2="14"/><line x1="17" y1="18" x2="3" y2="18"/></svg>Chat Bar<span style="flex:1"></span><button type="button" class="vis-reset-btn" data-vis-reset title="Reset this section to defaults" aria-label="Reset Chat Bar to defaults" style="background:none;border:none;padding:2px 4px;cursor:pointer;color:inherit;opacity:0.55;display:inline-flex;align-items:center;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="1 4 1 10 7 10"/><path d="M3.51 15a9 9 0 1 0 2.13-9.36L1 10"/></svg></button></h2> <div class="vis-toggles"> <label class="vis-row"> <span class="vis-icon"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="11" cy="11" r="8"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg></span> @@ -1820,9 +1873,6 @@ </label> </div> </div> - <div style="text-align:right;padding:0 4px;"> - <button type="button" class="admin-btn-sm" id="set-uiVisResetBtn" style="opacity:0.5;">Reset All</button> - </div> </div> <!-- ═══ THEME TAB ═══ --> @@ -1835,7 +1885,7 @@ <h2 style="margin:0;font-size:13px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><rect x="2" y="4" width="20" height="16" rx="2"/><path d="M6 8h.01M10 8h.01M14 8h.01M18 8h.01M8 12h.01M12 12h.01M16 12h.01M7 16h10"/></svg>Keyboard Shortcuts</h2> <p style="font-size:10px;opacity:0.4;margin:2px 0 0;">Click a shortcut to rebind. Press Escape to cancel.</p> </div> - <button type="button" class="shortcut-action-btn is-reset" id="shortcuts-reset-btn" title="Reset Shortcuts" style="width:28px;height:28px;font-size:15px;">↩</button> + <button type="button" class="vis-reset-btn" id="shortcuts-reset-btn" title="Reset shortcuts to defaults" aria-label="Reset shortcuts to defaults" style="background:none;border:none;padding:2px 4px;cursor:pointer;color:inherit;opacity:0.55;display:inline-flex;align-items:center;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="1 4 1 10 7 10"/><path d="M3.51 15a9 9 0 1 0 2.13-9.36L1 10"/></svg></button> </div> <div class="admin-card"> <div id="shortcuts-list"></div> @@ -1847,7 +1897,7 @@ <div data-settings-panel="account" class="hidden"> <div class="admin-card"> <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M20 21v-2a4 4 0 0 0-4-4H8a4 4 0 0 0-4 4v2"/><circle cx="12" cy="7" r="4"/></svg>Account</h2> - <div style="display:flex;align-items:center;gap:10px;margin:4px 0 12px;"> + <div style="display:flex;align-items:center;gap:10px;margin:12px 0 12px;"> <div class="user-bar-avatar" id="settings-account-avatar" style="width:32px;height:32px;font-size:14px;"></div> <div style="flex:1;"> <div id="settings-account-username" style="font-size:13px;font-weight:600;"></div> @@ -1863,7 +1913,7 @@ <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><rect x="3" y="11" width="18" height="11" rx="2" ry="2"/><path d="M7 11V7a5 5 0 0 1 10 0v4"/></svg>Change Password</h2> <div class="settings-col"> <input id="settings-pw-current" type="password" placeholder="Current password" autocomplete="current-password" style="padding:6px 8px;background:var(--bg);border:1px solid var(--border);border-radius:4px;color:var(--fg);font-family:inherit;font-size:12px;"> - <input id="settings-pw-new" type="password" placeholder="New password (min 8)" autocomplete="new-password" style="padding:6px 8px;background:var(--bg);border:1px solid var(--border);border-radius:4px;color:var(--fg);font-family:inherit;font-size:12px;"> + <input id="settings-pw-new" type="password" placeholder="New password" autocomplete="new-password" style="padding:6px 8px;background:var(--bg);border:1px solid var(--border);border-radius:4px;color:var(--fg);font-family:inherit;font-size:12px;"> <input id="settings-pw-confirm" type="password" placeholder="Confirm new password" autocomplete="new-password" style="padding:6px 8px;background:var(--bg);border:1px solid var(--border);border-radius:4px;color:var(--fg);font-family:inherit;font-size:12px;"> <div class="settings-row" style="margin-top:2px;justify-content:flex-end;"> <span id="settings-pw-msg" style="font-size:11px;margin-right:auto;"></span> @@ -1885,7 +1935,7 @@ <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><rect x="2" y="4" width="20" height="16" rx="2"/><path d="m22 7-8.97 5.7a1.94 1.94 0 0 1-2.06 0L2 7"/></svg>Email Accounts</h2> <div class="settings-row" style="align-items:center;"> <div class="admin-toggle-sub" style="margin:0;flex:1;">Add, edit, delete, and test accounts in Integrations.</div> - <button class="admin-btn-add" id="set-email-open-integrations" style="display:inline-flex;align-items:center;gap:6px;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="opacity:0.7"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"/><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"/></svg>Manage in Integrations</button> + <button class="admin-btn-add" id="set-email-open-integrations" style="display:inline-flex;align-items:center;gap:6px;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="opacity:0.7"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"/><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"/></svg>Open Integrations</button> </div> </div> @@ -1901,10 +1951,10 @@ <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.121 2.121 0 0 1 3 3L12 15l-4 1 1-4 9.5-9.5z"/></svg>Writing Style</h2> <div class="admin-toggle-sub" style="margin-bottom:8px">AI-extracted from your sent emails. Used when AI drafts replies.</div> <div class="settings-col"> - <textarea id="set-email-style" rows="4" class="settings-select" style="font-family:inherit;resize:vertical" placeholder="e.g. I write emails in this style. I don't use exclamation marks. I sign emails with: ..."></textarea> + <textarea id="set-email-style" rows="6" class="settings-select" style="font-family:inherit;resize:none" placeholder="e.g. I write emails in this style. I don't use exclamation marks. I sign emails with: ..."></textarea> <div class="settings-row" style="margin-top:4px"> <span id="set-email-style-msg" style="font-size:11px;"></span> - <button class="admin-btn-add" id="set-email-style-extract" style="margin-left:auto;">Extract from Sent (15 emails)</button> + <button class="admin-btn-add" id="set-email-style-extract" style="margin-left:auto;display:inline-flex;align-items:center;gap:5px;"><svg width="12" height="12" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>Extract from Sent (15 emails)</button> <button class="admin-btn-add" id="set-email-style-save">Save</button> </div> </div> @@ -1914,7 +1964,7 @@ <!-- ═══ REMINDERS TAB ═══ --> <div data-settings-panel="reminders" class="hidden"> <div class="admin-card"> - <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M18 8A6 6 0 0 0 6 8c0 7-3 9-3 9h18s-3-2-3-9"/><path d="M13.73 21a2 2 0 0 1-3.46 0"/></svg>How you're reminded</h2> + <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M18 8A6 6 0 0 0 6 8c0 7-3 9-3 9h18s-3-2-3-9"/><path d="M13.73 21a2 2 0 0 1-3.46 0"/></svg>How you're reminded<span style="flex:1"></span><span id="set-reminder-test-msg" style="font-size:11px;font-weight:normal;"></span><button class="admin-btn-sm" id="set-reminder-test-btn" style="font-size:11px;font-weight:normal;display:inline-flex;align-items:center;gap:4px;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polygon points="5 3 19 12 5 21 5 3"/></svg>Test</button></h2> <div class="admin-toggle-sub" style="margin-bottom:8px">Controls how fired note reminders are delivered.</div> <div class="settings-col"> <div class="settings-row"> @@ -1952,7 +2002,19 @@ </div> <div class="admin-card"> <h2 style="display:flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:1px;opacity:0.6;flex-shrink:0"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>AI Synthesis<span style="flex:1"></span><label class="admin-switch" title="Use the utility model to write reminder messages"><input type="checkbox" id="set-reminder-llm-toggle"><span class="admin-slider"></span></label></h2> - <div class="admin-toggle-sub" style="margin-bottom:8px">When on, the utility model writes a short, warm one-line reminder for browser, email, ntfy, AND webhook reminders instead of just the raw note content.</div> + <div class="admin-toggle-sub" style="margin-bottom:8px">When on, the utility model writes a short, warm one-line reminder for browser, email, ntfy, and webhook reminders instead of just the raw note content.</div> + <div class="settings-col"> + <div class="settings-row"> + <label class="settings-label" title="Optional — write the reminder in the voice of a saved character">Persona</label> + <select id="set-reminder-llm-persona" class="settings-select" style="flex:1;"> + <option value="">Default (warm, neutral)</option> + </select> + </div> + <div style="font-size:11px;opacity:0.7;margin-top:2px;"> + <a href="#" data-open-prompt-modal style="color:var(--accent, var(--red));text-decoration:underline;font-weight:600;">Edit persona settings here →</a> + </div> + <div id="set-reminder-llm-persona-msg" style="font-size:11px;color:color-mix(in srgb, var(--fg) 55%, transparent);"></div> + </div> </div> <div class="admin-card"> <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"/><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"/></svg>Public App URL</h2> @@ -1965,14 +2027,6 @@ <div id="set-app-public-url-msg" style="font-size:11px;color:color-mix(in srgb, var(--fg) 55%, transparent);"></div> </div> </div> - <div class="admin-card"> - <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M22 11.08V12a10 10 0 1 1-5.93-9.14"/><polyline points="22 4 12 14.01 9 11.01"/></svg>Test</h2> - <div class="admin-toggle-sub" style="margin-bottom:8px">Fire a test reminder using your current settings to verify everything works.</div> - <div class="settings-row"> - <span id="set-reminder-test-msg" style="font-size:11px;"></span> - <button class="admin-btn-add" id="set-reminder-test-btn" style="margin-left:auto;">Send Test Reminder</button> - </div> - </div> </div> <!-- ═══ ADMIN: USERS TAB ═══ --> @@ -1995,7 +2049,7 @@ <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M16 21v-2a4 4 0 0 0-4-4H5a4 4 0 0 0-4 4v2"/><circle cx="8.5" cy="7" r="4"/><line x1="20" y1="8" x2="20" y2="14"/><line x1="23" y1="11" x2="17" y2="11"/></svg>Add User</h2> <div class="admin-add-form"> <input id="adm-newUsername" type="text" placeholder="Username"> - <input id="adm-newPassword" type="password" placeholder="Password (min 8)"> + <input id="adm-newPassword" type="password" placeholder="Password"> <div class="admin-switch-inline" title="Grant full admin access"><label class="admin-switch"><input type="checkbox" id="adm-newIsAdmin"><span class="admin-slider"></span></label> Admin</div> </div> <div class="settings-row" style="margin-top:6px;"> @@ -2007,75 +2061,96 @@ <!-- ═══ SERVICES TAB ═══ --> <div data-settings-panel="services"> - <div class="admin-card"> - <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><rect x="2" y="2" width="20" height="8" rx="2"/><rect x="2" y="14" width="20" height="8" rx="2"/><circle cx="6" cy="6" r="1"/><circle cx="6" cy="18" r="1"/></svg>Add Models <span style="opacity:0.45;font-weight:normal;font-size:0.82em">(Endpoints)</span></h2> - <div class="admin-toggle-sub" style="margin-bottom:10px">Connect local models first, or add a cloud API.</div> - <!-- Local subsection --> - <div class="adm-add-section collapsible collapsed" id="adm-add-local"> - <div class="adm-ep-section-head adm-section-toggle" role="button" tabindex="0" aria-expanded="false"> - <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-1px;margin-right:4px;"><rect x="2" y="3" width="20" height="14" rx="2"/><path d="M8 21h8"/><path d="M12 17v4"/></svg> - <span>Local</span> - <svg class="adm-section-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg> + <!-- ── Local card ─────────────────────────────────────────── --> + <div class="admin-card"> + <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><rect x="2" y="3" width="20" height="14" rx="2"/><path d="M8 21h8"/><path d="M12 17v4"/></svg>Add Local Models <span style="opacity:0.45;font-weight:normal;font-size:0.82em">(Endpoint)</span> + <span style="flex:1"></span> + <button class="admin-btn-sm" id="adm-epLocalTestBtn" style="font-size:11px;font-weight:normal;display:inline-flex;align-items:center;gap:4px;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polygon points="5 3 19 12 5 21 5 3"/></svg>Test + </button> + <div style="position:relative;display:inline-block;"> + <button class="admin-btn-sm" id="adm-epLocalMoreBtn" title="More options" aria-haspopup="true" aria-expanded="false" style="font-size:11px;font-weight:normal;padding:4px 8px;line-height:1;"> + <svg width="14" height="4" viewBox="0 0 14 4" fill="currentColor"><circle cx="2" cy="2" r="1.4"/><circle cx="7" cy="2" r="1.4"/><circle cx="12" cy="2" r="1.4"/></svg> + </button> + <div id="adm-epLocalMoreMenu" style="display:none;position:absolute;top:calc(100% + 4px);right:0;z-index:50;min-width:170px;padding:4px;background:var(--panel,var(--bg));border:1px solid var(--border);border-radius:8px;box-shadow:0 6px 20px rgba(0,0,0,0.22);flex-direction:column;gap:1px;"> + <button class="admin-btn-sm adm-more-item" id="adm-epDiscoverBtn" title="Scan your network for running model servers" style="background:none;border:0;border-radius:5px;padding:7px 9px;display:flex;align-items:center;gap:8px;width:100%;text-align:left;font-size:12px;font-weight:normal;color:var(--fg);cursor:pointer;"> + <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><circle cx="11" cy="11" r="8"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg>Scan network + </button> + <button class="admin-btn-sm adm-more-item" id="adm-epOllamaBtn" title="Fill the default Ollama endpoint" style="background:none;border:0;border-radius:5px;padding:7px 9px;display:flex;align-items:center;gap:8px;width:100%;text-align:left;font-size:12px;font-weight:normal;color:var(--fg);cursor:pointer;"><span class="adm-ollama-logo" style="display:inline-flex;width:13px;height:13px;"></span>Add Ollama</button> + <button class="admin-btn-sm adm-more-item" id="adm-epLocalKeyBtn" title="Show / hide the API key field" aria-expanded="false" aria-controls="adm-epLocalApiKey-row" style="background:none;border:0;border-radius:5px;padding:7px 9px;display:flex;align-items:center;gap:8px;width:100%;text-align:left;font-size:12px;font-weight:normal;color:var(--fg);cursor:pointer;"> + <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 2l-9.6 9.6"/><circle cx="7.5" cy="15.5" r="5.5"/><path d="M15.5 7.5l3 3"/></svg>API key + </button> + </div> </div> + </h2> + <div class="admin-toggle-sub" style="margin:0 0 10px 2px;">Add a local model server (Ollama, llama.cpp, vLLM).</div> + <div class="adm-add-section"> <div class="admin-model-form"> <div class="admin-model-form-row"> - <input id="adm-epLocalUrl" type="text" placeholder="Paste endpoint URL, e.g. http://localhost:11434/v1" style="flex:1"> - </div> - <!-- API key row stays in the DOM but is collapsed until the - user clicks the Key button on the action row. Local - endpoints rarely need a key; hiding it by default keeps - the form a single visual line. --> - <div class="admin-model-form-row" id="adm-epLocalApiKey-row" style="display:none;"> - <input id="adm-epLocalApiKey" type="password" placeholder="API key (optional — for protected local endpoints)" autocomplete="off" style="flex:1"> - </div> - <!-- Action row: LLM/Image type, Quickstart buttons (Scan, - Ollama), Key reveal toggle, Test, Add — all inline so - the Quickstart fold is gone and Type sits with the - primary actions. --> - <div class="admin-model-form-row"> - <label style="display:inline-flex;align-items:center;gap:4px;font-size:11px;opacity:0.6;flex-shrink:0;">Type:<select id="adm-epLocalType" style="padding:5px;width:72px;flex-shrink:0;"> - <option value="llm" selected>LLM</option> - <option value="image">Image</option> - </select></label> - <button class="admin-btn-sm" id="adm-epDiscoverBtn" title="Scan your network for running model servers" style="display:inline-flex;align-items:center;gap:4px;"> - <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><circle cx="11" cy="11" r="8"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg>Scan - </button> - <button class="admin-btn-sm" id="adm-epOllamaBtn" title="Fill the default Ollama endpoint" style="display:inline-flex;align-items:center;gap:5px;"><span class="adm-ollama-logo" style="display:inline-flex;width:13px;height:13px;"></span>Ollama</button> - <span style="flex:1"></span> - <button class="admin-btn-sm" id="adm-epLocalKeyBtn" title="Show / hide the API key field" aria-expanded="false" aria-controls="adm-epLocalApiKey-row" style="opacity:0.75;display:inline-flex;align-items:center;gap:4px;"> - <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 2l-9.6 9.6"/><circle cx="7.5" cy="15.5" r="5.5"/><path d="M15.5 7.5l3 3"/></svg>API - </button> - <button class="admin-btn-sm" id="adm-epLocalTestBtn" style="min-width:55px;text-align:center;display:inline-flex;align-items:center;justify-content:center;gap:4px;"> - <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polygon points="5 3 19 12 5 21 5 3"/></svg>Test - </button> - <button class="admin-btn-add" id="adm-epLocalAddBtn" style="min-width:55px;text-align:center;display:inline-flex;align-items:center;justify-content:center;gap:4px;"> + <div class="adm-fused-group" style="display:flex;flex:1 1 180px;min-width:0;"> + <select id="adm-epLocalType" style="padding:5px;width:66px;flex-shrink:0;border-top-right-radius:0;border-bottom-right-radius:0;border-right:0;"> + <option value="llm" selected>LLM</option> + <option value="image">Image</option> + </select> + <input id="adm-epLocalUrl" type="text" placeholder="Paste endpoint URL, e.g. http://localhost:11434/v1" style="flex:1;min-width:0;border-top-left-radius:0;border-bottom-left-radius:0;"> + </div> + <button class="admin-btn-add" id="adm-epLocalAddBtn" style="min-width:55px;text-align:center;display:inline-flex;align-items:center;justify-content:center;gap:4px;flex-shrink:0;"> <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg>Add </button> </div> + <div class="admin-model-form-row" id="adm-epLocalApiKey-row" style="display:none;"> + <input id="adm-epLocalApiKey" type="password" placeholder="API key (optional — for protected local endpoints)" autocomplete="off" style="flex:1"> + </div> <div id="adm-epLocalMsg" class="adm-ep-inline-msg"></div> </div> </div> + </div> - <!-- API subsection --> - <div class="adm-add-section collapsible collapsed" id="adm-add-api" style="margin-top:14px"> - <div class="adm-ep-section-head adm-section-toggle" role="button" tabindex="0" aria-expanded="false"> - <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-1px;margin-right:4px;"><circle cx="12" cy="12" r="10"/><line x1="2" y1="12" x2="22" y2="12"/><path d="M12 2a15.3 15.3 0 0 1 4 10 15.3 15.3 0 0 1-4 10 15.3 15.3 0 0 1-4-10 15.3 15.3 0 0 1 4-10z"/></svg> - <span>API</span> - <svg class="adm-section-caret" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg> - </div> - <div class="admin-model-form"> - <!-- Custom picker (with logos). Hidden native <select> mirrors - its value so the existing JS that reads adm-epProvider - keeps working unchanged. --> - <div class="adm-provider-picker adm-provider-combo" id="adm-provider-picker"> - <input id="adm-epUrl" type="text" placeholder="Base URL or pick provider" autocomplete="off"> - <button type="button" class="adm-provider-btn" id="adm-provider-btn" title="Pick provider"> - <span class="adm-provider-current"><span class="adm-provider-logo"></span><span class="adm-provider-name">Provider</span></span> - <svg class="adm-provider-caret" width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg> + <!-- ── API card ───────────────────────────────────────────── --> + <div class="admin-card"> + <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><circle cx="12" cy="12" r="10"/><line x1="2" y1="12" x2="22" y2="12"/><path d="M12 2a15.3 15.3 0 0 1 4 10 15.3 15.3 0 0 1-4 10 15.3 15.3 0 0 1-4-10 15.3 15.3 0 0 1 4-10z"/></svg>Add API Models <span style="opacity:0.45;font-weight:normal;font-size:0.82em">(Endpoint)</span> + <span style="flex:1"></span> + <button class="admin-btn-sm" id="adm-epApiTestBtn" style="font-size:11px;font-weight:normal;display:inline-flex;align-items:center;gap:4px;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polygon points="5 3 19 12 5 21 5 3"/></svg>Test + </button> + <button class="admin-btn-sm hidden" id="adm-epApiCancelTestBtn" style="font-size:11px;font-weight:normal;">Cancel</button> + <div style="position:relative;display:inline-block;"> + <button class="admin-btn-sm" id="adm-epApiMoreBtn" title="More options" aria-haspopup="true" aria-expanded="false" style="font-size:11px;font-weight:normal;padding:4px 8px;line-height:1;"> + <svg width="14" height="4" viewBox="0 0 14 4" fill="currentColor"><circle cx="2" cy="2" r="1.4"/><circle cx="7" cy="2" r="1.4"/><circle cx="12" cy="2" r="1.4"/></svg> + </button> + <div id="adm-epApiMoreMenu" style="display:none;position:absolute;top:calc(100% + 4px);right:0;z-index:50;min-width:200px;padding:4px;background:var(--panel,var(--bg));border:1px solid var(--border);border-radius:8px;box-shadow:0 6px 20px rgba(0,0,0,0.22);flex-direction:column;gap:1px;"> + <div style="font-size:10px;text-transform:uppercase;letter-spacing:0.5px;opacity:0.55;padding:6px 9px 2px;">Connection mode</div> + <button class="admin-btn-sm adm-more-item adm-kind-opt" data-kind="proxy" style="background:none;border:0;border-radius:5px;padding:7px 9px;display:flex;align-items:center;gap:8px;width:100%;text-align:left;font-size:12px;font-weight:normal;color:var(--fg);cursor:pointer;"> + <svg class="adm-kind-check" width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg> + <span>Proxy</span> + <span style="margin-left:auto;opacity:0.5;font-size:10px;">routed via server</span> + </button> + <button class="admin-btn-sm adm-more-item adm-kind-opt" data-kind="api" style="background:none;border:0;border-radius:5px;padding:7px 9px;display:flex;align-items:center;gap:8px;width:100%;text-align:left;font-size:12px;font-weight:normal;color:var(--fg);cursor:pointer;"> + <svg class="adm-kind-check" width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round" style="visibility:hidden;"><polyline points="20 6 9 17 4 12"/></svg> + <span>API (direct)</span> + <span style="margin-left:auto;opacity:0.5;font-size:10px;">browser→provider</span> </button> - <div class="adm-provider-menu hidden" id="adm-provider-menu"></div> </div> + </div> + </h2> + <div class="admin-toggle-sub" style="margin:0 0 10px 2px;">Connect a cloud provider (OpenAI, Anthropic, DeepSeek, OpenRouter, etc.).</div> + <div class="adm-add-section"> + <div class="admin-model-form"> + <div class="admin-model-form-row"> + <div class="adm-provider-picker adm-provider-combo" id="adm-provider-picker" style="flex:1 1 220px;min-width:0;margin-bottom:0;"> + <button type="button" class="adm-provider-btn" id="adm-provider-btn" title="Pick provider" style="border-top-right-radius:0;border-bottom-right-radius:0;border-top-left-radius:6px;border-bottom-left-radius:6px;border-left:1px solid var(--border);border-right:1px solid var(--border);"> + <span class="adm-provider-current"><span class="adm-provider-logo"></span><span class="adm-provider-name">Provider</span></span> + <svg class="adm-provider-caret" width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg> + </button> + <input id="adm-epUrl" type="text" placeholder="Base URL or pick provider" autocomplete="off" style="border-left:0;border-top-left-radius:0;border-bottom-left-radius:0;border-top-right-radius:6px;border-bottom-right-radius:6px;"> + <div class="adm-provider-menu hidden" id="adm-provider-menu"></div> + </div> + </div> + <select id="adm-epKind" style="display:none"> + <option value="proxy">proxy</option> + <option value="api" selected>api</option> + </select> <select id="adm-epProvider" style="display:none"> <option value="">Custom URL</option> <option value="https://api.anthropic.com" data-logo="anthropic">Anthropic</option> @@ -2097,41 +2172,27 @@ <option value="https://api.z.ai/api/coding/paas/v4" data-logo="zhipu">Z.AI Coding Plan</option> <option value="https://integrate.api.nvidia.com/v1" data-logo="nvidia">NVIDIA</option> </select> - <!-- API key row stays in DOM, hidden until Key button is - clicked. Mirrors the Local section pattern: most users - paste a key via the provider preset flow rather than - typing it free-form, so the row only appears on demand. --> - <div class="admin-model-form-row" id="adm-epApiKey-row" style="display:none;"> - <input id="adm-epApiKey" type="password" placeholder="API key" autocomplete="off" style="flex:1"> - </div> - <div class="admin-model-form-row" style="margin-top:-4px;"> - <select id="adm-epKind" style="padding:5px;width:82px;"> - <option value="proxy">Proxy</option> - <option value="api">API</option> - </select> - <label style="display:inline-flex;align-items:center;gap:4px;font-size:11px;opacity:0.6;flex-shrink:0;">Type:<select id="adm-epType" style="padding:5px;width:80px;flex-shrink:0;"> - <option value="llm" selected>LLM</option> - <option value="image">Image</option> - </select></label> - <span style="flex:1"></span> - <button class="admin-btn-sm" id="adm-epApiKeyBtn" title="Show / hide the API key field" aria-expanded="false" aria-controls="adm-epApiKey-row" style="opacity:0.75;display:inline-flex;align-items:center;gap:4px;"> - <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 2l-9.6 9.6"/><circle cx="7.5" cy="15.5" r="5.5"/><path d="M15.5 7.5l3 3"/></svg>API - </button> - <button class="admin-btn-sm" id="adm-epApiTestBtn" style="min-width:55px;text-align:center;display:inline-flex;align-items:center;justify-content:center;gap:4px;"> - <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polygon points="5 3 19 12 5 21 5 3"/></svg>Test - </button> - <button class="admin-btn-sm hidden" id="adm-epApiCancelTestBtn" style="width:62px;text-align:center;">Cancel</button> - <button class="admin-btn-add" id="adm-epAddBtn" style="min-width:55px;text-align:center;display:inline-flex;align-items:center;justify-content:center;gap:4px;"> + <div class="admin-model-form-row" id="adm-epApiKey-row"> + <div style="position:relative;flex:1;display:flex;align-items:center;"> + <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="position:absolute;left:9px;top:50%;transform:translateY(-50%);opacity:0.55;pointer-events:none;"><path d="M21 2l-9.6 9.6"/><circle cx="7.5" cy="15.5" r="5.5"/><path d="M15.5 7.5l3 3"/></svg> + <input id="adm-epApiKey" type="password" placeholder="API key, e.g. sk-proj-AbCdEf…" autocomplete="off" style="flex:1;padding-left:28px;height:32px;box-sizing:border-box;"> + </div> + <button class="admin-btn-add" id="adm-epAddBtn" style="height:32px;min-width:55px;text-align:center;display:inline-flex;align-items:center;justify-content:center;gap:4px;flex-shrink:0;box-sizing:border-box;"> <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg>Add </button> </div> <div id="adm-epApiMsg" class="adm-ep-inline-msg"></div> - <div id="adm-deviceAuthStatus" class="adm-ep-inline-msg"></div> + <div id="adm-deviceAuthStatus" class="adm-ep-inline-msg" style="min-height:0;margin-top:0;"></div> </div> </div> </div> + + </div> + + <!-- ═══ ADDED MODELS TAB ═══ --> + <div data-settings-panel="added-models" class="hidden"> <div class="admin-card"> - <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><rect x="2" y="3" width="20" height="14" rx="2"/><line x1="8" y1="21" x2="16" y2="21"/><line x1="12" y1="17" x2="12" y2="21"/></svg>Added Models <span style="opacity:0.45;font-weight:normal;font-size:0.82em">(Endpoints)</span> + <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><polyline points="20 6 9 17 4 12"/></svg>Added Models <span style="opacity:0.45;font-weight:normal;font-size:0.82em">(Endpoints)</span> <span style="flex:1"></span> <button class="admin-btn-sm" id="adm-epProbeAllBtn" title="Re-test every endpoint and refresh online status" style="font-size:11px;font-weight:normal;display:inline-flex;align-items:center;gap:4px;"> <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round"><polyline points="23 4 23 10 17 10"/><polyline points="1 20 1 14 7 14"/><path d="M3.51 9a9 9 0 0 1 14.85-3.36L23 10M1 14l4.64 4.36A9 9 0 0 0 20.49 15"/></svg>Probe @@ -2140,20 +2201,18 @@ <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/></svg>Clear offline <span id="adm-epOfflineCount" style="opacity:0.6;margin-left:2px;"></span> </button> </h2> - <div class="admin-toggle-sub" style="margin-bottom:10px">Manage the endpoints you've added.</div> + <div class="admin-toggle-sub" style="margin-bottom:12px">Endpoints you've connected. Probe re-tests them all; Clear offline removes the dead ones.</div> <div class="adm-ep-section"> - <div class="adm-ep-section-head"> - <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-1px;margin-right:4px;"><rect x="2" y="3" width="20" height="14" rx="2"/><path d="M8 21h8"/><path d="M12 17v4"/></svg> - <span>Local</span> + <div class="adm-ep-section-head" style="font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:0.5px;opacity:0.7;margin-bottom:6px;display:inline-flex;align-items:center;gap:5px;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="2" y="3" width="20" height="14" rx="2"/><path d="M8 21h8"/><path d="M12 17v4"/></svg>Local </div> <div id="adm-epList-local"><div class="admin-empty">Loading...</div></div> </div> - <div class="adm-ep-section" style="margin-top:14px"> - <div class="adm-ep-section-head"> - <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-1px;margin-right:4px;"><circle cx="12" cy="12" r="10"/><line x1="2" y1="12" x2="22" y2="12"/><path d="M12 2a15.3 15.3 0 0 1 4 10 15.3 15.3 0 0 1-4 10 15.3 15.3 0 0 1-4-10 15.3 15.3 0 0 1 4-10z"/></svg> - <span>API</span> + <div class="adm-ep-section" style="margin-top:18px;"> + <div class="adm-ep-section-head" style="font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:0.5px;opacity:0.7;margin-bottom:6px;display:inline-flex;align-items:center;gap:5px;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="2" y1="12" x2="22" y2="12"/><path d="M12 2a15.3 15.3 0 0 1 4 10 15.3 15.3 0 0 1-4 10 15.3 15.3 0 0 1-4-10 15.3 15.3 0 0 1 4-10z"/></svg>API </div> - <div id="adm-epList-api"></div> + <div id="adm-epList-api"><div class="admin-empty">No API endpoints yet.</div></div> </div> </div> </div> @@ -2166,24 +2225,8 @@ <div class="admin-toggle-sub" style="margin-bottom:8px">All external service connections in one place.</div> <div id="unified-integrations-list"></div> <div id="unified-intg-form" style="display:none"></div> - <div style="text-align:center;padding:8px 0;"> - <button type="button" class="admin-btn-sm" id="unified-intg-add-btn" style="display:inline-flex;align-items:center;gap:6px;">+ Add Integration<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="opacity:0.7;"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"/><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"/></svg></button> - </div> - </div> - <div class="admin-card admin-only" style="margin-top:12px;"> - <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M21 2l-2 2m-7.61 7.61a5.5 5.5 0 1 1-7.778 7.778 5.5 5.5 0 0 1 7.777-7.777zm0 0L15.5 7.5m0 0l3 3L22 7l-3-3m-3.5 3.5L19 4"/></svg>API Tokens</h2> - <div class="admin-toggle-sub" style="margin-bottom:8px">Bearer tokens for external integrations (scripts, Codex, headless agent runs). Token value shown ONCE on create — copy it then.</div> - <div id="adm-tokenList" style="margin-bottom:8px;"></div> - <div style="display:flex;gap:6px;flex-wrap:wrap;align-items:flex-start;"> - <input type="text" id="adm-tokenName" placeholder="Token name (e.g. agent-test)" class="settings-select" style="flex:1;min-width:160px;"> - <input type="text" id="adm-tokenScopes" placeholder="scopes (comma-separated, blank = chat)" class="settings-select" style="flex:2;min-width:220px;" title="Allowed: chat, cookbook:read, cookbook:launch, documents:read|write, todos:read|write, email:read|draft|send, calendar:read|write, memory:read|write"> - <button class="admin-btn-add" id="adm-tokenAddBtn">Create token</button> - </div> - <div id="adm-tokenMsg" style="font-size:11px;margin-top:6px;"></div> - <div id="adm-tokenReveal" style="display:none;margin-top:8px;padding:8px 10px;background:color-mix(in srgb, var(--accent, var(--red)) 12%, transparent);border:1px solid color-mix(in srgb, var(--accent, var(--red)) 35%, transparent);border-radius:6px;"> - <div style="font-size:11px;font-weight:600;margin-bottom:4px;">Copy now — this is the only time you'll see it:</div> - <code id="adm-tokenValue" style="font-family:'Berkeley Mono','SF Mono','Fira Code',monospace;font-size:11px;word-break:break-all;display:block;background:var(--bg);padding:6px 8px;border-radius:4px;margin-bottom:6px;user-select:all;"></code> - <button class="admin-btn-sm" id="adm-tokenCopyBtn">Copy</button> + <div style="text-align:right;padding:8px 0;"> + <button type="button" class="admin-btn-add" id="unified-intg-add-btn" style="text-align:center;display:inline-flex;align-items:center;justify-content:center;gap:5px;flex-shrink:0;"><svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"/><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"/></svg>Add Integration</button> </div> </div> </div> @@ -2205,10 +2248,6 @@ <div id="set-agentMsg" style="font-size:11px;color:color-mix(in srgb, var(--fg) 45%, transparent);"></div> </div> </div> - <div class="admin-card" style="margin-bottom:12px;"> - <h2 style="display:flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:1px;opacity:0.6;flex-shrink:0"><path d="M9 11l3 3L22 4"/><path d="M21 12v7a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11"/></svg>Agent loop<span style="flex:1"></span><label class="admin-switch" title="On a failing effectful turn, climb verify → different-method → teacher → stop-and-summarize instead of silently quitting." style="flex-shrink:0"><input type="checkbox" id="set-agentSupervisorLadder"><span class="admin-slider"></span></label></h2> - <div class="admin-toggle-sub" style="margin-bottom:8px">Supervisor ladder. When on, every effectful agent turn that claims done is verified; on FAIL the ladder escalates verify → different method → teacher → stop-with-blocker, each rung visible in chat. Teacher rung requires <code>teacher_model</code> to be set.</div> - </div> <div class="admin-card" style="margin-bottom:12px;"> <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M14.7 6.3a1 1 0 0 0 0 1.4l1.6 1.6a1 1 0 0 0 1.4 0l3.77-3.77a6 6 0 0 1-7.94 7.94l-6.91 6.91a2.12 2.12 0 0 1-3-3l6.91-6.91a6 6 0 0 1 7.94-7.94l-3.76 3.76z"/></svg>Built-in Tools</h2> <div class="admin-toggle-sub" style="margin-bottom:8px">Enable or disable tools available to the AI agent.</div> @@ -2219,6 +2258,61 @@ <!-- ═══ SYSTEM TAB ═══ --> <div data-settings-panel="system" class="hidden"> + <div class="admin-card" id="settings-system-logs-card"> + <h2> + <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="settings-system-logs-svg"> + <polyline points="4 17 10 11 4 5"></polyline> + <line x1="12" y1="19" x2="20" y2="19"></line> + </svg> + Terminal Logs + </h2> + <div class="admin-toggle-sub settings-system-logs-toggle-sub">Live diagnostic logs and system output from the Odysseus process.</div> + + <div class="settings-col settings-system-logs-col"> + <!-- Controls row --> + <div class="settings-system-logs-controls"> + <!-- Search input --> + <input type="text" id="log-search-input" placeholder="Search logs..." class="settings-system-logs-search"> + + <!-- Level select --> + <select id="log-level-select" class="settings-system-logs-select"> + <option value="ALL">All Levels</option> + <option value="INFO">INFO</option> + <option value="WARNING">WARNING</option> + <option value="ERROR">ERROR</option> + <option value="DEBUG">DEBUG</option> + </select> + + <!-- Limit select --> + <select id="log-limit-select" class="settings-system-logs-select"> + <option value="100">100 lines</option> + <option value="200" selected>200 lines</option> + <option value="500">500 lines</option> + <option value="1000">1000 lines</option> + </select> + + <!-- Refresh Button --> + <button type="button" class="admin-btn-sm" id="log-refresh-btn"> + <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="settings-system-logs-refresh-svg"><path d="M21.5 2v6h-6M21.34 15.57a10 10 0 1 1-.57-8.38l5.67-5.67"/></svg> + Refresh + </button> + + <!-- Auto-refresh switch --> + <div class="settings-system-logs-autopoll-container"> + <label class="admin-switch" title="Auto-polling every 3 seconds"> + <input type="checkbox" id="log-auto-refresh-toggle"> + <span class="admin-slider"></span> + </label> + <span>Auto-poll</span> + </div> + </div> + + <!-- Console container --> + <div id="log-console-container"> + <div class="settings-system-logs-placeholder">Initializing logs terminal viewer...</div> + </div> + </div> + </div> <div class="admin-card"> <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="17 8 12 3 7 8"/><line x1="12" y1="3" x2="12" y2="15"/></svg>Data Backup</h2> <div class="admin-toggle-sub" style="margin-bottom:8px">Export or import your user data (memories, presets, settings, skills, preferences) as a JSON file.</div> @@ -2235,68 +2329,76 @@ <div style="display:flex;justify-content:space-between;align-items:center;"> <div> - <div class="admin-toggle-label">Wipe all chats</div> + <div class="admin-toggle-label">Delete all chats</div> <div class="admin-toggle-sub">Every session, message, and chat history. Documents/notes/etc. stay.</div> </div> - <button class="admin-btn-delete" data-wipe-kind="chats" style="white-space:nowrap;">Wipe</button> + <button class="admin-btn-delete" data-wipe-kind="chats" title="Delete all chats" aria-label="Delete all chats" style="display:inline-flex;align-items:center;gap:5px;white-space:nowrap;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>Delete</button> </div> <div style="display:flex;justify-content:space-between;align-items:center;margin-top:8px;"> <div> - <div class="admin-toggle-label">Wipe all memory</div> + <div class="admin-toggle-label">Delete all memory</div> <div class="admin-toggle-sub">Clears `memory.json`, the Memory table, and the vector store. Skills not affected.</div> </div> - <button class="admin-btn-delete" data-wipe-kind="memory" style="white-space:nowrap;">Wipe</button> + <button class="admin-btn-delete" data-wipe-kind="memory" title="Delete all memory" aria-label="Delete all memory" style="display:inline-flex;align-items:center;gap:5px;white-space:nowrap;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>Delete</button> </div> <div style="display:flex;justify-content:space-between;align-items:center;margin-top:8px;"> <div> - <div class="admin-toggle-label">Wipe all skills</div> + <div class="admin-toggle-label">Delete all skills</div> <div class="admin-toggle-sub">Drops `data/skills/` (all SKILL.md files). Memory not affected.</div> </div> - <button class="admin-btn-delete" data-wipe-kind="skills" style="white-space:nowrap;">Wipe</button> + <button class="admin-btn-delete" data-wipe-kind="skills" title="Delete all skills" aria-label="Delete all skills" style="display:inline-flex;align-items:center;gap:5px;white-space:nowrap;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>Delete</button> </div> <div style="display:flex;justify-content:space-between;align-items:center;margin-top:8px;"> <div> - <div class="admin-toggle-label">Wipe all notes</div> + <div class="admin-toggle-label">Delete all notes</div> <div class="admin-toggle-sub">Every note, todo, and checklist.</div> </div> - <button class="admin-btn-delete" data-wipe-kind="notes" style="white-space:nowrap;">Wipe</button> + <button class="admin-btn-delete" data-wipe-kind="notes" title="Delete all notes" aria-label="Delete all notes" style="display:inline-flex;align-items:center;gap:5px;white-space:nowrap;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>Delete</button> </div> <div style="display:flex;justify-content:space-between;align-items:center;margin-top:8px;"> <div> - <div class="admin-toggle-label">Wipe all tasks</div> + <div class="admin-toggle-label">Delete all tasks</div> <div class="admin-toggle-sub">Every scheduled task and its run history (Tasks tool).</div> </div> - <button class="admin-btn-delete" data-wipe-kind="tasks" style="white-space:nowrap;">Wipe</button> + <button class="admin-btn-delete" data-wipe-kind="tasks" title="Delete all tasks" aria-label="Delete all tasks" style="display:inline-flex;align-items:center;gap:5px;white-space:nowrap;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>Delete</button> </div> <div style="display:flex;justify-content:space-between;align-items:center;margin-top:8px;"> <div> - <div class="admin-toggle-label">Wipe all documents</div> + <div class="admin-toggle-label">Delete all documents</div> <div class="admin-toggle-sub">Every document and version. Drafts, exports, library — all gone.</div> </div> - <button class="admin-btn-delete" data-wipe-kind="documents" style="white-space:nowrap;">Wipe</button> + <button class="admin-btn-delete" data-wipe-kind="documents" title="Delete all documents" aria-label="Delete all documents" style="display:inline-flex;align-items:center;gap:5px;white-space:nowrap;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>Delete</button> </div> <div style="display:flex;justify-content:space-between;align-items:center;margin-top:8px;"> <div> - <div class="admin-toggle-label">Wipe all gallery</div> + <div class="admin-toggle-label">Delete all gallery</div> <div class="admin-toggle-sub">Every image record and the upload directory on disk.</div> </div> - <button class="admin-btn-delete" data-wipe-kind="gallery" style="white-space:nowrap;">Wipe</button> + <button class="admin-btn-delete" data-wipe-kind="gallery" title="Delete all gallery" aria-label="Delete all gallery" style="display:inline-flex;align-items:center;gap:5px;white-space:nowrap;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>Delete</button> </div> <div style="display:flex;justify-content:space-between;align-items:center;margin-top:8px;"> <div> - <div class="admin-toggle-label">Wipe all calendar</div> + <div class="admin-toggle-label">Delete all calendar</div> <div class="admin-toggle-sub">Every event and every calendar (incl. CalDAV-synced ones; resync to restore).</div> </div> - <button class="admin-btn-delete" data-wipe-kind="calendar" style="white-space:nowrap;">Wipe</button> + <button class="admin-btn-delete" data-wipe-kind="calendar" title="Delete all calendar" aria-label="Delete all calendar" style="display:inline-flex;align-items:center;gap:5px;white-space:nowrap;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>Delete</button> </div> + <hr style="border:0;border-top:1px solid color-mix(in srgb, #e55 25%, var(--border));margin:14px 0 10px;"> + <div style="display:flex;justify-content:space-between;align-items:center;"> + <div> + <div class="admin-toggle-label" style="color:#e55;">Delete everything</div> + <div class="admin-toggle-sub">All eight categories above, in one go. Same effect as wiping each one in sequence.</div> + </div> + <button class="admin-btn-delete" data-wipe-kind="__all__" title="Delete every category" aria-label="Delete everything" style="display:inline-flex;align-items:center;gap:5px;white-space:nowrap;font-weight:600;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>Delete All</button> + </div> <div id="adm-wipeMsg" style="margin-top:8px;"></div> </div> </div> @@ -2342,7 +2444,7 @@ <script type="module" src="/static/js/chatRenderer.js"></script> <script type="module" src="/static/js/codeRunner.js"></script> <script type="module" src="/static/js/chatStream.js"></script> -<script type="module" src="/static/js/chat.js?v=20260604s"></script> +<script type="module" src="/static/js/chat.js?v=20260609ws"></script> <script type="module" src="/static/js/cookbook.js"></script> <script src="/static/js/cookbookSchedule.js"></script> <script type="module" src="/static/js/search-chat.js"></script> diff --git a/static/js/admin.js b/static/js/admin.js index 82b90b737..bd63e10db 100644 --- a/static/js/admin.js +++ b/static/js/admin.js @@ -3,7 +3,7 @@ import uiModule from './ui.js'; import settingsModule from './settings.js'; -import { providerLogo } from './providers.js'; +import { providerLogo, providerLogoFromUrl } from './providers.js'; import { sortModelObjects } from './modelSort.js'; import { PROVIDER_DEVICE_FLOWS, formatDeviceFlowError, runProviderDeviceFlow } from './providerDeviceFlow.js'; @@ -13,6 +13,7 @@ let modalEl = null; // the endpoints list can flash a glow on that row. Cleared once the // animation fires. let _recentlyAddedEpId = null; +let _authPolicy = { password_min_length: 8, reserved_usernames: [] }; function el(id) { return document.getElementById(id); } function esc(s) { return uiModule.esc(s); } @@ -55,6 +56,7 @@ async function loadUsers() { </div> </div> <div style="display:flex;gap:8px;align-items:center;"> + <button class="admin-btn-sm" data-adm-toggle-admin="${esc(u.username)}" data-make-admin="${u.is_admin ? '0' : '1'}" style="font-size:11px;">${u.is_admin ? 'Revoke admin' : 'Make admin'}</button> <button class="admin-btn-sm" data-adm-rename-user="${esc(u.username)}" style="font-size:11px;">Rename</button> ${u.is_admin ? '' : `<button class="admin-btn-delete" data-adm-del-user="${esc(u.username)}" style="font-size:11px;">Remove</button>`} ${u.is_admin ? '' : '<svg class="admin-user-chevron" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="opacity:0.3;transition:transform 0.2s,opacity 0.2s;"><polyline points="6 9 12 15 18 9"/></svg>'} @@ -113,7 +115,7 @@ async function loadUsers() { // Toggle panel visibility + rotate chevron + load models let _modelsLoaded = false; header.addEventListener('click', (e) => { - if (e.target.closest('.admin-btn-delete, [data-adm-rename-user]')) return; + if (e.target.closest('.admin-btn-delete, [data-adm-rename-user], [data-adm-toggle-admin]')) return; privPanel.classList.toggle('hidden'); const chevron = header.querySelector('.admin-user-chevron'); if (chevron) { @@ -199,6 +201,42 @@ async function loadUsers() { }); } + // Promote / demote (admin toggle) — present on every row + const adminToggleBtn = row.querySelector('[data-adm-toggle-admin]'); + if (adminToggleBtn) { + adminToggleBtn.addEventListener('click', async (e) => { + e.stopPropagation(); + const username = adminToggleBtn.dataset.admToggleAdmin; + const makeAdmin = adminToggleBtn.dataset.makeAdmin === '1'; + const confirmMsg = makeAdmin + ? `Grant admin rights to "${username}"? They'll get full access to all settings and users — including the power to demote or remove other admins (you included).` + : `Revoke admin rights from "${username}"? They'll lose access to the admin panel.`; + if (!await uiModule.styledConfirm(confirmMsg, { confirmText: makeAdmin ? 'Make admin' : 'Revoke admin', danger: !makeAdmin })) return; + adminToggleBtn.disabled = true; + try { + const res = await fetch(`/api/auth/users/${encodeURIComponent(username)}/admin`, { + method: 'PUT', + credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ is_admin: makeAdmin }), + }); + const data = await res.json().catch(() => ({})); + if (!res.ok) { + uiModule.showError(data.detail || 'Failed to change admin status'); + adminToggleBtn.disabled = false; + return; + } + // Demoting yourself drops your own admin access — reload into the + // normal-user view (mirrors the rename-self reload above). + if (data.self) { window.location.reload(); return; } + loadUsers(); + } catch (err) { + uiModule.showError('Failed to change admin status'); + adminToggleBtn.disabled = false; + } + }); + } + list.appendChild(row); }); } catch (e) { list.innerHTML = '<div class="admin-error">Failed to load users</div>'; } @@ -306,6 +344,15 @@ function initSignupToggle() { } function initAddUser() { + fetch('/api/auth/policy', { credentials: 'same-origin' }) + .then(r => r.ok ? r.json() : null) + .then(policy => { + if (!policy) return; + _authPolicy = policy; + const admPw = el('adm-newPassword'); + if (admPw) admPw.placeholder = `Password (min ${policy.password_min_length})`; + }) + .catch(() => {}); el('adm-addBtn').addEventListener('click', async () => { const msg = el('adm-addMsg'); msg.textContent = ''; msg.className = ''; @@ -313,7 +360,8 @@ function initAddUser() { const password = el('adm-newPassword').value; const is_admin = el('adm-newIsAdmin').checked; if (!username) { msg.textContent = 'Username required'; msg.className = 'admin-error'; return; } - if (password.length < 8) { msg.textContent = 'Password must be at least 8 characters'; msg.className = 'admin-error'; return; } + if (password.length < _authPolicy.password_min_length) { msg.textContent = `Password must be at least ${_authPolicy.password_min_length} characters`; msg.className = 'admin-error'; return; } + if (_authPolicy.reserved_usernames.includes(username.toLowerCase())) { msg.textContent = 'This username is reserved'; msg.className = 'admin-error'; return; } el('adm-addBtn').disabled = true; try { const res = await fetch('/api/auth/users', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ username, password, is_admin }) }); @@ -449,13 +497,14 @@ async function loadEndpoints() { return ` <div class="admin-user-row${ep.is_enabled ? '' : ' admin-ep-disabled'}${justAddedClass}" data-adm-ep-id="${ep.id}"> <div style="display:flex;align-items:center;justify-content:space-between;${hasModels ? 'cursor:pointer;' : ''}padding:4px 0;" data-adm-ep-header="${ep.id}"> - <div class="admin-user-info" style="flex:1;flex-wrap:wrap;gap:0.3rem;"> + <div class="admin-user-info" style="flex:1;flex-wrap:wrap;gap:0.3rem;align-items:center;"> + <span class="adm-ep-row-logo" style="display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;flex-shrink:0;opacity:0.9;">${providerLogoFromUrl(ep.base_url) || ''}</span> <span class="admin-user-name">${esc(ep.name)}</span> ${ep.model_type === 'image' ? '<span class="admin-badge" style="background:color-mix(in srgb, var(--accent) 20%, transparent);color:var(--accent);">Image</span>' : ''} ${kindLabel ? `<span class="admin-badge">${esc(kindLabel)}</span>` : ''} ${statusBadge} ${ep.is_enabled ? '' : '<span class="admin-badge admin-badge-off">disabled</span>'} - ${hasModels ? '<span style="font-size:10px;opacity:0.4;">Click to manage models</span>' : ''} + ${hasModels ? `<span style="font-size:10px;opacity:0.4;${category === 'api' ? 'flex-basis:100%;' : ''}">Click to manage models</span>` : ''} </div> <div style="display:flex;gap:4px;align-items:center;"> <button class="admin-btn-sm" data-adm-toggle-ep="${ep.id}">${ep.is_enabled ? 'Disable' : 'Enable'}</button> @@ -828,6 +877,14 @@ function initEndpointForm() { document.addEventListener('click', (e) => { if (!picker.contains(e.target)) pickerMenu.classList.add('hidden'); }); + // Capture-phase Esc: dismiss the picker menu without bubbling to the + // settings-modal handler that would otherwise close the whole modal. + document.addEventListener('keydown', (e) => { + if (e.key !== 'Escape') return; + if (pickerMenu.classList.contains('hidden')) return; + e.stopPropagation(); + pickerMenu.classList.add('hidden'); + }, { capture: true }); } provider.addEventListener('change', () => { @@ -1022,14 +1079,15 @@ function initEndpointForm() { if (d.id) _recentlyAddedEpId = String(d.id); await loadEndpoints(); await _selectAddedModelInChat(d); + const goLink = ' <a href="#" data-go-added-models style="margin-left:6px;text-decoration:underline;color:inherit;font-weight:600;">Added Models →</a>'; if (!d.online) { - msg.textContent = 'Added (endpoint offline — will retry on next load)'; + msg.innerHTML = 'Added (endpoint offline — will retry on next load)' + goLink; msg.className = 'admin-error'; } else if (d.status === 'empty') { - msg.textContent = 'Added — endpoint reachable, no models found'; + msg.innerHTML = 'Added — endpoint reachable, no models found' + goLink; msg.className = 'admin-success'; } else { - msg.textContent = `Added — found ${count} model${count !== 1 ? 's' : ''}`; + msg.innerHTML = `Added — found ${count} model${count !== 1 ? 's' : ''}` + goLink; msg.className = 'admin-success'; } } else { msg.textContent = d.detail || 'Failed'; msg.className = 'admin-error'; } @@ -1168,7 +1226,125 @@ function initEndpointForm() { }); }; _wireKeyToggle('adm-epLocalKeyBtn', 'adm-epLocalApiKey-row'); - _wireKeyToggle('adm-epApiKeyBtn', 'adm-epApiKey-row'); + + // Delegated link handler for jumping between settings tabs. + // [data-go-added-models] → quick shortcut for the Added Models tab + // [data-go-settings-tab="X"] → any tab whose nav button has data-settings-tab="X" + // [data-go-scroll-to="#elementId"] → after switching, scroll the element into view + document.addEventListener('click', (e) => { + const explicit = e.target.closest('[data-go-settings-tab]'); + if (explicit) { + e.preventDefault(); + const tab = explicit.getAttribute('data-go-settings-tab'); + const scrollTo = explicit.getAttribute('data-go-scroll-to'); + const btn = document.querySelector(`[data-settings-tab="${tab}"]`); + if (btn) btn.click(); + if (scrollTo) { + // Defer to the next frame so the panel has actually become visible + // before we try to scroll into it. + requestAnimationFrame(() => { + const target = document.querySelector(scrollTo); + if (target) target.scrollIntoView({ behavior: 'smooth', block: 'start' }); + }); + } + return; + } + const link = e.target.closest('[data-go-added-models]'); + if (!link) return; + e.preventDefault(); + const btn = document.querySelector('[data-settings-tab="added-models"]'); + if (btn) btn.click(); + }); + + // Generic open/close helper for the kebab dropdowns in this card. + // Both the Local and API cards use the same shape: an h2-anchored button + // with id "<prefix>MoreBtn" toggles a sibling menu with id "<prefix>MoreMenu". + // Global Esc handler: close any currently-open kebab menu in the admin + // panel regardless of which _wireKebab instance owns it. Belt-and-braces + // backup for the per-instance handler below — registered once. + if (!document._admKebabEscWired) { + document._admKebabEscWired = true; + document.addEventListener('keydown', (e) => { + if (e.key !== 'Escape') return; + // Any visible kebab dropdown in the admin panel — match by id pattern + // so adding a new kebab elsewhere automatically benefits. + const menus = document.querySelectorAll( + '#adm-epLocalMoreMenu, #adm-epApiMoreMenu' + ); + let closed = false; + menus.forEach((m) => { + if (m && m.style.display !== 'none') { + m.style.display = 'none'; + // Sync the associated button's aria-expanded when we can find it. + const btn = document.getElementById(m.id.replace('Menu', 'Btn')); + if (btn) btn.setAttribute('aria-expanded', 'false'); + closed = true; + } + }); + if (closed) e.stopPropagation(); + }, { capture: true }); + } + + const _wireKebab = (btnId, menuId, onItem) => { + const btn = el(btnId); + const menu = el(menuId); + if (!btn || !menu) return; + const isOpen = () => menu.style.display !== 'none'; + const close = () => { menu.style.display = 'none'; btn.setAttribute('aria-expanded', 'false'); }; + const open = () => { menu.style.display = 'flex'; btn.setAttribute('aria-expanded', 'true'); }; + btn.addEventListener('click', (e) => { + e.stopPropagation(); + if (isOpen()) close(); else open(); + }); + menu.addEventListener('click', (e) => { + const item = e.target.closest('.adm-more-item'); + if (!item) return; + if (onItem) onItem(item, e); + close(); + }); + document.addEventListener('click', (e) => { + if (!isOpen()) return; + if (e.target.closest('#' + menuId + ', #' + btnId)) return; + close(); + }); + // Use capture phase so this fires before the settings-modal Esc handler + // (which is in bubble phase). stopPropagation prevents the modal from + // closing when the user only meant to dismiss this menu. + document.addEventListener('keydown', (e) => { + if (e.key === 'Escape' && isOpen()) { + e.stopPropagation(); + close(); + } + }, { capture: true }); + }; + + // API card "..." menu: contains the Proxy/API connection-mode toggle. + // Sync the visible checkmarks with the hidden #adm-epKind select so + // downstream code (which reads kindSel.value) keeps working. + (function wireApiKindMenu() { + const kind = el('adm-epKind'); + if (!kind) return; + const opts = document.querySelectorAll('#adm-epApiMoreMenu .adm-kind-opt'); + const sync = () => { + opts.forEach((o) => { + const check = o.querySelector('.adm-kind-check'); + if (check) check.style.visibility = (o.dataset.kind === kind.value) ? 'visible' : 'hidden'; + }); + }; + sync(); + kind.addEventListener('change', sync); + _wireKebab('adm-epApiMoreBtn', 'adm-epApiMoreMenu', (item) => { + const k = item.dataset.kind; + if (!k) return; + kind.value = k; + kind.dispatchEvent(new Event('change')); + }); + })(); + + // Local card "..." kebab: holds Scan network / Ollama / API key reveal. + // Item buttons keep their own click handlers; the helper just handles + // open/close + outside-click + Esc. + _wireKebab('adm-epLocalMoreBtn', 'adm-epLocalMoreMenu'); // ── Added Models toolbar: Probe + Clear offline ──────────────────── // Both buttons act over the currently-rendered endpoint list. The @@ -1180,10 +1356,10 @@ function initEndpointForm() { if (!lbl) return; const n = document.querySelectorAll('[data-adm-ep-id] [data-adm-ep-online="0"]').length; lbl.textContent = n > 0 ? `(${n})` : ''; - // Keep the button enabled even when there are no offline rows — a - // click on the empty case fires a toast instead of feeling dead. + // Hide the button entirely when there's nothing offline — no point + // showing an action that has nothing to act on. const btn = el('adm-epClearOfflineBtn'); - if (btn) btn.style.opacity = n === 0 ? '0.55' : '0.85'; + if (btn) btn.style.display = n === 0 ? 'none' : ''; }; // Wire after every loadEndpoints() run by patching the render hook — // simplest path: MutationObserver on the two list containers. @@ -1200,7 +1376,17 @@ function initEndpointForm() { probeAllBtn.addEventListener('click', async () => { probeAllBtn.disabled = true; const origHTML = probeAllBtn.innerHTML; - probeAllBtn.innerHTML = '<span style="opacity:0.7;">Probing…</span>'; + let _wp = null; + try { + const sp = window.spinnerModule || (await import('./spinner.js')).default; + _wp = sp.createWhirlpool(11); + _wp.element.style.cssText = 'display:inline-flex;width:11px;height:11px;margin:0 4px 0 0;'; + probeAllBtn.innerHTML = ''; + probeAllBtn.appendChild(_wp.element); + probeAllBtn.appendChild(document.createTextNode('Probing')); + } catch (_) { + probeAllBtn.innerHTML = '<span style="opacity:0.7;">Probing…</span>'; + } try { // Hit the bulk local probe (same one the model picker uses). await fetch('/api/model-endpoints/probe-local', { credentials: 'same-origin' }).catch(() => {}); @@ -1222,6 +1408,7 @@ function initEndpointForm() { await loadEndpoints(); if (uiModule && uiModule.showToast) uiModule.showToast('Endpoint status refreshed', 1800); } finally { + if (_wp) { try { _wp.destroy(); } catch (_) {} } probeAllBtn.innerHTML = origHTML; probeAllBtn.disabled = false; } @@ -1292,15 +1479,16 @@ function initEndpointForm() { const localTestBtn = el('adm-epLocalTestBtn'); if (localTestBtn) { localTestBtn.addEventListener('click', async () => { + const testOriginalHtml = localTestBtn.innerHTML || '>Test'; const msg = _endpointMsg('local'); - msg.textContent = ''; msg.className = ''; + msg.textContent = ''; msg.className = 'adm-ep-inline-msg'; const raw = (el('adm-epLocalUrl').value || '').trim(); if (!raw) { msg.textContent = 'Enter a base URL to test'; msg.className = 'admin-error'; return; } const url = _normalizeBaseUrl(raw); const keyEl = el('adm-epLocalApiKey'); const apiKey = keyEl ? keyEl.value.trim() : ''; localTestBtn.disabled = true; - localTestBtn.textContent = 'Testing...'; + localTestBtn.innerHTML = testOriginalHtml.replace(/>Test\s*$/, '>Testing...'); try { const fd = new FormData(); fd.append('base_url', url); @@ -1313,19 +1501,21 @@ function initEndpointForm() { msg.className = 'admin-error'; } localTestBtn.disabled = false; - localTestBtn.textContent = 'Test'; + localTestBtn.innerHTML = testOriginalHtml; }); } if (localAddBtn) { localAddBtn.addEventListener('click', async () => { + const addOriginalHtml = localAddBtn.innerHTML || '>Add'; const msg = _endpointMsg('local'); - msg.textContent = ''; msg.className = ''; + msg.textContent = ''; msg.className = 'adm-ep-inline-msg'; const raw = (el('adm-epLocalUrl').value || '').trim(); if (!raw) { msg.textContent = 'Enter a base URL (e.g. http://localhost:8002/v1)'; msg.className = 'admin-error'; return; } const url = _normalizeBaseUrl(raw); const keyEl = el('adm-epLocalApiKey'); const apiKey = keyEl ? keyEl.value.trim() : ''; - localAddBtn.disabled = true; localAddBtn.textContent = 'Adding...'; + localAddBtn.disabled = true; + localAddBtn.innerHTML = addOriginalHtml.replace(/>Add\s*$/, '>Adding...'); try { const fd = new FormData(); fd.append('base_url', url); @@ -1345,15 +1535,17 @@ function initEndpointForm() { await loadEndpoints(); await _selectAddedModelInChat(d); const count = (d.models || []).length; - msg.textContent = d.status === 'empty' + const baseText = d.status === 'empty' ? 'Added — Ollama is running, no models pulled yet' : d.online ? `Added — found ${count} model${count !== 1 ? 's' : ''}` : 'Added (offline — will retry on next load)'; + msg.innerHTML = `${baseText} <a href="#" data-go-added-models style="margin-left:6px;text-decoration:underline;color:inherit;font-weight:600;">Added Models →</a>`; msg.className = d.online ? 'admin-success' : 'admin-error'; } else { msg.textContent = d.detail || 'Failed'; msg.className = 'admin-error'; } } catch (e) { msg.textContent = 'Request failed'; msg.className = 'admin-error'; } - localAddBtn.disabled = false; localAddBtn.textContent = 'Add'; + localAddBtn.disabled = false; + localAddBtn.innerHTML = addOriginalHtml; }); } @@ -1379,10 +1571,7 @@ function initEndpointForm() { discoverBtn.addEventListener('click', async () => { const msg = _endpointMsg('local'); discoverBtn.disabled = true; - // Keep the button's icon as-is while scanning; the whirlpool + - // status text below is enough feedback. (Two spinning indicators - // at once looks busy.) - msg.className = ''; + msg.className = 'adm-ep-inline-msg'; msg.innerHTML = ''; try { const sp = window.spinnerModule || (await import('./spinner.js')).default; @@ -1393,7 +1582,7 @@ function initEndpointForm() { wrap.appendChild(wp.element); const txt = document.createElement('span'); txt.textContent = 'Scanning ports 8000-8020 and 11434 for model servers...'; - txt.style.cssText = 'font-size:12px;opacity:0.7;'; + txt.style.cssText = 'opacity:0.7;'; wrap.appendChild(txt); msg.appendChild(wrap); discoverBtn._wp = wp; @@ -1444,30 +1633,6 @@ function initEndpointForm() { }); } - // Collapsible Add-Models subsections (API / Local). Both start collapsed - // so the card is compact; the last-used state is remembered per section - // in localStorage so a frequent API-adder doesn't re-expand every time. - document.querySelectorAll('#adm-add-api, #adm-add-local').forEach((sec) => { - const head = sec.querySelector('.adm-section-toggle'); - if (!head) return; - const key = 'odysseus.addModels.' + sec.id + '.open'; - let open = false; - try { open = localStorage.getItem(key) === '1'; } catch {} - const apply = () => { - sec.classList.toggle('collapsed', !open); - head.setAttribute('aria-expanded', open ? 'true' : 'false'); - }; - apply(); - const toggle = () => { - open = !open; - try { localStorage.setItem(key, open ? '1' : '0'); } catch {} - apply(); - }; - head.addEventListener('click', toggle); - head.addEventListener('keydown', (e) => { - if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); toggle(); } - }); - }); document.querySelectorAll('.adm-quickstart-section').forEach((sec) => { const head = sec.querySelector('.adm-quickstart-toggle'); if (!head) return; @@ -2183,28 +2348,126 @@ function initRag() { /* ═══════════════════════════════════════════ SYSTEM TAB — Tokens ═══════════════════════════════════════════ */ +// Catalog mirrors the one in settings.js integration form. Keep keys in +// sync with the backend scope allowlist. +const _TOKEN_SCOPES = [ + { key: 'todos:read', label: 'Todos read', detail: 'Read notes and checklists' }, + { key: 'todos:write', label: 'Todos write', detail: 'Create, update, delete, and toggle todo items' }, + { key: 'documents:read', label: 'Documents read', detail: 'Read documents when a document API is enabled' }, + { key: 'documents:write', label: 'Documents write', detail: 'Create and update draft documents' }, + { key: 'email:read', label: 'Email read', detail: 'Read email when an email API is enabled' }, + { key: 'email:draft', label: 'Email draft', detail: 'Create email reply drafts without sending' }, + { key: 'email:send', label: 'Email send', detail: 'Send email directly' }, + { key: 'calendar:read', label: 'Calendar read', detail: 'Read calendar events when enabled' }, + { key: 'calendar:write', label: 'Calendar write', detail: 'Create and update calendar events' }, + { key: 'memory:read', label: 'Memory read', detail: 'Read memory when enabled' }, + { key: 'memory:write', label: 'Memory write', detail: 'Write memory when enabled' }, + { key: 'cookbook:read', label: 'Cookbook read', detail: 'List cookbook tasks + tail their tmux output' }, + { key: 'cookbook:launch', label: 'Cookbook launch', detail: 'Launch and stop cookbook serve tasks' }, +]; + +function _renderTokenScopeRows(t) { + const have = new Set(t.scopes || []); + return _TOKEN_SCOPES.map(s => { + const action = (s.key.split(':')[1] || '').toLowerCase(); + const pill = action === 'read' + ? 'background:rgba(150,150,150,0.18);color:var(--fg-muted,#888);' + : 'background:color-mix(in srgb, var(--accent, var(--red)) 18%, transparent);color:var(--accent, var(--red));'; + const tool = s.label.replace(/\s+(read|write|draft|send|launch)$/i, ''); + return ` + <label style="display:flex;align-items:center;gap:8px;min-height:28px;padding:1px 0;"> + <span class="settings-label" style="width:90px;flex-shrink:0;padding:0;font-size:12px;">${esc(tool)}</span> + <span style="font-size:9px;font-weight:600;text-transform:uppercase;letter-spacing:0.5px;padding:1px 7px;border-radius:999px;flex-shrink:0;min-width:44px;text-align:center;box-sizing:border-box;${pill}">${esc(action)}</span> + <span style="font-size:11px;line-height:1.35;opacity:0.62;flex:1;min-width:0;">${esc(s.detail)}</span> + <label class="admin-switch" style="margin-left:auto;flex-shrink:0;"><input type="checkbox" class="adm-tok-scope" data-token-id="${esc(t.id)}" data-scope="${esc(s.key)}" ${have.has(s.key) ? 'checked' : ''}><span class="admin-slider"></span></label> + </label>`; + }).join(''); +} + async function loadTokens() { const list = el('adm-tokenList'); + if (!list) return; try { const res = await fetch('/api/tokens', { credentials: 'same-origin' }); const tokens = await res.json(); - if (!tokens.length) { list.innerHTML = '<div class="admin-empty">No API tokens</div>'; return; } + if (!tokens.length) { list.innerHTML = '<div class="admin-empty" style="color:var(--accent, var(--red));opacity:0.7;font-size:10px;">No API tokens</div>'; return; } list.innerHTML = tokens.map(t => ` - <div class="admin-user-row"> - <div class="admin-user-info" style="flex:1;flex-wrap:wrap;gap:0.3rem;"> - <span class="admin-user-name">${esc(t.name)}</span> - <span class="admin-badge">${esc(t.token_prefix)}...</span> - <span class="admin-badge" title="Allowed API scopes">${esc((t.scopes || ['chat']).join(', '))}</span> - ${t.owner ? `<span style="font-size:0.75rem;opacity:0.5;">Owner: ${esc(t.owner)}</span>` : ''} - ${t.last_used_at ? `<span style="font-size:0.75rem;opacity:0.5;">Last used: ${new Date(t.last_used_at).toLocaleDateString()}</span>` : '<span style="font-size:0.75rem;opacity:0.4;">Never used</span>'} + <div class="admin-user-row" data-adm-tok-row="${esc(t.id)}" style="display:block;"> + <div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;"> + <div class="admin-user-info" style="flex:1;min-width:0;flex-wrap:wrap;gap:0.3rem;"> + <input type="text" class="adm-tok-rename" data-token-id="${esc(t.id)}" value="${esc(t.name || '')}" placeholder="Token name" style="font-size:13px;font-weight:600;padding:3px 6px;background:transparent;border:1px solid transparent;border-radius:4px;min-width:160px;" title="Click to rename"> + <span class="admin-badge">${esc(t.token_prefix)}...</span> + ${t.owner ? `<span style="font-size:0.75rem;opacity:0.5;">Owner: ${esc(t.owner)}</span>` : ''} + ${t.last_used_at ? `<span style="font-size:0.75rem;opacity:0.5;">Last used: ${new Date(t.last_used_at).toLocaleDateString()}</span>` : '<span style="font-size:0.75rem;opacity:0.4;">Never used</span>'} + </div> + <button class="admin-btn-sm" data-adm-tok-toggle="${esc(t.id)}" style="opacity:0.75;">Permissions</button> + <button class="admin-btn-delete" data-adm-del-token="${esc(t.id)}">Revoke</button> + </div> + <div data-adm-tok-perm="${esc(t.id)}" style="display:none;margin-top:8px;padding:8px 4px 0;border-top:1px solid var(--border);"> + ${_renderTokenScopeRows(t)} + <div class="adm-tok-scope-msg" data-token-id="${esc(t.id)}" style="font-size:11px;min-height:14px;margin-top:4px;"></div> </div> - <button class="admin-btn-delete" data-adm-del-token="${t.id}">Revoke</button> </div>`).join(''); + + // Revoke list.querySelectorAll('[data-adm-del-token]').forEach(btn => { btn.addEventListener('click', async () => { if (!await uiModule.styledConfirm('Revoke this API token? External integrations using it will stop working.', { confirmText: 'Revoke', danger: true })) return; await fetch(`/api/tokens/${btn.dataset.admDelToken}`, { method: 'DELETE', credentials: 'same-origin' }); loadTokens(); + // Codex / Claude integration cards on the Integrations panel are + // backed by these tokens — let them re-render so the deleted token + // disappears there too. + try { window.dispatchEvent(new CustomEvent('odysseus-integrations-changed')); } catch (_) {} + }); + }); + // Toggle permissions panel + list.querySelectorAll('[data-adm-tok-toggle]').forEach(btn => { + btn.addEventListener('click', () => { + const panel = list.querySelector(`[data-adm-tok-perm="${btn.dataset.admTokToggle}"]`); + if (!panel) return; + panel.style.display = panel.style.display === 'none' ? '' : 'none'; + }); + }); + // Rename + list.querySelectorAll('.adm-tok-rename').forEach(input => { + const original = input.value; + const commit = async () => { + const name = (input.value || '').trim(); + if (!name || name === original) return; + try { + const r = await fetch(`/api/tokens/${input.dataset.tokenId}`, { + method: 'PATCH', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name }), + }); + if (!r.ok) throw new Error('Save failed'); + loadTokens(); + } catch (_) { input.value = original; } + }; + input.addEventListener('blur', commit); + input.addEventListener('keydown', e => { if (e.key === 'Enter') { e.preventDefault(); input.blur(); } }); + }); + // Scope toggle change → PATCH the whole scopes array for this token. + list.querySelectorAll('.adm-tok-scope').forEach(cb => { + cb.addEventListener('change', async () => { + const tokenId = cb.dataset.tokenId; + const panel = list.querySelector(`[data-adm-tok-perm="${tokenId}"]`); + const msg = list.querySelector(`.adm-tok-scope-msg[data-token-id="${tokenId}"]`); + const scopes = Array.from(panel.querySelectorAll('.adm-tok-scope:checked')).map(input => input.dataset.scope); + try { + const r = await fetch(`/api/tokens/${tokenId}`, { + method: 'PATCH', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ scopes }), + }); + const d = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(d.detail || 'Failed'); + if (msg) { msg.textContent = 'Saved'; msg.style.color = 'var(--green, #50fa7b)'; setTimeout(() => { msg.textContent = ''; }, 1200); } + } catch (err) { + cb.checked = !cb.checked; + if (msg) { msg.textContent = (err && err.message) || 'Failed'; msg.style.color = 'var(--red)'; } + } }); }); } catch (e) { list.innerHTML = '<div class="admin-error">Failed to load tokens</div>'; } @@ -2236,11 +2499,20 @@ function initTokenForm() { else { msg.textContent = data.detail || 'Failed'; msg.className = 'admin-error'; } } catch (e) { msg.textContent = 'Request failed'; msg.className = 'admin-error'; } }); + const TOKEN_COPY_ICON = '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg>'; + const TOKEN_CHECK_ICON = '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg>'; el('adm-tokenCopyBtn').addEventListener('click', () => { const val = el('adm-tokenValue').textContent; + const btn = el('adm-tokenCopyBtn'); navigator.clipboard.writeText(val).then(() => { - el('adm-tokenCopyBtn').textContent = 'Copied!'; - setTimeout(() => { el('adm-tokenCopyBtn').textContent = 'Copy'; }, 2000); + btn.innerHTML = TOKEN_CHECK_ICON; + btn.style.color = 'var(--accent, var(--red))'; + btn.style.opacity = '1'; + setTimeout(() => { + btn.innerHTML = TOKEN_COPY_ICON; + btn.style.color = ''; + btn.style.opacity = '0.7'; + }, 1600); }); }); } @@ -2467,33 +2739,258 @@ function initDangerZone() { modalEl.querySelectorAll('[data-wipe-kind]').forEach(btn => { btn.addEventListener('click', async () => { const kind = btn.dataset.wipeKind; - const label = _LABELS[kind] || kind; - if (!await uiModule.styledConfirm(`Wipe ALL ${label}? This cannot be undone.`, { confirmText: 'Wipe', danger: true })) return; - if (!await uiModule.styledConfirm(`Really wipe every one of your ${label}?`, { confirmText: 'Yes, wipe everything', danger: true })) return; - btn.disabled = true; const prev = btn.textContent; btn.textContent = 'Wiping…'; + const isAll = kind === '__all__'; + const label = isAll ? 'data across every category' : (_LABELS[kind] || kind); + if (!await uiModule.styledConfirm(`Delete ALL ${label}? This cannot be undone.`, { confirmText: 'Delete', danger: true })) return; + if (!await uiModule.styledConfirm(`Really delete every one of your ${label}?`, { confirmText: isAll ? 'Yes, delete everything' : 'Yes, delete everything', danger: true })) return; + btn.disabled = true; + const prevHtml = btn.innerHTML; + btn.innerHTML = isAll ? 'Deleting all…' : 'Deleting…'; if (_wipeMsg) { _wipeMsg.textContent = ''; _wipeMsg.className = ''; } try { - const res = await fetch(`/api/admin/wipe/${kind}`, { method: 'DELETE', credentials: 'same-origin' }); - const data = await res.json().catch(() => ({})); - if (res.ok) { - if (_wipeMsg) { _wipeMsg.textContent = `Wiped ${data.count ?? 0} ${label}.`; _wipeMsg.className = 'admin-success'; } + if (isAll) { + // Iterate every known category. Failures in one shouldn't stop + // the rest — record per-category counts and surface a summary. + const kinds = Object.keys(_LABELS); + const results = []; + for (const k of kinds) { + try { + const r = await fetch(`/api/admin/wipe/${k}`, { method: 'DELETE', credentials: 'same-origin' }); + const d = await r.json().catch(() => ({})); + results.push({ k, ok: r.ok, count: d.count ?? 0, error: r.ok ? null : (d.detail || 'failed') }); + } catch (e) { + results.push({ k, ok: false, count: 0, error: e.message }); + } + } + const okCount = results.filter(r => r.ok).length; + const total = results.reduce((n, r) => n + (r.ok ? r.count : 0), 0); + const fails = results.filter(r => !r.ok).map(r => r.k); + if (_wipeMsg) { + if (!fails.length) { + _wipeMsg.textContent = `Deleted ${total} items across all ${okCount} categories.`; + _wipeMsg.className = 'admin-success'; + } else { + _wipeMsg.textContent = `Deleted ${total} items; failed: ${fails.join(', ')}.`; + _wipeMsg.className = 'admin-error'; + } + } } else { - if (_wipeMsg) { _wipeMsg.textContent = data.detail || 'Failed'; _wipeMsg.className = 'admin-error'; } + const res = await fetch(`/api/admin/wipe/${kind}`, { method: 'DELETE', credentials: 'same-origin' }); + const data = await res.json().catch(() => ({})); + if (res.ok) { + if (_wipeMsg) { _wipeMsg.textContent = `Deleted ${data.count ?? 0} ${label}.`; _wipeMsg.className = 'admin-success'; } + } else { + if (_wipeMsg) { _wipeMsg.textContent = data.detail || 'Failed'; _wipeMsg.className = 'admin-error'; } + } } } catch (e) { if (_wipeMsg) { _wipeMsg.textContent = 'Request failed: ' + e.message; _wipeMsg.className = 'admin-error'; } } - btn.disabled = false; btn.textContent = prev; + btn.disabled = false; btn.innerHTML = prevHtml; }); }); } +/* ═══════════════════════════════════════════ + TERMINAL LOGS VIEWER + ═══════════════════════════════════════════ */ +let logsPollInterval = null; +let isLogsPolling = false; +let cachedLogs = []; +let logsAbortController = null; + +function renderLogs(isAutoPoll = false) { + const consoleContainer = el('log-console-container'); + const levelSelect = el('log-level-select'); + const searchInput = el('log-search-input'); + + if (!consoleContainer) return; + + const levelFilter = levelSelect ? levelSelect.value : 'ALL'; + const searchQuery = searchInput ? searchInput.value.trim().toLowerCase() : ''; + + let logs = cachedLogs; + + // Filter by level locally + if (levelFilter !== 'ALL') { + logs = logs.filter(line => line.includes(` - ${levelFilter} - `)); + } + + // Filter by search query locally + if (searchQuery) { + logs = logs.filter(line => line.toLowerCase().includes(searchQuery)); + } + + if (logs.length === 0) { + consoleContainer.innerHTML = '<div class="settings-system-logs-placeholder">No logs found matching current filters.</div>'; + return; + } + + // Preserve scroll position if user is reading previous logs + const atBottom = consoleContainer.scrollHeight - consoleContainer.scrollTop - consoleContainer.clientHeight < 40; + + consoleContainer.innerHTML = logs.map(line => { + let levelClass = 'log-line-default'; + + if (line.includes(' - INFO - ')) { + levelClass = 'log-line-info'; + } else if (line.includes(' - WARNING - ')) { + levelClass = 'log-line-warning'; + } else if (line.includes(' - ERROR - ') || line.includes(' - CRITICAL - ')) { + levelClass = 'log-line-error'; + } else if (line.includes(' - DEBUG - ')) { + levelClass = 'log-line-debug'; + } + + // XSS safe escape + const escaped = line + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + + return `<div class="log-line ${levelClass}">${escaped}</div>`; + }).join(''); + + if (!isAutoPoll || atBottom) { + consoleContainer.scrollTop = consoleContainer.scrollHeight; + } +} + +async function loadLogs(isAutoPoll = false) { + const consoleContainer = el('log-console-container'); + const limitSelect = el('log-limit-select'); + + if (!consoleContainer) return; + + const limit = limitSelect ? limitSelect.value : 200; + + if (logsAbortController) { + logsAbortController.abort(); + } + logsAbortController = new AbortController(); + const { signal } = logsAbortController; + + try { + const res = await fetch(`/api/diagnostics/logs?limit=${limit}`, { + credentials: 'same-origin', + signal + }); + + if (!res.ok) { + if (!isAutoPoll) { + consoleContainer.innerHTML = ''; + const errDiv = document.createElement('div'); + errDiv.style.color = 'var(--red)'; + errDiv.style.fontWeight = '600'; + errDiv.textContent = `Failed to load logs: HTTP ${res.status}`; + consoleContainer.appendChild(errDiv); + } + return; + } + + const data = await res.json(); + if (data.status !== 'success' || !data.logs) { + if (!isAutoPoll) { + consoleContainer.innerHTML = ''; + const errDiv = document.createElement('div'); + errDiv.style.color = 'var(--red)'; + errDiv.style.fontWeight = '600'; + errDiv.textContent = 'Failed to parse logs data'; + consoleContainer.appendChild(errDiv); + } + return; + } + + cachedLogs = data.logs; + renderLogs(isAutoPoll); + } catch (err) { + if (err.name === 'AbortError') { + return; // Silently ignore deliberate abort + } + if (!isAutoPoll) { + consoleContainer.innerHTML = ''; + const errDiv = document.createElement('div'); + errDiv.style.color = 'var(--red)'; + errDiv.style.fontWeight = '600'; + errDiv.textContent = `Error retrieving logs: ${err.message}`; + consoleContainer.appendChild(errDiv); + } + } finally { + if (logsAbortController?.signal === signal) { + logsAbortController = null; + } + } +} + +function startLogsPolling() { + if (isLogsPolling) return; + isLogsPolling = true; + const toggle = el('log-auto-refresh-toggle'); + if (toggle) toggle.checked = true; + + logsPollInterval = setInterval(() => { + const modal = el('settings-modal'); + const systemPanel = el('settings-modal')?.querySelector('[data-settings-panel="system"]'); + + // Safe self-cleanup if modal or panel is hidden/closed + if (!modal || modal.classList.contains('hidden') || !systemPanel || systemPanel.classList.contains('hidden')) { + stopLogsPolling(); + return; + } + + loadLogs(true); + }, 3000); +} + +function stopLogsPolling() { + if (!isLogsPolling) return; + isLogsPolling = false; + if (logsPollInterval) { + clearInterval(logsPollInterval); + logsPollInterval = null; + } + const toggle = el('log-auto-refresh-toggle'); + if (toggle) toggle.checked = false; +} + +function initLogsView() { + const refreshBtn = el('log-refresh-btn'); + const levelSelect = el('log-level-select'); + const limitSelect = el('log-limit-select'); + const searchInput = el('log-search-input'); + const autoRefreshToggle = el('log-auto-refresh-toggle'); + + if (refreshBtn) refreshBtn.addEventListener('click', () => loadLogs(false)); + if (levelSelect) levelSelect.addEventListener('change', () => renderLogs(false)); + if (limitSelect) limitSelect.addEventListener('change', () => loadLogs(false)); + if (searchInput) searchInput.addEventListener('input', () => renderLogs(false)); + + if (autoRefreshToggle) { + autoRefreshToggle.addEventListener('change', (e) => { + if (e.target.checked) { + startLogsPolling(); + } else { + stopLogsPolling(); + } + }); + } + + // Initial fetch on view loading + loadLogs(false); +} + /* ═══════════════════════════════════════════ INIT & REFRESH ═══════════════════════════════════════════ */ function initAll() { modalEl = el('settings-modal'); - const inits = [initSignupToggle, initAddUser, initEndpointForm, initMcpForm, initCalDAV, initBackup, initDangerZone, initTokenForm, () => settingsModule.initIntegrations()]; + const inits = [ + initSignupToggle, initAddUser, initEndpointForm, initMcpForm, + initCalDAV, initBackup, initDangerZone, initTokenForm, initLogsView, + () => settingsModule.initIntegrations() + ]; for (const fn of inits) { try { fn(); } catch (e) { console.error('Admin init error in', fn.name || 'anonymous', e); } } @@ -2507,6 +3004,7 @@ function refreshAll() { loadBuiltinTools(); loadMcpServers(); loadTokens(); + loadLogs(false); } /* ═══════════════════════════════════════════ @@ -2523,6 +3021,7 @@ export function open(tab) { } export function close() { + stopLogsPolling(); settingsModule.close(); } diff --git a/static/js/calendar.js b/static/js/calendar.js index fec9f82c8..4c5c38564 100644 --- a/static/js/calendar.js +++ b/static/js/calendar.js @@ -9,7 +9,7 @@ import { makeWindowDraggable } from './windowDrag.js'; import { attachColorPicker } from './colorPicker.js'; import { bindMenuDismiss } from './escMenuStack.js'; import { - WEEKDAYS, MONTHS, MON_SHORT, + WEEKDAYS, WEEKDAYS_SUN, MONTHS, MON_SHORT, CAL_PALETTE, CAL_COLORS, _CAL_CUSTOM_GRADIENT, _TYPE_PALETTE, _trashIcon, _moreIcon, _bellIcon, _isCalBgImage, _calBgImageUrl, _calBgCss, @@ -64,6 +64,8 @@ let _hiddenTypes = new Set(); // event_type values to hide let _onlyImportant = false; let _filtersCollapsed = localStorage.getItem('cal-filters-collapsed') === '1'; +// Week-start preference: 'mon' (default, Mon=first col) or 'sun' (Sun=first col). +let _weekStartSun = localStorage.getItem('cal-week-start') === 'sun'; let _selectedDay = null; let _view = 'month'; let _searchQuery = ''; @@ -360,14 +362,14 @@ function _today() { return _ds(new Date()); } function _monthRange(d) { const y = d.getFullYear(), m = d.getMonth(); const first = new Date(y, m, 1); - const dow = (first.getDay() + 6) % 7; + const dow = _weekStartSun ? first.getDay() : (first.getDay() + 6) % 7; const gs = new Date(y, m, 1 - dow); const ge = new Date(gs); ge.setDate(gs.getDate() + 42); return [_ds(gs), _ds(ge)]; } function _weekRange(d) { - const dow = (d.getDay() + 6) % 7; + const dow = _weekStartSun ? d.getDay() : (d.getDay() + 6) % 7; const s = new Date(d); s.setDate(d.getDate() - dow); const e = new Date(s); e.setDate(s.getDate() + 7); return [_ds(s), _ds(e)]; @@ -630,6 +632,28 @@ function _getModal() { // ── Render dispatch ── +// Quick-add hint examples — the placeholder cycles through these every few +// seconds so users see different prompt shapes (events, deadlines, recurring). +const _QA_HINT_EXAMPLES = [ + 'return home to Ithaca 1pm tmrw', + 'dinner with Penelope Friday 8pm', + 'coffee with Athena 9am Saturday', + 'call Telemachus tomorrow morning', + 'dentist appointment 3pm next Tuesday', + 'finish the wooden horse by Friday EOD', + 'gym 7am every weekday', + 'flight to Athens Sunday 6:30am', + 'crew muster 10am daily', + 'council on Ithaca Monday 2pm', +]; +function _initQuickAddHintCycle() { + const span = document.getElementById('qa-hint-example'); + if (!span) return; + // Pick one random example per calendar open — no interval cycling. + const idx = Math.floor(Math.random() * _QA_HINT_EXAMPLES.length); + span.textContent = _QA_HINT_EXAMPLES[idx]; +} + // Stash the quick-add input's state (focus + caret + value) before a // re-render so background fetches don't kick the user out mid-type. Picked // up by _wireAll after the new DOM lands. @@ -844,7 +868,7 @@ function _headerHTML() { placeholder=" " autocomplete="off" /> - <span class="cal-quickadd-hint" id="cal-quickadd-hint" aria-hidden="true"><span class="qa-hint-accent">Quick add</span> — return home to Ithaca 1pm tmrw <svg class="qa-hint-enter" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="9 10 4 15 9 20"/><path d="M20 4v7a4 4 0 0 1-4 4H4"/></svg></span> + <span class="cal-quickadd-hint" id="cal-quickadd-hint" aria-hidden="true"><span class="qa-hint-accent">Quick add</span> — <span class="qa-hint-example" id="qa-hint-example">return home to Ithaca 1pm tmrw</span> <svg class="qa-hint-enter" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="9 10 4 15 9 20"/><path d="M20 4v7a4 4 0 0 1-4 4H4"/></svg></span> <span class="cal-quickadd-status" id="cal-quickadd-status"></span> </div>`; } @@ -928,11 +952,11 @@ async function _renderMonth() { _slideDir = 0; let h = _headerHTML() + _filtersRowHTML() + `<div class="cal-grid${slideClass}">`; h += '<div class="cal-week-headers">'; - for (const wd of WEEKDAYS) h += `<div class="cal-weekday">${wd}</div>`; + for (const wd of (_weekStartSun ? WEEKDAYS_SUN : WEEKDAYS)) h += `<div class="cal-weekday">${wd}</div>`; h += '</div>'; const first = new Date(y, m, 1); - const dow = (first.getDay() + 6) % 7; + const dow = _weekStartSun ? first.getDay() : (first.getDay() + 6) % 7; const gs = new Date(y, m, 1 - dow); const multiDay = _events.filter(e => { @@ -1141,13 +1165,13 @@ function _wkEventTopHeight(ev, dayStr) { // Date math if the string isn't shaped as expected. const _toMin = (iso, fallbackDate) => { if (!iso) return null; - const m = iso.match(/T(\d{2}):(\d{2})/); - if (m) { + const mins = _timeToMin(iso); + if (mins !== null && iso.includes('T')) { // If the event spans into a previous/next day, clamp to today's bounds. - const evDate = iso.slice(0, 10); + const evDate = _localDateOf(iso); if (evDate < fallbackDate) return 0; // event started before today if (evDate > fallbackDate) return 24 * 60; // event ends after today - return parseInt(m[1], 10) * 60 + parseInt(m[2], 10); + return mins; } // All-day or date-only — treat as start of day. return 0; @@ -1204,8 +1228,8 @@ async function _renderWeek() { const timedEvents = _eventsForDay(ds).filter(e => _eventVisible(e) && !e.all_day); const isSun = d.getDay() === 0; - colsHtml += `<div class="cal-wk-col${isToday ? ' cal-wk-today' : ''}${isSun ? ' cal-wk-sun' : ''}" data-date="${ds}">`; - colsHtml += `<div class="cal-wk-col-head"><span class="cal-wk-dn">${WEEKDAYS[idx]}</span><span class="cal-wk-dt">${d.getDate()}</span></div>`; + colsHtml += `<div class="cal-wk-col${isToday ? ' cal-wk-today' : ''}${isSun && !_weekStartSun ? ' cal-wk-sun' : ''}" data-date="${ds}">`; + colsHtml += `<div class="cal-wk-col-head"><span class="cal-wk-dn">${(_weekStartSun ? WEEKDAYS_SUN : WEEKDAYS)[idx]}</span><span class="cal-wk-dt">${d.getDate()}</span></div>`; // All-day strip colsHtml += `<div class="cal-wk-allday">`; for (const ev of allDayEvents) { @@ -1286,12 +1310,17 @@ async function _renderWeek() { if (!ev) return; const cols = Array.from(body.querySelectorAll('.cal-wk-grid')); if (!cols.length) return; - // Original timing - const m1 = (ev.dtstart || '').match(/T(\d{2}):(\d{2})/); - const m2 = (ev.dtend || '').match(/T(\d{2}):(\d{2})/); - const startMin0 = m1 ? parseInt(m1[1], 10) * 60 + parseInt(m1[2], 10) : 0; - const endMin0 = m2 ? parseInt(m2[1], 10) * 60 + parseInt(m2[2], 10) : startMin0 + 60; - const durationMin = Math.max(15, endMin0 - startMin0); + // Local/display timing + const startMin0 = _timeToMin(ev.dtstart) ?? 0; + const endMin0 = _timeToMin(ev.dtend) ?? startMin0 + 60; + + let durationMin = endMin0 - startMin0; + const startDs = _localDateOf(ev.dtstart); + const endDs = ev.dtend ? _localDateOf(ev.dtend) : startDs; + if (endDs > startDs && endMin0 <= startMin0) { + durationMin += 24 * 60; + } + durationMin = Math.max(15, durationMin); // Where did the cursor grab the block? (offset from block-top in px) const blockRect = block.getBoundingClientRect(); @@ -1365,7 +1394,7 @@ async function _renderWeek() { // a plain click (no movement) must still open the event. if (moved) block.dataset.justResized = '1'; // Decide whether anything actually moved. - const oldDs = (ev.dtstart || '').slice(0, 10); + const oldDs = _localDateOf(ev.dtstart); if (!nextDs) return; if (nextDs === oldDs && nextStartMin === startMin0) return; // Snapshot the original times so we can offer an Undo. @@ -1374,11 +1403,10 @@ async function _renderWeek() { const newEndMin = nextStartMin + durationMin; const hh = String(Math.floor(nextStartMin / 60)).padStart(2, '0'); const mm = String(nextStartMin % 60).padStart(2, '0'); - const hh2 = String(Math.floor(newEndMin / 60)).padStart(2, '0'); - const mm2 = String((newEndMin) % 60).padStart(2, '0'); - const _tz = _tzOffset(); + const newDtstartDate = new Date(`${nextDs}T${hh}:${mm}:00`); + const _tz = _tzOffsetForDate(newDtstartDate); const newDtstart = `${nextDs}T${hh}:${mm}:00${_tz}`; - const newDtend = `${nextDs}T${hh2}:${mm2}:00${_tz}`; + const newDtend = _addMinutesToLocalIso(newDtstart, durationMin); try { await _updateEvent(uid, { dtstart: newDtstart, dtend: newDtend }); _render(); @@ -1410,10 +1438,7 @@ async function _renderWeek() { const uid = block.dataset.uid; const ev = _events.find(x => x.uid === uid); if (!ev || !grid || !ds) return; - const startMin = (() => { - const m = (ev.dtstart || '').match(/T(\d{2}):(\d{2})/); - return m ? parseInt(m[1], 10) * 60 + parseInt(m[2], 10) : 0; - })(); + const startMin = _timeToMin(ev.dtstart) ?? 0; const initialTop = parseFloat(block.style.top || '0'); const gridRect = grid.getBoundingClientRect(); let newEndMin = startMin; @@ -1438,9 +1463,8 @@ async function _renderWeek() { if (resized) block.dataset.justResized = '1'; if (newEndMin === startMin) return; const prevDtend = ev.dtend; - const hh = String(Math.floor(newEndMin / 60)).padStart(2, '0'); - const mm = String(newEndMin % 60).padStart(2, '0'); - const newDtend = `${ds}T${hh}:${mm}:00${_tzOffset()}`; + const durationMin = newEndMin - startMin; + const newDtend = _addMinutesToLocalIso(ev.dtstart, durationMin); try { await _updateEvent(uid, { dtend: newDtend }); _render(); @@ -1724,9 +1748,9 @@ async function _renderYear() { for (let m = 0; m < 12; m++) { h += `<div class="cal-year-month" data-month="${m}"><div class="cal-year-month-title">${MON_SHORT[m]}</div>`; h += '<div class="cal-year-grid">'; - for (const wd of ['M', 'T', 'W', 'T', 'F', 'S', 'S']) h += `<div class="cal-year-wd">${wd}</div>`; + for (const wd of (_weekStartSun ? ['S','M','T','W','T','F','S'] : ['M','T','W','T','F','S','S'])) h += `<div class="cal-year-wd">${wd}</div>`; const first = new Date(y, m, 1); - const dow = (first.getDay() + 6) % 7; + const dow = _weekStartSun ? first.getDay() : (first.getDay() + 6) % 7; const daysInMonth = new Date(y, m + 1, 0).getDate(); for (let p = 0; p < dow; p++) h += '<div class="cal-year-cell"></div>'; for (let d = 1; d <= daysInMonth; d++) { @@ -1911,6 +1935,7 @@ function _wireAll(body) { // ── Quick-add input ───────────────────────────────────────────── const _qaInput = document.getElementById('cal-quickadd'); const _qaStatus = document.getElementById('cal-quickadd-status'); + _initQuickAddHintCycle(); if (_qaInput && !_qaInput._wired) { _qaInput._wired = true; const _submitQA = async () => { @@ -1966,10 +1991,10 @@ function _wireAll(body) { const ad = document.getElementById('cal-f-allday'); if (ad && !ad.checked) { ad.checked = true; ad.dispatchEvent(new Event('change')); } } else { - const t1 = (ev.dtstart || '').match(/T(\d{2}:\d{2})/); - const t2 = (ev.dtend || '').match(/T(\d{2}:\d{2})/); - if (t1) set('cal-f-start', t1[1]); - if (t2) set('cal-f-end', t2[1]); + const t1 = _fmtTime(ev.dtstart); + const t2 = _fmtTime(ev.dtend); + if (t1) set('cal-f-start', t1); + if (t2) set('cal-f-end', t2); document.getElementById('cal-f-start')?.dispatchEvent(new Event('input')); } // Make sure the details panel is open so the user can verify time. @@ -2474,6 +2499,13 @@ async function _showCalSettings() { </div> <div style="font-size:10px;opacity:0.4;margin-top:4px;">Download a calendar as .ics for backup or to import into another app.</div> </div> + <div style="border-top:1px solid var(--border);padding-top:12px;"> + <div style="font-size:11px;opacity:0.5;margin-bottom:6px;">Week starts on</div> + <div style="display:flex;gap:6px;"> + <button id="cal-wstart-mon" type="button" style="font-size:12px;padding:3px 10px;border-radius:4px;border:1px solid var(--border);background:${!_weekStartSun ? 'color-mix(in srgb, var(--accent,var(--red)) 18%, var(--panel))' : 'var(--panel)'};color:var(--fg);cursor:pointer;transition:background 0.1s,border-color 0.1s;outline:none;">Monday</button> + <button id="cal-wstart-sun" type="button" style="font-size:12px;padding:3px 10px;border-radius:4px;border:1px solid var(--border);background:${_weekStartSun ? 'color-mix(in srgb, var(--accent,var(--red)) 18%, var(--panel))' : 'var(--panel)'};color:var(--fg);cursor:pointer;transition:background 0.1s,border-color 0.1s;outline:none;">Sunday</button> + </div> + </div> <div style="border-top:1px solid var(--border);padding-top:12px;"> <div style="font-size:11px;opacity:0.5;margin-bottom:6px;">Sync</div> <div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap;"> @@ -2494,6 +2526,28 @@ async function _showCalSettings() { overlay.querySelector('#cal-settings-close').addEventListener('click', cleanup); overlay.addEventListener('click', (e) => { if (e.target === overlay) cleanup(); }); + // Week-start toggle: save to localStorage, update module state, re-render. + const _monBtn = overlay.querySelector('#cal-wstart-mon'); + const _sunBtn = overlay.querySelector('#cal-wstart-sun'); + const _activeStyle = 'color-mix(in srgb, var(--accent,var(--red)) 18%, var(--panel))'; + const _inactiveStyle = 'var(--panel)'; + const _applyWeekStartActive = () => { + if (_monBtn) _monBtn.style.background = _weekStartSun ? _inactiveStyle : _activeStyle; + if (_sunBtn) _sunBtn.style.background = _weekStartSun ? _activeStyle : _inactiveStyle; + }; + _monBtn?.addEventListener('click', () => { + _weekStartSun = false; + localStorage.setItem('cal-week-start', 'mon'); + _applyWeekStartActive(); + if (_open) _render(); + }); + _sunBtn?.addEventListener('click', () => { + _weekStartSun = true; + localStorage.setItem('cal-week-start', 'sun'); + _applyWeekStartActive(); + if (_open) _render(); + }); + // Create a new (local) calendar. Defaults the name + next palette color, then // reopens the panel so the user can rename it inline and pick a color. overlay.querySelector('#cal-settings-add')?.addEventListener('click', async (e) => { @@ -2918,35 +2972,68 @@ function _showEventForm(existing, defaultDate, defaultEndDate) { const startEl = document.getElementById('cal-f-start'); const endEl = document.getElementById('cal-f-end'); if (!startEl || !endEl) return; + const _toMin = (v) => { if (!v || !/^\d{2}:\d{2}$/.test(v)) return null; const [h, m] = v.split(':').map(n => parseInt(n, 10)); return h * 60 + m; }; + const _toHHMM = (mins) => { let m = ((mins % 1440) + 1440) % 1440; const hh = String(Math.floor(m / 60)).padStart(2, '0'); const mm = String(m % 60).padStart(2, '0'); return `${hh}:${mm}`; }; + + const _autoAdvanceEndDate = () => { + const isAD = document.getElementById('cal-f-allday')?.checked; + if (isAD) return; + + const dv = document.getElementById('cal-f-date')?.value; + const dvEndEl = document.getElementById('cal-f-date-end'); + if (!dv || !dvEndEl || dvEndEl.value !== dv) return; + + const sVal = startEl.value; + const eVal = endEl.value; + + if (sVal && eVal && eVal <= sVal) { + const d = new Date(`${dv}T00:00:00`); + d.setDate(d.getDate() + 1); + + dvEndEl.value = _ds(d); + } + }; + let prevStartMin = _toMin(startEl.value); - endEl.addEventListener('input', () => { endEl.dataset.userEdited = '1'; }); + + endEl.addEventListener('input', () => { + endEl.dataset.userEdited = '1'; + }); + + endEl.addEventListener('change', _autoAdvanceEndDate); + startEl.addEventListener('change', () => { const newStartMin = _toMin(startEl.value); const endMin = _toMin(endEl.value); - if (newStartMin == null) { prevStartMin = newStartMin; return; } - // Compute the duration before the change. Use the user's existing - // start→end gap, fallback to 1 hour. - let durationMin = 60; - if (prevStartMin != null && endMin != null && endMin > prevStartMin) { - durationMin = endMin - prevStartMin; - } else if (endMin != null && newStartMin != null && endMin > newStartMin && endEl.dataset.userEdited === '1') { - // User already set a custom end before changing start — leave it. + + if (newStartMin == null) { prevStartMin = newStartMin; return; } + + let durationMin = 60; + + if (prevStartMin != null && endMin != null && endMin > prevStartMin) { + durationMin = endMin - prevStartMin; + } else if (endMin != null && newStartMin != null && endMin > newStartMin && endEl.dataset.userEdited === '1') { + prevStartMin = newStartMin; + return; + } + endEl.value = _toHHMM(newStartMin + durationMin); prevStartMin = newStartMin; + _autoAdvanceEndDate(); }); })(); // Custom reminder picker @@ -3007,6 +3094,20 @@ function _showEventForm(existing, defaultDate, defaultEndDate) { // proper UTC instants (is_utc=True). Without this, naive "10:00" gets // re-interpreted as local elsewhere — the timezone-misfire bug. const _tz = _tzOffset(); + + if (!isAD) { + const startVal = document.getElementById('cal-f-start').value; + const endVal = document.getElementById('cal-f-end').value; + + const startDt = new Date(`${dv}T${startVal}:00`); + const endDt = new Date(`${dvEnd}T${endVal}:00`); + + if (endDt <= startDt) { + uiModule.showToast('End time must be after start time'); + return; + } + } + const payload = { summary, dtstart: isAD ? dv : `${dv}T${document.getElementById('cal-f-start').value}:00${_tz}`, @@ -3061,6 +3162,29 @@ function _showEventForm(existing, defaultDate, defaultEndDate) { // mode opens already expanded when there's any detail content to see. titleInput?.addEventListener('focus', () => setExpanded(true), { once: true }); + // Live time parse: typing a time like "11pm" or "15:30" into the title + // updates the hero clock + start input on the fly. The same parser still + // runs again on submit, but doing it live makes the hero clock track + // intent immediately instead of jumping at save. + if (titleInput) { + titleInput.addEventListener('input', () => { + if (document.getElementById('cal-f-allday')?.checked) return; + const tt = _parseTitleTime(titleInput.value); + if (!tt) return; + const startEl = document.getElementById('cal-f-start'); + const endEl = document.getElementById('cal-f-end'); + const newStart = `${String(tt.h).padStart(2, '0')}:${String(tt.m).padStart(2, '0')}`; + if (!startEl || startEl.value === newStart) return; + const toMin = (v) => { const p = (v || '').split(':'); return p.length === 2 ? (+p[0]) * 60 + (+p[1]) : null; }; + const s0 = toMin(startEl.value), e0 = toMin(endEl?.value); + const dur = (s0 != null && e0 != null && e0 > s0) ? e0 - s0 : 60; + startEl.value = newStart; + const endMin = (tt.h * 60 + tt.m + dur) % 1440; + if (endEl) endEl.value = `${String(Math.floor(endMin / 60)).padStart(2, '0')}:${String(endMin % 60).padStart(2, '0')}`; + startEl.dispatchEvent(new Event('input')); + }); + } + // Location → Apple Maps. The pin button next to the input is enabled // only when there's a non-empty location, and its href tracks the live // input value. Apple's universal URL opens the native Maps app on @@ -3215,6 +3339,37 @@ function _fmtTime(s) { } return s.slice(11, 16); } + +function _timeToMin(iso) { + const hm = _fmtTime(iso); + if (!hm) return null; + const m = hm.match(/^(\d{1,2}):(\d{2})$/); + if (!m) return null; + const h = parseInt(m[1], 10); + const min = parseInt(m[2], 10); + if (h < 0 || h > 23 || min < 0 || min > 59) return null; + return h * 60 + min; +} + +function _tzOffsetForDate(d) { + const off = -d.getTimezoneOffset(); + const sign = off >= 0 ? '+' : '-'; + const abs = Math.abs(off); + const hh = String(Math.floor(abs / 60)).padStart(2, '0'); + const mm = String(abs % 60).padStart(2, '0'); + return `${sign}${hh}:${mm}`; +} + +function _addMinutesToLocalIso(baseIso, addMinutes) { + const d = new Date(new Date(baseIso).getTime() + addMinutes * 60000); + const y = d.getFullYear(); + const mo = String(d.getMonth() + 1).padStart(2, '0'); + const da = String(d.getDate()).padStart(2, '0'); + const h = String(d.getHours()).padStart(2, '0'); + const m = String(d.getMinutes()).padStart(2, '0'); + return `${y}-${mo}-${da}T${h}:${m}:00${_tzOffsetForDate(d)}`; +} + function _e(s) { return uiModule.esc ? uiModule.esc(s || '') : (s || '').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"'); } // Linkify a location string: URLs become clickable, plain addresses get a Maps link. diff --git a/static/js/calendar/utils.js b/static/js/calendar/utils.js index a33cc1c66..7e6dd68e8 100644 --- a/static/js/calendar/utils.js +++ b/static/js/calendar/utils.js @@ -3,7 +3,9 @@ // Pure constants + zero-state helpers for the calendar UI. // No DOM, no fetch, no global mutable state — safe to import anywhere. -export const WEEKDAYS = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']; +export const WEEKDAYS = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']; +export const WEEKDAYS_SUN = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']; + export const MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; diff --git a/static/js/chat.js b/static/js/chat.js index 7ecefdb7d..c9b73a8f1 100644 --- a/static/js/chat.js +++ b/static/js/chat.js @@ -787,6 +787,19 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer try { await documentModule.saveDocument({ silent: true }); } catch (_e) { /* best-effort */ } fd.append('active_doc_id', documentModule.getCurrentDocId()); } + // Active email context — when an email reader is open, pass its + // uid/folder/account so "reply", "summarize", "what does this say" + // resolve to the email the user is actually looking at instead of + // making the agent invent a new markdown draft with fake headers. + try { + const getEmailCtx = window.__odysseusGetActiveEmailContext; + const emCtx = typeof getEmailCtx === 'function' ? getEmailCtx() : null; + if (emCtx && emCtx.uid) { + fd.append('active_email_uid', String(emCtx.uid)); + fd.append('active_email_folder', String(emCtx.folder || 'INBOX')); + if (emCtx.account) fd.append('active_email_account', String(emCtx.account)); + } + } catch (_e) { /* best-effort */ } // Web toggle: pre-search in Chat mode, tool permission in Agent mode const toggleState = Storage.loadToggleState(); let isAgentMode = (toggleState.mode || 'chat') === 'agent'; @@ -802,15 +815,15 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer } else { fd.append('use_web', 'true'); } + } else if (isAgentMode) { + fd.append('allow_web_search', 'false'); } if (el('research-toggle').checked) { fd.append('use_research', 'true'); // Research always runs in chat mode — override agent if set fd.set('mode', 'chat'); } - if (el('bash-toggle').checked) { - fd.append('allow_bash', 'true'); - } + fd.append('allow_bash', el('bash-toggle').checked ? 'true' : 'false'); const ragChk = el('rag-toggle'); if (ragChk && !ragChk.checked) { fd.append('use_rag', 'false'); @@ -819,6 +832,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer if (incognitoChk && incognitoChk.checked) { fd.append('incognito', 'true'); } + const _ws = (Storage.KEYS && Storage.get(Storage.KEYS.WORKSPACE, '')) || ''; + if (_ws) { + fd.append('workspace', _ws); + } if (presetsModule.getSelectedPreset()) { fd.append('preset_id', presetsModule.getSelectedPreset()); } @@ -1560,9 +1577,12 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer .replace(/<channel\|>/gi, ''); thinkText = thinkText.replace(/^\s*Thinking(?:\s+Process)?:\s*/i, ''); _liveThinkInner.innerHTML = markdownModule.mdToHtml(thinkText); - // Keep thinking box scrolled to bottom + // Keep thinking box scrolled to bottom, but let user scroll up var thinkBox = _liveThinkInner.closest('.thinking-content'); - if (thinkBox) thinkBox.scrollTop = thinkBox.scrollHeight; + if (thinkBox) { + var nearBottom = thinkBox.scrollHeight - thinkBox.clientHeight - thinkBox.scrollTop < 80; + if (nearBottom) thinkBox.scrollTop = thinkBox.scrollHeight; + } } uiModule.scrollHistory(); continue; @@ -1781,6 +1801,21 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer _sourcesData = json.data; _sourcesType = 'web'; _sourcesHtml = _buildSourcesBox(json.data, 'web'); } + } else if (json.type === 'workspace_rejected') { + // Server refused to bind the posted workspace (deleted folder, + // file path, sensitive dir, filesystem root). Clear the stored + // value so the pill stops claiming a confinement that is not in + // effect, and tell the user. + const _wsPath = (json.data && json.data.path) || ''; + import('./workspace.js').then((m) => { + const ws = m.default || m; + if (ws && ws.setWorkspace) ws.setWorkspace(''); + }); + uiModule.showToast( + `Workspace ${_wsPath || '(unknown)'} is no longer usable; running without confinement`, + 6000 + ); + continue; } else if (json.type === 'model_fallback') { // Model went offline — switched to fallback var _fbData = json.data || {}; @@ -3846,7 +3881,9 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer // Also submit on Enter (without shift) editor.addEventListener('keydown', (e) => { - if (e.key === 'Enter' && !e.shiftKey && !e.isComposing) { + const isMobile = window.innerWidth <= 768 + + if (e.key === 'Enter' && !e.shiftKey && !e.isComposing && !isMobile) { e.preventDefault(); saveBtn.click(); } @@ -3854,9 +3891,11 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer } /** - * Resend a user message — truncates history to that point and resubmits. + * Resend a user message. Normal resend appends a fresh copy at the end of + * the current thread; regenerate flows can opt into replacing from here. */ - export async function resendUserMessage(userMsgElement) { + export async function resendUserMessage(userMsgElement, opts = {}) { + const replaceFromHere = Boolean(opts && opts.replaceFromHere); const box = document.getElementById('chat-history'); const allMsgs = Array.from(box.querySelectorAll('.msg')); const msgIndex = allMsgs.indexOf(userMsgElement); @@ -3902,25 +3941,28 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer const sessionId = sessionModule.getCurrentSessionId(); if (!sessionId) return; - // Truncate backend to keep everything before this user message - const keepCount = msgIndex; try { - await fetch(`${API_BASE}/api/session/${sessionId}/truncate`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ keep_count: keepCount }) - }); + if (replaceFromHere) { + // Regenerate flows intentionally trim history to this point before + // resubmitting. The plain "Resend message" action must not do this. + const keepCount = msgIndex; + await fetch(`${API_BASE}/api/session/${sessionId}/truncate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ keep_count: keepCount }) + }); - // Drop the AI replies after the user message but KEEP the user bubble - // itself (so its photo stays visible). Then suppress the new user - // bubble that send would otherwise add — same pattern as regenerate. - let sibling = userMsgElement.nextSibling; - while (sibling) { - const next = sibling.nextSibling; - sibling.remove(); - sibling = next; + // Drop the AI replies after the user message but KEEP the user bubble + // itself (so its photo stays visible). Then suppress the new user + // bubble that send would otherwise add — same pattern as regenerate. + let sibling = userMsgElement.nextSibling; + while (sibling) { + const next = sibling.nextSibling; + sibling.remove(); + sibling = next; + } + _hideUserBubble = true; } - _hideUserBubble = true; _pendingRegenAttachments = _ids; // Resubmit @@ -4454,6 +4496,15 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer * Delete an AI message and its preceding user message from the conversation. */ export async function deleteMessage(msgElement) { + if (uiModule && uiModule.styledConfirm) { + const ok = await uiModule.styledConfirm('Delete this message?', { + confirmText: 'Delete', + cancelText: 'Cancel', + danger: true, + }); + if (!ok) return; + } + const box = document.getElementById('chat-history'); const allMsgs = Array.from(box.querySelectorAll('.msg')); const clickedIndex = allMsgs.indexOf(msgElement); diff --git a/static/js/chatRenderer.js b/static/js/chatRenderer.js index 7c6ecd096..ce98be4b9 100644 --- a/static/js/chatRenderer.js +++ b/static/js/chatRenderer.js @@ -362,7 +362,7 @@ function _openVisionEditor(att, userMsgEl) { await _saveVisionText(); _closeVisionEditor(); if (userMsgEl && window.chatModule?.resendUserMessage) { - window.chatModule.resendUserMessage(userMsgEl); + window.chatModule.resendUserMessage(userMsgEl, { replaceFromHere: true }); } else if (uiModule?.showToast) { uiModule.showToast('Saved'); } diff --git a/static/js/chatStream.js b/static/js/chatStream.js index 0cc14468e..fc62216ad 100644 --- a/static/js/chatStream.js +++ b/static/js/chatStream.js @@ -185,7 +185,7 @@ export function handleUIControl(uiData) { } else if (uiEvent === 'open_email_reply' || uiData.ui_event === 'open_email_reply') { import('./emailInbox.js').then(function(mod) { var fn = mod.openReplyDraft || (mod.default && mod.default.openReplyDraft); - if (fn) fn(uiData.uid, uiData.folder || 'INBOX', uiData.mode || 'reply'); + if (fn) fn(uiData.uid, uiData.folder || 'INBOX', uiData.mode || 'reply', uiData.body || ''); }).catch(function(e) { console.warn('open_email_reply failed:', e); }); diff --git a/static/js/compare/icons.js b/static/js/compare/icons.js index c2939f273..f6114b1a0 100644 --- a/static/js/compare/icons.js +++ b/static/js/compare/icons.js @@ -40,7 +40,7 @@ export const EVAL_PROMPTS = { chat: [ // ── ★ Featured — prompts that have actually broken frontier models ── { sub: '★ Featured', label: 'Sum digits 2^100', answer: '115', prompt: 'Compute the sum of the decimal digits of 2^100. Do NOT use code execution — work it out by reasoning about the number. Show every step, then end with the final number on its own line.' }, - { sub: '★ Featured', label: 'Three jugs', answer: '4 pours: 7→5, 5→3, 3→7, 5→3', prompt: 'You have three jugs of capacities 7, 5, and 3 liters. The 7-liter jug starts full; the others empty. Using only pouring (no markings), produce the shortest sequence of pours that leaves exactly 2 liters in the 3-liter jug. Output each step as `pour A → B` on its own line. Then state the total number of pours on a final line.' }, + { sub: '★ Featured', label: 'Three jugs', answer: '2 pours: 7→5, 7→3', prompt: 'You have three jugs of capacities 7, 5, and 3 liters. The 7-liter jug starts full; the others empty. Using only pouring (no markings), produce the shortest sequence of pours that leaves exactly 2 liters in the 3-liter jug. Output each step as `pour A → B` on its own line. Then state the total number of pours on a final line.' }, { sub: 'Visual', label: 'Draw SVG', prompt: 'Output a complete self-contained HTML file (```html block, no explanation, no other text) that centers a single SVG illustration on a simple background. The SVG must use only inline shapes — no <img>, no external assets, no JavaScript. Make it expressive and detailed. The SVG should depict: a friendly robot' }, { sub: 'Visual explain', label: 'Black hole HTML', prompt: 'Output a complete HTML file (```html block, no explanation outside the code) that visually explains how a black hole forms. Use four labeled "frames" laid out left-to-right (or stacked on small screens) showing: 1) a glowing massive star, 2) the star going supernova with shockwave rings, 3) collapse into a singularity, 4) the final black hole with a curved accretion disk and bent light around it. Use only vanilla HTML, CSS, and inline SVG — no JavaScript, no images. Each frame should have a one-sentence caption.' }, diff --git a/static/js/cookbook-deps-recipes.js b/static/js/cookbook-deps-recipes.js new file mode 100644 index 000000000..afb1b7287 --- /dev/null +++ b/static/js/cookbook-deps-recipes.js @@ -0,0 +1,97 @@ +// Per-backend × per-model install recipes for the Dependencies tab. +// +// Each entry says: when you're about to serve `model` on `backend`, here's +// the exact shell sequence to make the venv + install the right packages. +// Entries are matched first-hit; put the more specific patterns ABOVE the +// generic fallback for that backend. + +// Recipes carry two variants per entry: +// variants.pip → install into the configured venv via uv/pip +// variants.docker → pull the official container image +// +// The renderer prepends a `source <venv>/bin/activate` for the pip variant +// (env_prefix handles activation for Run). The docker variant skips the +// activate line — `docker pull` doesn't need a venv. + +const _RECIPES = [ + // ── vllm ────────────────────────────────────────────────────────────── + // MiniMax M2/M2.7 — same as the generic vllm install/image for now; + // kept as its own entry so future model-specific patches land in one + // obvious place without touching the catch-all. + { + backend: 'vllm', + label: 'MiniMax M2 / M2.7', + match: (m) => /minimax[-_]?m\s?2(\.7)?/i.test(m || ''), + variants: { + pip: { commands: ['uv pip install -U vllm --torch-backend auto'] }, + docker: { commands: ['docker pull vllm/vllm-openai:latest'] }, + }, + }, + // Generic vllm fallback. + { + backend: 'vllm', + label: 'Any vLLM model', + match: () => true, + variants: { + pip: { commands: ['uv pip install -U vllm --torch-backend auto'] }, + docker: { commands: ['docker pull vllm/vllm-openai:latest'] }, + }, + }, + + // ── sglang ──────────────────────────────────────────────────────────── + { + backend: 'sglang', + label: 'Any SGLang model', + match: () => true, + variants: { + pip: { commands: ['uv pip install -U "sglang[all]" --torch-backend auto'] }, + docker: { commands: ['docker pull lmsysorg/sglang:latest'] }, + }, + }, + + // ── llama.cpp ───────────────────────────────────────────────────────── + { + backend: 'llama_cpp', + label: 'Any GGUF model', + match: () => true, + variants: { + pip: { commands: ['CMAKE_ARGS="-DGGML_CUDA=on" uv pip install -U "llama-cpp-python[server]"'] }, + docker: { commands: ['docker pull ghcr.io/ggerganov/llama.cpp:server-cuda'] }, + }, + }, +]; + +export const RECIPE_VARIANTS = ['pip', 'docker']; +export const RECIPE_DEFAULT_VARIANT = 'pip'; + +// Get the commands array for a recipe + variant. Falls back to pip when +// the requested variant isn't defined for the recipe. +export function recipeCommands(recipe, variant) { + if (!recipe) return []; + const v = (recipe.variants || {})[variant] || (recipe.variants || {}).pip; + return (v && v.commands) || []; +} + +// Backends we surface a recipe panel for. Other rows in the Dependencies +// list keep the existing flat Install/Reinstall button without an expand +// affordance. +export const RECIPE_BACKENDS = new Set(['vllm', 'sglang', 'llama_cpp']); + +// All recipe entries for a given backend, in catalog order. The first one +// is the model-specific match (when present); the last is always the +// generic fallback. +export function recipesForBackend(backend) { + return _RECIPES.filter((r) => r.backend === backend); +} + +// Pick the best recipe for a backend + model id. Returns the catalog +// fallback when nothing more specific matches, or null if the backend +// isn't in the catalog at all. +export function pickRecipe(backend, modelId) { + const candidates = recipesForBackend(backend); + if (!candidates.length) return null; + for (const r of candidates) { + try { if (r.match(modelId)) return r; } catch (_) {} + } + return candidates[candidates.length - 1] || null; +} diff --git a/static/js/cookbook-diagnosis.js b/static/js/cookbook-diagnosis.js index 24d5770e7..933fbe621 100644 --- a/static/js/cookbook-diagnosis.js +++ b/static/js/cookbook-diagnosis.js @@ -65,7 +65,13 @@ import spinnerModule from './spinner.js'; // ── Error diagnosis ── -function _openCookbookDependencies(pkgName = '') { +// Re-exported so callers (Launch-tab pre-flight) can deep-link into the +// Dependencies tab + auto-expand a specific backend's recipe panel and +// pre-select the model they were trying to launch. +export function openCookbookDependencies(pkgName = '', opts = {}) { + _openCookbookDependencies(pkgName, opts); +} +function _openCookbookDependencies(pkgName = '', opts = {}) { const cookbook = window.cookbookModule; if (cookbook && typeof cookbook.open === 'function') { cookbook.open({ tab: 'Dependencies' }); @@ -94,6 +100,34 @@ function _openCookbookDependencies(pkgName = '') { row.scrollIntoView({ block: 'center' }); row.classList.add('cookbook-pkg-flash'); setTimeout(() => row.classList.remove('cookbook-pkg-flash'), 1800); + // Pre-flight deep link: auto-expand the recipe panel + pre-select + // the model the user was trying to launch. The dropdown values are + // now full model ids (sourced from _cachedModelIds), so we match by + // exact value first, then fall back to a substring match. + if (opts.expandRecipe) { + const caret = row.querySelector('[data-dep-recipe-toggle]'); + if (caret && caret.getAttribute('aria-expanded') !== 'true') caret.click(); + if (opts.model) { + const sel = document.querySelector(`[data-dep-recipe-pick="${CSS.escape(opts.expandRecipe)}"]`); + if (sel) { + const wanted = String(opts.model); + let matched = false; + for (let i = 0; i < sel.options.length; i++) { + if (sel.options[i].value === wanted) { + sel.value = wanted; matched = true; break; + } + } + if (!matched) { + for (let i = 0; i < sel.options.length; i++) { + if (sel.options[i].value && wanted.includes(sel.options[i].value)) { + sel.value = sel.options[i].value; matched = true; break; + } + } + } + if (matched) sel.dispatchEvent(new Event('change')); + } + } + } } }; tryHighlight(); @@ -320,6 +354,15 @@ export const ERROR_PATTERNS = [ }}, ], }, + { + pattern: /sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|Please ensure sgl_kernel is properly installed/i, + message: 'SGLang native dependencies are missing on this server.', + fixes: [ + { label: 'Copy OS package command', action: () => _copyText('sudo apt-get install -y libnuma-dev python3.12-dev build-essential') }, + { label: 'Copy kernel upgrade', action: () => _copyText('python3 -m pip install --upgrade sglang-kernel') }, + { label: 'Open Dependencies', action: () => _openCookbookDependencies('sglang') }, + ], + }, { pattern: /sglang.*command not found|No module named sglang|SGLang is not installed/i, message: 'SGLang is not installed or not in PATH.', @@ -406,7 +449,7 @@ export const ERROR_PATTERNS = [ { label: 'Repair kernel package', action: () => { const _vp = (_envState.env === 'venv' && _envState.envPath) ? `${_envState.envPath.replace(/\/+$/, '')}/bin/python3` : 'python3'; - _launchServeTask('repair-kernels', 'pip-update', `${_vp} -m pip install --user --break-system-packages kernels<0.15`); + _launchServeTask('repair-kernels', 'pip-update', `${_vp} -m pip install --user --break-system-packages "kernels<0.15"`); }}, { label: 'Open Dependencies', action: () => _openCookbookDependencies('sglang') }, ], @@ -617,7 +660,24 @@ export function _showDiagnosis(panel, diagnosis, sourceText) { // the full error+context for a forum/discord paste. const toolbar = document.createElement('div'); toolbar.className = 'cookbook-diag-toolbar'; - toolbar.style.cssText = 'display:flex;justify-content:flex-end;align-items:center;gap:4px;margin-bottom:-2px;'; + // Left side carries the diagnosis text (message + suggestion); buttons + // stay on the right. Was a separate body row below the toolbar, but + // the message reads more like "this is what the toolbar is for" when + // it sits inline with Copy / × Dismiss. + toolbar.style.cssText = 'display:flex;align-items:flex-start;gap:8px;margin-bottom:-2px;'; + + const textWrap = document.createElement('div'); + textWrap.style.cssText = 'flex:1;min-width:0;font-size:11px;line-height:1.35;'; + const msg = document.createElement('div'); + msg.className = 'cookbook-diag-message'; + msg.textContent = diagnosis.message; + textWrap.appendChild(msg); + const suggestion = document.createElement('div'); + suggestion.className = 'cookbook-diag-suggestion'; + suggestion.textContent = suggestionText; + suggestion.style.cssText = 'opacity:0.75;margin-top:1px;'; + textWrap.appendChild(suggestion); + toolbar.appendChild(textWrap); const copyBtn = document.createElement('button'); copyBtn.type = 'button'; @@ -651,18 +711,6 @@ export function _showDiagnosis(panel, diagnosis, sourceText) { toolbar.appendChild(dismissBtn); diag.appendChild(toolbar); - const body = document.createElement('div'); - body.className = 'cookbook-diag-body'; - const msg = document.createElement('div'); - msg.className = 'cookbook-diag-message'; - msg.textContent = diagnosis.message; - body.appendChild(msg); - const suggestion = document.createElement('div'); - suggestion.className = 'cookbook-diag-suggestion'; - suggestion.textContent = suggestionText; - body.appendChild(suggestion); - diag.appendChild(body); - const runFix = async (fix, button, busyLabel = fix.label, onStart = null, onDone = null) => { if (!fix || !button || button.dataset.busy) return; button.dataset.busy = '1'; @@ -709,7 +757,7 @@ export function _showDiagnosis(panel, diagnosis, sourceText) { }); row.appendChild(btn); } - body.appendChild(row); + diag.appendChild(row); } } diff --git a/static/js/cookbook-hwfit.js b/static/js/cookbook-hwfit.js index d8652d02e..243d3c9c7 100644 --- a/static/js/cookbook-hwfit.js +++ b/static/js/cookbook-hwfit.js @@ -31,6 +31,44 @@ import { } from './cookbook.js'; import uiModule from './ui.js'; import spinnerModule from './spinner.js'; +import { _loadTasks, _tmuxGracefulKill } from './cookbookRunning.js'; +import { openCookbookDependencies } from './cookbook-diagnosis.js'; + +// Map a serve-backend code (vllm / sglang / llamacpp) → the package name +// the Dependencies API reports. Used to look up "is this backend installed +// on the target server" before firing a launch. +const _BACKEND_PKG = { vllm: 'vllm', sglang: 'sglang', llamacpp: 'llama_cpp' }; + +// Pre-launch: ask the deps API whether the chosen backend is present on +// the target server. Returns true if it's good to go, false if we should +// block and route the user into Dependencies. +async function _ensureBackendInstalled(runBackend, host, port, envPath, modelName) { + const pkgName = _BACKEND_PKG[runBackend]; + if (!pkgName) return true; // unknown backend — don't block + try { + const params = new URLSearchParams(); + if (host) { + params.set('host', host); + if (port) params.set('ssh_port', String(port)); + if (envPath) params.set('venv', envPath); + } + const r = await fetch('/api/cookbook/packages' + (params.toString() ? '?' + params : '')); + const d = await r.json(); + const pkg = (d.packages || []).find(p => p.name === pkgName); + if (pkg && pkg.installed) return true; + } catch (_) { + // If we can't tell, don't block — the server's own serve route will + // surface a clearer error anyway. + return true; + } + const targetLabel = host || 'this server'; + uiModule.showToast( + `${pkgName} not installed on ${targetLabel}. Opening Dependencies — pick your model and click Run.`, + 6000 + ); + openCookbookDependencies(pkgName, { expandRecipe: pkgName, model: modelName }); + return false; +} // ── What Fits? (hardware model fitting) ── @@ -127,7 +165,12 @@ export function _renderGpuToggles(system) { _gpuToggleTotal = 0; return; } - if (!_gpuToggleTotal) _gpuToggleTotal = total; + // Update on every scan that returns a positive total — previously this + // only set on the first scan, so switching servers (e.g. local 1-GPU + // first, then a 4-GPU remote) left the Run-panel GPU buttons stuck on + // the original count. Zero/missing totals still don't clobber a known + // good value (avoids flicker during an in-flight re-probe). + if (total > 0) _gpuToggleTotal = total; container._groups = groups; if (container._activeGroup === undefined) container._activeGroup = 0; // auto = largest pool @@ -159,8 +202,17 @@ export function _renderGpuToggles(system) { // visual highlight. Before this, _activeCount stayed undefined → no // gpu_count param sent → backend's fallback could rank against RAM on // mixed-resource boxes ("tightest" sorted by RAM instead of GPU). - if (container._activeCount === undefined && validCounts.length) { - container._activeCount = maxGpu; + // + // On boxes where total RAM > total VRAM, default to RAM (count=0) instead + // — RAM is the dominant pool so it's the better starting filter. + if (container._activeCount === undefined) { + const ramGb = Number(system.total_ram_gb) || 0; + const vramGb = Number(system.gpu_vram_gb) || 0; + if (ramGb > vramGb) { + container._activeCount = 0; + } else if (validCounts.length) { + container._activeCount = maxGpu; + } } html += '<button class="hwfit-gpu-btn" data-count="0" title="CPU / RAM only">RAM</button>'; const hasExplicitCount = typeof container._activeCount === 'number'; @@ -363,7 +415,7 @@ function _scanSig() { hk: _currentServerValue(), u: document.getElementById('hwfit-usecase')?.value || '', s: document.getElementById('hwfit-search')?.value?.trim() || '', - o: sortEl?.value || 'score', + o: sortEl?.value || 'newest', r: sortEl?.dataset.reverse === '1' ? 1 : 0, q: document.getElementById('hwfit-quant')?.value || '', c: _ctxValue(), @@ -582,7 +634,7 @@ export async function _hwfitFetch(fresh = false) { }).catch(() => {}); } try { - const sortBy = document.getElementById('hwfit-sort')?.value || 'score'; + const sortBy = document.getElementById('hwfit-sort')?.value || 'newest'; const quantPref = document.getElementById('hwfit-quant')?.value || ''; const targetCtx = _ctxValue(); // Get active GPU count from toggles @@ -710,7 +762,7 @@ export async function _hwfitFetch(fresh = false) { // 1st click on a column = highest first; clicking it again = lowest first. if (!isImageMode) { const sortSel = document.getElementById('hwfit-sort'); - const sortKey = sortSel?.value || 'score'; + const sortKey = sortSel?.value || 'newest'; const asc = sortSel?.dataset.reverse === '1'; // reversed → ascending (lowest first) if (sortKey === 'fit') { // fit_level is categorical (perfect→good→marginal→too_tight), not numeric, @@ -723,6 +775,18 @@ export async function _hwfitFetch(fresh = false) { const as = Number(a.score) || 0, bs = Number(b.score) || 0; return asc ? as - bs : bs - as; }); + } else if (sortKey === 'newest') { + // release_date is an ISO-ish "YYYY-MM-DD" string — lexical sort is + // chronological. Default direction: newest first (reverse=undefined). + data.models.sort((a, b) => { + const ad = String(a.release_date || ''), bd = String(b.release_date || ''); + if (ad === bd) return 0; + // Empty dates land last regardless of direction so the column never + // floats undated rows above real releases. + if (!ad) return 1; + if (!bd) return -1; + return asc ? (ad < bd ? -1 : 1) : (ad < bd ? 1 : -1); + }); } else { const field = { score: 'score', vram: 'required_gb', speed: 'speed_tps', params: 'params_b', context: 'context' }[sortKey] || 'score'; data.models.sort((a, b) => { @@ -750,6 +814,80 @@ export async function _hwfitFetch(fresh = false) { } } +// Renders a non-blocking hardware visibility warning when Cookbook is using +// container-visible hardware that may not match the user's actual host machine. +function _renderHwVisibilityWarning(sys) { + const row = document.getElementById('hwfit-hw-row'); + if (!row) return; + + let box = document.getElementById('hwfit-hw-visibility-warning'); + + // Manual hardware is an explicit user override, so avoid showing stale + // container-detection warnings once the user has chosen a simulated profile. + const warning = sys?.manual_hardware ? null : sys?.hardware_visibility_warning; + + if (!warning) { + if (box) box.remove(); + return; + } + + if (!box) { + box = document.createElement('div'); + box.id = 'hwfit-hw-visibility-warning'; + box.className = 'hwfit-loading hwfit-hw-visibility-warning'; + row.insertAdjacentElement('afterend', box); + } + + box.innerHTML = ` + <div class="hwfit-hw-visibility-warning-title">${esc(warning.title || 'Hardware visibility note')}</div> + <div class="hwfit-hw-visibility-warning-body">${esc(warning.message || '')}</div> + <div class="hwfit-hw-visibility-warning-actions"> + <button type="button" class="hwfit-gpu-btn" data-hw-action="manual">Edit manual hardware</button> + <button type="button" class="hwfit-gpu-btn" data-hw-action="rescan">Rescan</button> + <button type="button" class="hwfit-gpu-btn" data-hw-action="copy">Copy diagnostics</button> + </div> + `; + + box.querySelector('[data-hw-action="manual"]')?.addEventListener('click', () => { + const panel = document.getElementById('hwfit-manual-panel'); + if (panel) panel.classList.remove('hidden'); + document.getElementById('hwfit-hw-manual-btn')?.scrollIntoView?.({ + behavior: 'smooth', + block: 'center', + }); + }); + + box.querySelector('[data-hw-action="rescan"]')?.addEventListener('click', () => { + _resetGpuToggleState(); + _hwfitCache = null; + _hwfitFetch(true); + }); + + box.querySelector('[data-hw-action="copy"]')?.addEventListener('click', () => { + // Keep diagnostics copy/paste friendly for GitHub issues and Docker support. + const text = [ + 'Odysseus Cookbook hardware diagnostics', + `probe_scope=${sys?.probe_scope || ''}`, + `containerized=${sys?.containerized === true}`, + `backend=${sys?.backend || ''}`, + `has_gpu=${sys?.has_gpu === true}`, + `gpu_name=${sys?.gpu_name || ''}`, + `gpu_count=${sys?.gpu_count || 0}`, + `gpu_vram_gb=${sys?.gpu_vram_gb || ''}`, + `ram=${sys?.available_ram_gb || '?'} / ${sys?.total_ram_gb || '?'} GB`, + `cpu_cores=${sys?.cpu_cores || ''}`, + `cpu_name=${sys?.cpu_name || ''}`, + '', + 'Useful checks:', + 'docker compose exec odysseus nvidia-smi -L', + 'docker compose exec odysseus cat /proc/meminfo | head', + 'docker compose exec odysseus python -c "from services.hwfit.hardware import detect_system; import json; print(json.dumps(detect_system(fresh=True), indent=2))"', + ].join('\n'); + + _copyText(text); + }); +} + export function _hwfitRenderHw(el, sys) { if (!el || !sys) return; // Cache system info globally so other modules can read VRAM without refetching @@ -838,6 +976,7 @@ export function _hwfitRenderHw(el, sys) { + chip('cores', cores) + chip('backend', esc(sys.backend || '')) + manualChip; + _renderHwVisibilityWarning(sys); // Body click → toggle "off" (dimmed, still visible). Membership of // _dismissedHwChips is what the ranker reads, so both add+remove // here also flips the model list. The manual chip is excluded — @@ -968,7 +1107,7 @@ function _modeLabel(model) { export const _hwfitColumns = [ { key: 'fit', label: 'Fit', cls: 'hwfit-fit' }, - { key: null, label: 'Model', cls: 'hwfit-name' }, + { key: 'newest', label: 'Model (latest)', cls: 'hwfit-name' }, { key: 'params',label: 'Param', cls: 'hwfit-c-params' }, { key: null, label: 'Quant', cls: 'hwfit-c-quant' }, { key: 'vram', label: 'VRAM', cls: 'hwfit-c-vram' }, @@ -998,7 +1137,7 @@ export function _hwfitRenderList(el, models) { return; } const sortSel = document.getElementById('hwfit-sort'); - const currentSort = sortSel?.value || 'score'; + const currentSort = sortSel?.value || 'newest'; const isReversed = sortSel?.dataset.reverse === '1'; // Active budget for the Fit column label \u2014 make it obvious whether the // ranking is against GPU or RAM so "tightest" can't be ambiguous on a @@ -1027,6 +1166,13 @@ export function _hwfitRenderList(el, models) { // (Budget tag removed — the GPU/RAM/N-GPU suffix next to "Fit" was noise; // the toggle row already shows which budget is active.) } + // The Model column's "(newest)" / "(oldest)" suffix flips with the sort + // direction so the user can see at a glance which way they're sorted. + if (col.key === 'newest' && col.key === currentSort) { + label = isReversed ? 'Model (oldest)' : 'Model (latest)'; + } else if (col.key === 'newest') { + label = 'Model (latest)'; + } html += `<span class="hwfit-col ${col.cls}${sortable}${active}"${dataAttr}>${label}${arrow}</span>`; } html += '</div>'; @@ -1181,6 +1327,72 @@ function _syncHostFromScanDropdown() { return host; } +// Minimum backend version a given model needs. Returns a semver string like +// "0.10.0" or null when the model has no known floor. Hardcoded for now — +// when the vLLM-recipes integration lands we can pull this from the upstream +// recipe page instead. Keep this conservative: a null return means "any +// installed version passes", so we don't false-positive launches. +function _minBackendVersion(modelName, backend) { + const n = (modelName || '').toLowerCase(); + if (backend === 'vllm') { + // MiniMax M2 / M2.5 / M2.7 — minimax_m2 parser shipped in 0.10.0 + if (n.includes('minimax') && n.match(/\bm2(?:\.\d)?\b/)) return '0.10.0'; + // MiniMax M3 — newer parser registered in 0.11.x + if (n.includes('minimax') && n.includes('m3')) return '0.11.0'; + // DeepSeek V3 / V3.1 / R1 — MoE expert-parallel paths matured in 0.7.0+ + if (n.includes('deepseek') && (n.includes('v3') || n.includes('r1'))) return '0.7.0'; + // Qwen3 reasoning models — qwen3 reasoning parser added in 0.7.0 + if (n.includes('qwen3') && !n.includes('coder') && !n.includes('instruct')) return '0.7.0'; + // GLM-4.5 / GLM-4.6 — glm45 reasoning parser added in 0.8.0 + if (n.includes('glm-4.5') || n.includes('glm-4.6') || n.includes('glm-5')) return '0.8.0'; + // gpt-oss reasoning models — gpt_oss parser + if (n.includes('gpt-oss')) return '0.10.0'; + // Llama-4 multimodal — landed in 0.7.0 + if (n.includes('llama-4') || n.includes('llama4')) return '0.7.0'; + } + return null; +} + +// Tiny semver compare: returns <0 / 0 / >0 like strcmp. Tolerates "0.10", +// "0.10.0", "0.10.0+cu124" — pre-release / build suffixes are stripped. +function _cmpSemver(a, b) { + const _parse = (s) => String(s || '').split(/[.+-]/).filter(p => /^\d+$/.test(p)).map(Number); + const A = _parse(a), B = _parse(b); + for (let i = 0; i < Math.max(A.length, B.length); i++) { + const av = A[i] || 0, bv = B[i] || 0; + if (av !== bv) return av - bv; + } + return 0; +} + +// Map the detected GPU + the model's quant to SGLang's URL-hash params so +// the cookbook page lands on the right preset. SGLang supports: +// hw = b200 | b300 | gb200 | gb300 | mi300x | mi325x | mi350x | mi355x | h200 +// quant = mxfp8 | bf16 +// variant = default strategy = balanced nodes = single +// We only set what we can confidently infer; anything missing degrades to +// SGLang's own default (which is `h200` + bf16 single-node balanced). +function _sglangHashFor(modelData) { + const sys = (typeof _hwfitCache !== 'undefined' ? _hwfitCache?.system : null) || {}; + const gpuName = String(sys.gpu_name || '').toLowerCase(); + let hw = ''; + if (/\bgb300/.test(gpuName)) hw = 'gb300'; + else if (/\bgb200/.test(gpuName)) hw = 'gb200'; + else if (/\bb300/.test(gpuName)) hw = 'b300'; + else if (/\bb200/.test(gpuName)) hw = 'b200'; + else if (/\bh200/.test(gpuName)) hw = 'h200'; + else if (/mi355/.test(gpuName)) hw = 'mi355x'; + else if (/mi350/.test(gpuName)) hw = 'mi350x'; + else if (/mi325/.test(gpuName)) hw = 'mi325x'; + else if (/mi300/.test(gpuName)) hw = 'mi300x'; + const qRaw = String(modelData?.quant || '').toLowerCase(); + // mxfp8 covers fp8 / mxfp8 / nvfp4; bf16 covers everything else cheap. + const quant = /fp8|mxfp|nvfp/.test(qRaw) ? 'mxfp8' : 'bf16'; + const parts = ['variant=default', `quant=${quant}`, 'strategy=balanced', 'nodes=single']; + if (hw) parts.unshift(`hw=${hw}`); + return '#' + parts.join('&'); +} + export function _expandModelRow(row, modelData) { const list = row.closest('.hwfit-list'); if (!list) return; @@ -1203,11 +1415,23 @@ export function _expandModelRow(row, modelData) { const dlSource = _downloadSourceRepo(modelData, backend); const hfUrl = `https://huggingface.co/${dlSource.repo}`; + // Official vendor recipe deep-links. These point to vLLM / SGLang's curated + // hardware-specific launch-command pages. They 404 for uncatalogued models \u2014 + // a known tradeoff; user just gets the vendor's "model not found" page. + const _recipeRepo = modelData.name || ''; + const _vllmUrl = _recipeRepo ? `https://recipes.vllm.ai/${_recipeRepo}` : ''; + const _sglangUrl = _recipeRepo ? `https://docs.sglang.io/cookbook/autoregressive/${_recipeRepo}${_sglangHashFor(modelData)}` : ''; let html = `<div class="hwfit-action-panel" data-model-name="${esc(modelData.name)}">`; html += `<div class="hwfit-panel-header">`; html += `<span class="hwfit-panel-model">${esc(modelData.name)}${dlSource.kind ? ` <span style="opacity:0.5;font-size:10px;">(${esc(dlSource.kind)} ${esc(modelData.quant || '')})</span>` : (modelData.quant_repo ? ` <span style="opacity:0.5;font-size:10px;">(${esc(modelData.quant)})</span>` : '')}</span>`; html += `<span class="hwfit-panel-badge">${esc(label)}</span>`; html += `<a href="${esc(hfUrl)}" target="_blank" rel="noopener" class="hwfit-panel-hf-link" title="View download source on HuggingFace">HF \u2197</a>`; + if (backend === 'vllm' && _vllmUrl) { + html += `<a href="${esc(_vllmUrl)}" target="_blank" rel="noopener" class="hwfit-panel-hf-link" title="vLLM official recipe (curated launch command). 404s if this model isn't in vLLM's recipes catalog.">vLLM \u2197</a>`; + } + if (backend === 'sglang' && _sglangUrl) { + html += `<a href="${esc(_sglangUrl)}" target="_blank" rel="noopener" class="hwfit-panel-hf-link" title="SGLang cookbook (hash pre-filled with your detected hardware). 404s if this model isn't in SGLang's cookbook catalog.">SGLang \u2197</a>`; + } html += `</div>`; html += `<div class="hwfit-panel-actions">`; html += `<button class="cookbook-btn hwfit-dl-btn">Download</button>`; @@ -1276,6 +1500,133 @@ export function _expandModelRow(row, modelData) { return; } + // ─── Pre-launch: stop the model already serving on this host ─────── + // Two servers can't share port 8000. Without this, the new launch + // silently collided and the user saw no feedback. We surface the + // conflict and offer to kill the running one first as the default + // action (it's almost always what the user wants). + try { + const _qrHostStr = _envState.remoteHost || ''; + const _activeServes = _loadTasks().filter(t => + t && t.type === 'serve' + && (t.remoteHost || '') === _qrHostStr + && (t.status === 'running' || t.status === 'ready' || t._serveReady) + ); + if (_activeServes.length) { + const _names = _activeServes.map(t => t.payload?.repo_id || t.repo || t.name || '?').filter(Boolean); + const _ok = await window.styledConfirm?.( + `${_names.length} model${_names.length === 1 ? '' : 's'} already serving on ${_qrHostStr || 'local'} (${_names.join(', ')}). Port 8000 will collide. Stop the running model and launch this one?`, + { confirmText: 'Stop & launch', cancelText: 'Cancel' } + ); + if (!_ok) return; + // Mark + kill each running serve, then wait briefly for the + // tmux session to actually go down before we kick off the new + // launch. Otherwise vLLM still races against the dying socket. + quickRunBtn.disabled = true; + quickRunBtn.textContent = 'Stopping…'; + for (const t of _activeServes) { + try { + // Use that task's own Stop button if it's rendered (handles + // endpoint cleanup, Ollama unload, fade-out). Falls back to + // a direct tmux kill if the Active tab isn't in the DOM yet. + const _taskEl = document.querySelector(`.cookbook-task[data-task-id="${t.sessionId}"]`); + const _stopBtn = _taskEl?.querySelector('.cookbook-task-action-stop'); + if (_stopBtn) { + _stopBtn.click(); + } else { + await fetch('/api/shell/exec', { + method: 'POST', + credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ command: _tmuxGracefulKill(t) }), + }); + } + } catch (_killErr) { /* best-effort */ } + } + // Give the OS a beat to release port 8000. + await new Promise(r => setTimeout(r, 2500)); + } + } catch (_e) { /* best-effort */ } + + // ─── Pre-launch driver check ───────────────────────────────────── + // vLLM/SGLang need a working CUDA/ROCm driver. nvidia-smi failures + // surface as system.gpu_error from our hardware probe; "no GPU + // detected" is the other common case. Bail with a clear message + // before kicking off the long install/launch chain — otherwise the + // user watches `pip install vllm` finish, then sees a cryptic CUDA + // error 10 minutes later. (llama.cpp / Ollama have CPU fallbacks + // so they skip this gate.) + const _qrBackendDetect = _detectBackend(modelData); + const _qrRunBackend = _qrBackendDetect.backend || 'vllm'; + if (_qrRunBackend === 'vllm' || _qrRunBackend === 'sglang') { + const _sys = _hwfitCache?.system || {}; + if (_sys.gpu_error) { + uiModule.showError(`Can't launch: GPU driver error — ${_sys.gpu_error}. Reinstall or repair the NVIDIA driver, then re-scan.`); + return; + } + if (!_sys.has_gpu || !(_sys.gpu_count > 0)) { + uiModule.showError(`Can't launch: no GPU detected by nvidia-smi. ${_qrRunBackend === 'vllm' ? 'vLLM' : 'SGLang'} needs a working CUDA or ROCm device.`); + return; + } + } + + // ─── Pre-launch install + version check ───────────────────────── + // Catches: + // a) "command not found" (binary not in PATH) + // b) "version too old" (model needs e.g. vllm >= 0.10.0 for the + // reasoning/tool parser registered for it). + // Both cases would otherwise fail 10s-3min into the launch with a + // cryptic shell error. Best-effort: a venv activated only by the + // launch wrapper can false-negative the PATH check, in which case + // the launch proceeds and the existing diagnosis layer handles it. + if (_qrRunBackend === 'vllm' || _qrRunBackend === 'sglang') { + try { + const _qrHostStr = _envState.remoteHost || ''; + const _coreCheck = _qrRunBackend === 'vllm' + ? "command -v vllm >/dev/null 2>&1 && vllm --version 2>&1 | grep -oE '[0-9]+\\.[0-9]+(\\.[0-9]+)?' | head -1 || echo MISSING" + : "python3 -c 'import sglang, sys; sys.stdout.write(sglang.__version__)' 2>/dev/null || echo MISSING"; + const _wrappedCheck = _qrHostStr + ? `ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ${_qrHostStr} "bash -lc ${JSON.stringify(_coreCheck)}"` + : `bash -lc ${JSON.stringify(_coreCheck)}`; + const _chkRes = await fetch('/api/shell/exec', { + method: 'POST', + credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ command: _wrappedCheck, timeout: 10 }), + }); + if (_chkRes.ok) { + const _chk = await _chkRes.json(); + const _stdout = String(_chk.stdout || '').trim(); + const _stderr = String(_chk.stderr || '').trim(); + const _out = `${_stdout}\n${_stderr}`; + if (_out.includes('MISSING')) { + const _pkg = _qrRunBackend === 'vllm' ? 'vLLM' : 'SGLang'; + const _hint = _qrRunBackend === 'vllm' + ? 'uv pip install -U vllm --torch-backend auto' + : "pip install -U 'sglang[all]'"; + uiModule.showError(`Can't launch: ${_pkg} isn't installed${_qrHostStr ? ' on ' + _qrHostStr : ''}. Install it first:\n${_hint}`); + return; + } + // Version-floor check. _minBackendVersion returns null when this + // model has no known requirement — in which case any installed + // version passes. + const _minVer = _minBackendVersion(modelData.name, _qrRunBackend); + const _verMatch = _stdout.match(/(\d+\.\d+(?:\.\d+)?)/); + const _curVer = _verMatch ? _verMatch[1] : ''; + if (_minVer && _curVer && _cmpSemver(_curVer, _minVer) < 0) { + const _pkg = _qrRunBackend === 'vllm' ? 'vLLM' : 'SGLang'; + const _hint = _qrRunBackend === 'vllm' + ? 'uv pip install -U vllm --torch-backend auto' + : "pip install -U 'sglang[all]'"; + uiModule.showError(`Can't launch: ${modelData.name} needs ${_pkg} ≥ ${_minVer}, but ${_curVer} is installed${_qrHostStr ? ' on ' + _qrHostStr : ''}. Upgrade:\n${_hint}`); + return; + } + } + } catch (_e) { + // Network/exec failed — fall through and let the launch try. + } + } + quickRunBtn.disabled = true; quickRunBtn.textContent = 'Starting...'; @@ -1353,6 +1704,23 @@ export function _expandModelRow(row, modelData) { // schema (repo_id + cmd) — sending `command`/`model` failed Pydantic // validation (422), which is why Run silently did nothing. const _srv = _serverByVal(_envState.remoteServerKey || host); + + // Pre-flight: if the backend isn't installed on the target server, + // route the user into Dependencies → recipe panel for that backend + // instead of launching into an obvious "command not found" failure. + const _ok = await _ensureBackendInstalled( + runBackend, + host, + (_srv && _srv.port) || undefined, + _envState.envPath || '', + modelData.name, + ); + if (!_ok) { + quickRunBtn.disabled = false; + quickRunBtn.textContent = 'Run'; + return; + } + const payload = { repo_id: modelData.name, cmd: cmd, @@ -1506,12 +1874,10 @@ export function _hwfitInit() { clearTimeout(_hwfitDebounce); _hwfitDebounce = setTimeout(() => _hwfitFetch(), 400); }); - // HF Token - const hfToken = document.getElementById('hwfit-hftoken'); - if (hfToken) { - hfToken.addEventListener('change', () => { _envState.hfToken = hfToken.value.trim(); _persistEnvState(); }); - hfToken.addEventListener('input', () => { _envState.hfToken = hfToken.value.trim(); }); - } + // HF token save is owned by cookbook.js (_wireTabEvents) — do not wire a + // second change/input handler here. The old duplicate ran after cookbook.js + // cleared the input on save and overwrote _envState.hfToken with "", so the + // debounced state sync never persisted the token to cookbook_state.json. // Rebuild all server select dropdowns with current servers function _rebuildServerSelect() { diff --git a/static/js/cookbook.js b/static/js/cookbook.js index 2abb263ba..fc05217c1 100644 --- a/static/js/cookbook.js +++ b/static/js/cookbook.js @@ -8,6 +8,7 @@ import spinnerModule from './spinner.js'; import { providerLogo } from './providers.js'; import { makeWindowDraggable } from './windowDrag.js'; import { _diagnose, _showDiagnosis, _clearDiagnosis, _runQuickCmd, ERROR_PATTERNS } from './cookbook-diagnosis.js'; +import { RECIPE_BACKENDS, recipesForBackend, pickRecipe, recipeCommands, RECIPE_DEFAULT_VARIANT } from './cookbook-deps-recipes.js'; import { _hwfitCache, _hwfitDebounce, _hwfitFetch, _hwfitInit, _hwfitRenderList, _hwfitRenderHw, _renderGpuToggles, _expandModelRow, _fitColors, _hwfitColumns, _cachedModelIds, _gpuToggleTotal, _resetGpuToggleState } from './cookbook-hwfit.js'; // Sub-modules @@ -233,22 +234,39 @@ function _detectModelOptimizations(modelName) { const n = (modelName || '').toLowerCase(); const opts = { envVars: [], flags: [], tips: [] }; - // Qwen3.5 MoE models + // Qwen3.5 MoE models — MoE-specific env vars + expert-parallel. + // The --reasoning-parser flag is added uniformly below via + // _detectReasoningParser, no longer hardcoded here. if (n.includes('qwen3.5') || n.includes('qwen3-') && (n.includes('a10b') || n.includes('a22b') || n.includes('a3b'))) { opts.envVars.push('VLLM_USE_DEEP_GEMM=0', 'VLLM_USE_FLASHINFER_MOE_FP16=1', 'VLLM_USE_FLASHINFER_SAMPLER=0', 'OMP_NUM_THREADS=4'); - opts.flags.push('--enable-expert-parallel', '--reasoning-parser qwen3'); + opts.flags.push('--enable-expert-parallel'); opts.tips.push('MoE optimizations: expert parallel + flashinfer MoE kernels'); } // Qwen3 MoE (non-3.5) else if (n.includes('qwen3') && (n.includes('a10b') || n.includes('a22b') || n.includes('a3b'))) { opts.envVars.push('VLLM_USE_DEEP_GEMM=0', 'VLLM_USE_FLASHINFER_MOE_FP16=1'); - opts.flags.push('--enable-expert-parallel', '--reasoning-parser qwen3'); + opts.flags.push('--enable-expert-parallel'); opts.tips.push('MoE optimizations: expert parallel'); } - // DeepSeek MoE - else if (n.includes('deepseek') && (n.includes('v3') || n.includes('r1'))) { + // DeepSeek MoE — V3 / V3.1 / V4 (and future Vx), R1 / R2 reasoning. + // Anything v-{integer} or r-{integer} family from DeepSeek is MoE in + // current architectures. These models also require fp8 KV cache to + // fit at meaningful context with current tensor-parallel layouts — + // the launch crashes otherwise (--kv-cache-dtype auto → bf16 OOMs). + else if (n.includes('deepseek') && /\b(v[3-9]|v\d{2,}|r[1-9])\b/.test(n)) { opts.flags.push('--enable-expert-parallel'); opts.tips.push('MoE expert parallel for DeepSeek'); + opts.kvCacheDtype = 'fp8'; + opts.tips.push('fp8 KV cache required — bf16 OOMs at usable context'); + } + // Reasoning parser — applies independently of MoE detection. Without this + // flag, models like MiniMax-M2.x, DeepSeek-R1, Qwen3 reasoning, GLM-4.x, + // gpt-oss leak <think> blocks as plain text instead of separating them + // into the reasoning_content channel. + const _reasoningParser = _detectReasoningParser(modelName); + if (_reasoningParser) { + opts.flags.push(`--reasoning-parser ${_reasoningParser}`); + opts.tips.push(`Reasoning parser (${_reasoningParser}): splits <think> tokens into a separate channel`); } // Speculative decoding — pick the right MTP method per model family. // opts.spec.{method,tokens} seed the UI dropdown/input; the actual flag is @@ -257,7 +275,7 @@ function _detectModelOptimizations(modelName) { if (n.includes('qwen3-next') || (n.includes('qwen3.5') && (n.includes('a10b') || n.includes('a22b')))) { specDefault = { method: 'qwen3_next_mtp', tokens: 2 }; } else if ( - (n.includes('deepseek') && (n.includes('v3') || n.includes('v3.1') || n.includes('r1'))) || + (n.includes('deepseek') && /\b(v[3-9]|v\d{2,}|r[1-9])\b/.test(n)) || n.includes('kimi-k2') || n.includes('kimi_k2') || n.includes('glm-4.5') || n.includes('glm4.5') || n.includes('minimax-m1') || n.includes('minimax_m1') @@ -273,6 +291,36 @@ function _detectModelOptimizations(modelName) { return opts; } +/** Detect the right vLLM --reasoning-parser based on model name. + * Returns the parser slug (matches vLLM's official list) or null when the + * model isn't a reasoning model. Without the right parser, thinking tokens + * leak as plain text instead of being split into a separate channel. + * Source: vllm/reasoning/__init__.py registered parsers. + */ +export function _detectReasoningParser(modelName) { + const n = (modelName || '').toLowerCase(); + // MiniMax M2 / M2.5 / M2.7 — released with a dedicated parser. Catch M2 + // before plain "minimax" so M2.x doesn't fall through to a wrong parser. + if (n.includes('minimax') && n.match(/\bm2(?:\.\d)?\b/)) return 'minimax_m2'; + // DeepSeek-R1 / V3-Thinking / V3.1-Thinking variants. Bare V3/V3.1 (non- + // thinking) skip this — they're not reasoning models. + if (n.includes('deepseek') && (n.includes('r1') || n.includes('thinking'))) return 'deepseek_r1'; + // Qwen3 / Qwen3.5 reasoning models. Qwen3-Coder + Qwen3-Instruct don't + // emit <think> blocks, so skip the parser there. + if (n.includes('qwen3') && !n.includes('coder') && !n.includes('instruct')) return 'qwen3'; + // GLM-4 / GLM-4.5 / GLM-4.6 with reasoning. + if (n.includes('glm-4') || n.includes('glm-5')) return 'glm45'; + // OpenAI gpt-oss family. + if (n.includes('gpt-oss')) return 'gpt_oss'; + // Hunyuan A13B reasoning. + if (n.includes('hunyuan') && n.includes('a13b')) return 'hunyuan_a13b'; + // IBM Granite reasoning. + if (n.includes('granite') && (n.includes('reason') || n.includes('think'))) return 'granite'; + // InternLM reasoning. + if (n.includes('internlm')) return 'internlm'; + return null; +} + /** Detect the right vLLM tool-call-parser based on model name. * Qwen tool-call formats split by generation: * - Qwen3-Coder → qwen3_coder (XML <tool_call> with named params) @@ -416,7 +464,10 @@ export function _buildServeCmd(f, modelName, backend) { const _py3Bin = _venvBin ? `${_venvBin}python3` : 'python3'; let cmd = ''; if (backend === 'vllm') { - const gpuId = f.gpu_id?.trim() || ''; + // GPU list comes from the Row-1 button strip (data-field="gpus") — + // the bare "auto" input that used to back gpu_id is gone, and the + // button strip is the only source for which devices to pin. + const gpuId = (f.gpus || f.gpu_id || '').toString().trim(); if (gpuId) cmd += `CUDA_VISIBLE_DEVICES=${gpuId} `; if (f.moe_env) { const _opts = _detectModelOptimizations(modelName); @@ -458,7 +509,10 @@ export function _buildServeCmd(f, modelName, backend) { cmd += ` --speculative-config '{"method":"${_specMethod}","num_speculative_tokens":${_specToks}}'`; } } else if (backend === 'sglang') { - const gpuId = f.gpu_id?.trim() || ''; + // GPU list comes from the Row-1 button strip (data-field="gpus") — + // the bare "auto" input that used to back gpu_id is gone, and the + // button strip is the only source for which devices to pin. + const gpuId = (f.gpus || f.gpu_id || '').toString().trim(); if (gpuId) cmd += `CUDA_VISIBLE_DEVICES=${gpuId} `; const _extraEnv = (f.extra_env ?? '').toString().replace(/\s+/g, ' ').trim(); if (_extraEnv) cmd += _extraEnv + ' '; @@ -475,7 +529,9 @@ export function _buildServeCmd(f, modelName, backend) { if (f.enforce_eager) cmd += ' --disable-cuda-graph'; } else if (backend === 'llamacpp') { const ggufPath = f._gguf_path || 'model.gguf'; - const gpuId = f.gpu_id?.trim() || ''; + // GPU list — read from gpus (button strip); fall back to gpu_id for + // backward-compat with older saved presets that pre-date the removal. + const gpuId = (f.gpus || f.gpu_id || '').toString().trim(); const py = _isWindows() ? 'python' : 'python3'; // CPU-only serve (-ngl 0): drop the GPU-only flags, otherwise the command // mixes "zero GPU layers" with CUDA unified-memory + flash-attn and fails to @@ -597,7 +653,8 @@ export function _buildServeCmd(f, modelName, backend) { } else if (backend === 'diffusers') { const gpuStr = f.gpus?.trim(); if (gpuStr) cmd += `CUDA_VISIBLE_DEVICES=${gpuStr} `; - cmd += `python3 scripts/diffusion_server.py --model ${modelName} --port ${f.port || '8100'}`; + const diffusersPy = _isWindows() ? 'python' : _py3Bin; + cmd += `${diffusersPy} scripts/diffusion_server.py --model ${modelName} --port ${f.port || '8100'}`; if (f.diff_dtype && f.diff_dtype !== 'bfloat16') cmd += ` --dtype ${f.diff_dtype}`; if (f.diff_device_map && f.diff_device_map !== 'balanced') cmd += ` --device-map ${f.diff_device_map}`; if (f.diff_steps) cmd += ` --steps ${f.diff_steps}`; @@ -718,7 +775,7 @@ async function _fetchDependencies() { const data = await resp.json(); const pkgs = data.packages || []; if (!pkgs.length) { list.innerHTML = '<div class="hwfit-loading">No packages found</div>'; return; } - const _winUnsupported = new Set(['diffusers', 'hf_transfer', 'vllm', 'rembg', 'gfpgan']); + const _winUnsupported = new Set(['hf_transfer', 'vllm', 'rembg', 'gfpgan']); const _statusTag = (pkg, isLocal, isSystemDep, winBlocked) => { if (winBlocked) return `<span class="cookbook-dep-tag cookbook-dep-na">N/A</span>`; @@ -736,6 +793,22 @@ async function _fetchDependencies() { return `<button class="cookbook-dep-tag cookbook-dep-install" data-dep-pip="${esc(pkg.pip)}" data-dep-target="${isLocal ? 'local' : 'remote'}">Install</button>`; }; + // Per-package inline glyphs — same accent-coloured marks used in the + // Backend picker on the Run page, so the Dependencies row visually + // matches the engine you're configuring. Unknown packages get no + // icon (the name alone is fine for librosa, hf_transfer, etc.). + const _DEP_GLYPHS = { + vllm: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M3 4l7 16 7-16"/><path d="M14 4l4 9 3-9"/></svg>', + sglang: '<svg width="13" height="13" viewBox="0 0 24 24" fill="currentColor" stroke="none" aria-hidden="true"><polygon points="13 2 3 14 12 14 11 22 21 10 12 10 13 2"/></svg>', + llama_cpp: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="9"/><path d="M8 12h8M12 8v8"/></svg>', + ollama: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 10a6 6 0 0 1 12 0v4a4 4 0 0 1-8 0v-1"/><circle cx="10" cy="9" r="1"/><circle cx="14" cy="9" r="1"/></svg>', + diffusers: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="4"/><path d="M12 2v3M12 19v3M2 12h3M19 12h3M5 5l2 2M17 17l2 2M5 19l2-2M17 7l2-2"/></svg>', + }; + const _depGlyphHtml = (name) => { + const g = _DEP_GLYPHS[name]; + return g ? `<span class="cookbook-dep-glyph" aria-hidden="true" style="display:inline-flex;align-items:center;justify-content:center;width:14px;height:14px;color:var(--accent, var(--red));margin-right:5px;vertical-align:-2px;">${g}</span>` : ''; + }; + const _depRow = (pkg) => { const isLocal = pkg.target === 'local'; const isSystemDep = pkg.kind === 'system'; @@ -756,9 +829,16 @@ async function _fetchDependencies() { } else if (pkg.name === 'sglang' && pkg.installed) { _rebuildBtn = `<button type="button" class="cookbook-dep-tag cookbook-dep-rebuild cookbook-dep-reinstall" data-reinstall-pkg="sglang" title="Force-reinstall SGLang (pulls a matching torch). Runs as a tmux task in the Running tab.">Reinstall</button>`; } + // For backends with a recipe catalog (vllm / sglang / llama_cpp), + // append a caret button that toggles a per-row recipe panel below. + const hasRecipe = RECIPE_BACKENDS.has(pkg.name); + const recipeCaret = hasRecipe + ? `<button class="cookbook-dep-tag cookbook-dep-recipe-caret" data-dep-recipe-toggle="${esc(pkg.name)}" title="Pick a model to see the exact install commands" aria-expanded="false" style="background:none;border:1px solid var(--border);padding:2px 6px;display:inline-flex;align-items:center;cursor:pointer;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="transition:transform 0.15s"><polyline points="6 9 12 15 18 9"/></svg></button>` + : ''; + const recipePanel = hasRecipe ? _recipePanelHtml(pkg.name) : ''; return `<div class="cookbook-dep-row${winBlocked ? ' cookbook-dep-blocked' : ''}" data-pkg-name="${esc(pkg.name)}" data-dep-pip="${esc(pkg.pip || '')}" data-dep-target="${isLocal ? 'local' : 'remote'}" data-dep-kind="${esc(pkg.kind || 'python')}">` + `<div class="cookbook-dep-info">` - + `<div class="memory-item-title">${esc(pkg.name)}</div>` + + `<div class="memory-item-title">${_depGlyphHtml(pkg.name)}${esc(pkg.name)}</div>` + `<div class="memory-item-meta" style="font-size:10px;opacity:0.5;margin-top:2px;">${esc(pkg.desc)}</div>` + note + updateNote @@ -766,9 +846,65 @@ async function _fetchDependencies() { + _rebuildBtn + `<span class="cookbook-dep-tag cookbook-dep-cat">${esc(pkg.category)}</span>` + _statusTag(pkg, isLocal, isSystemDep, winBlocked) - + `</div>`; + + recipeCaret + + `</div>` + + recipePanel; }; + // Prepend the configured venv's activate line (pip variant only) so + // the user sees a paste-ready sequence; Run keeps using env_prefix to + // activate the same venv before the pip command. Docker variant skips + // the activate line — `docker pull` doesn't need a venv. + function _recipeDisplayText(commands, variant) { + if (variant === 'docker') return commands.join('\n'); + const envPath = (_envState.envPath || '').replace(/\/+$/, ''); + const activate = envPath + ? `source ${envPath}${envPath.endsWith('/bin/activate') ? '' : '/bin/activate'}` + : '# (activate your venv first)'; + return [activate, ...commands].join('\n'); + } + + // Per-backend recipe panel (model picker + commands + Copy/Run). + // Lives directly below the row it expands and starts collapsed. + // The model picker lists every downloaded model from _cachedModelIds + // (the same set the Launch tab uses); pickRecipe() then finds the + // best-matching recipe for whatever the user selects, with the + // backend's generic entry as the fallback. + function _recipePanelHtml(backend) { + const candidates = recipesForBackend(backend); + if (!candidates.length) return ''; + const downloadedIds = _cachedModelIds ? Array.from(_cachedModelIds).sort() : []; + const modelOptions = downloadedIds.length + ? downloadedIds.map(id => `<option value="${esc(id)}">${esc(id)}</option>`).join('') + : ''; + // "Other" entry: user types/pastes an id, OR uses the generic fallback + // when no models have been downloaded yet. + const otherOpt = `<option value="">Other (generic ${esc(backend)} install)</option>`; + const opts = modelOptions + otherOpt; + // Initial recipe: the generic fallback (matches first time, no model id). + const initial = pickRecipe(backend, '') || candidates[0]; + const initialVariant = RECIPE_DEFAULT_VARIANT; + const initialCmds = recipeCommands(initial, initialVariant); + const rightActive = initialVariant === 'docker' ? ' mode-right' : ''; + return `<div class="cookbook-dep-recipe-panel" data-dep-recipe-panel="${esc(backend)}" data-dep-recipe-active-variant="${esc(initialVariant)}" style="display:none;margin:-4px 0 8px;padding:8px 12px 10px;background:rgba(0,0,0,0.04);border:1px solid var(--border);border-top:none;border-radius:0 0 6px 6px;"> + <div style="display:flex;align-items:center;gap:8px;margin-bottom:6px;"> + <span style="font-size:11px;opacity:0.75;flex-shrink:0;">Serving which model?</span> + <select class="settings-select cookbook-dep-recipe-pick" data-dep-recipe-pick="${esc(backend)}" style="flex:1;font-size:11px;padding:3px 6px;">${opts}</select> + <div class="mode-toggle${rightActive}" data-dep-recipe-variants="${esc(backend)}" style="flex-shrink:0;"> + <button type="button" class="mode-toggle-btn${initialVariant === 'pip' ? ' active' : ''}" data-dep-recipe-variant="${esc(backend)}" data-variant="pip" aria-pressed="${initialVariant === 'pip'}">Pip/uv</button> + <button type="button" class="mode-toggle-btn${initialVariant === 'docker' ? ' active' : ''}" data-dep-recipe-variant="${esc(backend)}" data-variant="docker" aria-pressed="${initialVariant === 'docker'}">Docker</button> + </div> + </div> + <div style="position:relative;"> + <pre class="cookbook-dep-recipe-cmds" data-dep-recipe-cmds="${esc(backend)}" data-dep-recipe-install="${esc(initialCmds.join('\n'))}" style="margin:0;padding:8px 36px 8px 10px;background:rgba(0,0,0,0.08);border-radius:4px;font-size:11px;line-height:1.5;overflow-x:auto;white-space:pre;">${esc(_recipeDisplayText(initialCmds, initialVariant))}</pre> + <button type="button" id="recipe-copy-${esc(backend)}" class="cookbook-dep-recipe-copy" data-dep-recipe-copy="${esc(backend)}" title="Copy" aria-label="Copy" style="position:absolute;top:6px;right:6px;padding:3px 5px;background:none;border:none;color:inherit;opacity:0.7;cursor:pointer;display:inline-flex;align-items:center;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg></button> + </div> + <div style="display:flex;gap:6px;justify-content:flex-end;margin-top:6px;"> + <button type="button" class="cookbook-dep-tag cookbook-dep-install cookbook-dep-recipe-run" data-dep-recipe-run="${esc(backend)}" style="display:inline-flex;align-items:center;gap:4px;cursor:pointer;"><svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor"><polygon points="5 3 19 12 5 21 5 3"/></svg>Run</button> + </div> + </div>`; + } + const _section = (title, note, items) => items.length ? `<div class="cookbook-dep-section"><span class="cookbook-dep-section-title">${title}</span><span class="cookbook-dep-section-note">${note}</span></div>` + items.map(_depRow).join('') @@ -865,7 +1001,7 @@ async function _fetchDependencies() { } // Wire install buttons (not-installed packages) - list.querySelectorAll('.cookbook-dep-install').forEach(btn => { + list.querySelectorAll('.cookbook-dep-install:not(.cookbook-dep-recipe-run)').forEach(btn => { btn.addEventListener('click', async (e) => { e.stopPropagation(); const pipName = btn.dataset.depPip; @@ -874,6 +1010,135 @@ async function _fetchDependencies() { }); }); + // ── Recipe panel wiring (per-backend dropdown with model + commands) ── + // Caret toggle: shows/hides the panel directly below the backend row. + list.querySelectorAll('[data-dep-recipe-toggle]').forEach(btn => { + btn.addEventListener('click', (e) => { + e.stopPropagation(); + const backend = btn.dataset.depRecipeToggle; + const panel = list.querySelector(`[data-dep-recipe-panel="${CSS.escape(backend)}"]`); + if (!panel) return; + const open = panel.style.display === 'none' || !panel.style.display; + panel.style.display = open ? 'block' : 'none'; + btn.setAttribute('aria-expanded', open ? 'true' : 'false'); + const caret = btn.querySelector('svg'); + if (caret) caret.style.transform = open ? 'rotate(180deg)' : ''; + }); + }); + // Re-render the <pre> for a backend using the currently-active variant + // (pip / docker) and the currently-picked model. Used by every input + // that changes which install sequence we should show. + function _refreshRecipePre(backend) { + const panel = list.querySelector(`[data-dep-recipe-panel="${CSS.escape(backend)}"]`); + if (!panel) return; + const variant = panel.dataset.depRecipeActiveVariant || RECIPE_DEFAULT_VARIANT; + const sel = panel.querySelector('[data-dep-recipe-pick]'); + const recipe = pickRecipe(backend, (sel && sel.value) || ''); + const cmds = recipeCommands(recipe, variant); + const pre = panel.querySelector('[data-dep-recipe-cmds]'); + if (pre) { + pre.textContent = _recipeDisplayText(cmds, variant); + pre.dataset.depRecipeInstall = cmds.join('\n'); + } + } + // Model select: pickRecipe matches the model id against the catalog. + list.querySelectorAll('[data-dep-recipe-pick]').forEach(sel => { + sel.addEventListener('change', () => _refreshRecipePre(sel.dataset.depRecipePick)); + }); + // Variant toggle (Pip/uv vs Docker): mirrors the agent/chat mode-toggle + // pattern — buttons get .active, container gets .mode-right when the + // right slot is selected so the sliding pill animates over. + list.querySelectorAll('[data-dep-recipe-variant]').forEach(btn => { + btn.addEventListener('click', (e) => { + e.stopPropagation(); + const backend = btn.dataset.depRecipeVariant; + const variant = btn.dataset.variant; + const panel = list.querySelector(`[data-dep-recipe-panel="${CSS.escape(backend)}"]`); + if (!panel) return; + panel.dataset.depRecipeActiveVariant = variant; + const container = panel.querySelector('.mode-toggle[data-dep-recipe-variants]'); + if (container) container.classList.toggle('mode-right', variant === 'docker'); + panel.querySelectorAll('[data-dep-recipe-variant]').forEach(b => { + const on = b.dataset.variant === variant; + b.classList.toggle('active', on); + b.setAttribute('aria-pressed', on ? 'true' : 'false'); + }); + _refreshRecipePre(backend); + }); + }); + // Copy: drop the visible command block on the clipboard. + list.querySelectorAll('[data-dep-recipe-copy]').forEach(btn => { + btn.addEventListener('click', async (e) => { + e.stopPropagation(); + const backend = btn.dataset.depRecipeCopy; + const pre = list.querySelector(`[data-dep-recipe-cmds="${CSS.escape(backend)}"]`); + if (!pre) return; + try { + await navigator.clipboard.writeText(pre.textContent); + uiModule.showToast('Copied'); + } catch { + // Fallback for non-secure contexts: select the pre's text so + // the user can Ctrl+C themselves. + const sel = window.getSelection(); const range = document.createRange(); + range.selectNodeContents(pre); sel.removeAllRanges(); sel.addRange(range); + } + }); + }); + // Run: launch the install command(s) as a tmux task on the currently- + // selected deps server. Activation comes from env_prefix (same plumbing + // the Install button uses) so the install lands in the configured venv + // instead of a fresh .venv in some random CWD. + list.querySelectorAll('[data-dep-recipe-run]').forEach(btn => { + btn.addEventListener('click', async (e) => { + e.stopPropagation(); + const backend = btn.dataset.depRecipeRun; + const pre = list.querySelector(`[data-dep-recipe-cmds="${CSS.escape(backend)}"]`); + if (!pre) return; + // Use the install-only command list (no activate line) — the + // displayed source line is for the user's reading; env_prefix + // handles it for the actual run. + const installRaw = pre.dataset.depRecipeInstall || pre.textContent; + const cmd = installRaw.split('\n').map(s => s.trim()).filter(Boolean).join(' && '); + const depsSel = document.getElementById('hwfit-deps-server'); + if (depsSel) _applyServerSelection(depsSel.value); + const targetHost = _envState.remoteHost || 'local'; + // Build env_prefix from the configured envPath (matches _installDep). + let envPrefix = ''; + if (_envState.env === 'venv' && _envState.envPath) { + const p = _envState.envPath; + envPrefix = 'source ' + _shellQuote(p.endsWith('/bin/activate') ? p : p + '/bin/activate'); + } else if (_envState.env === 'conda' && _envState.envPath) { + envPrefix = 'eval "$(conda shell.bash hook)" && conda activate ' + _shellQuote(_envState.envPath); + } + const reqBody = { + repo_id: `${backend} setup`, + cmd: cmd, + remote_host: _envState.remoteHost || undefined, + ssh_port: _getPort(_envState.remoteHost) || undefined, + env_prefix: envPrefix || undefined, + platform: _envState.platform || undefined, + }; + try { + const res = await fetch('/api/model/serve', { + method: 'POST', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(reqBody), + }); + const data = await res.json().catch(() => ({})); + if (!res.ok || !data.ok) { + uiModule.showToast('Run failed: ' + String(data.detail || data.error || `HTTP ${res.status}`).slice(0, 200)); + return; + } + const payload = { repo_id: `${backend} setup`, _cmd: cmd, remote_host: _envState.remoteHost || '', _dep: true }; + _addTask(data.session_id, `${backend} setup`, 'download', payload); + uiModule.showToast(`Running ${backend} setup on ${targetHost}…`); + } catch (err) { + uiModule.showToast('Run failed: ' + err.message); + } + }); + }); + + // Wire the ⋮ menu on installed packages — currently just "Update". function _showDepMenu(anchor) { document.querySelectorAll('.cookbook-dep-menu').forEach(d => d.remove()); @@ -1403,16 +1668,49 @@ function _wireTabEvents(body) { const dlFoldBody = document.getElementById('cookbook-dl-tab-fold-body'); const dlFoldChevron = document.getElementById('cookbook-dl-tab-chevron'); if (dlFold && dlFoldBody && dlFoldChevron) { + const _setFolded = (folded, persist = true) => { + // Toggle via class so CSS transition animates the height/opacity + // — display:none was an instant on/off and felt jarring. + dlFoldBody.classList.toggle('is-folded', folded); + dlFoldChevron.textContent = folded ? '▸' : '▾'; + dlFold.classList.toggle('is-folded', folded); + if (persist) { + try { localStorage.setItem('cookbook_dl_tab_folded_v1', folded ? '1' : '0'); } catch {} + } + }; dlFold.addEventListener('click', () => { - const folded = dlFoldBody.style.display === 'none'; - dlFoldBody.style.display = folded ? '' : 'none'; - dlFoldChevron.textContent = folded ? '▾' : '▸'; - // Toggle is-folded class on the h2 so the line under it only shows when - // the section is collapsed (the body's content normally provides - // separation; with no body visible, the line gives the h2 definition). - dlFold.classList.toggle('is-folded', !folded); - try { localStorage.setItem('cookbook_dl_tab_folded_v1', folded ? '0' : '1'); } catch {} + const folded = dlFoldBody.classList.contains('is-folded'); + _setFolded(!folded); }); + // Auto-fold on any downward scroll inside the cookbook modal, + // and auto-expand when the user scrolls all the way back to the + // top of whichever scroller they're in. The chevron ▸ still + // toggles manually. + const _maybeFold = () => { + if (dlFoldBody.classList.contains('is-folded')) return; + _setFolded(true, /* persist */ false); + }; + const _maybeExpand = () => { + if (!dlFoldBody.classList.contains('is-folded')) return; + _setFolded(false, /* persist */ false); + }; + // Capture phase so scrolls on nested scrollers (.hwfit-list, + // .cookbook-body, .modal-content) all hit us. + const _modal = dlFold.closest('#cookbook-modal') || document; + const _lastY = new WeakMap(); + _modal.addEventListener('scroll', (e) => { + const tgt = e.target; + if (!tgt || typeof tgt.scrollTop !== 'number') return; + // Ignore scrolls that originate INSIDE the Direct Download body + // (e.g. the Trending models list) — those are local to the + // section and shouldn't auto-fold the section that owns them. + if (dlFoldBody.contains && (tgt === dlFoldBody || dlFoldBody.contains(tgt))) return; + const y = tgt.scrollTop; + const prev = _lastY.get(tgt) || 0; + if (y > prev) _maybeFold(); + else if (y <= 0) _maybeExpand(); + _lastY.set(tgt, y); + }, true); } const hfToggle = document.getElementById('cookbook-hf-latest-toggle'); const hfArrow = document.getElementById('cookbook-hf-latest-arrow'); @@ -1570,9 +1868,9 @@ function _wireTabEvents(body) { document.getElementById('hwfit-server-select')?.addEventListener('change', _onServerChange); } - // Browse Ollama library — popular models from ollama.com via cached backend - // proxy. Click a row → fills the download input with `<name>:<size>` so the - // existing Download button kicks off `ollama pull`. + // Browse Ollama library popup removed — Engine = Ollama in the + // Scan / Download filter covers this use case. The handler below is a + // no-op now because the elements no longer exist. const olToggle = document.getElementById('cookbook-ollama-toggle'); const olArrow = document.getElementById('cookbook-ollama-arrow'); const olList = document.getElementById('cookbook-ollama-list'); @@ -1773,8 +2071,8 @@ function _renderRecipes() { // Tabs html += '<div class="cookbook-tabs">'; + html += '<button class="cookbook-tab" data-backend="Serve"><svg width="12" height="12" viewBox="0 0 24 24" fill="currentColor" stroke="none" style="vertical-align:-1px;margin-right:3px;"><polygon points="13 2 3 14 12 14 11 22 21 10 12 10 13 2"/></svg>Launch</button>'; html += '<button class="cookbook-tab active" data-backend="Search"><svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" style="vertical-align:-1px;margin-right:3px;"><polyline points="7 14 12 19 17 14"/><line x1="12" y1="19" x2="12" y2="5"/><line x1="5" y1="21" x2="19" y2="21"/></svg>Download</button>'; - html += '<button class="cookbook-tab" data-backend="Serve"><svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" style="vertical-align:-1px;margin-right:3px;"><rect x="2" y="2" width="20" height="8" rx="2"/><rect x="2" y="14" width="20" height="8" rx="2"/><circle cx="6" cy="6" r="1"/><circle cx="6" cy="18" r="1"/></svg>Serve</button>'; html += '<button class="cookbook-tab" data-backend="Dependencies"><svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" style="vertical-align:-1px;margin-right:3px;"><path d="M21 16V8a2 2 0 0 0-1-1.73l-7-4a2 2 0 0 0-2 0l-7 4A2 2 0 0 0 3 8v8a2 2 0 0 0 1 1.73l7 4a2 2 0 0 0 2 0l7-4A2 2 0 0 0 21 16z"/><polyline points="3.27 6.96 12 12.01 20.73 6.96"/><line x1="12" y1="22.08" x2="12" y2="12"/></svg>Dependencies</button>'; html += '<button class="cookbook-tab" data-backend="Settings"><svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" style="vertical-align:-1px;margin-right:3px;"><circle cx="12" cy="12" r="3"/><path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 1 1-2.83 2.83l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-4 0v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 1 1-2.83-2.83l.06-.06A1.65 1.65 0 0 0 4.68 15a1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1 0-4h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 1 1 2.83-2.83l.06.06A1.65 1.65 0 0 0 9 4.68a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 4 0v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 1 1 2.83 2.83l-.06.06A1.65 1.65 0 0 0 19.4 9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 0 4h-.09a1.65 1.65 0 0 0-1.51 1z"/></svg>Settings</button>'; html += '</div>'; @@ -1787,9 +2085,9 @@ function _renderRecipes() { // State persisted to localStorage so the fold survives reloads. const _dlTabFolded = (() => { try { return localStorage.getItem('cookbook_dl_tab_folded_v1') === '1'; } catch { return false; } })(); html += '<div style="display:flex;align-items:center;gap:8px;margin-bottom:2px;">'; - html += `<h2 id="cookbook-dl-tab-fold" class="${_dlTabFolded ? 'is-folded' : ''}" style="margin:0;padding:0;line-height:1;cursor:pointer;display:flex;align-items:center;justify-content:space-between;user-select:none;flex:1;">Download<span id="cookbook-dl-tab-chevron" style="display:inline-block;transition:transform 0.15s;font-size:1.1em;margin-left:8px;opacity:0.85;">${_dlTabFolded ? '▸' : '▾'}</span></h2>`; + html += `<h2 id="cookbook-dl-tab-fold" class="${_dlTabFolded ? 'is-folded' : ''}" style="margin:0;padding:0;line-height:1;cursor:pointer;display:flex;align-items:center;justify-content:space-between;user-select:none;flex:1;">Direct Download<span id="cookbook-dl-tab-chevron" style="display:inline-block;transition:transform 0.15s;font-size:1.1em;margin-left:8px;opacity:0.85;">${_dlTabFolded ? '▸' : '▾'}</span></h2>`; html += '</div>'; - html += `<div id="cookbook-dl-tab-fold-body" style="${_dlTabFolded ? 'display:none;' : ''}">`; + html += `<div id="cookbook-dl-tab-fold-body" class="${_dlTabFolded ? 'is-folded' : ''}">`; html += '<p class="memory-desc doclib-desc" style="margin-top:6px;">Download from <a href="https://huggingface.co/models" target="_blank" rel="noopener" style="color:var(--accent,var(--red));text-decoration:none;"><svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-1px;margin-right:1px;"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"/><polyline points="15 3 21 3 21 9"/><line x1="10" y1="14" x2="21" y2="3"/></svg>HuggingFace</a> by pasting model link, or download directly in the Scan section below.</p>'; html += '<div class="hwfit-container" id="hwfit-container">'; @@ -1819,42 +2117,34 @@ function _renderRecipes() { // silently sending downloads to the wrong server. An empty selection means Local; the user // chooses a remote server explicitly via the dropdown. - // Manual download input - html += `<div style="margin-top:7px;margin-bottom:2px;display:flex;gap:4px;align-items:center;">`; + // Manual download input — server picker on the same row as the repo input, + // on the left. The standalone "add server" button is gone (use Settings). + html += `<div class="cookbook-dl-input" style="margin-top:7px;display:flex;gap:4px;align-items:center;">`; if (_es.servers.length > 1) { - html += `<select class="cookbook-field-input hwfit-dl-server" id="hwfit-dl-server" style="height:28px;position:relative;top:0px;">`; + html += `<select class="cookbook-field-input hwfit-dl-server" id="hwfit-dl-server" style="height:28px;flex-shrink:0;">`; html += _buildServerOpts(true); html += `</select>`; } else { html += `<input type="hidden" id="hwfit-dl-server" value="local" />`; } - html += `<button class="memory-toolbar-btn cookbook-dl-add-server" title="Add server in Settings" style="height:28px;">add server</button>`; - html += `</div>`; - html += `<div class="cookbook-dl-input" style="margin-top:0;">`; - html += `<input type="text" class="cookbook-dl-repo" id="cookbook-dl-repo" placeholder="org/model-name, qwen2.5:14b, or HF URL" />`; + html += `<input type="text" class="cookbook-dl-repo" id="cookbook-dl-repo" placeholder="org/model-name, qwen2.5:14b, or HF URL" style="flex:1;min-width:0;" />`; html += `<button class="cookbook-btn cookbook-dl-btn" id="cookbook-dl-btn">Download</button>`; html += `</div>`; - // Browse Ollama library — fetches popular models from ollama.com via the - // /api/cookbook/ollama/library cached proxy, click → fills the input with - // `<name>:<size>` so the existing Download button kicks off `ollama pull`. - html += `<div style="margin-top:5px;position:relative;top:-3px;">`; - html += `<div style="display:flex;gap:4px;align-items:center;">`; - html += `<button type="button" class="memory-toolbar-btn" id="cookbook-ollama-toggle" style="flex:1;text-align:left;height:26px;display:flex;align-items:center;gap:6px;border-radius:4px;">`; - html += `<span id="cookbook-ollama-arrow" style="display:inline-block;transition:transform 0.15s;pointer-events:none;">▸</span>`; - html += `<span style="pointer-events:none;">Browse Ollama library</span>`; - html += `</button>`; - html += `<button type="button" class="memory-toolbar-btn" id="cookbook-ollama-refresh" title="Refresh" style="height:26px;width:26px;padding:0;border-radius:4px;">↻</button>`; - html += `</div>`; - html += `<div id="cookbook-ollama-list" style="display:none;margin-top:4px;max-height:320px;overflow-y:auto;flex-direction:column;gap:4px;"></div>`; - html += `</div>`; + // Ollama-library browse used to live here as its own collapsible dropdown, + // but that duplicated the Engine filter (which already has Ollama). The + // standalone UI is gone — to find Ollama models, set Engine = Ollama in + // the Scan / Download section below. // Latest HF models that fit — collapsible card list - html += `<div style="margin-top:5px;position:relative;top:-3px;">`; + html += `<div style="margin-top:5px;position:relative;top:-11px;">`; html += `<div style="display:flex;gap:4px;align-items:center;">`; - html += `<button type="button" class="memory-toolbar-btn" id="cookbook-hf-latest-toggle" style="flex:1;text-align:left;height:26px;display:flex;align-items:center;gap:6px;border-radius:4px;">`; - html += `<span id="cookbook-hf-latest-arrow" style="display:inline-block;transition:transform 0.15s;pointer-events:none;">\u25B8</span>`; - html += `<span style="pointer-events:none;">Trending models that fit your hardware</span>`; + html += `<button type="button" class="memory-toolbar-btn" id="cookbook-hf-latest-toggle" style="flex:1;text-align:left;height:28px;font-size:11px;display:flex;align-items:center;gap:6px;border-radius:5px;">`; + // Trending-up icon (accent) so the section reads as "what's hot". + html += `<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="var(--accent, var(--red))" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="flex-shrink:0;pointer-events:none;"><polyline points="23 6 13.5 15.5 8.5 10.5 1 18"/><polyline points="17 6 23 6 23 12"/></svg>`; + html += `<span style="pointer-events:none;flex:1;">Trending models that fit your hardware</span>`; + // Chevron moved to the RIGHT \u2014 collapsed = pointing right, expanded + // = rotated 90deg into a down chevron (handled by existing toggle CSS). + html += `<span id="cookbook-hf-latest-arrow" style="display:inline-block;transition:transform 0.15s;pointer-events:none;opacity:0.6;font-size:11px;">\u25B8</span>`; html += `</button>`; - html += `<button type="button" class="memory-toolbar-btn" id="cookbook-hf-latest-refresh" title="Refresh" style="height:26px;width:26px;padding:0;border-radius:4px;">\u21BB</button>`; html += `</div>`; html += `<div id="cookbook-hf-latest-list" style="display:none;margin-top:4px;max-height:320px;overflow-y:auto;flex-direction:column;gap:4px;"></div>`; html += `</div>`; @@ -1875,9 +2165,10 @@ function _renderRecipes() { // Image tab removed — text→image gen is gone from this build (only inpaint // remains, which uses its own settings panel). Vision (multimodal) stays. html += '<option value="multimodal">Vision</option></select>'; - // Engine sits next to the type filter so the "what category / which serving - // path" filters live together; Quant + Context are storage-format and budget - // levers, grouped to the right. + // Search moved next to the Type filter so the two primary picks + // (what category + free text) sit together; the more advanced + // levers (Engine / Quant / Context) live to the right. + html += '<input type="text" class="cookbook-field-input hwfit-search" id="hwfit-search" placeholder="Search models..." style="flex:1;" />'; html += '<span class="hwfit-engine-wrap">'; html += '<select class="cookbook-field-input hwfit-engine" id="hwfit-engine" style="height:28px;" title="Filter by serving engine">'; html += '<option value="">Engine</option>'; @@ -1892,7 +2183,7 @@ function _renderRecipes() { // quant for every model instead of silently filtering to Q4. html += '<span class="hwfit-quant-wrap">'; html += '<select class="cookbook-field-input hwfit-quant" id="hwfit-quant" style="height:28px;">'; - html += '<option value="" selected>Quant: All</option>'; + html += '<option value="" selected>Quant</option>'; html += '<option value="Q4_K_M">Q4</option><option value="Q8_0">Q8</option>'; html += '<option value="Q6_K">Q6</option><option value="Q5_K_M">Q5</option>'; html += '<option value="Q3_K_M">Q3</option><option value="Q2_K">Q2</option>'; @@ -1905,21 +2196,19 @@ function _renderRecipes() { html += '<label class="hwfit-ctx-control" title="Context length for fit estimates. Lower it to find more models that could fit your hardware.">'; html += '<span>Context</span><span class="hwfit-help-chip hwfit-help-chip-inline" title="Context length. Lower it to find more models that could fit your hardware; raise it when you need longer chats or documents.">?</span><input type="range" id="hwfit-context" min="0" max="5" step="1" value="3" />'; html += '<output id="hwfit-context-label">50k</output></label>'; - // Search lives at the far right of the toolbar so the controls (Type/Quant/ - // Engine/Context) read as a row of compact filters followed by free-text. - html += '<input type="text" class="cookbook-field-input hwfit-search" id="hwfit-search" placeholder="Search models..." style="flex:1;" />'; html += '</div>'; html += '<div class="hwfit-toolbar" style="margin-top:7px;">'; html += '<select class="cookbook-field-input hwfit-server-select" id="hwfit-server-select" style="height:28px;min-width:88px;position:relative;top:0px;">'; html += _buildServerOpts(false); html += '</select>'; html += '<div class="hwfit-gpu-toggles" id="hwfit-gpu-toggles"></div>'; - // Scan/refresh button (icon-only) where the quant dropdown used to sit. - html += '<button type="button" class="hwfit-gpu-btn" id="hwfit-rescan" title="Re-scan hardware" style="flex-shrink:0;position:relative;top:-3px;left:-1px;">↻ RESCAN</button>'; + // (Rescan button removed — Edit handles manual hardware updates; + // automatic re-probe runs on container restart.) html += '<button type="button" class="hwfit-gpu-btn hwfit-hw-manual-btn" id="hwfit-hw-manual-btn" title="Set hardware manually" style="flex-shrink:0;position:relative;top:-3px;left:-1px;display:inline-flex;align-items:center;gap:3px;"><svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" style="flex-shrink:0;"><path d="M12 20h9"/><path d="M16.5 3.5a2.121 2.121 0 0 1 3 3L7 19l-4 1 1-4Z"/></svg>EDIT</button>'; // Sort state — the clickable column headers read/write this (pewds' original // sort paradigm). Newest is reachable by clicking the Model column header. html += '<select class="cookbook-field-input hwfit-sort" id="hwfit-sort" style="display:none">'; + html += '<option value="newest" selected>Latest</option>'; html += '<option value="fit">Fit</option><option value="score">Score</option><option value="vram">VRAM</option>'; html += '<option value="speed">Speed</option><option value="params">Params</option>'; html += '<option value="context">Context</option></select>'; @@ -2173,10 +2462,13 @@ export async function open(opts) { // returned before hydration — and since close/reopen doesn't reset the page, // only a full reload recovered it. Re-rendering is cheap and the in-progress // Running tab is rendered separately just below. - _renderRecipes(); + // Guard the render passes: a single broken task card must not throw out of + // open() and leave the modal stuck hidden (it has no catch, so the panel + // would silently never appear). Show the window regardless; log and move on. + try { _renderRecipes(); } catch (e) { console.error('[cookbook] renderRecipes failed', e); } _rendered = true; _clearCookbookNotif(); - _renderRunningTab(); + try { _renderRunningTab(); } catch (e) { console.error('[cookbook] renderRunningTab failed', e); } // Self-heal: revive any download tasks whose tmux session is still alive // but were persisted as done/error (covers the "restarted server while a // big multi-shard download was in flight" case — the task survived in diff --git a/static/js/cookbookRunning.js b/static/js/cookbookRunning.js index b13856c08..28365d49e 100644 --- a/static/js/cookbookRunning.js +++ b/static/js/cookbookRunning.js @@ -793,9 +793,10 @@ function _winSessionCmd(task, tmuxArgs) { return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`; } if (tmuxArgs.includes('kill-session')) { + const stopTree = `function Stop-Tree([int]$Id) { Get-CimInstance Win32_Process -Filter "ParentProcessId = $Id" -ErrorAction SilentlyContinue | ForEach-Object { Stop-Tree ([int]$_.ProcessId) }; Stop-Process -Id $Id -Force -ErrorAction SilentlyContinue }`; const ps = host - ? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue` - : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`; + ? `${stopTree}; $p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue` + : `${stopTree}; $p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`; return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`; } if (tmuxArgs.includes('send-keys') && tmuxArgs.includes('C-c')) { @@ -807,7 +808,7 @@ function _winSessionCmd(task, tmuxArgs) { return host ? `ssh ${pf}${host} 'tmux ${tmuxArgs}' 2>/dev/null` : `tmux ${tmuxArgs} 2>/dev/null`; } -function _tmuxGracefulKill(task) { +export function _tmuxGracefulKill(task) { if (_isWindows(task)) { const host = task.remoteHost; const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux'; @@ -824,6 +825,48 @@ function _tmuxGracefulKill(task) { return `tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null`; } +// Force-kill escalation: SIGKILL the tmux pane's owning PID and any children, +// then nuke the session. Use AFTER the graceful kill when the process is +// still detected — vLLM sometimes ignores SIGINT during model init, and a +// stuck CUDA context can survive `tmux kill-session` alone. +export function _tmuxForceKill(task) { + if (_isWindows(task)) { + // Windows graceful path already does Stop-Process -Force, so the same + // command serves as the "force" variant. + return _tmuxGracefulKill(task); + } + const sid = task.sessionId; + const inner = + `PIDS=$(tmux list-panes -t ${sid} -F "#{pane_pid}" 2>/dev/null); ` + + `if [ -n "$PIDS" ]; then ` + + ` for P in $PIDS; do ` + + ` pkill -KILL -P "$P" 2>/dev/null; ` + + ` kill -9 "$P" 2>/dev/null; ` + + ` done; ` + + `fi; ` + + `tmux kill-session -t ${sid} 2>/dev/null`; + if (task.remoteHost) { + return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} ${_shQuote(inner)}`; + } + return inner; +} + +// Returns a shell snippet that prints "ALIVE" if the tmux session still +// exists (or its main PID is still listed in /proc), "DEAD" otherwise. +// Used by the Stop-all escalation to decide whether to force-kill. +export function _tmuxIsAliveCheck(task) { + if (_isWindows(task)) { + // Skip the check on Windows — the graceful path already force-kills. + return null; + } + const sid = task.sessionId; + const inner = `if tmux has-session -t ${sid} 2>/dev/null; then echo ALIVE; else echo DEAD; fi`; + if (task.remoteHost) { + return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} ${_shQuote(inner)}`; + } + return inner; +} + function _shQuote(value) { return "'" + String(value ?? '').replace(/'/g, "'\\''") + "'"; } @@ -1642,7 +1685,7 @@ export function _renderRunningTab() { runTab.className = 'cookbook-tab'; runTab.dataset.backend = 'Running'; const _errCount = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length; - runTab.innerHTML = `Running${activeCountHtml}${_errCount ? `<span class="cookbook-tab-error-dot"></span>` : ''}`; + runTab.innerHTML = `Active${activeCountHtml}${_errCount ? `<span class="cookbook-tab-error-dot"></span>` : ''}`; tabBar.insertBefore(runTab, tabBar.firstChild); runTab.addEventListener('click', () => { tabBar.querySelectorAll('.cookbook-tab').forEach(t => t.classList.remove('active')); @@ -1653,7 +1696,7 @@ export function _renderRunningTab() { }); } else if (runTab) { const _errCount2 = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length; - runTab.innerHTML = tasks.length ? `Running${activeCountHtml}${_errCount2 ? '<span class="cookbook-tab-error-dot"></span>' : ''}` : 'Running'; + runTab.innerHTML = tasks.length ? `Active${activeCountHtml}${_errCount2 ? '<span class="cookbook-tab-error-dot"></span>' : ''}` : 'Active'; if (!hasContent) { if (runTab.classList.contains('active')) { const wfTab = tabBar.querySelector('.cookbook-tab[data-backend="Search"]'); @@ -1668,9 +1711,13 @@ export function _renderRunningTab() { group = document.createElement('div'); group.className = 'cookbook-group hidden'; group.dataset.backendGroup = 'Running'; - group.innerHTML = '<div class="admin-card" style="flex:1;display:flex;flex-direction:column;overflow:hidden;">' + + // No `flex:1` on the card — with overflow:visible (forced via #cookbook-modal + // .cookbook-group > .admin-card), flex:1 collapsed the card to body height + // and the body's scrollHeight stopped tracking the overflowing children. + // Sized-to-content means cookbook-body's overflow-y:auto kicks in naturally. + group.innerHTML = '<div class="admin-card" style="display:flex;flex-direction:column;">' + '<div style="display:flex;align-items:baseline;gap:8px;margin-bottom:2px;">' + - '<h2 style="margin:0;padding:0;line-height:1;">Running <span id="running-count" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal">' + activeCount + '</span></h2>' + + '<h2 style="margin:0;padding:0;line-height:1;">Active <span id="running-count" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal">' + activeCount + '</span></h2>' + '</div>' + '<p class="memory-desc doclib-desc" style="margin-top:6px;">Active downloads and serving processes.</p>' + '</div>'; @@ -1750,7 +1797,7 @@ export function _renderRunningTab() { // green when reachable, red if any serve task on it is crashed/unreachable. const _secDot = (key && allTasks.some(_serveTaskFailed)) ? 'fail' : 'ok'; const _dotTitle = key ? (_secDot === 'fail' ? 'Server not responding' : 'Reachable') : 'Local (this machine)'; - sec.insertAdjacentHTML('afterbegin', `<div class="cookbook-section-header" data-collapse="${bodyId}"><svg class="cookbook-section-chevron" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"><polyline points="6 9 12 15 18 9"/></svg><span class="cookbook-srv-status ${_secDot}" title="${_dotTitle}" style="flex-shrink:0;position:relative;top:0px;"></span><span class="cookbook-section-title" style="margin:0;">${esc(sg.name)}</span><button class="cookbook-btn cookbook-stop-all-btn" data-stop-server="${esc(key)}">Stop all</button><button class="cookbook-btn cookbook-clear-btn" data-clear-server="${esc(key)}">Clear finished</button></div><div id="${bodyId}" class="cookbook-section-body"></div>`); + sec.insertAdjacentHTML('afterbegin', `<div class="cookbook-section-header" data-collapse="${bodyId}"><svg class="cookbook-section-chevron" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"><polyline points="6 9 12 15 18 9"/></svg><span class="cookbook-srv-status ${_secDot}" title="${_dotTitle}" style="flex-shrink:0;position:relative;top:0px;"></span><span class="cookbook-section-title" style="margin:0;">${esc(sg.name)}</span><button class="cookbook-btn cookbook-stop-all-btn" data-stop-server="${esc(key)}" title="Stop all running servers"><svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" stroke="none" aria-hidden="true" style="vertical-align:-1px;margin-right:4px;"><rect x="5" y="5" width="14" height="14" rx="1.5"/></svg>Stop all</button><button class="cookbook-btn cookbook-clear-btn" data-clear-server="${esc(key)}" title="Clear finished tasks"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="vertical-align:-1px;margin-right:4px;"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/></svg>Clear finished</button></div><div id="${bodyId}" class="cookbook-section-body"></div>`); } } @@ -1761,9 +1808,21 @@ export function _renderRunningTab() { btn.addEventListener('click', async (e) => { e.stopPropagation(); // don't toggle the section collapse (was an inline onclick, blocked by CSP) const host = btn.dataset.clearServer; - if (!await window.styledConfirm(`Clear finished tasks on ${_serverName(host)}?`, { confirmText: 'Clear' })) return; const allTasks = _loadTasks(); const toRemove = allTasks.filter(t => (t.remoteHost || '') === host && _canClearTask(t)); + // Bail with a clear message instead of silently doing nothing when + // every task on this server is still running (nothing finished to + // clear yet) — the previous behavior looked like the button was dead. + if (!toRemove.length) { + const stillRunning = allTasks.filter(t => (t.remoteHost || '') === host && t.status === 'running').length; + const _msg = stillRunning + ? `No finished tasks on ${_serverName(host)} — ${stillRunning} still running. Stop them first to clear.` + : `No finished tasks on ${_serverName(host)}.`; + if (window.uiModule?.showToast) window.uiModule.showToast(_msg); + else alert(_msg); + return; + } + if (!await window.styledConfirm(`Clear ${toRemove.length} finished task${toRemove.length === 1 ? '' : 's'} on ${_serverName(host)}?`, { confirmText: 'Clear' })) return; const remaining = allTasks.filter(t => (t.remoteHost || '') !== host || !_canClearTask(t)); _saveTasks(remaining); // Fade/slide each finished card out (same exit as the per-card clear) @@ -2099,57 +2158,43 @@ export function _renderRunningTab() { dropdown.className = 'cookbook-task-dropdown'; const items = []; + // ── Run section ───────────────────────────────────────────── // Queued download: let the user jump the queue and start it immediately // (downloads otherwise run one-at-a-time per server). if (task.type === 'download' && task.status === 'queued') { - items.push({ label: 'Start now', action: 'start-now', custom: () => { + items.push({ group: 'run', label: 'Start now', action: 'start-now', custom: () => { _startQueuedDownload(task); _renderRunningTab(); }}); } if (task.status !== 'running' && task.status !== 'queued') { - items.push({ label: 'Reconnect', action: 'reconnect' }); + items.push({ group: 'run', label: 'Reconnect tmux', action: 'reconnect' }); } if (task.status === 'running') { - items.push({ label: 'Stop', action: 'stop', danger: true }); + items.push({ group: 'run', label: 'Stop', action: 'stop', danger: true }); } - items.push({ label: 'Restart', action: 'retry' }); - // Edit serve — open the full serve panel (same as the edit icon), - // switching to this task's server first so the model is found. + items.push({ group: 'run', label: 'Restart', action: 'retry' }); + // ── Edit section ──────────────────────────────────────────── + // Merged "Edit & relaunch" — opens the structured serve panel + // pre-filled with this task's config. The old standalone "Edit + // cmd & relaunch" raw-text dialog is now reachable from inside + // that panel (Show command). Single entry-point per task. if (task.type === 'serve' && task.payload?.repo_id) { - items.push({ label: 'Edit in serve panel', action: 'edit-panel', tooltip: 'Open the full Serve config panel pre-filled with this task — pick a different backend, change GPUs, edit env vars, then Launch from there', custom: () => _openEdit() }); + items.push({ group: 'edit', label: 'Edit & relaunch', action: 'edit-panel', tooltip: 'Open the Serve config panel pre-filled with this task — pick a different backend, change GPUs, edit env vars or the raw cmd, then Launch.', custom: () => _openEdit() }); } - // Save serve — save current launch config as a preset. if (task.type === 'serve' && task.payload?._cmd) { - items.push({ label: 'Save serve', action: 'save', custom: () => { + items.push({ group: 'edit', label: 'Save serve', action: 'save', custom: () => { if (!_saveTaskAsPreset(task)) { uiModule.showToast('Already saved'); return; } uiModule.showToast('Saved to presets'); _renderRunningTab(); }}); } - // Edit command — only meaningful for serve tasks that aren't running. - // Lets the user tweak flags after a crash/error and relaunch. - if (task.type === 'serve' && task.status !== 'running' && task.payload?._cmd) { - items.push({ label: 'Edit cmd & relaunch', action: 'edit', tooltip: 'Edit the raw vllm/llama-server cmd string in a dialog and relaunch immediately on the same host', custom: async () => { - const newCmd = await _promptEditServeCmd(task.payload._cmd); - if (newCmd == null) return; // cancelled - try { - await fetch('/api/shell/exec', { - method: 'POST', credentials: 'same-origin', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ command: _tmuxGracefulKill(task) }), - }); - } catch {} - _removeTask(task.sessionId); - // Relaunch on the task's OWN host, not the current global selection. - _launchServeTask(task.name, task.payload.repo_id, newCmd, task.payload._fields, task.remoteHost || ''); - }}); - } + // ── Endpoint section ──────────────────────────────────────── // Manual endpoint registration — fallback for when auto-add fails // (e.g. probe timeout on a remote that's slow). Forces adding this // serve to the model-endpoints list regardless of prior flag state. if (task.type === 'serve' && task.payload?._cmd) { - items.push({ label: 'Register endpoint', action: 'register-endpoint', custom: async () => { + items.push({ group: 'endpoint', label: 'Register endpoint', action: 'register-endpoint', custom: async () => { const host = _connectHostFromRemote(task.remoteHost); const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/); const port = portMatch ? portMatch[1] : '8000'; @@ -2194,31 +2239,32 @@ export function _renderRunningTab() { } }}); } + // ── Copy section ──────────────────────────────────────────── if (_isWindows(task)) { const host = task.remoteHost; const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux'; const logCmd = host ? `ssh ${_sshPrefix(_getPort(task))}${host} "powershell -Command \\"Get-Content '${sd}\\${task.sessionId}.log' -Wait\\""` : `powershell -Command "Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${task.sessionId}.log') -Wait"`; - items.push({ label: 'Copy log cmd', action: 'copy-tmux', custom: () => { + items.push({ group: 'copy', label: 'Copy log cmd', action: 'copy-tmux', custom: () => { _copyText(logCmd); }}); } else { // Just the tmux command itself — no ssh wrapper. const tmuxAttach = `tmux attach -t ${task.sessionId}`; - items.push({ label: 'Copy tmux', action: 'copy-tmux', custom: () => { + items.push({ group: 'copy', label: 'Copy tmux', action: 'copy-tmux', custom: () => { _copyText(tmuxAttach); }}); } if (_shouldOfferCrashReport(task)) { - items.push({ label: 'Copy crash report', action: 'copy-crash-report', custom: () => { + items.push({ group: 'copy', label: 'Copy crash report', action: 'copy-crash-report', custom: () => { const out = (el.querySelector('.cookbook-output-pre')?.textContent || task.output || ''); _copyText(_buildCrashReport(task, out)); uiModule.showToast('Copied crash report'); }}); } // Copy the last 50 lines of the task's output/log. - items.push({ label: 'Copy last 50 lines', action: 'copy-log', custom: () => { + items.push({ group: 'copy', label: 'Copy last 50 lines', action: 'copy-log', custom: () => { const out = (el.querySelector('.cookbook-output-pre')?.textContent || task.output || ''); const last = out.split('\n').slice(-50).join('\n'); if (!last.trim()) { @@ -2232,8 +2278,10 @@ export function _renderRunningTab() { // the live tmux session and (for serve tasks) deletes the // matching model-endpoint, THEN animates the task card out. // Just "Remove" hid that it stops the live serve too. + // ── Danger section ────────────────────────────────────────── const _isLive = task.type === 'serve' && ['running', 'ready', 'loading', 'warming', 'starting'].includes(task.status || ''); items.push({ + group: 'danger', label: _isLive ? 'Stop and remove' : 'Remove', action: 'kill', tooltip: _isLive @@ -2241,10 +2289,8 @@ export function _renderRunningTab() { : 'Remove this row', danger: true, }); - // Cancel = mobile-only dismiss item. Same pattern as the email kebab: - // the `dropdown-cancel-mobile` class is hidden on desktop and styled - // as a separated bottom row on mobile (border-top + extra padding). - items.push({ label: 'Cancel', action: 'cancel', mobileOnly: true, custom: () => {} }); + // Cancel = mobile-only dismiss item. Same pattern as the email kebab. + items.push({ group: 'danger', label: 'Cancel', action: 'cancel', mobileOnly: true, custom: () => {} }); const _MENU_ICONS = { 'start-now': '<polygon points="6 4 20 12 6 20 6 4"/>', @@ -2261,7 +2307,18 @@ export function _renderRunningTab() { kill: '<path d="M3 6h18"/><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6m3 0V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"/>', cancel: '<line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/>', }; + let _lastGroup = null; for (const item of items) { + // Insert a thin divider whenever the group changes, so the + // user can visually scan Run / Edit / Endpoint / Copy / Danger + // blocks instead of one long undifferentiated list. + if (item.group && _lastGroup && item.group !== _lastGroup) { + const sep = document.createElement('div'); + sep.className = 'cookbook-dropdown-divider'; + sep.style.cssText = 'height:1px;margin:4px 6px;background:color-mix(in srgb, var(--fg) 12%, transparent);pointer-events:none;'; + dropdown.appendChild(sep); + } + _lastGroup = item.group || _lastGroup; const div = document.createElement('div'); div.className = 'dropdown-item-compact' + (item.danger ? ' cookbook-dropdown-danger' : '') @@ -2651,7 +2708,7 @@ async function _reconnectTask(el, task) { // capture-pane lets the existing _reconnectTask flow pick up // the real state (running, finished, or truly dead). const _reconnectFix = { - label: 'Reconnect', + label: 'Reconnect tmux', action: () => { _updateTask(task.sessionId, { status: 'running' }); el.dataset.status = 'running'; @@ -3532,12 +3589,22 @@ async function _pollBackgroundStatus() { // dead-session check inspects). Recover "done" from the retained output's // exit-0 sentinel so a clean install isn't downgraded to crashed. const depDone = !!task.payload?._dep && _depInstallSucceeded(task.output); + // A finished model download whose tmux pane is gone is also reported + // "stopped" (the dead-session check can miss the landed snapshot). + // Recover "done" from the terminal `DOWNLOAD_OK` sentinel — emitted + // only after the runner exits 0 — so a completed download isn't + // downgraded to crashed. This background poll runs blind (no live + // stream to debounce against), so unlike the reconnect loop it keys + // off the conclusive exit sentinel only, never the `/snapshots/` path, + // which can be printed mid-stream for multi-file downloads. + const downloadDone = task.type === 'download' + && String(task.output || '').includes('DOWNLOAD_OK'); const nextStatus = live.status === 'completed' ? 'done' : (live.status === 'error' ? 'error' : (live.status === 'stopped' - ? (depDone ? 'done' : (task.type === 'download' ? 'crashed' : 'stopped')) + ? ((depDone || downloadDone) ? 'done' : (task.type === 'download' ? 'crashed' : 'stopped')) : null)); if (nextStatus && task.status !== nextStatus) { updates.status = nextStatus; @@ -3547,6 +3614,7 @@ async function _pollBackgroundStatus() { updates.status = live.status === 'ready' ? 'ready' : 'running'; } if (live.progress && live.progress !== task.progress) updates.progress = live.progress; + if (live.exit_code != null && live.exit_code !== task.exit_code) updates.exit_code = live.exit_code; if (live.output_tail) { const previous = String(task.output || ''); const tail = String(live.output_tail || ''); diff --git a/static/js/cookbookServe.js b/static/js/cookbookServe.js index 2a5cc5b5b..33d56ef3c 100644 --- a/static/js/cookbookServe.js +++ b/static/js/cookbookServe.js @@ -9,6 +9,7 @@ import spinnerModule from './spinner.js'; import { providerLogo } from './providers.js'; import { modelColor } from './chatRenderer.js'; import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js'; +import { openCookbookDependencies } from './cookbook-diagnosis.js'; // Shared state/functions injected by init() let _envState; @@ -115,13 +116,28 @@ function _selectedServeTarget(panel) { : (server?.name || 'local server'); return { host, - port: host ? (_getPort(host) || server?.port || '') : '', + port: host ? (server?.port || _getPort(host) || '') : '', + env: server?.env || '', venv, platform: server?.platform || _envState.platform || '', label, }; } +function _remoteWindowsDiffusersUnsupported(target) { + return !!(target?.host && target?.platform === 'windows'); +} + +function _backendChoicesForTarget(target) { + if (target?.platform === 'windows') { + if (_remoteWindowsDiffusersUnsupported(target)) return [['llamacpp','llama.cpp']]; + return [['llamacpp','llama.cpp'],['diffusers','Diffusers']]; + } + return _isMetal() + ? [['llamacpp','llama.cpp'],['ollama','Ollama']] + : [['vllm','vLLM'],['sglang','SGLang'],['llamacpp','llama.cpp'],['ollama','Ollama'],['diffusers','Diffusers']]; +} + async function _fetchServeRuntimePackage(panel, backend) { const packageByBackend = { vllm: 'vllm', @@ -528,13 +544,14 @@ function _rerenderCachedModels() { const ss = (_byRepo[repo] && typeof _byRepo[repo] === 'object') ? _byRepo[repo] : (_lastUsed || (_isLegacyFlat ? _allSs : {})); + const _serveTarget = _selectedServeTarget(); + const _backendChoices = _backendChoicesForTarget(_serveTarget); + const _allowedBackends = new Set(_backendChoices.map(([v]) => v)); const detectedBackend = _detectBackend(m).backend; - const _allowedBackends = new Set(_isWindows() - ? ['llamacpp'] - : (_isMetal() ? ['llamacpp', 'ollama'] : ['vllm', 'sglang', 'llamacpp', 'ollama', 'diffusers'])); - const defaultBackend = (ss._forceBackend && ss.backend && _allowedBackends.has(ss.backend)) + let defaultBackend = (ss._forceBackend && ss.backend && _allowedBackends.has(ss.backend)) ? ss.backend : detectedBackend; + if (!_allowedBackends.has(defaultBackend)) defaultBackend = _backendChoices[0]?.[0] || detectedBackend; const savedMatchesBackend = !!ss._forceBackend || (ss.backend || 'vllm') === detectedBackend; const sv = (k, def) => (ss[k] !== undefined && savedMatchesBackend) ? ss[k] : def; const defaultTp = defaultBackend === 'llamacpp' ? '1' : sv('tp', '1'); @@ -546,7 +563,14 @@ function _rerenderCachedModels() { : (_es.gpus || detectedGpuIds)); const tpOpts = [1,2,4,8].map(n => `<option${defaultTp==String(n)?' selected':''}>${n}</option>`).join(''); const dtypeOpts = ['auto','float16','bfloat16'].map(d => `<option value="${d}"${sv('dtype','auto')===d?' selected':''}>${d}</option>`).join(''); - const vllmKvCacheOpts = ['auto','fp8'].map(d => `<option value="${d}"${sv('vllm_kv_cache_dtype','auto')===d?' selected':''}>${d}</option>`).join(''); + // KV cache default — most models are fine on auto, but a few + // (e.g. DeepSeek-V3/V4/R1 MoE) need fp8 explicitly or the launch + // OOMs. _detectModelOptimizations seeds opts.kvCacheDtype for + // those families; honour it unless the user has a saved override. + const _kvOptsCheck = _detectModelOptimizations(repo); + const _kvAutoDefault = (_kvOptsCheck && _kvOptsCheck.kvCacheDtype) || 'auto'; + const _kvSelected = sv('vllm_kv_cache_dtype', _kvAutoDefault); + const vllmKvCacheOpts = ['auto','fp8'].map(d => `<option value="${d}"${_kvSelected===d?' selected':''}>${d}</option>`).join(''); const _l = (name, tip) => `<span>${name}<span class="hwfit-hint" title="${tip}">?</span></span>`; const _ggufChoices = _runnableGgufFiles(m); const _savedGguf = String(sv('gguf_file', '') || ''); @@ -572,12 +596,22 @@ function _rerenderCachedModels() { const _arrowTitle = _modelPresets.length > 0 ? `${_modelPresets.length} saved launch config${_modelPresets.length === 1 ? '' : 's'} for ${_repoShort} — click ▾ to load or delete` : `No saved launch configs for ${_repoShort} yet — click Save to add one`; - let _slotsHtml = `<div class="cookbook-serve-slots cookbook-saved-split">` + // Wrap the Save split in a <label> so it picks up the same "field + // title + ?-help" treatment as Backend / venv / Port / GPUs sitting + // beside it in Row 1. Button text is "Save" (the action), label is + // "Settings" (what the saved blob represents). + let _slotsHtml = `<label>${_l('Settings','Saved launch configurations for this model — click ▾ to load or delete')}` + + `<div class="cookbook-serve-slots cookbook-saved-split">` + `<button type="button" class="cookbook-slot-btn cookbook-saved-save" title="Save current config"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M19 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11l5 5v11a2 2 0 0 1-2 2z"/><polyline points="17 21 17 13 7 13 7 21"/><polyline points="7 3 7 8 15 8"/></svg>Save</button>` + `<button type="button" class="cookbook-slot-btn cookbook-saved-arrow" title="${esc(_arrowTitle)}">${_arrowLabel}</button>` - + `</div>`; + + `</div></label>`; let panelHtml = `<div class="hwfit-serve-panel">`; + // Runtime-readiness note pinned at the top of the serve area so the + // user sees "vLLM ready on …" before scrolling into the configure + // form. Hidden until the readiness probe returns. The × button + // dismisses it for this panel only (re-shows on re-expand). + panelHtml += `<div class="hwfit-serve-runtime-note" style="display:none;font-size:11px;line-height:1.35;color:var(--fg-muted);margin:0 0 8px;padding:6px 28px 6px 10px;border-radius:5px;background:color-mix(in srgb, var(--fg) 4%, transparent);border:1px solid color-mix(in srgb, var(--border) 60%, transparent);position:relative;"><span class="hwfit-serve-runtime-text"></span><button type="button" class="hwfit-serve-runtime-close" title="Dismiss" aria-label="Dismiss" style="position:absolute;top:-8px;right:5px;background:none;border:0;color:inherit;cursor:pointer;padding:2px 4px;line-height:1;font-size:13px;opacity:0.6;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" aria-hidden="true"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg></button></div>`; // Warn when serving a model whose download hasn't fully completed — // the user CAN still hit Launch (vLLM/llama-server will start, then // crash trying to read missing shards), but they should know. @@ -589,16 +623,20 @@ function _rerenderCachedModels() { } // Row 1: Backend + Server + Env panelHtml += `<div class="hwfit-serve-row">`; - const _backendChoices = _isWindows() - ? [['llamacpp','llama.cpp']] - : _isMetal() - // Diffusers (diffusion_server.py) is CUDA-only — omit it on Metal. - ? [['llamacpp','llama.cpp'],['ollama','Ollama']] - : [['vllm','vLLM'],['sglang','SGLang'],['llamacpp','llama.cpp'],['ollama','Ollama'],['diffusers','Diffusers']]; const backendOpts = _backendChoices.map(([v,l]) => `<option value="${v}"${defaultBackend===v?' selected':''}>${l}</option>`).join(''); - panelHtml += `<label>${_l('Backend','Inference engine: vLLM, SGLang, llama.cpp, Ollama, or Diffusers')}<select class="hwfit-sf" data-field="backend">${backendOpts}</select></label>`; + // Custom Backend picker — native <select> can't host SVG inside + // options, so we render a button + menu that show the backend logo + // beside its name. The hidden <select.hwfit-sf data-field="backend"> + // stays as the source-of-truth so every existing change handler + // (updateBackendVisibility, runtime readiness, command builder) + // still fires via dispatchEvent('change') on selection. + panelHtml += `<label>${_l('Backend','Inference engine: vLLM, SGLang, llama.cpp, Ollama, or Diffusers')}<div class="hwfit-backend-picker" data-backend-picker style="position:relative;width:100%;"><select class="hwfit-sf hwfit-backend-source" data-field="backend" style="display:none;">${backendOpts}</select><button type="button" class="hwfit-backend-btn" data-backend-btn aria-haspopup="listbox" aria-expanded="false" style="display:flex;align-items:center;gap:6px;width:100%;height:28px;padding:0 8px;background:var(--bg);color:var(--fg);border:1px solid var(--border);border-radius:4px;font:inherit;font-size:11px;cursor:pointer;text-align:left;"><span class="hwfit-backend-btn-icon" data-backend-icon-slot aria-hidden="true" style="display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;color:var(--accent, var(--red));flex-shrink:0;"></span><span class="hwfit-backend-btn-label" data-backend-label style="flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;"></span><svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="opacity:0.6;flex-shrink:0;"><polyline points="6 9 12 15 18 9"/></svg></button><div class="hwfit-backend-menu" data-backend-menu role="listbox" hidden style="position:absolute;top:calc(100% + 4px);left:0;right:0;z-index:100;background:var(--panel, var(--bg));border:1px solid var(--border);border-radius:6px;box-shadow:0 6px 20px rgba(0,0,0,0.22);padding:4px;"></div></div></label>`; panelHtml += `<input type="hidden" class="hwfit-sf" data-field="host" value="${esc(_es.remoteHost || '')}" />`; panelHtml += `<label>${_l('venv','Path to Python venv or conda env activate script')}<input type="text" class="hwfit-sf hwfit-sf-wide" data-field="venv" value="${esc(sv('venv', _es.envPath || _srvVenv || ''))}" placeholder="~/venv" /></label>`; + // Dtype lives in Row 1 (next to venv) — it's the first knob people + // change when matching the model to the box, so it earns top-row + // real estate over Row 2's launch-tuning controls. + panelHtml += `<label>${_l('Dtype','Data type for weights. auto picks best for GPU')}<select class="hwfit-sf" data-field="dtype">${dtypeOpts}</select></label>`; const defaultPort = defaultBackend === 'ollama' ? '11434' : _nextAvailablePort(); panelHtml += `<label>${_l('Port','HTTP port for the API server')}<input type="text" class="hwfit-sf" data-field="port" value="${esc(sv('port', defaultPort))}" /></label>`; const _activeGpus = (defaultGpus || '').split(',').map(s => s.trim()).filter(Boolean); @@ -609,12 +647,16 @@ function _rerenderCachedModels() { const on = _activeGpus.includes(String(i)); _gpuBtnsHtml += `<button type="button" class="cookbook-gpu-btn${on ? ' active' : ''}" data-gpu="${i}">${i}</button>`; } - panelHtml += `<label>${_l('GPUs','Toggle which GPUs to use')}<div class="cookbook-gpu-group">${_gpuBtnsHtml}</div><input type="hidden" class="hwfit-sf" data-field="gpus" value="${esc(defaultGpus)}" /></label>`; - // Save / saved-configs split button — moved into Row 1 (next to GPUs) - // so it shares the same baseline as the rest of the top controls. + // GPUs button strip moved to Row 2 (next to GPU Mem) below. 4px + // margin on the left, 8px on the right — extra 4px right-side gap + // separates the GPU chiclets from the GPU Mem field that follows + // (asked-for breathing room; 4px on either side felt cramped on + // the GPU-Mem boundary). + const _gpusLabelHtml = `<label class="hwfit-gpus-label" style="margin:0 8px 0 4px;">${_l('GPUs','Toggle which GPUs to use')}<div class="cookbook-gpu-group">${_gpuBtnsHtml}</div><input type="hidden" class="hwfit-sf" data-field="gpus" value="${esc(defaultGpus)}" /></label>`; + // Save / saved-configs split button — sits at the right end of Row 1. panelHtml += _slotsHtml; panelHtml += `</div>`; - panelHtml += `<div class="hwfit-serve-runtime-note" style="display:none;font-size:11px;line-height:1.35;color:var(--fg-muted);margin-top:-4px;"></div>`; + // (hwfit-serve-runtime-note moved to the top of the panel — see above.) if (_ggufChoices.length > 1) { // Show the GGUF File dropdown for BOTH llama.cpp and Ollama — Ollama // also needs to know which exact .gguf to import via the new @@ -631,15 +673,22 @@ function _rerenderCachedModels() { // TP / Context / GPU / GPU Mem / Max Seqs / Dtype. Everything else // (Swap, KV Cache, Attention backend, Env vars, llama.cpp batch/ubatch) // moved to the Advanced fold below to keep this row scannable. - panelHtml += `<div class="hwfit-serve-row hwfit-backend-vllm hwfit-backend-sglang hwfit-backend-llamacpp hwfit-backend-ollama">`; + panelHtml += `<div class="hwfit-serve-row hwfit-serve-row-core hwfit-backend-vllm hwfit-backend-sglang hwfit-backend-llamacpp hwfit-backend-ollama">`; + // Order: TP → Context → Max Seqs → GPUs → GPU Mem. + // Dtype moved up to Row 1. GPUs moved here next to GPU Mem so the + // "which devices + how much of them" decisions sit adjacent. Max + // Seqs follows Context per the "request-shape" cluster. panelHtml += `<label class="hwfit-backend-vllm hwfit-backend-sglang">${_l('TP','Tensor Parallelism — split model across N GPUs')}<select class="hwfit-sf" data-field="tp">${tpOpts}</select></label>`; // ctx resets to the model's max on every panel open (the real ctx slider // lives in the Scan/Download toolbar — see cookbook.js .hwfit-ctx-control). panelHtml += `<label>${_l('Context','Max tokens per request — resets to the model max on every open. Lower = less VRAM')}<input type="text" class="hwfit-sf" data-field="ctx" value="${esc(m.context_length || m.context || '20000')}" /></label>`; - panelHtml += `<label>${_l('GPU','Which GPU to use. Leave empty for default')}<input type="text" class="hwfit-sf" data-field="gpu_id" value="${esc(sv('gpu_id', ''))}" placeholder="auto" style="width:50px;" /></label>`; - panelHtml += `<label class="hwfit-backend-vllm hwfit-backend-sglang">${_l('GPU Mem','Fraction of GPU memory (0.0–1.0). Lower if OOM')}<input type="text" class="hwfit-sf" data-field="gpu_mem" value="${esc(sv('gpu_mem', '0.90'))}" /></label>`; panelHtml += `<label class="hwfit-backend-vllm hwfit-backend-sglang">${_l('Max Seqs','Maximum concurrent requests. Lower = less memory. Default 4 — prosumer GPUs often OOM on vLLM default 256 during CUDA graph capture.')}<input type="text" class="hwfit-sf" data-field="max_seqs" value="${esc(sv('max_seqs', '4'))}" placeholder="4" /></label>`; - panelHtml += `<label>${_l('Dtype','Data type for weights. auto picks best for GPU')}<select class="hwfit-sf" data-field="dtype">${dtypeOpts}</select></label>`; + // GPU "auto" field removed — the GPU button strip below already + // writes data-field="gpus" (the canonical comma-separated device + // list) and the command builders now read from that single source. + panelHtml += `<label class="hwfit-backend-vllm hwfit-backend-sglang">${_l('GPU Mem','Fraction of GPU memory (0.0–1.0). Lower if OOM')}<input type="text" class="hwfit-sf" data-field="gpu_mem" value="${esc(sv('gpu_mem', '0.90'))}" /></label>`; + // GPUs button strip at the far right of Row 2. + panelHtml += _gpusLabelHtml; panelHtml += `</div>`; // ── Advanced (collapsed by default) ── // Everything below the fold is tuning users only touch occasionally: @@ -667,7 +716,10 @@ function _rerenderCachedModels() { // tuning, or any other KEY=VALUE pair that doesn't have a dedicated // field. After the venv activate runs, $VIRTUAL_ENV / $PATH / etc. are // already exported so they expand correctly here. - panelHtml += `<label class="hwfit-backend-vllm hwfit-backend-sglang" style="flex:1 1 100%;">${_l('Env','Extra KEY=VALUE env-var pairs prepended to the launch (space-separated). Example: CUDACXX=$VIRTUAL_ENV/lib/python3.10/site-packages/nvidia/cuda_nvcc/bin/nvcc — points flashinfer at the venv-bundled nvcc when the system one is too old for your GPU.')}<input type="text" class="hwfit-sf" data-field="extra_env" value="${esc(sv('extra_env',''))}" placeholder="CUDACXX=/path/to/nvcc NCCL_P2P_DISABLE=1" style="width:100%;" /></label>`; + // grid-column: 1 / -1 makes Env span every column of the Advanced + // row's CSS grid (the old flex:1 1 100% did nothing in a grid + // container — left an empty trailing column gap on wide modals). + panelHtml += `<label class="hwfit-backend-vllm hwfit-backend-sglang" style="grid-column:1 / -1;">${_l('Env','Extra KEY=VALUE env-var pairs prepended to the launch (space-separated). Example: CUDACXX=$VIRTUAL_ENV/lib/python3.10/site-packages/nvidia/cuda_nvcc/bin/nvcc — points flashinfer at the venv-bundled nvcc when the system one is too old for your GPU.')}<input type="text" class="hwfit-sf" data-field="extra_env" value="${esc(sv('extra_env',''))}" placeholder="CUDACXX=/path/to/nvcc NCCL_P2P_DISABLE=1" style="width:100%;" /></label>`; panelHtml += `</div>`; // Advanced llama.cpp row (Batch / UBatch — moved out of Core for the // same "rarely touched" reason as the vLLM extras above). @@ -686,11 +738,36 @@ function _rerenderCachedModels() { panelHtml += `<label>Height${_h('Default output height')} <input type="text" class="hwfit-sf" data-field="diff_height" value="${esc(sv('diff_height', ''))}" placeholder="1024" /></label>`; panelHtml += `</div>`; // Row 3: Checkboxes (vLLM) + // Order: Trust Remote → Auto Tool → Reasoning Parser (when the + // model has one) → Enforce Eager → Prefix Caching. Reasoning + // Parser was previously in a separate row below; the user wanted + // it inline with the other vLLM toggles between Auto Tool and + // Enforce Eager so the "what the model needs" decisions sit + // together at the top. + const _opts2_row3 = _detectModelOptimizations(repo); + const _rp_flag = _opts2_row3.flags.find(f => f.includes('--reasoning-parser')); + const _rp_name = _rp_flag ? _rp_flag.split(' ')[1] : ''; panelHtml += `<div class="hwfit-serve-checks hwfit-backend-vllm hwfit-backend-sglang">`; - panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="enforce_eager"${sv('enforce_eager',false)?' checked':''} /> Enforce Eager${_h('Disable CUDA graphs. Slower but uses less memory')}</label>`; panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="trust_remote"${sv('trust_remote',false)?' checked':''} /> Trust Remote Code${_h('Allow model to run custom code from HuggingFace')}</label>`; - panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="prefix_cache"${sv('prefix_cache',false)?' checked':''} /> Prefix Caching${_h('Cache shared prompt prefixes across requests')}</label>`; panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm"><input type="checkbox" class="hwfit-sf" data-field="auto_tool"${sv('auto_tool',false)?' checked':''} /> Auto Tool Choice${_h('Enable function/tool calling for agent mode')}</label>`; + if (_rp_name) panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm"><input type="checkbox" class="hwfit-sf" data-field="reasoning_parser" data-parser="${_rp_name}" /> Reasoning Parser <span class="hwfit-parser-tag">${_rp_name}</span></label>`; + panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="enforce_eager"${sv('enforce_eager',false)?' checked':''} /> Enforce Eager${_h('Disable CUDA graphs. Slower but uses less memory')}</label>`; + panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="prefix_cache"${sv('prefix_cache',false)?' checked':''} /> Prefix Caching${_h('Cache shared prompt prefixes across requests')}</label>`; + // Inline the previously-second vLLM checks row so Expert Parallel / + // Speculative / MoE Env sit next to Prefix Caching with no gap. All + // three are vLLM-only — class-gated so they hide on SGLang. + if (_opts2_row3.flags.includes('--enable-expert-parallel')) panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm"><input type="checkbox" class="hwfit-sf" data-field="expert_parallel" /> Expert Parallel</label>`; + { + const _specDef = _opts2_row3.spec || { method: 'mtp', tokens: 3 }; + const _specMethod = sv('spec_method', _specDef.method); + const _specTokens = sv('spec_tokens', String(_specDef.tokens)); + const _specMethods = ['mtp', 'qwen3_next_mtp', 'eagle', 'medusa', 'ngram']; + if (!_specMethods.includes(_specMethod)) _specMethods.unshift(_specMethod); + const _specOpts = _specMethods.map(m => + `<option value="${m}"${m === _specMethod ? ' selected' : ''}>${m}</option>`).join(''); + panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm hwfit-spec-group"><input type="checkbox" class="hwfit-sf" data-field="speculative" /> Speculative <select class="hwfit-sf hwfit-spec-method" data-field="spec_method" title="vLLM --speculative-config method">${_specOpts}</select><input type="number" class="hwfit-sf hwfit-spec-tokens hwfit-spec-tokens-bare" data-field="spec_tokens" value="${esc(_specTokens)}" min="1" max="10" title="num_speculative_tokens" style="width:44px;" /><span class="hwfit-help-chip hwfit-help-chip-inline" title="MTP / speculative decoding is supported on a few model families only — turn it on when the model card explicitly recommends it. On supported models it can boost inference throughput up to ~3×; on unsupported models it will either be ignored or fail to launch." style="margin-left:6px;">?</span></label>`; + } + if (_opts2_row3.envVars.length) panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm"><input type="checkbox" class="hwfit-sf" data-field="moe_env" /> MoE Env Vars</label>`; panelHtml += `</div>`; // Row 2c: llama.cpp fit/perf flags (set by Auto profiles, editable by hand) const _kvOpts = ['', 'q4_0', 'q8_0', 'f16'].map(k => `<option value="${k}"${sv('cache_type','')===k?' selected':''}>${k||'default'}</option>`).join(''); @@ -739,33 +816,16 @@ function _rerenderCachedModels() { panelHtml += `</div><div class="hwfit-serve-row hwfit-backend-diffusers">`; panelHtml += `<label>Harmonize GPU${_h('Separate GPU for img2img/harmonize. Leave empty to use same GPU')}<input type="text" class="hwfit-sf" data-field="diff_harmonize_gpu" value="${esc(sv('diff_harmonize_gpu', ''))}" placeholder="auto" style="width:50px;" /></label>`; panelHtml += `</div>`; - // Row 4: Extra args - panelHtml += `<div class="hwfit-serve-extra">`; - panelHtml += `<label>Extra args<input type="text" class="hwfit-sf" data-field="extra" value="${esc(sv('extra', ''))}" placeholder="--flag value" /></label>`; - panelHtml += `</div>`; // Model-specific optimizations. The checks row always renders for the // vLLM backend so the Speculative (MTP) control is ALWAYS reachable — // even for models the auto-detector doesn't recognize. Expert-parallel, // reasoning-parser and MoE-env still only appear when auto-detected. - const _opts2 = _detectModelOptimizations(repo); - panelHtml += `<div class="hwfit-serve-checks hwfit-backend-vllm">`; - if (_opts2.flags.includes('--enable-expert-parallel')) panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="expert_parallel" /> Expert Parallel</label>`; - if (_opts2.flags.some(f => f.includes('--reasoning-parser'))) { const rp = _opts2.flags.find(f => f.includes('--reasoning-parser')).split(' ')[1]; panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="reasoning_parser" data-parser="${rp}" /> Reasoning Parser <span class="hwfit-parser-tag">${rp}</span></label>`; } - { - // Speculative decoding (vLLM --speculative-config). Default OFF; the - // method/token defaults come from auto-detection when available, - // else fall back to MTP/3. Toggling the checkbox is what actually - // adds the flag at launch (see cookbook.js command builder). - const _specDef = _opts2.spec || { method: 'mtp', tokens: 3 }; - const _specMethod = sv('spec_method', _specDef.method); - const _specTokens = sv('spec_tokens', String(_specDef.tokens)); - const _specMethods = ['mtp', 'qwen3_next_mtp', 'eagle', 'medusa', 'ngram']; - if (!_specMethods.includes(_specMethod)) _specMethods.unshift(_specMethod); - const _specOpts = _specMethods.map(m => - `<option value="${m}"${m === _specMethod ? ' selected' : ''}>${m}</option>`).join(''); - panelHtml += `<label class="hwfit-sf-cb hwfit-spec-group"><input type="checkbox" class="hwfit-sf" data-field="speculative" /> Speculative <select class="hwfit-sf hwfit-spec-method" data-field="spec_method" title="vLLM --speculative-config method">${_specOpts}</select><span class="hwfit-numstep"><button type="button" class="hwfit-numstep-btn" data-step="-1" tabindex="-1" aria-label="Decrease">‹</button><input type="number" class="hwfit-sf hwfit-spec-tokens" data-field="spec_tokens" value="${esc(_specTokens)}" min="1" max="10" title="num_speculative_tokens" /><button type="button" class="hwfit-numstep-btn" data-step="1" tabindex="-1" aria-label="Increase">›</button></span><span class="hwfit-help-chip hwfit-help-chip-inline" title="MTP / speculative decoding is supported on a few model families only — turn it on when the model card explicitly recommends it. On supported models it can boost inference throughput up to ~3×; on unsupported models it will either be ignored or fail to launch." style="margin-left:6px;">?</span></label>`; - } - if (_opts2.envVars.length) panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="moe_env" /> MoE Env Vars</label>`; + // Expert Parallel / Speculative / MoE Env moved into Row 3 above so + // the vLLM-only toggles sit next to Prefix Caching with no gap. + // Extra args sits below the vLLM checks (Reasoning Parser + Spec) + // so it reads as "after the advanced toggles, any other flags". + panelHtml += `<div class="hwfit-serve-extra">`; + panelHtml += `<label>Extra args<input type="text" class="hwfit-sf" data-field="extra" value="${esc(sv('extra', ''))}" placeholder="--flag value" /></label>`; panelHtml += `</div>`; // ── End Advanced fold ── panelHtml += `</details>`; @@ -958,37 +1018,183 @@ function _rerenderCachedModels() { if (ok === false) clearInterval(_vramTimer); }, 4000); - // Show/hide backend-specific sections + // Backend icons — accent color, rendered via currentColor. vLLM gets + // a stylized double-V mark, the others fall back to a recognizable + // glyph for the engine family. Shown beside each option in the + // custom picker so the dropdown lists "[V] vLLM", "[⚡] SGLang", etc. + const _BACKEND_GLYPHS = { + vllm: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M3 4l7 16 7-16"/><path d="M14 4l4 9 3-9"/></svg>', + sglang: '<svg width="14" height="14" viewBox="0 0 24 24" fill="currentColor" stroke="none" aria-hidden="true"><polygon points="13 2 3 14 12 14 11 22 21 10 12 10 13 2"/></svg>', + llamacpp: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="9"/><path d="M8 12h8M12 8v8"/></svg>', + ollama: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M6 10a6 6 0 0 1 12 0v4a4 4 0 0 1-8 0v-1"/><circle cx="10" cy="9" r="1"/><circle cx="14" cy="9" r="1"/></svg>', + diffusers: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="4"/><path d="M12 2v3M12 19v3M2 12h3M19 12h3M5 5l2 2M17 17l2 2M5 19l2-2M17 7l2-2"/></svg>', + }; + + // ── Custom Backend picker wiring ──────────────────────────────── + // Reads the option list from the hidden <select.hwfit-backend-source> + // so the canonical (value, label) pairs come from one place. + const _backendPicker = panel.querySelector('[data-backend-picker]'); + const _backendSource = panel.querySelector('.hwfit-backend-source'); + const _backendBtn = panel.querySelector('[data-backend-btn]'); + const _backendMenu = panel.querySelector('[data-backend-menu]'); + const _backendBtnLabel = panel.querySelector('[data-backend-label]'); + const _backendBtnIconSlot = _backendBtn?.querySelector('[data-backend-icon-slot]'); + + function _setBackendBtnState(v) { + if (!_backendBtn) return; + const opt = _backendSource?.querySelector(`option[value="${CSS.escape(v)}"]`); + const label = opt ? opt.textContent : v; + if (_backendBtnLabel) _backendBtnLabel.textContent = label; + if (_backendBtnIconSlot) _backendBtnIconSlot.innerHTML = _BACKEND_GLYPHS[v] || _BACKEND_GLYPHS.vllm; + } + + function _renderBackendMenu() { + if (!_backendMenu || !_backendSource) return; + const items = Array.from(_backendSource.options).map(o => ({ value: o.value, label: o.textContent })); + _backendMenu.innerHTML = items.map(it => ` + <button type="button" role="option" class="hwfit-backend-item" data-value="${it.value}" style="all:unset;display:flex;align-items:center;gap:8px;width:100%;padding:6px 9px;border-radius:5px;font-size:12px;cursor:pointer;color:var(--fg);box-sizing:border-box;"> + <span class="hwfit-backend-item-icon" style="display:inline-flex;align-items:center;justify-content:center;width:14px;height:14px;color:var(--accent, var(--red));flex-shrink:0;">${_BACKEND_GLYPHS[it.value] || _BACKEND_GLYPHS.vllm}</span> + <span class="hwfit-backend-item-label" style="flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">${it.label}</span> + </button> + `).join(''); + // Hover styling (no global CSS rule — keep it self-contained). + _backendMenu.querySelectorAll('.hwfit-backend-item').forEach(btn => { + btn.addEventListener('mouseenter', () => { btn.style.background = 'color-mix(in srgb, var(--fg) 8%, transparent)'; }); + btn.addEventListener('mouseleave', () => { btn.style.background = ''; }); + btn.addEventListener('click', (ev) => { + ev.preventDefault(); + ev.stopPropagation(); + const v = btn.dataset.value; + if (_backendSource && _backendSource.value !== v) { + _backendSource.value = v; + _backendSource.dispatchEvent(new Event('change', { bubbles: true })); + } + _setBackendBtnState(v); + _closeBackendMenu(); + }); + }); + } + + function _openBackendMenu() { + if (!_backendMenu || !_backendBtn) return; + _backendMenu.hidden = false; + _backendBtn.setAttribute('aria-expanded', 'true'); + } + function _closeBackendMenu() { + if (!_backendMenu || !_backendBtn) return; + _backendMenu.hidden = true; + _backendBtn.setAttribute('aria-expanded', 'false'); + } + if (_backendBtn) { + _backendBtn.addEventListener('click', (ev) => { + ev.preventDefault(); + ev.stopPropagation(); + if (_backendMenu.hidden) _openBackendMenu(); + else _closeBackendMenu(); + }); + document.addEventListener('click', (ev) => { + if (!_backendMenu.hidden && !_backendPicker?.contains(ev.target)) _closeBackendMenu(); + }); + document.addEventListener('keydown', (ev) => { + if (ev.key === 'Escape' && !_backendMenu.hidden) { + ev.stopPropagation(); + _closeBackendMenu(); + } + }, { capture: true }); + } + _renderBackendMenu(); + _setBackendBtnState(_backendSource?.value || defaultBackend); + function updateBackendVisibility() { const b = panel.querySelector('[data-field="backend"]')?.value || 'vllm'; panel.querySelectorAll('[class*="hwfit-backend-"]').forEach(el => { + // Skip the entire backend-picker subtree — the picker's own + // classes (`hwfit-backend-picker`, `-btn`, `-menu`, `-item`, + // `-btn-icon`, `-btn-label`, `-item-icon`, `-item-label`) all + // match the wildcard and would get hidden as if they were + // "backend-specific form sections", which left the dropdown + // looking empty / collapsed. + if (el.closest('.hwfit-backend-picker')) return; const show = el.classList.contains(`hwfit-backend-${b}`); el.style.display = show ? '' : 'none'; }); + _setBackendBtnState(b); } updateBackendVisibility(); async function updateRuntimeReadinessNote() { const note = panel.querySelector('.hwfit-serve-runtime-note'); if (!note) return; + // Mirror the message into a small chip next to the model title at + // the top of the card, so the readiness state is visible without + // having to look down into the panel body. + // Clean up any title chip from previous versions — the readiness + // text now lives inside the panel at the top, not in the card title. + const card = panel.closest('.doclib-card, .memory-item'); + const titleEl = card ? card.querySelector('.memory-item-title') : null; + const titleChip = titleEl ? titleEl.querySelector('.hwfit-serve-runtime-chip') : null; + if (titleChip) titleChip.remove(); const backend = panel.querySelector('[data-field="backend"]')?.value || 'vllm'; + const noteText = note.querySelector('.hwfit-serve-runtime-text'); + const _writeNote = (s) => { if (noteText) noteText.textContent = s; else note.textContent = s; }; if (!['vllm', 'sglang', 'llamacpp', 'diffusers'].includes(backend)) { note.style.display = 'none'; - note.textContent = ''; + _writeNote(''); return; } + // Wire dismiss once per note element. + const _closeBtn = note.querySelector('.hwfit-serve-runtime-close'); + if (_closeBtn && !_closeBtn._wired) { + _closeBtn._wired = true; + _closeBtn.addEventListener('click', (ev) => { + ev.preventDefault(); + ev.stopPropagation(); + note.style.display = 'none'; + panel._runtimeNoteDismissed = true; + }); + } + // If the user dismissed it earlier on this panel, don't re-show. + if (panel._runtimeNoteDismissed) return; const seq = (panel._runtimeReadinessSeq || 0) + 1; panel._runtimeReadinessSeq = seq; note.style.display = ''; - note.textContent = 'Checking runtime on selected server...'; + _writeNote('Checking runtime on selected server…'); + note.style.borderColor = ''; + note.style.color = 'var(--fg-muted)'; try { const { pkg, target } = await _fetchServeRuntimePackage(panel, backend); if (panel._runtimeReadinessSeq !== seq) return; - note.textContent = _runtimeNoteText(backend, pkg, target); - note.style.color = pkg?.installed ? 'var(--fg-muted)' : 'var(--red)'; + _writeNote(_runtimeNoteText(backend, pkg, target)); + if (!pkg?.installed) { + note.style.color = 'var(--red)'; + note.style.borderColor = 'color-mix(in srgb, var(--red) 40%, transparent)'; + note.style.background = 'color-mix(in srgb, var(--red) 8%, transparent)'; + // Append an accent-color link straight to the Dependencies + // recipe panel for this backend so the user has one click + // to the fix instead of hunting for the right row. + if (noteText) { + const pkgName = pkg?.name || ({ vllm: 'vllm', sglang: 'sglang', llamacpp: 'llama_cpp', diffusers: 'diffusers' }[backend]); + const repo = (panel.closest('.doclib-card, .memory-item')?.dataset?.repo) || ''; + const link = document.createElement('a'); + link.href = '#'; + link.textContent = ' Install in Dependencies →'; + link.style.cssText = 'color:var(--accent, var(--red));text-decoration:underline;font-weight:600;margin-left:4px;'; + link.addEventListener('click', (ev) => { + ev.preventDefault(); + if (pkgName) openCookbookDependencies(pkgName, { expandRecipe: pkgName, model: repo }); + }); + noteText.appendChild(link); + } + } else { + // Healthy / ready → green so the user reads "good to go" at a + // glance instead of scanning fg-muted for a state. + note.style.color = 'var(--green, #4caf50)'; + note.style.borderColor = 'color-mix(in srgb, var(--green, #4caf50) 40%, transparent)'; + note.style.background = 'color-mix(in srgb, var(--green, #4caf50) 8%, transparent)'; + } } catch (err) { if (panel._runtimeReadinessSeq !== seq) return; - note.textContent = `Runtime readiness unavailable: ${err?.message || err}`; + _writeNote(`Runtime readiness unavailable: ${err?.message || err}`); note.style.color = 'var(--fg-muted)'; } } @@ -1688,15 +1894,39 @@ function _rerenderCachedModels() { // Cancel button — collapses the serve config panel (same effect as // tapping the row to toggle it shut). Mobile users wanted an explicit // "back out" affordance next to Launch. - panel.querySelector('.hwfit-serve-cancel')?.addEventListener('click', (ev) => { - ev.stopPropagation(); + const _collapsePanel = () => { panel._cleanupRuntimeReadiness?.(); panel.remove(); item.classList.remove('doclib-card-expanded'); item.style.flexDirection = ''; item.style.alignItems = ''; if (list) { list.style.minHeight = ''; list.style.maxHeight = ''; } + }; + panel.querySelector('.hwfit-serve-cancel')?.addEventListener('click', (ev) => { + ev.stopPropagation(); + _collapsePanel(); }); + // Esc anywhere on the page closes the open serve panel. Skips when + // the user is typing in a field — they want Esc to deselect / blur + // those, not collapse the form they're configuring. + const _onEscClose = (ev) => { + if (ev.key !== 'Escape') return; + if (!panel.isConnected) { + document.removeEventListener('keydown', _onEscClose, true); + return; + } + const t = ev.target; + const inField = t && ( + t.tagName === 'INPUT' || t.tagName === 'TEXTAREA' || t.tagName === 'SELECT' || t.isContentEditable + ); + if (inField) return; + // Skip when one of the dropdown/menu popovers is open — the + // popovers handle their own Esc and use stopPropagation, so any + // Esc that bubbles here means nothing else claimed it. + ev.stopPropagation(); + _collapsePanel(); + }; + document.addEventListener('keydown', _onEscClose, true); // Launch button panel.querySelector('.hwfit-serve-launch').addEventListener('click', async (ev) => { @@ -1751,6 +1981,56 @@ function _rerenderCachedModels() { else serveState[el.dataset.field] = el.value; }); serveState.backend = serveState.backend || (_detectBackend(m).backend) || 'vllm'; + const launchTarget = _selectedServeTarget(panel); + if (serveState.backend === 'diffusers' && _remoteWindowsDiffusersUnsupported(launchTarget)) { + _restoreLaunchBtn(); + uiModule.showToast('Diffusers serving is not supported on remote Windows servers yet. Use local Windows or a Linux server.', 9000); + return; + } + + // Pre-launch: check our own task list for a serve already running + // on this host. Offer to stop+launch as the default action — the + // SSH-based port probe below is more thorough but it can miss + // when SSH glitches or `ss` isn't installed. This catches the + // common case instantly without waiting for a network round-trip. + try { + const _runningMod = await import('./cookbookRunning.js'); + const _hostStr = launchTarget.host || ''; + const _active = (_runningMod._loadTasks ? _runningMod._loadTasks() : []).filter(t => + t && t.type === 'serve' + && (t.remoteHost || '') === _hostStr + && (t.status === 'running' || t.status === 'ready' || t._serveReady) + ); + if (_active.length) { + const _names = _active.map(t => t.payload?.repo_id || t.repo || t.name || '?').filter(Boolean); + const _ok = await window.styledConfirm( + `${_active.length} model${_active.length === 1 ? '' : 's'} already serving on ${_hostStr || 'local'} (${_names.join(', ')}). Port 8000 will collide. Stop the running model and launch this one?`, + { title: 'Server already running', confirmText: 'Stop & launch', cancelText: 'Cancel' }, + ); + if (!_ok) { _restoreLaunchBtn(); return; } + // Kill each active serve; prefer the rendered Stop button so + // endpoint cleanup + Ollama unload run normally. Fall back to + // a raw tmux kill when the Active tab isn't in the DOM. + for (const t of _active) { + try { + const _el = document.querySelector(`.cookbook-task[data-task-id="${t.sessionId}"]`); + const _btn = _el?.querySelector('.cookbook-task-action-stop'); + if (_btn) { + _btn.click(); + } else if (_runningMod._tmuxGracefulKill) { + await fetch('/api/shell/exec', { + method: 'POST', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ command: _runningMod._tmuxGracefulKill(t) }), + }); + } + } catch (_killErr) { /* best-effort */ } + } + // Give the OS a beat to release port 8000. + await new Promise(r => setTimeout(r, 2500)); + } + } catch (_e) { /* best-effort */ } + const backendWarning = _serveBackendWarning(m, repo, serveState.backend, serveState); if (backendWarning) { _restoreLaunchBtn(); @@ -1769,12 +2049,11 @@ function _rerenderCachedModels() { || (serveState.backend === 'diffusers'); if (_needsGpu) { try { - const _probeHost = (_envState.remoteHost || '').trim(); + const _probeHost = (launchTarget.host || '').trim(); const _probeParams = new URLSearchParams(); if (_probeHost) { _probeParams.set('host', _probeHost); - const _sp = (_serverByVal?.(_envState.remoteServerKey || _probeHost) || {}).port; - if (_sp) _probeParams.set('ssh_port', _sp); + if (launchTarget.port) _probeParams.set('ssh_port', launchTarget.port); } const _probeRes = await fetch('/api/cookbook/gpus' + (_probeParams.toString() ? '?' + _probeParams : ''), { credentials: 'same-origin' }); const _probeData = await _probeRes.json(); @@ -1807,10 +2086,10 @@ function _rerenderCachedModels() { || launchCmd.match(/OLLAMA_HOST=[^:\s]+:(\d{2,5})\b/); const _port = _portMatch ? _portMatch[1] : ''; if (_port) { - const _portHost = (_envState.remoteHost || '').trim(); + const _portHost = (launchTarget.host || '').trim(); const _checkInner = `ss -tlnp 2>/dev/null | awk '$4 ~ /:${_port}$/ {print; exit}' || netstat -tlnp 2>/dev/null | awk '$4 ~ /:${_port}$/ {print; exit}'`; const _cmd = _portHost - ? `ss h ${_portHost} <<<"" 2>/dev/null; ssh -o ConnectTimeout=4 -o StrictHostKeyChecking=no ${_portHost} ${JSON.stringify(_checkInner)}` + ? `ssh -o ConnectTimeout=4 -o StrictHostKeyChecking=no ${_sshPrefix(launchTarget.port)}${_portHost} ${JSON.stringify(_checkInner)}` : _checkInner; const _res = await fetch('/api/shell/exec', { method: 'POST', credentials: 'same-origin', @@ -1867,20 +2146,8 @@ function _rerenderCachedModels() { // Resolve the target host from the visible Server dropdown — the reliable // source. Relying on _envState.remoteHost silently sent serves to Local // when that value was stale/empty. Pass it explicitly to the launcher. - let serveHost = _envState.remoteHost || ''; - let _srvEnv = '', _srvEnvPath = ''; - const _ssEl = document.getElementById('hwfit-server-select') || document.getElementById('hwfit-dl-server'); - if (_ssEl && _ssEl.value != null) { - if (_ssEl.value === 'local') serveHost = ''; - else { - const _srv = _serverByVal?.(_ssEl.value) || _envState.servers[parseInt(_ssEl.value)]; - if (_srv) { - serveHost = _srv.host; - _srvEnv = _srv.env || ''; - _srvEnvPath = _srv.envPath || ''; - } - } - } + let serveHost = launchTarget.host || ''; + let _srvEnv = launchTarget.env || '', _srvEnvPath = launchTarget.venv || ''; // The venv field wins; otherwise fall back to the env configured for the // selected server in Settings, so the activation isn't silently dropped // when the field is left blank (the per-server venv wasn't being applied). diff --git a/static/js/document.js b/static/js/document.js index 86ecf2880..20d77788a 100644 --- a/static/js/document.js +++ b/static/js/document.js @@ -87,7 +87,8 @@ import * as Modals from './modalManager.js'; } function _accountCanSend(account) { - return !!(account && account.smtp_host && account.smtp_user && account.has_smtp_password); + if (!account || !account.smtp_host || !account.smtp_user) return false; + return !!(account.has_smtp_password || account.oauth_provider); } async function _resolveComposeSendAccountId() { @@ -2472,6 +2473,8 @@ import * as Modals from './modalManager.js'; } // Hide toolbar items that have no clean WYSIWYG equivalent in email (Code). document.querySelectorAll('.md-toolbar-email-hide').forEach(el => { el.style.display = 'none'; }); + // Show email-only toolbar items (AI reply button). + document.querySelectorAll('.md-toolbar-email-only').forEach(el => { el.style.display = 'inline-flex'; }); if (emailHeader) emailHeader.style.display = ''; if (emailActions) emailActions.style.display = ''; // Emails have their own complete footer (Close / More / Send), so hide the @@ -2864,6 +2867,8 @@ import * as Modals from './modalManager.js'; if (emailActions) emailActions.style.display = 'none'; // Restore toolbar items that were hidden for email (Code dropdown). document.querySelectorAll('.md-toolbar-email-hide').forEach(el => { el.style.display = ''; }); + // Re-hide email-only toolbar items (AI reply button). + document.querySelectorAll('.md-toolbar-email-only').forEach(el => { el.style.display = 'none'; }); // Restore the generic documents action bar + its bottom footer (Close / // Copy / Export) for non-email docs. const docActions = document.getElementById('doc-editor-actions'); @@ -3206,7 +3211,95 @@ import * as Modals from './modalManager.js'; renderTabs(); } - async function _aiReply() { + // Fast/Full + optional context popover for the doc-editor email Reply button. + // Mirrors the email reader's AI reply choice popover so the UX is identical: + // textarea for an optional steering note, then Fast (lightning) or Full + // (concentric dot) buttons; both feed into _aiReply with the chosen mode. + let _docAiReplyChoiceMenu = null; + function _closeDocAiReplyChoice() { + if (_docAiReplyChoiceMenu) { + try { _docAiReplyChoiceMenu.remove(); } catch (_) {} + _docAiReplyChoiceMenu = null; + } + } + function _showDocAiReplyChoice(btn) { + _closeDocAiReplyChoice(); + if (!btn) return; + const rect = btn.getBoundingClientRect(); + const menu = document.createElement('div'); + menu.className = 'doc-ai-reply-choice'; + const menuMaxW = Math.min(240, window.innerWidth - 16); + const left = Math.max(8, Math.min(rect.left, window.innerWidth - menuMaxW - 8)); + const estHeight = 150; + const spaceBelow = window.innerHeight - rect.bottom - 8; + const spaceAbove = rect.top - 8; + const top = (spaceBelow >= estHeight || spaceBelow >= spaceAbove) + ? Math.max(8, Math.min(rect.bottom + 6, window.innerHeight - estHeight - 8)) + : Math.max(8, rect.top - estHeight - 6); + menu.style.cssText = [ + 'position:fixed', + `left:${left}px`, + `top:${top}px`, + `max-width:${menuMaxW}px`, + 'box-sizing:border-box', + 'z-index:10060', + 'display:flex', + 'gap:6px', + 'padding:6px', + 'background:var(--bg,#111)', + 'border:1px solid var(--border,#333)', + 'border-radius:7px', + 'box-shadow:0 8px 24px rgba(0,0,0,.28)', + ].join(';'); + menu.innerHTML = ` + <div style="display:flex;flex-direction:column;gap:6px;min-width:200px;"> + <textarea data-note-input rows="2" placeholder="Add context (optional)" style="width:100%;box-sizing:border-box;resize:vertical;min-height:42px;font-family:inherit;font-size:11px;padding:5px 6px;border-radius:5px;border:1px solid var(--border,#333);background:var(--bg-elev,#1a1a1a);color:var(--fg);"></textarea> + <div style="display:flex;align-items:center;gap:4px;"> + <button class="memory-toolbar-btn" data-mode="ai-reply-fast" title="Shorter, faster draft" style="display:inline-flex;align-items:center;justify-content:center;gap:5px;flex:1;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="var(--accent, var(--red))" aria-hidden="true"><polygon points="13 2 3 14 12 14 11 22 21 10 12 10 13 2"/></svg> + Fast + </button> + <button class="memory-toolbar-btn" data-mode="ai-reply-full" title="Fuller reply with more context" style="display:inline-flex;align-items:center;justify-content:center;gap:5px;flex:1;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true" style="color:var(--accent, var(--red));"><circle cx="12" cy="12" r="6"/></svg> + Full + </button> + </div> + </div> + `; + const noteInput = menu.querySelector('[data-note-input]'); + setTimeout(() => noteInput?.focus(), 0); + menu.addEventListener('mousedown', (ev) => ev.stopPropagation()); + menu.addEventListener('click', async (ev) => { + const choice = ev.target.closest('[data-mode]'); + if (!choice) return; + ev.preventDefault(); + ev.stopPropagation(); + const mode = choice.getAttribute('data-mode') || 'ai-reply-fast'; + const noteHint = (noteInput?.value || '').trim(); + _closeDocAiReplyChoice(); + await _aiReply({ mode, noteHint }); + }); + document.body.appendChild(menu); + _docAiReplyChoiceMenu = menu; + const outsideClose = (ev) => { + if (menu.contains(ev.target)) return; + document.removeEventListener('click', outsideClose, true); + _closeDocAiReplyChoice(); + }; + setTimeout(() => document.addEventListener('click', outsideClose, true), 0); + // Esc to close. + const escClose = (ev) => { + if (ev.key === 'Escape') { + ev.stopPropagation(); + document.removeEventListener('keydown', escClose, true); + _closeDocAiReplyChoice(); + } + }; + document.addEventListener('keydown', escClose, true); + } + + async function _aiReply(opts = {}) { + const { mode = 'auto', noteHint = '' } = (opts || {}); const to = document.getElementById('doc-email-to')?.value?.trim() || ''; const subject = document.getElementById('doc-email-subject')?.value?.trim() || ''; const textarea = document.getElementById('doc-editor-textarea'); @@ -3251,32 +3344,43 @@ import * as Modals from './modalManager.js'; if (btn) { btn.disabled = true; btn.innerHTML = '<svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;margin-right:3px"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>Drafting...'; } try { + // Empty-compose path: if there's no original body, send a placeholder + // so the backend's "no body" guard doesn't fail. The user_hint carries + // the user's compose intent; the model uses To/Subject + that hint. + const bodyForApi = currentBody || (noteHint ? '(no prior email — compose a new message based on the To, Subject, and user instructions)' : currentBody); + const fastFlag = mode === 'ai-reply-fast' ? true + : mode === 'ai-reply-full' ? false + : shouldUseFastAiReply(); const res = await fetch(`${API_BASE}/api/email/ai-reply`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ to: to, subject: subject, - original_body: currentBody, + original_body: bodyForApi, model: currentModel, session_id: currentSessionId, message_id: inReplyTo, uid: sourceUid, folder: sourceFolder, - fast: shouldUseFastAiReply(), + fast: fastFlag, + user_hint: noteHint || '', }), }); const data = await res.json(); if (data.success && data.reply) { - const cleanReply = cleanAiReplyText(data.reply); - const lines = currentBody.split('\n'); - const quoteIdx = lines.findIndex(l => l.startsWith('On ') && l.includes(' wrote:')); - let newBody = ''; - if (quoteIdx > 0) { - newBody = cleanReply + '\n\n' + lines.slice(quoteIdx).join('\n'); - } else { - newBody = cleanReply + (currentBody ? '\n\n' + currentBody : ''); - } + let cleanReply = cleanAiReplyText(data.reply); + // Strip any "On <date>, <name> wrote:" attribution + everything + // after it from the AI's output — the model sometimes re-quotes + // the original thread, and we already have the real quote in + // currentBody. Without this, AI's invented quote stacked on top + // of the real one and looked like the history had been "edited". + cleanReply = cleanReply.replace(/\n*On\b[\s\S]*?\bwrote:[\s\S]*$/m, '').trim(); + // Never overwrite the existing draft (user's typed text + the + // quoted history below it). Always prepend the AI suggestion so + // the user can read it, copy parts, or delete it — but their + // own work and the original quote are untouched. + const newBody = currentBody ? cleanReply + '\n\n' + currentBody : cleanReply; await _streamEmailBodyText(textarea, newBody); if (uiModule) uiModule.showToast(`AI draft inserted (${data.model_used || 'AI'})`); } else { @@ -3285,7 +3389,7 @@ import * as Modals from './modalManager.js'; } catch (e) { if (uiModule) uiModule.showError('Failed to generate AI reply'); } finally { - if (btn) { btn.disabled = false; btn.innerHTML = '<svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;margin-right:3px"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>AI Reply'; } + if (btn) { btn.disabled = false; btn.innerHTML = '<svg width="12" height="12" viewBox="0 0 24 24" fill="currentColor" style="color:var(--accent, var(--red));flex-shrink:0;position:relative;top:-1px;"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg><span style="font-size:11px;margin-left:4px;">Reply</span>'; } } } @@ -3813,7 +3917,6 @@ import * as Modals from './modalManager.js'; <button id="doc-export-pdf-btn" class="doc-action-icon-btn" title="Export PDF" style="display:none;opacity:0.7;gap:4px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"/><polyline points="14 2 14 8 20 8"/><line x1="12" y1="18" x2="12" y2="12"/><polyline points="9 15 12 18 15 15"/></svg> <span style="font-size:11px;">Export PDF</span></button> <button id="doc-pdf-view-btn" class="doc-action-icon-btn" title="Toggle PDF view" style="display:none;opacity:0.7;gap:4px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"/><polyline points="14 2 14 8 20 8"/></svg> <span style="font-size:11px;">PDF</span></button> <select id="doc-language-select" class="doc-language-select"> - <option value="">type</option> <option value="python">python</option> <option value="javascript">javascript</option> <option value="typescript">typescript</option> @@ -3851,22 +3954,24 @@ import * as Modals from './modalManager.js'; </button> <div id="doc-email-fields" class="doc-email-fields"> <div class="email-field" style="position:relative"> - <label>To</label> + <span class="email-field-prefix">To</span> <input type="text" id="doc-email-to" placeholder="recipient@example.com" autocomplete="off" /> <div id="doc-email-to-suggestions" class="email-autocomplete" style="display:none"></div> <button type="button" id="doc-email-show-cc" class="email-cc-toggle" title="Show Cc/Bcc">Cc</button> </div> <div class="email-field" id="doc-email-cc-row" style="display:none;position:relative"> - <label>Cc</label> - <input type="text" id="doc-email-cc" placeholder="cc@example.com" autocomplete="off" /> + <span class="email-field-prefix">Cc</span> + <input type="text" id="doc-email-cc" placeholder="cc@example.com, example2" autocomplete="off" /> <div id="doc-email-cc-suggestions" class="email-autocomplete" style="display:none"></div> + <button type="button" class="email-cc-close" data-cc-close title="Hide Cc/Bcc" aria-label="Hide Cc/Bcc"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg></button> </div> <div class="email-field" id="doc-email-bcc-row" style="display:none;position:relative"> - <label>Bcc</label> + <span class="email-field-prefix">Bcc</span> <input type="text" id="doc-email-bcc" placeholder="bcc@example.com" autocomplete="off" /> <div id="doc-email-bcc-suggestions" class="email-autocomplete" style="display:none"></div> + <button type="button" class="email-cc-close" data-cc-close title="Hide Cc/Bcc" aria-label="Hide Cc/Bcc"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg></button> </div> - <div class="email-field"><label>Subject</label><input type="text" id="doc-email-subject" placeholder="Subject" /></div> + <div class="email-field" style="position:relative"><span class="email-field-prefix">Subject</span><input type="text" id="doc-email-subject" placeholder="" /></div> <div id="doc-email-attachments" class="email-attachments" style="display:none"></div> <div id="doc-email-compose-atts" class="email-compose-atts" style="display:none"></div> </div> @@ -3879,13 +3984,14 @@ import * as Modals from './modalManager.js'; <div class="doc-md-toolbar" id="doc-md-toolbar" style="display:none"> <div class="md-toolbar-items" id="md-toolbar-items"> <span class="md-view-toggle" id="doc-md-view-toggle" style="display:none" role="group" aria-label="Edit or preview"> - <button type="button" class="md-view-opt" data-mdview="edit" title="Edit source"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.12 2.12 0 0 1 3 3L12 15l-4 1 1-4 9.5-9.5z"/></svg></button> - <button type="button" class="md-view-opt" data-mdview="preview" title="Preview"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"/><circle cx="12" cy="12" r="3"/></svg></button> + <button type="button" class="md-view-opt" data-mdview="edit" title="Edit source (Ctrl+Alt+M to toggle)"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.12 2.12 0 0 1 3 3L12 15l-4 1 1-4 9.5-9.5z"/></svg></button> + <button type="button" class="md-view-opt" data-mdview="preview" title="Preview (Ctrl+Alt+M to toggle)"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"/><circle cx="12" cy="12" r="3"/></svg></button> </span> <span class="md-view-toggle" id="doc-render-view-toggle" style="display:none" role="group" aria-label="Code or run"> <button type="button" class="md-view-opt" data-renderview="code" title="Edit code"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="16 18 22 12 16 6"/><polyline points="8 6 2 12 8 18"/></svg></button> <button type="button" class="md-view-opt" data-renderview="run" title="Run / Preview"><svg width="13" height="13" viewBox="0 0 24 24" fill="currentColor" stroke="none"><polygon points="5 3 19 12 5 21 5 3"/></svg></button> </span> + <button id="doc-email-ai-reply-btn" class="doc-action-icon-btn md-toolbar-email-only" type="button" title="Draft a reply with AI (Fast / Full + optional context)" style="display:none;align-items:center;gap:4px;"><svg width="12" height="12" viewBox="0 0 24 24" fill="currentColor" style="color:var(--accent, var(--red));flex-shrink:0;position:relative;top:-1px;"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg><span style="font-size:11px;">Reply</span></button> <button id="doc-fontsize-btn" class="doc-action-icon-btn" title="Font size" style="position:relative;width:28px;height:26px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="opacity:0.7;"><path d="M4 7V4h16v3"/><path d="M12 4v16"/><path d="M8 20h8"/></svg><span class="doc-fontsize-levels"><i data-sz="s">S</i><i data-sz="m">M</i><i data-sz="l">L</i></span></button> <button id="doc-diff-toggle-btn" class="doc-action-icon-btn" title="Compare changes" style="opacity:0.7;display:none;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 3v18"/><path d="M5 12H2l5-5 5 5H9"/><path d="M19 12h3l-5 5-5-5h3"/></svg></button> <span class="md-toolbar-sep"></span> @@ -4395,6 +4501,24 @@ import * as Modals from './modalManager.js'; } }); } + // Ctrl+Alt+M (and Cmd+Opt+M on mac) flips Edit ↔ Preview on a markdown + // doc. Bound once globally; gated on the doc panel being open and the + // active doc being markdown so it doesn't fire while the user is typing + // in a non-markdown context. + if (!window._docMdToggleBound) { + window._docMdToggleBound = true; + document.addEventListener('keydown', (e) => { + if ((e.ctrlKey || e.metaKey) && e.altKey && !e.shiftKey && (e.key === 'm' || e.key === 'M' || e.code === 'KeyM')) { + if (!isOpen) return; + const doc = activeDocId && docs.get(activeDocId); + const lang = (doc?.language || 'markdown').toLowerCase(); + if (lang !== 'markdown') return; + e.preventDefault(); + toggleMarkdownPreview(); + _syncHeaderActions(); + } + }); + } document.getElementById('doc-email-draft-btn')?.addEventListener('click', () => { document.getElementById('doc-email-more-menu').style.display = 'none'; _saveDraft(); @@ -4409,7 +4533,11 @@ import * as Modals from './modalManager.js'; document.getElementById('doc-email-more-menu').style.display = 'none'; _scheduleSend(anchor); }); - document.getElementById('doc-email-ai-reply-btn')?.addEventListener('click', _aiReply); + document.getElementById('doc-email-ai-reply-btn')?.addEventListener('click', (ev) => { + ev.preventDefault(); + ev.stopPropagation(); + _showDocAiReplyChoice(ev.currentTarget); + }); const collapseBtn = document.getElementById('doc-email-collapse-btn'); if (collapseBtn && !collapseBtn._emailCollapseWired) { @@ -4489,6 +4617,25 @@ import * as Modals from './modalManager.js'; _syncEmailHeaderSummary(); }); + // Cc/Bcc close — X buttons inside the Cc and Bcc fields hide both + // rows + clear their inputs + restore the Cc opener on the To row. + document.querySelectorAll('[data-cc-close]').forEach(closeBtn => { + closeBtn.addEventListener('click', (ev) => { + ev.stopPropagation(); + const ccRow = document.getElementById('doc-email-cc-row'); + const bccRow = document.getElementById('doc-email-bcc-row'); + const ccInput = document.getElementById('doc-email-cc'); + const bccInput = document.getElementById('doc-email-bcc'); + if (ccRow) ccRow.style.display = 'none'; + if (bccRow) bccRow.style.display = 'none'; + if (ccInput) ccInput.value = ''; + if (bccInput) bccInput.value = ''; + const ccToggle = document.getElementById('doc-email-show-cc'); + if (ccToggle) ccToggle.style.display = ''; + _syncEmailHeaderSummary(); + }); + }); + // Autocomplete for To / Cc / Bcc — typed fragment after the last // comma triggers contact search; Enter / Tab / click on a suggestion // appends "<email>, " so the user can keep typing more recipients. @@ -8527,6 +8674,19 @@ import * as Modals from './modalManager.js'; // `body:has(.doc-editor-pane.doc-fullscreen) .doc-divider-collapse` slides // it into a forced-inside position). Hiding the divider here would hide // the chevron with it. + + // Hide the tab bar during the layout shift so any in-flight smooth + // scroll / reflow doesn't visibly "fly" the active tab across the + // pane as it expands. Restored after the layout settles. + const tabBar = document.getElementById('doc-tab-bar'); + if (tabBar) { + tabBar.style.visibility = 'hidden'; + clearTimeout(tabBar._fsHideTimer); + tabBar._fsHideTimer = setTimeout(() => { + tabBar.style.visibility = ''; + }, 240); + } + if (pane.classList.contains('doc-fullscreen')) { pane.classList.remove('doc-fullscreen'); if (container) container.style.display = ''; diff --git a/static/js/emailInbox.js b/static/js/emailInbox.js index 8ca1a6a3c..1b4d67a4e 100644 --- a/static/js/emailInbox.js +++ b/static/js/emailInbox.js @@ -22,8 +22,8 @@ const _replyIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" const _archiveIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="2" y="3" width="20" height="5" rx="1"/><path d="M4 8v11a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8"/><path d="M10 12h4"/></svg>'; const _deleteIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 6h18"/><path d="M8 6V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"/><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6"/></svg>'; const _unreadIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><circle cx="12" cy="12" r="3" fill="currentColor"/></svg>'; -const _starIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>'; -const _starFilledIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="currentColor" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>'; +const _starIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M19 21l-7-5-7 5V5a2 2 0 0 1 2-2h10a2 2 0 0 1 2 2z"/></svg>'; +const _starFilledIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="currentColor" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M19 21l-7-5-7 5V5a2 2 0 0 1 2-2h10a2 2 0 0 1 2 2z"/></svg>'; const _bellIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 8A6 6 0 0 0 6 8c0 7-3 9-3 9h18s-3-2-3-9"/><path d="M13.73 21a2 2 0 0 1-3.46 0"/></svg>'; const _icon = (svg) => `<span class="dropdown-icon">${svg}</span>`; const _replySeparator = '---------- Previous message ----------'; @@ -74,6 +74,11 @@ window.addEventListener('email-answered', (e) => { item.classList.remove('email-unread'); const check = item.querySelector('.email-done-check'); if (check) check.classList.add('active'); + // Auto-mark from sending a reply — flash the row so the user sees the + // state change without staring at it. Class self-removes after the + // animation so it doesn't replay on re-renders. + item.classList.add('email-auto-done-flash'); + setTimeout(() => item.classList.remove('email-auto-done-flash'), 1200); }); }); let _loading = false; @@ -113,19 +118,19 @@ export function init(documentModule) { } catch (_) {} if (opts.compose) { _composeNew(); return; } if (opts.email) { - await _openEmail(opts.email, null, opts.emailData, opts.mode || 'reply'); + await _openEmail(opts.email, null, opts.emailData, opts.mode || 'reply', opts.noteHint || ''); } }, }); _watchDocOpenToReDockEmail(); } -export async function openReplyDraft(uid, folder = 'INBOX', mode = 'reply') { +export async function openReplyDraft(uid, folder = 'INBOX', mode = 'reply', prefilledBody = '') { if (!uid) return; const previousFolder = _currentFolder; _currentFolder = folder || 'INBOX'; try { - await _openEmail({ uid: String(uid), subject: '' }, null, null, mode || 'reply'); + await _openEmail({ uid: String(uid), subject: '' }, null, null, mode || 'reply', '', prefilledBody || ''); } finally { _currentFolder = previousFolder || _currentFolder; } @@ -525,11 +530,6 @@ function _createEmailItem(em) { </div> <div class="email-subject">${_esc(em.subject)}${unreadIcon}${attachIcon}${tagPills}${spamTag}</div> </div> - <div class="email-menu-wrap"> - <button class="hamburger email-menu-btn" title="Actions"> - <svg width="14" height="14" viewBox="0 0 24 24" fill="currentColor"><circle cx="12" cy="5" r="2"/><circle cx="12" cy="12" r="2"/><circle cx="12" cy="19" r="2"/></svg> - </button> - </div> `; // Click sender name → filter list to that sender @@ -562,17 +562,10 @@ function _createEmailItem(em) { // Click to open — do NOT close sidebar item.addEventListener('click', (e) => { - if (e.target.closest('.email-menu-wrap')) return; if (item.dataset.swipeBlock === '1') return; _openEmail(em, item); }); - const menuWrap = item.querySelector('.email-menu-wrap'); - menuWrap.addEventListener('click', (e) => { - e.stopPropagation(); - _showEmailMenu(em, menuWrap, item); - }); - // Swipe left to archive (mobile). Mirrors sidebar-layout.js swipe pattern. if ('ontouchstart' in window) { let startX = 0, startY = 0, dx = 0, dy = 0, swiping = false, swiped = false; @@ -580,7 +573,6 @@ function _createEmailItem(em) { const VERT_CANCEL = 30; // px vertical motion cancels swipe (treat as scroll) item.addEventListener('touchstart', (e) => { - if (e.target.closest('.email-menu-wrap')) return; const t = e.touches[0]; startX = t.clientX; startY = t.clientY; dx = 0; dy = 0; swiping = true; swiped = false; @@ -638,10 +630,13 @@ function _createEmailItem(em) { return item; } -async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply') { +async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply', noteHint = '', prefilledBody = '') { const aiReplyMode = mode === 'ai-reply-fast' ? 'fast' : (mode === 'ai-reply-full' ? 'full' : ''); const wantsAiReply = mode === 'ai-reply' || !!aiReplyMode; - let aiSuggestedBody = null; + // Body pre-fill from the agent's open_email_reply tool call takes the + // same insertion slot as an AI-suggested body — both land just before + // the quoted-original block. + let aiSuggestedBody = (typeof prefilledBody === 'string' && prefilledBody.trim()) ? prefilledBody.trim() : null; if (wantsAiReply) { // Fall through to reply-all (not plain reply) so the generated AI // draft addresses everyone on the original thread. On single- @@ -698,6 +693,7 @@ async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply') { uid: String(em.uid || ''), folder: _currentFolder, fast: aiReplyMode ? aiReplyMode === 'fast' : _shouldUseFastAiReply(data), + user_hint: (noteHint || '').trim() || undefined, }), }); const result = await res.json(); diff --git a/static/js/emailLibrary.js b/static/js/emailLibrary.js index 4dd2f720d..7beb6a122 100644 --- a/static/js/emailLibrary.js +++ b/static/js/emailLibrary.js @@ -22,6 +22,7 @@ import { _tryFoldHintSig, _foldSignature, _SIG_ICON, _QUOTE_ICON, } from './emailLibrary/signatureFold.js'; import { state } from './emailLibrary/state.js'; +import { collapseSidebarToRail } from './modalSnap.js'; const API_BASE = window.location.origin; let _emailUnreadChipClickWired = false; @@ -29,6 +30,7 @@ let _libLoadSeq = 0; let _libFolderSeq = 0; let _libSearchSeq = 0; let _libSearchHadResults = false; +let _libSearchInFlight = false; let _activeEmailReaderForSelectAll = null; function _isEmailTypingTarget(t) { @@ -61,6 +63,52 @@ function _markEmailReaderActive(reader) { reader.addEventListener('focusin', () => { _activeEmailReaderForSelectAll = reader; }, true); } +// Stash the email identity (uid + folder + account) on the reader element +// so chat submits and other code paths can ask "what email is the user +// currently looking at?" without re-deriving from the DOM hierarchy. +function _stampReaderContext(reader, em, folder, account) { + if (!reader || !em) return; + reader.dataset.emailUid = String(em.uid || ''); + reader.dataset.emailFolder = String(folder || state._libFolder || 'INBOX'); + reader.dataset.emailAccount = String(account || state._libAccountId || ''); + if (em.subject) reader.dataset.emailSubject = String(em.subject); + if (em.from_address || em.from_name) { + reader.dataset.emailFrom = String(em.from_address || em.from_name); + } +} + +// Returns { uid, folder, account, subject, from } for the email the user +// is most likely referring to — the last reader they interacted with, then +// any open reader-modal as a fallback. Returns null when no email reader +// is open. Exported below for chat.js to read on submit. +function _getActiveEmailContext() { + const candidates = []; + if (_activeEmailReaderForSelectAll && _activeEmailReaderForSelectAll.isConnected) { + candidates.push(_activeEmailReaderForSelectAll); + } + // Visible reader-tab modals (popped-out windows). + document.querySelectorAll('.modal[id^="email-reader-"]:not(.hidden):not(.modal-minimized) .email-card-reader').forEach(el => candidates.push(el)); + // Expanded inline reader in the library list. + document.querySelectorAll('#email-lib-modal:not(.hidden) .doclib-card.email-card-expanded .email-card-reader').forEach(el => candidates.push(el)); + for (const r of candidates) { + const uid = r?.dataset?.emailUid; + if (uid) { + return { + uid, + folder: r.dataset.emailFolder || 'INBOX', + account: r.dataset.emailAccount || '', + subject: r.dataset.emailSubject || '', + from: r.dataset.emailFrom || '', + }; + } + } + return null; +} + +// Frontend reads via the global so chat.js doesn't need a separate import +// path (emailLibrary loads lazily in some entry points). +try { window.__odysseusGetActiveEmailContext = _getActiveEmailContext; } catch (_) {} + const _COPY_EMAIL_ICON = '<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg>'; function _decodeAttrValue(v) { @@ -128,6 +176,20 @@ async function _copyTextToClipboard(text) { } } +function _wireMetaToggle(root) { + const toggle = root && root.querySelector('.email-reader-meta-toggle'); + const details = root && root.querySelector('.email-reader-meta-details'); + if (!toggle || !details) return; + toggle.addEventListener('click', (ev) => { + ev.stopPropagation(); + const open = details.hasAttribute('hidden'); + if (open) details.removeAttribute('hidden'); + else details.setAttribute('hidden', ''); + toggle.setAttribute('aria-expanded', String(open)); + toggle.classList.toggle('open', open); + }); +} + function _recipientChipHtml(full, label, extraClass = '') { const fullText = String(full || '').trim(); const addr = _emailAddressFromRecipientText(fullText); @@ -234,9 +296,9 @@ function _syncEmailReadState(uid, isRead = true) { dot.className = 'email-card-unread-dot'; dot.style.cssText = `width:6px;height:6px;border-radius:50%;background:${_senderColor(senderName)};flex-shrink:0;margin-left:2px;`; const done = titleRow.querySelector('.email-card-done'); - const rightCluster = titleRow.querySelector('.email-card-header-menu')?.parentElement; + const navArrows = titleRow.querySelector('.email-card-nav-arrows'); if (done) done.insertAdjacentElement('afterend', dot); - else if (rightCluster) titleRow.insertBefore(dot, rightCluster); + else if (navArrows) titleRow.insertBefore(dot, navArrows); else titleRow.appendChild(dot); }); } @@ -300,14 +362,8 @@ function _renderAccountsLoading() { try { const wp = spinnerModule.createWhirlpool(14); wp.element.classList.add('email-accounts-loading-whirlpool'); - const label = document.createElement('span'); - label.className = 'email-accounts-loading-label'; - label.textContent = 'Accounts'; strip.appendChild(wp.element); - strip.appendChild(label); - } catch (_) { - strip.textContent = 'Accounts...'; - } + } catch (_) {} } function _syncEmailReminderBellVisibility(enabled) { @@ -326,6 +382,12 @@ async function _loadEmailReminderBellVisibility() { _syncEmailReminderBellVisibility(false); } } +// Live-update the bell when the reminder channel changes in Settings, +// so the user doesn't have to reopen Email to see the change apply. +window.addEventListener('odysseus-reminder-channel-changed', (e) => { + const ch = e?.detail?.channel; + _syncEmailReminderBellVisibility(ch === 'email'); +}); function _readCssPx(name) { const v = getComputedStyle(document.documentElement).getPropertyValue(name); @@ -406,7 +468,14 @@ function _clearEmailDocumentSplit() { ].forEach(prop => docPane.style.removeProperty(prop)); } -function _hasDesktopRoomForEmailAndDocument(modal) { +// Compute the left-edge x assuming the wide sidebar has collapsed to the +// rail. Used by the "try collapsing the sidebar first" path so we can decide +// whether collapsing recovers enough room before minimizing email. +function _emailSplitLeftEdgeIfSidebarCollapsed() { + return _readCssPx('--icon-rail-w'); +} + +function _hasDesktopRoomForEmailAndDocument(modal, opts = {}) { if (window.innerWidth <= 768) return false; if (window.innerWidth >= 1100) return true; const content = modal?.querySelector?.('.modal-content'); @@ -416,18 +485,31 @@ function _hasDesktopRoomForEmailAndDocument(modal) { const emailWidth = isFullscreen ? Math.min(440, Math.max(360, Math.round(window.innerWidth * 0.30))) : Math.max(360, Math.round(rect?.width || 440)); - const docMinWidth = 560; - const breathingRoom = 72; - const leftEdge = isFullscreen ? _emailSplitLeftEdge() : Math.max(0, Math.round(rect?.left || _emailSplitLeftEdge())); + // Relaxed thresholds — the old 560 + 72 forced an unnecessary tab-down + // on ~1200–1300px viewports where there was visually plenty of room. + const docMinWidth = 460; + const breathingRoom = 40; + const leftEdgeNow = isFullscreen ? _emailSplitLeftEdge() : Math.max(0, Math.round(rect?.left || _emailSplitLeftEdge())); + const leftEdge = opts.assumeSidebarCollapsed ? _emailSplitLeftEdgeIfSidebarCollapsed() : leftEdgeNow; return (window.innerWidth - leftEdge - emailWidth) >= (docMinWidth + breathingRoom); } function _prepareEmailWindowForDocument(modal) { if (window.innerWidth <= 768) return true; if (!modal) return false; + // Try to make breathing room by collapsing the wide sidebar to the rail + // when there isn't enough horizontal space for both panes. The + // route-collapse marker that collapseSidebarToRail() sets means the + // sidebar will auto-restore when the doc closes. Crucially, we no + // longer fall back to clearing the split when even that isn't enough — + // the user opted out of auto-tab-down, so we proceed with the dock + // even if it's cramped. if (!_hasDesktopRoomForEmailAndDocument(modal)) { - _clearEmailDocumentSplit(); - return true; + const sidebar = document.getElementById('sidebar'); + const sidebarWasOpen = sidebar && !sidebar.classList.contains('hidden'); + if (sidebarWasOpen && _hasDesktopRoomForEmailAndDocument(modal, { assumeSidebarCollapsed: true })) { + try { collapseSidebarToRail(); } catch (_) {} + } } if (modal.classList.contains('modal-left-docked')) { const content = modal.querySelector('.modal-content'); @@ -675,7 +757,7 @@ async function _prewarmDefaultEmailView() { } catch (_) {} const accountQS = accountId ? `&account_id=${encodeURIComponent(accountId)}` : ''; - const res = await fetch(`${API_BASE}/api/email/list?folder=${encodeURIComponent(folder)}${accountQS}&limit=100&offset=0&filter=${filter}`, { + const res = await fetch(`${API_BASE}/api/email/list?folder=${encodeURIComponent(folder)}${accountQS}&limit=500&offset=0&filter=${filter}`, { credentials: 'same-origin', }); if (!res.ok) return; @@ -752,6 +834,13 @@ export function openEmailLibrary(opts = {}) { state._libEmails = []; state._libOffset = 0; state._libSearch = ''; + state._libSearchDraft = ''; + // Reset select-mode on each open so the toolbar Select button + // never opens already-toggled-on after a previous session. + state._selectMode = false; + if (state._selectedUids) state._selectedUids.clear(); + state._libSearchPills = []; + _libSuggestionCache = null; state._libFilter = 'all'; state._libHasAttachments = false; // Animate the very first card render with a domino cascade (same as the @@ -786,7 +875,6 @@ export function openEmailLibrary(opts = {}) { </div> <div class="modal-body" style="display:flex;flex-direction:column;gap:10px;overflow:hidden;"> <div class="admin-card" style="flex:1;flex-direction:column;display:flex;overflow:hidden;"> - <p class="memory-desc doclib-desc">All emails. Click to open as a document.</p> <div class="email-accounts-row"> <div id="email-lib-accounts" style="display:flex;gap:4px;flex:1;min-width:0;"></div> <button class="memory-toolbar-btn email-compose-jiggle" id="email-lib-compose-btn"> @@ -799,7 +887,10 @@ export function openEmailLibrary(opts = {}) { <select class="memory-sort-select" id="email-lib-folder" style="flex:1;min-width:0;text-overflow:ellipsis;"> <option value="INBOX">Inbox</option> </select> - <select class="memory-sort-select" id="email-lib-filter" style="flex:1;min-width:0;"> + <!-- Hidden native select kept as the source of truth — all + existing change handlers still fire via the custom picker + dispatching 'change' on it. --> + <select class="memory-sort-select" id="email-lib-filter" style="display:none;"> <option value="all">All</option> <option value="unread">Unread</option> <option value="favorites">Favorites</option> @@ -816,7 +907,14 @@ export function openEmailLibrary(opts = {}) { <option value="tag:marketing">Marketing</option> </optgroup> </select> - <button class="memory-toolbar-btn email-filter-select-btn" id="email-lib-select-btn">Select</button> + <div class="email-filter-picker" id="email-filter-picker" style="flex:1;min-width:0;position:relative;"> + <button type="button" class="email-filter-btn" id="email-filter-btn" aria-haspopup="listbox" aria-expanded="false"> + <span class="email-filter-current"><span class="email-filter-icon"></span><span class="email-filter-label">All</span></span> + <svg class="email-filter-caret" width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg> + </button> + <div class="email-filter-menu" id="email-filter-menu" role="listbox" hidden></div> + </div> + <button class="memory-toolbar-btn email-filter-select-btn" id="email-lib-select-btn"><svg class="memory-select-btn-icon" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:3px;"><circle cx="12" cy="12" r="10"/><circle cx="12" cy="12" r="3" fill="currentColor" stroke="none"/></svg>Select</button> <button class="memory-toolbar-btn email-filter-refresh-btn" id="email-lib-refresh-btn" title="Refresh"> <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-1px;"><path d="M1 4v6h6"/><path d="M23 20v-6h-6"/><path d="M20.49 9A9 9 0 0 0 5.64 5.64L1 10m22 4l-4.64 4.36A9 9 0 0 1 3.51 15"/></svg> </button> @@ -827,7 +925,12 @@ export function openEmailLibrary(opts = {}) { </div> <div class="email-search-row" style="display:flex;gap:6px;align-items:flex-start;"> <div class="email-search-wrap" style="position:relative;flex:1;min-width:140px;"> - <input type="text" id="email-lib-search" placeholder="Search emails\u2026" class="memory-search-input" style="width:100%;padding-right:96px;" /> + <div class="email-lib-chip-bar memory-search-input" id="email-lib-chip-bar" style="width:100%;padding-right:96px;padding-left:26px;display:flex;align-items:center;flex-wrap:wrap;gap:4px;cursor:text;min-height:30px;position:relative;"> + <svg class="email-lib-chip-bar-icon" width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="position:absolute;left:8px;top:50%;transform:translateY(-50%);pointer-events:none;color:var(--accent, var(--red));"><circle cx="11" cy="11" r="7"/><path d="M21 21l-4.35-4.35"/></svg> + <span id="email-lib-pills" style="display:contents"></span> + <input type="text" id="email-lib-search" placeholder="Search by name or text" autocomplete="off" style="flex:1;min-width:80px;border:0;outline:none;background:transparent;color:inherit;font:inherit;padding:0;position:relative;top:-1px;" /> + </div> + <div id="email-lib-suggest" style="display:none;position:absolute;top:calc(100% + 2px);left:0;right:0;z-index:60;background:var(--panel,var(--bg));border:1px solid var(--border);border-radius:6px;box-shadow:0 6px 18px rgba(0,0,0,0.25);max-height:240px;overflow-y:auto;"></div> <button class="memory-toolbar-btn email-undone-toggle email-undone-toggle-inline" id="email-undone-btn" title="Show only emails not marked as done (undone)"> <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg> </button> @@ -841,7 +944,7 @@ export function openEmailLibrary(opts = {}) { </div> </div> <div id="email-lib-bulk" class="memory-bulk-bar hidden" style="margin-bottom:5px;"> - <label class="memory-bulk-check-all" style="position:relative;top:2px;"><input type="checkbox" id="email-lib-select-all"> All</label> + <label class="memory-bulk-check-all" style="position:relative;top:0px;"><input type="checkbox" id="email-lib-select-all"> All</label> <span id="email-lib-selected-count" style="position:relative;top:1px;">0 Selected</span> <button class="memory-toolbar-btn" id="email-lib-bulk-actions" style="position:relative;top:-2px;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-1px;margin-right:3px;"><line x1="8" y1="6" x2="21" y2="6"/><line x1="8" y1="12" x2="21" y2="12"/><line x1="8" y1="18" x2="21" y2="18"/><line x1="3" y1="6" x2="3.01" y2="6"/><line x1="3" y1="12" x2="3.01" y2="12"/><line x1="3" y1="18" x2="3.01" y2="18"/></svg>Actions <span style="opacity:0.55;font-size:9px;">▼</span></button> <button class="memory-toolbar-btn" id="email-lib-bulk-delete" style="position:relative;top:-2px;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-1px;margin-right:3px;"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/></svg>Delete</button> @@ -970,7 +1073,10 @@ export function openEmailLibrary(opts = {}) { // Sync quick-toggle active states so they mirror the dropdown. document.getElementById('email-undone-btn')?.classList.toggle('active', state._libFilter === 'undone'); document.getElementById('email-reminder-btn')?.classList.toggle('active', state._libFilter === 'reminders'); + // Mirror the picker label/icon. + _renderFilterPickerCurrent(); }); + _initFilterPicker(); document.getElementById('email-attach-btn')?.addEventListener('click', () => { const btn = document.getElementById('email-attach-btn'); state._libHasAttachments = !state._libHasAttachments; @@ -1048,12 +1154,11 @@ export function openEmailLibrary(opts = {}) { // \Flagged search). _libSort stays at its 'recent' default so the grid keeps // the API's newest-first order. - let searchTimer = null; - document.getElementById('email-lib-search').addEventListener('input', (e) => { - state._libSearch = e.target.value; - if (searchTimer) clearTimeout(searchTimer); - searchTimer = setTimeout(_doSearch, 350); - }); + // Chip-bar search: pills represent contact + free-text filters; the live + // input below drives the autocomplete dropdown. Old behavior — instant + // local filter on every keystroke + server-side IMAP search after 350ms + // — is replaced by deterministic local filtering against the snapshot. + _initEmailSearchChipBar(); document.getElementById('email-lib-refresh-btn').addEventListener('click', async () => { const btn = document.getElementById('email-lib-refresh-btn'); @@ -1179,10 +1284,20 @@ export function openEmailLibrary(opts = {}) { } } - // Select mode toggle + // Select mode toggle — icon + label swap matches the brain memories + // select button (dot+Select ↔ X+Cancel). + const _SELECT_BTN_DOT_SVG = '<svg class="memory-select-btn-icon" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:3px;"><circle cx="12" cy="12" r="10"/><circle cx="12" cy="12" r="3" fill="currentColor" stroke="none"/></svg>'; + const _SELECT_BTN_X_SVG = '<svg class="memory-select-btn-icon" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" style="vertical-align:-2px;margin-right:3px;"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>'; + const _setSelectBtnState = (on) => { + const btn = document.getElementById('email-lib-select-btn'); + if (!btn) return; + if (on) { btn.classList.add('active'); btn.innerHTML = _SELECT_BTN_X_SVG + 'Cancel'; } + else { btn.classList.remove('active'); btn.innerHTML = _SELECT_BTN_DOT_SVG + 'Select'; } + }; document.getElementById('email-lib-select-btn').addEventListener('click', () => { state._selectMode = !state._selectMode; state._selectedUids.clear(); + _setSelectBtnState(state._selectMode); _updateBulkBar(); _renderGrid(); }); @@ -1202,6 +1317,7 @@ export function openEmailLibrary(opts = {}) { document.getElementById('email-lib-bulk-cancel')?.addEventListener('click', () => { state._selectMode = false; state._selectedUids.clear(); + _setSelectBtnState(false); _updateBulkBar(); _renderGrid(); }); @@ -1284,10 +1400,15 @@ export function openEmailLibrary(opts = {}) { document.addEventListener('keydown', state._libEscHandler, true); _renderAccountsLoading(); - _loadAccounts(); - _loadFolders(); - _loadEmailReminderBellVisibility(); - _loadEmails(); + // Await accounts before loading emails so the list request carries the + // right account_id from the very first fetch (now that we auto-select + // an explicit account instead of relying on a 'Default' chip). + (async () => { + await _loadAccounts(); + _loadFolders(); + _loadEmailReminderBellVisibility(); + _loadEmails(); + })(); } async function _loadAccounts() { @@ -1297,6 +1418,15 @@ async function _loadAccounts() { const d = await r.json(); state._libAccounts = d.accounts || []; } catch (_) { state._libAccounts = []; } + // The 'Default' chip is gone — pick an explicit account so the email + // list and any per-email actions (open in new tab, mark read, etc.) + // always carry an account_id and can't desync from the server's + // is_default state. + if (!state._libAccountId && state._libAccounts.length) { + const def = state._libAccounts.find(a => a.is_default) || state._libAccounts[0]; + state._libAccountId = def.id; + _publishActiveAccount(); + } _renderAccountsStrip(); } @@ -1305,12 +1435,23 @@ function _renderAccountsStrip() { if (!strip) return; strip.style.display = 'flex'; const esc = s => String(s || '').replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); - const allActive = !state._libAccountId ? ' active' : ''; - let html = `<button class="memory-toolbar-btn gallery-chip${allActive}" data-acc-id="">All (default)</button>`; + // The 'Default' chip caused desync bugs (changing the server-side + // default via the dot while still on the cached 'default' view would + // open the wrong account's emails). Each account renders as its own + // chip; the active one is selected explicitly via _loadAccounts. + let html = ''; + // 6px dot — matches the sidebar notification-dot size. + const _dotFilled = '<svg width="6" height="6" viewBox="0 0 24 24" fill="currentColor"><circle cx="12" cy="12" r="10"/></svg>'; + const _dotHollow = '<svg width="6" height="6" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3"><circle cx="12" cy="12" r="9"/></svg>'; for (const a of state._libAccounts) { const active = state._libAccountId === a.id ? ' active' : ''; const label = a.name || a.from_address || a.imap_user || 'account'; - html += `<button class="memory-toolbar-btn gallery-chip${active}" data-acc-id="${esc(a.id)}" title="${esc(a.from_address || a.imap_user || '')}${a.is_default ? ' (default)' : ''}">${esc(label)}</button>`; + const dot = a.is_default ? _dotFilled : _dotHollow; + const dotTitle = a.is_default ? 'Default account' : 'Set as default'; + html += `<span class="gallery-chip-wrap" style="position:relative;display:inline-flex;align-items:center;">` + + `<button class="memory-toolbar-btn gallery-chip${active}" data-acc-id="${esc(a.id)}" title="${esc(a.from_address || a.imap_user || '')}${a.is_default ? ' (default)' : ''}" style="padding-right:24px;">${esc(label)}</button>` + + `<button class="email-lib-default-dot${a.is_default ? ' is-default' : ''}" data-set-default="${esc(a.id)}" title="${dotTitle}" aria-label="${dotTitle}" style="position:absolute;right:6px;top:calc(50% - 3px);transform:translateY(-50%);background:none;border:0;padding:0;width:18px;height:18px;cursor:pointer;color:${a.is_default ? 'var(--accent, var(--red))' : 'inherit'};opacity:${a.is_default ? '1' : '0.45'};display:inline-flex;align-items:center;justify-content:center;line-height:0;">${dot}</button>` + + `</span>`; } strip.innerHTML = html; strip.querySelectorAll('button[data-acc-id]').forEach(btn => { @@ -1323,6 +1464,72 @@ function _renderAccountsStrip() { _loadEmails({ force: true, useCache: false }); }); }); + // Star handler: POST set-default, then reload accounts + re-render so + // the chip stars reflect the new default. + strip.querySelectorAll('button[data-set-default]').forEach(btn => { + btn.addEventListener('click', async (e) => { + e.stopPropagation(); + const acctId = btn.dataset.setDefault; + if (!acctId) return; + try { + await fetch(`${API_BASE}/api/email/accounts/${encodeURIComponent(acctId)}/set-default`, { + method: 'POST', credentials: 'same-origin', + }); + // Refresh the local accounts cache and re-render the strip. + for (const a of state._libAccounts) a.is_default = (a.id === acctId); + _renderAccountsStrip(); + } catch (err) { + console.error('Set default account failed:', err); + } + }); + }); + // Idempotent — wire wheel + grab-drag scroll once per strip element. + if (!strip._scrollWired) { + strip._scrollWired = true; + // Vertical wheel → horizontal scroll. Only intercept when there's + // actually horizontal overflow to scroll through, otherwise let the + // page do its normal vertical scroll. + strip.addEventListener('wheel', (e) => { + if (strip.scrollWidth <= strip.clientWidth) return; + if (Math.abs(e.deltaY) <= Math.abs(e.deltaX)) return; + e.preventDefault(); + strip.scrollLeft += e.deltaY; + }, { passive: false }); + // Click-and-drag scroll. Track mousedown, then mousemove deltas + // bump scrollLeft. Cancel a chip click if the user actually dragged + // more than a few pixels. + let dragging = false; + let startX = 0; + let startScroll = 0; + let moved = 0; + strip.style.cursor = 'grab'; + strip.addEventListener('mousedown', (e) => { + if (e.button !== 0) return; + dragging = true; + moved = 0; + startX = e.pageX; + startScroll = strip.scrollLeft; + strip.style.cursor = 'grabbing'; + strip.style.userSelect = 'none'; + }); + window.addEventListener('mousemove', (e) => { + if (!dragging) return; + const dx = e.pageX - startX; + moved = Math.max(moved, Math.abs(dx)); + strip.scrollLeft = startScroll - dx; + }); + window.addEventListener('mouseup', () => { + if (!dragging) return; + dragging = false; + strip.style.cursor = 'grab'; + strip.style.userSelect = ''; + }); + // Swallow chip clicks fired after a real drag — the user meant to scroll, + // not select. + strip.addEventListener('click', (e) => { + if (moved > 5) { e.stopPropagation(); e.preventDefault(); moved = 0; } + }, true); + } _publishActiveAccount(); } @@ -1416,6 +1623,12 @@ function _makeDraggable(content, modal, fsClass) { function _snapEmailModalToLeftSidebar(modal) { if (!modal) return false; if (window.innerWidth < 900) return false; + // "Open in new tab" reader modals (id="email-view-…") are explicitly + // floating windows the user already positioned. Replying from one + // shouldn't yank it to the left edge — leave it on top in its current + // spot. Reply still opens the compose document; the user can drag the + // reader away or close it themselves. + if ((modal.id || '').startsWith('email-view-')) return false; const content = modal.querySelector('.modal-content'); if (!content) return false; // Only dock if currently fullscreen — for a manually-sized window the @@ -1525,6 +1738,609 @@ function _crossFolderCandidates() { return Array.from(new Set(candidates.filter(Boolean))); } +// Snapshot of state._libEmails taken right before search starts so we +// can both filter locally and restore on clear without re-fetching. +let _libPreSearchEmails = null; +let _libPreSearchTotal = 0; + +// Cached contact suggestions for the chip-input autocomplete. Built on +// first focus / first keystroke from contacts + currently-loaded senders. +let _libSuggestionCache = null; +let _libSuggestionFocusIdx = 0; + +async function _buildSuggestionSource() { + // Combine the contacts list with senders/recipients visible in the + // loaded email list. Dedup by lowercased email address; prefer + // contact-supplied display names where present. + const map = new Map(); + const _add = (name, email) => { + const key = String(email || '').trim().toLowerCase(); + if (!key) return; + const prev = map.get(key); + if (!prev || (name && !prev.name)) { + map.set(key, { name: (name || '').trim(), email: key }); + } + }; + // 1) Senders / recipients already in the loaded grid. + for (const em of (state._libEmails || [])) { + _add(em.from_name, em.from_address); + const _parse = (s) => String(s || '').split(',').forEach(seg => { + const m = seg.match(/^\s*"?([^"<]*)"?\s*<?([^>]+)>?\s*$/); + if (m) _add(m[1], m[2]); + }); + _parse(em.to); + _parse(em.cc); + } + // 2) Address book — best-effort. + try { + const r = await fetch(`${API_BASE}/api/contacts/list`, { credentials: 'same-origin' }); + if (r.ok) { + const d = await r.json(); + for (const c of (d.contacts || [])) { + const email = c.email || (c.emails && c.emails[0]) || ''; + _add(c.name || c.full_name, email); + } + } + } catch (_) {} + return Array.from(map.values()).filter(x => x.email); +} + +function _scoreSuggestion(s, needle) { + // Crude relevance: startsWith on name or email wins big; substring is fine. + const n = (s.name || '').toLowerCase(); + const e = (s.email || '').toLowerCase(); + if (n.startsWith(needle) || e.startsWith(needle)) return 3; + if (n.includes(needle) || e.includes(needle)) return 2; + return 0; +} + +// Filter / attachment suggestions surfaced inside the same chip-bar +// dropdown. Typing 'attachment', 'unread', 'urgent' etc. surfaces the +// corresponding filter row with its icon; picking it pins a filter +// pill that drives state._libFilter or the has-attachments toggle. +const _LIB_FILTER_OPTIONS = [ + { value: 'filter:has-attachments', label: 'Has attachments', keywords: ['attachment', 'attachments', 'has attachment', 'attach'] }, + { value: 'filter:unread', label: 'Unread', keywords: ['unread', 'new', 'unseen'] }, + { value: 'filter:favorites', label: 'Favorites', keywords: ['favorite', 'favorites', 'starred', 'star', 'flagged'] }, + { value: 'filter:undone', label: 'Undone', keywords: ['undone', 'pending', 'todo'] }, + { value: 'filter:reminders', label: 'Reminders', keywords: ['reminder', 'reminders'] }, + { value: 'filter:unanswered', label: 'Unanswered', keywords: ['unanswered', 'unreplied', 'no reply'] }, + { value: 'filter:pending_30d', label: 'Pending · 30d', keywords: ['pending 30d', 'pending', 'recent pending'] }, + { value: 'filter:stale_30d', label: 'Stale · >30d', keywords: ['stale', 'old', 'stale 30d'] }, + { value: 'filter:tag:urgent', label: 'Urgent', keywords: ['urgent', 'critical'] }, + { value: 'filter:tag:reply-soon', label: 'Reply soon', keywords: ['reply soon', 'reply', 'follow up'] }, + { value: 'filter:tag:spam', label: 'Spam', keywords: ['spam', 'junk'] }, + { value: 'filter:tag:newsletter', label: 'Newsletter', keywords: ['newsletter', 'newsletters', 'subscriptions'] }, + { value: 'filter:tag:marketing', label: 'Marketing', keywords: ['marketing', 'promo', 'promotional'] }, +]; + +function _libFilterIconFor(value) { + // value is 'filter:<X>' — strip prefix and reuse the existing icon map. + const v = String(value || '').replace(/^filter:/, ''); + if (v === 'has-attachments') return '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="m21.44 11.05-9.19 9.19a6 6 0 0 1-8.49-8.49l8.57-8.57A4 4 0 1 1 17.93 8.8l-8.59 8.57a2 2 0 0 1-2.83-2.83l8.49-8.48"/></svg>'; + return _EMAIL_FILTER_ICONS[v] || _EMAIL_FILTER_ICONS['all']; +} + +function _scoreFilterOption(opt, needle) { + for (const kw of opt.keywords) { + if (kw === needle) return 4; + if (kw.startsWith(needle)) return 3; + if (kw.includes(needle)) return 2; + } + if (opt.label.toLowerCase().includes(needle)) return 2; + return 0; +} + +function _filterSuggestions(needle, limit = 10) { + const n = String(needle || '').trim().toLowerCase(); + if (!n) return []; + // Filter / attachment matches first — typing 'unread' should surface + // the filter row before contact suggestions, since 'unread' isn't a + // person. + const filterMatches = _LIB_FILTER_OPTIONS + .map(opt => ({ s: { kind: 'filter', value: opt.value, label: opt.label, icon: _libFilterIconFor(opt.value) }, score: _scoreFilterOption(opt, n) })) + .filter(x => x.score > 0); + const src = _libSuggestionCache || []; + const contactMatches = src + .map(s => ({ s: { kind: 'contact', ...s }, score: _scoreSuggestion(s, n) })) + .filter(x => x.score > 0); + // Email subject / sender-name matches — use the snapshot (unfiltered + // list) when available so suggestions don't shrink as pills narrow the + // visible grid. Cap to 4 so contacts + filters stay visible. + const emails = _libPreSearchEmails || state._libEmails || []; + const emailMatches = []; + for (const em of emails) { + const subj = String(em.subject || '').toLowerCase(); + const fromN = String(em.from_name || '').toLowerCase(); + let score = 0; + if (subj.startsWith(n) || fromN.startsWith(n)) score = 3; + else if (subj.includes(n) || fromN.includes(n)) score = 1; + if (score > 0) emailMatches.push({ s: { kind: 'email', uid: em.uid, subject: em.subject || '(no subject)', from_name: em.from_name || em.from_address || '' }, score }); + if (emailMatches.length >= 4) break; + } + return filterMatches.concat(contactMatches).concat(emailMatches) + .sort((a, b) => b.score - a.score) + .slice(0, limit) + .map(x => x.s); +} + +function _emailMatchesPill(em, pill) { + if (!pill) return false; + if (pill.type === 'contact') { + const target = (pill.email || '').toLowerCase(); + if (!target) return false; + if (String(em.from_address || '').toLowerCase() === target) return true; + if (String(em.to || '').toLowerCase().includes(target)) return true; + if (String(em.cc || '').toLowerCase().includes(target)) return true; + return false; + } + if (pill.type === 'filter') { + // Filter pills delegate to the server-side filter (state._libFilter) + // or the has-attachments toggle. The list is already pre-filtered by + // those when this runs, so the pill is effectively always-true here + // — it lives in the pill bar purely as a visible affordance. + return true; + } + // text pill — broad local-match + const q = (pill.text || '').toLowerCase(); + if (!q) return true; + return _matchesQuery(em, q); +} + +function _matchesQuery(em, q) { + const needle = q.toLowerCase(); + return ( + String(em.subject || '').toLowerCase().includes(needle) || + String(em.from_name || '').toLowerCase().includes(needle) || + String(em.from_address || '').toLowerCase().includes(needle) || + String(em.snippet || em.preview || '').toLowerCase().includes(needle) + ); +} + +// Apply the active pill filter to the snapshot. Each pill is OR-ed; an +// email shows up if ANY pill matches (a contact pill matches by from/to/cc +// equality, a text pill matches by the broad _matchesQuery substring). +function _applyPillFilter() { + const pills = state._libSearchPills || []; + const draft = (state._libSearchDraft || '').trim(); + const noPills = pills.length === 0; + const noDraft = draft.length === 0; + // First time we apply with anything active: snapshot the loaded list. + if (!noPills || draft.length >= 1) { + if (!_libPreSearchEmails) { + _libPreSearchEmails = (state._libEmails || []).slice(); + _libPreSearchTotal = state._libTotal; + } + } + if (noPills && noDraft) { + if (_libPreSearchEmails) { + state._libEmails = _libPreSearchEmails; + state._libTotal = _libPreSearchTotal; + _libPreSearchEmails = null; + _libPreSearchTotal = 0; + } + _renderGrid(); + return; + } + const source = _libPreSearchEmails || state._libEmails || []; + // If the active server search covers a piece of text (either the live + // draft OR an Enter-committed text pill), skip the local re-filter for + // it — _emailMatchesPill only checks subject/from_name/from_address/ + // snippet (no BODY), so it was dropping legitimate server hits where + // the match was in body text. Real pills (contact, filter chips) still + // apply, and other text pills with different strings still apply. + const libSearchLower = (_libSearchHadResults ? (state._libSearch || '').trim().toLowerCase() : ''); + const serverHandledDraft = !!(libSearchLower && draft && libSearchLower === draft.toLowerCase()); + const draftPill = (!serverHandledDraft && draft.length >= 1) ? { type: 'text', text: draft } : null; + // Filter out text pills whose text matches the active server search — + // those were the trigger for the IMAP query and don't need re-checking. + const effectiveBasePills = libSearchLower + ? pills.filter(p => !(p.type === 'text' && (p.text || '').toLowerCase() === libSearchLower)) + : pills; + const effective = draftPill ? effectiveBasePills.concat([draftPill]) : effectiveBasePills; + // AND across pills — "alice + bob" should mean both alice AND bob are + // somewhere on the email (from/to/cc), not "from alice OR from bob". + const filtered = source.filter(em => effective.every(p => _emailMatchesPill(em, p))); + state._libEmails = filtered; + _renderGrid(); +} +// Back-compat shim: older call sites still expect _localSearchFilter. +function _localSearchFilter(query) { + state._libSearchDraft = String(query || ''); + _applyPillFilter(); +} + +// Render the active pills inside the chip bar. Each pill carries a × to +// remove individually. Backspace on empty input also pops the last one. +function _renderSearchPills() { + const wrap = document.getElementById('email-lib-pills'); + if (!wrap) return; + const pills = state._libSearchPills || []; + const esc = s => String(s || '').replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); + wrap.innerHTML = pills.map((p, i) => { + // Filter pills render as icon-only (the icon is the affordance); + // contact + text pills carry their label as text. + if (p.type === 'filter') { + const titleAttr = `${(p.label || p.value).replace(/"/g, '"')}`; + return `<span class="email-lib-pill" data-pill-idx="${i}" title="${titleAttr}" style="display:inline-flex;align-items:center;gap:2px;padding:0 4px 0 6px;border-radius:999px;background:color-mix(in srgb, var(--accent, var(--red)) 14%, transparent);color:var(--accent, var(--red));line-height:18px;height:18px;flex-shrink:0;"> + <span style="display:inline-flex;align-items:center;width:11px;height:11px;flex-shrink:0;">${_libFilterIconFor(p.value)}</span> + <button type="button" class="email-lib-pill-x" data-pill-idx="${i}" title="Remove" style="background:transparent;border:0;color:inherit;cursor:pointer;font-size:11px;line-height:1;padding:0 2px;opacity:0.7;position:relative;top:-4px;">×</button> + </span>`; + } + const label = p.type === 'contact' ? (p.name || p.email || '?') : (p.text || ''); + return `<span class="email-lib-pill" data-pill-idx="${i}" style="display:inline-flex;align-items:center;gap:3px;padding:0 4px 0 6px;border-radius:999px;background:color-mix(in srgb, var(--accent, var(--red)) 14%, transparent);color:var(--accent, var(--red));font-size:10px;line-height:18px;height:18px;font-weight:600;max-width:160px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;flex-shrink:0;"> + <span style="overflow:hidden;text-overflow:ellipsis;">${esc(label)}</span> + <button type="button" class="email-lib-pill-x" data-pill-idx="${i}" title="Remove" style="background:transparent;border:0;color:inherit;cursor:pointer;font-size:11px;line-height:1;padding:0 2px;opacity:0.7;position:relative;top:-4px;">×</button> + </span>`; + }).join(''); + wrap.querySelectorAll('.email-lib-pill-x').forEach(btn => { + btn.addEventListener('click', (e) => { + e.stopPropagation(); + const idx = Number(btn.dataset.pillIdx); + if (Number.isFinite(idx)) _removeSearchPillAt(idx); + }); + }); +} + +function _applyFilterPillSideEffect(pill) { + // Filter pills drive the existing has-attachments toggle / filter + // dropdown so the server returns the right list. Only one filter + // pill is active at a time (see _addSearchPill). + const sel = document.getElementById('email-lib-filter'); + const attachBtn = document.getElementById('email-attach-btn'); + if (pill.value === 'filter:has-attachments') { + if (!state._libHasAttachments) { + state._libHasAttachments = true; + if (attachBtn) attachBtn.classList.add('active'); + } + if (sel && sel.value !== 'all') { sel.value = 'all'; sel.dispatchEvent(new Event('change')); } + return; + } + // Any other filter pill — set the dropdown value, clear attachments + if (state._libHasAttachments) { + state._libHasAttachments = false; + if (attachBtn) attachBtn.classList.remove('active'); + } + if (sel) { + const v = pill.value.replace(/^filter:/, ''); + if (sel.value !== v) { sel.value = v; sel.dispatchEvent(new Event('change')); } + } +} + +function _clearFilterPillSideEffect() { + const sel = document.getElementById('email-lib-filter'); + const attachBtn = document.getElementById('email-attach-btn'); + if (state._libHasAttachments) { + state._libHasAttachments = false; + if (attachBtn) attachBtn.classList.remove('active'); + } + if (sel && sel.value !== 'all') { + sel.value = 'all'; sel.dispatchEvent(new Event('change')); + } +} + +function _addSearchPill(pill) { + if (!pill) return; + if (!Array.isArray(state._libSearchPills)) state._libSearchPills = []; + // Dedup by email (contact), text (text pill), or filter value. + if (pill.type === 'contact') { + const key = (pill.email || '').toLowerCase(); + if (!key) return; + if (state._libSearchPills.some(p => p.type === 'contact' && (p.email || '').toLowerCase() === key)) return; + } else if (pill.type === 'text') { + const t = (pill.text || '').toLowerCase(); + if (!t) return; + if (state._libSearchPills.some(p => p.type === 'text' && (p.text || '').toLowerCase() === t)) return; + } else if (pill.type === 'filter') { + // Single-filter rule — drop any existing filter pill before adding. + state._libSearchPills = state._libSearchPills.filter(p => p.type !== 'filter'); + state._libSearchPills.push(pill); + _applyFilterPillSideEffect(pill); + _renderSearchPills(); + return; + } + state._libSearchPills.push(pill); + _renderSearchPills(); + _applyPillFilter(); +} + +function _removeSearchPillAt(idx) { + if (!Array.isArray(state._libSearchPills)) return; + const removed = state._libSearchPills[idx]; + state._libSearchPills.splice(idx, 1); + if (removed && removed.type === 'filter') _clearFilterPillSideEffect(); + _renderSearchPills(); + // Pill cleared all the way: if we got into search-result mode via the + // IMAP search, the pre-search snapshot is now those results too (set + // in _doSearch). Restoring from it would leave the user staring at + // the same results with the pill bar empty. Re-fetch the real inbox + // so removing the last pill genuinely "goes back". + const noPillsLeft = (state._libSearchPills || []).length === 0 + && !(state._libSearchDraft || '').trim(); + if (noPillsLeft && _libSearchHadResults) { + _libSearchHadResults = false; + _libPreSearchEmails = null; + _libPreSearchTotal = 0; + state._libSearch = ''; + state._libOffset = 0; + const _searchInput = document.getElementById('email-lib-search'); + if (_searchInput) _searchInput.value = ''; + _loadEmails({ useCache: true }); + return; + } + _applyPillFilter(); +} + +// Render the autocomplete dropdown below the input. focusIdx highlights +// the active row; Tab autocompletes / Enter accepts that row. +function _renderSearchSuggestions(items) { + const menu = document.getElementById('email-lib-suggest'); + if (!menu) return; + if (!items.length) { menu.style.display = 'none'; menu.innerHTML = ''; return; } + const esc = s => String(s || '').replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); + menu.innerHTML = items.map((s, i) => { + const highlight = i === _libSuggestionFocusIdx ? 'background:color-mix(in srgb, var(--fg) 8%, transparent);' : ''; + if (s.kind === 'filter') { + return `<div class="email-lib-suggest-item" data-idx="${i}" style="display:flex;align-items:center;gap:8px;padding:6px 10px;cursor:pointer;font-size:12px;${highlight}"> + <span style="display:inline-flex;align-items:center;width:13px;height:13px;color:var(--accent, var(--red));flex-shrink:0;">${s.icon}</span> + <span style="font-weight:600;">${esc(s.label)}</span> + </div>`; + } + if (s.kind === 'email') { + return `<div class="email-lib-suggest-item" data-idx="${i}" style="display:flex;align-items:center;gap:6px;padding:6px 10px;cursor:pointer;font-size:12px;${highlight}"> + <span style="display:inline-flex;align-items:center;width:13px;height:13px;color:var(--fg-muted, var(--fg));opacity:0.55;flex-shrink:0;"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="2" y="4" width="20" height="16" rx="2"/><polyline points="2 6 12 13 22 6"/></svg></span> + <span style="font-weight:600;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">${esc(s.subject)}</span> + ${s.from_name ? `<span style="opacity:0.55;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">— ${esc(s.from_name)}</span>` : ''} + </div>`; + } + return `<div class="email-lib-suggest-item" data-idx="${i}" style="display:flex;align-items:center;gap:6px;padding:6px 10px;cursor:pointer;font-size:12px;${highlight}"> + <span style="font-weight:600;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">${esc(s.name || s.email)}</span> + ${s.name ? `<span style="opacity:0.55;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">${esc(s.email)}</span>` : ''} + </div>`; + }).join(''); + menu.style.display = ''; + menu.querySelectorAll('.email-lib-suggest-item').forEach(row => { + row.addEventListener('mousedown', (e) => { + // mousedown (not click) so we beat the input blur handler that hides the menu. + e.preventDefault(); + const idx = Number(row.dataset.idx); + const item = items[idx]; + if (item) _acceptSuggestion(item); + }); + }); +} + +function _hideSearchSuggestions() { + const menu = document.getElementById('email-lib-suggest'); + if (menu) { menu.style.display = 'none'; menu.innerHTML = ''; } + _libSuggestionFocusIdx = 0; +} + +function _acceptSuggestion(s) { + const input = document.getElementById('email-lib-search'); + if (s.kind === 'filter') { + _addSearchPill({ type: 'filter', value: s.value, label: s.label }); + } else if (s.kind === 'email') { + // Clear the draft + dropdown and open the matching card directly. + if (input) input.value = ''; + state._libSearchDraft = ''; + _hideSearchSuggestions(); + _applyPillFilter(); + const grid = document.getElementById('email-lib-grid'); + const card = grid?.querySelector(`.doclib-card[data-uid="${CSS.escape(String(s.uid))}"]`); + const em = (state._libEmails || []).find(x => String(x.uid) === String(s.uid)) + || (_libPreSearchEmails || []).find(x => String(x.uid) === String(s.uid)); + if (card && em) { + card.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); + _toggleCardPreview(card, em); + } + return; + } else { + _addSearchPill({ type: 'contact', name: s.name, email: s.email }); + // Same as the text-pill path in the Enter handler: trigger the IMAP + // search so unloaded emails (older than the current page) show up + // when picking a contact. The local pill filter then narrows the + // search results to that contact's address. + const _q = (s.email || s.name || '').trim(); + if (_q && _q.length >= 2) { + state._libSearch = _q; + _doSearch(); + } + } + if (input) input.value = ''; + state._libSearchDraft = ''; + _hideSearchSuggestions(); + _applyPillFilter(); + if (input) input.focus(); +} + +async function _initEmailSearchChipBar() { + const bar = document.getElementById('email-lib-chip-bar'); + const input = document.getElementById('email-lib-search'); + if (!bar || !input) return; + state._libSearchPills = state._libSearchPills || []; + state._libSearchDraft = ''; + _renderSearchPills(); + + // Lazy-load suggestion source on first focus / keystroke. + const _ensureSuggestionCache = async () => { + if (_libSuggestionCache) return; + _libSuggestionCache = await _buildSuggestionSource(); + }; + + // Click anywhere in the bar lands the cursor in the input field. + bar.addEventListener('click', (e) => { + if (e.target.closest('.email-lib-pill-x')) return; + input.focus(); + }); + + let _itemsRef = []; + const _refreshSuggestions = async () => { + await _ensureSuggestionCache(); + _itemsRef = _filterSuggestions(input.value); + // Default to no focused suggestion — text typing should feel like + // regular search; the user has to ArrowDown / Tab explicitly to + // pick a contact. Enter without a focused row commits as text. + _libSuggestionFocusIdx = -1; + _renderSearchSuggestions(_itemsRef); + }; + + input.addEventListener('focus', _refreshSuggestions); + // Debounced IMAP search — fires ~500ms after the user stops typing so + // searches for names/text not in the current inbox page actually surface + // hits, instead of just locally filtering the visible window. + // + // Live local filtering on EVERY keystroke was clobbering server hits: + // _emailMatchesPill / _matchesQuery check subject/from_name/from_address/ + // snippet but never body, so intermediate text like "sam" reduced the + // 61 server results to whatever matched just those four fields (often + // 0). User saw "no emails" while typing. So local filter is gone from + // the typing path — debounced server search drives the grid. Pill + // add/remove still re-runs the local filter through _applyPillFilter + // directly. + let _libSearchTypingTimer = null; + input.addEventListener('input', async () => { + state._libSearchDraft = input.value; + try { console.log('[email-search] input event, value=', JSON.stringify(input.value)); } catch {} + await _refreshSuggestions(); + if (_libSearchTypingTimer) clearTimeout(_libSearchTypingTimer); + const v = input.value.trim(); + if (v.length >= 2) { + _libSearchTypingTimer = setTimeout(() => { + const cur = (input.value || '').trim(); + if (cur === v && cur.length >= 2) { + state._libSearch = cur; + try { console.log('[email-search] firing _doSearch for', cur); } catch {} + _doSearch(); + } else { + try { console.log('[email-search] debounce expired but input changed (was', v, 'now', cur, ')'); } catch {} + } + }, 500); + } else if (!v && _libSearchHadResults) { + // Cleared the input → restore the inbox the same way the pill-clear + // path does. Otherwise the stale search results stayed up after the + // user backspaced everything out. + _libSearchHadResults = false; + _libPreSearchEmails = null; + _libPreSearchTotal = 0; + state._libSearch = ''; + state._libOffset = 0; + _loadEmails({ useCache: true }); + } + }); + input.addEventListener('blur', () => { + // Delay so click/mousedown on a suggestion fires first. + setTimeout(_hideSearchSuggestions, 120); + }); + input.addEventListener('keydown', (e) => { + const menu = document.getElementById('email-lib-suggest'); + const menuOpen = menu && menu.style.display !== 'none'; + if (e.key === 'Backspace' && !input.value && (state._libSearchPills || []).length) { + e.preventDefault(); + _removeSearchPillAt(state._libSearchPills.length - 1); + return; + } + if (e.key === 'ArrowDown' && menuOpen) { + e.preventDefault(); + // -1 → 0 → 1 → … → length-1, then wraps back to -1 (no selection) + const next = _libSuggestionFocusIdx + 1; + _libSuggestionFocusIdx = next >= _itemsRef.length ? -1 : next; + _renderSearchSuggestions(_itemsRef); + return; + } + if (e.key === 'ArrowUp' && menuOpen) { + e.preventDefault(); + // -1 → length-1 → length-2 → … → 0 → -1 + const next = _libSuggestionFocusIdx - 1; + _libSuggestionFocusIdx = next < -1 ? _itemsRef.length - 1 : next; + _renderSearchSuggestions(_itemsRef); + return; + } + if (e.key === 'Tab' && menuOpen) { + // Tab autocompletes the FIRST suggestion (most-relevant), regardless + // of whether the user arrowed down yet — matches the user's mental + // model of "type a name and tab to pick". + const pick = _libSuggestionFocusIdx >= 0 ? _itemsRef[_libSuggestionFocusIdx] : _itemsRef[0]; + if (pick) { e.preventDefault(); _acceptSuggestion(pick); return; } + } + if (e.key === 'Enter') { + e.preventDefault(); + // Only commit a contact if the user explicitly focused one. Plain + // Enter should default to a text pill so regular text search works + // without forcing a contact pick. + if (menuOpen && _libSuggestionFocusIdx >= 0 && _itemsRef[_libSuggestionFocusIdx]) { + _acceptSuggestion(_itemsRef[_libSuggestionFocusIdx]); + return; + } + const v = input.value.trim(); + if (v) { + _addSearchPill({ type: 'text', text: v }); + input.value = ''; + state._libSearchDraft = ''; + _hideSearchSuggestions(); + // Pill-only filtering used to only check emails already loaded into + // state._libEmails (the visible page of the inbox). Searches for + // names/text that aren't in the current page returned "no emails" + // even when matches existed on the server. Trigger the IMAP + // search so state._libEmails is replaced with the actual hits, + // then the pill filter narrows to matches. + state._libSearch = v; + _doSearch(); + } + return; + } + if (e.key === 'Escape') { + if (menuOpen) { + // Just close the dropdown — let the modal Esc handler run on the + // next Esc to actually dismiss the library. + e.preventDefault(); + e.stopPropagation(); + _hideSearchSuggestions(); + } else { + // Blur first so the modal Esc handler doesn't get suppressed by + // any IME / typing-target check, and let the event propagate. + try { input.blur(); } catch (_) {} + } + } + }); +} + +// Click-to-add: clicking a recipient-chip in the email reader OR a +// .email-meta-sender in the library list drops the person into the +// library search as a contact pill so the user can pivot to "everything +// from / to this person" in one tap. +window.addEventListener('click', (e) => { + const lib = document.getElementById('email-lib-modal'); + // 1) Recipient chips inside the email reader area + const chip = e.target.closest && e.target.closest('.recipient-chip'); + if (chip && chip.closest('.email-reader-header, .email-card-reader, .email-reader-tab-modal')) { + // Don't pivot to library search for chips in the From / To / Cc + // meta — clicking those should just toggle the expanded address + // view via the per-reader handler. + if (chip.closest('.email-reader-meta')) return; + const email = (chip.dataset && chip.dataset.email) || ''; + const name = (chip.dataset && chip.dataset.name) || (chip.textContent || '').trim(); + if (!email) return; + e.preventDefault(); + e.stopPropagation(); + try { window.openEmailLibrary && window.openEmailLibrary(); } catch (_) {} + _addSearchPill({ type: 'contact', name, email }); + return; + } + // 2) Sender name in a library list card row (only when the library is open) + if (lib && !lib.classList.contains('hidden')) { + const senderEl = e.target.closest && e.target.closest('.email-meta-sender'); + if (senderEl && senderEl.closest('#email-lib-grid')) { + const email = (senderEl.dataset && senderEl.dataset.email) || ''; + const name = (senderEl.dataset && senderEl.dataset.name) || (senderEl.textContent || '').trim(); + if (!email) return; + e.preventDefault(); + e.stopPropagation(); + _addSearchPill({ type: 'contact', name, email }); + } + } +}, true); + async function _doSearch() { const seq = ++_libSearchSeq; const q = state._libSearch.trim(); @@ -1540,17 +2356,26 @@ async function _doSearch() { _renderGrid(); return; } - const grid = document.getElementById('email-lib-grid'); - if (!grid) return; - const sp = _renderEmailLoading(grid); const accountAtStart = state._libAccountId || ''; const folderAtStart = state._libFolder || 'INBOX'; + // No grid-blanking spinner — the local filter already painted something + // useful. Surface progress in the stats badge instead so the user knows + // the server search is still grinding. + const stats = document.getElementById('email-lib-stats'); + const originalStatsText = stats?.textContent || ''; + if (stats) stats.textContent = 'Searching…'; + _libSearchInFlight = true; + // Force a re-render so the "Searching…" empty-state shows (and any + // existing "No emails" gets replaced) while the fetch is in flight. + _renderGrid(); + const accountQS = accountAtStart ? `&account_id=${encodeURIComponent(accountAtStart)}` : ''; try { - const accountQS = accountAtStart ? `&account_id=${encodeURIComponent(accountAtStart)}` : ''; + // Single fast fetch — limit=100 so the IMAP fetch loop doesn't spend + // 60 s pulling 500 headers serially. We can wire "Load more" later + // off `state._libTotal` if needed. const res = await fetch(`${API_BASE}/api/email/search?folder=${encodeURIComponent(folderAtStart)}${accountQS}&q=${encodeURIComponent(q)}&limit=100`); const data = await res.json(); - sp.destroy(); if ( seq !== _libSearchSeq || q !== state._libSearch.trim() || @@ -1564,16 +2389,140 @@ async function _doSearch() { const results = data.emails || []; _libSearchHadResults = true; state._libEmails = results; // temporarily replace with search results + state._libTotal = data.total || results.length; + // Refresh the pre-search snapshot so any subsequent _applyPillFilter + // call (focus / pill edit / etc.) sources from the actual search + // results, not the stale inbox page that was loaded before the + // search ran. Without this, active pills (a contact pill from the + // suggestion the user just clicked) would filter the inbox snapshot + // → near-always empty → user sees "no emails" even though the + // server search succeeded. + _libPreSearchEmails = results.slice(); + _libPreSearchTotal = state._libTotal; + // If pills are active (and they usually are after a contact-pill or + // text-pill add), re-run the pill filter so the visible grid is the + // pill-narrowed intersection of the new search results. Otherwise + // _renderGrid below would render the raw server response, which + // might not match the active pills the user just added. + if ((state._libSearchPills || []).length) { + _applyPillFilter(); + // Fall back to rendering the raw results if the pill intersection + // hid everything but the user just confirmed they want this query. + if (!(state._libEmails || []).length) state._libEmails = results; + } _renderGrid(); - const stats = document.getElementById('email-lib-stats'); - if (stats) stats.textContent = `${data.total || results.length} match${(data.total || results.length) === 1 ? '' : 'es'}`; + const count = data.total || results.length; + if (stats) stats.textContent = `${count} match${count === 1 ? '' : 'es'} on server`; + try { console.log('[email-search]', JSON.stringify({ q, folder: folderAtStart, count, returned: results.length })); } catch {} } catch (e) { - sp.destroy(); - grid.innerHTML = '<div class="email-loading">Search failed</div>'; + if (stats) stats.textContent = originalStatsText || 'Search failed'; + try { console.error('[email-search] fetch failed:', e); } catch {} + } finally { + _libSearchInFlight = false; } } +// Custom dropdown for the email filter (All/Unread/Favorites/...). Replaces +// the native <select> so each row can carry an SVG icon. The hidden +// <select id="email-lib-filter"> stays as the value source — clicking a +// menu item updates its value and dispatches 'change', so every existing +// listener keeps working. +const _EMAIL_FILTER_ICONS = { + 'all': '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><line x1="8" y1="6" x2="21" y2="6"/><line x1="8" y1="12" x2="21" y2="12"/><line x1="8" y1="18" x2="21" y2="18"/><line x1="3" y1="6" x2="3.01" y2="6"/><line x1="3" y1="12" x2="3.01" y2="12"/><line x1="3" y1="18" x2="3.01" y2="18"/></svg>', + 'unread': '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"/><line x1="8" y1="16" x2="16" y2="8"/><line x1="8" y1="8" x2="16" y2="16"/></svg>', + 'favorites': '<svg width="13" height="13" viewBox="0 0 24 24" fill="currentColor" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M19 21l-7-5-7 5V5a2 2 0 0 1 2-2h10a2 2 0 0 1 2 2z"/></svg>', + 'undone': '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="3" width="18" height="18" rx="3"/></svg>', + 'reminders': '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M10.268 21a2 2 0 0 0 3.464 0"/><path d="M3.262 15.326A1 1 0 0 0 4 17h16a1 1 0 0 0 .74-1.673C19.41 13.956 18 12.499 18 8A6 6 0 0 0 6 8c0 4.499-1.411 5.956-2.738 7.326"/></svg>', + 'unanswered': '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="9 17 4 12 9 7"/><path d="M20 18v-2a4 4 0 0 0-4-4H4"/></svg>', + 'pending_30d': '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><polyline points="12 6 12 12 16 14"/></svg>', + 'stale_30d': '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="4" width="18" height="18" rx="2"/><line x1="16" y1="2" x2="16" y2="6"/><line x1="8" y1="2" x2="8" y2="6"/><line x1="3" y1="10" x2="21" y2="10"/><line x1="10" y1="14" x2="14" y2="18"/><line x1="14" y1="14" x2="10" y2="18"/></svg>', + 'tag:urgent': '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M10.29 3.86 1.82 18a2 2 0 0 0 1.71 3h16.94a2 2 0 0 0 1.71-3L13.71 3.86a2 2 0 0 0-3.42 0z"/><line x1="12" y1="9" x2="12" y2="13"/><line x1="12" y1="17" x2="12.01" y2="17"/></svg>', + 'tag:reply-soon':'<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="9 17 4 12 9 7"/><path d="M20 18v-2a4 4 0 0 0-4-4H4"/><circle cx="18" cy="6" r="2" fill="currentColor" stroke="none"/></svg>', + 'tag:spam': '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="4.93" y1="4.93" x2="19.07" y2="19.07"/></svg>', + 'tag:newsletter':'<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 22h16a2 2 0 0 0 2-2V4a2 2 0 0 0-2-2H8a2 2 0 0 0-2 2v16a2 2 0 0 1-2 2zm0 0a2 2 0 0 1-2-2v-9c0-1.1.9-2 2-2h2"/><path d="M18 14h-8"/><path d="M15 18h-5"/><path d="M10 6h8v4h-8V6z"/></svg>', + 'tag:marketing': '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 11l18-5v12L3 14v-3z"/><path d="M11.6 16.8a3 3 0 1 1-5.8-1.6"/></svg>', +}; + +function _filterIcon(value) { + return _EMAIL_FILTER_ICONS[value] || _EMAIL_FILTER_ICONS['all']; +} + +function _renderFilterPickerCurrent() { + const sel = document.getElementById('email-lib-filter'); + const btn = document.getElementById('email-filter-btn'); + if (!sel || !btn) return; + const value = sel.value || 'all'; + const opt = sel.querySelector(`option[value="${CSS.escape(value)}"]`); + const label = opt ? opt.textContent : value; + const iconWrap = btn.querySelector('.email-filter-icon'); + const labelEl = btn.querySelector('.email-filter-label'); + if (iconWrap) iconWrap.innerHTML = _filterIcon(value); + if (labelEl) labelEl.textContent = label; +} + +function _initFilterPicker() { + const sel = document.getElementById('email-lib-filter'); + const picker = document.getElementById('email-filter-picker'); + const btn = document.getElementById('email-filter-btn'); + const menu = document.getElementById('email-filter-menu'); + if (!sel || !picker || !btn || !menu || picker._wired) return; + picker._wired = true; + + // Build menu from the hidden <select> contents (preserves optgroup labels). + const items = []; + for (const child of sel.children) { + if (child.tagName === 'OPTGROUP') { + items.push({ group: child.label }); + for (const o of child.children) { + items.push({ value: o.value, label: o.textContent, group: child.label }); + } + } else if (child.tagName === 'OPTION') { + items.push({ value: child.value, label: child.textContent }); + } + } + menu.innerHTML = items.map(it => { + if (!it.value) { + return `<div class="email-filter-group">${it.group}</div>`; + } + return `<button type="button" role="option" class="email-filter-item" data-value="${it.value}"> + <span class="email-filter-item-icon">${_filterIcon(it.value)}</span> + <span class="email-filter-item-label">${it.label}</span> + </button>`; + }).join(''); + + const close = () => { + menu.hidden = true; + btn.setAttribute('aria-expanded', 'false'); + }; + const open = () => { + menu.hidden = false; + btn.setAttribute('aria-expanded', 'true'); + }; + btn.addEventListener('click', (e) => { + e.stopPropagation(); + if (menu.hidden) open(); else close(); + }); + menu.addEventListener('click', (e) => { + const item = e.target.closest('.email-filter-item'); + if (!item) return; + sel.value = item.dataset.value; + sel.dispatchEvent(new Event('change', { bubbles: true })); + close(); + }); + document.addEventListener('click', (e) => { + if (!menu.hidden && !picker.contains(e.target)) close(); + }); + document.addEventListener('keydown', (e) => { + if (e.key === 'Escape' && !menu.hidden) { + e.stopPropagation(); + close(); + } + }, { capture: true }); + + _renderFilterPickerCurrent(); +} + function _renderEmailLoading(grid) { if (!grid) return null; grid.innerHTML = ''; @@ -1697,7 +2646,18 @@ async function _loadEmails({ force = false, useCache = true } = {}) { state._libEmails = data.emails || []; state._libTotal = data.total || 0; if (sp) sp.destroy(); - _renderGrid(); + // If chip-bar pills are active, swap the snapshot to the freshly + // loaded list and re-apply the filter so pills persist across + // refreshes / folder switches instead of getting wiped. + const _activePills = (state._libSearchPills || []).length > 0 + || (state._libSearchDraft || '').length > 0; + if (_activePills) { + _libPreSearchEmails = state._libEmails.slice(); + _libPreSearchTotal = state._libTotal; + _applyPillFilter(); + } else { + _renderGrid(); + } const stats = document.getElementById('email-lib-stats'); if (stats) stats.textContent = `${state._libTotal} emails`; _refreshUnreadBadge(); @@ -1788,6 +2748,7 @@ function _renderGrid() { grid.innerHTML = ''; let filtered = state._libEmails; + try { console.log('[email-search] _renderGrid: state._libEmails.length=', (state._libEmails || []).length, 'pills=', (state._libSearchPills || []).length, 'draft=', JSON.stringify(state._libSearchDraft || ''), 'libSearch=', JSON.stringify(state._libSearch || '')); } catch {} // Apply sort if (state._libSort === 'unread') { @@ -1796,8 +2757,39 @@ function _renderGrid() { filtered = [...filtered].sort((a, b) => Number(b.is_flagged) - Number(a.is_flagged)); } // 'recent' is the default order from the API + // Stable secondary sort: favorited (is_flagged) emails ALWAYS bubble to + // the top of whatever order the sort above produced. This pins the + // user's flagged items so they're the first thing in the inbox no + // matter which sort mode is active. + filtered = [...filtered].sort((a, b) => Number(!!b.is_flagged) - Number(!!a.is_flagged)); if (filtered.length === 0) { + // Active search — don't flash "No emails": the IMAP fetch is still + // running. Show a "Searching…" placeholder until _doSearch resolves + // and renders again. Without this the user saw an empty state + // smiley for ~500ms between the optimistic pill-filter clear and + // the server response landing. + if (_libSearchInFlight) { + grid.innerHTML = ''; + const wrap = document.createElement('div'); + wrap.className = 'email-loading'; + wrap.style.cssText = 'display:flex;align-items:center;justify-content:center;gap:8px;flex-wrap:wrap;padding:24px;opacity:0.75;'; + grid.appendChild(wrap); + // Whirlpool spinner for parity with the rest of the cookbook / + // doclib loaders. Falls back to plain text if the import fails. + import('./spinner.js').then(sp => { + if (!wrap.isConnected) return; + const w = sp.default.createWhirlpool(20); + w.element.style.cssText = 'margin:0;display:block;'; + wrap.appendChild(w.element); + const lbl = document.createElement('span'); + lbl.textContent = 'Searching…'; + wrap.appendChild(lbl); + }).catch(() => { + wrap.textContent = 'Searching…'; + }); + return; + } // Inbox-zero is a win — pair the message with a small smiley so the // empty state reads as "all caught up", not "something's broken". const _smileyIco = '<span style="vertical-align:-3px;margin-left:6px;">' + emptyStateIcon('smiley') + '</span>'; @@ -1892,10 +2884,16 @@ function _createCard(em) { // hides the actually useful info. Outside Sent, show the sender as before. const isSentFolderEarly = /sent/i.test(state._libFolder); let senderName; + let senderAddress; if (isSentFolderEarly) { senderName = _formatRecipients(em.to) || em.to || '(no recipient)'; + // First address out of em.to for click-to-pill targeting. + const _firstTo = String(em.to || '').split(',')[0] || ''; + const _m = _firstTo.match(/<([^>]+)>/); + senderAddress = (_m ? _m[1] : _firstTo).trim(); } else { senderName = em.from_name || em.from_address; + senderAddress = em.from_address || ''; } const color = _senderColor(senderName); @@ -1988,7 +2986,7 @@ function _createCard(em) { const star = document.createElement('span'); star.title = 'Favorited'; star.style.cssText = 'color:var(--accent, var(--red));opacity:0.85;flex-shrink:0;display:inline-flex;'; - star.innerHTML = '<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>'; + star.innerHTML = '<svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M19 21l-7-5-7 5V5a2 2 0 0 1 2-2h10a2 2 0 0 1 2 2z"/></svg>'; titleRow.appendChild(star); } @@ -2016,27 +3014,10 @@ function _createCard(em) { await _toggleCardPreview(sibling, nextEm); sibling.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); }); - // Right cluster: expanded-only actions menu + nav arrows. The normal - // `.memory-item-actions` menu is hidden while expanded, so this keeps - // the same email actions available beside the previous/next controls. - const rightCluster = document.createElement('span'); - rightCluster.style.cssText = 'margin-left:auto;display:inline-flex;align-items:center;gap:6px;'; - const headerMenuBtn = document.createElement('button'); - headerMenuBtn.type = 'button'; - headerMenuBtn.className = 'email-card-header-menu'; - headerMenuBtn.title = 'Actions'; - headerMenuBtn.innerHTML = '<svg width="14" height="14" viewBox="0 0 24 24" fill="currentColor"><circle cx="12" cy="5" r="2"/><circle cx="12" cy="12" r="2"/><circle cx="12" cy="19" r="2"/></svg>'; - headerMenuBtn.addEventListener('click', (e) => { - e.stopPropagation(); - _showCardMenu(em, headerMenuBtn); - }); - // The CSS rule on .email-card-nav-arrows still sets margin-left:auto - // (needed when the arrows live alone in the title row). Inside this - // wrapper, we want the cluster's gap to apply, so cancel that auto. - navArrows.style.marginLeft = '0'; - rightCluster.appendChild(headerMenuBtn); - rightCluster.appendChild(navArrows); - titleRow.appendChild(rightCluster); + // Just the nav arrows here — the per-card `.memory-item-actions` menu + // at the bottom of the card stays visible while expanded (see the CSS + // override below), so duplicating it in the header was redundant. + titleRow.appendChild(navArrows); content.appendChild(titleRow); @@ -2044,7 +3025,7 @@ function _createCard(em) { meta.className = 'memory-item-meta'; meta.style.cssText = 'font-size:10px;opacity:0.7;margin-top:2px;'; const senderPrefix = isSentFolderEarly ? 'to ' : ''; - meta.innerHTML = `<span class="email-meta-sender"><span style="opacity:0.55">${senderPrefix}</span><span style="color:${color};font-weight:600">${_esc(senderName)}</span></span><span class="email-meta-sep"> · </span><span class="email-meta-date">${_esc(dateStr)}</span>`; + meta.innerHTML = `<span class="email-meta-sender" data-email="${_esc(senderAddress || '')}" data-name="${_esc(senderName || '')}"><span style="opacity:0.55">${senderPrefix}</span><span style="color:${color};font-weight:600">${_esc(senderName)}</span></span><span class="email-meta-sep"> · </span><span class="email-meta-date">${_esc(dateStr)}</span>`; content.appendChild(meta); card.appendChild(content); @@ -2145,13 +3126,19 @@ function _prefetchAdjacentEmails(card, count = 1) { const target = targets.find(t => t?.dataset?.uid); const uid = target?.dataset?.uid; if (!uid) return; - const key = `${state._libAccountId || ''}|${state._libFolder}|${uid}`; + // Use the email's actual folder when it was stamped by the search + // endpoint; otherwise default to the currently-selected folder. + const _emFold = (() => { + const emObj = (state._libEmails || []).find(e => String(e.uid) === String(uid)); + return (emObj && emObj.folder) || state._libFolder || 'INBOX'; + })(); + const key = `${state._libAccountId || ''}|${_emFold}|${uid}`; if (_emailReadPrefetching.has(key) || _emailReadPrefetching.size > 0) return; if (_emailReadPrefetchTimer) clearTimeout(_emailReadPrefetchTimer); _emailReadPrefetchTimer = setTimeout(() => { _emailReadPrefetchTimer = null; _emailReadPrefetching.add(key); - fetch(`${API_BASE}/api/email/read/${encodeURIComponent(uid)}?folder=${encodeURIComponent(state._libFolder)}${_acct()}&mark_seen=false`) + fetch(`${API_BASE}/api/email/read/${encodeURIComponent(uid)}?folder=${encodeURIComponent(_emFold)}${_acct()}&mark_seen=false`) .catch(() => {}) .finally(() => _emailReadPrefetching.delete(key)); }, 900); @@ -2159,7 +3146,10 @@ function _prefetchAdjacentEmails(card, count = 1) { async function _toggleCardPreview(card, em) { const accountAtStart = state._libAccountId || ''; - const folderAtStart = state._libFolder || 'INBOX'; + // Prefer the per-email folder stamped by the search endpoint (results + // from "All Mail" carry folder="[Gmail]/All Mail"). Falls back to the + // currently-selected folder for normal inbox cards. + const folderAtStart = (em && em.folder) || state._libFolder || 'INBOX'; const uidAtStart = String(em?.uid || card?.dataset?.uid || ''); const grid = card.closest('.doclib-grid'); const gridRect = grid?.getBoundingClientRect?.(); @@ -2199,6 +3189,13 @@ async function _toggleCardPreview(card, em) { card.classList.add('email-card-expanded'); card.classList.add('doclib-card-expanded'); card.style.minHeight = `${Math.round(stableOpenHeight)}px`; + // Pull the card into view in case the user clicked an email further up + // the list whose top is partially scrolled off the viewport. Wait for + // the layout to settle (minHeight just changed) before scrolling so + // the browser scrolls toward the post-expansion position. + requestAnimationFrame(() => { + try { card.scrollIntoView({ behavior: 'smooth', block: 'start' }); } catch (_) {} + }); if (!em.is_read) { _syncEmailReadState(em.uid, true); fetch(`${API_BASE}/api/email/mark-read/${em.uid}?folder=${encodeURIComponent(folderAtStart)}${_acct()}`, { method: 'POST' }) @@ -2244,6 +3241,7 @@ async function _toggleCardPreview(card, em) { // Mark as read locally _syncEmailReadState(em.uid, true); _prefetchAdjacentEmails(card); + _stampReaderContext(reader, { ...em, ...data }, state._libFolder, state._libAccountId); // Build the attachments wrap using the shared helper so the signature- // image filter (small inline PNGs/JPGs, Outlook image001 placeholders, @@ -2282,20 +3280,20 @@ async function _toggleCardPreview(card, em) { reader.innerHTML = ` <div class="email-reader-header"> <div class="email-reader-meta"> - <div class="email-reader-meta-row"><strong>From:</strong><span class="recipient-chips">${fromChip}</span></div> - ${data.to ? `<div class="email-reader-meta-row"><strong>To:</strong><span class="recipient-chips">${buildRecipients(data.to)}</span></div>` : ''} - ${data.cc ? `<div class="email-reader-meta-row"><strong>Cc:</strong><span class="recipient-chips">${buildRecipients(data.cc)}</span></div>` : ''} - </div> - <div class="email-reader-actions"> - <div class="email-reader-actions-row email-reader-actions-row-primary"> + <div class="email-reader-meta-row email-reader-meta-from"> + <strong>From:</strong> + <span class="recipient-chips">${fromChip}${(data.to || data.cc) ? `<button class="email-reader-meta-toggle" type="button" aria-expanded="false" title="Show recipients"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg></button>` : ''}</span> + </div> + ${(data.to || data.cc) ? `<div class="email-reader-meta-details" hidden> + ${data.to ? `<div class="email-reader-meta-row"><strong>To:</strong><span class="recipient-chips">${buildRecipients(data.to)}</span></div>` : ''} + ${data.cc ? `<div class="email-reader-meta-row"><strong>Cc:</strong><span class="recipient-chips">${buildRecipients(data.cc)}</span></div>` : ''} + </div>` : ''} + <div class="email-reader-actions-inline"> + <button class="memory-toolbar-btn reader-icon-btn" data-act="ai-reply" title="${data.cached_ai_reply ? 'AI Reply (cached draft ready)' : 'AI Reply (suggest a draft)'}">${_aiReplyIcon(data)}<span class="reader-btn-label">AI reply</span></button> <button class="memory-toolbar-btn reader-icon-btn" data-act="reply" title="Reply"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="9 17 4 12 9 7"/><path d="M20 18v-2a4 4 0 0 0-4-4H4"/></svg><span class="reader-btn-label">Reply</span></button> ${_hasMultipleRecipients(data) ? `<button class="memory-toolbar-btn reader-icon-btn" data-act="reply-all" title="Reply All"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="7 17 2 12 7 7"/><polyline points="12 17 7 12 12 7"/><path d="M22 18v-2a4 4 0 0 0-4-4H7"/></svg><span class="reader-btn-label">Reply all</span></button>` : ''} <button class="memory-toolbar-btn reader-icon-btn" data-act="forward" title="Forward"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="15 17 20 12 15 7"/><path d="M4 18v-2a4 4 0 0 1 4-4h12"/></svg><span class="reader-btn-label">Forward</span></button> - </div> - <div class="email-reader-actions-row email-reader-actions-row-secondary"> - <button class="memory-toolbar-btn reader-icon-btn" data-act="ai-reply" title="${data.cached_ai_reply ? 'AI Reply (cached draft ready)' : 'AI Reply (suggest a draft)'}">${_aiReplyIcon(data)}<span class="reader-btn-label">AI reply</span></button> <button class="memory-toolbar-btn reader-icon-btn" data-act="summarize" title="Summarize">${_summaryIcon(data)}<span class="reader-btn-label">Summary</span></button> - <button class="memory-toolbar-btn reader-icon-btn" data-act="from-sender" title="Search text in this thread"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><circle cx="11" cy="11" r="7"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg><span class="reader-btn-label">Search</span></button> <div class="email-reader-more-wrap" style="position:relative"> <button class="memory-toolbar-btn reader-icon-btn" data-act="more" title="More actions"><svg width="14" height="14" viewBox="0 0 24 24" fill="currentColor"><circle cx="12" cy="5" r="2"/><circle cx="12" cy="12" r="2"/><circle cx="12" cy="19" r="2"/></svg><span class="reader-btn-label">More</span></button> </div> @@ -2363,6 +3361,7 @@ async function _toggleCardPreview(card, em) { ev.stopPropagation(); await _summarizeEmail(reader, data, ev.currentTarget); }); + _wireMetaToggle(reader); // from-sender / thread-search Search button is DISABLED for now — // the search + threaded sidebar UX is too buggy to ship. Physically // remove it from every reader render path. Re-enable by deleting @@ -3625,8 +4624,24 @@ function _wireAttachmentHandlers(reader, folder) { window.open(url, '_blank'); return; } - const orig = chip.style.opacity; - chip.style.opacity = '0.6'; + // Swap the paperclip icon for a whirlpool spinner while the + // download is in flight, so large attachments give a clear cue + // they're loading. Restore on completion. + const iconSvg = chip.querySelector(':scope > svg'); + const origIconHtml = iconSvg ? iconSvg.outerHTML : ''; + let _wp = null; + let _spinnerHost = null; + try { + const sp = window.spinnerModule || (await import('./spinner.js')).default; + _wp = sp.createWhirlpool(12); + _spinnerHost = document.createElement('span'); + _spinnerHost.className = 'email-attachment-spinner'; + _spinnerHost.style.cssText = 'display:inline-flex;width:12px;height:12px;align-items:center;justify-content:center;flex-shrink:0;position:relative;top:-2px;'; + _spinnerHost.appendChild(_wp.element); + if (iconSvg) iconSvg.replaceWith(_spinnerHost); + } catch (_) {} + const origOpacity = chip.style.opacity; + chip.style.opacity = '0.85'; try { const res = await fetch(url, { credentials: 'same-origin' }); if (!res.ok) { @@ -3647,7 +4662,14 @@ function _wireAttachmentHandlers(reader, folder) { console.error('attachment download error', e); location.href = url; } finally { - chip.style.opacity = orig; + chip.style.opacity = origOpacity; + if (_spinnerHost && _spinnerHost.parentNode && origIconHtml) { + const tmp = document.createElement('div'); + tmp.innerHTML = origIconHtml; + const restored = tmp.firstChild; + if (restored) _spinnerHost.replaceWith(restored); + } + if (_wp) { try { _wp.destroy(); } catch (_) {} } } }); }); @@ -3942,6 +4964,7 @@ async function _openEmailAsTab(em, folder) { return; } _syncEmailReadState(em.uid, true); + _stampReaderContext(reader, { ...em, ...data }, useFolder, state._libAccountId); const buildChips = (str) => { if (!str) return ''; return _splitRecipientList(str).map(a => { @@ -3955,18 +4978,19 @@ async function _openEmailAsTab(em, folder) { reader.innerHTML = ` <div class="email-reader-header"> <div class="email-reader-meta"> - <div class="email-reader-meta-row"><strong>From:</strong><span class="recipient-chips">${fromChip}</span></div> - ${data.to ? `<div class="email-reader-meta-row"><strong>To:</strong><span class="recipient-chips">${buildChips(data.to)}</span></div>` : ''} - ${data.cc ? `<div class="email-reader-meta-row"><strong>Cc:</strong><span class="recipient-chips">${buildChips(data.cc)}</span></div>` : ''} - </div> - <div class="email-reader-actions"> - <div class="email-reader-actions-row email-reader-actions-row-primary"> + <div class="email-reader-meta-row email-reader-meta-from"> + <strong>From:</strong> + <span class="recipient-chips">${fromChip}${(data.to || data.cc) ? `<button class="email-reader-meta-toggle" type="button" aria-expanded="false" title="Show recipients"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg></button>` : ''}</span> + </div> + ${(data.to || data.cc) ? `<div class="email-reader-meta-details" hidden> + ${data.to ? `<div class="email-reader-meta-row"><strong>To:</strong><span class="recipient-chips">${buildChips(data.to)}</span></div>` : ''} + ${data.cc ? `<div class="email-reader-meta-row"><strong>Cc:</strong><span class="recipient-chips">${buildChips(data.cc)}</span></div>` : ''} + </div>` : ''} + <div class="email-reader-actions-inline"> + <button class="memory-toolbar-btn reader-icon-btn" data-act="ai-reply" title="${data.cached_ai_reply ? 'AI Reply (cached draft ready)' : 'AI Reply'}">${_aiReplyIcon(data)}<span class="reader-btn-label">AI reply</span></button> <button class="memory-toolbar-btn reader-icon-btn" data-act="reply" title="Reply"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="9 17 4 12 9 7"/><path d="M20 18v-2a4 4 0 0 0-4-4H4"/></svg><span class="reader-btn-label">Reply</span></button> ${_hasMultipleRecipients(data) ? `<button class="memory-toolbar-btn reader-icon-btn" data-act="reply-all" title="Reply All"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="7 17 2 12 7 7"/><polyline points="12 17 7 12 12 7"/><path d="M22 18v-2a4 4 0 0 0-4-4H7"/></svg><span class="reader-btn-label">Reply all</span></button>` : ''} <button class="memory-toolbar-btn reader-icon-btn" data-act="forward" title="Forward"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="15 17 20 12 15 7"/><path d="M4 18v-2a4 4 0 0 1 4-4h12"/></svg><span class="reader-btn-label">Forward</span></button> - </div> - <div class="email-reader-actions-row email-reader-actions-row-secondary"> - <button class="memory-toolbar-btn reader-icon-btn" data-act="ai-reply" title="${data.cached_ai_reply ? 'AI Reply (cached draft ready)' : 'AI Reply'}">${_aiReplyIcon(data)}<span class="reader-btn-label">AI reply</span></button> <button class="memory-toolbar-btn reader-icon-btn" data-act="summarize" title="Summarize">${_summaryIcon(data)}<span class="reader-btn-label">Summary</span></button> <button class="memory-toolbar-btn reader-icon-btn" data-act="from-sender" title="Search text in this thread"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><circle cx="11" cy="11" r="7"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg><span class="reader-btn-label">Search</span></button> <div class="email-reader-more-wrap" style="position:relative"> @@ -4005,6 +5029,7 @@ async function _openEmailAsTab(em, folder) { ev.stopPropagation(); try { await _summarizeEmail(reader, data, ev.currentTarget); } catch {} }); + _wireMetaToggle(reader); reader.querySelector('[data-act="from-sender"]')?.remove(); reader.querySelector('[data-act="from-sender"]')?.addEventListener('click', async (ev) => { ev.stopPropagation(); @@ -4109,20 +5134,20 @@ async function _openEmailWindow(em, folder) { bodyEl.innerHTML = ` <div class="email-reader-header"> <div class="email-reader-meta"> - <div class="email-reader-meta-row"><strong>From:</strong><span class="recipient-chips">${fromChip}</span></div> - ${data.to ? `<div class="email-reader-meta-row"><strong>To:</strong><span class="recipient-chips">${_chipsFor(data.to)}</span></div>` : ''} - ${data.cc ? `<div class="email-reader-meta-row"><strong>Cc:</strong><span class="recipient-chips">${_chipsFor(data.cc)}</span></div>` : ''} - </div> - <div class="email-reader-actions"> - <div class="email-reader-actions-row email-reader-actions-row-primary"> + <div class="email-reader-meta-row email-reader-meta-from"> + <strong>From:</strong> + <span class="recipient-chips">${fromChip}${(data.to || data.cc) ? `<button class="email-reader-meta-toggle" type="button" aria-expanded="false" title="Show recipients"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg></button>` : ''}</span> + </div> + ${(data.to || data.cc) ? `<div class="email-reader-meta-details" hidden> + ${data.to ? `<div class="email-reader-meta-row"><strong>To:</strong><span class="recipient-chips">${_chipsFor(data.to)}</span></div>` : ''} + ${data.cc ? `<div class="email-reader-meta-row"><strong>Cc:</strong><span class="recipient-chips">${_chipsFor(data.cc)}</span></div>` : ''} + </div>` : ''} + <div class="email-reader-actions-inline"> + <button class="memory-toolbar-btn reader-icon-btn" data-act="ai-reply" title="${data.cached_ai_reply ? 'AI Reply (cached draft ready)' : 'AI Reply (suggest a draft)'}">${_aiReplyIcon(data)}<span class="reader-btn-label">AI reply</span></button> <button class="memory-toolbar-btn reader-icon-btn" data-act="reply" title="Reply"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="9 17 4 12 9 7"/><path d="M20 18v-2a4 4 0 0 0-4-4H4"/></svg><span class="reader-btn-label">Reply</span></button> ${_hasMultipleRecipients(data) ? `<button class="memory-toolbar-btn reader-icon-btn" data-act="reply-all" title="Reply All"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="7 17 2 12 7 7"/><polyline points="12 17 7 12 12 7"/><path d="M22 18v-2a4 4 0 0 0-4-4H7"/></svg><span class="reader-btn-label">Reply all</span></button>` : ''} <button class="memory-toolbar-btn reader-icon-btn" data-act="forward" title="Forward"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="15 17 20 12 15 7"/><path d="M4 18v-2a4 4 0 0 1 4-4h12"/></svg><span class="reader-btn-label">Forward</span></button> - </div> - <div class="email-reader-actions-row email-reader-actions-row-secondary"> - <button class="memory-toolbar-btn reader-icon-btn" data-act="ai-reply" title="${data.cached_ai_reply ? 'AI Reply (cached draft ready)' : 'AI Reply (suggest a draft)'}">${_aiReplyIcon(data)}<span class="reader-btn-label">AI reply</span></button> <button class="memory-toolbar-btn reader-icon-btn" data-act="summarize" title="Summarize">${_summaryIcon(data)}<span class="reader-btn-label">Summary</span></button> - <button class="memory-toolbar-btn reader-icon-btn" data-act="from-sender" title="Search text in this thread"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><circle cx="11" cy="11" r="7"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg><span class="reader-btn-label">Search</span></button> <div class="email-reader-more-wrap" style="position:relative"> <button class="memory-toolbar-btn reader-icon-btn" data-act="more" title="More actions"><svg width="14" height="14" viewBox="0 0 24 24" fill="currentColor"><circle cx="12" cy="5" r="2"/><circle cx="12" cy="12" r="2"/><circle cx="12" cy="19" r="2"/></svg><span class="reader-btn-label">More</span></button> </div> @@ -4160,6 +5185,7 @@ async function _openEmailWindow(em, folder) { ev.stopPropagation(); try { await _summarizeEmail(bodyEl, data, ev.currentTarget); } catch {} }); + _wireMetaToggle(bodyEl); bodyEl.querySelector('[data-act="from-sender"]')?.remove(); bodyEl.querySelector('[data-act="from-sender"]')?.addEventListener('click', async (ev) => { ev.stopPropagation(); @@ -4483,6 +5509,10 @@ function _showReaderMoreMenu(em, card, reader, anchor) { const _bubblesIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"/></svg>'; const _contactIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"/><circle cx="9" cy="7" r="4"/><line x1="19" y1="8" x2="19" y2="14"/><line x1="22" y1="11" x2="16" y2="11"/></svg>'; + // Three groups separated by dividers: + // 1. Open / Mark Unread / Remind — the per-email view actions + // 2. Save sender / Not Done / Archive — non-destructive state changes + // 3. Move to Spam / Move to Trash / Delete — destructive const actions = [ { label: 'Open in new tab', @@ -4492,6 +5522,77 @@ function _showReaderMoreMenu(em, card, reader, anchor) { await _openEmailAsTab(em, folder); }, }, + { + label: 'Remind to reply', + icon: _bellIcon, + submenu: 'remind', + }, + { separator: true }, + { + label: em.is_read ? 'Mark as Unread' : 'Mark as Read', + icon: _unreadIcon, + action: async () => { + const newRead = !em.is_read; + _syncEmailReadState(em.uid, newRead); + try { + if (newRead) { + await fetch(`${API_BASE}/api/email/mark-read/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); + } else { + await fetch(`${API_BASE}/api/email/mark-unread/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); + } + } catch (e) { console.error(e); } + _renderGrid(); + }, + }, + { + // Favorite (pin to top). Same bookmark glyph we use for the + // sidebar-pin / favorites filter so the visual language stays + // consistent. Toggling updates em.is_flagged and re-sorts via + // _renderGrid (favorited rows are always pinned at the top). + label: em.is_flagged ? 'Unfavorite' : 'Favorite (pin to top)', + icon: '<svg width="14" height="14" viewBox="0 0 24 24" fill="' + (em.is_flagged ? 'currentColor' : 'none') + '" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M19 21l-7-5-7 5V5a2 2 0 0 1 2-2h10a2 2 0 0 1 2 2z"/></svg>', + action: async () => { + const next = !em.is_flagged; + em.is_flagged = next; + _renderGrid(); + try { + await fetch(`${API_BASE}/api/email/flag/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}&on=${next ? 'true' : 'false'}`, { method: 'POST' }); + } catch (e) { + // Roll back the optimistic flip if the server didn't take it. + em.is_flagged = !next; + _renderGrid(); + console.error('Failed to toggle favorite:', e); + } + }, + }, + { + label: em.is_answered ? 'Mark as Not Done' : 'Mark as Done', + icon: _checkIcon, + action: async () => { + const newState = !em.is_answered; + em.is_answered = newState; + if (newState) _syncEmailReadState(em.uid, true); + try { + if (newState) { + await fetch(`${API_BASE}/api/email/mark-answered/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); + await fetch(`${API_BASE}/api/email/mark-read/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); + } else { + await fetch(`${API_BASE}/api/email/clear-answered/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); + } + } catch (e) { console.error('Failed to toggle done:', e); } + _renderGrid(); + }, + }, + { + label: 'Move to Archive', + icon: _archIcon, + action: async () => { + try { + await fetch(`${API_BASE}/api/email/archive/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); + } catch (e) { console.error(e); } + await closeAndRemove(); + }, + }, { // Save the sender to CardDAV contacts. Pulls name + address off the // list-item (em); falls back to splitting the local-part for a name. @@ -4522,58 +5623,7 @@ function _showReaderMoreMenu(em, card, reader, anchor) { } }, }, - // Threaded ⇄ Plain-text view toggle removed — threaded view disabled - // for now (too buggy). Emails always render plain text. Restore this - // menu item + _bubblesDisabled() localStorage logic to bring it back. - { - label: em.is_read ? 'Mark Unread' : 'Mark Read', - icon: _unreadIcon, - action: async () => { - const newRead = !em.is_read; - _syncEmailReadState(em.uid, newRead); - try { - if (newRead) { - await fetch(`${API_BASE}/api/email/mark-read/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); - } else { - await fetch(`${API_BASE}/api/email/mark-unread/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); - } - } catch (e) { console.error(e); } - _renderGrid(); - }, - }, - { - label: em.is_answered ? 'Not Done' : 'Done', - icon: _checkIcon, - action: async () => { - const newState = !em.is_answered; - em.is_answered = newState; - if (newState) _syncEmailReadState(em.uid, true); - try { - if (newState) { - await fetch(`${API_BASE}/api/email/mark-answered/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); - await fetch(`${API_BASE}/api/email/mark-read/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); - } else { - await fetch(`${API_BASE}/api/email/clear-answered/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); - } - } catch (e) { console.error('Failed to toggle done:', e); } - _renderGrid(); - }, - }, - { - label: 'Archive', - icon: _archIcon, - action: async () => { - try { - await fetch(`${API_BASE}/api/email/archive/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); - } catch (e) { console.error(e); } - await closeAndRemove(); - }, - }, - { - label: 'Remind to reply', - icon: _bellIcon, - submenu: 'remind', - }, + { separator: true }, { label: 'Move to Spam', icon: _spamIcon, @@ -4614,6 +5664,12 @@ function _showReaderMoreMenu(em, card, reader, anchor) { ]; for (const a of actions) { + if (a.separator) { + const sep = document.createElement('div'); + sep.className = 'dropdown-divider'; + dropdown.appendChild(sep); + continue; + } const item = document.createElement('div'); item.className = 'dropdown-item-compact' + (a.danger ? ' dropdown-item-danger' : ''); const arrow = a.submenu ? '<span style="margin-left:auto;opacity:0.5;">›</span>' : ''; @@ -4723,6 +5779,22 @@ function _showCardMenu(em, anchor) { } }, }); + actions.push({ + label: em.is_flagged ? 'Unfavorite' : 'Favorite (pin to top)', + icon: '<svg width="14" height="14" viewBox="0 0 24 24" fill="' + (em.is_flagged ? 'currentColor' : 'none') + '" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M19 21l-7-5-7 5V5a2 2 0 0 1 2-2h10a2 2 0 0 1 2 2z"/></svg>', + action: async () => { + const next = !em.is_flagged; + em.is_flagged = next; + _renderGrid(); + try { + await fetch(`${API_BASE}/api/email/flag/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}&on=${next ? 'true' : 'false'}`, { method: 'POST' }); + } catch (e) { + em.is_flagged = !next; + _renderGrid(); + console.error('Failed to toggle favorite:', e); + } + }, + }); actions.push({ label: 'Archive', icon: _archIcon, @@ -4735,6 +5807,22 @@ function _showCardMenu(em, anchor) { }, }); } else { + actions.push({ + label: em.is_flagged ? 'Unfavorite' : 'Favorite (pin to top)', + icon: '<svg width="14" height="14" viewBox="0 0 24 24" fill="' + (em.is_flagged ? 'currentColor' : 'none') + '" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M19 21l-7-5-7 5V5a2 2 0 0 1 2-2h10a2 2 0 0 1 2 2z"/></svg>', + action: async () => { + const next = !em.is_flagged; + em.is_flagged = next; + _renderGrid(); + try { + await fetch(`${API_BASE}/api/email/flag/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}&on=${next ? 'true' : 'false'}`, { method: 'POST' }); + } catch (e) { + em.is_flagged = !next; + _renderGrid(); + console.error('Failed to toggle favorite:', e); + } + }, + }); actions.push({ label: 'Archive', icon: _archIcon, @@ -4909,67 +5997,123 @@ async function _bulkAction(action) { const originalDeleteHtml = deleteBtn?.innerHTML || ''; const originalCountText = countEl?.textContent || ''; let busySpinner = null; - if (action === 'delete') { - if (deleteBtn) { - deleteBtn.disabled = true; - deleteBtn.classList.add('email-bulk-loading'); - deleteBtn.innerHTML = '<span class="email-bulk-loading-label">Deleting</span>'; - busySpinner = spinnerModule.create('', 'clean', 'whirlpool'); - const spEl = busySpinner.createElement(); - spEl.classList.add('email-bulk-whirlpool'); - deleteBtn.appendChild(spEl); - busySpinner.start(); - } - if (actionsBtn) actionsBtn.disabled = true; - if (cancelBtn) cancelBtn.disabled = true; - if (selectAll) selectAll.disabled = true; - if (countEl) countEl.textContent = `Deleting ${uids.length}...`; + // Loading state for every bulk action, not just delete — large + // selections (e.g. 90+ Dones) used to silently hammer the server + // with sequential requests and the user got zero feedback. Now the + // Actions button (or Delete button) shows a whirlpool + verb-ing + // label, and the count surfaces progress. + const verbing = { + delete: 'Deleting', + archive: 'Archiving', + done: 'Marking done', + read: 'Marking read', + unread: 'Marking unread', + }[action] || 'Updating'; + const targetBtn = action === 'delete' ? deleteBtn : actionsBtn; + let originalTargetHtml = ''; + if (targetBtn) { + originalTargetHtml = targetBtn.innerHTML; + targetBtn.disabled = true; + targetBtn.classList.add('email-bulk-loading'); + targetBtn.innerHTML = `<span class="email-bulk-loading-label">${verbing}</span>`; + busySpinner = spinnerModule.create('', 'clean', 'whirlpool'); + const spEl = busySpinner.createElement(); + spEl.classList.add('email-bulk-whirlpool'); + targetBtn.appendChild(spEl); + busySpinner.start(); } + if (action !== 'delete' && deleteBtn) deleteBtn.disabled = true; + if (action === 'delete' && actionsBtn) actionsBtn.disabled = true; + if (cancelBtn) cancelBtn.disabled = true; + if (selectAll) selectAll.disabled = true; + if (countEl) countEl.textContent = `${verbing} ${uids.length}…`; + + // Single-uid worker. + const handleOne = async (uid) => { + try { + if (action === 'archive') { + await fetch(`${API_BASE}/api/email/archive/${uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); + } else if (action === 'delete') { + await fetch(`${API_BASE}/api/email/delete/${uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'DELETE' }); + } else if (action === 'done') { + // uid may come back from the Set as a string while em.uid is + // numeric (or vice versa) — coerce both sides so the in-memory + // state actually flips and the post-loop re-render shows the + // done checkmark. + const em = state._libEmails.find(e => String(e.uid) === String(uid)); + if (em) { em.is_answered = true; em.is_read = true; } + const ansRes = await fetch(`${API_BASE}/api/email/mark-answered/${uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); + const readRes = await fetch(`${API_BASE}/api/email/mark-read/${uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); + if (!ansRes.ok || !readRes.ok) throw new Error(`mark-done HTTP ${ansRes.status}/${readRes.status}`); + } else if (action === 'read' || action === 'unread') { + const endpoint = action === 'read' ? 'mark-read' : 'mark-unread'; + const res = await fetch(`${API_BASE}/api/email/${endpoint}/${uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); + let data = null; + try { data = await res.json(); } catch (_) {} + if (!res.ok || data?.success === false) { + throw new Error(data?.error || `HTTP ${res.status}`); + } + _syncEmailReadState(uid, action === 'read'); + } + } catch (e) { + if (action === 'read' || action === 'unread') failedReadSync += 1; + console.error(`Failed to ${action} ${uid}:`, e); + } + }; try { - for (const uid of uids) { - try { - if (action === 'archive') { - await fetch(`${API_BASE}/api/email/archive/${uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); - } else if (action === 'delete') { - await fetch(`${API_BASE}/api/email/delete/${uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'DELETE' }); - } else if (action === 'done') { - const em = state._libEmails.find(e => e.uid === uid); - if (em) { - em.is_answered = true; - em.is_read = true; - } - await fetch(`${API_BASE}/api/email/mark-answered/${uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); - await fetch(`${API_BASE}/api/email/mark-read/${uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); - } else if (action === 'read' || action === 'unread') { - const endpoint = action === 'read' ? 'mark-read' : 'mark-unread'; - const res = await fetch(`${API_BASE}/api/email/${endpoint}/${uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' }); - let data = null; - try { data = await res.json(); } catch (_) {} - if (!res.ok || data?.success === false) { - throw new Error(data?.error || `HTTP ${res.status}`); - } - _syncEmailReadState(uid, action === 'read'); + // Run in parallel with a concurrency cap so 92 emails don't take + // 30 seconds sequentially but we also don't open 92 simultaneous + // connections. + const CONCURRENCY = 6; + const queue = uids.slice(); + let inFlight = 0; + let nextSlot = 0; + let finishedCount = 0; + await new Promise((resolve) => { + const launch = () => { + while (inFlight < CONCURRENCY && nextSlot < queue.length) { + const uid = queue[nextSlot++]; + inFlight++; + handleOne(uid).finally(() => { + inFlight--; + finishedCount++; + if (countEl) countEl.textContent = `${verbing} ${finishedCount}/${queue.length}…`; + if (nextSlot >= queue.length && inFlight === 0) resolve(); + else launch(); + }); } - } catch (e) { - if (action === 'read' || action === 'unread') failedReadSync += 1; - console.error(`Failed to ${action} ${uid}:`, e); - } - } + if (queue.length === 0) resolve(); + }; + launch(); + }); if (action === 'archive' || action === 'delete') { await _animateEmailCardRemoval(uids); const removed = new Set(uids.map(uid => String(uid))); state._libEmails = state._libEmails.filter(e => !removed.has(String(e.uid))); + } else if (action === 'done' && state._libFilter === 'undone') { + // The undone filter is a "show only not-done" view — after marking + // selected emails done, they no longer match. Animate them out and + // drop them from the local list so the view reflects the filter + // instead of leaving freshly-done cards sitting there. + await _animateEmailCardRemoval(uids); + const removed = new Set(uids.map(uid => String(uid))); + state._libEmails = state._libEmails.filter(e => !removed.has(String(e.uid))); } } finally { if (busySpinner) busySpinner.destroy(); - if (deleteBtn) { - deleteBtn.disabled = false; - deleteBtn.classList.remove('email-bulk-loading'); - deleteBtn.innerHTML = originalDeleteHtml; + // Restore whichever button we hijacked (delete vs actions). + if (targetBtn) { + targetBtn.disabled = false; + targetBtn.classList.remove('email-bulk-loading'); + targetBtn.innerHTML = originalTargetHtml || targetBtn.innerHTML; } - if (actionsBtn) actionsBtn.disabled = false; + if (deleteBtn && deleteBtn !== targetBtn) { + deleteBtn.disabled = false; + deleteBtn.innerHTML = originalDeleteHtml || deleteBtn.innerHTML; + } + if (actionsBtn && actionsBtn !== targetBtn) actionsBtn.disabled = false; if (cancelBtn) cancelBtn.disabled = false; if (selectAll) selectAll.disabled = false; if (countEl) countEl.textContent = originalCountText; @@ -5000,7 +6144,7 @@ function _summaryIcon(data) { return `<svg width="14" height="14" viewBox="0 0 24 24" fill="${fill}"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>`; } -async function _runAiReplyFromButton(btn, em, data, mode) { +async function _runAiReplyFromButton(btn, em, data, mode, noteHint = '') { _snapEmailModalToLeftSidebar(btn.closest('.modal')); btn.disabled = true; const orig = btn.innerHTML; @@ -5012,7 +6156,7 @@ async function _runAiReplyFromButton(btn, em, data, mode) { btn.appendChild(wp.element); } catch (_) {} try { - if (state._onEmailClick) await state._onEmailClick({ email: em, emailData: data, mode }); + if (state._onEmailClick) await state._onEmailClick({ email: em, emailData: data, mode, noteHint }); } finally { try { wp && wp.stop(); } catch (_) {} btn.disabled = false; @@ -5030,10 +6174,31 @@ function _showAiReplyChoice(btn, em, data) { const rect = btn.getBoundingClientRect(); const menu = document.createElement('div'); menu.className = 'email-ai-reply-choice'; + /* Clamp width to viewport minus 16px margin so the menu (textarea + + Fast/Full buttons) never spills off the right edge on narrow + mobile screens. */ + const menuMaxW = Math.min(220, window.innerWidth - 16); + const left = Math.max(8, Math.min(rect.left, window.innerWidth - menuMaxW - 8)); + /* Vertical placement: prefer below the button, but flip above if + there's not enough room (e.g. button near bottom of viewport). + Estimated menu height is ~150px (textarea + buttons + padding). */ + const estHeight = 150; + const spaceBelow = window.innerHeight - rect.bottom - 8; + const spaceAbove = rect.top - 8; + let top; + if (spaceBelow >= estHeight || spaceBelow >= spaceAbove) { + top = Math.max(8, Math.min(rect.bottom + 6, window.innerHeight - estHeight - 8)); + } else { + top = Math.max(8, rect.top - estHeight - 6); + } menu.style.cssText = [ 'position:fixed', - `left:${Math.max(8, Math.min(rect.left, window.innerWidth - 190))}px`, - `top:${Math.min(window.innerHeight - 96, rect.bottom + 6)}px`, + `left:${left}px`, + `top:${top}px`, + `max-width:${menuMaxW}px`, + `max-height:${window.innerHeight - 16}px`, + 'overflow:auto', + 'box-sizing:border-box', 'z-index:10060', 'display:flex', 'gap:6px', @@ -5043,30 +6208,66 @@ function _showAiReplyChoice(btn, em, data) { 'border-radius:7px', 'box-shadow:0 8px 24px rgba(0,0,0,.28)', ].join(';'); + // Fast = lightning bolt (already used as a 'fast' glyph elsewhere in the app). + // Full = layered concentric circles to suggest "more, deeper" — not a fully + // filled circle so it reads as a complement to the lightning, not as a "stop". menu.innerHTML = ` - <button class="memory-toolbar-btn" data-mode="ai-reply-fast" title="Shorter, faster draft">Fast</button> - <button class="memory-toolbar-btn" data-mode="ai-reply-full" title="Uses the fuller reply context">Full</button> + <div class="email-ai-reply-row" style="display:flex;flex-direction:column;gap:6px;min-width:180px;"> + <textarea data-note-input rows="2" placeholder="Add context (optional)" style="width:100%;box-sizing:border-box;resize:vertical;min-height:42px;font-family:inherit;font-size:11px;padding:5px 6px;border-radius:5px;border:1px solid var(--border,#333);background:var(--bg-elev,#1a1a1a);color:var(--fg);"></textarea> + <div style="display:flex;align-items:center;gap:4px;"> + <button class="memory-toolbar-btn" data-mode="ai-reply-fast" title="Shorter, faster draft" style="display:inline-flex;align-items:center;justify-content:center;gap:5px;flex:1;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="var(--accent, var(--red))" aria-hidden="true"><polygon points="13 2 3 14 12 14 11 22 21 10 12 10 13 2"/></svg> + Fast + </button> + <button class="memory-toolbar-btn" data-mode="ai-reply-full" title="Uses the fuller reply context" style="display:inline-flex;align-items:center;justify-content:center;gap:5px;flex:1;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true" style="color:var(--accent, var(--red));"><circle cx="12" cy="12" r="6"/></svg> + Full + </button> + </div> + </div> `; + const noteInput = menu.querySelector('[data-note-input]'); + setTimeout(() => noteInput.focus(), 0); menu.addEventListener('click', async (ev) => { const choice = ev.target.closest('[data-mode]'); if (!choice) return; ev.preventDefault(); ev.stopPropagation(); const mode = choice.getAttribute('data-mode') || 'ai-reply'; + const noteHint = (noteInput.value || '').trim(); _closeAiReplyChoice(); - await _runAiReplyFromButton(btn, em, data, mode); + await _runAiReplyFromButton(btn, em, data, mode, noteHint); }); + // Esc closes the popover; ignore plain clicks inside the menu so the + // textarea stays focused. + menu.addEventListener('mousedown', (ev) => ev.stopPropagation()); document.body.appendChild(menu); - setTimeout(() => document.addEventListener('click', _closeAiReplyChoice, true), 0); + // Outside-click closer: only fires when the click target is OUTSIDE + // the menu. The original handler closed on any click which made + // focusing the textarea immediately dismiss the popover. + const outsideClose = (ev) => { + if (menu.contains(ev.target)) return; + document.removeEventListener('click', outsideClose, true); + _closeAiReplyChoice(); + }; + setTimeout(() => document.addEventListener('click', outsideClose, true), 0); } function _handleAiReplyButton(ev, em, data) { ev.stopPropagation(); const btn = ev.currentTarget; - if (data?.cached_ai_reply) { + // First click on a cached email surfaces the cached draft. Second + // click clears the cache and opens the Fast/Full + context menu so + // the user can ask for a fresh draft (with new steering). + if (data?.cached_ai_reply && !btn.dataset.shownOnce) { + btn.dataset.shownOnce = '1'; _runAiReplyFromButton(btn, em, data, 'ai-reply'); return; } + if (data?.cached_ai_reply) { + data.cached_ai_reply = null; + btn.dataset.shownOnce = ''; + } _showAiReplyChoice(btn, em, data); } @@ -5152,11 +6353,57 @@ function _showLibRemindSubmenu(em, parentDropdown) { tmp.addEventListener('blur', () => setTimeout(() => tmp.remove(), 200)); }); parentDropdown.appendChild(customItem); + // "Note" — prompts for free-text and saves it as a note without a + // due_date, so no timer/reminder fires. + const noteItem = document.createElement('div'); + noteItem.className = 'dropdown-item-compact'; + noteItem.innerHTML = '<span>Note</span>'; + noteItem.addEventListener('click', (e) => { + e.stopPropagation(); + parentDropdown.remove(); + _promptEmailNote(em); + }); + parentDropdown.appendChild(noteItem); } -async function _createEmailReplyReminder(em, dueDate) { +function _promptEmailNote(em) { + const overlay = document.createElement('div'); + overlay.style.cssText = 'position:fixed;inset:0;z-index:99998;background:rgba(0,0,0,0.45);display:flex;align-items:center;justify-content:center;padding:16px;'; + const card = document.createElement('div'); + card.style.cssText = 'background:var(--bg);border:1px solid var(--border);border-radius:8px;padding:14px;min-width:280px;max-width:min(420px, 92vw);display:flex;flex-direction:column;gap:8px;box-shadow:0 12px 32px rgba(0,0,0,0.4);'; + const subject = em.subject || '(no subject)'; + card.innerHTML = ` + <div style="font-size:11px;opacity:0.6;">Note about ${_esc(subject)}</div> + <textarea data-note placeholder="Write your note…" rows="4" style="resize:vertical;min-height:80px;font-family:inherit;font-size:12px;padding:7px 8px;border-radius:6px;border:1px solid var(--border);background:var(--bg-elev,#1a1a1a);color:var(--fg);box-sizing:border-box;width:100%;"></textarea> + <div style="display:flex;gap:6px;justify-content:flex-end;"> + <button class="memory-toolbar-btn" data-act="cancel">Cancel</button> + <button class="memory-toolbar-btn active" data-act="save">Save</button> + </div> + `; + overlay.appendChild(card); + document.body.appendChild(overlay); + const ta = card.querySelector('[data-note]'); + setTimeout(() => ta.focus(), 0); + const close = () => overlay.remove(); + overlay.addEventListener('click', (ev) => { if (ev.target === overlay) close(); }); + card.querySelector('[data-act="cancel"]').addEventListener('click', close); + card.querySelector('[data-act="save"]').addEventListener('click', async () => { + const text = (ta.value || '').trim(); + if (!text) { ta.focus(); return; } + close(); + await _createEmailReplyReminder(em, null, text); + }); + ta.addEventListener('keydown', (ev) => { + if (ev.key === 'Escape') close(); + else if ((ev.ctrlKey || ev.metaKey) && ev.key === 'Enter') card.querySelector('[data-act="save"]').click(); + }); +} + +async function _createEmailReplyReminder(em, dueDate, customText = '') { const pad = n => String(n).padStart(2,'0'); - const iso = `${dueDate.getFullYear()}-${pad(dueDate.getMonth()+1)}-${pad(dueDate.getDate())}T${pad(dueDate.getHours())}:${pad(dueDate.getMinutes())}`; + const iso = dueDate + ? `${dueDate.getFullYear()}-${pad(dueDate.getMonth()+1)}-${pad(dueDate.getDate())}T${pad(dueDate.getHours())}:${pad(dueDate.getMinutes())}` + : null; const fullFrom = em.from || em.sender || ''; // Extract just the first name from "First Last <email@x>" or fall back to email local part let from = 'someone'; @@ -5171,17 +6418,18 @@ async function _createEmailReplyReminder(em, dueDate) { const subject = em.subject || '(no subject)'; const folder = state._libFolder || 'INBOX'; const deepLink = `${window.location.origin}/#email=${encodeURIComponent(folder)}:${em.uid}`; + const itemText = customText || `Reply to ${from}: ${subject}`; const payload = { title: `Reply: ${subject}`, note_type: 'todo', items: [ - { text: `Reply to ${from}: ${subject}`, checked: false }, + { text: itemText, checked: false }, ], content: `Open email: ${deepLink}`, label: 'email reminder', - due_date: iso, source: 'email', }; + if (iso) payload.due_date = iso; try { const res = await fetch(`${API_BASE}/api/notes`, { method: 'POST', credentials: 'same-origin', @@ -5190,8 +6438,12 @@ async function _createEmailReplyReminder(em, dueDate) { }); if (!res.ok) throw new Error('Failed'); const { showToast } = await import('./ui.js'); - const fmt = dueDate.toLocaleString([], { month:'short', day:'numeric', hour:'numeric', minute:'2-digit' }); - showToast(`Todo reminder set for ${fmt}`); + if (dueDate) { + const fmt = dueDate.toLocaleString([], { month:'short', day:'numeric', hour:'numeric', minute:'2-digit' }); + showToast(`Todo reminder set for ${fmt}`); + } else { + showToast('Reply note saved'); + } if ('Notification' in window && Notification.permission === 'default') { try { Notification.requestPermission(); } catch {} } diff --git a/static/js/memory.js b/static/js/memory.js index 1df76a37a..b5de2bfe6 100644 --- a/static/js/memory.js +++ b/static/js/memory.js @@ -18,6 +18,80 @@ let selectedIds = new Set(); const MEMORY_CATEGORIES = ['fact', 'identity', 'preference', 'contact', 'project', 'goal', 'task']; +// Sort-option icons for the custom Memory sort picker (and Skills picker +// once it reuses the same markup). Each value maps to a 13px Feather-style +// SVG so the icon visually distinguishes Newest / Oldest / A-Z / Most used. +const _MEMORY_SORT_ICONS = { + newest: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><polyline points="12 6 12 12 16 14"/></svg>', + oldest: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 12a9 9 0 1 0 3-6.7L3 8"/><polyline points="3 3 3 8 8 8"/><polyline points="12 7 12 12 16 14"/></svg>', + alpha: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 4h6"/><path d="M3 10h6"/><path d="M3 16h4"/><path d="M14 4l4 12"/><path d="M16 12h4"/><polyline points="17 18 21 14 17 10"/><line x1="21" y1="14" x2="13" y2="14"/></svg>', + uses: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M8.5 14.5A2.5 2.5 0 0 0 11 12c0-1.38-.5-2-1-3-1.072-2.143-.224-4.054 2-6 .5 2.5 2 4.9 4 6.5 2 1.6 3 3.5 3 5.5a7 7 0 1 1-14 0c0-1.153.433-2.294 1-3a2.5 2.5 0 0 0 2.5 2.5z"/></svg>', +}; + +function _memorySortIcon(value) { + return _MEMORY_SORT_ICONS[value] || _MEMORY_SORT_ICONS.newest; +} + +function _renderMemorySortPickerCurrent() { + const sel = document.getElementById('memory-sort'); + const btn = document.getElementById('memory-sort-btn'); + if (!sel || !btn) return; + const value = sel.value || 'newest'; + const opt = sel.querySelector(`option[value="${CSS.escape(value)}"]`); + const label = opt ? opt.textContent : value; + const iconWrap = btn.querySelector('.memory-sort-icon-cur'); + const labelEl = btn.querySelector('.memory-sort-label'); + if (iconWrap) iconWrap.innerHTML = _memorySortIcon(value); + if (labelEl) labelEl.textContent = label; +} + +function _initMemorySortPicker() { + const sel = document.getElementById('memory-sort'); + const picker = document.getElementById('memory-sort-picker'); + const btn = document.getElementById('memory-sort-btn'); + const menu = document.getElementById('memory-sort-menu'); + if (!sel || !picker || !btn || !menu || picker._wired) return; + picker._wired = true; + + const items = Array.from(sel.children) + .filter(o => o.tagName === 'OPTION') + .map(o => ({ value: o.value, label: o.textContent })); + + menu.innerHTML = items.map(it => ` + <button type="button" role="option" class="memory-sort-item" data-value="${it.value}"> + <span class="memory-sort-item-icon">${_memorySortIcon(it.value)}</span> + <span class="memory-sort-item-label">${it.label}</span> + </button> + `).join(''); + + const close = () => { menu.hidden = true; btn.setAttribute('aria-expanded', 'false'); }; + const open = () => { menu.hidden = false; btn.setAttribute('aria-expanded', 'true'); }; + + btn.addEventListener('click', (e) => { + e.stopPropagation(); + if (menu.hidden) open(); else close(); + }); + menu.addEventListener('click', (e) => { + const item = e.target.closest('.memory-sort-item'); + if (!item) return; + sel.value = item.dataset.value; + sel.dispatchEvent(new Event('change', { bubbles: true })); + _renderMemorySortPickerCurrent(); + close(); + }); + document.addEventListener('click', (e) => { + if (!menu.hidden && !picker.contains(e.target)) close(); + }); + document.addEventListener('keydown', (e) => { + if (e.key === 'Escape' && !menu.hidden) { + e.stopPropagation(); + close(); + } + }, { capture: true }); + + _renderMemorySortPickerCurrent(); +} + function _ensureNewMemoryCategorySelect() { const sel = document.getElementById('new-memory-category'); if (!sel || sel.dataset.wired === '1') return; @@ -334,13 +408,16 @@ export async function loadMemories() { // ---- Bulk select mode ---- +const _SELECT_BTN_DOT_SVG = '<svg class="memory-select-btn-icon" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:3px;"><circle cx="12" cy="12" r="10"/><circle cx="12" cy="12" r="3" fill="currentColor" stroke="none"/></svg>'; +const _SELECT_BTN_X_SVG = '<svg class="memory-select-btn-icon" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" style="vertical-align:-2px;margin-right:3px;"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>'; + function enterSelectMode() { selectMode = true; selectedIds.clear(); const bulkBar = document.getElementById('memory-bulk-bar'); const selectBtn = document.getElementById('memory-select-btn'); if (bulkBar) bulkBar.classList.remove('hidden'); - if (selectBtn) { selectBtn.classList.add('active'); selectBtn.textContent = 'Cancel'; } + if (selectBtn) { selectBtn.classList.add('active'); selectBtn.innerHTML = _SELECT_BTN_X_SVG + 'Cancel'; } updateBulkCount(); renderMemoryList(); } @@ -352,7 +429,7 @@ function exitSelectMode() { const selectBtn = document.getElementById('memory-select-btn'); const selectAll = document.getElementById('memory-select-all'); if (bulkBar) bulkBar.classList.add('hidden'); - if (selectBtn) { selectBtn.classList.remove('active'); selectBtn.textContent = 'Select'; } + if (selectBtn) { selectBtn.classList.remove('active'); selectBtn.innerHTML = _SELECT_BTN_DOT_SVG + 'Select'; } if (selectAll) selectAll.checked = false; renderMemoryList(); } @@ -449,7 +526,7 @@ export async function tidyMemories() { const data = await res.json(); if ((data.removed || 0) === 0) { if (tidySpinner) tidySpinner.destroy(); - if (tidyBtn) { tidyBtn.disabled = false; tidyBtn.innerHTML = '<svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;margin-right:2px;"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg> Tidy'; } + if (tidyBtn) { tidyBtn.disabled = false; tidyBtn.innerHTML = '<svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;margin-right:2px;color:var(--accent, var(--red));"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg> Tidy'; } showToast('Already clean'); return; } @@ -492,7 +569,7 @@ export async function tidyMemories() { tidyBtn.disabled = false; tidyBtn.style.border = ''; tidyBtn.style.background = ''; - tidyBtn.innerHTML = '<svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;margin-right:2px;"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg> Tidy'; + tidyBtn.innerHTML = '<svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:-1px;margin-right:2px;color:var(--accent, var(--red));"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg> Tidy'; } } } @@ -1387,6 +1464,7 @@ document.addEventListener('DOMContentLoaded', () => { renderMemoryList(); }); } + _initMemorySortPicker(); const tidyBtn = document.getElementById('memory-tidy-btn'); if (tidyBtn) tidyBtn.addEventListener('click', tidyMemories); diff --git a/static/js/modalSnap.js b/static/js/modalSnap.js index e7cce55dd..48b3922ac 100644 --- a/static/js/modalSnap.js +++ b/static/js/modalSnap.js @@ -302,6 +302,7 @@ function _anchorLeftDock(content) { } } +export function collapseSidebarToRail() { return _collapseSidebarToRail(); } function _collapseSidebarToRail() { const sidebar = document.getElementById('sidebar'); const rail = document.getElementById('icon-rail'); @@ -808,7 +809,10 @@ export function makeEdgeDockController(modal, side = 'right', dockClass) { handle.style.bottom = '0'; handle.style.width = '10px'; handle.style.cursor = 'col-resize'; - handle.style.background = 'linear-gradient(to right, transparent 0 3px, color-mix(in srgb, var(--accent, var(--red)) 35%, transparent) 3px 7px, transparent 7px 10px)'; + // Invisible at rest, accent stripe fades in on hover (see + // .edge-dock-resize-handle CSS rule). + handle.style.background = 'transparent'; + handle.style.transition = 'background 0.18s ease'; handle.style.pointerEvents = 'auto'; handle.style.touchAction = 'none'; handle.style.display = 'none'; @@ -994,7 +998,7 @@ export function makeEdgeDockController(modal, side = 'right', dockClass) { stripe.style.bottom = '0'; stripe.style.width = '10px'; stripe.style.cursor = 'col-resize'; - stripe.style.zIndex = '9999'; + stripe.style.zIndex = '261'; stripe.style.background = 'linear-gradient(to right, transparent 0 3px, color-mix(in srgb, var(--accent, var(--red)) 35%, transparent) 3px 7px, transparent 7px 10px)'; stripe.style.pointerEvents = 'auto'; stripe.style.touchAction = 'none'; diff --git a/static/js/notes.js b/static/js/notes.js index e64e5035c..58dff6e7f 100644 --- a/static/js/notes.js +++ b/static/js/notes.js @@ -1099,6 +1099,9 @@ export function openPanel() { if (_open) return; _open = true; _editingId = null; + // Reset the search filter — the rebuilt pane's search input renders empty, so a + // stale _searchQuery would silently hide non-matching notes after a reopen. + _searchQuery = ''; _clearViewedReminderGlows(); _firedDotDismissedAt = Date.now(); try { localStorage.setItem(REMINDER_DISMISSED_AT_KEY, String(_firedDotDismissedAt)); } catch {} diff --git a/static/js/providers.js b/static/js/providers.js index f42afcd67..da50fff89 100644 --- a/static/js/providers.js +++ b/static/js/providers.js @@ -147,4 +147,31 @@ export function providerLabel(endpointUrl) { return host.replace(/^api\./i, ""); } -export default { providerLogo, providerLabel }; +// Map endpoint URL → logo SVG using the same model-id regex catalog. +// Tests host + port + path so loopback servers (e.g. Ollama on +// localhost:11434) still match by port. Falls back to null when nothing +// recognises the URL, so callers can render a neutral placeholder. +export function providerLogoFromUrl(url) { + if (!url) return null; + let host = '', port = '', path = ''; + try { + const u = new URL(url); + host = u.hostname; port = u.port; path = u.pathname || ''; + } catch (_) { + const raw = String(url).replace(/^[a-z]+:\/\//i, ''); + const slashIdx = raw.indexOf('/'); + const hostport = slashIdx >= 0 ? raw.slice(0, slashIdx) : raw; + path = slashIdx >= 0 ? raw.slice(slashIdx) : ''; + const colon = hostport.lastIndexOf(':'); + host = colon >= 0 ? hostport.slice(0, colon) : hostport; + port = colon >= 0 ? hostport.slice(colon + 1) : ''; + } + // Build candidate strings to test against the provider catalog. + const candidates = [host, port ? `${host}:${port}` : '', port ? `:${port}` : '', path].filter(Boolean); + for (const [re, svg] of _PROVIDERS) { + if (candidates.some(c => re.test(c))) return svg; + } + return null; +} + +export default { providerLogo, providerLabel, providerLogoFromUrl }; diff --git a/static/js/research/panel.js b/static/js/research/panel.js index d515580ad..3abf75fb1 100644 --- a/static/js/research/panel.js +++ b/static/js/research/panel.js @@ -7,6 +7,26 @@ import createResearchSynapse from '../researchSynapse.js'; import spinnerModule from '../spinner.js'; import { sortModelIds } from '../modelSort.js'; +// Rotating research textarea placeholders — pick one at random each +// time the panel is rendered so the example keeps feeling fresh. +const _RESEARCH_HINTS = [ + "e.g. Trace Odysseus's ten-year journey home from Troy — every island, monster, and detour, and why each one cost him", + "e.g. Compare Rust and Go for building a high-throughput web API in 2026", + "e.g. Fact-check whether honey actually never spoils", + "e.g. How to roast a duck so the skin stays crispy", + "e.g. The collapse of Bronze Age civilizations — leading theories and the evidence behind each", + "e.g. Best M.2 NVMe SSDs under $200 for a home AI workstation", + "e.g. Why do cats knead with their paws? Cover the leading behavioural explanations", + "e.g. Side effects and benefits of long-term creatine supplementation", + "e.g. How does end-to-end encryption work in Signal, step by step", + "e.g. The history of the printing press in East Asia, 700 CE → 1600 CE", +]; +function _pickResearchHint() { + const i = Math.floor(Math.random() * _RESEARCH_HINTS.length); + // Escape double-quotes so we can safely splice into a placeholder="…" attribute. + return _RESEARCH_HINTS[i].replace(/"/g, '"'); +} + // jobId -> { synapse, status } — survives across _renderJobs() rebuilds so // the SVG keeps its accumulated nodes/edges between progress events. const _jobSynapses = new Map(); @@ -49,13 +69,12 @@ try { _settingsCollapsed = localStorage.getItem(_COLLAPSE_KEY) === '1'; } catch function _saveSettingsToStorage() { try { - const activeCat = document.querySelector('.research-cat.active'); localStorage.setItem(_SETTINGS_KEY, JSON.stringify({ max_rounds: document.getElementById('research-rounds')?.value || '0', search_provider: document.getElementById('research-search-provider')?.value || '', endpoint_id: document.getElementById('research-endpoint')?.value || '', model: document.getElementById('research-model')?.value || '', - category: activeCat?.dataset.cat || '', + category: document.getElementById('research-category')?.value || '', })); } catch {} } @@ -346,15 +365,14 @@ function _buildPanelHTML() { </div> <div class="modal-body research-pane-body" data-no-swipe-dismiss> <div class="research-new-job"> - <div style="display:flex;align-items:baseline;gap:8px;margin-bottom:2px;"> - <h2 style="margin:0;padding:0;line-height:1;">Research <span id="research-stats" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal"></span></h2> + <div style="display:flex;align-items:center;gap:8px;margin-bottom:2px;"> + <h2 style="margin:0;padding:0;line-height:1;display:inline-flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="var(--accent, var(--red))" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="flex-shrink:0;"><path d="M6 18h8"/><path d="M3 22h18"/><path d="M14 22a7 7 0 1 0 0-14h-1"/><path d="M9 14h2"/><path d="M9 12a2 2 0 0 1-2-2V6h4v4a2 2 0 0 1-2 2Z"/><path d="M12 6V3a1 1 0 0 0-1-1H9a1 1 0 0 0-1 1v3"/></svg>Research <span id="research-stats" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal;position:relative;top:4px;"></span></h2> </div> - <p class="memory-desc doclib-desc" style="margin-top:6px;display:flex;align-items:center;gap:6px;"> - <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="flex-shrink:0;opacity:0.8;"><path d="M6 18h8"/><path d="M3 22h18"/><path d="M14 22a7 7 0 1 0 0-14h-1"/><path d="M9 14h2"/><path d="M9 12a2 2 0 0 1-2-2V6h4v4a2 2 0 0 1-2 2Z"/><path d="M12 6V3a1 1 0 0 0-1-1H9a1 1 0 0 0-1 1v3"/></svg> + <p class="memory-desc doclib-desc" style="margin-top:2px;display:flex;align-items:center;gap:6px;flex-wrap:wrap;"> <span>Multi-step web research with an LLM-in-the-loop agent</span> + <span id="research-no-past-hint" style="display:none;font-size:11px;opacity:0.7;position:relative;top:-4px;">— past runs in <button type="button" class="research-library-link" style="background:none;border:none;padding:0;font:inherit;color:var(--accent, var(--red));cursor:pointer;text-decoration:underline;">Library, Research</button></span> </p> - <div id="research-no-past-hint" class="memory-desc doclib-desc" style="display:none;margin-top:-2px;font-size:11px;opacity:0.7;">All past research found in <button type="button" class="research-library-link">Library, Research</button></div> - <textarea id="research-query" class="research-query" placeholder="e.g. Trace Odysseus's ten-year journey home from Troy — every island, monster, and detour, and why each one cost him" rows="4"></textarea> + <textarea id="research-query" class="research-query" placeholder="${_pickResearchHint()}" rows="4"></textarea> <div class="research-category-row" id="research-category-row"> <button class="research-cat active" data-cat="" title="LLM auto-detects the best format">Auto</button> <button class="research-cat" data-cat="product">Product</button> @@ -363,13 +381,23 @@ function _buildPanelHTML() { <button class="research-cat" data-cat="factcheck">Fact-check</button> </div> <button id="research-settings-toggle" class="research-settings-toggle${chevronCls}"> - Settings<span class="research-settings-chevron">${_chevronIcon}</span> + <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:4px;opacity:0.85;flex-shrink:0;"><circle cx="12" cy="12" r="3"/><path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1-2-2 2 2 0 0 1 2-2h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 2-2 2 2 0 0 1 2 2v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-.09a1.65 1.65 0 0 0-1.51 1z"/></svg>Settings<span class="research-settings-chevron">${_chevronIcon}</span> </button> <div id="research-settings-body" class="research-settings-row"${settingsHidden}> <label class="research-setting"> - <span class="research-setting-label">Rounds</span> + <span class="research-setting-label">Rounds <span class="hwfit-help-chip hwfit-help-chip-inline" title="How many search → read → reflect rounds the agent runs. More rounds = deeper coverage, longer wait, more tokens.">?</span></span> <select id="research-rounds">${roundOpts}</select> </label> + <label class="research-setting"> + <span class="research-setting-label">Format <span class="hwfit-help-chip hwfit-help-chip-inline" title="Auto lets the LLM pick the output shape. Override when you specifically want a Compare table, How-to, Product, or Fact-check.">?</span></span> + <select id="research-category"> + <option value="" selected>Auto</option> + <option value="product">Product</option> + <option value="comparison">Compare</option> + <option value="howto">How-to</option> + <option value="factcheck">Fact-check</option> + </select> + </label> <label class="research-setting"> <span class="research-setting-label">Search engine</span> <select id="research-search-provider">${providerOpts}</select> @@ -418,8 +446,8 @@ function _dismissKeyboard(input) { /** Reset the category selector back to "Auto" (called after each start). */ function _resetCategoryToAuto() { - document.querySelectorAll('.research-cat').forEach(b => - b.classList.toggle('active', (b.dataset.cat || '') === '')); + const sel = document.getElementById('research-category'); + if (sel) sel.value = ''; } function _wireEvents(pane) { @@ -433,13 +461,6 @@ function _wireEvents(pane) { pane.querySelector('#research-start-btn').addEventListener('click', _handleStart); pane.querySelector('#research-add-btn').addEventListener('click', _handleAdd); - pane.querySelectorAll('.research-cat').forEach(btn => { - btn.addEventListener('click', () => { - pane.querySelectorAll('.research-cat').forEach(b => b.classList.remove('active')); - btn.classList.add('active'); - }); - }); - pane.querySelector('#research-settings-toggle').addEventListener('click', () => { const body = document.getElementById('research-settings-body'); const btn = document.getElementById('research-settings-toggle'); @@ -465,8 +486,7 @@ function _wireEvents(pane) { } function _readSettings() { - const activeCat = document.querySelector('.research-cat.active'); - const category = activeCat?.dataset.cat || undefined; + const category = document.getElementById('research-category')?.value || undefined; const settings = { max_rounds: parseInt(document.getElementById('research-rounds')?.value || '0', 10), search_provider: document.getElementById('research-search-provider')?.value || undefined, @@ -505,9 +525,8 @@ function _editJob(job) { } // Restore category const cat = job.category || ''; - document.querySelectorAll('.research-cat').forEach(b => { - b.classList.toggle('active', b.dataset.cat === cat); - }); + const catSel = document.getElementById('research-category'); + if (catSel) catSel.value = cat; // Restore settings const s = job.settings || {}; const roundsEl = document.getElementById('research-rounds'); @@ -594,9 +613,8 @@ function _restoreSavedSettings() { const saved = _loadSettingsFromStorage(); if (!saved) return; if (saved.category !== undefined) { - document.querySelectorAll('.research-cat').forEach(b => { - b.classList.toggle('active', b.dataset.cat === saved.category); - }); + const catSel = document.getElementById('research-category'); + if (catSel) catSel.value = saved.category; } // Rounds intentionally defaults to "Auto" on every open — don't restore. // Users can pick a specific cap each time if needed. @@ -785,22 +803,26 @@ function _renderJobs() { }); const body = document.createElement('div'); body.className = 'research-section-body'; - // Hint inside the "Past research" header (second line, styled like the main - // Research description) — past research is kept in the Library's Research tab. + // Past Research header: link goes INLINE next to the title instead + // of on a second row. Append it to the title span as a small chip. if (key === 'past') { - const hint = document.createElement('div'); - hint.className = 'memory-desc doclib-desc research-library-hint'; - hint.innerHTML = 'All past research found in <button type="button" class="research-library-link">Library, Research</button>'; - hint.querySelector('.research-library-link').addEventListener('click', (e) => { - e.stopPropagation(); - // Close the research panel first so the Library opens ABOVE it on mobile - // (otherwise it stacks under the full-screen panel). - closePanel(); - if (window.documentModule && window.documentModule.openLibrary) { - window.documentModule.openLibrary({ tab: 'research' }); - } - }); - header.appendChild(hint); + const titleEl = header.querySelector('.research-section-title'); + if (titleEl) { + const hint = document.createElement('span'); + hint.className = 'research-library-hint research-library-hint-inline'; + hint.style.cssText = 'margin-left:8px;font-size:10.5px;opacity:0.65;font-weight:normal;'; + hint.innerHTML = '— all in <button type="button" class="research-library-link" style="background:none;border:none;padding:0;font:inherit;color:var(--accent, var(--red));cursor:pointer;text-decoration:underline;">Library, Research</button>'; + hint.querySelector('.research-library-link').addEventListener('click', (e) => { + e.stopPropagation(); + // Close the research panel first so the Library opens ABOVE it on mobile + // (otherwise it stacks under the full-screen panel). + closePanel(); + if (window.documentModule && window.documentModule.openLibrary) { + window.documentModule.openLibrary({ tab: 'research' }); + } + }); + titleEl.appendChild(hint); + } } arr.forEach(j => body.appendChild(_buildJobCard(j))); sec.appendChild(header); diff --git a/static/js/sessions.js b/static/js/sessions.js index 15dfde08a..a337be105 100644 --- a/static/js/sessions.js +++ b/static/js/sessions.js @@ -2258,8 +2258,8 @@ if (document.readyState === 'loading') { // Shared global listener to close all session dropdowns on click-away or Escape function _initDropdownDismiss() { document.addEventListener('click', (e) => { - if (e.target.closest('.session-dropdown-menu')) return; - document.querySelectorAll('.session-dropdown-menu').forEach(d => d.style.display = 'none'); + if (e.target.closest('.session-dropdown-menu, .session-folder-submenu')) return; + document.querySelectorAll('.session-dropdown-menu, .session-folder-submenu').forEach(d => d.style.display = 'none'); }); // Watch the sidebar — when it's hidden (any path: hamburger, swipe, mobile // collapse), close any open session dropdowns so they don't orphan over @@ -2268,14 +2268,16 @@ function _initDropdownDismiss() { if (_sb) { new MutationObserver(() => { if (_sb.classList.contains('hidden')) { - document.querySelectorAll('.session-dropdown-menu, .folder-submenu').forEach(d => d.style.display = 'none'); + document.querySelectorAll('.session-dropdown-menu, .session-folder-submenu').forEach(d => d.style.display = 'none'); } }).observe(_sb, { attributes: true, attributeFilter: ['class'] }); } document.addEventListener('keydown', (e) => { - if (e.key === 'Escape') { - document.querySelectorAll('.session-dropdown-menu').forEach(d => d.style.display = 'none'); - } + if (e.key !== 'Escape') return; + // Esc must dismiss both the parent dropdown AND the Move-to-folder + // submenu in one keypress — previously only the dropdown closed and + // the submenu was left orphaned on screen. + document.querySelectorAll('.session-dropdown-menu, .session-folder-submenu').forEach(d => d.style.display = 'none'); }); } diff --git a/static/js/settings.js b/static/js/settings.js index 6d0906c9e..1dbe7f6d7 100644 --- a/static/js/settings.js +++ b/static/js/settings.js @@ -6,10 +6,12 @@ import searchModule from './search.js'; import { makeWindowDraggable } from './windowDrag.js'; import { clearDockSide } from './modalSnap.js'; import { sortModelIds } from './modelSort.js'; +import { providerLogo } from './providers.js'; import { isAltGrEvent } from './platform.js'; let initialized = false; let modalEl = null; +let _authPolicy = { password_min_length: 8 }; function el(id) { return document.getElementById(id); } function esc(s) { return uiModule.esc(s); } @@ -81,6 +83,29 @@ function resetWindowPlacement() { ].forEach(prop => content.style.removeProperty(prop)); } +/* ── Delegated link: close Settings + open the Prompt (characters) modal ── */ +function initOpenPromptModalLink() { + document.addEventListener('click', async (e) => { + const link = e.target.closest('[data-open-prompt-modal]'); + if (!link) return; + e.preventDefault(); + // Close settings first so the prompt modal isn't stacked on top. + if (modalEl && !modalEl.classList.contains('hidden')) close(); + try { + const m = await import('./presets.js'); + const fn = m.openCustomPresetModal || (m.default && m.default.openCustomPresetModal); + if (typeof fn === 'function') fn(); + } catch (_) { + const modal = document.getElementById('custom-preset-modal'); + if (modal) modal.classList.remove('hidden'); + } + // Force the Persona tab (data-chartab="character") since the link's + // whole purpose is editing personas — not landing on Inject by default. + const personaTab = document.querySelector('#custom-preset-modal .preset-tab[data-chartab="character"]'); + if (personaTab) personaTab.click(); + }); +} + /* ── Close on backdrop / X ── */ function initClose() { modalEl.querySelector('.close-btn').addEventListener('click', close); @@ -90,6 +115,16 @@ function initClose() { }); document.addEventListener('keydown', e => { if (e.key !== 'Escape' || !modalEl || modalEl.classList.contains('hidden')) return; + // Bail when a transient popover inside the modal is open — Esc should + // dismiss just that, not the whole modal. Same-document listeners fire + // in registration order regardless of capture/bubble, so the popover's + // own handler can't pre-empt ours; we have to opt out here. + const popoverOpen = modalEl.querySelector( + '#adm-epLocalMoreMenu, #adm-epApiMoreMenu, #adm-provider-menu, #search-provider-menu, [data-popover-open="1"]' + ); + if (popoverOpen && popoverOpen.style.display !== 'none' && !popoverOpen.classList.contains('hidden')) { + return; + } // If an integration edit/add form is open inside the modal, close // just that — don't dismiss the whole settings modal. (Pressing // ESC mid-edit and losing the modal was a fast-typing footgun.) @@ -205,6 +240,41 @@ function _fillEndpointSelect(selectEl, endpoints, selected, keepBlank) { } else if (blankText !== null) { selectEl.value = ''; } + _syncEndpointLogo(selectEl); +} + +// Mirror the selected model's provider logo into a sibling <span id="<selectId>-logo">. +// Wires the change listener exactly once so we can call this every time the +// select is repopulated without piling on duplicate handlers. +function _syncModelLogo(selectEl) { + if (!selectEl) return; + const logoEl = document.getElementById(selectEl.id + '-logo'); + if (!logoEl) return; + const apply = () => { logoEl.innerHTML = providerLogo(selectEl.value) || ''; }; + apply(); + if (!selectEl.dataset.logoSync) { + selectEl.dataset.logoSync = '1'; + selectEl.addEventListener('change', apply); + } +} + +// Same idea but for endpoint dropdowns where the <option value="…"> +// is an opaque endpoint UUID — fall back to the option's text label +// so providerLogo() can pattern-match (Anthropic, OpenAI, Ollama, …). +function _syncEndpointLogo(selectEl) { + if (!selectEl) return; + const logoEl = document.getElementById(selectEl.id + '-logo'); + if (!logoEl) return; + const apply = () => { + const opt = selectEl.options[selectEl.selectedIndex]; + const label = (opt && opt.textContent) || selectEl.value || ''; + logoEl.innerHTML = providerLogo(label) || ''; + }; + apply(); + if (!selectEl.dataset.epLogoSync) { + selectEl.dataset.epLogoSync = '1'; + selectEl.addEventListener('change', apply); + } } function _fillModelSelect(selectEl, models, selected, keepBlank) { @@ -231,6 +301,7 @@ function _fillModelSelect(selectEl, models, selected, keepBlank) { } else if (blankText !== null) { selectEl.value = ''; } + _syncModelLogo(selectEl); } function _registerAiEndpointRefresh(fn) { @@ -338,7 +409,7 @@ function _bindFallbackWidget(opts) { rm.type = 'button'; rm.className = 'settings-fallback-remove'; rm.title = 'Remove fallback'; - rm.innerHTML = '×'; + rm.innerHTML = '<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>'; rm.addEventListener('click', function() { current.splice(idx, 1); render(); @@ -442,7 +513,7 @@ async function initDefaultChat() { rm.type = 'button'; rm.className = 'settings-fallback-remove'; rm.title = 'Remove fallback'; - rm.innerHTML = '×'; + rm.innerHTML = '<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>'; rm.addEventListener('click', function() { _fallbacks.splice(idx, 1); renderFallbacks(); @@ -706,7 +777,7 @@ async function initImageSettings() { const settings = await settingsRes.json(); if (settings.image_model) modelSel.value = settings.image_model; if (settings.image_quality) qualSel.value = settings.image_quality; - if (enabledToggle) enabledToggle.checked = settings.image_gen_enabled !== false; + if (enabledToggle) enabledToggle.checked = settings.image_gen_enabled === true; } catch (e) { console.warn('Failed to load settings', e); } function syncImgDisabled() { @@ -720,7 +791,7 @@ async function initImageSettings() { async function saveSettings() { try { await fetch('/api/auth/settings', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ image_gen_enabled: enabledToggle ? enabledToggle.checked : true, image_model: modelSel.value, image_quality: qualSel.value }) }); + body: JSON.stringify({ image_gen_enabled: enabledToggle ? enabledToggle.checked : false, image_model: modelSel.value, image_quality: qualSel.value }) }); msg.textContent = 'Saved'; msg.style.color = 'var(--fg)'; setTimeout(() => { msg.textContent = ''; }, 2000); } catch (e) { msg.textContent = 'Failed to save'; msg.style.color = 'var(--red)'; } } @@ -767,6 +838,7 @@ async function initVisionSettings() { const settingsRes = await fetch('/api/auth/settings', { credentials: 'same-origin' }); const settings = await settingsRes.json(); if (settings.vision_model) vlSel.value = settings.vision_model; + _syncModelLogo(vlSel); if (enabledToggle) enabledToggle.checked = settings.vision_enabled !== false; visionFallbackWidget = _bindFallbackWidget({ containerId: 'set-visionFallbacks', @@ -1057,13 +1129,16 @@ async function initSttSettings() { SEARCH TAB ═══════════════════════════════════════════ */ +var _LINK = function(href, text) { + return '<a href="' + href + '" target="_blank" rel="noopener noreferrer" style="color:var(--accent, var(--red));text-decoration:underline;">' + text + '</a>'; +}; var _searchProviderHints = { - searxng: 'Self-hosted SearXNG instance. Leave URL empty to use the SEARXNG_INSTANCE env var.', - duckduckgo: 'Free search — no API key required. Works out of the box.', - brave: 'Get your API key from brave.com/search/api', - google_pse: 'Requires a Google API key and a Programmable Search Engine ID (CX). Create one at programmablesearchengine.google.com', - tavily: 'AI-optimized search. 1,000 free credits/month at tavily.com', - serper: 'Google results via API. 2,500 free queries at serper.dev', + searxng: 'Private, self-hosted instance. Leave URL empty to use the SEARXNG_INSTANCE env var.', + duckduckgo: 'No API key needed, but rate-limited — heavy use can return empty results. Configure a fallback below.', + brave: 'Get your API key from ' + _LINK('https://brave.com/search/api/', 'brave.com/search/api'), + google_pse: 'Requires a Google API key and a Programmable Search Engine ID (CX). Create one at ' + _LINK('https://programmablesearchengine.google.com/', 'programmablesearchengine.google.com'), + tavily: 'AI-optimized search. 1,000 free credits/month at ' + _LINK('https://tavily.com/', 'tavily.com'), + serper: 'Google results via API. 2,500 free queries at ' + _LINK('https://serper.dev/', 'serper.dev'), disabled: 'Web search and deep research tools will be unavailable.', }; var _searchNeedsKey = { brave: 1, google_pse: 1, tavily: 1, serper: 1 }; @@ -1102,7 +1177,7 @@ async function initSearchSettings() { urlRow.style.display = prov === 'searxng' ? 'flex' : 'none'; keyRow.style.display = _searchNeedsKey[prov] ? 'flex' : 'none'; cxRow.style.display = prov === 'google_pse' ? 'flex' : 'none'; - hint.textContent = _searchProviderHints[prov] || ''; + hint.innerHTML = _searchProviderHints[prov] || ''; if (prov === 'brave') keyInput.placeholder = 'Brave API key'; else if (prov === 'google_pse') keyInput.placeholder = 'Google API key'; else if (prov === 'tavily') keyInput.placeholder = 'Tavily API key'; @@ -1266,63 +1341,81 @@ async function initSearchSettings() { .map(function(o) { return { value: o.value, label: o.textContent, logo: o.dataset.searchLogo }; }) .filter(function(o) { return !inChain.has(o.value); }); } + var addBtn = el('set-searchAddFallback'); + var TRASH_SVG = '<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg>'; function _renderFallbackChain() { if (!fbWrap) return; var chain = (_settings.search_fallback_chain || []).slice(); - var esc = function(s) { return String(s || '').replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); }; - var chipsHtml = chain.map(function(p, i) { - var label = _searchLabels[p] || p; - var logo = _SEARCH_PROVIDER_LOGOS[p] || ''; - return '<span class="search-fb-chip" draggable="true" data-idx="' + i + '" data-value="' + esc(p) + '">' + - '<span class="search-fb-grip" title="Drag to reorder">⋮⋮</span>' + - '<span class="search-fb-logo">' + logo + '</span>' + - '<span>' + esc(label) + '</span>' + - '<button type="button" class="search-fb-remove" data-value="' + esc(p) + '" title="Remove">×</button>' + - '</span>'; - }).join(''); - var addOptions = _availableFallbackOptions(); - var addSelect = addOptions.length - ? '<select class="search-fb-add" id="search-fb-add"><option value="">+ Add</option>' + - addOptions.map(function(o) { return '<option value="' + esc(o.value) + '">' + esc(o.label) + '</option>'; }).join('') + - '</select>' - : ''; - fbWrap.innerHTML = chipsHtml + addSelect; - // Wire chip remove + drag-reorder + add - fbWrap.querySelectorAll('.search-fb-remove').forEach(function(btn) { - btn.addEventListener('click', function() { - var next = (_settings.search_fallback_chain || []).filter(function(p) { return p !== btn.dataset.value; }); - _saveFallbackChain(next); + fbWrap.innerHTML = ''; + chain.forEach(function(p, idx) { + var row = document.createElement('div'); + row.className = 'settings-fallback-row'; + + var num = document.createElement('span'); + num.className = 'settings-fallback-num'; + num.textContent = (idx + 1) + '.'; + row.appendChild(num); + + // Inline logo so the row identifies its provider at a glance even + // before opening the dropdown. The <select> below still drives + // selection; we just mirror its value into the logo span. + var logoWrap = document.createElement('span'); + logoWrap.style.cssText = 'display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;flex-shrink:0;color:var(--fg);'; + var setLogo = function(val) { + var srcOpt = Array.from(provSel.options).find(function(o) { return o.value === val; }); + logoWrap.innerHTML = srcOpt ? _searchProviderLogoSvg(srcOpt.dataset.searchLogo) : ''; + }; + setLogo(p); + row.appendChild(logoWrap); + + var sel = document.createElement('select'); + sel.className = 'settings-select'; + // Options: this row's current value + every other provider not yet in the chain (and not the primary or 'disabled'). + var primary = provSel.value; + var others = new Set(chain.filter(function(x) { return x !== p; }).concat([primary, 'disabled'])); + Array.from(provSel.options).forEach(function(o) { + if (o.value !== p && others.has(o.value)) return; + var opt = document.createElement('option'); + opt.value = o.value; + opt.textContent = o.textContent; + sel.appendChild(opt); }); - }); - var addSel = el('search-fb-add'); - if (addSel) { - addSel.addEventListener('change', function() { - if (!addSel.value) return; + sel.value = p; + sel.addEventListener('change', function() { + setLogo(sel.value); var next = (_settings.search_fallback_chain || []).slice(); - if (!next.includes(addSel.value)) next.push(addSel.value); + next[idx] = sel.value; _saveFallbackChain(next); }); + row.appendChild(sel); + + var rm = document.createElement('button'); + rm.type = 'button'; + rm.className = 'settings-fallback-remove'; + rm.title = 'Remove fallback'; + rm.innerHTML = TRASH_SVG; + rm.addEventListener('click', function() { + var next = (_settings.search_fallback_chain || []).filter(function(x, i) { return i !== idx; }); + _saveFallbackChain(next); + }); + row.appendChild(rm); + + fbWrap.appendChild(row); + }); + // Add-fallback button: disabled when there are no remaining providers to add. + if (addBtn) { + var hasMore = _availableFallbackOptions().length > 0; + addBtn.style.display = hasMore ? '' : 'none'; } - // Drag-reorder - var dragging = null; - fbWrap.querySelectorAll('.search-fb-chip').forEach(function(chip) { - chip.addEventListener('dragstart', function() { - dragging = chip; chip.classList.add('dragging'); - }); - chip.addEventListener('dragend', function() { - if (dragging) dragging.classList.remove('dragging'); - dragging = null; - // Persist new order - var order = Array.from(fbWrap.querySelectorAll('.search-fb-chip')).map(function(c) { return c.dataset.value; }); - _saveFallbackChain(order); - }); - chip.addEventListener('dragover', function(e) { - e.preventDefault(); - if (!dragging || dragging === chip) return; - var rect = chip.getBoundingClientRect(); - var after = (e.clientX - rect.left) > rect.width / 2; - chip.parentNode.insertBefore(dragging, after ? chip.nextSibling : chip); - }); + } + if (addBtn && !addBtn._wired) { + addBtn._wired = true; + addBtn.addEventListener('click', function() { + var avail = _availableFallbackOptions(); + if (!avail.length) return; + var next = (_settings.search_fallback_chain || []).slice(); + next.push(avail[0].value); + _saveFallbackChain(next); }); } async function _saveFallbackChain(chain) { @@ -1355,8 +1448,18 @@ async function initSearchSettings() { // Persist current form values first so the test uses what's on screen. await saveSearch(); testBtn.disabled = true; - var orig = testBtn.textContent; - testBtn.textContent = 'Testing...'; + var origHtml = testBtn.innerHTML; + var wp = null; + try { + var sp = window.spinnerModule || (await import('./spinner.js')).default; + wp = sp.createWhirlpool(11); + wp.element.style.cssText = 'display:inline-flex;width:11px;height:11px;margin:0 4px 0 0;'; + testBtn.innerHTML = ''; + testBtn.appendChild(wp.element); + testBtn.appendChild(document.createTextNode('Testing')); + } catch (_) { + testBtn.innerHTML = origHtml.replace(/>Test\s*$/, '>Testing...'); + } msg.textContent = ''; var t0 = performance.now(); try { @@ -1382,7 +1485,8 @@ async function initSearchSettings() { msg.textContent = '✗ Test failed: ' + (e && e.message ? e.message : e); msg.style.color = 'var(--red)'; } finally { - testBtn.disabled = false; testBtn.textContent = orig; + if (wp) { try { wp.destroy(); } catch (_) {} } + testBtn.disabled = false; testBtn.innerHTML = origHtml; } }); } @@ -1515,6 +1619,14 @@ async function initResearchSettings() { async function initResearchSearchSettings() { var searchSel = el('set-researchSearch'); var msg = el('set-researchSearchMsg'); + var logoEl = el('set-researchSearch-logo'); + + function updateSearchLogo() { + if (!logoEl) return; + var opt = searchSel.selectedOptions[0]; + var key = opt && opt.dataset ? opt.dataset.searchLogo : ''; + logoEl.innerHTML = key ? (_SEARCH_PROVIDER_LOGOS[key] || '') : ''; + } function updateSearchOptions(settings) { var options = searchSel.querySelectorAll('option'); @@ -1539,6 +1651,7 @@ async function initResearchSearchSettings() { var settings = await res.json(); if (settings.research_search_provider) searchSel.value = settings.research_search_provider; updateSearchOptions(settings); + updateSearchLogo(); } catch (e) { console.warn('Failed to load research search settings', e); } async function saveResearchSearch() { @@ -1552,7 +1665,7 @@ async function initResearchSearchSettings() { } catch (e) { msg.textContent = 'Failed to save'; msg.style.color = 'var(--red)'; } } - searchSel.addEventListener('change', saveResearchSearch); + searchSel.addEventListener('change', function() { updateSearchLogo(); saveResearchSearch(); }); } /* ── Agent Settings (AI tab) ── */ @@ -1607,6 +1720,25 @@ async function initAgentSettings() { msg.textContent = (cur > 0 ? 'Limit: ' + cur + ' tool calls' : 'Unlimited tool calls') + (curR != null ? ' · ' + curR + ' steps/message' : '') + (supInput && supInput.checked ? ' · supervisor on' : ''); + + // Standalone Email Safety toggle (separate card on the AI Defaults tab). + // Default to ON if the setting isn't present so a fresh install is safe. + var emailConfirm = el('set-agentEmailConfirm'); + if (emailConfirm) { + try { + var s = await fetch('/api/auth/settings', { credentials: 'same-origin' }).then(r => r.json()); + emailConfirm.checked = s.agent_email_confirm !== false; + } catch (_) {} + emailConfirm.addEventListener('change', async () => { + try { + await fetch('/api/auth/settings', { + method: 'POST', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ agent_email_confirm: !!emailConfirm.checked }), + }); + } catch (_) {} + }); + } } /* ═══════════════════════════════════════════ @@ -1667,15 +1799,25 @@ function initAppearance() { }); }); - var resetBtn = el('set-uiVisResetBtn'); - if (resetBtn) { - resetBtn.addEventListener('click', function() { - localStorage.removeItem('odysseus-ui-visibility'); + // Per-section reset buttons (arrow-circle-back icon in each card's h2). + // Removes only the keys belonging to this section from the persisted + // visibility map so other sections keep their user settings. + modalEl.querySelectorAll('[data-vis-reset]').forEach(function(btn) { + btn.addEventListener('click', function() { + var card = btn.closest('.admin-card'); + if (!card) return; + var keys = Array.from(card.querySelectorAll('[data-ui-key]')) + .map(function(c) { return c.dataset.uiKey; }) + .filter(Boolean); + if (!keys.length) return; + var s = window.loadUIVis ? window.loadUIVis() : {}; + keys.forEach(function(k) { delete s[k]; }); + if (window.saveUIVis) window.saveUIVis(s); syncAppearanceCheckboxes(); syncPrivacyCheckboxes(); - window.applyUIVis({}); + if (window.applyUIVis) window.applyUIVis(s); }); - } + }); } function syncAppearanceCheckboxes() { @@ -1862,7 +2004,9 @@ async function initShortcuts() { <span class="shortcut-hint" hidden></span> <button class="shortcut-key${combo ? '' : ' shortcut-key-unset'}" data-action="${action}" title="Click to rebind">${keyContent}</button> <button class="shortcut-action-btn ${isCustom ? 'is-reset' : ''}" data-action="${action}" title="${isCustom ? 'Reset to default' : 'Confirm'}" style="${isCustom ? '' : 'visibility:hidden'}"> - ${isCustom ? '\u21A9' : '\u2713'} + ${isCustom + ? '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="1 4 1 10 7 10"/><path d="M3.51 15a9 9 0 1 0 2.13-9.36L1 10"/></svg>' + : '\u2713'} </button> </div> `; @@ -1934,7 +2078,7 @@ async function initShortcuts() { btn.innerHTML = _formatKeyCaps(keybinds[action]); const isCustom = keybinds[action] !== SHORTCUT_DEFAULTS[action]; if (isCustom) { - actionBtn.textContent = '\u21A9'; + actionBtn.innerHTML = '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="1 4 1 10 7 10"/><path d="M3.51 15a9 9 0 1 0 2.13-9.36L1 10"/></svg>'; actionBtn.classList.add('is-reset'); actionBtn.title = 'Reset to default'; } else { @@ -2017,6 +2161,16 @@ function initAccount() { } }).catch(() => {}); + // Update password placeholder and policy from server + fetch('/api/auth/policy', { credentials: 'same-origin' }) + .then(r => r.ok ? r.json() : null) + .then(policy => { + if (!policy) return; + _authPolicy = policy; + const pwNew = el('settings-pw-new'); + if (pwNew) pwNew.placeholder = `New password (min ${policy.password_min_length})`; + }).catch(() => {}); + // Change password const saveBtn = el('settings-pw-save'); const msgEl = el('settings-pw-msg'); @@ -2027,7 +2181,7 @@ function initAccount() { const conf = el('settings-pw-confirm').value; msgEl.style.color = ''; if (!cur || !nw) { msgEl.textContent = 'Fill in all fields'; msgEl.style.color = 'var(--red)'; return; } - if (nw.length < 8) { msgEl.textContent = 'Min 8 characters'; msgEl.style.color = 'var(--red)'; return; } + if (nw.length < _authPolicy.password_min_length) { msgEl.textContent = `Min ${_authPolicy.password_min_length} characters`; msgEl.style.color = 'var(--red)'; return; } if (nw !== conf) { msgEl.textContent = 'Passwords don\'t match'; msgEl.style.color = 'var(--red)'; return; } saveBtn.disabled = true; try { @@ -2182,6 +2336,7 @@ function initAll() { initTabs(); initDrag(); initClose(); + initOpenPromptModalLink(); initOpacityToggle(); initialized = true; initDefaultChat(); @@ -2436,7 +2591,7 @@ async function initReminderSettings() { // users don't think they have to choose between channels. const CHANNEL_HINTS = { browser: 'Reminders appear as browser notifications inside Odysseus.', - email: 'Reminders are emailed AND shown as a browser notification.', + email: 'Reminders are emailed and shown as a browser notification.', ntfy: 'Reminders are pushed via ntfy AND shown as a browser notification.', webhook: 'Reminders are POSTed to the selected integration AND shown as a browser notification. Use {{title}} and {{message}} in the payload template.', }; @@ -2466,6 +2621,38 @@ async function initReminderSettings() { if (savedChannel === 'webhook' && !webhookConfigured) savedChannel = 'browser'; channelSel.value = savedChannel; llmToggle.checked = !!s.reminder_llm_synthesis; + // Persona dropdown — populate from built-in PROMPT_TEMPLATES (characters) + // plus any custom character preset. Selected value persists to + // reminder_llm_persona (backend hook lives in src/notes.py once + // /api/notes/fire-reminder lands). + const personaSel = el('set-reminder-llm-persona'); + if (personaSel) { + try { + const presetsMod = await import('./presets.js'); + const tpl = presetsMod.PROMPT_TEMPLATES || []; + const chars = tpl.filter(t => t.isCharacter); + for (const c of chars) { + const opt = document.createElement('option'); + opt.value = c.id; + opt.textContent = c.name; + personaSel.appendChild(opt); + } + // Custom character (single-slot preset) + try { + const all = (presetsMod.getAllPresets && presetsMod.getAllPresets()) || {}; + if (all.custom && all.custom.character_name) { + const opt = document.createElement('option'); + opt.value = 'custom'; + opt.textContent = all.custom.character_name + ' (custom)'; + personaSel.appendChild(opt); + } + } catch (_) {} + } catch (_) {} + personaSel.value = s.reminder_llm_persona || ''; + personaSel.addEventListener('change', () => { + save({ reminder_llm_persona: personaSel.value }); + }); + } if (emailToIn) emailToIn.value = s.reminder_email_to || ''; if (ntfyTopicIn) ntfyTopicIn.value = s.reminder_ntfy_topic || 'Reminders'; populateWebhookIntegrations(s.reminder_webhook_integration_id || ''); @@ -2508,6 +2695,9 @@ async function initReminderSettings() { if (hint) hint.textContent = CHANNEL_HINTS[channelSel.value] || ''; syncChannelRows(); save({ reminder_channel: channelSel.value }); + // Email reminder bell visibility tracks this — broadcast so the + // email library can re-evaluate without waiting for a re-open. + try { window.dispatchEvent(new CustomEvent('odysseus-reminder-channel-changed', { detail: { channel: channelSel.value } })); } catch (_) {} }); if (emailToIn) { let emailDebounce; @@ -2567,8 +2757,8 @@ async function initReminderSettings() { if (testBtn) { testBtn.addEventListener('click', async () => { testBtn.disabled = true; - if (testMsg) { testMsg.textContent = 'Sending…'; testMsg.style.color = 'var(--fg)'; } - // Whirlpool loader right next to the "Sending…" text while it sends. + if (testMsg) { testMsg.textContent = 'Sending'; testMsg.style.color = 'var(--fg)'; } + // Whirlpool loader right next to the "Sending" text while it sends. let _testSpin = null; try { const _sp = (await import('./spinner.js')).default; @@ -2578,6 +2768,9 @@ async function initReminderSettings() { } catch (_) {} const _stopTestSpin = () => { try { _testSpin && _testSpin.stop(); _testSpin && _testSpin.element.remove(); } catch (_) {} }; try { + // Persona picker is in a different scope (Reminders init), look it up + // by id so we can pass whatever is currently selected on screen. + const personaSel = el('set-reminder-llm-persona'); const res = await fetch('/api/notes/fire-reminder', { method: 'POST', credentials: 'same-origin', @@ -2587,6 +2780,11 @@ async function initReminderSettings() { title: 'Test Reminder', body: 'This is a test reminder to verify your settings are working.', channel: channelSel.value, + // Mirror the in-UI AI Synthesis toggle + persona so the test never + // races a pending save and lets the user preview changes before + // hitting Save. + llm_synthesis: !!(llmToggle && llmToggle.checked), + llm_persona: (personaSel && personaSel.value) || '', ...(channelSel.value === 'webhook' ? { webhook_integration_id: webhookIntgSel?.value || '', webhook_payload_template: webhookTemplateIn?.value.trim() || '', @@ -2646,7 +2844,7 @@ async function initEmailAccountsSettings() { try { const mod = await import('./tasks.js'); const openTasks = mod.openTasks || (mod.default && mod.default.openTasks); - if (typeof openTasks === 'function') openTasks(); + if (typeof openTasks === 'function') openTasks(null, { filter: 'Email' }); else document.getElementById('tool-tasks-btn')?.click(); } catch (_) { document.getElementById('tool-tasks-btn')?.click(); @@ -2726,13 +2924,14 @@ async function initEmailAccountsSettings() { // IMAP and SMTP. Dovecot is IMAP-only here; the host is intentionally // blank because it may live on another machine (DNS, LAN, Tailscale). const PROVIDERS = { - gmail: { label: 'Gmail', imap: { host: 'imap.gmail.com', port: 993, starttls: false }, smtp: { host: 'smtp.gmail.com', port: 465 } }, - migadu: { label: 'Migadu', imap: { host: 'imap.migadu.com', port: 993, starttls: false }, smtp: { host: 'smtp.migadu.com', port: 465 } }, - icloud: { label: 'iCloud', imap: { host: 'imap.mail.me.com', port: 993, starttls: false }, smtp: { host: 'smtp.mail.me.com', port: 587 } }, - outlook: { label: 'Outlook / Office 365', imap: { host: 'outlook.office365.com', port: 993, starttls: false }, smtp: { host: 'smtp.office365.com', port: 587 } }, - fastmail: { label: 'Fastmail', imap: { host: 'imap.fastmail.com', port: 993, starttls: false }, smtp: { host: 'smtp.fastmail.com', port: 465 } }, - yahoo: { label: 'Yahoo', imap: { host: 'imap.mail.yahoo.com', port: 993, starttls: false }, smtp: { host: 'smtp.mail.yahoo.com', port: 465 } }, - dovecot: { label: 'Dovecot IMAP (no SMTP)', imap: { host: '', port: 31143, starttls: false }, smtp: { host: '', port: 465 } }, + gmail: { label: 'Gmail', imap: { host: 'imap.gmail.com', port: 993, starttls: false }, smtp: { host: 'smtp.gmail.com', port: 465 } }, + google_workspace: { label: 'Google Workspace / .edu', imap: { host: 'imap.gmail.com', port: 993, starttls: false }, smtp: { host: 'smtp.gmail.com', port: 587 }, oauth: 'google' }, + migadu: { label: 'Migadu', imap: { host: 'imap.migadu.com', port: 993, starttls: false }, smtp: { host: 'smtp.migadu.com', port: 465 } }, + icloud: { label: 'iCloud', imap: { host: 'imap.mail.me.com', port: 993, starttls: false }, smtp: { host: 'smtp.mail.me.com', port: 587 } }, + outlook: { label: 'Outlook / Office 365', imap: { host: 'outlook.office365.com', port: 993, starttls: false }, smtp: { host: 'smtp.office365.com', port: 587 } }, + fastmail: { label: 'Fastmail', imap: { host: 'imap.fastmail.com', port: 993, starttls: false }, smtp: { host: 'smtp.fastmail.com', port: 465 } }, + yahoo: { label: 'Yahoo', imap: { host: 'imap.mail.yahoo.com', port: 993, starttls: false }, smtp: { host: 'smtp.mail.yahoo.com', port: 465 } }, + dovecot: { label: 'Dovecot IMAP (no SMTP)', imap: { host: '', port: 31143, starttls: false }, smtp: { host: '', port: 465 } }, }; const _providerOptions = Object.entries(PROVIDERS) .map(([k, v]) => `<option value="${k}">${esc(v.label)}</option>`) @@ -2745,11 +2944,17 @@ async function initEmailAccountsSettings() { <div id="eaf-provider-note" style="display:none;font-size:11px;line-height:1.5;padding:8px 10px;margin:2px 0 4px;border:1px solid color-mix(in srgb, var(--fg) 15%, transparent);border-left:3px solid var(--accent, var(--red));border-radius:4px;background:color-mix(in srgb, var(--fg) 4%, transparent);"></div> <div class="settings-row"><label class="settings-label">Name${_hint('Optional label for this account (e.g. “Work” or “Personal”). Leave blank to use the email address.')}</label><input id="eaf-name" class="settings-input" placeholder="(optional — leave blank to use email)" value="${esc(a.name || '')}"></div> <div class="settings-row"><label class="settings-label">Email${_hint('Your email address. Used as the From: header on outgoing mail and as the display label when Name is blank.')}</label><input id="eaf-from" class="settings-input" placeholder="you@example.com" value="${esc(a.from_address || '')}"></div> + <div class="settings-row"><label class="settings-label">Display Name${_hint('Your name as it appears in the From: field of emails you send, e.g. Jane Smith. Auto-filled from Google during OAuth.')}</label><input id="eaf-display-name" class="settings-input" placeholder="Your Name" value="${esc(a.display_name || '')}"></div> + <div id="eaf-oauth-section" style="display:none;margin:8px 0;padding:10px;border:1px solid var(--border);border-radius:6px;background:color-mix(in srgb,var(--accent,#50fa7b) 6%,transparent)"> + <div style="font-size:11px;font-weight:600;margin-bottom:6px">Google OAuth2 — required for Workspace / .edu accounts</div> + <div id="eaf-oauth-status" style="font-size:11px;opacity:0.7;margin-bottom:6px">${a.oauth_provider === 'google' ? '✓ Connected via Google OAuth' : 'Not connected — click below to authorize'}</div> + <button type="button" id="eaf-oauth-btn" class="admin-btn-add" style="font-size:11px">${a.oauth_provider === 'google' ? 'Reconnect with Google' : 'Connect with Google'}</button> + </div> <div style="font-size:11px;font-weight:600;opacity:0.6;margin:6px 0 2px">IMAP (Receiving)</div> <div class="settings-row"><label class="settings-label">Host${_hint('Your IMAP server, e.g. imap.gmail.com, imap.migadu.com, a LAN host, or a Tailscale IP for Dovecot.')}</label><input id="eaf-imap-host" class="settings-input" value="${esc(a.imap_host || '')}"></div> <div class="settings-row"><label class="settings-label">Port${_hint('993 for IMAPS (most providers), 143 for plain or STARTTLS. Local servers often use a custom port like 31143.')}</label><input id="eaf-imap-port" class="settings-input" type="number" value="${esc(a.imap_port || 993)}" style="max-width:100px"></div> <div class="settings-row"><label class="settings-label">Username${_hint('Usually your full email address.')}</label><input id="eaf-imap-user" class="settings-input" value="${esc(a.imap_user || '')}"></div> - <div class="settings-row"><label class="settings-label">Password${_hint('Your IMAP login password. Use an app-specific password if your provider requires 2FA. Outlook / Office 365 generally requires OAuth and will not work with a normal password here.')}</label><input id="eaf-imap-pass" class="settings-input" type="password" placeholder="${isEdit && a.has_imap_password ? '(unchanged)' : ''}"></div> + <div class="eaf-password-section"><div class="settings-row"><label class="settings-label">Password${_hint('Your IMAP login password. Use an app-specific password if your provider requires 2FA. Outlook / Office 365 generally requires OAuth and will not work with a normal password here.')}</label><input id="eaf-imap-pass" class="settings-input" type="password" placeholder="${isEdit && a.has_imap_password ? '(unchanged)' : ''}"></div></div> <div class="settings-row"><label class="settings-label">STARTTLS${_hint('Turn ON for port 143/587 to upgrade plain to TLS. Turn OFF for port 993 (IMAPS — already encrypted) or a local server with no TLS configured.')}</label><label class="admin-switch"><input type="checkbox" id="eaf-imap-starttls" ${a.imap_starttls !== false ? 'checked' : ''}><span class="admin-slider"></span></label></div> <div style="font-size:11px;font-weight:600;opacity:0.6;margin:8px 0 2px">SMTP (Sending) <span style="font-weight:normal;opacity:0.7">— optional, leave blank for read-only</span></div> <div class="settings-row"><label class="settings-label">Host${_hint('Your outgoing-mail server, e.g. smtp.gmail.com, smtp.migadu.com. Leave blank to make this account read-only.')}</label><input id="eaf-smtp-host" class="settings-input" value="${esc(a.smtp_host || '')}"></div> @@ -2772,6 +2977,16 @@ async function initEmailAccountsSettings() { </div> `; + // Show/hide OAuth section and password fields based on provider selection. + function _syncOauthUI(providerKey) { + const p = PROVIDERS[providerKey]; + const isOauth = !!(p && p.oauth); + el('eaf-oauth-section').style.display = isOauth ? '' : 'none'; + formEl.querySelectorAll('.eaf-password-section').forEach(r => { + r.style.display = isOauth ? 'none' : ''; + }); + } + const eafProviderNotes = { outlook: { title: 'Outlook / Office 365 needs OAuth', @@ -2796,13 +3011,41 @@ async function initEmailAccountsSettings() { el('eaf-provider').addEventListener('change', (e) => { _renderEafProviderNote(e.target.value); const p = PROVIDERS[e.target.value]; - if (!p) return; + if (!p) { _syncOauthUI(''); return; } el('eaf-imap-host').value = p.imap.host; el('eaf-imap-port').value = p.imap.port; el('eaf-imap-starttls').checked = !!p.imap.starttls; el('eaf-smtp-host').value = p.smtp.host; el('eaf-smtp-port').value = p.smtp.port; el('eaf-smtp-security').value = p.smtp.security || ((parseInt(p.smtp.port || 465) === 587) ? 'starttls' : 'ssl'); + _syncOauthUI(e.target.value); + }); + + // Init OAuth UI for accounts already connected via OAuth. + if (a.oauth_provider === 'google') _syncOauthUI('google_workspace'); + + // "Connect with Google" button — save the account first, then redirect to OAuth. + el('eaf-oauth-btn').addEventListener('click', async () => { + // Must save the account first to get an account_id to pass to the OAuth flow. + const body = { + name: el('eaf-name').value.trim() || el('eaf-from').value.trim(), + from_address: el('eaf-from').value.trim(), + imap_host: el('eaf-imap-host').value.trim(), + imap_port: parseInt(el('eaf-imap-port').value) || 993, + imap_user: el('eaf-imap-user').value.trim(), + imap_starttls: el('eaf-imap-starttls').checked, + smtp_host: el('eaf-smtp-host').value.trim(), + smtp_port: parseInt(el('eaf-smtp-port').value) || 587, + smtp_user: el('eaf-imap-user').value.trim(), + }; + if (!body.name) { el('eaf-msg').textContent = 'Enter a Name or Email first'; el('eaf-msg').style.color = 'var(--red)'; return; } + const url = isEdit ? `/api/email/accounts/${a.id}` : '/api/email/accounts'; + const method = isEdit ? 'PUT' : 'POST'; + const r = await fetch(url, { method, credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const d = await r.json(); + if (!d.ok) { el('eaf-msg').textContent = d.error || 'Save failed'; el('eaf-msg').style.color = 'var(--red)'; return; } + const accId = isEdit ? a.id : d.id; + window.location.href = `/api/email/oauth/google/authorize?account_id=${encodeURIComponent(accId)}`; }); el('eaf-smtp-security').value = _smtpSecurity(a); @@ -2822,6 +3065,7 @@ async function initEmailAccountsSettings() { const body = { name: el('eaf-name').value.trim(), from_address: el('eaf-from').value.trim(), + display_name: el('eaf-display-name').value.trim(), imap_host: el('eaf-imap-host').value.trim(), imap_port: parseInt(el('eaf-imap-port').value) || 993, imap_user: el('eaf-imap-user').value.trim(), @@ -3261,7 +3505,7 @@ const AGENT_CONFIGS = { namePrefix: 'codex agent', defaultName: 'Codex Agent', pluginPath: '/api/codex/plugin.zip', - setupDescription: 'Downloads the plugin bundle and registers it with Codex. Sets <code>ODYSSEUS_URL</code> + <code>ODYSSEUS_API_TOKEN</code>, fetches the plugin from <a href="/api/codex/plugin.zip" style="color:var(--accent,var(--red));">this Odysseus instance</a>, and runs <code>codex plugin add odysseus@personal</code>.', + setupDescription: 'Downloads a plugin bundle and registers it.', buildSetup: (origin, token) => `export ODYSSEUS_URL=${origin} export ODYSSEUS_API_TOKEN='${token}' mkdir -p ~/plugins @@ -3299,7 +3543,7 @@ python3 ~/plugins/odysseus/scripts/odysseus_api.py capabilities`, namePrefix: 'claude agent', defaultName: 'Claude Agent', pluginPath: '/api/claude/plugin.zip', - setupDescription: 'Downloads the skill bundle into <code>~/.claude/skills/odysseus/</code>. Sets <code>ODYSSEUS_URL</code> + <code>ODYSSEUS_API_TOKEN</code>, fetches the skill from <a href="/api/claude/plugin.zip" style="color:var(--accent,var(--red));">this Odysseus instance</a>. Claude Code auto-loads the skill on next start.', + setupDescription: 'Downloads a plugin bundle and registers it.', buildSetup: (origin, token) => `export ODYSSEUS_URL=${origin} export ODYSSEUS_API_TOKEN='${token}' mkdir -p ~/.claude @@ -3321,6 +3565,21 @@ async function initUnifiedIntegrations() { if (!listEl) return; let integrationNotice = ''; + // Hide the "+ Add Integration" button whenever the per-type create form + // is open so it doesn't compete visually with the in-progress form. + // Many call sites toggle formEl.style.display directly; observe instead + // of patching every one of them. + if (formEl && addBtn && addBtn.parentElement && !formEl._addBtnObserved) { + formEl._addBtnObserved = true; + const addBtnWrap = addBtn.parentElement; + const _syncAddBtnWrap = () => { + const formOpen = formEl.style.display && formEl.style.display !== 'none'; + addBtnWrap.style.display = formOpen ? 'none' : ''; + }; + new MutationObserver(_syncAddBtnWrap).observe(formEl, { attributes: true, attributeFilter: ['style'] }); + _syncAddBtnWrap(); + } + function _openEmailSettings() { open('email'); } @@ -3407,7 +3666,7 @@ async function initUnifiedIntegrations() { ? '<span style="width:8px;height:8px;border-radius:50%;background:var(--color-success,#50fa7b);flex-shrink:0;--notif-glow:var(--color-success,#50fa7b);animation:cookbook-notif-pulse 2s ease-in-out infinite;" title="Active"></span>' : '<span style="width:8px;height:8px;border-radius:50%;background:var(--fg);opacity:0.3;flex-shrink:0" title="Disabled"></span>'; return `<div class="intg-card" data-intg-id="${item.id}" data-intg-type="${item.type}" style="display:flex;align-items:center;gap:10px;padding:8px 10px;border:1px solid var(--border);border-radius:8px;background:color-mix(in srgb, var(--fg) 3%, transparent);margin-bottom:6px;cursor:pointer;transition:all 0.15s;" title="Click to edit"> - <span style="opacity:0.6;flex-shrink:0">${t.icon}</span> + <span style="color:var(--accent, var(--red));flex-shrink:0">${t.icon}</span> <div style="flex:1;min-width:0"> <div style="font-size:12px;font-weight:600;display:flex;align-items:center;gap:6px">${item.name} <span style="font-size:9px;text-transform:uppercase;letter-spacing:0.5px;padding:1px 5px;border:1px solid color-mix(in srgb, var(--accent, var(--red)) 50%, transparent);border-radius:3px;color:var(--accent, var(--red));background:color-mix(in srgb, var(--accent, var(--red)) 12%, transparent);">${t.label}</span></div> <div style="font-size:11px;opacity:0.5;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${item.detail || ''}</div> @@ -3441,9 +3700,11 @@ async function initUnifiedIntegrations() { if (e.target.closest('.intg-del-btn')) return; const type = card.dataset.intgType; const id = card.dataset.intgId; - const items2 = listEl.querySelectorAll('.intg-card'); - items2.forEach(c => c.style.borderColor = ''); - card.style.borderColor = 'var(--accent)'; + // Toggle a class instead of mutating inline borderColor — the + // inline border shorthand made the reset unreliable, leaving + // stale accent borders on previously-clicked cards. + listEl.querySelectorAll('.intg-card.intg-card-active').forEach(c => c.classList.remove('intg-card-active')); + card.classList.add('intg-card-active'); showForm(type, id); }); }); @@ -3555,7 +3816,12 @@ async function initUnifiedIntegrations() { <div class="settings-row"><label class="settings-label">Auth${_apiHint('How this service expects the credential to be sent. <b>Bearer</b> = sends "Authorization: Bearer YOUR_KEY" (most modern APIs, ntfy, OpenAI-style). <b>Header</b> = sends YOUR_KEY verbatim under a header name you choose (Miniflux uses X-Auth-Token). <b>Basic</b> = HTTP basic auth (user:pass). <b>None</b> = the API is open / no auth.')}</label><select id="uf-api-auth" class="settings-input"><option value="bearer">Bearer (most common)</option><option value="header">Header</option><option value="basic">Basic</option><option value="none">None</option></select></div> <div class="settings-row" id="uf-api-header-row"><label class="settings-label">Header${_apiHint('The HTTP header name the key goes under (Miniflux: X-Auth-Token; most others: Authorization). Only used when Auth = Header.')}</label><input id="uf-api-header" class="settings-input" placeholder="X-Auth-Token"></div> <div class="settings-row"><label class="settings-label">API Key${_apiHint('The secret token the service issued you (generated in its admin panel / settings). Used to prove your identity on each request. Required for any Auth mode except None.')}</label><input id="uf-api-key" class="settings-input" type="password" placeholder="Token/key"></div> - <div class="settings-row" style="margin-top:4px"><button class="admin-btn-sm" id="uf-api-save">Save</button><button class="admin-btn-sm" id="uf-api-test" style="opacity:0.7">Test</button><button class="admin-btn-sm" id="uf-api-cancel" style="opacity:0.7">Cancel</button><span id="uf-api-msg" style="font-size:11px"></span></div> + <div class="settings-row" style="margin-top:10px;align-items:center;justify-content:flex-end;gap:6px;"> + <span id="uf-api-msg" style="font-size:11px;flex:1;margin-right:8px"></span> + <button class="admin-btn-add" id="uf-api-test" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Test</button> + <button class="admin-btn-add" id="uf-api-save" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));font-weight:600;">Save</button> + <button class="admin-btn-add" id="uf-api-cancel" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Cancel</button> + </div> </div> </div>`; // Custom preset dropdown wire-up (hidden select stays as data source). @@ -3644,7 +3910,11 @@ async function initUnifiedIntegrations() { el('uf-api-cancel').addEventListener('click', () => { formEl.style.display = 'none'; }); el('uf-api-save').addEventListener('click', async () => { const presetKey = preset.value || undefined; - const body = { name: name.value, base_url: url.value, auth_type: auth.value, auth_header: header.value, preset: presetKey }; + const nameValue = name.value.trim(); + const urlValue = url.value.trim(); + if (!nameValue) { el('uf-api-msg').textContent = 'Name required'; el('uf-api-msg').style.color = 'var(--red)'; return; } + if (!urlValue) { el('uf-api-msg').textContent = 'Base URL required'; el('uf-api-msg').style.color = 'var(--red)'; return; } + const body = { name: nameValue, base_url: urlValue, auth_type: auth.value, auth_header: header.value, preset: presetKey }; if (key.value) body.api_key = key.value; try { const u = _editId ? `/api/auth/integrations/${_editId}` : '/api/auth/integrations'; @@ -3691,7 +3961,12 @@ async function initUnifiedIntegrations() { <div class="settings-row"><label class="settings-label">Server URL</label><input id="uf-caldav-url" class="settings-input" placeholder="https://www.google.com/calendar/dav/you@gmail.com/user/"></div> <div class="settings-row"><label class="settings-label">Username</label><input id="uf-caldav-user" class="settings-input" placeholder="you@example.com"></div> <div class="settings-row"><label class="settings-label">Password</label><input id="uf-caldav-pass" class="settings-input" type="password" placeholder="${isNew ? '' : 'Leave blank to keep existing'}"></div> - <div class="settings-row" style="margin-top:4px"><button class="admin-btn-sm" id="uf-caldav-save">Save</button><button class="admin-btn-sm" id="uf-caldav-test" style="opacity:0.7">Test</button><button class="admin-btn-sm" id="uf-caldav-cancel" style="opacity:0.7">Cancel</button><span id="uf-caldav-msg" style="font-size:11px;margin-left:6px"></span></div> + <div class="settings-row" style="margin-top:10px;align-items:center;justify-content:flex-end;gap:6px;"> + <span id="uf-caldav-msg" style="font-size:11px;flex:1;margin-right:8px"></span> + <button class="admin-btn-add" id="uf-caldav-test" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Test</button> + <button class="admin-btn-add" id="uf-caldav-save" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));font-weight:600;">Save</button> + <button class="admin-btn-add" id="uf-caldav-cancel" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Cancel</button> + </div> </div> </div>`; @@ -3795,13 +4070,13 @@ async function initUnifiedIntegrations() { <div class="settings-row"><label class="settings-label">URL</label><input id="uf-carddav-url" class="settings-input" placeholder="http://localhost:5232/user/contacts/"></div> <div class="settings-row"><label class="settings-label">Username</label><input id="uf-carddav-user" class="settings-input"></div> <div class="settings-row"><label class="settings-label">Password</label><input id="uf-carddav-pass" class="settings-input" type="password"></div> - <div class="settings-row" style="margin-top:8px;align-items:center;"> - <button class="admin-btn-add" id="uf-carddav-save" style="background:var(--red);border-color:var(--red);color:#fff;display:inline-flex;align-items:center;gap:5px;font-weight:600;"> + <div class="settings-row" style="margin-top:10px;align-items:center;justify-content:flex-end;gap:6px;"> + <span id="uf-carddav-msg" style="font-size:11px;flex:1;margin-right:8px"></span> + <button class="admin-btn-add" id="uf-carddav-save" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));font-weight:600;"> <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="20 6 9 17 4 12"/></svg> Save </button> - <span id="uf-carddav-msg" style="font-size:11px;flex:1;margin-left:8px"></span> - <button class="admin-btn-add" id="uf-carddav-cancel" style="opacity:0.7;display:inline-flex;align-items:center;gap:5px;position:relative;top:1px;margin-left:auto;"> + <button class="admin-btn-add" id="uf-carddav-cancel" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));"> <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg> Cancel </button> @@ -3817,16 +4092,24 @@ async function initUnifiedIntegrations() { <button class="admin-btn-sm" id="cm-add-toggle">+ Add</button> <input type="file" id="cm-import-file" accept=".vcf,.csv,text/vcard,text/csv" multiple style="display:none"> </div> - <div id="cm-add-row" class="contacts-add-row" style="display:none;"> - <input id="cm-add-name" class="settings-input" placeholder="Name" style="flex:1;min-width:0;"> - <input id="cm-add-email" class="settings-input" placeholder="email@example.com" style="flex:1;min-width:0;"> - <button class="admin-btn-sm" id="cm-add-save">Save</button> + <div id="cm-add-row" class="contacts-add-row" style="display:none;flex-direction:column;gap:4px;"> + <input id="cm-add-name" class="settings-input" placeholder="Name"> + <input id="cm-add-email" class="settings-input" placeholder="email@example.com"> + <input id="cm-add-phone" class="settings-input" placeholder="Phone (optional)"> + <input id="cm-add-address" class="settings-input" placeholder="Address (optional)"> + <div style="display:flex;gap:6px;justify-content:flex-end;"><button class="admin-btn-sm" id="cm-add-save">Save</button></div> </div> + <input type="text" id="cm-search" class="settings-input" placeholder="Search contacts (name, email, phone, address)" style="margin-top:6px;"> <div id="cm-list" class="contacts-list"><div style="opacity:0.4;font-size:11px;padding:8px 2px;">Loading…</div></div> </div>`; try { const r = await fetch('/api/contacts/config', { credentials: 'same-origin' }); const d = await r.json(); el('uf-carddav-url').value = d.url || ''; el('uf-carddav-user').value = d.username || ''; + // Server masks the password as '***' when one is saved (or '' when + // none). Surface that state via the input's placeholder so users + // can tell their password is already on file without us echoing it. + const passInput = el('uf-carddav-pass'); + if (passInput && d.password) passInput.placeholder = '(unchanged)'; } catch (_) {} el('uf-carddav-cancel').addEventListener('click', () => { formEl.style.display = 'none'; }); el('uf-carddav-save').addEventListener('click', async () => { @@ -3857,11 +4140,18 @@ async function initUnifiedIntegrations() { el('cm-add-save')?.addEventListener('click', async () => { const name = el('cm-add-name').value.trim(); const email = el('cm-add-email').value.trim(); - if (!email) { el('cm-add-email').focus(); return; } + const phone = el('cm-add-phone')?.value.trim() || ''; + const address = el('cm-add-address')?.value.trim() || ''; + // Need at least a name or email; address-only entries without a + // name aren't useful as a contact. + if (!name && !email) { (name ? el('cm-add-email') : el('cm-add-name')).focus(); return; } try { - await fetch('/api/contacts/add', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ name, email }) }); + await fetch('/api/contacts/add', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ name, email, phone, address }) }); } catch (_) {} - el('cm-add-name').value = ''; el('cm-add-email').value = ''; + el('cm-add-name').value = ''; + el('cm-add-email').value = ''; + if (el('cm-add-phone')) el('cm-add-phone').value = ''; + if (el('cm-add-address')) el('cm-add-address').value = ''; el('cm-add-row').style.display = 'none'; await _renderContactsManager(); }); @@ -3960,33 +4250,66 @@ async function initUnifiedIntegrations() { } // Sort by name for a stable list. contacts.sort((a, b) => (a.name || '').localeCompare(b.name || '')); - list.innerHTML = contacts.map(c => { - const emails = (c.emails || []).join(', '); - const phones = (c.phones || []).join(', '); - const sub = [emails, phones].filter(Boolean).join(' · '); - return `<div class="contact-row" data-uid="${esc(c.uid)}"> - <div class="contact-row-view" style="display:flex;align-items:center;gap:8px;"> - <div style="flex:1;min-width:0;"> - <div class="contact-name" style="font-size:12px;font-weight:600;">${esc(c.name || '(no name)')}</div> - <div class="contact-sub" style="font-size:10px;opacity:0.55;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">${esc(sub)}</div> + + // Live filter — search across name/emails/phones/address. + const searchInput = el('cm-search'); + const q = (searchInput?.value || '').trim().toLowerCase(); + const filtered = !q ? contacts : contacts.filter(c => { + const hay = [ + c.name || '', + (c.emails || []).join(' '), + (c.phones || []).join(' '), + c.address || '', + ].join(' ').toLowerCase(); + return hay.includes(q); + }); + if (cnt) cnt.textContent = contacts.length ? `(${filtered.length}/${contacts.length})` : ''; + + if (!filtered.length) { + list.innerHTML = `<div style="opacity:0.4;font-size:11px;padding:8px 2px;">${q ? 'No matches.' : 'No contacts yet.'}</div>`; + } else { + list.innerHTML = filtered.map(c => { + const emails = (c.emails || []).join(', '); + const phones = (c.phones || []).join(', '); + const address = c.address || ''; + const sub = [emails, phones, address].filter(Boolean).join(' · '); + return `<div class="contact-row" data-uid="${esc(c.uid)}"> + <div class="contact-row-view" style="display:flex;align-items:center;gap:8px;"> + <div style="flex:1;min-width:0;"> + <div class="contact-name" style="font-size:12px;font-weight:600;">${esc(c.name || '(no name)')}</div> + <div class="contact-sub" style="font-size:10px;opacity:0.55;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">${esc(sub)}</div> + </div> + <button class="admin-btn-sm contact-edit" title="Edit" style="display:inline-flex;align-items:center;gap:4px;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 35%, var(--border));"> + <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.12 2.12 0 0 1 3 3L12 15l-4 1 1-4 9.5-9.5z"/></svg> + Edit + </button> + <button class="admin-btn-sm contact-del" title="Delete" style="opacity:0.85;display:inline-flex;align-items:center;gap:4px;"> + <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M8 6V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"/></svg> + Delete + </button> </div> - <button class="admin-btn-sm contact-edit" title="Edit" style="display:inline-flex;align-items:center;gap:4px;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 35%, var(--border));"> - <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.12 2.12 0 0 1 3 3L12 15l-4 1 1-4 9.5-9.5z"/></svg> - Edit - </button> - <button class="admin-btn-sm contact-del" title="Delete" style="opacity:0.85;display:inline-flex;align-items:center;gap:4px;"> - <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M8 6V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"/></svg> - Delete - </button> - </div> - <div class="contact-row-edit" style="display:none;flex-direction:column;gap:4px;"> - <input class="settings-input contact-edit-name" value="${esc(c.name || '')}" placeholder="Name"> - <input class="settings-input contact-edit-emails" value="${esc(emails)}" placeholder="email1, email2"> - <input class="settings-input contact-edit-phones" value="${esc(phones)}" placeholder="phone1, phone2"> - <div style="display:flex;gap:6px;"><button class="admin-btn-sm contact-save">Save</button><button class="admin-btn-sm contact-cancel" style="opacity:0.7;">Cancel</button></div> - </div> - </div>`; - }).join(''); + <div class="contact-row-edit" style="display:none;flex-direction:column;gap:4px;"> + <input class="settings-input contact-edit-name" value="${esc(c.name || '')}" placeholder="Name"> + <input class="settings-input contact-edit-emails" value="${esc(emails)}" placeholder="email1, email2"> + <input class="settings-input contact-edit-phones" value="${esc(phones)}" placeholder="phone1, phone2"> + <input class="settings-input contact-edit-address" value="${esc(address)}" placeholder="Address"> + <div style="display:flex;gap:6px;"><button class="admin-btn-sm contact-save">Save</button><button class="admin-btn-sm contact-cancel" style="opacity:0.7;">Cancel</button></div> + </div> + </div>`; + }).join(''); + } + + // Wire the search input — debounced so we don't refetch on every key. + if (searchInput && !searchInput._wired) { + searchInput._wired = true; + let _t; + searchInput.addEventListener('input', () => { + clearTimeout(_t); + _t = setTimeout(() => _renderContactsManager(), 80); + }); + } + // Stash latest contacts so the search input doesn't have to refetch. + list._lastContacts = contacts; // Wire each row's edit / delete / save / cancel. list.querySelectorAll('.contact-row').forEach(row => { const uid = row.dataset.uid; @@ -4005,6 +4328,7 @@ async function initUnifiedIntegrations() { name: row.querySelector('.contact-edit-name').value.trim(), emails: row.querySelector('.contact-edit-emails').value.split(',').map(s => s.trim()).filter(Boolean), phones: row.querySelector('.contact-edit-phones').value.split(',').map(s => s.trim()).filter(Boolean), + address: row.querySelector('.contact-edit-address')?.value.trim() || '', }; try { await fetch('/api/contacts/' + encodeURIComponent(uid), { method: 'PUT', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); @@ -4050,6 +4374,7 @@ async function initUnifiedIntegrations() { // it may be remote (DNS, LAN, Tailscale), not localhost. const PROVIDERS = { gmail: { label: 'Gmail', emailEx: 'you@gmail.com', imap: { host: 'imap.gmail.com', port: 993, starttls: false }, smtp: { host: 'smtp.gmail.com', port: 465 } }, + google_workspace: { label: 'Google Workspace / .edu', emailEx: 'you@yourschool.edu', imap: { host: 'imap.gmail.com', port: 993, starttls: false }, smtp: { host: 'smtp.gmail.com', port: 587 }, oauth: 'google' }, migadu: { label: 'Migadu', emailEx: 'you@yourdomain.com', imap: { host: 'imap.migadu.com', port: 993, starttls: false }, smtp: { host: 'smtp.migadu.com', port: 465 } }, icloud: { label: 'iCloud', emailEx: 'you@icloud.com', imap: { host: 'imap.mail.me.com', port: 993, starttls: false }, smtp: { host: 'smtp.mail.me.com', port: 587 } }, outlook: { label: 'Outlook / Office 365', emailEx: 'you@outlook.com', imap: { host: 'outlook.office365.com', port: 993, starttls: false }, smtp: { host: 'smtp.office365.com', port: 587 } }, @@ -4067,6 +4392,7 @@ async function initUnifiedIntegrations() { const PROV_LOGO = { '': _customLogo, gmail: _letterLogo('G', '#ea4335'), + google_workspace: _letterLogo('G', '#ea4335'), migadu: _letterLogo('M', '#3aa39d'), icloud: _letterLogo('i', '#3693f3'), outlook: _letterLogo('O', '#0078d4'), @@ -4095,11 +4421,17 @@ async function initUnifiedIntegrations() { <div id="uf-email-provider-note" style="display:none;font-size:11px;line-height:1.5;padding:8px 10px;margin:2px 0 4px;border:1px solid color-mix(in srgb, var(--fg) 15%, transparent);border-left:3px solid var(--accent, var(--red));border-radius:4px;background:color-mix(in srgb, var(--fg) 4%, transparent);"></div> <div class="settings-row"><label class="settings-label">Name${_hint('Optional label for this account (e.g. “Work” or “Personal”). Leave blank to use the email address.')}</label><input id="uf-email-name" class="settings-input" placeholder="(optional — leave blank to use email)"></div> <div class="settings-row"><label class="settings-label">Email${_hint('Your email address. Used as the From: header on outgoing mail and as the display label when Name is blank.')}</label><input id="uf-email-from" class="settings-input" placeholder="you@example.com"></div> + <div class="settings-row"><label class="settings-label">Display Name${_hint('Your name as it appears in the From: field of emails you send, e.g. Jane Smith. Auto-filled from Google during OAuth.')}</label><input id="uf-display-name" class="settings-input" placeholder="Your Name"></div> + <div id="uf-oauth-section" style="display:none;margin:8px 0;padding:10px;border:1px solid var(--border);border-radius:6px;background:color-mix(in srgb,var(--accent,#50fa7b) 6%,transparent)"> + <div style="font-size:11px;font-weight:600;margin-bottom:6px">Google OAuth2 — required for Workspace / .edu accounts</div> + <div id="uf-oauth-status" style="font-size:11px;opacity:0.7;margin-bottom:6px">${existing && existing.oauth_provider === 'google' ? '✓ Connected via Google OAuth' : 'Not connected — click below to authorize'}</div> + <button type="button" id="uf-oauth-btn" class="admin-btn-add" style="font-size:11px">${existing && existing.oauth_provider === 'google' ? 'Reconnect with Google' : 'Connect with Google'}</button> + </div> <div style="font-size:11px;font-weight:600;opacity:0.6;margin:4px 0 2px;display:flex;align-items:center;gap:5px;"><svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="color:var(--accent, var(--red));flex-shrink:0;" aria-hidden="true"><polyline points="22 12 16 12 14 15 10 15 8 12 2 12"/><path d="M5.45 5.11 2 12v6a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2v-6l-3.45-6.89A2 2 0 0 0 16.76 4H7.24a2 2 0 0 0-1.79 1.11z"/></svg>IMAP (Receiving)</div> <div class="settings-row"><label class="settings-label">Host${_hint('Your IMAP server, e.g. imap.gmail.com, imap.migadu.com, a LAN host, or a Tailscale IP for Dovecot.')}</label><input id="uf-imap-host" class="settings-input" placeholder="imap.example.com"></div> <div class="settings-row"><label class="settings-label">Port${_hint('993 for IMAPS (most providers), 143 for plain or STARTTLS. Local servers often use a custom port like 31143.')}</label><input id="uf-imap-port" class="settings-input" type="number" placeholder="993" style="max-width:100px"></div> <div class="settings-row"><label class="settings-label">Username${_hint('Yes — your full email address goes here too (e.g. you@gmail.com). Same as the Email field above for almost every provider.')}</label><input id="uf-imap-user" class="settings-input" placeholder="you@example.com"></div> - <div class="settings-row"><label class="settings-label">Password${_hint('For Gmail, iCloud, and Yahoo: paste your App Password (NOT your normal account password). For Migadu and Fastmail, your mailbox password usually works. Outlook / Office 365 generally requires OAuth and will not work with this password form.')}</label><input id="uf-imap-pass" class="settings-input" type="password" placeholder="${placeholderPass}"></div> + <div class="uf-password-section"><div class="settings-row"><label class="settings-label">Password${_hint('For Gmail, iCloud, and Yahoo: paste your App Password (NOT your normal account password). For Migadu and Fastmail, your mailbox password usually works. Outlook / Office 365 generally requires OAuth and will not work with this password form.')}</label><input id="uf-imap-pass" class="settings-input" type="password" placeholder="${placeholderPass}"></div></div> <div class="settings-row"><label class="settings-label">STARTTLS${_hint('Turn ON for port 143/587 to upgrade plain to TLS. Turn OFF for port 993 (IMAPS — already encrypted) or a local server with no TLS configured.')}</label><label class="admin-switch" style="margin-left:0"><input type="checkbox" id="uf-imap-starttls" checked><span class="admin-slider"></span></label></div> <div style="font-size:11px;font-weight:600;opacity:0.6;margin:8px 0 2px;display:flex;align-items:center;gap:5px;"><svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="color:var(--accent, var(--red));flex-shrink:0;" aria-hidden="true"><line x1="22" y1="2" x2="11" y2="13"/><polygon points="22 2 15 22 11 13 2 9 22 2"/></svg>SMTP (Sending) <span style="font-weight:normal;opacity:0.7">— optional, leave blank for read-only</span></div> <div class="settings-row"><label class="settings-label">Host${_hint('Your outgoing-mail server, e.g. smtp.gmail.com. Leave blank to make this account read-only.')}</label><input id="uf-smtp-host" class="settings-input" placeholder="smtp.example.com"></div> @@ -4109,21 +4441,21 @@ async function initUnifiedIntegrations() { <div class="settings-row uf-smtp-creds"><label class="settings-label">Username${_hint('Usually the same as your IMAP username (your email address).')}</label><input id="uf-smtp-user" class="settings-input"></div> <div class="settings-row uf-smtp-creds"><label class="settings-label">Password${_hint('Your SMTP password — often the same as your IMAP password. Outlook / Office 365 generally requires OAuth and will not work with this password form.')}</label><input id="uf-smtp-pass" class="settings-input" type="password" placeholder="${placeholderPass}"></div> <div class="settings-row" style="margin-top:4px"><label class="settings-label">Default${_hint('Use this account whenever no specific account is chosen.')}</label><label class="admin-switch" style="margin-left:0"><input type="checkbox" id="uf-email-default"><span class="admin-slider"></span></label><span style="font-size:10px;opacity:0.5;margin-left:6px">Used when nothing else is selected</span></div> - <div class="settings-row" style="margin-top:10px;align-items:center;"> - <button class="admin-btn-add" id="uf-email-save" style="background:var(--red);border-color:var(--red);color:#fff;display:inline-flex;align-items:center;gap:5px;font-weight:600;"> - <span class="uf-email-save-ico" style="display:inline-flex;width:11px;height:11px;align-items:center;justify-content:center;"> - <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="20 6 9 17 4 12"/></svg> - </span> - <span class="uf-email-save-label">${isEdit ? 'Save' : 'Create'}</span> - </button> - <button class="admin-btn-add" id="uf-email-test" style="display:inline-flex;align-items:center;gap:5px;opacity:0.85;"> + <div class="settings-row" style="margin-top:10px;align-items:center;justify-content:flex-end;gap:6px;"> + <span id="uf-email-msg" style="font-size:11px;flex:1;margin-right:8px"></span> + <button class="admin-btn-add" id="uf-email-test" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));"> <span class="uf-email-test-ico" style="display:inline-flex;width:11px;height:11px;align-items:center;justify-content:center;"> <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="22 4 12 14.01 9 11.01"/><path d="M21 12v7a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11"/></svg> </span> Test </button> - <span id="uf-email-msg" style="font-size:11px;flex:1;margin-left:8px"></span> - <button class="admin-btn-add" id="uf-email-cancel" style="opacity:0.7;display:inline-flex;align-items:center;gap:5px;position:relative;top:1px;margin-left:auto;"> + <button class="admin-btn-add" id="uf-email-save" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));font-weight:600;"> + <span class="uf-email-save-ico" style="display:inline-flex;width:11px;height:11px;align-items:center;justify-content:center;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="20 6 9 17 4 12"/></svg> + </span> + <span class="uf-email-save-label">${isEdit ? 'Save' : 'Create'}</span> + </button> + <button class="admin-btn-add" id="uf-email-cancel" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));"> <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg> Cancel </button> @@ -4224,6 +4556,16 @@ async function initUnifiedIntegrations() { </div>`; }; + // Show/hide the OAuth section and password fields based on provider selection. + function _syncOauthUI(providerKey) { + const p = PROVIDERS[providerKey]; + const isOauth = !!(p && p.oauth); + el('uf-oauth-section').style.display = isOauth ? '' : 'none'; + formEl.querySelectorAll('.uf-password-section').forEach(r => { + r.style.display = isOauth ? 'none' : ''; + }); + } + // Custom dropdown wire-up — the native <select> stays in the DOM as the // data source and accessibility target, but the visible UI is a button + // popup so each provider row can render with its SVG logo. Selecting an @@ -4280,6 +4622,7 @@ async function initUnifiedIntegrations() { el('uf-email-provider').addEventListener('change', (e) => { const key = e.target.value; _renderProviderNote(key); + _syncOauthUI(key); const p = PROVIDERS[key]; if (!p) return; el('uf-imap-host').value = p.imap.host; @@ -4295,6 +4638,23 @@ async function initUnifiedIntegrations() { } }); + // Init OAuth UI for accounts already connected via OAuth. + if (existing && existing.oauth_provider === 'google') _syncOauthUI('google_workspace'); + + // "Connect with Google" — save the account first, then redirect to OAuth. + el('uf-oauth-btn').addEventListener('click', async () => { + const body = _collectBody(); + if (!body.name) body.name = body.from_address; + if (!body.name) { el('uf-email-msg').textContent = 'Enter a Name or Email first'; el('uf-email-msg').style.color = 'var(--red)'; return; } + const url = isEdit ? `/api/email/accounts/${editId}` : '/api/email/accounts'; + const method = isEdit ? 'PUT' : 'POST'; + const r = await fetch(url, { method, credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const d = await r.json(); + if (!(d.ok || d.id)) { el('uf-email-msg').textContent = d.error || 'Save failed'; el('uf-email-msg').style.color = 'var(--red)'; return; } + const accId = isEdit ? editId : d.id; + window.location.href = `/api/email/oauth/google/authorize?account_id=${encodeURIComponent(accId)}`; + }); + // "Same as IMAP" toggle — hide the SMTP creds rows when on. const _syncSmtpSame = () => { const same = el('uf-smtp-same').checked; @@ -4307,6 +4667,7 @@ async function initUnifiedIntegrations() { if (existing) { el('uf-email-name').value = existing.name || ''; el('uf-email-from').value = existing.from_address || ''; + el('uf-display-name').value = existing.display_name || ''; el('uf-imap-host').value = existing.imap_host || ''; el('uf-imap-port').value = existing.imap_port || 993; el('uf-imap-user').value = existing.imap_user || ''; @@ -4355,6 +4716,7 @@ async function initUnifiedIntegrations() { const body = { name: el('uf-email-name').value.trim(), from_address: el('uf-email-from').value.trim(), + display_name: el('uf-display-name').value.trim(), imap_host: el('uf-imap-host').value.trim(), imap_port: parseInt(el('uf-imap-port').value) || 993, imap_user: el('uf-imap-user').value.trim(), @@ -4396,7 +4758,15 @@ async function initUnifiedIntegrations() { btn.style.color = ''; btn.style.boxShadow = ''; btn.style.animation = ''; - ico.innerHTML = _spinner; + // Use the canonical whirlpool spinner so this matches Probe / Test + // elsewhere; fall back to the inline CSS ring if the module fails. + try { + const sp = window.spinnerModule || (await import('./spinner.js')).default; + const wp = sp.createWhirlpool(11); + wp.element.style.cssText = 'display:inline-flex;width:11px;height:11px;position:relative;top:-2px;'; + ico.innerHTML = ''; + ico.appendChild(wp.element); + } catch (_) { ico.innerHTML = _spinner; } msg.textContent = ''; msg.style.color = ''; try { @@ -4499,14 +4869,14 @@ async function initUnifiedIntegrations() { <div class="settings-row"><label class="settings-label">Server URL</label><input id="uf-vault-url" class="settings-input" placeholder="https://vault.example.com"></div> <div class="settings-row"><label class="settings-label">Email</label><input id="uf-vault-email" class="settings-input" placeholder="you@example.com"></div> <div class="settings-row"><label class="settings-label">Master Password</label><input id="uf-vault-pass" class="settings-input" type="password" placeholder="Only required for Login / Unlock"></div> - <div class="settings-row" style="margin-top:4px;flex-wrap:wrap;gap:4px"> - <button class="admin-btn-sm" id="uf-vault-save">Save Config</button> - <button class="admin-btn-sm" id="uf-vault-login">Login</button> - <button class="admin-btn-sm" id="uf-vault-unlock">Unlock</button> - <button class="admin-btn-sm" id="uf-vault-lock" style="opacity:0.7">Lock</button> - <button class="admin-btn-sm" id="uf-vault-logout" style="opacity:0.7">Logout</button> - <button class="admin-btn-sm" id="uf-vault-cancel" style="opacity:0.7">Cancel</button> - <span id="uf-vault-msg" style="font-size:11px;margin-left:4px"></span> + <div class="settings-row" style="margin-top:10px;align-items:center;justify-content:flex-end;gap:6px;flex-wrap:wrap;"> + <span id="uf-vault-msg" style="font-size:11px;flex:1;margin-right:8px"></span> + <button class="admin-btn-add" id="uf-vault-save" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));font-weight:600;">Save Config</button> + <button class="admin-btn-add" id="uf-vault-login" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Login</button> + <button class="admin-btn-add" id="uf-vault-unlock" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Unlock</button> + <button class="admin-btn-add" id="uf-vault-lock" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Lock</button> + <button class="admin-btn-add" id="uf-vault-logout" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Logout</button> + <button class="admin-btn-add" id="uf-vault-cancel" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Cancel</button> </div> <div style="font-size:10px;opacity:0.5;margin-top:6px;line-height:1.4"> <strong>Login</strong> registers this device with your Vaultwarden account (once per account).<br> @@ -4699,12 +5069,12 @@ async function initUnifiedIntegrations() { <span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:${statusColor}"></span> <span style="font-size:11px;opacity:0.7">${statusText}</span> </div> - <div style="display:flex;gap:6px;flex-wrap:wrap;margin-bottom:8px"> - ${srv.needs_oauth ? `<a href="/api/mcp/oauth/authorize/${srv.id}" target="_blank" class="admin-btn-sm" style="background:var(--red);color:#fff;text-decoration:none">Authorize</a>` : ''} - <button class="admin-btn-sm" id="uf-mcp-reconnect">Reconnect</button> - <button class="admin-btn-sm" id="uf-mcp-toggle">${srv.is_enabled ? 'Disable' : 'Enable'}</button> - <button class="admin-btn-sm" id="uf-mcp-cancel" style="opacity:0.7">Close</button> - <span id="uf-mcp-msg" style="font-size:11px"></span> + <div style="display:flex;align-items:center;gap:6px;flex-wrap:wrap;margin-bottom:8px;justify-content:flex-end;"> + <span id="uf-mcp-msg" style="font-size:11px;flex:1;margin-right:8px"></span> + ${srv.needs_oauth ? `<a href="/api/mcp/oauth/authorize/${srv.id}" target="_blank" class="admin-btn-add" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));text-decoration:none;font-weight:600;">Authorize</a>` : ''} + <button class="admin-btn-add" id="uf-mcp-reconnect" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Reconnect</button> + <button class="admin-btn-add" id="uf-mcp-toggle" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">${srv.is_enabled ? 'Disable' : 'Enable'}</button> + <button class="admin-btn-add" id="uf-mcp-cancel" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Close</button> </div> <div id="uf-mcp-tools-panel"></div> </div>`; @@ -4766,7 +5136,11 @@ async function initUnifiedIntegrations() { <div id="uf-mcp-sse-fields" style="display:none;flex-direction:column;gap:6px;"> <div class="settings-row"><label class="settings-label">URL</label><input id="uf-mcp-url" class="settings-input" placeholder="http://localhost:3001/sse"></div> </div> - <div class="settings-row" style="margin-top:4px"><button class="admin-btn-sm" id="uf-mcp-save">Save</button><button class="admin-btn-sm" id="uf-mcp-cancel" style="opacity:0.7">Cancel</button><span id="uf-mcp-msg" style="font-size:11px"></span></div> + <div class="settings-row" style="margin-top:10px;align-items:center;justify-content:flex-end;gap:6px;"> + <span id="uf-mcp-msg" style="font-size:11px;flex:1;margin-right:8px"></span> + <button class="admin-btn-add" id="uf-mcp-save" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));font-weight:600;">Save</button> + <button class="admin-btn-add" id="uf-mcp-cancel" style="background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));">Cancel</button> + </div> </div> </div>`; el('uf-mcp-transport').addEventListener('change', () => { @@ -4875,79 +5249,233 @@ async function initUnifiedIntegrations() { </label>`; }).join(''); }; - const tokenRows = agentTokens.length ? agentTokens.map(t => ` - <div class="uf-codex-token" data-token-id="${esc(t.id)}" style="border:1px solid var(--border);border-radius:6px;padding:9px 10px;margin-top:8px;"> - <div style="display:flex;align-items:center;gap:8px;margin-bottom:8px;"> - <div style="flex:1;min-width:0;"> - <input type="text" class="uf-codex-rename settings-input" data-token-id="${esc(t.id)}" value="${esc(t.name || cfg.defaultName)}" placeholder="${esc(cfg.defaultName)} (e.g. ${esc(cfg.word)} on laptop)" style="font-size:12px;font-weight:600;padding:3px 6px;width:100%;background:transparent;border:1px solid transparent;border-radius:4px;" title="Click to rename this agent"> - <div style="font-size:10px;opacity:0.52;margin-top:2px;">${esc(t.token_prefix || 'token')}...${t.last_used_at ? ` · Last used ${new Date(t.last_used_at).toLocaleDateString()}` : ' · Never used'}</div> - </div> - <button class="admin-btn-sm uf-codex-copy-prefix" data-token-prefix="${esc(t.token_prefix || '')}" title="Copy token prefix (full token only shown once, at creation)" style="opacity:0.7">Copy</button> - <button class="admin-btn-delete uf-codex-revoke" data-token-id="${esc(t.id)}">Revoke</button> - </div> - <div style="font-size:11px;font-weight:600;opacity:0.62;margin-bottom:4px;">Tool access</div> - ${scopeToggles(t)} - <div class="uf-codex-scope-msg" data-token-id="${esc(t.id)}" style="font-size:11px;min-height:14px;"></div> - </div>`).join('') : `<div style="opacity:0.45;font-size:11px;padding:8px 0;">No ${esc(cfg.word)} tokens yet.</div>`; const origin = window.location.origin || ''; const setupForToken = (token) => cfg.buildSetup(origin, token); + // Inline editor for the existing token the user clicked into (current). + // Shows the rename input, the prefix/last-used, and scope toggles that + // PATCH /api/tokens/{id} on change. The integration row's trash button + // handles revoke, so no Revoke button in here. + const editExistingHtml = current ? ` + <div style="border:1px solid var(--border);border-radius:6px;padding:9px 10px;margin-bottom:8px;"> + <div style="display:flex;align-items:center;gap:8px;margin-bottom:8px;"> + <input type="text" id="uf-codex-existing-rename" data-token-id="${esc(current.id)}" value="${esc(current.name || cfg.defaultName)}" style="font-size:12px;font-weight:600;padding:3px 6px;flex:1;background:transparent;border:1px solid transparent;border-radius:4px;" title="Click to rename"> + <span style="font-size:10px;opacity:0.55;">${esc(current.token_prefix || 'token')}...${current.last_used_at ? ` · Last used ${new Date(current.last_used_at).toLocaleDateString()}` : ' · Never used'}</span> + </div> + <div style="font-size:11px;font-weight:600;opacity:0.62;margin-bottom:4px;">Permissions</div> + ${scopeToggles(current)} + <div id="uf-codex-existing-msg" style="font-size:11px;min-height:14px;margin-top:4px;"></div> + </div>` : ''; + formEl.innerHTML = ` <div class="admin-card" style="margin-top:8px"> - <h2 style="font-size:13px">${esc(cfg.label)}</h2> - <div style="font-size:11px;opacity:0.65;line-height:1.45;margin:-2px 0 8px;">Generates a scoped token + setup commands so ${esc(cfg.word)} on your own machine can read/write your Odysseus data (todos, email, calendar, etc.). The agent runs in your terminal — it isn't streamed inside Odysseus.</div> <div class="settings-col"> - <div id="uf-codex-pending" style="display:${current ? 'none' : 'block'};font-size:11px;opacity:0.6;padding:6px 0;">Creating agent...</div> - <div id="uf-codex-reveal" style="display:none;padding:10px 12px;border:1px solid var(--border);border-left:3px solid var(--accent, var(--red));border-radius:6px;background:rgba(0,0,0,0.04);width:100%;box-sizing:border-box;"> - <div style="font-weight:600;font-size:12px;margin-bottom:6px;">${esc(cfg.word)} setup</div> + ${editExistingHtml} + <div id="uf-codex-prompt" style="display:${current ? 'none' : 'block'};padding:6px 0;"> + <div style="font-size:11px;opacity:0.7;margin-bottom:6px;">Name this ${esc(cfg.word)} agent so you can tell it apart from other ones (e.g. "${esc(cfg.defaultName)} — laptop").</div> + <input type="text" id="uf-codex-name-input" class="settings-select" placeholder="${esc(cfg.defaultName)}" style="width:100%;font-size:12px;padding:6px 8px;"> + </div> + <div id="uf-codex-pending" style="display:none;align-items:center;gap:8px;padding:6px 0;font-size:11px;opacity:0.7;"></div> + <div id="uf-codex-reveal" style="display:none;width:100%;box-sizing:border-box;"> + <div style="font-weight:600;font-size:12px;margin-bottom:6px;">Token</div> - <div style="font-size:11px;opacity:0.62;margin-bottom:4px;">Copy this token now — it will not be shown again.</div> - <code id="uf-codex-token" style="display:block;word-break:break-all;font-size:11px;padding:6px 8px;background:rgba(0,0,0,0.08);border-radius:4px;"></code> - <div style="margin-top:6px;"> - <button class="admin-btn-sm" id="uf-codex-copy-token">Copy token</button> + <div style="font-size:11px;opacity:0.62;margin-bottom:4px;">Copy this token, it won't be shown again.</div> + <div style="position:relative;"> + <code id="uf-codex-token" style="display:block;word-break:break-all;font-size:11px;padding:6px 30px 6px 8px;background:rgba(0,0,0,0.08);border-radius:4px;"></code> + <button type="button" class="admin-btn-sm" id="uf-codex-copy-token" title="Copy token" aria-label="Copy token" style="position:absolute;right:4px;top:50%;transform:translateY(-50%);padding:3px 5px;background:none;border:none;color:inherit;opacity:0.7;cursor:pointer;display:inline-flex;align-items:center;"> + <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg> + </button> </div> - <div style="margin-top:14px;font-weight:600;font-size:11px;margin-bottom:4px;">Quickstart — or copy setup directly in your terminal</div> + <div style="margin-top:14px;font-weight:600;font-size:11px;margin-bottom:4px;">Quickstart — simply paste directly in your terminal.</div> <div style="font-size:11px;opacity:0.62;margin-bottom:6px;">${cfg.setupDescription}</div> <pre style="margin:0;white-space:pre;overflow-x:auto;max-height:220px;overflow-y:auto;font-size:10px;line-height:1.45;padding:8px 10px;background:rgba(0,0,0,0.08);border-radius:4px;width:100%;box-sizing:border-box;"><code id="uf-codex-setup-code"></code></pre> - <div style="margin-top:6px;"> - <button class="admin-btn-sm" id="uf-codex-copy-setup">Copy setup</button> - </div> - <div style="margin-top:14px;font-weight:600;font-size:11px;margin-bottom:4px;">Configure access</div> - <div style="font-size:11px;opacity:0.62;margin-bottom:6px;">Toggle which Odysseus tools this agent can use. New agents start with chat only.</div> - <div id="uf-codex-inline-scopes"></div> + <div style="margin-top:14px;display:flex;align-items:center;gap:8px;"> + <span style="font-weight:600;font-size:11px;">Configure access</span> + <span style="flex:1"></span> + <button type="button" class="admin-btn-sm" id="uf-codex-copy-setup" title="Copy setup" aria-label="Copy setup" style="font-size:11px;font-weight:normal;display:inline-flex;align-items:center;gap:5px;opacity:0.85;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg> + <span>Copy</span> + </button> + <button type="button" class="admin-btn-sm" id="uf-codex-toggle-config" aria-expanded="false" style="font-size:11px;font-weight:normal;display:inline-flex;align-items:center;gap:5px;opacity:0.85;"> + <svg id="uf-codex-toggle-config-caret" width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="transition:transform 0.15s"><polyline points="6 9 12 15 18 9"/></svg> + <span>Configure</span> + </button> + </div> + <div id="uf-codex-config-body" style="display:none;"> + <div style="font-size:11px;opacity:0.62;margin:4px 0 6px;">Toggle which Odysseus tools this agent can use. New agents start with chat only.</div> + <div id="uf-codex-inline-scopes"></div> + </div> </div> - <div style="font-size:11px;font-weight:600;opacity:0.62;margin-top:10px;">${agentTokens.length ? 'Existing agents' : 'Agents'}</div> - <div id="uf-codex-token-list">${tokenRows}</div> - <div class="settings-row" style="margin-top:10px;align-items:center;"> - <button class="admin-btn-add" id="uf-codex-save" style="background:var(--red);border-color:var(--red);color:#fff;display:inline-flex;align-items:center;gap:5px;font-weight:600;"> - <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="20 6 9 17 4 12"/></svg> - Save - </button> - <span id="uf-codex-msg" style="font-size:11px;flex:1;margin-left:8px"></span> - <button class="admin-btn-add" id="uf-codex-cancel" style="opacity:0.7;display:inline-flex;align-items:center;gap:5px;position:relative;top:1px;margin-left:auto;"> + <div class="settings-row" style="margin-top:10px;align-items:center;gap:6px;"> + <button class="admin-btn-add" id="uf-codex-cancel" style="display:inline-flex;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));"> <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg> Cancel </button> + <span id="uf-codex-msg" style="font-size:11px;flex:1;text-align:center;"></span> + <button class="admin-btn-add" id="uf-codex-revoke" style="display:none;align-items:center;gap:5px;background:color-mix(in srgb, var(--color-error) 10%, transparent);color:var(--color-error);border:1px solid var(--color-error);font-weight:600;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/><path d="M9 6V4a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v2"/></svg> + Revoke + </button> + <button class="admin-btn-add" id="uf-codex-create-btn" style="display:${current ? 'none' : 'inline-flex'};align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M21 2l-9.6 9.6"/><circle cx="7.5" cy="15.5" r="5.5"/><path d="M15.5 7.5l3 3"/></svg> + Create token + </button> + <button class="admin-btn-add" id="uf-codex-save" style="display:none;align-items:center;gap:5px;background:transparent;color:var(--accent, var(--red));border-color:color-mix(in srgb, var(--accent, var(--red)) 45%, var(--border));font-weight:600;"> + <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="20 6 9 17 4 12"/></svg> + Save + </button> </div> </div> </div>`; + // Editing an existing token: surface Revoke alongside Cancel, and stash + // the id so the Revoke handler knows what to DELETE. + if (current) { + formEl.dataset.createdTokenId = String(current.id); + const revokeBtn = el('uf-codex-revoke'); + if (revokeBtn) revokeBtn.style.display = 'inline-flex'; + // Inline rename + per-scope PATCH on change. + const renameInput = el('uf-codex-existing-rename'); + if (renameInput) { + const original = renameInput.value; + const commit = async () => { + const name = (renameInput.value || '').trim(); + if (!name || name === original) return; + try { + const r = await fetch(`/api/tokens/${renameInput.dataset.tokenId}`, { + method: 'PATCH', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name }), + }); + if (!r.ok) throw new Error('Save failed'); + notifyIntegrationsChanged(); + } catch (_) { renameInput.value = original; } + }; + renameInput.addEventListener('blur', commit); + renameInput.addEventListener('keydown', e => { if (e.key === 'Enter') { e.preventDefault(); renameInput.blur(); } }); + } + formEl.querySelectorAll('.uf-codex-scope').forEach(cb => { + cb.addEventListener('change', async () => { + const msg = el('uf-codex-existing-msg'); + const scopes = ['chat'].concat( + Array.from(formEl.querySelectorAll('.uf-codex-scope:checked')).map(input => input.dataset.scope) + ); + try { + const r = await fetch(`/api/tokens/${cb.dataset.tokenId}`, { + method: 'PATCH', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ scopes }), + }); + const d = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(d.detail || 'Failed'); + if (msg) { msg.textContent = 'Saved'; msg.style.color = 'var(--green, #50fa7b)'; setTimeout(() => { msg.textContent = ''; }, 1200); } + notifyIntegrationsChanged(); + } catch (err) { + cb.checked = !cb.checked; + if (msg) { msg.textContent = (err && err.message) || 'Failed'; msg.style.color = 'var(--red)'; } + } + }); + }); + } + el('uf-codex-cancel')?.addEventListener('click', () => { formEl.style.display = 'none'; }); - el('uf-codex-save')?.addEventListener('click', () => { + + // Configure access — collapsed by default so the reveal panel doesn't + // dump 13 toggles at once. Click reveals + rotates the caret. + el('uf-codex-toggle-config')?.addEventListener('click', () => { + const body = el('uf-codex-config-body'); + const btn = el('uf-codex-toggle-config'); + const caret = el('uf-codex-toggle-config-caret'); + if (!body || !btn) return; + const open = body.style.display === 'none'; + body.style.display = open ? '' : 'none'; + btn.setAttribute('aria-expanded', open ? 'true' : 'false'); + if (caret) caret.style.transform = open ? 'rotate(180deg)' : ''; + }); + + el('uf-codex-save')?.addEventListener('click', async () => { const msg = el('uf-codex-msg'); - if (msg) { msg.textContent = 'Saved'; msg.style.color = 'var(--green, #50fa7b)'; } - setTimeout(() => { formEl.style.display = 'none'; }, 350); + const tokenId = formEl.dataset.createdTokenId; + if (!tokenId) { formEl.style.display = 'none'; return; } + const scopes = ['chat'].concat( + Array.from(formEl.querySelectorAll('#uf-codex-inline-scopes .uf-codex-scope:checked')) + .map(input => input.dataset.scope) + ); + try { + const r = await fetch(`/api/tokens/${tokenId}`, { + method: 'PATCH', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ scopes }), + }); + const d = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(d.detail || 'Failed'); + if (msg) { msg.textContent = 'Saved'; msg.style.color = 'var(--green, #50fa7b)'; } + await renderList(); + setTimeout(() => { formEl.style.display = 'none'; }, 350); + } catch (err) { + if (msg) { msg.textContent = err?.message || 'Save failed'; msg.style.color = 'var(--red)'; } + } + }); + + // Revoke = delete this agent token entirely. Confirmation prompt keeps + // it from being a one-click footgun. Closes the form on success. + el('uf-codex-revoke')?.addEventListener('click', async () => { + const tokenId = formEl.dataset.createdTokenId; + if (!tokenId) return; + const ok = window.styledConfirm + ? await window.styledConfirm(`Revoke this ${cfg.word} agent token? Integrations using it will lose access.`, { confirmText: 'Revoke', danger: true }) + : confirm(`Revoke this ${cfg.word} agent token? Integrations using it will lose access.`); + if (!ok) return; + const msg = el('uf-codex-msg'); + try { + const r = await fetch(`/api/tokens/${tokenId}`, { method: 'DELETE', credentials: 'same-origin' }); + if (!r.ok) throw new Error('Revoke failed'); + if (msg) { msg.textContent = 'Revoked'; msg.style.color = 'var(--color-error)'; } + await renderList(); + setTimeout(() => { formEl.style.display = 'none'; }, 350); + } catch (err) { + if (msg) { msg.textContent = err?.message || 'Revoke failed'; msg.style.color = 'var(--red)'; } + } }); const _autoCreateCodex = async () => { const msg = el('uf-codex-msg'); + const prompt = el('uf-codex-prompt'); const pending = el('uf-codex-pending'); + const createBtn = el('uf-codex-create-btn'); + if (prompt) prompt.style.display = 'none'; + if (createBtn) createBtn.style.display = 'none'; + // Whirlpool spinner while the POST is in flight. + let _wp = null; + if (pending) { + pending.innerHTML = ''; + pending.style.display = 'flex'; + try { + const sp = window.spinnerModule || (await import('./spinner.js')).default; + _wp = sp.createWhirlpool(14); + _wp.element.style.cssText = 'display:inline-flex;width:14px;height:14px;margin:0 4px 0 0;'; + pending.appendChild(_wp.element); + pending.appendChild(document.createTextNode('Creating token…')); + } catch (_) { + pending.textContent = 'Creating token…'; + } + } const existingNames = new Set(agentTokens.map(t => (t.name || '').trim())); - let name = cfg.defaultName; - let n = 2; - while (existingNames.has(name)) { name = `${cfg.defaultName} ${n++}`; } + const nameInput = el('uf-codex-name-input'); + // User-typed name wins. Empty / whitespace falls back to the default, + // auto-suffixed with " 2", " 3"… so two tokens never collide. + let name = (nameInput && nameInput.value || '').trim() || cfg.defaultName; + if (existingNames.has(name)) { + let n = 2; + const base = name; + while (existingNames.has(name)) { name = `${base} ${n++}`; } + } + // Minimum scope on creation so the token isn't effectively saved + // with everything granted before the user has clicked Save. The + // UI toggles below are pre-checked as a preview of what *will* + // be granted; nothing else is persisted server-side until Save. const fd = new FormData(); fd.append('name', name); fd.append('scopes', 'chat'); @@ -4955,6 +5483,7 @@ async function initUnifiedIntegrations() { const r = await fetch('/api/tokens', { method: 'POST', credentials: 'same-origin', body: fd }); const d = await r.json(); if (!r.ok) throw new Error(d.detail || 'Failed'); + if (_wp) { try { _wp.destroy(); } catch (_) {} } if (pending) pending.style.display = 'none'; el('uf-codex-token').textContent = d.token || ''; el('uf-codex-reveal').style.display = ''; @@ -4962,23 +5491,31 @@ async function initUnifiedIntegrations() { if (setupBtn) setupBtn.dataset.token = d.token || ''; const setupCode = el('uf-codex-setup-code'); if (setupCode) setupCode.textContent = setupForToken(d.token || ''); - // Populate inline scope toggles for the just-created token (Configure access already open) - const newToken = { id: d.id, name, scopes: d.scopes || ['chat'] }; + // Populate inline scope toggles for the just-created token with + // ALL scopes pre-checked as a UI preview — the underlying token + // still only has 'chat' until the user clicks Save below. + const uiToken = { id: d.id, scopes: ['chat'].concat(toolScopes.map(s => s.key)) }; const inlineEl = el('uf-codex-inline-scopes'); if (inlineEl) { inlineEl.innerHTML = ` - <div class="uf-codex-token" data-token-id="${esc(newToken.id)}"> - ${scopeToggles(newToken)} - <div class="uf-codex-scope-msg" data-token-id="${esc(newToken.id)}" style="font-size:11px;min-height:14px;"></div> + <div class="uf-codex-token" data-token-id="${esc(uiToken.id)}"> + ${scopeToggles(uiToken)} + <div class="uf-codex-scope-msg" data-token-id="${esc(uiToken.id)}" style="font-size:11px;min-height:14px;"></div> </div>`; - _wireScopeChange(inlineEl); + // No auto-PATCH: scope toggles only persist on Save click below. } + // Now that the token exists, surface the Save button. + const saveBtn = el('uf-codex-save'); + if (saveBtn) saveBtn.style.display = 'inline-flex'; + // Remember the created token id so Save can PATCH its scopes. + formEl.dataset.createdTokenId = String(uiToken.id); if (msg) { msg.textContent = `Created "${name}".`; msg.style.color = 'var(--green, #50fa7b)'; } await renderList(); } catch (err) { + if (_wp) { try { _wp.destroy(); } catch (_) {} } if (pending) pending.style.display = 'none'; if (msg) { msg.textContent = err?.message || 'Failed'; @@ -4986,7 +5523,8 @@ async function initUnifiedIntegrations() { } } }; - if (!current) _autoCreateCodex(); + // Bind the explicit Create button; no auto-creation. + el('uf-codex-create-btn')?.addEventListener('click', () => { _autoCreateCodex(); }); const _copyCodexToken = async (text) => { const value = String(text || ''); if (!value) return false; @@ -5020,88 +5558,43 @@ async function initUnifiedIntegrations() { selection.removeAllRanges(); selection.addRange(range); }; + const COPY_ICON = '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg>'; + const CHECK_ICON = '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg>'; el('uf-codex-copy-setup')?.addEventListener('click', async () => { const token = el('uf-codex-copy-setup')?.dataset.token || ''; const btn = el('uf-codex-copy-setup'); - if (!token) { - if (btn) { - btn.textContent = 'Add agent first'; - setTimeout(() => { const latest = el('uf-codex-copy-setup'); if (latest) latest.textContent = 'Copy setup'; }, 1600); - } - return; - } + if (!token) return; const setup = setupForToken(token); const ok = await _copyCodexToken(setup); if (!btn) return; - btn.textContent = ok ? 'Copied setup' : 'Select setup'; - if (!ok) _selectTextFallback(setup, 'uf-codex-reveal'); - setTimeout(() => { const latest = el('uf-codex-copy-setup'); if (latest) latest.textContent = 'Copy setup'; }, 1600); + if (ok) { + btn.innerHTML = CHECK_ICON; + btn.style.color = 'var(--accent, var(--red))'; + btn.style.opacity = '1'; + } else { + _selectTextFallback(setup, 'uf-codex-reveal'); + } + setTimeout(() => { + const latest = el('uf-codex-copy-setup'); + if (latest) { latest.innerHTML = COPY_ICON; latest.style.color = ''; latest.style.opacity = '0.7'; } + }, 1600); }); el('uf-codex-copy-token')?.addEventListener('click', async () => { const token = el('uf-codex-token')?.textContent || ''; const ok = await _copyCodexToken(token); const btn = el('uf-codex-copy-token'); if (!btn) return; - btn.textContent = ok ? 'Copied token' : 'Select token'; - if (!ok) _selectTextFallback(token, 'uf-codex-reveal'); - setTimeout(() => { const latest = el('uf-codex-copy-token'); if (latest) latest.textContent = 'Copy token'; }, 1600); - }); - formEl.querySelectorAll('.uf-codex-revoke').forEach(btn => { - btn.addEventListener('click', async () => { - if (!await window.styledConfirm(`Revoke this ${cfg.word} token? Integrations using it will lose access.`, { confirmText: 'Revoke', danger: true })) return; - await fetch(`/api/tokens/${btn.dataset.tokenId}`, { method: 'DELETE', credentials: 'same-origin' }); - formEl.style.display = 'none'; - await renderList(); - }); - }); - // Rename: PATCH the token's name when the user blurs the input (or hits Enter). - formEl.querySelectorAll('.uf-codex-rename').forEach(input => { - const original = input.value; - const commit = async () => { - const name = (input.value || '').trim(); - if (!name || name === original) return; - try { - const r = await fetch(`/api/tokens/${input.dataset.tokenId}`, { - method: 'PATCH', - credentials: 'same-origin', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ name }), - }); - if (!r.ok) throw new Error('Save failed'); - input.style.borderColor = 'var(--green, #50fa7b)'; - setTimeout(() => { input.style.borderColor = 'transparent'; }, 800); - await renderList(); - } catch (_) { - input.value = original; - input.style.borderColor = 'var(--red)'; - setTimeout(() => { input.style.borderColor = 'transparent'; }, 1200); - } - }; - input.addEventListener('blur', commit); - input.addEventListener('keydown', (e) => { if (e.key === 'Enter') { e.preventDefault(); input.blur(); } }); - }); - // Copy token prefix (full token irrecoverable after the one-time creation reveal). - formEl.querySelectorAll('.uf-codex-copy-prefix').forEach(btn => { - btn.addEventListener('click', async () => { - const prefix = btn.dataset.tokenPrefix || ''; - if (!prefix) return; - try { - if (navigator.clipboard && window.isSecureContext) { - await navigator.clipboard.writeText(prefix); - } else { - const ta = document.createElement('textarea'); - ta.value = prefix; - ta.style.cssText = 'position:fixed;left:0;top:0;width:1px;height:1px;opacity:0;'; - document.body.appendChild(ta); - ta.select(); - try { document.execCommand('copy'); } catch (_) {} - ta.remove(); - } - const label = btn.textContent; - btn.textContent = 'Copied prefix'; - setTimeout(() => { btn.textContent = label; }, 1400); - } catch (_) {} - }); + if (ok) { + btn.innerHTML = CHECK_ICON; + btn.style.color = 'var(--accent, var(--red))'; + btn.style.opacity = '1'; + } else { + _selectTextFallback(token, 'uf-codex-reveal'); + } + setTimeout(() => { + const latest = el('uf-codex-copy-token'); + if (latest) { latest.innerHTML = COPY_ICON; latest.style.color = ''; latest.style.opacity = '0.7'; } + }, 1600); }); function _wireScopeChange(scope) { scope.querySelectorAll('.uf-codex-scope').forEach(cb => { @@ -5130,76 +5623,68 @@ async function initUnifiedIntegrations() { }); }); } - _wireScopeChange(formEl); + // Note: don't call _wireScopeChange(formEl) here. The existing-token + // editor (current) already wires its own change handler that PATCHes + // immediately. The inline scopes for a *just-created* token should + // remain unwired so they only persist on Save click below. } - // ── Add button with type picker ── + // ── Add button now drops a type-picker menu directly anchored to itself ── if (addBtn) { - addBtn.addEventListener('click', () => { - formEl.style.display = ''; - const _typeOptions = [ - ['api', 'API Service'], - ['caldav', 'CalDAV Calendar'], - ['claude', 'Claude Agent'], - ['codex', 'Codex Agent'], - ['carddav', 'Contacts (CardDAV)'], - ['contacts', 'Contacts Import'], - ['email', 'Email (IMAP/SMTP)'], - ['mcp', 'MCP Tool Server'], - ]; - const _iconFor = (k) => (INTG_TYPES[k]?.icon || '').replace(/width="14"/, 'width="16"').replace(/height="14"/, 'height="16"'); - const _rowsHtml = _typeOptions.map(([k, label]) => `<button type="button" class="uf-type-option" data-value="${k}" style="display:flex;align-items:center;gap:10px;width:100%;padding:8px 10px;background:transparent;border:0;color:var(--fg);font:inherit;cursor:pointer;text-align:left;"><span style="display:inline-flex;color:var(--accent, var(--red));flex-shrink:0;">${_iconFor(k)}</span><span>${esc(label)}</span></button>`).join(''); - formEl.innerHTML = ` - <div class="admin-card" style="margin-top:8px"> - <h2 style="font-size:13px;display:flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="color:var(--accent, var(--red));flex-shrink:0;"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"/><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"/></svg>Add Integration</h2> - <div class="settings-col"> - <div class="settings-row"><label class="settings-label">Type</label> - <div style="position:relative;flex:1;min-width:0;"> - <button type="button" id="uf-type-trigger" class="settings-select" style="display:flex;align-items:center;gap:10px;cursor:pointer;text-align:left;width:100%;padding-right:24px;position:relative;"> - <span class="uf-type-icon" style="display:inline-flex;color:var(--accent, var(--red));"></span> - <span class="uf-type-label" style="flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;opacity:0.65;">Select...</span> - <span aria-hidden="true" style="position:absolute;right:8px;top:50%;transform:translateY(-50%);opacity:0.5;font-size:10px;pointer-events:none;">▾</span> - </button> - <div id="uf-type-menu" style="display:none;position:absolute;top:calc(100% + 2px);left:0;right:0;z-index:1000;background:var(--panel);border:1px solid var(--border);border-radius:6px;max-height:340px;overflow-y:auto;box-shadow:0 6px 18px rgba(0,0,0,0.25);">${_rowsHtml}</div> - </div> - </div> - </div> - </div>`; - const trigger = el('uf-type-trigger'); - const menu = el('uf-type-menu'); - const labelEl = trigger.querySelector('.uf-type-label'); - const iconEl = trigger.querySelector('.uf-type-icon'); - const _closeMenu = () => { menu.style.display = 'none'; }; - const _openMenu = () => { - menu.style.display = 'block'; - // Drop-up when there's not enough room below the trigger (mobile - // landscape / docked keyboard / long lists near the bottom of screen). - const tRect = trigger.getBoundingClientRect(); - const mRect = menu.getBoundingClientRect(); - const below = window.innerHeight - tRect.bottom; - const above = tRect.top; - if (mRect.height > below && above > below) { - menu.style.top = 'auto'; menu.style.bottom = 'calc(100% + 2px)'; - } else { - menu.style.top = 'calc(100% + 2px)'; menu.style.bottom = 'auto'; - } - const onDoc = (ev) => { if (!menu.contains(ev.target) && ev.target !== trigger) { _closeMenu(); document.removeEventListener('click', onDoc, true); } }; - setTimeout(() => document.addEventListener('click', onDoc, true), 0); - }; - trigger.addEventListener('click', (e) => { e.stopPropagation(); menu.style.display === 'block' ? _closeMenu() : _openMenu(); }); + const _typeOptions = [ + ['api', 'API Service'], + ['caldav', 'CalDAV Calendar'], + ['claude', 'Claude Agent'], + ['codex', 'Codex Agent'], + ['carddav', 'Contacts (CardDAV)'], + ['contacts', 'Contacts Import'], + ['email', 'Email (IMAP/SMTP)'], + ['mcp', 'MCP Tool Server'], + ]; + const _iconFor = (k) => (INTG_TYPES[k]?.icon || '').replace(/width="14"/, 'width="16"').replace(/height="14"/, 'height="16"'); + const _rowsHtml = _typeOptions.map(([k, label]) => `<button type="button" class="uf-type-option" data-value="${k}" style="display:flex;align-items:center;gap:10px;width:100%;padding:8px 10px;background:transparent;border:0;color:var(--fg);font:inherit;cursor:pointer;text-align:left;"><span style="display:inline-flex;color:var(--accent, var(--red));flex-shrink:0;">${_iconFor(k)}</span><span>${esc(label)}</span></button>`).join(''); + + // Anchor wrapper so the absolutely-positioned menu lands directly under + // the add button. The button is the wrapper's only sibling. + if (!addBtn.parentElement.classList.contains('uf-add-anchor')) { + addBtn.parentElement.style.position = 'relative'; + addBtn.parentElement.classList.add('uf-add-anchor'); + } + let _menuEl = null; + const _closeMenu = () => { if (_menuEl) { _menuEl.remove(); _menuEl = null; } }; + addBtn.addEventListener('click', (e) => { + e.stopPropagation(); + if (_menuEl) { _closeMenu(); return; } + const menu = document.createElement('div'); + menu.className = 'uf-add-menu'; + menu.innerHTML = _rowsHtml; + menu.style.cssText = 'position:absolute;right:0;z-index:1000;background:var(--panel);border:1px solid var(--border);border-radius:6px;max-height:340px;overflow-y:auto;box-shadow:0 6px 18px rgba(0,0,0,0.25);min-width:220px;'; + addBtn.parentElement.appendChild(menu); + _menuEl = menu; + // Drop-up when there isn't enough room below the button (modal near + // the viewport bottom, mobile keyboard up, etc.). + const tRect = addBtn.getBoundingClientRect(); + const mRect = menu.getBoundingClientRect(); + const below = window.innerHeight - tRect.bottom; + const above = tRect.top; + if (mRect.height > below && above > below) { + menu.style.top = 'auto'; menu.style.bottom = 'calc(100% + 2px)'; + } else { + menu.style.top = 'calc(100% + 2px)'; menu.style.bottom = 'auto'; + } menu.querySelectorAll('.uf-type-option').forEach(btn => { btn.addEventListener('mouseenter', () => { btn.style.background = 'color-mix(in srgb, var(--fg) 8%, transparent)'; }); btn.addEventListener('mouseleave', () => { btn.style.background = 'transparent'; }); - btn.addEventListener('click', (e) => { - e.stopPropagation(); + btn.addEventListener('click', (ev) => { + ev.stopPropagation(); const k = btn.dataset.value; - const lbl = btn.querySelector('span:last-child')?.textContent || ''; - if (labelEl) { labelEl.textContent = lbl; labelEl.style.opacity = '1'; } - if (iconEl) iconEl.innerHTML = _iconFor(k); _closeMenu(); + formEl.style.display = ''; showForm(k, 'new'); }); }); + const onDoc = (ev) => { if (!menu.contains(ev.target) && ev.target !== addBtn) { _closeMenu(); document.removeEventListener('click', onDoc, true); } }; + setTimeout(() => document.addEventListener('click', onDoc, true), 0); }); } @@ -5260,6 +5745,40 @@ export function close() { } } +// Handle redirect back from Google OAuth2 — open settings to integrations and show status. +(function _handleOauthRedirect() { + const sp = new URLSearchParams(window.location.search); + if (!sp.has('email_oauth_success') && !sp.has('email_oauth_error')) return; + // Strip params from URL without a page reload. + const clean = window.location.pathname + window.location.hash; + window.history.replaceState(null, '', clean); + const success = sp.has('email_oauth_success'); + const errMsg = sp.get('email_oauth_error') || ''; + // Open settings → integrations after the app has initialised. + function _tryOpen() { + if (window.settingsModule && typeof window.settingsModule.open === 'function') { + window.settingsModule.open('integrations'); + // Brief toast-style banner. + const banner = document.createElement('div'); + banner.textContent = success + ? '✓ Google account connected — email is ready' + : `Google OAuth failed: ${errMsg || 'unknown error'}`; + Object.assign(banner.style, { + position: 'fixed', bottom: '24px', left: '50%', transform: 'translateX(-50%)', + background: success ? 'var(--accent, #50fa7b)' : 'var(--red, #ff5555)', + color: '#000', padding: '8px 18px', borderRadius: '6px', fontSize: '12px', + fontWeight: '600', zIndex: '99999', pointerEvents: 'none', + boxShadow: '0 2px 12px rgba(0,0,0,0.3)', + }); + document.body.appendChild(banner); + setTimeout(() => banner.remove(), 4000); + } else { + setTimeout(_tryOpen, 100); + } + } + _tryOpen(); +})(); + const settingsModule = { open, close, initIntegrations, initUnifiedIntegrations, syncAdminVisibility, refreshAiModelEndpoints }; diff --git a/static/js/skills.js b/static/js/skills.js index 8eac3954c..104d684a1 100644 --- a/static/js/skills.js +++ b/static/js/skills.js @@ -91,7 +91,18 @@ export async function loadSkills(cascade = false) { try { const res = await fetch(`${API}/api/skills`); const data = await res.json(); - skills = data.skills || []; + // Dedupe by name (case-insensitive) — the API has occasionally + // returned the same skill twice (built-in shadow + user copy, or + // a write-then-read race), and rendering both made the duplicate + // detector mark BOTH entries as the "recommended" keeper. + const _seen = new Set(); + skills = (data.skills || []).filter(sk => { + const k = String(sk?.name || sk?.id || '').toLowerCase(); + if (!k) return true; + if (_seen.has(k)) return false; + _seen.add(k); + return true; + }); _loadSkillApprovalThreshold(); // Built-in capabilities are no longer surfaced in the Skills menu. loaded = true; @@ -392,21 +403,11 @@ function _openSkillMenu(btn, card, sk, name, isPublished) { }; if (isPublished) mk(_ICON.unpublish, 'Unpublish', {}, () => _setSkillStatus(name, 'draft')); else mk(_ICON.approve, 'Publish', {}, () => _setSkillStatus(name, 'published')); - mk(_ICON.edit, 'Edit', {}, async () => { - if (!card.classList.contains('doclib-card-expanded')) await _expandSkillCard(card, name); - _toggleSkillEdit(card, name); - }); - mk(_ICON.test, 'Test', {}, () => _testSkill(card, name)); - // Audit kicks off the bulk audit-all loop (test → judge → fix → retry → demote). - // Starts at the top of the list and walks down. - mk(_ICON.test, 'Audit', {}, () => _auditAllSkills()); - mk(_ICON.del, 'Delete', { danger: true }, () => _deleteSkill(name, card)); - - // Select — enters bulk-select mode and pre-selects this skill. Same pattern - // as the email/documents/brain Select item, with the email bullet icon. + // Select — moved up to 2nd so it sits next to Publish/Unpublish + // (bulk actions cluster at the top of the menu). const selItem = document.createElement('button'); selItem.className = 'skill-kebab-item'; - selItem.innerHTML = '<span style="display:inline-flex;width:14px;height:14px;align-items:center;justify-content:center;"><span style="font-size:16px;line-height:1;">●</span></span><span>Select</span>'; + selItem.innerHTML = '<svg class="memory-select-btn-icon" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="flex-shrink:0;"><circle cx="12" cy="12" r="10"/><circle cx="12" cy="12" r="3" fill="currentColor" stroke="none"/></svg><span>Select</span>'; selItem.addEventListener('click', (e) => { e.stopPropagation(); menu.remove(); @@ -416,6 +417,15 @@ function _openSkillMenu(btn, card, sk, name, isPublished) { }); menu.appendChild(selItem); + mk(_ICON.edit, 'Edit', {}, async () => { + if (!card.classList.contains('doclib-card-expanded')) await _expandSkillCard(card, name); + _toggleSkillEdit(card, name); + }); + mk(_ICON.test, 'Test', {}, () => _testSkill(card, name)); + // Audit kicks off the bulk audit-all loop (test → judge → fix → retry → demote). + mk(_ICON.test, 'Audit', {}, () => _auditAllSkills()); + mk(_ICON.del, 'Delete', { danger: true }, () => _deleteSkill(name, card)); + // Mobile-only Cancel — mirrors the email/documents/brain popup pattern. // CSS hides `.dropdown-cancel-mobile` on desktop where outside-click // already dismisses cleanly. @@ -514,6 +524,8 @@ function _buildBuiltinCards() { card.addEventListener('click', (e) => { if (e.target.closest('button, input, textarea')) return; + // Editing in progress → don't collapse on an outside-the-textarea click. + if (card.querySelector('.skill-md-editor')) return; _expandBuiltinCard(card, b.name); }); return card; @@ -786,6 +798,10 @@ function renderSkillsList() { card.addEventListener('click', (e) => { if (card._suppressNextClick) { card._suppressNextClick = false; return; } if (e.target.closest('button, input, textarea')) return; + // While editing, a click on the card body (outside the textarea) must + // NOT collapse the card — that silently discards unsaved edits. Only + // Save/Cancel exit edit mode. + if (card.querySelector('.skill-md-editor')) return; if (_selectMode) { const cb = card.querySelector('.skill-select-cb'); if (cb) { cb.checked = !cb.checked; cb.dispatchEvent(new Event('change')); } @@ -1591,13 +1607,16 @@ function _renderAuditPanel(panel, st) { // ---- Select mode / bulk actions ---- +const _SKILLS_SELECT_BTN_DOT_SVG = '<svg class="memory-select-btn-icon" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:3px;"><circle cx="12" cy="12" r="10"/><circle cx="12" cy="12" r="3" fill="currentColor" stroke="none"/></svg>'; +const _SKILLS_SELECT_BTN_X_SVG = '<svg class="memory-select-btn-icon" width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" style="vertical-align:-2px;margin-right:3px;"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>'; + function _enterSelectMode() { _selectMode = true; _selectedNames.clear(); const bar = document.getElementById('skills-bulk-bar'); const btn = document.getElementById('skills-select-btn'); if (bar) bar.classList.remove('hidden'); - if (btn) { btn.classList.add('active'); btn.textContent = 'Cancel'; } + if (btn) { btn.classList.add('active'); btn.innerHTML = _SKILLS_SELECT_BTN_X_SVG + 'Cancel'; } _updateBulkBar(); renderSkillsList(); } @@ -1609,7 +1628,7 @@ function _exitSelectMode() { const btn = document.getElementById('skills-select-btn'); const all = document.getElementById('skills-select-all'); if (bar) bar.classList.add('hidden'); - if (btn) { btn.classList.remove('active'); btn.textContent = 'Select'; } + if (btn) { btn.classList.remove('active'); btn.innerHTML = _SKILLS_SELECT_BTN_DOT_SVG + 'Select'; } if (all) all.checked = false; renderSkillsList(); } diff --git a/static/js/slashCommands.js b/static/js/slashCommands.js index 79b037cf4..07d96dc9d 100644 --- a/static/js/slashCommands.js +++ b/static/js/slashCommands.js @@ -17,6 +17,7 @@ import chatRenderer from './chatRenderer.js'; import spinnerModule from './spinner.js'; import themeModule from './theme.js'; import documentModule from './document.js'; +import workspaceModule from './workspace.js'; import settingsModule from './settings.js'; import cookbookModule from './cookbook.js'; import { EVAL_PROMPTS } from './compare/index.js'; @@ -338,10 +339,13 @@ function _submitComposedMessage(text) { const msgInput = document.getElementById('message'); const form = document.getElementById('chat-form'); if (!msgInput || !form) return false; - msgInput.value = text; - msgInput.dispatchEvent(new Event('input', { bubbles: true })); - if (typeof form.requestSubmit === 'function') form.requestSubmit(); - else form.dispatchEvent(new Event('submit', { cancelable: true, bubbles: true })); + // The slash handler and app-level form debounce must both release before + // sending the pinned prompt, otherwise the follow-up submit is dropped. + setTimeout(() => { + msgInput.value = text; + msgInput.dispatchEvent(new Event('input', { bubbles: true })); + form.dispatchEvent(new Event('submit', { cancelable: true, bubbles: true })); + }, 350); return true; } @@ -1229,6 +1233,40 @@ async function _cmdToggleDoc(args, ctx) { return true; } +// Workspace: confine the agent's file/shell tools to a folder. Not a boolean - +// show / set <path> / clear / pick (open the directory browser). +async function _cmdWorkspace(args, ctx) { + const sub = (args[0] || '').toLowerCase(); + const rest = args.slice(1).join(' ').trim(); + const cur = workspaceModule.getWorkspace(); + if (!sub || sub === 'show' || sub === 'status' || sub === 'info') { + slashReply(cur ? `Workspace: <code>${uiModule.esc(cur)}</code>` : 'No workspace set. <code>/workspace pick</code> or <code>/workspace set /path</code>.'); + return true; + } + if (sub === 'set' || sub === 'cd' || sub === 'use') { + if (!rest) { slashReply('Usage: <code>/workspace set /absolute/path</code>'); return true; } + // Validate server-side before persisting so the pill never claims a + // workspace the backend will refuse to bind (typo, file path, deleted + // folder, sensitive dir, filesystem root). + workspaceModule.vetAndSetWorkspace(rest).then(({ ok, path }) => { + if (ok) slashReply(`Workspace set: <code>${uiModule.esc(path)}</code>`); + else slashReply(`Not a usable workspace folder: <code>${uiModule.esc(rest)}</code>. It must be an existing directory, not a filesystem root or sensitive path.`); + }); + return true; + } + if (sub === 'clear' || sub === 'off' || sub === 'none' || sub === 'unset') { + workspaceModule.clearWorkspace(); + slashReply('Workspace cleared.'); + return true; + } + if (sub === 'pick' || sub === 'browse' || sub === 'open') { + workspaceModule.openWorkspaceBrowser(); + return true; + } + slashReply('Usage: <code>/workspace</code> · <code>set /path</code> · <code>clear</code> · <code>pick</code>'); + return true; +} + async function _cmdToggleShow(args, ctx) { const name = (args[0] || '').toLowerCase(); const val = (args[1] || '').toLowerCase(); @@ -5731,6 +5769,14 @@ const COMMANDS = { '_show': { handler: _cmdToggleShow, alias: [], help: 'Show all toggle states', usage: '/toggle' } } }, + workspace: { + alias: ['ws'], + category: 'Agent', + help: 'Set the folder the agent works in', + handler: _cmdWorkspace, + noUserBubble: true, + usage: '/workspace [set <path> | clear | pick]', + }, memory: { alias: ['m'], category: 'Memory', diff --git a/static/js/storage.js b/static/js/storage.js index c72a5dbb1..7ff9c6bd5 100644 --- a/static/js/storage.js +++ b/static/js/storage.js @@ -23,7 +23,8 @@ export const KEYS = { MCP_ACTIVE: 'odysseus-mcp-active', SECTION_ORDER: 'sidebar-section-order', ADMIN_LAST_TAB: 'admin-last-tab', - DENSITY: 'odysseus-density' + DENSITY: 'odysseus-density', + WORKSPACE: 'odysseus-workspace' }; /** diff --git a/static/js/tasks.js b/static/js/tasks.js index 03e426f73..a74bde151 100644 --- a/static/js/tasks.js +++ b/static/js/tasks.js @@ -1077,9 +1077,23 @@ function _showForm(existing, initTaskType, initTriggerType) { typeOpts.innerHTML = ''; if (taskType === 'llm' || taskType === 'research') { const placeholder = taskType === 'research' ? 'What should be researched?' : 'What should the AI do?'; + const _personaOpts = [ + ['', 'Default (no persona)'], + ['socrates', 'Socrates'], + ['razor', 'Razor'], + ['nietzsche', 'Nietzsche'], + ['spark', 'Spark'], + ['odysseus', 'Odysseus'], + ]; + const _curPersona = (existing?.character_id || '').toLowerCase(); + const _personaOptsHtml = _personaOpts.map(([v, label]) => + `<option value="${v}" ${v === _curPersona ? 'selected' : ''}>${label}</option>`).join(''); typeOpts.innerHTML = ` <label class="task-form-label">${taskType === 'research' ? 'Research question' : 'Prompt'}</label> <textarea id="task-form-prompt" class="task-form-input task-form-textarea" rows="4" placeholder="${placeholder}">${existing?.prompt || ''}</textarea> + + <label class="task-form-label">Persona <span style="opacity:0.5;font-weight:normal;font-size:10px;">(optional — biases the output voice)</span></label> + <select id="task-form-persona" class="task-form-input">${_personaOptsHtml}</select> `; } else { typeOpts.innerHTML = ` @@ -1437,7 +1451,11 @@ function _showForm(existing, initTaskType, initTriggerType) { return; } payload.prompt = prompt; + const personaVal = document.getElementById('task-form-persona')?.value || ''; + payload.character_id = personaVal; } else { + // Non-llm/research tasks: explicitly clear any persona on switch. + payload.character_id = ''; const action = document.getElementById('task-form-action')?.value; if (!action) { if (uiModule) uiModule.showError('Select an action'); @@ -2482,12 +2500,15 @@ function _renderMainView() { // ---- Modal ---- -export function openTasks(focusId) { +export function openTasks(focusId, opts) { + const o = opts || {}; if (_open) { - // Already open — just focus the requested task. + // Already open — just focus the requested task / apply filter. + if (o.filter !== undefined) { _taskFilter = o.filter; _renderList(); } if (focusId) _focusTask(focusId); return; } + if (o.filter !== undefined) _taskFilter = o.filter; _pendingFocusTaskId = focusId || null; _open = true; _tasksCascadeNext = true; diff --git a/static/js/tileManager.js b/static/js/tileManager.js index e70e13e80..3ce1b1238 100644 --- a/static/js/tileManager.js +++ b/static/js/tileManager.js @@ -6,16 +6,13 @@ * when the cursor is near a snap zone. On release, snaps the modal-content * to fill that zone with a springy animation. * - * Snap zones (9): - * - top edge (10% strip) → maximize - * - top-left corner → top-left quarter - * - top-right corner → top-right quarter + * Snap zones: + * - over top edge → fullscreen + * - top strip → maximize + * - top edge → top half * - left edge → left half * - right edge → right half - * - bottom-left corner → bottom-left quarter - * - bottom-right corner → bottom-right quarter * - bottom edge → bottom half - * - sidebar edge (if present) → snap next to the sidebar * * Mobile (≤768px) is excluded — the swipe-dismiss UX takes precedence. * @@ -24,7 +21,6 @@ */ const EDGE_THRESHOLD_PX = 24; // how close to an edge counts as "near" -const CORNER_THRESHOLD_PX = 64; // corner box size const TOP_FULL_STRIP_PX = 8; // top strip → maximize let _ghost = null; @@ -111,9 +107,13 @@ function _zoneForPointer(x, y) { return { name: 'maximize', rect: { left: safe.left, top: safe.top, width: W, height: H } }; } - // Corner quarter-snaps DISABLED (user request) — only the top strip - // (maximize) and the right/bottom half-snaps remain. The LEFT-half snap - // is also disabled (the sidebar lives there; docking over it is awkward). + // Symmetric edge half-snaps. The safe rect already starts to the right of + // the sidebar/rail, so left-half fills the left side of the workspace + // without covering navigation. + if (y <= safe.top + EDGE_THRESHOLD_PX) + return { name: 'top-half', rect: { left: safe.left, top: safe.top, width: W, height: H / 2 } }; + if (x <= safe.left + EDGE_THRESHOLD_PX) + return { name: 'left-half', rect: { left: safe.left, top: safe.top, width: W / 2, height: H } }; if (x >= safe.right - EDGE_THRESHOLD_PX) return { name: 'right-half', rect: { left: safe.left + W / 2, top: safe.top, width: W / 2, height: H } }; if (y >= safe.bottom - EDGE_THRESHOLD_PX) @@ -131,8 +131,7 @@ function _zoneForContent(content, x, y) { // flip to top tabs via CSS when the window gets narrow. if (modal && modal.id === 'settings-modal' && zone.name !== 'right-half') return null; if (modal && (modal.id === 'cookbook-modal' - || modal.id === 'theme-modal' - || modal.id === 'memory-modal') + || modal.id === 'theme-modal') && zone.name !== 'fullscreen') return null; return zone; } @@ -304,6 +303,7 @@ function _reclampAll(animate = false) { switch (name) { case 'fullscreen': r = { left: 0, top: 0, width: window.innerWidth, height: window.innerHeight }; break; case 'maximize': r = { left: safe.left, top: safe.top, width: W, height: H }; break; + case 'top-half': r = { left: safe.left, top: safe.top, width: W, height: H/2 }; break; case 'left-half': r = { left: safe.left, top: safe.top, width: W/2, height: H }; break; case 'right-half': r = { left: safe.left + W/2, top: safe.top, width: W/2, height: H }; break; case 'bottom-half': r = { left: safe.left, top: safe.top + H/2, width: W, height: H/2 }; break; @@ -374,6 +374,14 @@ export function clearPreview() { _activeZone = null; } +export function _zoneForPointerForTests(x, y) { + return _zoneForPointer(x, y); +} + +export function _zoneForContentForTests(content, x, y) { + return _zoneForContent(content, x, y); +} + // Snap a modal (its .modal-content) into a previously-detected zone. export function snapModalToZone(modal, zone) { if (!modal || !zone) return; diff --git a/static/js/windowDrag.js b/static/js/windowDrag.js index 5e7cb0c9d..5f2b62f3c 100644 --- a/static/js/windowDrag.js +++ b/static/js/windowDrag.js @@ -61,7 +61,7 @@ export function makeWindowDraggable(modal, options = {}) { const fsClass = options.fsClass || null; const onEnterFullscreen = options.onEnterFullscreen || null; const onExitFullscreen = options.onExitFullscreen || null; - const enableFullscreen = options.enableFullscreen !== false && !!onEnterFullscreen; + const enableFullscreen = false; const onDragEnd = options.onDragEnd || null; const onDragStart = options.onDragStart || null; const skipSelector = options.skipSelector || 'button, input, select'; diff --git a/static/js/workspace.js b/static/js/workspace.js new file mode 100644 index 000000000..fd6ab4184 --- /dev/null +++ b/static/js/workspace.js @@ -0,0 +1,208 @@ +// static/js/workspace.js +// +// Workspace picker: browse server directories in a draggable modal, choose a +// folder, and show it as a removable pill in the chat input bar. While set, the +// chat request sends `workspace` so the agent's file/shell tools are confined +// to that folder (see routes/chat_routes.py + src/tool_execution.py). + +import Storage, { KEYS } from './storage.js'; +import uiModule from './ui.js'; +import { makeWindowDraggable } from './windowDrag.js'; + +const API_BASE = window.location.origin; +// Same folder glyph as the overflow menu item + pill (not an emoji). +const _FOLDER_SVG = '<svg class="workspace-row-icon" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 7a2 2 0 0 1 2-2h4l2 2h8a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2z"/></svg>'; +let _modal = null; +let _curPath = ''; + +export function getWorkspace() { + return Storage.get(KEYS.WORKSPACE, '') || ''; +} + +function _basename(p) { + if (!p) return ''; + // Handle both POSIX (/) and Windows (\) separators. + const parts = p.replace(/[\\/]+$/, '').split(/[\\/]/); + return parts[parts.length - 1] || p; +} + +// Workspace only applies to agent mode (it scopes the file/shell tools), so the +// pill + overflow entry are hidden in chat mode, like the bash toggle. +function _isChatMode() { + const b = document.getElementById('mode-chat-btn'); + return !!(b && b.classList.contains('active')); +} + +export function syncWorkspaceIndicator(path) { + const chat = _isChatMode(); + const pill = document.getElementById('workspace-indicator-btn'); + const name = document.getElementById('workspace-indicator-name'); + const overflow = document.getElementById('overflow-workspace-btn'); + if (pill) { + pill.style.display = (path && !chat) ? '' : 'none'; + pill.classList.toggle('active', !!path); + if (path) pill.title = `Workspace: ${path}\nFile tools are confined here; shell commands start here but are not sandboxed and can reach outside it.\nClick to clear.`; + } + if (name) name.textContent = path ? _basename(path) : ''; + if (overflow) { + overflow.style.display = chat ? 'none' : ''; + overflow.classList.toggle('active', !!path); + } + // Recompute the "+" overflow dot (app.js owns updatePlusDot via this event). + try { document.dispatchEvent(new CustomEvent('overflow-state-change')); } catch (_) {} +} + +// Called by the agent/chat mode toggle so the pill + overflow entry follow mode. +export function applyMode(_mode) { + syncWorkspaceIndicator(getWorkspace()); +} + +export function setWorkspace(path) { + if (path) Storage.set(KEYS.WORKSPACE, path); + else Storage.remove(KEYS.WORKSPACE); + syncWorkspaceIndicator(path || ''); +} + +/** + * Validate a manually entered path server-side, then persist the canonical + * form. Returns {ok, path|null}. Without this, a typo / file path / deleted + * folder / filesystem root would be stored and shown as active while the + * backend silently refuses to bind it on every send. + */ +export async function vetAndSetWorkspace(path) { + try { + const res = await fetch(`${API_BASE}/api/workspace/vet?path=${encodeURIComponent(path)}`, { credentials: 'same-origin' }); + if (!res.ok) return { ok: false, path: null }; + const data = await res.json(); + if (data.ok && data.path) { + setWorkspace(data.path); + return { ok: true, path: data.path }; + } + return { ok: false, path: null }; + } catch (e) { + return { ok: false, path: null }; + } +} + +export function clearWorkspace() { + setWorkspace(''); + if (uiModule && uiModule.showToast) uiModule.showToast('Workspace cleared'); +} + +async function _load(path) { + const url = `${API_BASE}/api/workspace/browse${path ? `?path=${encodeURIComponent(path)}` : ''}`; + const res = await fetch(url, { credentials: 'same-origin' }); + if (!res.ok) throw new Error(`browse failed: ${res.status}`); + return res.json(); +} + +function _render(data) { + _curPath = data.path; + const body = _modal.querySelector('#workspace-body'); + const pathEl = _modal.querySelector('#workspace-cur-path'); + if (pathEl) { + // Reflect the resolved (realpath) location back into the editable field. + pathEl.value = data.path; + pathEl.title = data.path; + } + let rows = ''; + if (data.parent) { + rows += `<div class="workspace-row workspace-up" data-path="${encodeURIComponent(data.parent)}">↑ ..</div>`; + } + for (const d of data.dirs) { + // Backend supplies the full child path (os.path.join → cross-platform). + rows += `<div class="workspace-row" data-path="${encodeURIComponent(d.path)}">${_FOLDER_SVG}<span>${uiModule.esc(d.name)}</span></div>`; + } + if (data.truncated) { + rows += '<div class="workspace-empty">Too many folders to list. Type or paste a path above to jump in.</div>'; + } + if (!data.dirs.length && !data.parent) rows = '<div class="workspace-empty">No subfolders</div>'; + body.innerHTML = rows || '<div class="workspace-empty">No subfolders</div>'; + body.querySelectorAll('.workspace-row').forEach((row) => { + row.addEventListener('click', () => _navigate(decodeURIComponent(row.dataset.path))); + }); + // Filesystem roots (and sensitive dirs) can be browsed through but never + // bound as the workspace; the backend rejects them too. + const useBtn = _modal.querySelector('#workspace-use'); + if (useBtn) { + useBtn.disabled = data.selectable === false; + useBtn.title = data.selectable === false ? 'This folder cannot be used as a workspace' : ''; + } +} + +async function _navigate(path) { + try { + _render(await _load(path)); + } catch (e) { + if (uiModule && uiModule.showError) uiModule.showError('Could not open folder'); + } +} + +function _getModal() { + if (_modal) return _modal; + _modal = document.createElement('div'); + _modal.id = 'workspace-modal'; + _modal.className = 'modal'; + _modal.style.display = 'none'; + _modal.innerHTML = ` + <div class="modal-content"> + <div class="modal-header"> + <h4><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:6px"><path d="M3 7a2 2 0 0 1 2-2h4l2 2h8a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2z"/></svg>Select workspace</h4> + <button class="close-btn" id="workspace-close" aria-label="Close">✖</button> + </div> + <input type="text" class="styled-prompt-input workspace-cur" id="workspace-cur-path" + spellcheck="false" autocomplete="off" autocapitalize="off" autocorrect="off" + placeholder="Type or paste a folder path, then press Enter" /> + <p class="muted workspace-note">File tools are <strong>confined</strong> to this folder. Shell commands start here but are <strong>not sandboxed</strong> and can reach outside it. A workspace scopes the tools; it is not a security boundary.</p> + <div class="modal-body workspace-body" id="workspace-body"></div> + <div class="modal-footer workspace-footer"> + <button type="button" class="confirm-btn confirm-btn-secondary" id="workspace-cancel">Cancel</button> + <button type="button" class="confirm-btn confirm-btn-primary" id="workspace-use">Use this folder</button> + </div> + </div>`; + document.body.appendChild(_modal); + _modal.querySelector('#workspace-close').addEventListener('click', closeWorkspaceBrowser); + _modal.querySelector('#workspace-cancel').addEventListener('click', closeWorkspaceBrowser); + // Editable path bar: Enter navigates to a typed/pasted folder. + _modal.querySelector('#workspace-cur-path').addEventListener('keydown', (e) => { + if (e.key === 'Enter') { + e.preventDefault(); + const v = e.target.value.trim(); + if (v) _navigate(v); + } + }); + _modal.querySelector('#workspace-use').addEventListener('click', () => { + setWorkspace(_curPath); + if (uiModule && uiModule.showToast) uiModule.showToast(`Workspace set: ${_basename(_curPath)}`); + closeWorkspaceBrowser(); + }); + const content = _modal.querySelector('.modal-content'); + const header = _modal.querySelector('.modal-header'); + if (content && header) makeWindowDraggable(_modal, { content, header }); + return _modal; +} + +export async function openWorkspaceBrowser() { + const modal = _getModal(); + modal.style.display = 'flex'; + try { + _render(await _load(getWorkspace() || '')); + } catch (e) { + if (uiModule && uiModule.showError) uiModule.showError('Could not browse folders'); + } +} + +export function closeWorkspaceBrowser() { + if (_modal) _modal.style.display = 'none'; +} + +export function initWorkspace() { + // Restore persisted workspace into the pill on load. + syncWorkspaceIndicator(getWorkspace()); + const overflow = document.getElementById('overflow-workspace-btn'); + if (overflow) overflow.addEventListener('click', openWorkspaceBrowser); + const pill = document.getElementById('workspace-indicator-btn'); + if (pill) pill.addEventListener('click', clearWorkspace); +} + +export default { initWorkspace, openWorkspaceBrowser, getWorkspace, setWorkspace, vetAndSetWorkspace, clearWorkspace, syncWorkspaceIndicator, applyMode }; diff --git a/static/login.html b/static/login.html index 90ebb499a..eeece7cc3 100644 --- a/static/login.html +++ b/static/login.html @@ -4,6 +4,9 @@ <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0, interactive-widget=resizes-visual"> <title>Odysseus — Login + + +