mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-06-16 01:35:36 -04:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 68f19a889a | |||
| 422f23fb12 | |||
| 0f966d6b9f | |||
| 7b09491557 | |||
| fafaf089c5 |
@@ -6,6 +6,7 @@ Imports MemoryManager and MemoryVectorStore from the Odysseus codebase.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -23,6 +24,55 @@ _memory_manager = None
|
||||
_memory_vector = None
|
||||
_initialized = False
|
||||
|
||||
_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_MEMORY_OWNER", "ODYSSEUS_MEMORY_OWNER")
|
||||
_OWNER_SCOPE_ERROR = (
|
||||
"Error: Memory MCP owner is not configured for an owner-scoped memory store. "
|
||||
"Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
|
||||
)
|
||||
|
||||
|
||||
def _configured_owner() -> str | None:
|
||||
for key in _OWNER_ENV_KEYS:
|
||||
owner = os.environ.get(key, "").strip()
|
||||
if owner:
|
||||
return owner
|
||||
return None
|
||||
|
||||
|
||||
def _entry_owner(entry: dict) -> str | None:
|
||||
owner = entry.get("owner")
|
||||
if owner is None:
|
||||
return None
|
||||
owner_text = str(owner).strip()
|
||||
return owner_text or None
|
||||
|
||||
|
||||
def _owner_scoped_store(entries: list[dict]) -> bool:
|
||||
return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
|
||||
|
||||
|
||||
def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]:
|
||||
"""Return configured owner, all entries, visible entries, and optional error."""
|
||||
entries = _memory_manager.load_all()
|
||||
owner = _configured_owner()
|
||||
if owner is None and _owner_scoped_store(entries):
|
||||
return None, entries, [], _OWNER_SCOPE_ERROR
|
||||
if owner is None:
|
||||
visible = [
|
||||
entry for entry in entries
|
||||
if isinstance(entry, dict) and _entry_owner(entry) is None
|
||||
]
|
||||
else:
|
||||
visible = [
|
||||
entry for entry in entries
|
||||
if isinstance(entry, dict) and _entry_owner(entry) == owner
|
||||
]
|
||||
return owner, entries, visible, None
|
||||
|
||||
|
||||
def _text_result(text: str) -> list[TextContent]:
|
||||
return [TextContent(type="text", text=text)]
|
||||
|
||||
|
||||
def _ensure_init():
|
||||
"""Lazy-init memory managers on first use."""
|
||||
@@ -75,24 +125,26 @@ async def list_tools() -> list[Tool]:
|
||||
@server.call_tool()
|
||||
async def call_tool(name: str, arguments: dict) -> list[TextContent]:
|
||||
if name != "manage_memory":
|
||||
return [TextContent(type="text", text=f"Unknown tool: {name}")]
|
||||
return _text_result(f"Unknown tool: {name}")
|
||||
|
||||
_ensure_init()
|
||||
if not _memory_manager:
|
||||
return [TextContent(type="text", text="Error: Memory manager not available")]
|
||||
return _text_result("Error: Memory manager not available")
|
||||
|
||||
action = arguments.get("action", "")
|
||||
|
||||
if action == "list":
|
||||
category_filter = arguments.get("category", "")
|
||||
memories = _memory_manager.load()
|
||||
_owner, _all_memories, memories, scope_error = _scope_entries()
|
||||
if scope_error:
|
||||
return _text_result(scope_error)
|
||||
if category_filter:
|
||||
memories = [m for m in memories if m.get("category", "").lower() == category_filter.lower()]
|
||||
if not memories:
|
||||
msg = "No memories found"
|
||||
if category_filter:
|
||||
msg += f" in category '{category_filter}'"
|
||||
return [TextContent(type="text", text=msg + ".")]
|
||||
return _text_result(msg + ".")
|
||||
|
||||
lines = [f"Found {len(memories)} memory entries:\n"]
|
||||
for m in memories:
|
||||
@@ -102,15 +154,17 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
|
||||
if len(text) > 150:
|
||||
text = text[:150] + "..."
|
||||
lines.append(f"- [{cat}] `{mid}` — {text}")
|
||||
return [TextContent(type="text", text="\n".join(lines))]
|
||||
return _text_result("\n".join(lines))
|
||||
|
||||
elif action == "add":
|
||||
text = arguments.get("text", "")
|
||||
category = arguments.get("category", "fact")
|
||||
if not text:
|
||||
return [TextContent(type="text", text="Error: Memory text cannot be empty")]
|
||||
entry = _memory_manager.add_entry(text, source="ai_agent", category=category)
|
||||
memories = _memory_manager.load_all()
|
||||
return _text_result("Error: Memory text cannot be empty")
|
||||
owner, memories, _visible, scope_error = _scope_entries()
|
||||
if scope_error:
|
||||
return _text_result(scope_error)
|
||||
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
|
||||
memories.append(entry)
|
||||
_memory_manager.save(memories)
|
||||
if _memory_vector and _memory_vector.healthy:
|
||||
@@ -118,25 +172,28 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
|
||||
_memory_vector.add(entry["id"], text)
|
||||
except Exception:
|
||||
pass
|
||||
return [TextContent(type="text", text=f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")]
|
||||
return _text_result(f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")
|
||||
|
||||
elif action == "edit":
|
||||
memory_id = arguments.get("memory_id", "")
|
||||
new_text = arguments.get("text", "")
|
||||
if not memory_id or not new_text:
|
||||
return [TextContent(type="text", text="Error: edit needs memory_id and text")]
|
||||
memories = _memory_manager.load_all()
|
||||
found = False
|
||||
return _text_result("Error: edit needs memory_id and text")
|
||||
_owner, memories, visible, scope_error = _scope_entries()
|
||||
if scope_error:
|
||||
return _text_result(scope_error)
|
||||
full_id = None
|
||||
for m in memories:
|
||||
for m in visible:
|
||||
if m.get("id", "").startswith(memory_id):
|
||||
m["text"] = new_text
|
||||
m["timestamp"] = int(time.time())
|
||||
found = True
|
||||
full_id = m["id"]
|
||||
break
|
||||
if not found:
|
||||
return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")]
|
||||
if not full_id:
|
||||
return _text_result(f"Error: Memory '{memory_id}' not found")
|
||||
for m in memories:
|
||||
if m.get("id") == full_id:
|
||||
m["text"] = new_text
|
||||
m["timestamp"] = int(time.time())
|
||||
break
|
||||
_memory_manager.save(memories)
|
||||
if _memory_vector and _memory_vector.healthy and full_id:
|
||||
try:
|
||||
@@ -144,24 +201,26 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
|
||||
_memory_vector.add(full_id, new_text)
|
||||
except Exception:
|
||||
pass
|
||||
return [TextContent(type="text", text=f"Memory updated: {new_text}")]
|
||||
return _text_result(f"Memory updated: {new_text}")
|
||||
|
||||
elif action == "delete":
|
||||
memory_id = arguments.get("memory_id", "")
|
||||
if not memory_id:
|
||||
return [TextContent(type="text", text="Error: delete needs memory_id")]
|
||||
memories = _memory_manager.load_all()
|
||||
return _text_result("Error: delete needs memory_id")
|
||||
_owner, memories, visible, scope_error = _scope_entries()
|
||||
if scope_error:
|
||||
return _text_result(scope_error)
|
||||
full_id = None
|
||||
deleted_text = ""
|
||||
deleted_category = ""
|
||||
for m in memories:
|
||||
for m in visible:
|
||||
if m.get("id", "").startswith(memory_id):
|
||||
full_id = m["id"]
|
||||
deleted_text = m.get("text", "")
|
||||
deleted_category = m.get("category", "")
|
||||
break
|
||||
if not full_id:
|
||||
return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")]
|
||||
return _text_result(f"Error: Memory '{memory_id}' not found")
|
||||
memories = [m for m in memories if m.get("id") != full_id]
|
||||
_memory_manager.save(memories)
|
||||
if _memory_vector and _memory_vector.healthy and full_id:
|
||||
@@ -171,30 +230,32 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
|
||||
pass
|
||||
cat = f"[{deleted_category}] " if deleted_category else ""
|
||||
snippet = deleted_text if len(deleted_text) <= 120 else deleted_text[:117] + "..."
|
||||
return [TextContent(type="text", text=f"Memory deleted: {cat}{snippet} (id: {memory_id})")]
|
||||
return _text_result(f"Memory deleted: {cat}{snippet} (id: {memory_id})")
|
||||
|
||||
elif action == "search":
|
||||
query = arguments.get("text", "")
|
||||
if not query:
|
||||
return [TextContent(type="text", text="Error: search needs text (query)")]
|
||||
memories = _memory_manager.load()
|
||||
return _text_result("Error: search needs text (query)")
|
||||
_owner, _all_memories, memories, scope_error = _scope_entries()
|
||||
if scope_error:
|
||||
return _text_result(scope_error)
|
||||
if hasattr(_memory_manager, 'get_relevant_memories'):
|
||||
results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
|
||||
else:
|
||||
query_lower = query.lower()
|
||||
results = [m for m in memories if query_lower in m.get("text", "").lower()][:20]
|
||||
if not results:
|
||||
return [TextContent(type="text", text=f"No memories found matching '{query}'.")]
|
||||
return _text_result(f"No memories found matching '{query}'.")
|
||||
lines = [f"Found {len(results)} matching memories:\n"]
|
||||
for m in results:
|
||||
cat = m.get("category", "fact")
|
||||
mid = m.get("id", "?")[:8]
|
||||
text = m.get("text", "")
|
||||
lines.append(f"- [{cat}] `{mid}` — {text}")
|
||||
return [TextContent(type="text", text="\n".join(lines))]
|
||||
return _text_result("\n".join(lines))
|
||||
|
||||
else:
|
||||
return [TextContent(type="text", text=f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")]
|
||||
return _text_result(f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")
|
||||
|
||||
|
||||
async def run():
|
||||
|
||||
@@ -15,7 +15,7 @@ from urllib.parse import urljoin, urlparse
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES
|
||||
from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
|
||||
|
||||
from .analytics import RateLimitError, error_logger
|
||||
from .cache import (
|
||||
@@ -369,7 +369,7 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
|
||||
# Fetch
|
||||
try:
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
||||
"User-Agent": WEB_FETCH_USER_AGENT,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
# identity so the streamed size cap in _get_public_url stays honest
|
||||
|
||||
@@ -9,7 +9,7 @@ from urllib.parse import urljoin, urlparse, parse_qs
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from src.constants import SEARXNG_INSTANCE, REQUEST_TIMEOUT
|
||||
from src.constants import SEARXNG_INSTANCE, REQUEST_TIMEOUT, WEB_FETCH_USER_AGENT
|
||||
from .analytics import RateLimitError, error_logger
|
||||
from .query import build_enhanced_query
|
||||
|
||||
@@ -138,7 +138,7 @@ def searxng_search_api(query: str, count: Optional[int] = None, categories: str
|
||||
count = count if count is not None else _get_result_count()
|
||||
instance = _get_search_instance()
|
||||
api_key = ""
|
||||
headers = {"User-Agent": "Mozilla/5.0"}
|
||||
headers = {"User-Agent": WEB_FETCH_USER_AGENT}
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
# News/fresh queries do badly in the 'general' category — it favours
|
||||
@@ -250,7 +250,7 @@ def searxng_search(query, max_results=10):
|
||||
"""Search using SearXNG instance - parsing HTML."""
|
||||
instance = _get_search_instance()
|
||||
api_key = ""
|
||||
req_headers = {"User-Agent": "Mozilla/5.0"}
|
||||
req_headers = {"User-Agent": WEB_FETCH_USER_AGENT}
|
||||
if api_key:
|
||||
req_headers["Authorization"] = f"Bearer {api_key}"
|
||||
try:
|
||||
@@ -389,7 +389,7 @@ def duckduckgo_search(query: str, count: Optional[int] = None, time_filter: Opti
|
||||
response = httpx.get(
|
||||
"https://html.duckduckgo.com/html/",
|
||||
params={"q": query, "kp": _safesearch_for("duckduckgo_html")},
|
||||
headers={"User-Agent": "Mozilla/5.0"},
|
||||
headers={"User-Agent": WEB_FETCH_USER_AGENT},
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
+14
-1
@@ -57,7 +57,13 @@ MEMORY_VECTORS_DIR = os.path.join(DATA_DIR, "memory_vectors")
|
||||
|
||||
# Paths with an intentional dedicated env override, defaulting under DATA_DIR.
|
||||
MAIL_ATTACHMENTS_DIR = os.getenv("ODYSSEUS_MAIL_ATTACHMENTS_DIR", os.path.join(DATA_DIR, "mail-attachments"))
|
||||
FASTEMBED_CACHE_DIR = os.getenv("FASTEMBED_CACHE_PATH", os.path.join(DATA_DIR, "fastembed_cache"))
|
||||
# `or` (not os.getenv's default arg) so a PRESENT-but-EMPTY value falls back to
|
||||
# the default. docker-compose.yml injects `FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-}`,
|
||||
# which sets the var to "" when the host hasn't defined it. os.getenv(name, default)
|
||||
# only returns the default when the var is ABSENT, so the empty string would win →
|
||||
# os.makedirs("") raises [Errno 2] No such file or directory: '' → FastEmbed fails to
|
||||
# init and all vector features (RAG, semantic memory, tool index) silently degrade.
|
||||
FASTEMBED_CACHE_DIR = os.getenv("FASTEMBED_CACHE_PATH") or os.path.join(DATA_DIR, "fastembed_cache")
|
||||
|
||||
# Agent tool output limits (single source of truth — imported by tool_execution.py,
|
||||
# tool_implementations.py, agent_tools.py, and any other module that needs them)
|
||||
@@ -78,6 +84,13 @@ MAX_CONTEXT_MESSAGES = 90
|
||||
REQUEST_TIMEOUT = 20
|
||||
OPENAI_COMPAT_PATH = "/v1/chat/completions"
|
||||
|
||||
# Outbound UA for web_fetch / web_search scraping; common desktop UA so pages serve normal HTML.
|
||||
WEB_FETCH_USER_AGENT = os.environ.get(
|
||||
"WEB_FETCH_USER_AGENT",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36",
|
||||
)
|
||||
|
||||
# Environment variables with defaults
|
||||
DEFAULT_HOST = os.getenv("LLM_HOST", "localhost")
|
||||
LLM_HOSTS = [h.strip() for h in os.getenv("LLM_HOSTS", "").split(",") if h.strip()]
|
||||
|
||||
+24
-5
@@ -236,6 +236,29 @@ def _digest_windows(now):
|
||||
]
|
||||
|
||||
|
||||
def _checkin_calendar_events(db, owner, start, end):
|
||||
"""Calendar events in [start, end] for ONE owner, for the check-in digest.
|
||||
|
||||
Ownership lives on CalendarCal.owner; events inherit it via calendar_id.
|
||||
The digest query had no owner scope, so it pulled EVERY user's events into
|
||||
one user's check-in (a cross-tenant leak of summaries/locations). Scope it
|
||||
by joining CalendarCal, mirroring routes/calendar_routes.list_events.
|
||||
"""
|
||||
from core.database import CalendarEvent as _CE, CalendarCal as _CC
|
||||
return (
|
||||
db.query(_CE)
|
||||
.join(_CC, _CE.calendar_id == _CC.id)
|
||||
.filter(
|
||||
_CC.owner == owner,
|
||||
_CE.dtstart >= start,
|
||||
_CE.dtstart <= end,
|
||||
_CE.status != "cancelled",
|
||||
)
|
||||
.order_by(_CE.dtstart)
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
class TaskScheduler:
|
||||
def __init__(self, session_manager):
|
||||
self._session_manager = session_manager
|
||||
@@ -1127,11 +1150,7 @@ class TaskScheduler:
|
||||
# Strip timezone for naive DB comparison
|
||||
_s = start.replace(tzinfo=None) if start.tzinfo else start
|
||||
_e = end.replace(tzinfo=None) if end.tzinfo else end
|
||||
evs = _db.query(_CE).filter(
|
||||
_CE.dtstart >= _s,
|
||||
_CE.dtstart <= _e,
|
||||
_CE.status != "cancelled",
|
||||
).order_by(_CE.dtstart).all()
|
||||
evs = _checkin_calendar_events(_db, task.owner, _s, _e)
|
||||
if not evs:
|
||||
continue
|
||||
# Group by importance for richer output
|
||||
|
||||
@@ -1,326 +0,0 @@
|
||||
# Oversized Test File Split Plan
|
||||
|
||||
## Purpose
|
||||
|
||||
This document plans future oversized test-file splits using current repo data.
|
||||
It does not move files, rewrite assertions, extract helpers, or change CI.
|
||||
|
||||
## Roadmap context
|
||||
|
||||
- Issue: #3983
|
||||
- Parent tracker: #2523
|
||||
- Follows #3973 / #3982, the report-only order-sensitivity diagnostics slice.
|
||||
|
||||
## Methodology
|
||||
|
||||
Metrics were generated from the current test tree using:
|
||||
|
||||
- physical line counts for every recursive `test_*.py` file under `tests/`;
|
||||
- AST counts for `test_*` functions and `Test*` classes;
|
||||
- one `pytest --collect-only -q tests` run to count collected items per file;
|
||||
- current taxonomy classification from `tests._taxonomy.classify_test_path`; and
|
||||
- static setup-signal scans for route/API, DB/session, import-state, security, filesystem, subprocess/script, async/threading, and UI/static indicators.
|
||||
|
||||
Static signals are not proof of risk. They are review prompts.
|
||||
Future split PRs must still inspect each file manually before editing.
|
||||
|
||||
## Current summary
|
||||
|
||||
- test files scanned: 583
|
||||
- collected pytest items counted: 3586
|
||||
- large-file threshold: 300 lines
|
||||
- large-collected threshold: 20 collected items
|
||||
|
||||
Area distribution:
|
||||
|
||||
| Value | Files |
|
||||
|---|---:|
|
||||
| cli | 28 |
|
||||
| helpers | 1 |
|
||||
| js | 39 |
|
||||
| routes | 23 |
|
||||
| security | 77 |
|
||||
| services | 144 |
|
||||
| uncategorized | 234 |
|
||||
| unit | 37 |
|
||||
|
||||
Sub-area distribution:
|
||||
|
||||
| Value | Files |
|
||||
|---|---:|
|
||||
| api | 6 |
|
||||
| atomic | 3 |
|
||||
| auth | 9 |
|
||||
| calendar | 10 |
|
||||
| cli | 28 |
|
||||
| confinement | 7 |
|
||||
| cookbook | 13 |
|
||||
| document | 11 |
|
||||
| email | 12 |
|
||||
| embedding | 3 |
|
||||
| gallery | 5 |
|
||||
| history | 3 |
|
||||
| js | 39 |
|
||||
| llm | 16 |
|
||||
| mcp | 8 |
|
||||
| memory | 15 |
|
||||
| nondict | 7 |
|
||||
| nonstring | 22 |
|
||||
| owner | 14 |
|
||||
| owner_scope | 23 |
|
||||
| parse | 4 |
|
||||
| provider | 6 |
|
||||
| research | 16 |
|
||||
| route | 6 |
|
||||
| routes | 9 |
|
||||
| scheduler | 3 |
|
||||
| scope | 5 |
|
||||
| security | 9 |
|
||||
| session | 16 |
|
||||
| ssrf | 3 |
|
||||
| webhook | 3 |
|
||||
| xss | 5 |
|
||||
|
||||
Values below 2 files: 244 values covering 244 files.
|
||||
|
||||
## Top files by collected pytest items
|
||||
|
||||
| File | Lines | Collected tests | Test defs | Test classes | Area | Sub-area | Signals |
|
||||
|---|---:|---:|---:|---:|---|---|---|
|
||||
| `tests/test_model_routes.py` | 1778 | 139 | 116 | 10 | routes | routes | route/api, db/session, import-state, async/threading |
|
||||
| `tests/test_security_regressions.py` | 1224 | 92 | 68 | 0 | security | security | route/api, db/session, import-state, security, filesystem, async/threading, ui/static |
|
||||
| `tests/test_provider_classification.py` | 188 | 67 | 21 | 4 | services | provider | - |
|
||||
| `tests/test_cookbook_helpers.py` | 912 | 65 | 65 | 0 | services | cookbook | route/api, filesystem, subprocess/script, async/threading, ui/static |
|
||||
| `tests/test_shell_routes.py` | 481 | 63 | 48 | 8 | routes | routes | route/api, import-state, filesystem |
|
||||
| `tests/test_pr_blocker_audit.py` | 964 | 58 | 58 | 0 | uncategorized | pr_blocker_audit | import-state, security, filesystem |
|
||||
| `tests/test_provider_endpoints.py` | 241 | 58 | 18 | 1 | services | provider | subprocess/script |
|
||||
| `tests/test_agent_loop.py` | 469 | 52 | 52 | 5 | uncategorized | agent_loop | db/session, import-state |
|
||||
| `tests/test_service_health.py` | 472 | 47 | 42 | 0 | uncategorized | service_health | async/threading |
|
||||
| `tests/test_run_focus.py` | 399 | 47 | 44 | 0 | uncategorized | run_focus | security, filesystem, subprocess/script, ui/static |
|
||||
| `tests/test_llm_core_temperature.py` | 196 | 41 | 17 | 0 | services | llm | - |
|
||||
| `tests/test_endpoint_probing.py` | 411 | 34 | 30 | 6 | uncategorized | endpoint_probing | route/api, db/session, import-state |
|
||||
| `tests/test_llm_core_anthropic_temp_omit.py` | 94 | 32 | 6 | 0 | services | llm | db/session |
|
||||
| `tests/test_chat_helpers.py` | 264 | 31 | 18 | 0 | uncategorized | chat_helpers | route/api |
|
||||
| `tests/test_provider_detection.py` | 148 | 31 | 31 | 5 | services | provider | - |
|
||||
| `tests/test_model_context.py` | 251 | 30 | 30 | 4 | uncategorized | model_context | db/session, import-state |
|
||||
| `tests/test_endpoint_resolver.py` | 148 | 30 | 30 | 6 | uncategorized | endpoint_resolver | - |
|
||||
| `tests/test_embedding_lanes.py` | 1104 | 29 | 29 | 0 | services | embedding | filesystem |
|
||||
| `tests/test_upload_limits_centralized.py` | 110 | 29 | 5 | 0 | uncategorized | upload_limits_centralized | import-state, filesystem |
|
||||
| `tests/test_email_oauth.py` | 580 | 28 | 25 | 0 | services | email | route/api, db/session, security, async/threading |
|
||||
| `tests/test_review_regressions.py` | 930 | 26 | 26 | 0 | uncategorized | review_regressions | route/api, db/session, import-state, filesystem, async/threading |
|
||||
| `tests/test_rename_user_owner_sync.py` | 686 | 26 | 26 | 0 | security | owner | route/api, db/session, import-state, filesystem, async/threading |
|
||||
| `tests/test_helpers_import_state.py` | 426 | 26 | 26 | 0 | helpers | helpers | route/api, db/session, import-state |
|
||||
| `tests/test_taxonomy.py` | 145 | 26 | 16 | 0 | uncategorized | taxonomy | security, ui/static |
|
||||
| `tests/test_tool_path_confinement.py` | 282 | 24 | 24 | 0 | security | confinement | import-state, filesystem, async/threading |
|
||||
| `tests/test_copilot.py` | 170 | 23 | 16 | 0 | uncategorized | copilot | - |
|
||||
| `tests/test_research_utils.py` | 97 | 23 | 23 | 2 | services | research | - |
|
||||
| `tests/test_api_chat_security.py` | 401 | 22 | 8 | 0 | security | security | route/api, db/session, import-state, filesystem, async/threading |
|
||||
| `tests/test_tool_support_heuristic.py` | 166 | 22 | 22 | 3 | uncategorized | tool_support_heuristic | - |
|
||||
| `tests/test_platform_compat.py` | 318 | 21 | 21 | 0 | uncategorized | platform_compat | import-state, filesystem, subprocess/script |
|
||||
|
||||
## Top files by physical line count
|
||||
|
||||
| File | Lines | Collected tests | Test defs | Test classes | Area | Sub-area | Signals |
|
||||
|---|---:|---:|---:|---:|---|---|---|
|
||||
| `tests/test_model_routes.py` | 1778 | 139 | 116 | 10 | routes | routes | route/api, db/session, import-state, async/threading |
|
||||
| `tests/test_security_regressions.py` | 1224 | 92 | 68 | 0 | security | security | route/api, db/session, import-state, security, filesystem, async/threading, ui/static |
|
||||
| `tests/test_embedding_lanes.py` | 1104 | 29 | 29 | 0 | services | embedding | filesystem |
|
||||
| `tests/test_pr_blocker_audit.py` | 964 | 58 | 58 | 0 | uncategorized | pr_blocker_audit | import-state, security, filesystem |
|
||||
| `tests/test_review_regressions.py` | 930 | 26 | 26 | 0 | uncategorized | review_regressions | route/api, db/session, import-state, filesystem, async/threading |
|
||||
| `tests/test_cookbook_helpers.py` | 912 | 65 | 65 | 0 | services | cookbook | route/api, filesystem, subprocess/script, async/threading, ui/static |
|
||||
| `tests/test_rename_user_owner_sync.py` | 686 | 26 | 26 | 0 | security | owner | route/api, db/session, import-state, filesystem, async/threading |
|
||||
| `tests/test_email_oauth.py` | 580 | 28 | 25 | 0 | services | email | route/api, db/session, security, async/threading |
|
||||
| `tests/test_api_token_routes.py` | 578 | 17 | 17 | 0 | routes | api_routes | route/api, db/session, import-state, async/threading |
|
||||
| `tests/test_shell_routes.py` | 481 | 63 | 48 | 8 | routes | routes | route/api, import-state, filesystem |
|
||||
| `tests/test_email_owner_scope.py` | 474 | 9 | 9 | 0 | security | owner_scope | route/api, db/session, filesystem, async/threading |
|
||||
| `tests/test_service_health.py` | 472 | 47 | 42 | 0 | uncategorized | service_health | async/threading |
|
||||
| `tests/test_agent_loop.py` | 469 | 52 | 52 | 5 | uncategorized | agent_loop | db/session, import-state |
|
||||
| `tests/test_kv_cache_invalidation_2927.py` | 463 | 8 | 8 | 0 | uncategorized | kv_cache_invalidation_2927 | route/api, db/session, import-state, async/threading |
|
||||
| `tests/test_helpers_import_state.py` | 426 | 26 | 26 | 0 | helpers | helpers | route/api, db/session, import-state |
|
||||
| `tests/test_endpoint_owner_scope_followup.py` | 414 | 11 | 11 | 0 | security | owner_scope | route/api, db/session, filesystem |
|
||||
| `tests/test_endpoint_probing.py` | 411 | 34 | 30 | 6 | uncategorized | endpoint_probing | route/api, db/session, import-state |
|
||||
| `tests/test_imap_leak_fixes.py` | 404 | 15 | 15 | 0 | uncategorized | imap_leak_fixes | route/api, db/session, security, filesystem |
|
||||
| `tests/test_companion_readonly.py` | 402 | 17 | 17 | 0 | uncategorized | companion_readonly | db/session, import-state |
|
||||
| `tests/test_api_chat_security.py` | 401 | 22 | 8 | 0 | security | security | route/api, db/session, import-state, filesystem, async/threading |
|
||||
| `tests/test_upload_handler_atomicity.py` | 401 | 9 | 9 | 0 | uncategorized | upload_handler_atomicity | filesystem, async/threading |
|
||||
| `tests/test_run_focus.py` | 399 | 47 | 44 | 0 | uncategorized | run_focus | security, filesystem, subprocess/script, ui/static |
|
||||
| `tests/test_auth_regressions.py` | 375 | 15 | 15 | 0 | security | auth | route/api, db/session, import-state, async/threading |
|
||||
| `tests/test_calendar_owner_scope.py` | 345 | 7 | 7 | 0 | security | owner_scope | route/api, db/session, import-state, filesystem, async/threading, ui/static |
|
||||
| `tests/test_null_owner_gates.py` | 342 | 20 | 20 | 0 | security | owner | route/api, db/session, import-state |
|
||||
| `tests/test_agent_migration_manifest.py` | 340 | 15 | 15 | 0 | uncategorized | agent_migration_manifest | import-state, filesystem |
|
||||
| `tests/test_calendar_recurrence.py` | 338 | 19 | 19 | 0 | services | calendar | - |
|
||||
| `tests/test_tool_policy.py` | 330 | 13 | 13 | 0 | uncategorized | tool_policy | import-state, async/threading |
|
||||
| `tests/test_workspace_confine.py` | 328 | 18 | 18 | 0 | uncategorized | workspace_confine | route/api, filesystem, subprocess/script, async/threading |
|
||||
| `tests/test_diffusion_server_security.py` | 325 | 14 | 14 | 0 | security | security | route/api, import-state, security, filesystem, async/threading, ui/static |
|
||||
|
||||
## Split planning candidates
|
||||
|
||||
This section is generated from metrics, not from manual judgement.
|
||||
Files are included when they meet at least one threshold:
|
||||
|
||||
- at least 300 physical lines; or
|
||||
- at least 20 collected pytest items.
|
||||
|
||||
These are planning candidates only. A later split PR still needs a focused manual review of each file before moving tests.
|
||||
|
||||
| File | Why included | Setup/risk signals | Suggested handling |
|
||||
|---|---|---|---|
|
||||
| `tests/test_model_routes.py` | 1778 lines, 139 collected tests | route/api, db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_security_regressions.py` | 1224 lines, 92 collected tests | route/api, db/session, import-state, security, filesystem, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_provider_classification.py` | 67 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_cookbook_helpers.py` | 912 lines, 65 collected tests | route/api, filesystem, subprocess/script, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_shell_routes.py` | 481 lines, 63 collected tests | route/api, import-state, filesystem | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_pr_blocker_audit.py` | 964 lines, 58 collected tests | import-state, security, filesystem | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_provider_endpoints.py` | 58 collected tests | subprocess/script | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_agent_loop.py` | 469 lines, 52 collected tests | db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_service_health.py` | 472 lines, 47 collected tests | async/threading | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_run_focus.py` | 399 lines, 47 collected tests | security, filesystem, subprocess/script, ui/static | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_llm_core_temperature.py` | 41 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_endpoint_probing.py` | 411 lines, 34 collected tests | route/api, db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_llm_core_anthropic_temp_omit.py` | 32 collected tests | db/session | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_chat_helpers.py` | 31 collected tests | route/api | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_provider_detection.py` | 31 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_model_context.py` | 30 collected tests | db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_endpoint_resolver.py` | 30 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_embedding_lanes.py` | 1104 lines, 29 collected tests | filesystem | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_upload_limits_centralized.py` | 29 collected tests | import-state, filesystem | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_email_oauth.py` | 580 lines, 28 collected tests | route/api, db/session, security, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_review_regressions.py` | 930 lines, 26 collected tests | route/api, db/session, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_rename_user_owner_sync.py` | 686 lines, 26 collected tests | route/api, db/session, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_helpers_import_state.py` | 426 lines, 26 collected tests | route/api, db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_taxonomy.py` | 26 collected tests | security, ui/static | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_tool_path_confinement.py` | 24 collected tests | import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_copilot.py` | 23 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_research_utils.py` | 23 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_api_chat_security.py` | 401 lines, 22 collected tests | route/api, db/session, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_tool_support_heuristic.py` | 22 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_platform_compat.py` | 318 lines, 21 collected tests | import-state, filesystem, subprocess/script | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_context_compactor.py` | 21 collected tests | db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_prompt_security.py` | 21 collected tests | No obvious setup signals from static scan. | Good first manual-review candidate if test themes are cohesive. |
|
||||
| `tests/test_null_owner_gates.py` | 342 lines, 20 collected tests | route/api, db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_youtube_handler_consolidation.py` | 20 collected tests | route/api, import-state | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_calendar_recurrence.py` | 338 lines | No obvious setup signals from static scan. | Plan split boundaries before editing. |
|
||||
| `tests/test_workspace_confine.py` | 328 lines | route/api, filesystem, subprocess/script, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_api_token_routes.py` | 578 lines | route/api, db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_companion_readonly.py` | 402 lines | db/session, import-state | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_set_admin.py` | 317 lines | route/api, import-state, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_imap_leak_fixes.py` | 404 lines | route/api, db/session, security, filesystem | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_auth_regressions.py` | 375 lines | route/api, db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_agent_migration_manifest.py` | 340 lines | import-state, filesystem | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_diffusion_server_security.py` | 325 lines | route/api, import-state, security, filesystem, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_tool_policy.py` | 330 lines | import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_endpoint_owner_scope_followup.py` | 414 lines | route/api, db/session, filesystem | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_upload_routes_owner_scope.py` | 315 lines | route/api, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_email_owner_scope.py` | 474 lines | route/api, db/session, filesystem, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_upload_handler_atomicity.py` | 401 lines | filesystem, async/threading | Plan split boundaries before editing. |
|
||||
| `tests/test_kv_cache_invalidation_2927.py` | 463 lines | route/api, db/session, import-state, async/threading | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_calendar_owner_scope.py` | 345 lines | route/api, db/session, import-state, filesystem, async/threading, ui/static | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
| `tests/test_skills_manager_owner_isolation.py` | 306 lines | import-state, filesystem | Defer mechanical split until setup/risk boundaries are mapped. |
|
||||
|
||||
## Taxonomy coverage gaps among split candidates
|
||||
|
||||
`uncategorized` is a current taxonomy area, not a builder failure.
|
||||
This plan does not reclassify tests because taxonomy changes should be reviewed separately from oversized-file split planning.
|
||||
|
||||
Before using any of these files as a split target, first decide whether the taxonomy should be refined in a separate focused issue/PR.
|
||||
|
||||
| File | Lines | Collected tests | Sub-area | Signals | Suggested follow-up |
|
||||
|---|---:|---:|---|---|---|
|
||||
| `tests/test_pr_blocker_audit.py` | 964 | 58 | pr_blocker_audit | import-state, security, filesystem | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_agent_loop.py` | 469 | 52 | agent_loop | db/session, import-state | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_service_health.py` | 472 | 47 | service_health | async/threading | Review taxonomy mapping before using as a split target. |
|
||||
| `tests/test_run_focus.py` | 399 | 47 | run_focus | security, filesystem, subprocess/script, ui/static | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_endpoint_probing.py` | 411 | 34 | endpoint_probing | route/api, db/session, import-state | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_chat_helpers.py` | 264 | 31 | chat_helpers | route/api | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_model_context.py` | 251 | 30 | model_context | db/session, import-state | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_endpoint_resolver.py` | 148 | 30 | endpoint_resolver | - | Review taxonomy mapping before using as a split target. |
|
||||
| `tests/test_upload_limits_centralized.py` | 110 | 29 | upload_limits_centralized | import-state, filesystem | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_review_regressions.py` | 930 | 26 | review_regressions | route/api, db/session, import-state, filesystem, async/threading | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_taxonomy.py` | 145 | 26 | taxonomy | security, ui/static | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_copilot.py` | 170 | 23 | copilot | - | Review taxonomy mapping before using as a split target. |
|
||||
| `tests/test_tool_support_heuristic.py` | 166 | 22 | tool_support_heuristic | - | Review taxonomy mapping before using as a split target. |
|
||||
| `tests/test_platform_compat.py` | 318 | 21 | platform_compat | import-state, filesystem, subprocess/script | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_context_compactor.py` | 233 | 21 | context_compactor | db/session, import-state, async/threading | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_youtube_handler_consolidation.py` | 104 | 20 | youtube_handler_consolidation | route/api, import-state | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_workspace_confine.py` | 328 | 18 | workspace_confine | route/api, filesystem, subprocess/script, async/threading | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_companion_readonly.py` | 402 | 17 | companion_readonly | db/session, import-state | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_set_admin.py` | 317 | 17 | set_admin | route/api, import-state, filesystem, async/threading | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_imap_leak_fixes.py` | 404 | 15 | imap_leak_fixes | route/api, db/session, security, filesystem | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_agent_migration_manifest.py` | 340 | 15 | agent_migration_manifest | import-state, filesystem | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_tool_policy.py` | 330 | 13 | tool_policy | import-state, async/threading | Review taxonomy and setup/risk boundaries before any split. |
|
||||
| `tests/test_upload_handler_atomicity.py` | 401 | 9 | upload_handler_atomicity | filesystem, async/threading | Review taxonomy mapping before using as a split target. |
|
||||
| `tests/test_kv_cache_invalidation_2927.py` | 463 | 8 | kv_cache_invalidation_2927 | route/api, db/session, import-state, async/threading | Review taxonomy and setup/risk boundaries before any split. |
|
||||
|
||||
## Suggested first manual-review candidates
|
||||
|
||||
These are not automatic split approvals. They are categorized candidates with enough size/collection value and no route/API, DB/session, import-state, or security signal from the static scan.
|
||||
|
||||
Files still in the `uncategorized` taxonomy area are listed separately below so taxonomy review does not get mixed into the first split decision.
|
||||
|
||||
| File | Lines | Collected tests | Area | Sub-area | Signals | Why this is a candidate |
|
||||
|---|---:|---:|---|---|---|---|
|
||||
| `tests/test_provider_classification.py` | 188 | 67 | services | provider | - | 67 collected tests |
|
||||
| `tests/test_provider_endpoints.py` | 241 | 58 | services | provider | subprocess/script | 58 collected tests |
|
||||
| `tests/test_llm_core_temperature.py` | 196 | 41 | services | llm | - | 41 collected tests |
|
||||
| `tests/test_provider_detection.py` | 148 | 31 | services | provider | - | 31 collected tests |
|
||||
| `tests/test_embedding_lanes.py` | 1104 | 29 | services | embedding | filesystem | 1104 lines, 29 collected tests |
|
||||
| `tests/test_research_utils.py` | 97 | 23 | services | research | - | 23 collected tests |
|
||||
| `tests/test_prompt_security.py` | 203 | 21 | security | security | - | 21 collected tests |
|
||||
| `tests/test_calendar_recurrence.py` | 338 | 19 | services | calendar | - | 338 lines |
|
||||
|
||||
## High-risk candidates to defer first
|
||||
|
||||
These files may still be split later, but not as the first implementation slice without a separate manual boundary review.
|
||||
|
||||
| File | Lines | Collected tests | High-risk signals |
|
||||
|---|---:|---:|---|
|
||||
| `tests/test_model_routes.py` | 1778 | 139 | db/session, import-state, route/api |
|
||||
| `tests/test_security_regressions.py` | 1224 | 92 | db/session, import-state, route/api, security |
|
||||
| `tests/test_cookbook_helpers.py` | 912 | 65 | route/api |
|
||||
| `tests/test_shell_routes.py` | 481 | 63 | import-state, route/api |
|
||||
| `tests/test_pr_blocker_audit.py` | 964 | 58 | import-state, security |
|
||||
| `tests/test_agent_loop.py` | 469 | 52 | db/session, import-state |
|
||||
| `tests/test_run_focus.py` | 399 | 47 | security |
|
||||
| `tests/test_endpoint_probing.py` | 411 | 34 | db/session, import-state, route/api |
|
||||
| `tests/test_llm_core_anthropic_temp_omit.py` | 94 | 32 | db/session |
|
||||
| `tests/test_chat_helpers.py` | 264 | 31 | route/api |
|
||||
| `tests/test_model_context.py` | 251 | 30 | db/session, import-state |
|
||||
| `tests/test_upload_limits_centralized.py` | 110 | 29 | import-state |
|
||||
| `tests/test_email_oauth.py` | 580 | 28 | db/session, route/api, security |
|
||||
| `tests/test_review_regressions.py` | 930 | 26 | db/session, import-state, route/api |
|
||||
| `tests/test_rename_user_owner_sync.py` | 686 | 26 | db/session, import-state, route/api |
|
||||
|
||||
## Rules for future split PRs
|
||||
|
||||
- One file or one coherent file-family per PR.
|
||||
- No assertion rewrites mixed with file moves.
|
||||
- No helper extraction mixed with file moves.
|
||||
- No production code changes.
|
||||
- No CI workflow changes.
|
||||
- Preserve existing markers and taxonomy unless the split issue explicitly says otherwise.
|
||||
- Validate the original file's collected tests before and after the split.
|
||||
- Validate any neighboring taxonomy/focused-runner behavior if paths change.
|
||||
- Treat files with route/API, DB/session, import-state, or security signals as higher-risk until manually reviewed.
|
||||
|
||||
## Suggested next step
|
||||
|
||||
Use this plan to choose the first actual oversized-file split issue.
|
||||
The first split should prefer a file with high review value and low setup risk.
|
||||
Do not start a split PR from this planning issue alone if the file's boundaries are still ambiguous.
|
||||
|
||||
## Reproduction command
|
||||
|
||||
This document was generated with:
|
||||
|
||||
```bash
|
||||
.venv/bin/python tests/tools/build_oversized_test_split_plan.py
|
||||
```
|
||||
|
||||
## Freshness check
|
||||
|
||||
After editing the builder or rebasing the branch, regenerate the plan and confirm no unexpected plan drift:
|
||||
|
||||
```bash
|
||||
.venv/bin/python tests/tools/build_oversized_test_split_plan.py
|
||||
git diff --exit-code -- tests/OVERSIZED_TEST_SPLIT_PLAN.md
|
||||
```
|
||||
@@ -219,6 +219,9 @@ class _WebhookManager:
|
||||
async def fire(self, event, payload):
|
||||
return None
|
||||
|
||||
def fire_and_forget(self, event, payload):
|
||||
return None
|
||||
|
||||
|
||||
def _install_sync_chat_stubs(monkeypatch):
|
||||
# FastAPI checks for python_multipart at import time when Form is used;
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Check-in calendar digest must be scoped to the task owner.
|
||||
|
||||
The digest query selected CalendarEvent with no owner scope, so a scheduled
|
||||
check-in for one user pulled EVERY user's calendar events (summaries,
|
||||
locations) into their digest — a cross-tenant leak. Ownership lives on
|
||||
CalendarCal.owner; the query must join it, like routes/calendar_routes.
|
||||
"""
|
||||
import tempfile
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
import core.database as cdb
|
||||
from core.database import CalendarEvent, CalendarCal
|
||||
from src.task_scheduler import _checkin_calendar_events
|
||||
|
||||
_TMPDB = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
|
||||
_ENGINE = create_engine(f"sqlite:///{_TMPDB.name}", connect_args={"check_same_thread": False}, poolclass=NullPool)
|
||||
cdb.Base.metadata.create_all(_ENGINE)
|
||||
_TS = sessionmaker(bind=_ENGINE, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
def _seed():
|
||||
db = _TS()
|
||||
try:
|
||||
db.query(CalendarEvent).delete(); db.query(CalendarCal).delete()
|
||||
db.add(CalendarCal(id="calA", owner="alice", name="A"))
|
||||
db.add(CalendarCal(id="calB", owner="bob", name="B"))
|
||||
db.add(CalendarEvent(uid="a1", calendar_id="calA", summary="Alice mtg",
|
||||
dtstart=datetime(2026, 6, 10, 9, 0),
|
||||
dtend=datetime(2026, 6, 10, 10, 0), status="confirmed"))
|
||||
db.add(CalendarEvent(uid="b1", calendar_id="calB", summary="Bob secret",
|
||||
dtstart=datetime(2026, 6, 10, 10, 0),
|
||||
dtend=datetime(2026, 6, 10, 11, 0), status="confirmed"))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_digest_only_returns_owner_events():
|
||||
_seed()
|
||||
db = _TS()
|
||||
try:
|
||||
s, e = datetime(2026, 6, 1), datetime(2026, 6, 30)
|
||||
alice = _checkin_calendar_events(db, "alice", s, e)
|
||||
assert [ev.summary for ev in alice] == ["Alice mtg"] # not Bob's
|
||||
bob = _checkin_calendar_events(db, "bob", s, e)
|
||||
assert [ev.summary for ev in bob] == ["Bob secret"]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_cancelled_excluded_and_window_respected():
|
||||
_seed()
|
||||
db = _TS()
|
||||
try:
|
||||
db2 = _TS()
|
||||
db2.add(CalendarEvent(uid="a2", calendar_id="calA", summary="cancelled",
|
||||
dtstart=datetime(2026, 6, 11),
|
||||
dtend=datetime(2026, 6, 11, 1, 0), status="cancelled"))
|
||||
db2.commit(); db2.close()
|
||||
s, e = datetime(2026, 6, 1), datetime(2026, 6, 30)
|
||||
out = _checkin_calendar_events(db, "alice", s, e)
|
||||
assert "cancelled" not in [ev.summary for ev in out]
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Regression: FASTEMBED_CACHE_DIR must tolerate a PRESENT-but-EMPTY
|
||||
FASTEMBED_CACHE_PATH.
|
||||
|
||||
docker-compose.yml injects ``FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-}``,
|
||||
which sets the variable to ``""`` when the host has not defined it. The old
|
||||
``os.getenv("FASTEMBED_CACHE_PATH", default)`` only used the default when the
|
||||
variable was ABSENT, so an empty value made ``FASTEMBED_CACHE_DIR == ""`` →
|
||||
``os.makedirs("")`` raised ``[Errno 2] No such file or directory: ''`` →
|
||||
FastEmbed failed to initialise and every vector feature (RAG, semantic memory,
|
||||
tool index) silently degraded on the default Docker stack.
|
||||
|
||||
These tests pin the fix: empty is treated like absent → use the DATA_DIR
|
||||
default, while an explicit non-empty override is still honoured.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import os
|
||||
|
||||
import src.constants as constants
|
||||
|
||||
|
||||
def _reload_with(monkeypatch, value):
|
||||
"""Reload src.constants with FASTEMBED_CACHE_PATH set to ``value`` (or
|
||||
removed when ``value`` is None) and return the reloaded module."""
|
||||
if value is None:
|
||||
monkeypatch.delenv("FASTEMBED_CACHE_PATH", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("FASTEMBED_CACHE_PATH", value)
|
||||
return importlib.reload(constants)
|
||||
|
||||
|
||||
def _restore(monkeypatch):
|
||||
"""Return the module to its env-default state so reloading it here does
|
||||
not leak a test-specific FASTEMBED_CACHE_DIR into other tests."""
|
||||
monkeypatch.delenv("FASTEMBED_CACHE_PATH", raising=False)
|
||||
importlib.reload(constants)
|
||||
|
||||
|
||||
def test_empty_fastembed_cache_path_falls_back_to_default(monkeypatch):
|
||||
"""The bug: an empty FASTEMBED_CACHE_PATH (exactly what Docker injects)
|
||||
must fall back to the DATA_DIR default, never the empty string."""
|
||||
try:
|
||||
mod = _reload_with(monkeypatch, "")
|
||||
assert mod.FASTEMBED_CACHE_DIR, "empty env must not yield an empty path"
|
||||
assert mod.FASTEMBED_CACHE_DIR == os.path.join(mod.DATA_DIR, "fastembed_cache")
|
||||
finally:
|
||||
_restore(monkeypatch)
|
||||
|
||||
|
||||
def test_unset_fastembed_cache_path_uses_default(monkeypatch):
|
||||
"""Sanity: an absent variable also resolves to the default."""
|
||||
try:
|
||||
mod = _reload_with(monkeypatch, None)
|
||||
assert mod.FASTEMBED_CACHE_DIR == os.path.join(mod.DATA_DIR, "fastembed_cache")
|
||||
finally:
|
||||
_restore(monkeypatch)
|
||||
|
||||
|
||||
def test_explicit_fastembed_cache_path_is_respected(monkeypatch):
|
||||
"""A real explicit override must still win — the fix only changes the
|
||||
empty-value handling, not the documented FASTEMBED_CACHE_PATH override."""
|
||||
custom = os.path.join("custom", "fastembed-cache")
|
||||
try:
|
||||
mod = _reload_with(monkeypatch, custom)
|
||||
assert mod.FASTEMBED_CACHE_DIR == custom
|
||||
finally:
|
||||
_restore(monkeypatch)
|
||||
@@ -0,0 +1,150 @@
|
||||
import asyncio
|
||||
|
||||
import mcp_servers.memory_server as memory_server
|
||||
from src.memory import MemoryManager
|
||||
|
||||
|
||||
class FakeVector:
|
||||
healthy = True
|
||||
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
self.removed = []
|
||||
|
||||
def add(self, memory_id, text):
|
||||
self.added.append((memory_id, text))
|
||||
|
||||
def remove(self, memory_id):
|
||||
self.removed.append(memory_id)
|
||||
|
||||
|
||||
def _tool_text(arguments):
|
||||
result = asyncio.run(memory_server.call_tool("manage_memory", arguments))
|
||||
return result[0].text
|
||||
|
||||
|
||||
def _entry(manager, text, owner=None, memory_id=None, category="fact"):
|
||||
entry = manager.add_entry(text, owner=owner, category=category)
|
||||
if memory_id:
|
||||
entry["id"] = memory_id
|
||||
return entry
|
||||
|
||||
|
||||
def _configure_server(monkeypatch, manager, vector=None):
|
||||
monkeypatch.setattr(memory_server, "_memory_manager", manager)
|
||||
monkeypatch.setattr(memory_server, "_memory_vector", vector)
|
||||
monkeypatch.setattr(memory_server, "_initialized", True)
|
||||
for key in memory_server._OWNER_ENV_KEYS:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
|
||||
def test_mcp_memory_uses_configured_owner_for_all_operations(monkeypatch, tmp_path):
|
||||
manager = MemoryManager(str(tmp_path))
|
||||
vector = FakeVector()
|
||||
alice = _entry(
|
||||
manager,
|
||||
"Alice likes green tea",
|
||||
owner="alice",
|
||||
memory_id="aaaaaaaa-0000-0000-0000-000000000000",
|
||||
)
|
||||
bob = _entry(
|
||||
manager,
|
||||
"Bob likes espresso",
|
||||
owner="bob",
|
||||
memory_id="bbbbbbbb-0000-0000-0000-000000000000",
|
||||
)
|
||||
manager.save([alice, bob])
|
||||
_configure_server(monkeypatch, manager, vector)
|
||||
monkeypatch.setenv("ODYSSEUS_MCP_MEMORY_OWNER", "alice")
|
||||
|
||||
list_text = _tool_text({"action": "list"})
|
||||
assert "Alice likes green tea" in list_text
|
||||
assert "Bob likes espresso" not in list_text
|
||||
|
||||
search_text = _tool_text({"action": "search", "text": "likes"})
|
||||
assert "Alice likes green tea" in search_text
|
||||
assert "Bob likes espresso" not in search_text
|
||||
|
||||
add_text = _tool_text({
|
||||
"action": "add",
|
||||
"text": "Alice prefers concise notes",
|
||||
"category": "preference",
|
||||
})
|
||||
assert "Memory added" in add_text
|
||||
added = next(
|
||||
entry for entry in manager.load_all()
|
||||
if entry["text"] == "Alice prefers concise notes"
|
||||
)
|
||||
assert added["owner"] == "alice"
|
||||
assert vector.added == [(added["id"], "Alice prefers concise notes")]
|
||||
|
||||
edit_text = _tool_text({
|
||||
"action": "edit",
|
||||
"memory_id": bob["id"][:8],
|
||||
"text": "Bob changed",
|
||||
})
|
||||
assert edit_text == "Error: Memory 'bbbbbbbb' not found"
|
||||
bob_after_edit = next(
|
||||
entry for entry in manager.load_all()
|
||||
if entry["id"] == bob["id"]
|
||||
)
|
||||
assert bob_after_edit["text"] == "Bob likes espresso"
|
||||
|
||||
delete_text = _tool_text({"action": "delete", "memory_id": bob["id"][:8]})
|
||||
assert delete_text == "Error: Memory 'bbbbbbbb' not found"
|
||||
assert any(entry["id"] == bob["id"] for entry in manager.load_all())
|
||||
|
||||
|
||||
def test_mcp_memory_fails_closed_without_owner_for_owner_scoped_store(monkeypatch, tmp_path):
|
||||
manager = MemoryManager(str(tmp_path))
|
||||
alice = _entry(manager, "Alice private memory", owner="alice", memory_id="aaaaaaaa-0000")
|
||||
bob = _entry(manager, "Bob private memory", owner="bob", memory_id="bbbbbbbb-0000")
|
||||
manager.save([alice, bob])
|
||||
_configure_server(monkeypatch, manager, FakeVector())
|
||||
before = manager.load_all()
|
||||
|
||||
actions = [
|
||||
{"action": "list"},
|
||||
{"action": "search", "text": "private"},
|
||||
{"action": "add", "text": "new ownerless memory"},
|
||||
{"action": "edit", "memory_id": alice["id"][:8], "text": "changed"},
|
||||
{"action": "delete", "memory_id": alice["id"][:8]},
|
||||
]
|
||||
|
||||
for arguments in actions:
|
||||
assert _tool_text(arguments).startswith("Error: Memory MCP owner is not configured")
|
||||
|
||||
assert manager.load_all() == before
|
||||
|
||||
|
||||
def test_mcp_memory_preserves_ownerless_local_behavior(monkeypatch, tmp_path):
|
||||
manager = MemoryManager(str(tmp_path))
|
||||
legacy = _entry(
|
||||
manager,
|
||||
"Legacy local memory",
|
||||
memory_id="llllllll-0000-0000-0000-000000000000",
|
||||
)
|
||||
manager.save([legacy])
|
||||
_configure_server(monkeypatch, manager, FakeVector())
|
||||
|
||||
assert "Legacy local memory" in _tool_text({"action": "list"})
|
||||
assert "Legacy local memory" in _tool_text({"action": "search", "text": "legacy"})
|
||||
|
||||
add_text = _tool_text({"action": "add", "text": "Another local memory"})
|
||||
assert "Memory added" in add_text
|
||||
added = next(
|
||||
entry for entry in manager.load_all()
|
||||
if entry["text"] == "Another local memory"
|
||||
)
|
||||
assert "owner" not in added
|
||||
|
||||
assert _tool_text({
|
||||
"action": "edit",
|
||||
"memory_id": legacy["id"][:8],
|
||||
"text": "Updated local memory",
|
||||
}) == "Memory updated: Updated local memory"
|
||||
assert any(entry["text"] == "Updated local memory" for entry in manager.load_all())
|
||||
|
||||
delete_text = _tool_text({"action": "delete", "memory_id": legacy["id"][:8]})
|
||||
assert delete_text.startswith("Memory deleted:")
|
||||
assert all(entry["id"] != legacy["id"] for entry in manager.load_all())
|
||||
@@ -0,0 +1,18 @@
|
||||
"""The web scraping path routes its User-Agent through one constant.
|
||||
|
||||
Guards the dedup: web_fetch / web_search outbound UAs go through
|
||||
WEB_FETCH_USER_AGENT, so a stale or bare Mozilla string cannot be re-inlined in
|
||||
the search sources.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
_SEARCH = Path(__file__).resolve().parent.parent / "services" / "search"
|
||||
|
||||
|
||||
def test_search_sources_have_no_inline_mozilla_ua():
|
||||
offenders = [
|
||||
str(py.relative_to(_SEARCH.parent.parent))
|
||||
for py in _SEARCH.rglob("*.py")
|
||||
if "Mozilla/" in py.read_text(encoding="utf-8")
|
||||
]
|
||||
assert not offenders, f"inline Mozilla UA found; use WEB_FETCH_USER_AGENT: {offenders}"
|
||||
@@ -1,574 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the oversized test-file split plan for issue #3983.
|
||||
|
||||
The output is a planning document only. It does not move tests, rewrite
|
||||
assertions, extract helpers, or change CI.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
TESTS_DIR = ROOT / "tests"
|
||||
OUTPUT = TESTS_DIR / "OVERSIZED_TEST_SPLIT_PLAN.md"
|
||||
RAW_OUTPUT = Path("/tmp/oversized-test-file-metrics.json")
|
||||
|
||||
LARGE_LINE_THRESHOLD = 300
|
||||
LARGE_NODE_THRESHOLD = 20
|
||||
TOP_LIMIT = 30
|
||||
|
||||
HIGH_RISK_SIGNALS = {"route/api", "db/session", "import-state", "security"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FileMetric:
|
||||
path: str
|
||||
lines: int
|
||||
nonblank: int
|
||||
test_defs: int
|
||||
test_classes: int
|
||||
collected: int
|
||||
area: str
|
||||
sub_area: str
|
||||
signals: tuple[str, ...]
|
||||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
def count_ast_tests(text: str) -> tuple[int, int]:
|
||||
tree = ast.parse(text)
|
||||
test_defs = 0
|
||||
test_classes = 0
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
if node.name.startswith("test_"):
|
||||
test_defs += 1
|
||||
elif isinstance(node, ast.ClassDef):
|
||||
if node.name.startswith("Test"):
|
||||
test_classes += 1
|
||||
|
||||
return test_defs, test_classes
|
||||
|
||||
|
||||
def load_taxonomy_classifier():
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from tests._taxonomy import classify_test_path
|
||||
|
||||
return classify_test_path
|
||||
|
||||
|
||||
def classify(path: Path, classify_test_path) -> tuple[str, str]:
|
||||
rel_path = Path(path.relative_to(ROOT).as_posix())
|
||||
|
||||
try:
|
||||
result = classify_test_path(rel_path)
|
||||
except Exception:
|
||||
return "unknown", "unknown"
|
||||
|
||||
return getattr(result, "area", "unknown"), getattr(result, "sub_area", "unknown")
|
||||
|
||||
|
||||
def collect_node_counts() -> Counter[str]:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pytest",
|
||||
"--collect-only",
|
||||
"-q",
|
||||
"tests",
|
||||
]
|
||||
env = dict(os.environ)
|
||||
env["PY_COLORS"] = "0"
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(result.stdout)
|
||||
print(result.stderr, file=sys.stderr)
|
||||
raise SystemExit(result.returncode)
|
||||
|
||||
counts: Counter[str] = Counter()
|
||||
for line in result.stdout.splitlines():
|
||||
line = line.strip()
|
||||
if "::" not in line:
|
||||
continue
|
||||
if not line.startswith("tests/"):
|
||||
continue
|
||||
file_path = line.split("::", 1)[0]
|
||||
counts[file_path] += 1
|
||||
|
||||
return counts
|
||||
|
||||
|
||||
def detect_signals(text: str, path: str) -> tuple[str, ...]:
|
||||
signal_patterns = {
|
||||
"route/api": [
|
||||
r"\bTestClient\b",
|
||||
r"\bapp\.",
|
||||
r"\broutes\.",
|
||||
r"\bfrom routes\b",
|
||||
r"\bimport routes\b",
|
||||
],
|
||||
"db/session": [
|
||||
r"\bSessionLocal\b",
|
||||
r"\bsqlite\b",
|
||||
r"\bDATABASE_URL\b",
|
||||
r"\bcore\.database\b",
|
||||
r"\bdb\.query\b",
|
||||
r"\bcommit\(",
|
||||
],
|
||||
"import-state": [
|
||||
r"\bsys\.modules\b",
|
||||
r"\bimportlib\b",
|
||||
r"\bclear_module\b",
|
||||
r"\bpreserve_import_state\b",
|
||||
r"\bmonkeypatch\.setitem\b",
|
||||
],
|
||||
"security": [
|
||||
r"\bsecurity\b",
|
||||
r"\bssrf\b",
|
||||
r"\bpath traversal\b",
|
||||
r"\bcsrf\b",
|
||||
r"\bpermission\b",
|
||||
],
|
||||
"filesystem": [
|
||||
r"\btmp_path\b",
|
||||
r"\bTemporaryDirectory\b",
|
||||
r"\bPath\(",
|
||||
r"\bmkdir\b",
|
||||
r"\bwrite_text\b",
|
||||
r"\bread_text\b",
|
||||
],
|
||||
"subprocess/script": [
|
||||
r"\bsubprocess\b",
|
||||
r"\brunpy\b",
|
||||
r"\bload_script\b",
|
||||
r"\bsys\.argv\b",
|
||||
],
|
||||
"async/threading": [
|
||||
r"\basyncio\b",
|
||||
r"\bthreading\b",
|
||||
r"\bconcurrent\.futures\b",
|
||||
r"\bThreadPoolExecutor\b",
|
||||
],
|
||||
"ui/static": [
|
||||
r"\bstatic/",
|
||||
r"\bjsdom\b",
|
||||
r"\bnode\b",
|
||||
r"\.js\b",
|
||||
],
|
||||
}
|
||||
|
||||
signals = []
|
||||
for name, patterns in signal_patterns.items():
|
||||
if any(re.search(pattern, text, flags=re.IGNORECASE) for pattern in patterns):
|
||||
signals.append(name)
|
||||
|
||||
if path.startswith("tests/cli/"):
|
||||
signals.append("cli-directory")
|
||||
|
||||
return tuple(signals)
|
||||
|
||||
|
||||
def metric_for(path: Path, node_counts: Counter[str], classify_test_path) -> FileMetric:
|
||||
rel = path.relative_to(ROOT).as_posix()
|
||||
text = read_text(path)
|
||||
lines = len(text.splitlines())
|
||||
nonblank = sum(1 for line in text.splitlines() if line.strip())
|
||||
test_defs, test_classes = count_ast_tests(text)
|
||||
area, sub_area = classify(path, classify_test_path)
|
||||
|
||||
return FileMetric(
|
||||
path=rel,
|
||||
lines=lines,
|
||||
nonblank=nonblank,
|
||||
test_defs=test_defs,
|
||||
test_classes=test_classes,
|
||||
collected=node_counts.get(rel, 0),
|
||||
area=area,
|
||||
sub_area=sub_area,
|
||||
signals=detect_signals(text, rel),
|
||||
)
|
||||
|
||||
|
||||
def test_files() -> list[Path]:
|
||||
return sorted(TESTS_DIR.rglob("test_*.py"))
|
||||
|
||||
|
||||
def as_metric_row(metric: FileMetric) -> str:
|
||||
signals = ", ".join(metric.signals) if metric.signals else "-"
|
||||
return (
|
||||
f"| `{metric.path}` | {metric.lines} | {metric.collected} | "
|
||||
f"{metric.test_defs} | {metric.test_classes} | "
|
||||
f"{metric.area} | {metric.sub_area} | {signals} |"
|
||||
)
|
||||
|
||||
|
||||
def metric_table(title: str, metrics: list[FileMetric]) -> list[str]:
|
||||
lines = [
|
||||
f"## {title}",
|
||||
"",
|
||||
"| File | Lines | Collected tests | Test defs | Test classes | Area | Sub-area | Signals |",
|
||||
"|---|---:|---:|---:|---:|---|---|---|",
|
||||
]
|
||||
lines.extend(as_metric_row(metric) for metric in metrics)
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def candidate_metrics(metrics: list[FileMetric]) -> list[FileMetric]:
|
||||
return [
|
||||
metric
|
||||
for metric in metrics
|
||||
if metric.lines >= LARGE_LINE_THRESHOLD
|
||||
or metric.collected >= LARGE_NODE_THRESHOLD
|
||||
]
|
||||
|
||||
|
||||
def include_reasons(metric: FileMetric) -> str:
|
||||
reasons = []
|
||||
if metric.lines >= LARGE_LINE_THRESHOLD:
|
||||
reasons.append(f"{metric.lines} lines")
|
||||
if metric.collected >= LARGE_NODE_THRESHOLD:
|
||||
reasons.append(f"{metric.collected} collected tests")
|
||||
return ", ".join(reasons)
|
||||
|
||||
|
||||
def risk_notes(metric: FileMetric) -> str:
|
||||
if not metric.signals:
|
||||
return "No obvious setup signals from static scan."
|
||||
return ", ".join(metric.signals)
|
||||
|
||||
|
||||
def suggested_handling(metric: FileMetric) -> str:
|
||||
if HIGH_RISK_SIGNALS.intersection(metric.signals):
|
||||
return "Defer mechanical split until setup/risk boundaries are mapped."
|
||||
if metric.collected >= LARGE_NODE_THRESHOLD:
|
||||
return "Good first manual-review candidate if test themes are cohesive."
|
||||
return "Plan split boundaries before editing."
|
||||
|
||||
|
||||
def candidate_section(metrics: list[FileMetric]) -> list[str]:
|
||||
lines = [
|
||||
"## Split planning candidates",
|
||||
"",
|
||||
"This section is generated from metrics, not from manual judgement.",
|
||||
"Files are included when they meet at least one threshold:",
|
||||
"",
|
||||
f"- at least {LARGE_LINE_THRESHOLD} physical lines; or",
|
||||
f"- at least {LARGE_NODE_THRESHOLD} collected pytest items.",
|
||||
"",
|
||||
"These are planning candidates only. A later split PR still needs a focused manual review of each file before moving tests.",
|
||||
"",
|
||||
"| File | Why included | Setup/risk signals | Suggested handling |",
|
||||
"|---|---|---|---|",
|
||||
]
|
||||
|
||||
for metric in metrics:
|
||||
lines.append(
|
||||
f"| `{metric.path}` | {include_reasons(metric)} | "
|
||||
f"{risk_notes(metric)} | {suggested_handling(metric)} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def first_manual_review_section(metrics: list[FileMetric]) -> list[str]:
|
||||
low_risk = [
|
||||
metric
|
||||
for metric in metrics
|
||||
if metric.area != "uncategorized"
|
||||
and not HIGH_RISK_SIGNALS.intersection(metric.signals)
|
||||
]
|
||||
low_risk = sorted(low_risk, key=lambda m: (m.collected, m.lines), reverse=True)
|
||||
|
||||
lines = [
|
||||
"## Suggested first manual-review candidates",
|
||||
"",
|
||||
"These are not automatic split approvals. They are categorized candidates with enough size/collection value and no route/API, DB/session, import-state, or security signal from the static scan.",
|
||||
"",
|
||||
"Files still in the `uncategorized` taxonomy area are listed separately below so taxonomy review does not get mixed into the first split decision.",
|
||||
"",
|
||||
"| File | Lines | Collected tests | Area | Sub-area | Signals | Why this is a candidate |",
|
||||
"|---|---:|---:|---|---|---|---|",
|
||||
]
|
||||
|
||||
if not low_risk:
|
||||
lines.append("| _None_ | - | - | - | - | - | - |")
|
||||
|
||||
for metric in low_risk[:10]:
|
||||
signals = ", ".join(metric.signals) if metric.signals else "-"
|
||||
lines.append(
|
||||
f"| `{metric.path}` | {metric.lines} | {metric.collected} | "
|
||||
f"{metric.area} | {metric.sub_area} | {signals} | {include_reasons(metric)} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def taxonomy_gap_section(metrics: list[FileMetric]) -> list[str]:
|
||||
uncategorized = [
|
||||
metric
|
||||
for metric in metrics
|
||||
if metric.area == "uncategorized"
|
||||
]
|
||||
uncategorized = sorted(
|
||||
uncategorized,
|
||||
key=lambda m: (m.collected, m.lines),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
lines = [
|
||||
"## Taxonomy coverage gaps among split candidates",
|
||||
"",
|
||||
"`uncategorized` is a current taxonomy area, not a builder failure.",
|
||||
"This plan does not reclassify tests because taxonomy changes should be reviewed separately from oversized-file split planning.",
|
||||
"",
|
||||
"Before using any of these files as a split target, first decide whether the taxonomy should be refined in a separate focused issue/PR.",
|
||||
"",
|
||||
"| File | Lines | Collected tests | Sub-area | Signals | Suggested follow-up |",
|
||||
"|---|---:|---:|---|---|---|",
|
||||
]
|
||||
|
||||
if not uncategorized:
|
||||
lines.append("| _None_ | - | - | - | - | - |")
|
||||
|
||||
for metric in uncategorized:
|
||||
signals = ", ".join(metric.signals) if metric.signals else "-"
|
||||
follow_up = "Review taxonomy mapping before using as a split target."
|
||||
if HIGH_RISK_SIGNALS.intersection(metric.signals):
|
||||
follow_up = "Review taxonomy and setup/risk boundaries before any split."
|
||||
lines.append(
|
||||
f"| `{metric.path}` | {metric.lines} | {metric.collected} | "
|
||||
f"{metric.sub_area} | {signals} | {follow_up} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def deferred_section(metrics: list[FileMetric]) -> list[str]:
|
||||
deferred = [
|
||||
metric
|
||||
for metric in metrics
|
||||
if HIGH_RISK_SIGNALS.intersection(metric.signals)
|
||||
]
|
||||
deferred = sorted(deferred, key=lambda m: (m.collected, m.lines), reverse=True)
|
||||
|
||||
lines = [
|
||||
"## High-risk candidates to defer first",
|
||||
"",
|
||||
"These files may still be split later, but not as the first implementation slice without a separate manual boundary review.",
|
||||
"",
|
||||
"| File | Lines | Collected tests | High-risk signals |",
|
||||
"|---|---:|---:|---|",
|
||||
]
|
||||
|
||||
for metric in deferred[:15]:
|
||||
signals = ", ".join(sorted(HIGH_RISK_SIGNALS.intersection(metric.signals)))
|
||||
lines.append(
|
||||
f"| `{metric.path}` | {metric.lines} | {metric.collected} | {signals} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def write_distribution(
|
||||
lines: list[str],
|
||||
title: str,
|
||||
values: Counter[str],
|
||||
*,
|
||||
min_count: int = 1,
|
||||
) -> None:
|
||||
displayed = [
|
||||
(value, count)
|
||||
for value, count in sorted(values.items())
|
||||
if count >= min_count
|
||||
]
|
||||
omitted_values = sum(1 for count in values.values() if count < min_count)
|
||||
omitted_files = sum(count for count in values.values() if count < min_count)
|
||||
|
||||
lines.extend([
|
||||
f"{title}:",
|
||||
"",
|
||||
"| Value | Files |",
|
||||
"|---|---:|",
|
||||
])
|
||||
for value, count in displayed:
|
||||
lines.append(f"| {value} | {count} |")
|
||||
|
||||
if omitted_values:
|
||||
lines.extend([
|
||||
"",
|
||||
f"Values below {min_count} files: {omitted_values} values covering {omitted_files} files.",
|
||||
])
|
||||
|
||||
lines.append("")
|
||||
|
||||
|
||||
def write_report(metrics: list[FileMetric], node_count_total: int) -> None:
|
||||
by_lines = sorted(metrics, key=lambda m: (m.lines, m.collected), reverse=True)
|
||||
by_collected = sorted(metrics, key=lambda m: (m.collected, m.lines), reverse=True)
|
||||
candidates = sorted(
|
||||
candidate_metrics(metrics),
|
||||
key=lambda m: (m.collected, m.lines),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
areas = Counter(metric.area for metric in metrics)
|
||||
sub_areas = Counter(metric.sub_area for metric in metrics)
|
||||
|
||||
lines = [
|
||||
"# Oversized Test File Split Plan",
|
||||
"",
|
||||
"## Purpose",
|
||||
"",
|
||||
"This document plans future oversized test-file splits using current repo data.",
|
||||
"It does not move files, rewrite assertions, extract helpers, or change CI.",
|
||||
"",
|
||||
"## Roadmap context",
|
||||
"",
|
||||
"- Issue: #3983",
|
||||
"- Parent tracker: #2523",
|
||||
"- Follows #3973 / #3982, the report-only order-sensitivity diagnostics slice.",
|
||||
"",
|
||||
"## Methodology",
|
||||
"",
|
||||
"Metrics were generated from the current test tree using:",
|
||||
"",
|
||||
"- physical line counts for every recursive `test_*.py` file under `tests/`;",
|
||||
"- AST counts for `test_*` functions and `Test*` classes;",
|
||||
"- one `pytest --collect-only -q tests` run to count collected items per file;",
|
||||
"- current taxonomy classification from `tests._taxonomy.classify_test_path`; and",
|
||||
"- static setup-signal scans for route/API, DB/session, import-state, security, filesystem, subprocess/script, async/threading, and UI/static indicators.",
|
||||
"",
|
||||
"Static signals are not proof of risk. They are review prompts.",
|
||||
"Future split PRs must still inspect each file manually before editing.",
|
||||
"",
|
||||
"## Current summary",
|
||||
"",
|
||||
f"- test files scanned: {len(metrics)}",
|
||||
f"- collected pytest items counted: {node_count_total}",
|
||||
f"- large-file threshold: {LARGE_LINE_THRESHOLD} lines",
|
||||
f"- large-collected threshold: {LARGE_NODE_THRESHOLD} collected items",
|
||||
"",
|
||||
]
|
||||
|
||||
write_distribution(lines, "Area distribution", areas)
|
||||
write_distribution(lines, "Sub-area distribution", sub_areas, min_count=2)
|
||||
|
||||
lines.extend(metric_table("Top files by collected pytest items", by_collected[:TOP_LIMIT]))
|
||||
lines.extend(metric_table("Top files by physical line count", by_lines[:TOP_LIMIT]))
|
||||
lines.extend(candidate_section(candidates))
|
||||
lines.extend(taxonomy_gap_section(candidates))
|
||||
lines.extend(first_manual_review_section(candidates))
|
||||
lines.extend(deferred_section(candidates))
|
||||
|
||||
lines.extend([
|
||||
"## Rules for future split PRs",
|
||||
"",
|
||||
"- One file or one coherent file-family per PR.",
|
||||
"- No assertion rewrites mixed with file moves.",
|
||||
"- No helper extraction mixed with file moves.",
|
||||
"- No production code changes.",
|
||||
"- No CI workflow changes.",
|
||||
"- Preserve existing markers and taxonomy unless the split issue explicitly says otherwise.",
|
||||
"- Validate the original file's collected tests before and after the split.",
|
||||
"- Validate any neighboring taxonomy/focused-runner behavior if paths change.",
|
||||
"- Treat files with route/API, DB/session, import-state, or security signals as higher-risk until manually reviewed.",
|
||||
"",
|
||||
"## Suggested next step",
|
||||
"",
|
||||
"Use this plan to choose the first actual oversized-file split issue.",
|
||||
"The first split should prefer a file with high review value and low setup risk.",
|
||||
"Do not start a split PR from this planning issue alone if the file's boundaries are still ambiguous.",
|
||||
"",
|
||||
"## Reproduction command",
|
||||
"",
|
||||
"This document was generated with:",
|
||||
"",
|
||||
"```bash",
|
||||
".venv/bin/python tests/tools/build_oversized_test_split_plan.py",
|
||||
"```",
|
||||
"",
|
||||
"## Freshness check",
|
||||
"",
|
||||
"After editing the builder or rebasing the branch, regenerate the plan and confirm no unexpected plan drift:",
|
||||
"",
|
||||
"```bash",
|
||||
".venv/bin/python tests/tools/build_oversized_test_split_plan.py",
|
||||
"git diff --exit-code -- tests/OVERSIZED_TEST_SPLIT_PLAN.md",
|
||||
"```",
|
||||
"",
|
||||
])
|
||||
|
||||
OUTPUT.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def write_raw(metrics: list[FileMetric]) -> None:
|
||||
raw = [
|
||||
{
|
||||
"area": metric.area,
|
||||
"collected": metric.collected,
|
||||
"lines": metric.lines,
|
||||
"nonblank": metric.nonblank,
|
||||
"path": metric.path,
|
||||
"signals": list(metric.signals),
|
||||
"sub_area": metric.sub_area,
|
||||
"test_classes": metric.test_classes,
|
||||
"test_defs": metric.test_defs,
|
||||
}
|
||||
for metric in metrics
|
||||
]
|
||||
RAW_OUTPUT.write_text(json.dumps(raw, indent=2, sort_keys=True), encoding="utf-8")
|
||||
|
||||
|
||||
def assert_taxonomy_worked(metrics: list[FileMetric]) -> None:
|
||||
if not metrics:
|
||||
raise SystemExit("ERROR: no test files were scanned")
|
||||
|
||||
unknown = sum(1 for metric in metrics if metric.area == "unknown")
|
||||
if unknown == len(metrics):
|
||||
raise SystemExit("ERROR: taxonomy classification returned unknown for every file")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not TESTS_DIR.exists():
|
||||
print("ERROR: tests/ directory not found", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
classify_test_path = load_taxonomy_classifier()
|
||||
node_counts = collect_node_counts()
|
||||
metrics = [metric_for(path, node_counts, classify_test_path) for path in test_files()]
|
||||
|
||||
assert_taxonomy_worked(metrics)
|
||||
write_report(metrics, sum(node_counts.values()))
|
||||
write_raw(metrics)
|
||||
|
||||
print(f"Wrote {OUTPUT.relative_to(ROOT)}")
|
||||
print(f"Wrote {RAW_OUTPUT}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user