diff --git a/pyproject.toml b/pyproject.toml index 116b1376c..58161958f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,18 @@ [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" +# Test-taxonomy markers added at collection time by tests/conftest.py. The +# stable area_* markers are declared here; the dynamic sub_ +# markers are registered before collection by pytest_configure in +# tests/conftest.py, so unknown-mark warnings still flag genuine typos outside +# the taxonomy. See tests/_taxonomy.py and tests/README.md. +markers = [ + "area_security: tests covering auth, owner-scope, SSRF, XSS, confinement, redaction", + "area_routes: tests covering HTTP route / API behavior", + "area_services: tests covering service-layer behavior (llm, cookbook, email, calendar, ...)", + "area_cli: tests covering CLI / script behavior", + "area_js: JavaScript / Node-backed tests", + "area_helpers: self-tests for the shared test helpers in tests/helpers/", + "area_unit: pure parser / utility tests that do not clearly belong elsewhere", + "area_uncategorized: tests not yet matched by the taxonomy (fallback)", +] diff --git a/tests/README.md b/tests/README.md index b1b99de72..bfdc27366 100644 --- a/tests/README.md +++ b/tests/README.md @@ -13,6 +13,26 @@ behavioral-vs-source-text policy, and helper/factory extraction rules - see [`TESTING_STANDARD.md`](./TESTING_STANDARD.md). This file is the concrete helper reference; that file is the standard the refactor works toward. +## Running focused subsets (taxonomy markers) + +`tests/conftest.py` tags every test at collection time with two markers derived +from its filename by `tests/_taxonomy.py`: an `area_*` marker (e.g. +`area_security`) and a finer `sub_*` marker (e.g. `sub_owner_scope`). This adds +markers only - it moves no files and changes no test behavior. Use them to run a +focused slice: + +```bash +python3 -m pytest -m area_security +python3 -m pytest -m "area_services and sub_cookbook" +``` + +Areas are `security`, `routes`, `services`, `cli`, `js`, `helpers`, `unit`, and +`uncategorized`. Classification is conservative and token-based: a file that +matches no area keyword falls back to `area_uncategorized` with its filename as +the sub-area. The `area_*` names are registered in `pyproject.toml`; the dynamic +`sub_*` names are registered before collection by `pytest_configure` in +`tests/conftest.py`, so unknown-mark warnings still flag genuine typos. + ## Core principles - Keep PRs small and homogeneous: one kind of change per PR. diff --git a/tests/_taxonomy.py b/tests/_taxonomy.py new file mode 100644 index 000000000..cc99cdbc1 --- /dev/null +++ b/tests/_taxonomy.py @@ -0,0 +1,162 @@ +"""Conservative test taxonomy: classify test files by area and sub-area. + +This module is the single source of truth for the collection-time markers added +in ``tests/conftest.py``. It performs no inference beyond simple, exact matching +of filename tokens against small, explicit keyword sets. A file is matched to +the first area (in priority order) whose keyword set intersects its filename +tokens; files that match no area fall back to ``uncategorized`` with the +filename itself as the sub-area. + +The categories mirror ``tests/TESTING_STANDARD.md``. This module imports nothing +from the application - only the standard library - and changes no test behavior. +""" +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path + +# Area keyword sets. Keep these small and explicit; prefer leaving a file +# ``uncategorized`` over guessing. Matching is exact, token-by-token. +SECURITY_KEYWORDS = frozenset({ + "security", "auth", "owner", "scope", + "ssrf", "xss", "confinement", "permission", "redaction", +}) +CLI_KEYWORDS = frozenset({"cli"}) +ROUTES_KEYWORDS = frozenset({"route", "routes", "api"}) +SERVICES_KEYWORDS = frozenset({ + "llm", "provider", "cookbook", "session", "history", "email", + "calendar", "memory", "gallery", "document", "research", "mcp", + "scheduler", "webhook", "embedding", +}) +UNIT_KEYWORDS = frozenset({ + "parse", "parser", "parsing", "nonstring", "nondict", + "atomic", "regex", "tokenize", +}) + +# Keyword-matched areas, in priority order (first match wins). Security is a +# cross-cutting concern and intentionally outranks the feature areas, so e.g. +# ``test_email_owner_scope.py`` classifies as ``security``, not ``services``. +# ``js`` and ``helpers`` are matched by dedicated rules in ``_match_area``. +KEYWORD_AREAS = ( + ("security", SECURITY_KEYWORDS), + ("cli", CLI_KEYWORDS), + ("routes", ROUTES_KEYWORDS), + ("services", SERVICES_KEYWORDS), + ("unit", UNIT_KEYWORDS), +) + +# File extensions that indicate a JavaScript/Node-backed test. +JS_EXTENSIONS = frozenset({".js", ".mjs", ".ts"}) + +UNCATEGORIZED = "uncategorized" + + +@dataclass(frozen=True) +class TestClassification: + """Area and sub-area for a single test file.""" + + area: str + sub_area: str + + +def normalize_marker_name(value: str) -> str: + """Lowercase ``value`` and reduce it to a marker-safe ``[a-z0-9_]`` token.""" + lowered = value.lower() + collapsed = re.sub(r"[^a-z0-9]+", "_", lowered) + return collapsed.strip("_") + + +def _stem(path: str | Path) -> str: + """Filename without its extension chain (``invariant.test.mjs`` -> ``invariant``).""" + return Path(path).name.split(".", 1)[0] + + +def _extension(path: str | Path) -> str: + """Lowercased final file extension, e.g. ``.py`` or ``.mjs``.""" + return Path(path).suffix.lower() + + +def _filename_tokens(path: str | Path) -> tuple[str, ...]: + """Underscore tokens of the filename stem, with a leading ``test`` dropped.""" + tokens = tuple(t for t in normalize_marker_name(_stem(path)).split("_") if t) + if tokens and tokens[0] == "test": + tokens = tokens[1:] + return tokens + + +def _matched_keywords(tokens: tuple[str, ...], keywords: frozenset[str]) -> tuple[str, ...]: + """Filename tokens that appear in ``keywords``, in order, de-duplicated.""" + matched: list[str] = [] + for token in tokens: + if token in keywords and token not in matched: + matched.append(token) + return tuple(matched) + + +def _match_area(tokens: tuple[str, ...], extension: str) -> tuple[str, tuple[str, ...]]: + """Return ``(area, matched_keywords)`` using the conservative priority order.""" + if extension in JS_EXTENSIONS or "js" in tokens: + return "js", ("js",) + if tokens and tokens[0] == "helpers": + return "helpers", ("helpers",) + for area, keywords in KEYWORD_AREAS: + matched = _matched_keywords(tokens, keywords) + if matched: + return area, matched + return UNCATEGORIZED, () + + +def _sub_area(area: str, matched: tuple[str, ...], tokens: tuple[str, ...]) -> str: + """Derive the sub-area: matched keywords for a known area, else the filename.""" + if area == UNCATEGORIZED: + return "_".join(tokens) + return "_".join(matched) + + +def _in_helpers_dir(path: str | Path) -> bool: + """True if ``path`` is under the test helper dir ``tests/helpers/``. + + Matches the exact adjacent ``tests``/``helpers`` component pair, so an + unrelated ancestor directory merely named ``helpers`` does not count. + """ + parts = Path(path).parent.parts + adjacent_pairs = list(zip(parts, parts[1:])) + return ("tests", "helpers") in adjacent_pairs + + +def classify_test_path(path: str | Path) -> TestClassification: + """Classify a test file path into an area and a sub-area. + + A test file under a ``helpers`` directory is a helper self-test regardless of + its filename, which complements the filename first-token rule in + ``_match_area`` (e.g. ``test_helpers_import_state.py`` in ``tests/``). + """ + if _in_helpers_dir(path): + return TestClassification(area="helpers", sub_area="helpers") + tokens = _filename_tokens(path) + area, matched = _match_area(tokens, _extension(path)) + sub_area = _sub_area(area, matched, tokens) or UNCATEGORIZED + return TestClassification(area=area, sub_area=sub_area) + + +def markers_for_path(path: str | Path) -> tuple[str, ...]: + """Return the ``(area_*, sub_*)`` marker names for a test file path.""" + classification = classify_test_path(path) + area_marker = normalize_marker_name(f"area_{classification.area}") + sub_marker = normalize_marker_name(f"sub_{classification.sub_area}") + return (area_marker, sub_marker) + + +def discover_markers(paths: Iterable[str | Path]) -> tuple[str, ...]: + """Distinct ``area_*`` / ``sub_*`` marker names for ``paths``, sorted. + + Pure: it derives names from the given paths only and performs no filesystem + access of its own. The caller decides which paths to scan. Used at + ``pytest_configure`` time to register the dynamic ``sub_*`` markers. + """ + names: set[str] = set() + for path in paths: + names.update(markers_for_path(path)) + return tuple(sorted(names)) diff --git a/tests/conftest.py b/tests/conftest.py index b30774e0e..4567aae80 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -"""Shared test configuration — ensure project root is on sys.path and stub heavy deps.""" +"""Shared test configuration - ensure project root is on sys.path and stub heavy deps.""" import sys import os import types @@ -9,12 +9,12 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # Importing core.database below runs init_db() at import time, and its default # (sqlite:///./data/app.db) can't be opened in a clean worktree because SQLite -# won't create the missing ./data parent dir — pytest then dies during +# won't create the missing ./data parent dir - pytest then dies during # collection, before any test module loads. Default to an in-memory DB for the # test session so collection is deterministic and writes no repo-local # artifacts. An explicit DATABASE_URL (a real test/CI database) is preserved. # This only unblocks collection/import-time init; it does not provide a shared -# file-backed DB across processes — tests needing that must set DATABASE_URL. +# file-backed DB across processes - tests needing that must set DATABASE_URL. os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:") # Pre-import real heavy modules BEFORE any test file's module-level stubs can @@ -27,7 +27,7 @@ try: import sqlalchemy.orm # noqa: F401 import core.database # noqa: F401 except ImportError: - pass # not installed — the stubs below will handle it + pass # not installed - the stubs below will handle it def _has_module(mod_name: str) -> bool: try: @@ -54,3 +54,37 @@ if "src.database" not in sys.modules: _db.SessionLocal = MagicMock() _db.ModelEndpoint = MagicMock() sys.modules["src.database"] = _db + + +def pytest_configure(config): + """Register the dynamic taxonomy ``sub_*`` markers before collection. + + The stable ``area_*`` markers are declared in ``pyproject.toml``. The + per-file ``sub_*`` markers are derived from the test filenames here so that + unknown-mark warnings still surface genuine typos outside the taxonomy. This + only registers marker names; it imports no production module. + """ + import pathlib + from tests._taxonomy import discover_markers + + tests_dir = pathlib.Path(__file__).parent + paths = list(tests_dir.rglob("test_*.py")) + list(tests_dir.rglob("*_test.py")) + for marker_name in discover_markers(paths): + if marker_name.startswith("sub_"): + config.addinivalue_line("markers", f"{marker_name}: taxonomy sub-area marker") + + +def pytest_collection_modifyitems(config, items): + """Tag each collected test with its taxonomy ``area_*`` and ``sub_*`` markers. + + Collection-time only: this adds markers and nothing else. It does not skip, + reorder, or deselect tests, mutate fixtures or the environment, or import any + production module. See ``tests/_taxonomy.py`` for the classification rules. + """ + import pytest + from tests._taxonomy import markers_for_path + + for item in items: + path = getattr(item, "path", None) or item.fspath + for marker_name in markers_for_path(path): + item.add_marker(getattr(pytest.mark, marker_name)) diff --git a/tests/test_taxonomy.py b/tests/test_taxonomy.py new file mode 100644 index 000000000..9b00201e4 --- /dev/null +++ b/tests/test_taxonomy.py @@ -0,0 +1,145 @@ +"""Unit tests for tests/_taxonomy.py - the test-taxonomy classification module. + +These tests pin the conservative classification behavior directly, without +running pytest collection. They import only the module under test (a test-support +module, not production code) and touch no filesystem. +""" +import re + +import pytest + +from tests._taxonomy import ( + classify_test_path, + discover_markers, + markers_for_path, + normalize_marker_name, +) + + +# --- normalize_marker_name --------------------------------------------------- + +def test_normalize_lowercases(): + assert normalize_marker_name("Area_Security") == "area_security" + + +def test_normalize_converts_nonalphanumeric_runs_to_underscore(): + assert normalize_marker_name("owner--scope..test") == "owner_scope_test" + + +def test_normalize_strips_leading_and_trailing_underscores(): + assert normalize_marker_name("__owner-scope__") == "owner_scope" + + +# --- classify_test_path: one example per area -------------------------------- + +@pytest.mark.parametrize("filename, expected_area, expected_sub", [ + ("test_owner_scope.py", "security", "owner_scope"), + ("test_cookbook_helpers.py", "services", "cookbook"), + ("test_routes_sessions.py", "routes", "routes"), + ("test_backup_cli.py", "cli", "cli"), + ("test_compare_js.py", "js", "js"), + ("segmenter.test.mjs", "js", "js"), + ("segmenter.test.js", "js", "js"), + ("segmenter.test.ts", "js", "js"), + ("test_helpers_import_state.py", "helpers", "helpers"), + ("test_atomic_io.py", "unit", "atomic"), +]) +def test_classify_examples(filename, expected_area, expected_sub): + result = classify_test_path(filename) + assert result.area == expected_area + assert result.sub_area == expected_sub + + +# --- classify_test_path: fallback -------------------------------------------- + +def test_unknown_filename_is_uncategorized(): + result = classify_test_path("test_widget_gizmo_thing.py") + assert result.area == "uncategorized" + + +def test_uncategorized_sub_area_is_derived_from_filename_tokens(): + result = classify_test_path("test_archived_sessions_model_filter.py") + assert result.area == "uncategorized" + assert result.sub_area == "archived_sessions_model_filter" + + +# --- markers_for_path -------------------------------------------------------- + +def test_markers_for_path_returns_one_area_and_one_sub(): + markers = markers_for_path("test_owner_scope.py") + assert markers == ("area_security", "sub_owner_scope") + assert len([m for m in markers if m.startswith("area_")]) == 1 + assert len([m for m in markers if m.startswith("sub_")]) == 1 + + +def test_markers_for_path_are_normalized(): + markers = markers_for_path("test_foo-bar.py") + assert markers == ("area_uncategorized", "sub_foo_bar") + for marker in markers: + assert re.fullmatch(r"[a-z0-9_]+", marker) + + +# --- discover_markers -------------------------------------------------------- + +def test_discover_markers_is_sorted_and_deduplicated(): + paths = [ + "test_owner_scope.py", + "test_owner_scope.py", + "test_cookbook_helpers.py", + ] + markers = discover_markers(paths) + assert markers == tuple(sorted(set(markers))) + assert markers == ( + "area_security", + "area_services", + "sub_cookbook", + "sub_owner_scope", + ) + + +def test_discover_markers_includes_area_and_sub(): + markers = discover_markers(["test_owner_scope.py"]) + assert any(m.startswith("area_") for m in markers) + assert any(m.startswith("sub_") for m in markers) + + +# --- edge cases -------------------------------------------------------------- + +def test_normalize_all_symbols_becomes_empty(): + assert normalize_marker_name("@@@") == "" + + +def test_bare_test_filename_is_fully_uncategorized(): + result = classify_test_path("tests/test.py") + assert result.area == "uncategorized" + assert result.sub_area == "uncategorized" + + +def test_markers_for_bare_test_filename(): + markers = markers_for_path("tests/test.py") + assert "area_uncategorized" in markers + assert "sub_uncategorized" in markers + + +@pytest.mark.parametrize("path", [ + "tests/helpers/test_module_isolation.py", + "/work/repo/tests/helpers/test_module_isolation.py", +]) +def test_file_under_helpers_dir_is_helpers(path): + result = classify_test_path(path) + assert result.area == "helpers" + assert result.sub_area == "helpers" + + +# --- priority contract ------------------------------------------------------- + +def test_security_beats_services_when_both_tokens_present(): + result = classify_test_path("test_email_owner_scope.py") + assert result.area == "security" + assert result.sub_area == "owner_scope" + + +def test_unrelated_helpers_ancestor_is_not_helpers(): + result = classify_test_path("/work/helpers/odysseus/tests/test_owner_scope.py") + assert result.area == "security" + assert result.sub_area == "owner_scope"