diff --git a/tests/README.md b/tests/README.md index bfdc27366..66a720b9b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -33,6 +33,20 @@ the sub-area. The `area_*` names are registered in `pyproject.toml`; the dynamic `sub_*` names are registered before collection by `pytest_configure` in `tests/conftest.py`, so unknown-mark warnings still flag genuine typos. +For common focused runs, use `tests/run_focus.py`. It validates area and +sub-area names, accepts sub-areas with or without the `sub_` prefix, and passes +extra pytest arguments after `--`: + +```bash +python3 tests/run_focus.py --area security +python3 tests/run_focus.py --area services --sub-area cookbook +python3 tests/run_focus.py --sub-area sub_cookbook +python3 tests/run_focus.py --keyword taxonomy +python3 tests/run_focus.py --last-failed +python3 tests/run_focus.py --dry-run --area services --sub-area cookbook +python3 tests/run_focus.py --area services -- --maxfail=1 -q +``` + ## Core principles - Keep PRs small and homogeneous: one kind of change per PR. diff --git a/tests/run_focus.py b/tests/run_focus.py new file mode 100644 index 000000000..c09035f39 --- /dev/null +++ b/tests/run_focus.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Focused test selection runner for the pytest taxonomy markers (issue #3442). + +This wraps ``pytest -m`` selection over the ``area_*`` / ``sub_*`` markers that +``tests/conftest.py`` adds at collection time (issue #3491) so focused +validation is repeatable and less error-prone than hand-written marker +expressions. It builds a pytest command line and either prints it (``--dry-run``) +or runs it. + +Examples: + tests/run_focus.py --area security + tests/run_focus.py --area services --sub-area cookbook + tests/run_focus.py --keyword taxonomy -- --maxfail=1 -q + +This script imports no production code and changes no test behavior. It only +constructs and (optionally) executes a pytest invocation. +""" +from __future__ import annotations + +import argparse +import shlex +import subprocess +import sys +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +TESTS_DIR = Path(__file__).resolve().parent +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from tests._taxonomy import discover_markers, normalize_marker_name # noqa: E402 + +# The canonical taxonomy areas, mirroring the ``area_*`` markers declared in +# pyproject.toml and produced by tests/_taxonomy.py. +AREAS: tuple[str, ...] = ( + "security", + "routes", + "services", + "cli", + "js", + "helpers", + "unit", + "uncategorized", +) + + +def normalize_sub_area(value: str) -> str: + """Normalize a CLI sub-area value and remove an optional ``sub_`` prefix.""" + token = normalize_marker_name(value) + if token.startswith("sub_"): + token = token.removeprefix("sub_") + if not token: + raise argparse.ArgumentTypeError( + f"invalid sub-area {value!r}: must contain at least one letter or digit" + ) + return token + + +def discover_sub_areas(tests_dir: Path = TESTS_DIR) -> frozenset[str]: + """Discover valid taxonomy sub-areas from Python test filenames.""" + paths = list(tests_dir.rglob("test_*.py")) + paths += list(tests_dir.rglob("*_test.py")) + markers = discover_markers(paths) + return frozenset( + marker.removeprefix("sub_") + for marker in markers + if marker.startswith("sub_") + ) + + +def sub_area_type(valid_sub_areas: frozenset[str]) -> Callable[[str], str]: + """Build an argparse converter that accepts only discovered sub-areas.""" + + def validate(value: str) -> str: + sub_area = normalize_sub_area(value) + if sub_area not in valid_sub_areas: + raise argparse.ArgumentTypeError( + f"unknown sub-area {value!r}; choose a discovered taxonomy sub-area" + ) + return sub_area + + return validate + + +@dataclass(frozen=True) +class FocusSelection: + """A single focused-selection request, decoupled from argparse and pytest.""" + + area: str | None = None + sub_area: str | None = None + keyword: str | None = None + last_failed: bool = False + pytest_args: tuple[str, ...] = field(default_factory=tuple) + + @property + def has_focus(self) -> bool: + """True when at least one focusing selector (not just pass-through) is set.""" + return bool(self.area or self.sub_area or self.keyword or self.last_failed) + + +def build_marker_expression(area: str | None, sub_area: str | None) -> str | None: + """Build the ``-m`` marker expression from an area and/or sub-area. + + Returns ``None`` when neither is given so the caller can omit ``-m``. + """ + parts: list[str] = [] + if area: + parts.append(f"area_{area}") + if sub_area: + parts.append(f"sub_{sub_area}") + if not parts: + return None + return " and ".join(parts) + + +def build_pytest_command( + selection: FocusSelection, python: str | None = None +) -> list[str]: + """Build the pytest argv list for ``selection``. + + No shell is involved; the result is a plain argv list for subprocess. The + interpreter defaults to the one running this script (the project venv when + invoked as ``.venv/bin/python tests/run_focus.py``). + """ + command = [python or sys.executable, "-m", "pytest"] + marker_expression = build_marker_expression(selection.area, selection.sub_area) + if marker_expression: + command += ["-m", marker_expression] + if selection.keyword: + command += ["-k", selection.keyword] + if selection.last_failed: + command += ["--last-failed", "--last-failed-no-failures=none"] + command += list(selection.pytest_args) + return command + + +def selection_from_args(namespace: argparse.Namespace) -> FocusSelection: + """Convert parsed argparse values into a ``FocusSelection``.""" + return FocusSelection( + area=namespace.area, + sub_area=namespace.sub_area, + keyword=namespace.keyword, + last_failed=namespace.last_failed, + pytest_args=tuple(namespace.pytest_args), + ) + + +def build_parser( + valid_sub_areas: frozenset[str] | None = None, +) -> argparse.ArgumentParser: + """Build the argument parser for the focused runner.""" + if valid_sub_areas is None: + valid_sub_areas = discover_sub_areas() + parser = argparse.ArgumentParser( + prog="run_focus.py", + description=( + "Run a focused subset of the test suite using the area_*/sub_* " + "taxonomy markers. Combine --area and --sub-area to intersect them." + ), + epilog=( + "Pass extra pytest arguments after a literal -- separator, e.g.: " + "run_focus.py --area services -- --maxfail=1 -q" + ), + ) + parser.add_argument( + "--area", + choices=AREAS, + help="select tests in one taxonomy area (marker area_)", + ) + parser.add_argument( + "--sub-area", + type=sub_area_type(valid_sub_areas), + metavar="NAME", + help="select tests in a sub-area (marker sub_); combinable with --area", + ) + parser.add_argument( + "-k", + "--keyword", + help="pass a keyword expression through to pytest -k", + ) + parser.add_argument( + "--last-failed", + action="store_true", + help="re-run only tests that failed on the last run (pytest --last-failed)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="print the pytest command without executing it", + ) + parser.add_argument( + "pytest_args", + nargs="*", + metavar="-- PYTEST_ARGS", + help="extra arguments forwarded to pytest after a literal --", + ) + return parser + + +def run( + argv: Sequence[str] | None = None, + executor: Callable[[list[str]], int] = subprocess.call, +) -> int: + """Parse ``argv``, build the pytest command, and run or print it. + + ``executor`` is injected so tests can assert on the constructed command + without spawning a process. It must accept an argv list and return an exit + code, matching ``subprocess.call``. + """ + parser = build_parser() + namespace = parser.parse_args(argv) + selection = selection_from_args(namespace) + if not selection.has_focus: + parser.error( + "no focus selected: pass at least one of --area, --sub-area, " + "--keyword, or --last-failed" + ) + command = build_pytest_command(selection) + if namespace.dry_run: + print(shlex.join(command)) + return 0 + return executor(command) + + +def main() -> int: + """Console entry point.""" + return run(sys.argv[1:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_run_focus.py b/tests/test_run_focus.py new file mode 100644 index 000000000..959ee0ca5 --- /dev/null +++ b/tests/test_run_focus.py @@ -0,0 +1,218 @@ +"""Direct tests for the focused test-selection runner (tests/run_focus.py). + +Command construction is tested separately from process execution: the pure +builder functions are asserted directly, and ``run`` is exercised with an +injected fake executor so no pytest subprocess is ever spawned. +""" +from __future__ import annotations + +import argparse +import sys + +import pytest + +from tests.run_focus import ( + FocusSelection, + build_marker_expression, + build_pytest_command, + discover_sub_areas, + normalize_sub_area, + run, +) + +PY = "PY" # placeholder interpreter for deterministic command assertions + + +def _cmd(**kwargs) -> list[str]: + """Build a pytest command for a FocusSelection made from kwargs.""" + return build_pytest_command(FocusSelection(**kwargs), python=PY) + + +# --- marker expression building ------------------------------------------- + + +def test_area_only_marker_expression(): + assert build_marker_expression("security", None) == "area_security" + + +def test_sub_area_only_marker_expression(): + assert build_marker_expression(None, "cookbook") == "sub_cookbook" + + +def test_area_and_sub_area_marker_expression(): + assert build_marker_expression("services", "cookbook") == "area_services and sub_cookbook" + + +def test_no_selection_marker_expression_is_none(): + assert build_marker_expression(None, None) is None + + +# --- command construction -------------------------------------------------- + + +def test_area_only_command(): + assert _cmd(area="security") == [PY, "-m", "pytest", "-m", "area_security"] + + +def test_sub_area_only_command(): + assert _cmd(sub_area="cookbook") == [PY, "-m", "pytest", "-m", "sub_cookbook"] + + +def test_area_and_sub_area_command(): + assert _cmd(area="services", sub_area="cookbook") == [ + PY, "-m", "pytest", "-m", "area_services and sub_cookbook", + ] + + +def test_keyword_only_command(): + assert _cmd(keyword="taxonomy") == [PY, "-m", "pytest", "-k", "taxonomy"] + + +def test_area_and_keyword_command(): + assert _cmd(area="services", keyword="cookbook") == [ + PY, "-m", "pytest", "-m", "area_services", "-k", "cookbook", + ] + + +def test_passthrough_pytest_args_appended_last(): + command = _cmd(area="services", pytest_args=("--maxfail=1", "-q")) + assert command == [PY, "-m", "pytest", "-m", "area_services", "--maxfail=1", "-q"] + + +def test_last_failed_appends_safe_flags(): + assert _cmd(last_failed=True) == [ + PY, + "-m", + "pytest", + "--last-failed", + "--last-failed-no-failures=none", + ] + + +def test_default_python_is_current_interpreter(): + command = build_pytest_command(FocusSelection(area="cli")) + assert command[0] == sys.executable + + +# --- sub-area normalization ------------------------------------------------ + + +def test_normalize_sub_area_lowercases_and_collapses(): + assert normalize_sub_area("Cook Book") == "cook_book" + + +def test_normalize_sub_area_strips_separators(): + assert normalize_sub_area("--owner.scope--") == "owner_scope" + + +def test_normalize_sub_area_removes_marker_prefix(): + assert normalize_sub_area("sub_cookbook") == "cookbook" + + +def test_normalize_sub_area_rejects_empty_after_normalization(): + with pytest.raises(argparse.ArgumentTypeError): + normalize_sub_area("!!!") + + +def test_discover_sub_areas_from_test_filename(tmp_path): + (tmp_path / "test_cookbook_helpers.py").write_text("", encoding="utf-8") + + assert discover_sub_areas(tmp_path) == frozenset({"cookbook"}) + + +# --- run(): dry-run, execution, validation --------------------------------- + + +class _FakeExecutor: + """Records the command it was asked to run and returns a fixed code.""" + + def __init__(self, returncode: int = 0): + self.returncode = returncode + self.calls: list[list[str]] = [] + + def __call__(self, command: list[str]) -> int: + self.calls.append(command) + return self.returncode + + +def test_dry_run_prints_command_and_does_not_execute(capsys): + executor = _FakeExecutor() + code = run( + ["--dry-run", "--area", "services", "--sub-area", "cookbook"], + executor=executor, + ) + out = capsys.readouterr().out + assert code == 0 + assert executor.calls == [] + assert out == ( + f"{sys.executable} -m pytest " + "-m 'area_services and sub_cookbook'\n" + ) + + +def test_dry_run_last_failed_prints_safe_flags(capsys): + executor = _FakeExecutor() + code = run(["--dry-run", "--last-failed"], executor=executor) + out = capsys.readouterr().out + assert code == 0 + assert executor.calls == [] + assert out == ( + f"{sys.executable} -m pytest " + "--last-failed --last-failed-no-failures=none\n" + ) + + +def test_run_invokes_executor_with_built_command(): + executor = _FakeExecutor(returncode=3) + code = run(["--keyword", "taxonomy", "--", "--maxfail=1"], executor=executor) + assert code == 3 + assert executor.calls == [[sys.executable, "-m", "pytest", "-k", "taxonomy", "--maxfail=1"]] + + +def test_run_last_failed_only(): + executor = _FakeExecutor() + run(["--last-failed"], executor=executor) + assert executor.calls == [[ + sys.executable, + "-m", + "pytest", + "--last-failed", + "--last-failed-no-failures=none", + ]] + + +@pytest.mark.parametrize("value", ["cookbook", "sub_cookbook"]) +def test_run_accepts_both_sub_area_forms(value): + executor = _FakeExecutor() + run(["--sub-area", value], executor=executor) + assert executor.calls == [[ + sys.executable, + "-m", + "pytest", + "-m", + "sub_cookbook", + ]] + + +def test_invalid_area_exits_with_error(): + with pytest.raises(SystemExit) as excinfo: + run(["--area", "bogus"], executor=_FakeExecutor()) + assert excinfo.value.code == 2 + + +def test_invalid_sub_area_exits_with_error(capsys): + with pytest.raises(SystemExit) as excinfo: + run( + ["--sub-area", "definitely_not_a_real_sub_area"], + executor=_FakeExecutor(), + ) + assert excinfo.value.code == 2 + assert "unknown sub-area" in capsys.readouterr().err + + +def test_no_focus_selector_is_rejected(): + executor = _FakeExecutor() + with pytest.raises(SystemExit) as excinfo: + run(["--", "-q"], executor=executor) + assert excinfo.value.code == 2 + assert executor.calls == []