test: add fast lane and duration visibility (#3659)

This commit is contained in:
Alexandre Teixeira
2026-06-09 19:11:47 +01:00
committed by GitHub
parent 9e74a327f8
commit cdfda4bd16
5 changed files with 252 additions and 7 deletions
+4
View File
@@ -15,4 +15,8 @@ markers = [
"area_helpers: self-tests for the shared test helpers in tests/helpers/", "area_helpers: self-tests for the shared test helpers in tests/helpers/",
"area_unit: pure parser / utility tests that do not clearly belong elsewhere", "area_unit: pure parser / utility tests that do not clearly belong elsewhere",
"area_uncategorized: tests not yet matched by the taxonomy (fallback)", "area_uncategorized: tests not yet matched by the taxonomy (fallback)",
# Fast-lane marker (issue #3443). Opt-in and orthogonal to the area_*/sub_*
# taxonomy. The fast lane runs `not slow`; mark a test slow only with
# duration evidence (see tests/run_focus.py --durations and tests/README.md).
"slow: opt-in marker for known-slow tests; excluded by the fast lane (not slow)",
] ]
+29
View File
@@ -47,6 +47,35 @@ python3 tests/run_focus.py --dry-run --area services --sub-area cookbook
python3 tests/run_focus.py --area services -- --maxfail=1 -q python3 tests/run_focus.py --area services -- --maxfail=1 -q
``` ```
### Fast lane and duration visibility
`--fast` runs the fast lane: the tests that are *not* marked `slow` (it adds the
marker expression `not slow`). It composes with `--area`/`--sub-area` using
`and`. Because no tests may be marked `slow` yet, `--fast` can initially match
the full focused selection; it becomes a real speed-up as `slow` marks are added
from duration evidence. Use it for quick local or reviewer feedback; it does not
replace broader focused or full-suite validation before merge.
`--durations N` and `--durations-min FLOAT` add pytest's slowest-test reporting
so you can see where time goes. They are reporting only and do not count as a
focus selector, so `--durations` must be combined with a real selector
(`--area`, `--sub-area`, `--keyword`, `--last-failed`, or `--fast`).
Activate or otherwise use the project Python environment before running these
commands. The examples use `python3` intentionally to avoid hard-coding a local
venv path.
```bash
python3 tests/run_focus.py --fast
python3 tests/run_focus.py --area services --fast
python3 tests/run_focus.py --area services --durations 25
python3 tests/run_focus.py --area services --fast --durations 25 --durations-min 0.05
```
The `slow` marker is opt-in. Mark a test `slow` only with duration evidence
(from `--durations`), not by guessing - see the fast-lane policy in
`TESTING_STANDARD.md`.
## Core principles ## Core principles
- Keep PRs small and homogeneous: one kind of change per PR. - Keep PRs small and homogeneous: one kind of change per PR.
+10
View File
@@ -74,6 +74,16 @@ A test that genuinely spans categories (e.g. a route test that also pins a
security invariant) is classified by its **primary** assertion target and may be security invariant) is classified by its **primary** assertion target and may be
split if it grows. split if it grows.
## Fast lane policy
The fast lane is `not slow`: `tests/run_focus.py --fast` selects every test that
is not marked `slow`. The `slow` marker is **opt-in**, and slow marks must be
**evidence-driven from `--durations` output** - mark a test slow only when its
measured duration shows it is genuinely expensive, never by guessing. The fast
lane exists for quick local and reviewer feedback; it is **not** a replacement
for broader focused or full-suite validation before merge, and a test must never
be marked `slow` to hide a failure or skip coverage.
## Determinism & isolation rules ## Determinism & isolation rules
Do not mutate shared process state without a controlled helper and guaranteed Do not mutate shared process state without a controlled helper and guaranteed
+74 -7
View File
@@ -11,6 +11,8 @@ Examples:
tests/run_focus.py --area security tests/run_focus.py --area security
tests/run_focus.py --area services --sub-area cookbook tests/run_focus.py --area services --sub-area cookbook
tests/run_focus.py --keyword taxonomy -- --maxfail=1 -q tests/run_focus.py --keyword taxonomy -- --maxfail=1 -q
tests/run_focus.py --fast
tests/run_focus.py --area services --fast --durations 25
This script imports no production code and changes no test behavior. It only This script imports no production code and changes no test behavior. It only
constructs and (optionally) executes a pytest invocation. constructs and (optionally) executes a pytest invocation.
@@ -70,6 +72,22 @@ def discover_sub_areas(tests_dir: Path = TESTS_DIR) -> frozenset[str]:
) )
def non_negative_int(value: str) -> int:
"""argparse type: a non-negative int (0 means "show all" for --durations)."""
number = int(value)
if number < 0:
raise argparse.ArgumentTypeError(f"must be >= 0, got {value!r}")
return number
def non_negative_float(value: str) -> float:
"""argparse type: a non-negative float (seconds threshold for --durations-min)."""
number = float(value)
if number < 0:
raise argparse.ArgumentTypeError(f"must be >= 0, got {value!r}")
return number
def sub_area_type(valid_sub_areas: frozenset[str]) -> Callable[[str], str]: def sub_area_type(valid_sub_areas: frozenset[str]) -> Callable[[str], str]:
"""Build an argparse converter that accepts only discovered sub-areas.""" """Build an argparse converter that accepts only discovered sub-areas."""
@@ -92,24 +110,42 @@ class FocusSelection:
sub_area: str | None = None sub_area: str | None = None
keyword: str | None = None keyword: str | None = None
last_failed: bool = False last_failed: bool = False
fast: bool = False
durations: int | None = None
durations_min: float | None = None
pytest_args: tuple[str, ...] = field(default_factory=tuple) pytest_args: tuple[str, ...] = field(default_factory=tuple)
@property @property
def has_focus(self) -> bool: def has_focus(self) -> bool:
"""True when at least one focusing selector (not just pass-through) is set.""" """True when at least one focusing selector (not just pass-through) is set.
return bool(self.area or self.sub_area or self.keyword or self.last_failed)
Duration visibility (``durations`` / ``durations_min``) is reporting
only, not a selector, so it does not count as focus on its own.
"""
return bool(
self.area
or self.sub_area
or self.keyword
or self.last_failed
or self.fast
)
def build_marker_expression(area: str | None, sub_area: str | None) -> str | None: def build_marker_expression(
"""Build the ``-m`` marker expression from an area and/or sub-area. area: str | None, sub_area: str | None, fast: bool = False
) -> str | None:
"""Build the ``-m`` marker expression from area, sub-area, and the fast lane.
Returns ``None`` when neither is given so the caller can omit ``-m``. The fast lane adds ``not slow`` and composes with any area/sub-area with
``and``. Returns ``None`` when nothing is given so the caller can omit ``-m``.
""" """
parts: list[str] = [] parts: list[str] = []
if area: if area:
parts.append(f"area_{area}") parts.append(f"area_{area}")
if sub_area: if sub_area:
parts.append(f"sub_{sub_area}") parts.append(f"sub_{sub_area}")
if fast:
parts.append("not slow")
if not parts: if not parts:
return None return None
return " and ".join(parts) return " and ".join(parts)
@@ -125,13 +161,19 @@ def build_pytest_command(
invoked as ``.venv/bin/python tests/run_focus.py``). invoked as ``.venv/bin/python tests/run_focus.py``).
""" """
command = [python or sys.executable, "-m", "pytest"] command = [python or sys.executable, "-m", "pytest"]
marker_expression = build_marker_expression(selection.area, selection.sub_area) marker_expression = build_marker_expression(
selection.area, selection.sub_area, selection.fast
)
if marker_expression: if marker_expression:
command += ["-m", marker_expression] command += ["-m", marker_expression]
if selection.keyword: if selection.keyword:
command += ["-k", selection.keyword] command += ["-k", selection.keyword]
if selection.last_failed: if selection.last_failed:
command += ["--last-failed", "--last-failed-no-failures=none"] command += ["--last-failed", "--last-failed-no-failures=none"]
if selection.durations is not None:
command += [f"--durations={selection.durations}"]
if selection.durations_min is not None:
command += [f"--durations-min={selection.durations_min}"]
command += list(selection.pytest_args) command += list(selection.pytest_args)
return command return command
@@ -143,6 +185,9 @@ def selection_from_args(namespace: argparse.Namespace) -> FocusSelection:
sub_area=namespace.sub_area, sub_area=namespace.sub_area,
keyword=namespace.keyword, keyword=namespace.keyword,
last_failed=namespace.last_failed, last_failed=namespace.last_failed,
fast=namespace.fast,
durations=namespace.durations,
durations_min=namespace.durations_min,
pytest_args=tuple(namespace.pytest_args), pytest_args=tuple(namespace.pytest_args),
) )
@@ -185,6 +230,23 @@ def build_parser(
action="store_true", action="store_true",
help="re-run only tests that failed on the last run (pytest --last-failed)", help="re-run only tests that failed on the last run (pytest --last-failed)",
) )
parser.add_argument(
"--fast",
action="store_true",
help="fast lane: exclude tests marked slow (adds 'not slow'); composable with --area/--sub-area",
)
parser.add_argument(
"--durations",
type=non_negative_int,
metavar="N",
help="report the N slowest tests (pytest --durations=N, 0 shows all); not a focus selector",
)
parser.add_argument(
"--durations-min",
type=non_negative_float,
metavar="SECONDS",
help="minimum duration to report with --durations (pytest --durations-min)",
)
parser.add_argument( parser.add_argument(
"--dry-run", "--dry-run",
action="store_true", action="store_true",
@@ -215,7 +277,12 @@ def run(
if not selection.has_focus: if not selection.has_focus:
parser.error( parser.error(
"no focus selected: pass at least one of --area, --sub-area, " "no focus selected: pass at least one of --area, --sub-area, "
"--keyword, or --last-failed" "--keyword, --last-failed, or --fast (--durations is reporting only)"
)
if selection.durations_min is not None and selection.durations is None:
parser.error(
"--durations-min has no effect without --durations; pass "
"--durations N as well"
) )
command = build_pytest_command(selection) command = build_pytest_command(selection)
if namespace.dry_run: if namespace.dry_run:
+135
View File
@@ -47,6 +47,21 @@ def test_no_selection_marker_expression_is_none():
assert build_marker_expression(None, None) is None assert build_marker_expression(None, None) is None
def test_fast_only_marker_expression():
assert build_marker_expression(None, None, fast=True) == "not slow"
def test_fast_composes_with_area():
assert build_marker_expression("services", None, fast=True) == "area_services and not slow"
def test_fast_composes_with_area_and_sub_area():
assert (
build_marker_expression("services", "cookbook", fast=True)
== "area_services and sub_cookbook and not slow"
)
# --- command construction -------------------------------------------------- # --- command construction --------------------------------------------------
@@ -94,6 +109,47 @@ def test_default_python_is_current_interpreter():
assert command[0] == sys.executable assert command[0] == sys.executable
# --- fast lane and duration visibility -------------------------------------
def test_fast_only_command():
assert _cmd(fast=True) == [PY, "-m", "pytest", "-m", "not slow"]
def test_fast_with_area_command():
assert _cmd(area="services", fast=True) == [
PY, "-m", "pytest", "-m", "area_services and not slow",
]
def test_fast_with_area_and_sub_area_command():
assert _cmd(area="services", sub_area="cookbook", fast=True) == [
PY, "-m", "pytest", "-m", "area_services and sub_cookbook and not slow",
]
def test_durations_appends_flag():
assert _cmd(fast=True, durations=25) == [
PY, "-m", "pytest", "-m", "not slow", "--durations=25",
]
def test_durations_min_appends_flag():
assert _cmd(fast=True, durations=25, durations_min=0.05) == [
PY, "-m", "pytest", "-m", "not slow", "--durations=25", "--durations-min=0.05",
]
def test_durations_is_not_a_focus_selector():
assert FocusSelection(durations=25).has_focus is False
assert FocusSelection(fast=True).has_focus is True
def test_durations_kept_before_passthrough_args():
command = _cmd(fast=True, durations=25, pytest_args=("-q",))
assert command == [PY, "-m", "pytest", "-m", "not slow", "--durations=25", "-q"]
# --- sub-area normalization ------------------------------------------------ # --- sub-area normalization ------------------------------------------------
@@ -216,3 +272,82 @@ def test_no_focus_selector_is_rejected():
run(["--", "-q"], executor=executor) run(["--", "-q"], executor=executor)
assert excinfo.value.code == 2 assert excinfo.value.code == 2
assert executor.calls == [] assert executor.calls == []
def test_fast_run_invokes_executor_with_not_slow():
executor = _FakeExecutor()
run(["--fast"], executor=executor)
assert executor.calls == [[sys.executable, "-m", "pytest", "-m", "not slow"]]
def test_fast_with_durations_run_invokes_executor():
executor = _FakeExecutor()
run(["--area", "services", "--fast", "--durations", "25"], executor=executor)
assert executor.calls == [[
sys.executable,
"-m",
"pytest",
"-m",
"area_services and not slow",
"--durations=25",
]]
def test_fast_durations_dry_run_prints_command(capsys):
executor = _FakeExecutor()
code = run(["--dry-run", "--fast", "--durations", "25"], executor=executor)
out = capsys.readouterr().out
assert code == 0
assert executor.calls == []
assert out == f"{sys.executable} -m pytest -m 'not slow' --durations=25\n"
def test_durations_alone_is_rejected_before_executor():
executor = _FakeExecutor()
with pytest.raises(SystemExit) as excinfo:
run(["--durations", "25"], executor=executor)
assert excinfo.value.code == 2
assert executor.calls == []
def test_durations_zero_is_allowed_means_show_all():
executor = _FakeExecutor()
run(["--fast", "--durations", "0"], executor=executor)
assert executor.calls == [[
sys.executable, "-m", "pytest", "-m", "not slow", "--durations=0",
]]
@pytest.mark.parametrize("flag,value", [("--durations", "-1"), ("--durations-min", "-0.5")])
def test_negative_duration_values_are_rejected(flag, value):
executor = _FakeExecutor()
with pytest.raises(SystemExit) as excinfo:
run(["--fast", flag, value], executor=executor)
assert excinfo.value.code == 2
assert executor.calls == []
@pytest.mark.parametrize("argv", [
["--fast", "--durations-min", "0.05"],
["--area", "services", "--durations-min", "0.05"],
])
def test_durations_min_without_durations_is_rejected(argv):
executor = _FakeExecutor()
with pytest.raises(SystemExit) as excinfo:
run(argv, executor=executor)
assert excinfo.value.code == 2
assert executor.calls == []
def test_durations_min_with_durations_is_allowed():
executor = _FakeExecutor()
run(["--fast", "--durations", "25", "--durations-min", "0.05"], executor=executor)
assert executor.calls == [[
sys.executable,
"-m",
"pytest",
"-m",
"not slow",
"--durations=25",
"--durations-min=0.05",
]]