test: add fast lane and duration visibility (#3659)

This commit is contained in:
Alexandre Teixeira
2026-06-09 19:11:47 +01:00
committed by GitHub
parent 9e74a327f8
commit cdfda4bd16
5 changed files with 252 additions and 7 deletions
+4
View File
@@ -15,4 +15,8 @@ markers = [
"area_helpers: self-tests for the shared test helpers in tests/helpers/",
"area_unit: pure parser / utility tests that do not clearly belong elsewhere",
"area_uncategorized: tests not yet matched by the taxonomy (fallback)",
# Fast-lane marker (issue #3443). Opt-in and orthogonal to the area_*/sub_*
# taxonomy. The fast lane runs `not slow`; mark a test slow only with
# duration evidence (see tests/run_focus.py --durations and tests/README.md).
"slow: opt-in marker for known-slow tests; excluded by the fast lane (not slow)",
]
+29
View File
@@ -47,6 +47,35 @@ python3 tests/run_focus.py --dry-run --area services --sub-area cookbook
python3 tests/run_focus.py --area services -- --maxfail=1 -q
```
### Fast lane and duration visibility
`--fast` runs the fast lane: the tests that are *not* marked `slow` (it adds the
marker expression `not slow`). It composes with `--area`/`--sub-area` using
`and`. Because no tests may be marked `slow` yet, `--fast` can initially match
the full focused selection; it becomes a real speed-up as `slow` marks are added
from duration evidence. Use it for quick local or reviewer feedback; it does not
replace broader focused or full-suite validation before merge.
`--durations N` and `--durations-min FLOAT` add pytest's slowest-test reporting
so you can see where time goes. They are reporting only and do not count as a
focus selector, so `--durations` must be combined with a real selector
(`--area`, `--sub-area`, `--keyword`, `--last-failed`, or `--fast`).
Activate or otherwise use the project Python environment before running these
commands. The examples use `python3` intentionally to avoid hard-coding a local
venv path.
```bash
python3 tests/run_focus.py --fast
python3 tests/run_focus.py --area services --fast
python3 tests/run_focus.py --area services --durations 25
python3 tests/run_focus.py --area services --fast --durations 25 --durations-min 0.05
```
The `slow` marker is opt-in. Mark a test `slow` only with duration evidence
(from `--durations`), not by guessing - see the fast-lane policy in
`TESTING_STANDARD.md`.
## Core principles
- Keep PRs small and homogeneous: one kind of change per PR.
+10
View File
@@ -74,6 +74,16 @@ A test that genuinely spans categories (e.g. a route test that also pins a
security invariant) is classified by its **primary** assertion target and may be
split if it grows.
## Fast lane policy
The fast lane is `not slow`: `tests/run_focus.py --fast` selects every test that
is not marked `slow`. The `slow` marker is **opt-in**, and slow marks must be
**evidence-driven from `--durations` output** - mark a test slow only when its
measured duration shows it is genuinely expensive, never by guessing. The fast
lane exists for quick local and reviewer feedback; it is **not** a replacement
for broader focused or full-suite validation before merge, and a test must never
be marked `slow` to hide a failure or skip coverage.
## Determinism & isolation rules
Do not mutate shared process state without a controlled helper and guaranteed
+74 -7
View File
@@ -11,6 +11,8 @@ Examples:
tests/run_focus.py --area security
tests/run_focus.py --area services --sub-area cookbook
tests/run_focus.py --keyword taxonomy -- --maxfail=1 -q
tests/run_focus.py --fast
tests/run_focus.py --area services --fast --durations 25
This script imports no production code and changes no test behavior. It only
constructs and (optionally) executes a pytest invocation.
@@ -70,6 +72,22 @@ def discover_sub_areas(tests_dir: Path = TESTS_DIR) -> frozenset[str]:
)
def non_negative_int(value: str) -> int:
"""argparse type: a non-negative int (0 means "show all" for --durations)."""
number = int(value)
if number < 0:
raise argparse.ArgumentTypeError(f"must be >= 0, got {value!r}")
return number
def non_negative_float(value: str) -> float:
"""argparse type: a non-negative float (seconds threshold for --durations-min)."""
number = float(value)
if number < 0:
raise argparse.ArgumentTypeError(f"must be >= 0, got {value!r}")
return number
def sub_area_type(valid_sub_areas: frozenset[str]) -> Callable[[str], str]:
"""Build an argparse converter that accepts only discovered sub-areas."""
@@ -92,24 +110,42 @@ class FocusSelection:
sub_area: str | None = None
keyword: str | None = None
last_failed: bool = False
fast: bool = False
durations: int | None = None
durations_min: float | None = None
pytest_args: tuple[str, ...] = field(default_factory=tuple)
@property
def has_focus(self) -> bool:
"""True when at least one focusing selector (not just pass-through) is set."""
return bool(self.area or self.sub_area or self.keyword or self.last_failed)
"""True when at least one focusing selector (not just pass-through) is set.
Duration visibility (``durations`` / ``durations_min``) is reporting
only, not a selector, so it does not count as focus on its own.
"""
return bool(
self.area
or self.sub_area
or self.keyword
or self.last_failed
or self.fast
)
def build_marker_expression(area: str | None, sub_area: str | None) -> str | None:
"""Build the ``-m`` marker expression from an area and/or sub-area.
def build_marker_expression(
area: str | None, sub_area: str | None, fast: bool = False
) -> str | None:
"""Build the ``-m`` marker expression from area, sub-area, and the fast lane.
Returns ``None`` when neither is given so the caller can omit ``-m``.
The fast lane adds ``not slow`` and composes with any area/sub-area with
``and``. Returns ``None`` when nothing is given so the caller can omit ``-m``.
"""
parts: list[str] = []
if area:
parts.append(f"area_{area}")
if sub_area:
parts.append(f"sub_{sub_area}")
if fast:
parts.append("not slow")
if not parts:
return None
return " and ".join(parts)
@@ -125,13 +161,19 @@ def build_pytest_command(
invoked as ``.venv/bin/python tests/run_focus.py``).
"""
command = [python or sys.executable, "-m", "pytest"]
marker_expression = build_marker_expression(selection.area, selection.sub_area)
marker_expression = build_marker_expression(
selection.area, selection.sub_area, selection.fast
)
if marker_expression:
command += ["-m", marker_expression]
if selection.keyword:
command += ["-k", selection.keyword]
if selection.last_failed:
command += ["--last-failed", "--last-failed-no-failures=none"]
if selection.durations is not None:
command += [f"--durations={selection.durations}"]
if selection.durations_min is not None:
command += [f"--durations-min={selection.durations_min}"]
command += list(selection.pytest_args)
return command
@@ -143,6 +185,9 @@ def selection_from_args(namespace: argparse.Namespace) -> FocusSelection:
sub_area=namespace.sub_area,
keyword=namespace.keyword,
last_failed=namespace.last_failed,
fast=namespace.fast,
durations=namespace.durations,
durations_min=namespace.durations_min,
pytest_args=tuple(namespace.pytest_args),
)
@@ -185,6 +230,23 @@ def build_parser(
action="store_true",
help="re-run only tests that failed on the last run (pytest --last-failed)",
)
parser.add_argument(
"--fast",
action="store_true",
help="fast lane: exclude tests marked slow (adds 'not slow'); composable with --area/--sub-area",
)
parser.add_argument(
"--durations",
type=non_negative_int,
metavar="N",
help="report the N slowest tests (pytest --durations=N, 0 shows all); not a focus selector",
)
parser.add_argument(
"--durations-min",
type=non_negative_float,
metavar="SECONDS",
help="minimum duration to report with --durations (pytest --durations-min)",
)
parser.add_argument(
"--dry-run",
action="store_true",
@@ -215,7 +277,12 @@ def run(
if not selection.has_focus:
parser.error(
"no focus selected: pass at least one of --area, --sub-area, "
"--keyword, or --last-failed"
"--keyword, --last-failed, or --fast (--durations is reporting only)"
)
if selection.durations_min is not None and selection.durations is None:
parser.error(
"--durations-min has no effect without --durations; pass "
"--durations N as well"
)
command = build_pytest_command(selection)
if namespace.dry_run:
+135
View File
@@ -47,6 +47,21 @@ def test_no_selection_marker_expression_is_none():
assert build_marker_expression(None, None) is None
def test_fast_only_marker_expression():
assert build_marker_expression(None, None, fast=True) == "not slow"
def test_fast_composes_with_area():
assert build_marker_expression("services", None, fast=True) == "area_services and not slow"
def test_fast_composes_with_area_and_sub_area():
assert (
build_marker_expression("services", "cookbook", fast=True)
== "area_services and sub_cookbook and not slow"
)
# --- command construction --------------------------------------------------
@@ -94,6 +109,47 @@ def test_default_python_is_current_interpreter():
assert command[0] == sys.executable
# --- fast lane and duration visibility -------------------------------------
def test_fast_only_command():
assert _cmd(fast=True) == [PY, "-m", "pytest", "-m", "not slow"]
def test_fast_with_area_command():
assert _cmd(area="services", fast=True) == [
PY, "-m", "pytest", "-m", "area_services and not slow",
]
def test_fast_with_area_and_sub_area_command():
assert _cmd(area="services", sub_area="cookbook", fast=True) == [
PY, "-m", "pytest", "-m", "area_services and sub_cookbook and not slow",
]
def test_durations_appends_flag():
assert _cmd(fast=True, durations=25) == [
PY, "-m", "pytest", "-m", "not slow", "--durations=25",
]
def test_durations_min_appends_flag():
assert _cmd(fast=True, durations=25, durations_min=0.05) == [
PY, "-m", "pytest", "-m", "not slow", "--durations=25", "--durations-min=0.05",
]
def test_durations_is_not_a_focus_selector():
assert FocusSelection(durations=25).has_focus is False
assert FocusSelection(fast=True).has_focus is True
def test_durations_kept_before_passthrough_args():
command = _cmd(fast=True, durations=25, pytest_args=("-q",))
assert command == [PY, "-m", "pytest", "-m", "not slow", "--durations=25", "-q"]
# --- sub-area normalization ------------------------------------------------
@@ -216,3 +272,82 @@ def test_no_focus_selector_is_rejected():
run(["--", "-q"], executor=executor)
assert excinfo.value.code == 2
assert executor.calls == []
def test_fast_run_invokes_executor_with_not_slow():
executor = _FakeExecutor()
run(["--fast"], executor=executor)
assert executor.calls == [[sys.executable, "-m", "pytest", "-m", "not slow"]]
def test_fast_with_durations_run_invokes_executor():
executor = _FakeExecutor()
run(["--area", "services", "--fast", "--durations", "25"], executor=executor)
assert executor.calls == [[
sys.executable,
"-m",
"pytest",
"-m",
"area_services and not slow",
"--durations=25",
]]
def test_fast_durations_dry_run_prints_command(capsys):
executor = _FakeExecutor()
code = run(["--dry-run", "--fast", "--durations", "25"], executor=executor)
out = capsys.readouterr().out
assert code == 0
assert executor.calls == []
assert out == f"{sys.executable} -m pytest -m 'not slow' --durations=25\n"
def test_durations_alone_is_rejected_before_executor():
executor = _FakeExecutor()
with pytest.raises(SystemExit) as excinfo:
run(["--durations", "25"], executor=executor)
assert excinfo.value.code == 2
assert executor.calls == []
def test_durations_zero_is_allowed_means_show_all():
executor = _FakeExecutor()
run(["--fast", "--durations", "0"], executor=executor)
assert executor.calls == [[
sys.executable, "-m", "pytest", "-m", "not slow", "--durations=0",
]]
@pytest.mark.parametrize("flag,value", [("--durations", "-1"), ("--durations-min", "-0.5")])
def test_negative_duration_values_are_rejected(flag, value):
executor = _FakeExecutor()
with pytest.raises(SystemExit) as excinfo:
run(["--fast", flag, value], executor=executor)
assert excinfo.value.code == 2
assert executor.calls == []
@pytest.mark.parametrize("argv", [
["--fast", "--durations-min", "0.05"],
["--area", "services", "--durations-min", "0.05"],
])
def test_durations_min_without_durations_is_rejected(argv):
executor = _FakeExecutor()
with pytest.raises(SystemExit) as excinfo:
run(argv, executor=executor)
assert excinfo.value.code == 2
assert executor.calls == []
def test_durations_min_with_durations_is_allowed():
executor = _FakeExecutor()
run(["--fast", "--durations", "25", "--durations-min", "0.05"], executor=executor)
assert executor.calls == [[
sys.executable,
"-m",
"pytest",
"-m",
"not slow",
"--durations=25",
"--durations-min=0.05",
]]