mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-06-15 17:25:26 -04:00
test: add fast lane and duration visibility (#3659)
This commit is contained in:
committed by
GitHub
parent
9e74a327f8
commit
cdfda4bd16
@@ -15,4 +15,8 @@ markers = [
|
||||
"area_helpers: self-tests for the shared test helpers in tests/helpers/",
|
||||
"area_unit: pure parser / utility tests that do not clearly belong elsewhere",
|
||||
"area_uncategorized: tests not yet matched by the taxonomy (fallback)",
|
||||
# Fast-lane marker (issue #3443). Opt-in and orthogonal to the area_*/sub_*
|
||||
# taxonomy. The fast lane runs `not slow`; mark a test slow only with
|
||||
# duration evidence (see tests/run_focus.py --durations and tests/README.md).
|
||||
"slow: opt-in marker for known-slow tests; excluded by the fast lane (not slow)",
|
||||
]
|
||||
|
||||
@@ -47,6 +47,35 @@ python3 tests/run_focus.py --dry-run --area services --sub-area cookbook
|
||||
python3 tests/run_focus.py --area services -- --maxfail=1 -q
|
||||
```
|
||||
|
||||
### Fast lane and duration visibility
|
||||
|
||||
`--fast` runs the fast lane: the tests that are *not* marked `slow` (it adds the
|
||||
marker expression `not slow`). It composes with `--area`/`--sub-area` using
|
||||
`and`. Because no tests may be marked `slow` yet, `--fast` can initially match
|
||||
the full focused selection; it becomes a real speed-up as `slow` marks are added
|
||||
from duration evidence. Use it for quick local or reviewer feedback; it does not
|
||||
replace broader focused or full-suite validation before merge.
|
||||
|
||||
`--durations N` and `--durations-min FLOAT` add pytest's slowest-test reporting
|
||||
so you can see where time goes. They are reporting only and do not count as a
|
||||
focus selector, so `--durations` must be combined with a real selector
|
||||
(`--area`, `--sub-area`, `--keyword`, `--last-failed`, or `--fast`).
|
||||
|
||||
Activate or otherwise use the project Python environment before running these
|
||||
commands. The examples use `python3` intentionally to avoid hard-coding a local
|
||||
venv path.
|
||||
|
||||
```bash
|
||||
python3 tests/run_focus.py --fast
|
||||
python3 tests/run_focus.py --area services --fast
|
||||
python3 tests/run_focus.py --area services --durations 25
|
||||
python3 tests/run_focus.py --area services --fast --durations 25 --durations-min 0.05
|
||||
```
|
||||
|
||||
The `slow` marker is opt-in. Mark a test `slow` only with duration evidence
|
||||
(from `--durations`), not by guessing - see the fast-lane policy in
|
||||
`TESTING_STANDARD.md`.
|
||||
|
||||
## Core principles
|
||||
|
||||
- Keep PRs small and homogeneous: one kind of change per PR.
|
||||
|
||||
@@ -74,6 +74,16 @@ A test that genuinely spans categories (e.g. a route test that also pins a
|
||||
security invariant) is classified by its **primary** assertion target and may be
|
||||
split if it grows.
|
||||
|
||||
## Fast lane policy
|
||||
|
||||
The fast lane is `not slow`: `tests/run_focus.py --fast` selects every test that
|
||||
is not marked `slow`. The `slow` marker is **opt-in**, and slow marks must be
|
||||
**evidence-driven from `--durations` output** - mark a test slow only when its
|
||||
measured duration shows it is genuinely expensive, never by guessing. The fast
|
||||
lane exists for quick local and reviewer feedback; it is **not** a replacement
|
||||
for broader focused or full-suite validation before merge, and a test must never
|
||||
be marked `slow` to hide a failure or skip coverage.
|
||||
|
||||
## Determinism & isolation rules
|
||||
|
||||
Do not mutate shared process state without a controlled helper and guaranteed
|
||||
|
||||
+74
-7
@@ -11,6 +11,8 @@ Examples:
|
||||
tests/run_focus.py --area security
|
||||
tests/run_focus.py --area services --sub-area cookbook
|
||||
tests/run_focus.py --keyword taxonomy -- --maxfail=1 -q
|
||||
tests/run_focus.py --fast
|
||||
tests/run_focus.py --area services --fast --durations 25
|
||||
|
||||
This script imports no production code and changes no test behavior. It only
|
||||
constructs and (optionally) executes a pytest invocation.
|
||||
@@ -70,6 +72,22 @@ def discover_sub_areas(tests_dir: Path = TESTS_DIR) -> frozenset[str]:
|
||||
)
|
||||
|
||||
|
||||
def non_negative_int(value: str) -> int:
|
||||
"""argparse type: a non-negative int (0 means "show all" for --durations)."""
|
||||
number = int(value)
|
||||
if number < 0:
|
||||
raise argparse.ArgumentTypeError(f"must be >= 0, got {value!r}")
|
||||
return number
|
||||
|
||||
|
||||
def non_negative_float(value: str) -> float:
|
||||
"""argparse type: a non-negative float (seconds threshold for --durations-min)."""
|
||||
number = float(value)
|
||||
if number < 0:
|
||||
raise argparse.ArgumentTypeError(f"must be >= 0, got {value!r}")
|
||||
return number
|
||||
|
||||
|
||||
def sub_area_type(valid_sub_areas: frozenset[str]) -> Callable[[str], str]:
|
||||
"""Build an argparse converter that accepts only discovered sub-areas."""
|
||||
|
||||
@@ -92,24 +110,42 @@ class FocusSelection:
|
||||
sub_area: str | None = None
|
||||
keyword: str | None = None
|
||||
last_failed: bool = False
|
||||
fast: bool = False
|
||||
durations: int | None = None
|
||||
durations_min: float | None = None
|
||||
pytest_args: tuple[str, ...] = field(default_factory=tuple)
|
||||
|
||||
@property
|
||||
def has_focus(self) -> bool:
|
||||
"""True when at least one focusing selector (not just pass-through) is set."""
|
||||
return bool(self.area or self.sub_area or self.keyword or self.last_failed)
|
||||
"""True when at least one focusing selector (not just pass-through) is set.
|
||||
|
||||
Duration visibility (``durations`` / ``durations_min``) is reporting
|
||||
only, not a selector, so it does not count as focus on its own.
|
||||
"""
|
||||
return bool(
|
||||
self.area
|
||||
or self.sub_area
|
||||
or self.keyword
|
||||
or self.last_failed
|
||||
or self.fast
|
||||
)
|
||||
|
||||
|
||||
def build_marker_expression(area: str | None, sub_area: str | None) -> str | None:
|
||||
"""Build the ``-m`` marker expression from an area and/or sub-area.
|
||||
def build_marker_expression(
|
||||
area: str | None, sub_area: str | None, fast: bool = False
|
||||
) -> str | None:
|
||||
"""Build the ``-m`` marker expression from area, sub-area, and the fast lane.
|
||||
|
||||
Returns ``None`` when neither is given so the caller can omit ``-m``.
|
||||
The fast lane adds ``not slow`` and composes with any area/sub-area with
|
||||
``and``. Returns ``None`` when nothing is given so the caller can omit ``-m``.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
if area:
|
||||
parts.append(f"area_{area}")
|
||||
if sub_area:
|
||||
parts.append(f"sub_{sub_area}")
|
||||
if fast:
|
||||
parts.append("not slow")
|
||||
if not parts:
|
||||
return None
|
||||
return " and ".join(parts)
|
||||
@@ -125,13 +161,19 @@ def build_pytest_command(
|
||||
invoked as ``.venv/bin/python tests/run_focus.py``).
|
||||
"""
|
||||
command = [python or sys.executable, "-m", "pytest"]
|
||||
marker_expression = build_marker_expression(selection.area, selection.sub_area)
|
||||
marker_expression = build_marker_expression(
|
||||
selection.area, selection.sub_area, selection.fast
|
||||
)
|
||||
if marker_expression:
|
||||
command += ["-m", marker_expression]
|
||||
if selection.keyword:
|
||||
command += ["-k", selection.keyword]
|
||||
if selection.last_failed:
|
||||
command += ["--last-failed", "--last-failed-no-failures=none"]
|
||||
if selection.durations is not None:
|
||||
command += [f"--durations={selection.durations}"]
|
||||
if selection.durations_min is not None:
|
||||
command += [f"--durations-min={selection.durations_min}"]
|
||||
command += list(selection.pytest_args)
|
||||
return command
|
||||
|
||||
@@ -143,6 +185,9 @@ def selection_from_args(namespace: argparse.Namespace) -> FocusSelection:
|
||||
sub_area=namespace.sub_area,
|
||||
keyword=namespace.keyword,
|
||||
last_failed=namespace.last_failed,
|
||||
fast=namespace.fast,
|
||||
durations=namespace.durations,
|
||||
durations_min=namespace.durations_min,
|
||||
pytest_args=tuple(namespace.pytest_args),
|
||||
)
|
||||
|
||||
@@ -185,6 +230,23 @@ def build_parser(
|
||||
action="store_true",
|
||||
help="re-run only tests that failed on the last run (pytest --last-failed)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fast",
|
||||
action="store_true",
|
||||
help="fast lane: exclude tests marked slow (adds 'not slow'); composable with --area/--sub-area",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--durations",
|
||||
type=non_negative_int,
|
||||
metavar="N",
|
||||
help="report the N slowest tests (pytest --durations=N, 0 shows all); not a focus selector",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--durations-min",
|
||||
type=non_negative_float,
|
||||
metavar="SECONDS",
|
||||
help="minimum duration to report with --durations (pytest --durations-min)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
@@ -215,7 +277,12 @@ def run(
|
||||
if not selection.has_focus:
|
||||
parser.error(
|
||||
"no focus selected: pass at least one of --area, --sub-area, "
|
||||
"--keyword, or --last-failed"
|
||||
"--keyword, --last-failed, or --fast (--durations is reporting only)"
|
||||
)
|
||||
if selection.durations_min is not None and selection.durations is None:
|
||||
parser.error(
|
||||
"--durations-min has no effect without --durations; pass "
|
||||
"--durations N as well"
|
||||
)
|
||||
command = build_pytest_command(selection)
|
||||
if namespace.dry_run:
|
||||
|
||||
@@ -47,6 +47,21 @@ def test_no_selection_marker_expression_is_none():
|
||||
assert build_marker_expression(None, None) is None
|
||||
|
||||
|
||||
def test_fast_only_marker_expression():
|
||||
assert build_marker_expression(None, None, fast=True) == "not slow"
|
||||
|
||||
|
||||
def test_fast_composes_with_area():
|
||||
assert build_marker_expression("services", None, fast=True) == "area_services and not slow"
|
||||
|
||||
|
||||
def test_fast_composes_with_area_and_sub_area():
|
||||
assert (
|
||||
build_marker_expression("services", "cookbook", fast=True)
|
||||
== "area_services and sub_cookbook and not slow"
|
||||
)
|
||||
|
||||
|
||||
# --- command construction --------------------------------------------------
|
||||
|
||||
|
||||
@@ -94,6 +109,47 @@ def test_default_python_is_current_interpreter():
|
||||
assert command[0] == sys.executable
|
||||
|
||||
|
||||
# --- fast lane and duration visibility -------------------------------------
|
||||
|
||||
|
||||
def test_fast_only_command():
|
||||
assert _cmd(fast=True) == [PY, "-m", "pytest", "-m", "not slow"]
|
||||
|
||||
|
||||
def test_fast_with_area_command():
|
||||
assert _cmd(area="services", fast=True) == [
|
||||
PY, "-m", "pytest", "-m", "area_services and not slow",
|
||||
]
|
||||
|
||||
|
||||
def test_fast_with_area_and_sub_area_command():
|
||||
assert _cmd(area="services", sub_area="cookbook", fast=True) == [
|
||||
PY, "-m", "pytest", "-m", "area_services and sub_cookbook and not slow",
|
||||
]
|
||||
|
||||
|
||||
def test_durations_appends_flag():
|
||||
assert _cmd(fast=True, durations=25) == [
|
||||
PY, "-m", "pytest", "-m", "not slow", "--durations=25",
|
||||
]
|
||||
|
||||
|
||||
def test_durations_min_appends_flag():
|
||||
assert _cmd(fast=True, durations=25, durations_min=0.05) == [
|
||||
PY, "-m", "pytest", "-m", "not slow", "--durations=25", "--durations-min=0.05",
|
||||
]
|
||||
|
||||
|
||||
def test_durations_is_not_a_focus_selector():
|
||||
assert FocusSelection(durations=25).has_focus is False
|
||||
assert FocusSelection(fast=True).has_focus is True
|
||||
|
||||
|
||||
def test_durations_kept_before_passthrough_args():
|
||||
command = _cmd(fast=True, durations=25, pytest_args=("-q",))
|
||||
assert command == [PY, "-m", "pytest", "-m", "not slow", "--durations=25", "-q"]
|
||||
|
||||
|
||||
# --- sub-area normalization ------------------------------------------------
|
||||
|
||||
|
||||
@@ -216,3 +272,82 @@ def test_no_focus_selector_is_rejected():
|
||||
run(["--", "-q"], executor=executor)
|
||||
assert excinfo.value.code == 2
|
||||
assert executor.calls == []
|
||||
|
||||
|
||||
def test_fast_run_invokes_executor_with_not_slow():
|
||||
executor = _FakeExecutor()
|
||||
run(["--fast"], executor=executor)
|
||||
assert executor.calls == [[sys.executable, "-m", "pytest", "-m", "not slow"]]
|
||||
|
||||
|
||||
def test_fast_with_durations_run_invokes_executor():
|
||||
executor = _FakeExecutor()
|
||||
run(["--area", "services", "--fast", "--durations", "25"], executor=executor)
|
||||
assert executor.calls == [[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pytest",
|
||||
"-m",
|
||||
"area_services and not slow",
|
||||
"--durations=25",
|
||||
]]
|
||||
|
||||
|
||||
def test_fast_durations_dry_run_prints_command(capsys):
|
||||
executor = _FakeExecutor()
|
||||
code = run(["--dry-run", "--fast", "--durations", "25"], executor=executor)
|
||||
out = capsys.readouterr().out
|
||||
assert code == 0
|
||||
assert executor.calls == []
|
||||
assert out == f"{sys.executable} -m pytest -m 'not slow' --durations=25\n"
|
||||
|
||||
|
||||
def test_durations_alone_is_rejected_before_executor():
|
||||
executor = _FakeExecutor()
|
||||
with pytest.raises(SystemExit) as excinfo:
|
||||
run(["--durations", "25"], executor=executor)
|
||||
assert excinfo.value.code == 2
|
||||
assert executor.calls == []
|
||||
|
||||
|
||||
def test_durations_zero_is_allowed_means_show_all():
|
||||
executor = _FakeExecutor()
|
||||
run(["--fast", "--durations", "0"], executor=executor)
|
||||
assert executor.calls == [[
|
||||
sys.executable, "-m", "pytest", "-m", "not slow", "--durations=0",
|
||||
]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag,value", [("--durations", "-1"), ("--durations-min", "-0.5")])
|
||||
def test_negative_duration_values_are_rejected(flag, value):
|
||||
executor = _FakeExecutor()
|
||||
with pytest.raises(SystemExit) as excinfo:
|
||||
run(["--fast", flag, value], executor=executor)
|
||||
assert excinfo.value.code == 2
|
||||
assert executor.calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("argv", [
|
||||
["--fast", "--durations-min", "0.05"],
|
||||
["--area", "services", "--durations-min", "0.05"],
|
||||
])
|
||||
def test_durations_min_without_durations_is_rejected(argv):
|
||||
executor = _FakeExecutor()
|
||||
with pytest.raises(SystemExit) as excinfo:
|
||||
run(argv, executor=executor)
|
||||
assert excinfo.value.code == 2
|
||||
assert executor.calls == []
|
||||
|
||||
|
||||
def test_durations_min_with_durations_is_allowed():
|
||||
executor = _FakeExecutor()
|
||||
run(["--fast", "--durations", "25", "--durations-min", "0.05"], executor=executor)
|
||||
assert executor.calls == [[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pytest",
|
||||
"-m",
|
||||
"not slow",
|
||||
"--durations=25",
|
||||
"--durations-min=0.05",
|
||||
]]
|
||||
|
||||
Reference in New Issue
Block a user