mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-06-17 10:15:27 -04:00
test: report cwd in order-sensitivity runner
This commit is contained in:
committed by
Kenny Van de Maele
parent
3372539e74
commit
b27b6fc0b6
+35
-5
@@ -97,11 +97,41 @@ python3 tests/run_order_report.py --seed 123 -- tests/cli/ -q
|
|||||||
python3 tests/run_order_report.py -- tests/cli/ -q # generates and prints a seed
|
python3 tests/run_order_report.py -- tests/cli/ -q # generates and prints a seed
|
||||||
```
|
```
|
||||||
|
|
||||||
The same seed reproduces the same order, so to reproduce a failing run, re-run
|
The same seed reproduces the same order when the reported working directory,
|
||||||
with the seed it printed (the runner prints an exact reproduction command at
|
pytest target arguments, and test environment are also the same. The runner
|
||||||
the top of each run). Failures discovered this way are real isolation bugs:
|
prints all command arguments with shell-safe POSIX quoting and uses the
|
||||||
fix them in separate scoped PRs - do not silence them with `skip`/`xfail`, and
|
invoking Python interpreter.
|
||||||
do not "fix" them by depending on a particular order.
|
|
||||||
|
A generated-seed run starts with output like:
|
||||||
|
|
||||||
|
```text
|
||||||
|
[order-report] working directory: /path/to/odysseus
|
||||||
|
[order-report] shuffling test order with seed 284734921
|
||||||
|
[order-report] reproduce from this working directory with the same test environment:
|
||||||
|
[order-report] reproduce with: /path/to/odysseus/.venv/bin/python /path/to/odysseus/tests/run_order_report.py --seed 284734921 -- tests/cli/ -q
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the printed command from the reported working directory to reproduce the
|
||||||
|
same fixed-seed order:
|
||||||
|
|
||||||
|
```text
|
||||||
|
[order-report] working directory: /path/to/odysseus
|
||||||
|
[order-report] shuffling test order with seed 284734921
|
||||||
|
[order-report] reproduce from this working directory with the same test environment:
|
||||||
|
[order-report] reproduce with: /path/to/odysseus/.venv/bin/python /path/to/odysseus/tests/run_order_report.py --seed 284734921 -- tests/cli/ -q
|
||||||
|
```
|
||||||
|
|
||||||
|
Pytest output remains visible between the report header and footer. A failing
|
||||||
|
run ends with pytest's normal failure report followed by:
|
||||||
|
|
||||||
|
```text
|
||||||
|
FAILED tests/example_test.py::test_example - AssertionError
|
||||||
|
[order-report] seed 284734921: pytest exit code 1 (report-only; fix order-sensitive failures in separate scoped PRs)
|
||||||
|
```
|
||||||
|
|
||||||
|
Failures discovered this way are real isolation bugs: fix them in separate
|
||||||
|
scoped PRs - do not silence them with `skip`/`xfail`, and do not "fix" them by
|
||||||
|
depending on a particular order.
|
||||||
|
|
||||||
The runner propagates pytest's exit code, so it composes with normal local
|
The runner propagates pytest's exit code, so it composes with normal local
|
||||||
workflows; "report-only" means it is not a CI gate, not that failures are
|
workflows; "report-only" means it is not a CI gate, not that failures are
|
||||||
|
|||||||
@@ -16,7 +16,8 @@ Examples:
|
|||||||
|
|
||||||
The shuffle is applied through a local ``pytest_collection_modifyitems`` hook
|
The shuffle is applied through a local ``pytest_collection_modifyitems`` hook
|
||||||
passed to ``pytest.main`` as an in-process plugin; no conftest or global
|
passed to ``pytest.main`` as an in-process plugin; no conftest or global
|
||||||
plugin is involved. The exit code is pytest's own.
|
plugin is involved. Reproduction requires the reported working directory,
|
||||||
|
seed, pytest arguments, and test environment. The exit code is pytest's own.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -105,7 +106,12 @@ def print_report_header(seed: int, pytest_args: Sequence[str]) -> None:
|
|||||||
"--",
|
"--",
|
||||||
*pytest_args,
|
*pytest_args,
|
||||||
]
|
]
|
||||||
|
print(f"[order-report] working directory: {Path.cwd()}")
|
||||||
print(f"[order-report] shuffling test order with seed {seed}")
|
print(f"[order-report] shuffling test order with seed {seed}")
|
||||||
|
print(
|
||||||
|
"[order-report] reproduce from this working directory with the same "
|
||||||
|
"test environment:"
|
||||||
|
)
|
||||||
print(f"[order-report] reproduce with: {shlex.join(repro)}")
|
print(f"[order-report] reproduce with: {shlex.join(repro)}")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -125,6 +125,13 @@ def test_explicit_seed_is_printed_with_repro_command(capsys):
|
|||||||
assert f"reproduce with: {repro}" in out
|
assert f"reproduce with: {repro}" in out
|
||||||
|
|
||||||
|
|
||||||
|
def test_working_directory_is_reported(capsys, monkeypatch, tmp_path):
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
run(["--seed", "123", "--", "-q"], pytest_main=_FakePytestMain())
|
||||||
|
out = capsys.readouterr().out
|
||||||
|
assert f"[order-report] working directory: {tmp_path}" in out
|
||||||
|
|
||||||
|
|
||||||
def test_footer_repeats_seed_and_outcome(capsys):
|
def test_footer_repeats_seed_and_outcome(capsys):
|
||||||
run(["--seed", "123", "--", "-q"], pytest_main=_FakePytestMain(returncode=1))
|
run(["--seed", "123", "--", "-q"], pytest_main=_FakePytestMain(returncode=1))
|
||||||
out = capsys.readouterr().out
|
out = capsys.readouterr().out
|
||||||
|
|||||||
Reference in New Issue
Block a user