mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-06-16 09:45:24 -04:00
test: report cwd in order-sensitivity runner
This commit is contained in:
committed by
Kenny Van de Maele
parent
3372539e74
commit
b27b6fc0b6
+35
-5
@@ -97,11 +97,41 @@ python3 tests/run_order_report.py --seed 123 -- tests/cli/ -q
|
||||
python3 tests/run_order_report.py -- tests/cli/ -q # generates and prints a seed
|
||||
```
|
||||
|
||||
The same seed reproduces the same order, so to reproduce a failing run, re-run
|
||||
with the seed it printed (the runner prints an exact reproduction command at
|
||||
the top of each run). Failures discovered this way are real isolation bugs:
|
||||
fix them in separate scoped PRs - do not silence them with `skip`/`xfail`, and
|
||||
do not "fix" them by depending on a particular order.
|
||||
The same seed reproduces the same order when the reported working directory,
|
||||
pytest target arguments, and test environment are also the same. The runner
|
||||
prints all command arguments with shell-safe POSIX quoting and uses the
|
||||
invoking Python interpreter.
|
||||
|
||||
A generated-seed run starts with output like:
|
||||
|
||||
```text
|
||||
[order-report] working directory: /path/to/odysseus
|
||||
[order-report] shuffling test order with seed 284734921
|
||||
[order-report] reproduce from this working directory with the same test environment:
|
||||
[order-report] reproduce with: /path/to/odysseus/.venv/bin/python /path/to/odysseus/tests/run_order_report.py --seed 284734921 -- tests/cli/ -q
|
||||
```
|
||||
|
||||
Run the printed command from the reported working directory to reproduce the
|
||||
same fixed-seed order:
|
||||
|
||||
```text
|
||||
[order-report] working directory: /path/to/odysseus
|
||||
[order-report] shuffling test order with seed 284734921
|
||||
[order-report] reproduce from this working directory with the same test environment:
|
||||
[order-report] reproduce with: /path/to/odysseus/.venv/bin/python /path/to/odysseus/tests/run_order_report.py --seed 284734921 -- tests/cli/ -q
|
||||
```
|
||||
|
||||
Pytest output remains visible between the report header and footer. A failing
|
||||
run ends with pytest's normal failure report followed by:
|
||||
|
||||
```text
|
||||
FAILED tests/example_test.py::test_example - AssertionError
|
||||
[order-report] seed 284734921: pytest exit code 1 (report-only; fix order-sensitive failures in separate scoped PRs)
|
||||
```
|
||||
|
||||
Failures discovered this way are real isolation bugs: fix them in separate
|
||||
scoped PRs - do not silence them with `skip`/`xfail`, and do not "fix" them by
|
||||
depending on a particular order.
|
||||
|
||||
The runner propagates pytest's exit code, so it composes with normal local
|
||||
workflows; "report-only" means it is not a CI gate, not that failures are
|
||||
|
||||
@@ -16,7 +16,8 @@ Examples:
|
||||
|
||||
The shuffle is applied through a local ``pytest_collection_modifyitems`` hook
|
||||
passed to ``pytest.main`` as an in-process plugin; no conftest or global
|
||||
plugin is involved. The exit code is pytest's own.
|
||||
plugin is involved. Reproduction requires the reported working directory,
|
||||
seed, pytest arguments, and test environment. The exit code is pytest's own.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -105,7 +106,12 @@ def print_report_header(seed: int, pytest_args: Sequence[str]) -> None:
|
||||
"--",
|
||||
*pytest_args,
|
||||
]
|
||||
print(f"[order-report] working directory: {Path.cwd()}")
|
||||
print(f"[order-report] shuffling test order with seed {seed}")
|
||||
print(
|
||||
"[order-report] reproduce from this working directory with the same "
|
||||
"test environment:"
|
||||
)
|
||||
print(f"[order-report] reproduce with: {shlex.join(repro)}")
|
||||
|
||||
|
||||
|
||||
@@ -125,6 +125,13 @@ def test_explicit_seed_is_printed_with_repro_command(capsys):
|
||||
assert f"reproduce with: {repro}" in out
|
||||
|
||||
|
||||
def test_working_directory_is_reported(capsys, monkeypatch, tmp_path):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
run(["--seed", "123", "--", "-q"], pytest_main=_FakePytestMain())
|
||||
out = capsys.readouterr().out
|
||||
assert f"[order-report] working directory: {tmp_path}" in out
|
||||
|
||||
|
||||
def test_footer_repeats_seed_and_outcome(capsys):
|
||||
run(["--seed", "123", "--", "-q"], pytest_main=_FakePytestMain(returncode=1))
|
||||
out = capsys.readouterr().out
|
||||
|
||||
Reference in New Issue
Block a user