test: add report-only order-sensitivity runner (#3982)

* test: add report-only order-sensitivity runner

* test: report cwd in order-sensitivity runner

tests/test_run_order_report.py

@@ -0,0 +1,245 @@
+"""Direct tests for the order-sensitivity report runner (tests/run_order_report.py).
+
+The shuffle and argument plumbing are tested without spawning pytest: the
+shuffle helpers are asserted directly and ``run`` is exercised with an
+injected fake ``pytest.main``. A small subprocess test then proves the seed is
+applied end to end (reproducible, seed visible) against a throwaway test file,
+never the real suite.
+"""
+from __future__ import annotations
+
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from tests.run_order_report import (
+    SEED_MAX,
+    OrderShuffle,
+    generate_seed,
+    run,
+    shuffle_items,
+)
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+RUNNER = REPO_ROOT / "tests" / "run_order_report.py"
+
+
+class _FakePytestMain:
+    """Records forwarded args and plugins and returns a fixed exit code."""
+
+    def __init__(self, returncode: int = 0):
+        self.returncode = returncode
+        self.calls: list[tuple[list[str], list]] = []
+
+    def __call__(self, args: list[str], plugins: list) -> int:
+        self.calls.append((list(args), list(plugins)))
+        return self.returncode
+
+
+# --- shuffle determinism -----------------------------------------------------
+
+
+def test_same_seed_shuffles_identically():
+    first = list(range(20))
+    second = list(range(20))
+    shuffle_items(first, seed=123)
+    shuffle_items(second, seed=123)
+    assert first == second
+
+
+def test_different_seeds_shuffle_differently():
+    first = list(range(20))
+    second = list(range(20))
+    shuffle_items(first, seed=123)
+    shuffle_items(second, seed=321)
+    assert first != second
+
+
+def test_shuffle_preserves_items():
+    items = list(range(20))
+    shuffle_items(items, seed=123)
+    assert sorted(items) == list(range(20))
+
+
+def test_plugin_hook_matches_shuffle_items():
+    hooked = list(range(20))
+    expected = list(range(20))
+    OrderShuffle(seed=7).pytest_collection_modifyitems(hooked)
+    shuffle_items(expected, seed=7)
+    assert hooked == expected
+
+
+# --- argument parsing and pytest invocation ----------------------------------
+
+
+def test_pytest_args_after_separator_are_forwarded():
+    fake = _FakePytestMain()
+    run(["--seed", "123", "--", "tests/cli/", "-q"], pytest_main=fake)
+    (args, plugins), = fake.calls
+    assert args == ["tests/cli/", "-q"]
+    assert [type(p) for p in plugins] == [OrderShuffle]
+
+
+def test_explicit_seed_reaches_plugin():
+    fake = _FakePytestMain()
+    run(["--seed", "123", "--", "-q"], pytest_main=fake)
+    (_, plugins), = fake.calls
+    assert plugins[0].seed == 123
+
+
+def test_pytest_exit_code_is_propagated():
+    fake = _FakePytestMain(returncode=3)
+    assert run(["--seed", "123", "--", "-q"], pytest_main=fake) == 3
+
+
+@pytest.mark.parametrize("value", ["abc", "-1", str(SEED_MAX + 1)])
+def test_invalid_seed_is_rejected_before_pytest(value):
+    fake = _FakePytestMain()
+    with pytest.raises(SystemExit) as excinfo:
+        run(["--seed", value, "--", "-q"], pytest_main=fake)
+    assert excinfo.value.code == 2
+    assert fake.calls == []
+
+
+# --- seed reporting -----------------------------------------------------------
+
+
+def test_explicit_seed_is_printed_with_repro_command(capsys):
+    run(["--seed", "123", "--", "tests/cli/", "-q"], pytest_main=_FakePytestMain())
+    out = capsys.readouterr().out
+    assert "[order-report] shuffling test order with seed 123" in out
+    repro = shlex.join(
+        [
+            sys.executable,
+            str(RUNNER),
+            "--seed",
+            "123",
+            "--",
+            "tests/cli/",
+            "-q",
+        ]
+    )
+    assert f"reproduce with: {repro}" in out
+
+
+def test_working_directory_is_reported(capsys, monkeypatch, tmp_path):
+    monkeypatch.chdir(tmp_path)
+    run(["--seed", "123", "--", "-q"], pytest_main=_FakePytestMain())
+    out = capsys.readouterr().out
+    assert f"[order-report] working directory: {tmp_path}" in out
+
+
+def test_footer_repeats_seed_and_outcome(capsys):
+    run(["--seed", "123", "--", "-q"], pytest_main=_FakePytestMain(returncode=1))
+    out = capsys.readouterr().out
+    assert "[order-report] seed 123: pytest exit code 1" in out
+
+
+def test_generated_seed_is_printed_and_used(capsys):
+    fake = _FakePytestMain()
+    run(["--", "-q"], pytest_main=fake)
+    out = capsys.readouterr().out
+    seed_line = next(line for line in out.splitlines() if "with seed" in line)
+    seed = int(seed_line.rsplit("seed ", 1)[1])
+    assert 0 <= seed <= SEED_MAX
+    (_, plugins), = fake.calls
+    assert plugins[0].seed == seed
+
+
+def test_generate_seed_is_within_range():
+    assert all(0 <= generate_seed() <= SEED_MAX for _ in range(5))
+
+
+# --- end-to-end: the seed really drives collection order (real subprocess) ---
+
+_SAMPLE_TESTS = "".join(
+    f"def test_{name}():\n    pass\n\n"
+    for name in ("alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel")
+)
+
+
+@pytest.fixture(scope="module")
+def sample_suite(tmp_path_factory) -> Path:
+    """A throwaway directory with eight trivial tests, outside the repo rootdir."""
+    suite = tmp_path_factory.mktemp("order_report_suite")
+    (suite / "test_sample.py").write_text(_SAMPLE_TESTS, encoding="utf-8")
+    return suite
+
+
+def _collect_order(sample_suite: Path, seed: int) -> tuple[list[str], str]:
+    """Run the runner with ``--collect-only`` and return (test ids, stdout)."""
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(RUNNER),
+            "--seed",
+            str(seed),
+            "--",
+            "--collect-only",
+            "-q",
+            "-p",
+            "no:cacheprovider",
+            "test_sample.py",
+        ],
+        cwd=sample_suite,
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    ids = [line for line in result.stdout.splitlines() if "::" in line]
+    assert len(ids) == 8, result.stdout
+    return ids, result.stdout
+
+
+def test_subprocess_same_seed_is_reproducible(sample_suite):
+    first, out = _collect_order(sample_suite, seed=123)
+    second, _ = _collect_order(sample_suite, seed=123)
+    assert first == second
+    assert "[order-report] shuffling test order with seed 123" in out
+
+
+def test_subprocess_different_seeds_change_order(sample_suite):
+    first, _ = _collect_order(sample_suite, seed=123)
+    second, _ = _collect_order(sample_suite, seed=321)
+    assert first != second
+
+
+def test_subprocess_failure_exit_code_and_footer(tmp_path):
+    """A real failing pytest run keeps pytest's exit code and reports the seed."""
+    (tmp_path / "test_failure.py").write_text(
+        "def test_failure():\n    assert False\n",
+        encoding="utf-8",
+    )
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(RUNNER),
+            "--seed",
+            "123",
+            "--",
+            "test_failure.py",
+            "-q",
+        ],
+        cwd=tmp_path,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 1
+    repro = shlex.join(
+        [
+            sys.executable,
+            str(RUNNER),
+            "--seed",
+            "123",
+            "--",
+            "test_failure.py",
+            "-q",
+        ]
+    )
+    assert f"reproduce with: {repro}" in result.stdout
+    assert "[order-report] seed 123: pytest exit code 1" in result.stdout