odysseus/tests/test_run_order_report.py

"""Direct tests for the order-sensitivity report runner (tests/run_order_report.py).

The shuffle and argument plumbing are tested without spawning pytest: the
shuffle helpers are asserted directly and ``run`` is exercised with an
injected fake ``pytest.main``. A small subprocess test then proves the seed is
applied end to end (reproducible, seed visible) against a throwaway test file,
never the real suite.
"""
from __future__ import annotations

import shlex
import subprocess
import sys
from pathlib import Path

import pytest

from tests.run_order_report import (
    SEED_MAX,
    OrderShuffle,
    generate_seed,
    run,
    shuffle_items,
)

REPO_ROOT = Path(__file__).resolve().parents[1]
RUNNER = REPO_ROOT / "tests" / "run_order_report.py"


class _FakePytestMain:
    """Records forwarded args and plugins and returns a fixed exit code."""

    def __init__(self, returncode: int = 0):
        self.returncode = returncode
        self.calls: list[tuple[list[str], list]] = []

    def __call__(self, args: list[str], plugins: list) -> int:
        self.calls.append((list(args), list(plugins)))
        return self.returncode


# --- shuffle determinism -----------------------------------------------------


def test_same_seed_shuffles_identically():
    first = list(range(20))
    second = list(range(20))
    shuffle_items(first, seed=123)
    shuffle_items(second, seed=123)
    assert first == second


def test_different_seeds_shuffle_differently():
    first = list(range(20))
    second = list(range(20))
    shuffle_items(first, seed=123)
    shuffle_items(second, seed=321)
    assert first != second


def test_shuffle_preserves_items():
    items = list(range(20))
    shuffle_items(items, seed=123)
    assert sorted(items) == list(range(20))


def test_plugin_hook_matches_shuffle_items():
    hooked = list(range(20))
    expected = list(range(20))
    OrderShuffle(seed=7).pytest_collection_modifyitems(hooked)
    shuffle_items(expected, seed=7)
    assert hooked == expected


# --- argument parsing and pytest invocation ----------------------------------


def test_pytest_args_after_separator_are_forwarded():
    fake = _FakePytestMain()
    run(["--seed", "123", "--", "tests/cli/", "-q"], pytest_main=fake)
    (args, plugins), = fake.calls
    assert args == ["tests/cli/", "-q"]
    assert [type(p) for p in plugins] == [OrderShuffle]


def test_explicit_seed_reaches_plugin():
    fake = _FakePytestMain()
    run(["--seed", "123", "--", "-q"], pytest_main=fake)
    (_, plugins), = fake.calls
    assert plugins[0].seed == 123


def test_pytest_exit_code_is_propagated():
    fake = _FakePytestMain(returncode=3)
    assert run(["--seed", "123", "--", "-q"], pytest_main=fake) == 3


@pytest.mark.parametrize("value", ["abc", "-1", str(SEED_MAX + 1)])
def test_invalid_seed_is_rejected_before_pytest(value):
    fake = _FakePytestMain()
    with pytest.raises(SystemExit) as excinfo:
        run(["--seed", value, "--", "-q"], pytest_main=fake)
    assert excinfo.value.code == 2
    assert fake.calls == []


# --- seed reporting -----------------------------------------------------------


def test_explicit_seed_is_printed_with_repro_command(capsys):
    run(["--seed", "123", "--", "tests/cli/", "-q"], pytest_main=_FakePytestMain())
    out = capsys.readouterr().out
    assert "[order-report] shuffling test order with seed 123" in out
    repro = shlex.join(
        [
            sys.executable,
            str(RUNNER),
            "--seed",
            "123",
            "--",
            "tests/cli/",
            "-q",
        ]
    )
    assert f"reproduce with: {repro}" in out


def test_working_directory_is_reported(capsys, monkeypatch, tmp_path):
    monkeypatch.chdir(tmp_path)
    run(["--seed", "123", "--", "-q"], pytest_main=_FakePytestMain())
    out = capsys.readouterr().out
    assert f"[order-report] working directory: {tmp_path}" in out


def test_footer_repeats_seed_and_outcome(capsys):
    run(["--seed", "123", "--", "-q"], pytest_main=_FakePytestMain(returncode=1))
    out = capsys.readouterr().out
    assert "[order-report] seed 123: pytest exit code 1" in out


def test_generated_seed_is_printed_and_used(capsys):
    fake = _FakePytestMain()
    run(["--", "-q"], pytest_main=fake)
    out = capsys.readouterr().out
    seed_line = next(line for line in out.splitlines() if "with seed" in line)
    seed = int(seed_line.rsplit("seed ", 1)[1])
    assert 0 <= seed <= SEED_MAX
    (_, plugins), = fake.calls
    assert plugins[0].seed == seed


def test_generate_seed_is_within_range():
    assert all(0 <= generate_seed() <= SEED_MAX for _ in range(5))


# --- end-to-end: the seed really drives collection order (real subprocess) ---

_SAMPLE_TESTS = "".join(
    f"def test_{name}():\n    pass\n\n"
    for name in ("alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel")
)


@pytest.fixture(scope="module")
def sample_suite(tmp_path_factory) -> Path:
    """A throwaway directory with eight trivial tests, outside the repo rootdir."""
    suite = tmp_path_factory.mktemp("order_report_suite")
    (suite / "test_sample.py").write_text(_SAMPLE_TESTS, encoding="utf-8")
    return suite


def _collect_order(sample_suite: Path, seed: int) -> tuple[list[str], str]:
    """Run the runner with ``--collect-only`` and return (test ids, stdout)."""
    result = subprocess.run(
        [
            sys.executable,
            str(RUNNER),
            "--seed",
            str(seed),
            "--",
            "--collect-only",
            "-q",
            "-p",
            "no:cacheprovider",
            "test_sample.py",
        ],
        cwd=sample_suite,
        capture_output=True,
        text=True,
    )
    assert result.returncode == 0, result.stderr or result.stdout
    ids = [line for line in result.stdout.splitlines() if "::" in line]
    assert len(ids) == 8, result.stdout
    return ids, result.stdout


def test_subprocess_same_seed_is_reproducible(sample_suite):
    first, out = _collect_order(sample_suite, seed=123)
    second, _ = _collect_order(sample_suite, seed=123)
    assert first == second
    assert "[order-report] shuffling test order with seed 123" in out


def test_subprocess_different_seeds_change_order(sample_suite):
    first, _ = _collect_order(sample_suite, seed=123)
    second, _ = _collect_order(sample_suite, seed=321)
    assert first != second


def test_subprocess_failure_exit_code_and_footer(tmp_path):
    """A real failing pytest run keeps pytest's exit code and reports the seed."""
    (tmp_path / "test_failure.py").write_text(
        "def test_failure():\n    assert False\n",
        encoding="utf-8",
    )

    result = subprocess.run(
        [
            sys.executable,
            str(RUNNER),
            "--seed",
            "123",
            "--",
            "test_failure.py",
            "-q",
        ],
        cwd=tmp_path,
        capture_output=True,
        text=True,
    )

    assert result.returncode == 1
    repro = shlex.join(
        [
            sys.executable,
            str(RUNNER),
            "--seed",
            "123",
            "--",
            "test_failure.py",
            "-q",
        ]
    )
    assert f"reproduce with: {repro}" in result.stdout
    assert "[order-report] seed 123: pytest exit code 1" in result.stdout