odysseus/tests/run_order_report.py

#!/usr/bin/env python3
"""Report-only randomized test-order runner (issue #3973).

Runs pytest with the collected test items shuffled by a seeded RNG so
order-sensitive tests (hidden coupling through shared import state, module
caches, databases, etc.) surface locally. The seed is always printed, so any
failing order is reproducible with ``--seed``.

This runner is report-only: it is not wired into CI, adds no gate, and does
not change normal pytest collection or ordering. Failures it discovers should
be fixed in separate scoped PRs, not silenced here.

Examples:
    python3 tests/run_order_report.py --seed 123 -- tests/cli/ -q
    python3 tests/run_order_report.py -- tests/cli/ -q   # generates and prints a seed

The shuffle is applied through a local ``pytest_collection_modifyitems`` hook
passed to ``pytest.main`` as an in-process plugin; no conftest or global
plugin is involved. Reproduction requires the reported working directory,
seed, pytest arguments, and test environment. The exit code is pytest's own.
"""
from __future__ import annotations

import argparse
import random
import shlex
import sys
from collections.abc import Callable, Sequence
from pathlib import Path

# Seeds are kept in the non-negative 32-bit range so they stay short enough to
# copy from a report line into a reproduction command.
SEED_MAX = 2**32 - 1


def shuffle_items(items: list, seed: int) -> None:
    """Deterministically shuffle ``items`` in place using ``seed``."""
    random.Random(seed).shuffle(items)


class OrderShuffle:
    """Local pytest plugin that shuffles collected items with a fixed seed."""

    def __init__(self, seed: int):
        self.seed = seed

    def pytest_collection_modifyitems(self, items: list) -> None:
        shuffle_items(items, self.seed)


def generate_seed() -> int:
    """Generate a fresh seed for a run that did not pass ``--seed``."""
    return random.SystemRandom().randint(0, SEED_MAX)


def seed_type(value: str) -> int:
    """argparse type: a seed in ``[0, SEED_MAX]``."""
    number = int(value)
    if not 0 <= number <= SEED_MAX:
        raise argparse.ArgumentTypeError(
            f"seed must be between 0 and {SEED_MAX}, got {value!r}"
        )
    return number


def build_parser() -> argparse.ArgumentParser:
    """Build the argument parser for the order-sensitivity runner."""
    parser = argparse.ArgumentParser(
        prog="run_order_report.py",
        description=(
            "Run pytest with randomized test order to surface order-sensitive "
            "tests. Report-only: prints the seed used and propagates pytest's "
            "exit code; it changes no normal pytest behavior."
        ),
        epilog=(
            "Pass pytest targets and options after a literal -- separator, "
            "e.g.: run_order_report.py --seed 123 -- tests/cli/ -q"
        ),
    )
    parser.add_argument(
        "--seed",
        type=seed_type,
        help="shuffle seed; omitted: a seed is generated and printed",
    )
    parser.add_argument(
        "pytest_args",
        nargs="*",
        metavar="-- PYTEST_ARGS",
        help="pytest targets/options forwarded after a literal --",
    )
    return parser


def runner_path() -> str:
    """Return an absolute path for copy-pasteable reproduction commands."""
    return str(Path(__file__).resolve())


def print_report_header(seed: int, pytest_args: Sequence[str]) -> None:
    """Print the seed and an exact reproduction command before running."""
    repro = [
        sys.executable,
        runner_path(),
        "--seed",
        str(seed),
        "--",
        *pytest_args,
    ]
    print(f"[order-report] working directory: {Path.cwd()}")
    print(f"[order-report] shuffling test order with seed {seed}")
    print(
        "[order-report] reproduce from this working directory with the same "
        "test environment:"
    )
    print(f"[order-report] reproduce with: {shlex.join(repro)}")


def print_report_footer(seed: int, exit_code: int) -> None:
    """Print the outcome with the seed again, after possibly long pytest output."""
    outcome = "no failures" if exit_code == 0 else f"pytest exit code {exit_code}"
    print(
        f"[order-report] seed {seed}: {outcome} "
        "(report-only; fix order-sensitive failures in separate scoped PRs)"
    )


def run(
    argv: Sequence[str] | None = None,
    pytest_main: Callable[..., int] | None = None,
) -> int:
    """Parse ``argv``, run pytest with shuffled item order, and report the seed.

    ``pytest_main`` is injected so tests can assert on the forwarded arguments
    and plugin without running a nested pytest. It must match ``pytest.main``:
    accept ``(args, plugins=...)`` and return an exit code.
    """
    namespace = build_parser().parse_args(argv)
    seed = namespace.seed if namespace.seed is not None else generate_seed()
    pytest_args = list(namespace.pytest_args)
    print_report_header(seed, pytest_args)
    if pytest_main is None:
        import pytest

        pytest_main = pytest.main
    exit_code = int(pytest_main(pytest_args, plugins=[OrderShuffle(seed)]))
    print_report_footer(seed, exit_code)
    return exit_code


def main() -> int:
    """Console entry point."""
    return run(sys.argv[1:])


if __name__ == "__main__":
    raise SystemExit(main())