Bench Report

benchmatrix.bench_report

Versioned, loadable comparison report documents.

BenchmarkPolicyProvenance dataclass

Configuration provenance embedded in a comparison report.

Source code in src/benchmatrix/bench_report.py

@dataclass(frozen=True, slots=True)
class BenchmarkPolicyProvenance:
    """Configuration provenance embedded in a comparison report."""

    selection: PolicySelection
    configuration_file: str | None = None
    configured_fields: tuple[str, ...] = ()
    cli_overrides: tuple[str, ...] = ()

    def __post_init__(self) -> None:
        """Validate and normalize policy provenance."""
        if self.selection not in {"defaults", "disabled", "discovered", "explicit"}:
            raise ValueError(f"Unsupported policy selection: {self.selection!r}.")
        if self.configuration_file is not None and (
            not isinstance(self.configuration_file, str) or not self.configuration_file
        ):
            raise ValueError("configuration_file must be a non-empty string or None.")
        configured_fields = tuple(self.configured_fields)
        cli_overrides = tuple(self.cli_overrides)
        if any(not isinstance(field, str) or not field for field in (*configured_fields, *cli_overrides)):
            raise ValueError("Policy provenance fields must be non-empty strings.")
        object.__setattr__(self, "configured_fields", configured_fields)
        object.__setattr__(self, "cli_overrides", cli_overrides)

__post_init__

__post_init__() -> None

Validate and normalize policy provenance.

Source code in src/benchmatrix/bench_report.py

def __post_init__(self) -> None:
    """Validate and normalize policy provenance."""
    if self.selection not in {"defaults", "disabled", "discovered", "explicit"}:
        raise ValueError(f"Unsupported policy selection: {self.selection!r}.")
    if self.configuration_file is not None and (
        not isinstance(self.configuration_file, str) or not self.configuration_file
    ):
        raise ValueError("configuration_file must be a non-empty string or None.")
    configured_fields = tuple(self.configured_fields)
    cli_overrides = tuple(self.cli_overrides)
    if any(not isinstance(field, str) or not field for field in (*configured_fields, *cli_overrides)):
        raise ValueError("Policy provenance fields must be non-empty strings.")
    object.__setattr__(self, "configured_fields", configured_fields)
    object.__setattr__(self, "cli_overrides", cli_overrides)

BenchmarkThresholdProvenance dataclass

Rule scope and origin supplying one reported cell threshold.

Source code in src/benchmatrix/bench_report.py

@dataclass(frozen=True, slots=True)
class BenchmarkThresholdProvenance:
    """Rule scope and origin supplying one reported cell threshold."""

    scope: RegressionThresholdScope
    origin: ThresholdOrigin
    field: str

    def __post_init__(self) -> None:
        """Validate threshold provenance."""
        if self.scope not in {"cell", "case", "implementation", "metric", "default"}:
            raise ValueError(f"Unsupported threshold scope: {self.scope!r}.")
        if self.origin not in {"built_in", "configuration", "cli"}:
            raise ValueError(f"Unsupported threshold origin: {self.origin!r}.")
        if not isinstance(self.field, str) or not self.field:
            raise ValueError("Threshold provenance field must be a non-empty string.")

__post_init__

__post_init__() -> None

Validate threshold provenance.

Source code in src/benchmatrix/bench_report.py

def __post_init__(self) -> None:
    """Validate threshold provenance."""
    if self.scope not in {"cell", "case", "implementation", "metric", "default"}:
        raise ValueError(f"Unsupported threshold scope: {self.scope!r}.")
    if self.origin not in {"built_in", "configuration", "cli"}:
        raise ValueError(f"Unsupported threshold origin: {self.origin!r}.")
    if not isinstance(self.field, str) or not self.field:
        raise ValueError("Threshold provenance field must be a non-empty string.")

BenchmarkCollectionSnapshot dataclass

Portable collection provenance embedded in a comparison report.

Source code in src/benchmatrix/bench_report.py

@dataclass(frozen=True, slots=True)
class BenchmarkCollectionSnapshot:
    """Portable collection provenance embedded in a comparison report."""

    manifest: str
    created_at: str
    command: tuple[str, ...]
    cwd: str
    commit: str | None
    environment_fingerprint: str | None
    requested_runs: int
    expected_cells: tuple[BenchmarkCell, ...]
    records: tuple[BenchmarkRunRecord, ...]

    def __post_init__(self) -> None:
        """Validate and normalize the portable collection snapshot."""
        if not isinstance(self.manifest, str) or not self.manifest:
            raise ValueError("Collection manifest must be a non-empty string.")
        if not isinstance(self.created_at, str) or not self.created_at:
            raise ValueError("Collection created_at must be a non-empty string.")
        command = tuple(self.command)
        if not command or any(not isinstance(argument, str) or not argument for argument in command):
            raise ValueError("Collection command must contain non-empty strings.")
        if not isinstance(self.cwd, str) or not self.cwd:
            raise ValueError("Collection cwd must be a non-empty string.")
        if self.commit is not None and (not isinstance(self.commit, str) or not self.commit):
            raise ValueError("Collection commit must be a non-empty string or None.")
        if self.environment_fingerprint is not None and (
            not isinstance(self.environment_fingerprint, str) or not self.environment_fingerprint
        ):
            raise ValueError("Collection environment fingerprint must be a non-empty string or None.")
        if (
            isinstance(self.requested_runs, bool)
            or not isinstance(self.requested_runs, int)
            or self.requested_runs <= 0
        ):
            raise ValueError("Collection requested_runs must be a positive integer.")
        expected_cells = tuple(self.expected_cells)
        records = tuple(self.records)
        if tuple(record.index for record in records) != tuple(range(1, len(records) + 1)):
            raise ValueError("Collection record indexes must be contiguous and one-based.")
        successful_runs = sum(record.status == "succeeded" for record in records)
        if successful_runs > self.requested_runs:
            raise ValueError("Collection has more successful runs than requested.")
        if len(set(expected_cells)) != len(expected_cells):
            raise ValueError("Collection expected_cells must not contain duplicates.")
        for implementation, case, metric in expected_cells:
            if not implementation or not case or metric not in KNOWN_METRICS:
                raise ValueError(f"Invalid collection expected cell: {(implementation, case, metric)!r}.")
        object.__setattr__(self, "command", command)
        object.__setattr__(self, "expected_cells", expected_cells)
        object.__setattr__(self, "records", records)

    @classmethod
    def from_group(cls, group: BenchmarkRunGroup) -> BenchmarkCollectionSnapshot:
        """Create a portable snapshot from a loaded run group."""
        return cls(
            manifest=str(group.manifest_path),
            created_at=group.created_at,
            command=group.command,
            cwd=str(group.cwd),
            commit=group.commit,
            environment_fingerprint=group.environment_fingerprint,
            requested_runs=group.requested_runs,
            expected_cells=group.expected_cells,
            records=group.records,
        )

    @property
    def attempted_runs(self) -> int:
        """Return the number of completed collection attempts."""
        return len(self.records)

    @property
    def successful_runs(self) -> int:
        """Return the number of accepted collection attempts."""
        return sum(record.status == "succeeded" for record in self.records)

    @property
    def failed_runs(self) -> int:
        """Return the number of failed collection attempts."""
        return sum(record.status == "failed" for record in self.records)

    @property
    def pending_runs(self) -> int:
        """Return initial collection slots that were never attempted."""
        return max(0, self.requested_runs - self.attempted_runs)

    @property
    def retry_attempts(self) -> int:
        """Return attempts appended after the initial collection slots."""
        return max(0, self.attempted_runs - self.requested_runs)

    @property
    def remaining_runs(self) -> int:
        """Return additional successful runs needed for completeness."""
        return self.requested_runs - self.successful_runs

    @property
    def complete(self) -> bool:
        """Return whether the requested successful-run target was reached."""
        return self.successful_runs == self.requested_runs

    def to_dict(self) -> dict[str, JsonValue]:
        """Return the stable JSON representation of this collection."""
        return {
            "manifest": self.manifest,
            "created_at": self.created_at,
            "command": list(self.command),
            "cwd": self.cwd,
            "commit": self.commit,
            "environment_fingerprint": self.environment_fingerprint,
            "requested_runs": self.requested_runs,
            "attempted_runs": self.attempted_runs,
            "successful_runs": self.successful_runs,
            "failed_runs": self.failed_runs,
            "complete": self.complete,
            "expected_cells": [
                {
                    "implementation_name": implementation,
                    "case_name": case,
                    "metric_name": metric,
                }
                for implementation, case, metric in self.expected_cells
            ],
            "runs": [_record_dict(record) for record in self.records],
        }

attempted_runs property

attempted_runs: int

Return the number of completed collection attempts.

successful_runs property

successful_runs: int

Return the number of accepted collection attempts.

failed_runs property

failed_runs: int

Return the number of failed collection attempts.

pending_runs property

pending_runs: int

Return initial collection slots that were never attempted.

retry_attempts property

retry_attempts: int

Return attempts appended after the initial collection slots.

remaining_runs property

remaining_runs: int

Return additional successful runs needed for completeness.

complete property

complete: bool

Return whether the requested successful-run target was reached.

__post_init__

__post_init__() -> None

Validate and normalize the portable collection snapshot.

Source code in src/benchmatrix/bench_report.py

def __post_init__(self) -> None:
    """Validate and normalize the portable collection snapshot."""
    if not isinstance(self.manifest, str) or not self.manifest:
        raise ValueError("Collection manifest must be a non-empty string.")
    if not isinstance(self.created_at, str) or not self.created_at:
        raise ValueError("Collection created_at must be a non-empty string.")
    command = tuple(self.command)
    if not command or any(not isinstance(argument, str) or not argument for argument in command):
        raise ValueError("Collection command must contain non-empty strings.")
    if not isinstance(self.cwd, str) or not self.cwd:
        raise ValueError("Collection cwd must be a non-empty string.")
    if self.commit is not None and (not isinstance(self.commit, str) or not self.commit):
        raise ValueError("Collection commit must be a non-empty string or None.")
    if self.environment_fingerprint is not None and (
        not isinstance(self.environment_fingerprint, str) or not self.environment_fingerprint
    ):
        raise ValueError("Collection environment fingerprint must be a non-empty string or None.")
    if (
        isinstance(self.requested_runs, bool)
        or not isinstance(self.requested_runs, int)
        or self.requested_runs <= 0
    ):
        raise ValueError("Collection requested_runs must be a positive integer.")
    expected_cells = tuple(self.expected_cells)
    records = tuple(self.records)
    if tuple(record.index for record in records) != tuple(range(1, len(records) + 1)):
        raise ValueError("Collection record indexes must be contiguous and one-based.")
    successful_runs = sum(record.status == "succeeded" for record in records)
    if successful_runs > self.requested_runs:
        raise ValueError("Collection has more successful runs than requested.")
    if len(set(expected_cells)) != len(expected_cells):
        raise ValueError("Collection expected_cells must not contain duplicates.")
    for implementation, case, metric in expected_cells:
        if not implementation or not case or metric not in KNOWN_METRICS:
            raise ValueError(f"Invalid collection expected cell: {(implementation, case, metric)!r}.")
    object.__setattr__(self, "command", command)
    object.__setattr__(self, "expected_cells", expected_cells)
    object.__setattr__(self, "records", records)

from_group classmethod

from_group(
    group: BenchmarkRunGroup,
) -> BenchmarkCollectionSnapshot

Create a portable snapshot from a loaded run group.

Source code in src/benchmatrix/bench_report.py

@classmethod
def from_group(cls, group: BenchmarkRunGroup) -> BenchmarkCollectionSnapshot:
    """Create a portable snapshot from a loaded run group."""
    return cls(
        manifest=str(group.manifest_path),
        created_at=group.created_at,
        command=group.command,
        cwd=str(group.cwd),
        commit=group.commit,
        environment_fingerprint=group.environment_fingerprint,
        requested_runs=group.requested_runs,
        expected_cells=group.expected_cells,
        records=group.records,
    )

to_dict

to_dict() -> dict[str, JsonValue]

Return the stable JSON representation of this collection.

Source code in src/benchmatrix/bench_report.py

def to_dict(self) -> dict[str, JsonValue]:
    """Return the stable JSON representation of this collection."""
    return {
        "manifest": self.manifest,
        "created_at": self.created_at,
        "command": list(self.command),
        "cwd": self.cwd,
        "commit": self.commit,
        "environment_fingerprint": self.environment_fingerprint,
        "requested_runs": self.requested_runs,
        "attempted_runs": self.attempted_runs,
        "successful_runs": self.successful_runs,
        "failed_runs": self.failed_runs,
        "complete": self.complete,
        "expected_cells": [
            {
                "implementation_name": implementation,
                "case_name": case,
                "metric_name": metric,
            }
            for implementation, case, metric in self.expected_cells
        ],
        "runs": [_record_dict(record) for record in self.records],
    }

BenchmarkPairedCollectionSnapshot dataclass

Portable paired AB/BA collection provenance embedded in a report.

Source code in src/benchmatrix/bench_report.py

@dataclass(frozen=True, slots=True)
class BenchmarkPairedCollectionSnapshot:
    """Portable paired AB/BA collection provenance embedded in a report."""

    manifest: str
    created_at: str
    baseline_command: tuple[str, ...]
    candidate_command: tuple[str, ...]
    baseline_cwd: str
    candidate_cwd: str
    baseline_commit: str | None
    candidate_commit: str | None
    baseline_environment_fingerprint: str | None
    candidate_environment_fingerprint: str | None
    requested_pairs: int
    random_seed: int
    expected_cells: tuple[BenchmarkCell, ...]
    records: tuple[BenchmarkPairedRunRecord, ...]
    automatic_pairs: bool = False

    def __post_init__(self) -> None:
        """Validate and normalize the portable paired snapshot."""
        if not isinstance(self.manifest, str) or not self.manifest:
            raise ValueError("Paired collection manifest must be a non-empty string.")
        if not isinstance(self.created_at, str) or not self.created_at:
            raise ValueError("Paired collection created_at must be a non-empty string.")
        baseline_command = tuple(self.baseline_command)
        candidate_command = tuple(self.candidate_command)
        for name, command in (
            ("baseline_command", baseline_command),
            ("candidate_command", candidate_command),
        ):
            if not command or any(not isinstance(argument, str) or not argument for argument in command):
                raise ValueError(f"Paired collection {name} must contain non-empty strings.")
        for name, cwd in (("baseline_cwd", self.baseline_cwd), ("candidate_cwd", self.candidate_cwd)):
            if not isinstance(cwd, str) or not cwd:
                raise ValueError(f"Paired collection {name} must be a non-empty string.")
        for name, value in (
            ("baseline_commit", self.baseline_commit),
            ("candidate_commit", self.candidate_commit),
            ("baseline_environment_fingerprint", self.baseline_environment_fingerprint),
            ("candidate_environment_fingerprint", self.candidate_environment_fingerprint),
        ):
            if value is not None and (not isinstance(value, str) or not value):
                raise ValueError(f"Paired collection {name} must be a non-empty string or None.")
        if (
            isinstance(self.requested_pairs, bool)
            or not isinstance(self.requested_pairs, int)
            or self.requested_pairs <= 0
        ):
            raise ValueError("Paired collection requested_pairs must be a positive integer.")
        if isinstance(self.random_seed, bool) or not isinstance(self.random_seed, int) or self.random_seed < 0:
            raise ValueError("Paired collection random_seed must be a non-negative integer.")
        if not isinstance(self.automatic_pairs, bool):
            raise TypeError("Paired collection automatic_pairs must be a boolean.")

        expected_cells = tuple(self.expected_cells)
        records = tuple(self.records)
        if len(set(expected_cells)) != len(expected_cells):
            raise ValueError("Paired collection expected_cells must not contain duplicates.")
        for implementation, case, metric in expected_cells:
            if not implementation or not case or metric not in KNOWN_METRICS:
                raise ValueError(f"Invalid paired collection expected cell: {(implementation, case, metric)!r}.")
        if any(not isinstance(record, BenchmarkPairedRunRecord) for record in records):
            raise TypeError("Paired collection records must contain BenchmarkPairedRunRecord values.")
        if tuple(record.index for record in records) != tuple(range(1, len(records) + 1)):
            raise ValueError("Paired collection record indexes must be contiguous and one-based.")
        canonical_paths = tuple(record.path.resolve() for record in records)
        if len(set(canonical_paths)) != len(canonical_paths):
            raise ValueError("Paired collection record paths must be unique.")

        successful_records = tuple(record for record in records if record.status == "succeeded")
        if successful_records and not expected_cells:
            raise ValueError("Paired collection with successful records requires expected cells.")
        if expected_cells and not successful_records:
            raise ValueError("Paired collection without successful records cannot define expected cells.")
        if self.automatic_pairs:
            if expected_cells:
                supercycle = balanced_order_supercycle_length(len(expected_cells))
                minimum_pairs = EvidencePolicy().minimum_runs
                expected_pair_target = ((minimum_pairs + supercycle - 1) // supercycle) * supercycle
                if self.requested_pairs != expected_pair_target:
                    raise ValueError(
                        "Automatic paired collection target must be the smallest complete joint-design "
                        "supercycle at or above the evidence minimum."
                    )
            elif self.requested_pairs != 6:
                raise ValueError("Automatic paired collection without successful records must retain target 6.")

        schedule = {
            entry.pair_index: entry
            for entry in make_paired_ab_ba_schedule(
                self.requested_pairs,
                random_seed=self.random_seed,
                cell_count=(len(expected_cells) if expected_cells else None),
            )
        }
        block_variants: dict[tuple[int, int], set[str]] = {}
        attempts_by_pair: dict[int, set[int]] = {}
        latest_attempt_by_pair: dict[int, int] = {}
        first_seen_pairs: set[int] = set()
        matrix_anchor_seen = False
        for record in records:
            if record.pair_index > self.requested_pairs:
                raise ValueError("Paired collection record pair_index exceeds requested_pairs.")
            if record.pair_index not in first_seen_pairs:
                expected_pair_index = len(first_seen_pairs) + 1
                if record.pair_index != expected_pair_index:
                    raise ValueError("Paired collection target pairs must first appear as a one-based prefix.")
                first_seen_pairs.add(record.pair_index)
            if record.pair_index > 1 and not matrix_anchor_seen:
                raise ValueError("Paired collection cannot start later target pairs before a successful matrix exists.")
            expected = schedule[record.pair_index]
            if record.pair_order != expected.pair_order or record.cell_order_index != expected.cell_order_index:
                raise ValueError("Paired collection record differs from its deterministic schedule.")
            previous_attempt = latest_attempt_by_pair.get(record.pair_index, record.block_attempt)
            if record.block_attempt < previous_attempt:
                raise ValueError("Paired collection block attempts must be chronological per pair.")
            latest_attempt_by_pair[record.pair_index] = record.block_attempt
            key = (record.pair_index, record.block_attempt)
            variants = block_variants.setdefault(key, set())
            if record.variant in variants:
                raise ValueError("Paired collection block attempts cannot repeat a variant.")
            variants.add(record.variant)
            attempts_by_pair.setdefault(record.pair_index, set()).add(record.block_attempt)
            matrix_anchor_seen = matrix_anchor_seen or record.status == "succeeded"
        records_by_block: dict[tuple[int, int], list[BenchmarkPairedRunRecord]] = {}
        for record in records:
            records_by_block.setdefault((record.pair_index, record.block_attempt), []).append(record)
        for (pair_index, _block_attempt), block_records in records_by_block.items():
            scheduled_variants = schedule[pair_index].variants
            observed_variants = tuple(record.variant for record in block_records)
            if observed_variants != scheduled_variants[: len(observed_variants)]:
                raise ValueError("Paired collection block records are not in scheduled AB/BA order.")
            if len(block_records) == 2 and block_records[1].index != block_records[0].index + 1:
                raise ValueError("Paired collection block members must be adjacent records.")
            if len(block_records) == 1 and block_records[0].index < len(records):
                following = records[block_records[0].index]
                if following.pair_index != pair_index or following.block_attempt != block_records[0].block_attempt + 1:
                    raise ValueError("Paired collection a partial block must be followed by a retry of the same pair.")
        for pair_index, attempts in attempts_by_pair.items():
            if sorted(attempts) != list(range(1, max(attempts) + 1)):
                raise ValueError("Paired collection block attempts must be contiguous per pair.")
            complete_attempts = sum(
                {
                    record.variant
                    for record in records
                    if record.pair_index == pair_index
                    and record.block_attempt == block_attempt
                    and record.status == "succeeded"
                }
                == {"baseline", "candidate"}
                for block_attempt in attempts
            )
            if complete_attempts > 1:
                raise ValueError("Paired collection cannot contain two complete blocks for one pair.")
            complete_attempt = next(
                (
                    block_attempt
                    for block_attempt in attempts
                    if {
                        record.variant
                        for record in records
                        if record.pair_index == pair_index
                        and record.block_attempt == block_attempt
                        and record.status == "succeeded"
                    }
                    == {"baseline", "candidate"}
                ),
                None,
            )
            if complete_attempt is not None and any(attempt > complete_attempt for attempt in attempts):
                raise ValueError("Paired collection cannot retry a pair after a complete block.")

        for variant, commit, fingerprint in (
            (
                "baseline",
                self.baseline_commit,
                self.baseline_environment_fingerprint,
            ),
            (
                "candidate",
                self.candidate_commit,
                self.candidate_environment_fingerprint,
            ),
        ):
            variant_records = tuple(record for record in successful_records if record.variant == variant)
            if not variant_records:
                if commit is not None or fingerprint is not None:
                    raise ValueError(f"Paired collection {variant} anchors require a successful record.")
                continue
            if variant_records[0].environment_fingerprint != fingerprint:
                raise ValueError(f"First successful {variant} fingerprint does not match the collection anchor.")
            if any(record.commit != commit for record in variant_records):
                raise ValueError(f"Successful {variant} commits do not match the collection anchor.")

        object.__setattr__(self, "baseline_command", baseline_command)
        object.__setattr__(self, "candidate_command", candidate_command)
        object.__setattr__(self, "expected_cells", expected_cells)
        object.__setattr__(self, "records", records)

    @classmethod
    def from_group(cls, group: BenchmarkPairedRunGroup) -> BenchmarkPairedCollectionSnapshot:
        """Create a portable snapshot from a loaded paired run group."""
        return cls(
            manifest=str(group.manifest_path),
            created_at=group.created_at,
            baseline_command=group.baseline_command,
            candidate_command=group.candidate_command,
            baseline_cwd=str(group.baseline_cwd),
            candidate_cwd=str(group.candidate_cwd),
            baseline_commit=group.baseline_commit,
            candidate_commit=group.candidate_commit,
            baseline_environment_fingerprint=group.baseline_environment_fingerprint,
            candidate_environment_fingerprint=group.candidate_environment_fingerprint,
            requested_pairs=group.requested_pairs,
            random_seed=group.random_seed,
            expected_cells=group.expected_cells,
            records=group.records,
            automatic_pairs=group.automatic_pairs,
        )

    @property
    def attempted_commands(self) -> int:
        """Return the number of completed command attempts."""
        return len(self.records)

    @property
    def successful_commands(self) -> int:
        """Return the number of accepted command attempts."""
        return sum(record.status == "succeeded" for record in self.records)

    @property
    def failed_commands(self) -> int:
        """Return the number of failed command attempts."""
        return sum(record.status == "failed" for record in self.records)

    @property
    def complete_pair_count(self) -> int:
        """Return the number of target pairs with one complete atomic block."""
        return len(self.complete_pair_records)

    @property
    def complete_pair_records(
        self,
    ) -> tuple[tuple[BenchmarkPairedRunRecord, BenchmarkPairedRunRecord], ...]:
        """Return baseline/candidate records for complete blocks in pair order."""
        successful = {
            (record.pair_index, record.block_attempt, record.variant): record
            for record in self.records
            if record.status == "succeeded"
        }
        pairs: list[tuple[BenchmarkPairedRunRecord, BenchmarkPairedRunRecord]] = []
        for pair_index in range(1, self.requested_pairs + 1):
            attempts = sorted(
                block_attempt
                for pair, block_attempt, variant in successful
                if pair == pair_index and variant == "baseline"
            )
            for block_attempt in attempts:
                baseline = successful.get((pair_index, block_attempt, "baseline"))
                candidate = successful.get((pair_index, block_attempt, "candidate"))
                if baseline is not None and candidate is not None:
                    pairs.append((baseline, candidate))
                    break
        return tuple(pairs)

    @property
    def baseline_sources(self) -> tuple[str, ...]:
        """Return complete-block baseline paths in target-pair order."""
        return tuple(str(baseline.path) for baseline, _candidate in self.complete_pair_records)

    @property
    def candidate_sources(self) -> tuple[str, ...]:
        """Return complete-block candidate paths in target-pair order."""
        return tuple(str(candidate.path) for _baseline, candidate in self.complete_pair_records)

    @property
    def orphan_success_count(self) -> int:
        """Return successful commands excluded from complete pairs."""
        return self.successful_commands - 2 * self.complete_pair_count

    @property
    def complete(self) -> bool:
        """Return whether every requested pair has a complete block."""
        return self.complete_pair_count == self.requested_pairs

    def to_dict(self) -> dict[str, JsonValue]:
        """Return the stable JSON representation of this paired collection."""
        return {
            "manifest": self.manifest,
            "created_at": self.created_at,
            "baseline_command": list(self.baseline_command),
            "candidate_command": list(self.candidate_command),
            "baseline_cwd": self.baseline_cwd,
            "candidate_cwd": self.candidate_cwd,
            "baseline_commit": self.baseline_commit,
            "candidate_commit": self.candidate_commit,
            "baseline_environment_fingerprint": self.baseline_environment_fingerprint,
            "candidate_environment_fingerprint": self.candidate_environment_fingerprint,
            "requested_pairs": self.requested_pairs,
            "random_seed": self.random_seed,
            "automatic_pairs": self.automatic_pairs,
            "attempted_commands": self.attempted_commands,
            "successful_commands": self.successful_commands,
            "failed_commands": self.failed_commands,
            "complete_pairs": self.complete_pair_count,
            "orphan_successes": self.orphan_success_count,
            "complete": self.complete,
            "expected_cells": [
                {
                    "implementation_name": implementation,
                    "case_name": case,
                    "metric_name": metric,
                }
                for implementation, case, metric in self.expected_cells
            ],
            "runs": [_paired_record_dict(record) for record in self.records],
        }

attempted_commands property

attempted_commands: int

Return the number of completed command attempts.

successful_commands property

successful_commands: int

Return the number of accepted command attempts.

failed_commands property

failed_commands: int

Return the number of failed command attempts.

complete_pair_count property

complete_pair_count: int

Return the number of target pairs with one complete atomic block.

complete_pair_records property

complete_pair_records: tuple[
    tuple[
        BenchmarkPairedRunRecord, BenchmarkPairedRunRecord
    ],
    ...,
]

Return baseline/candidate records for complete blocks in pair order.

baseline_sources property

baseline_sources: tuple[str, ...]

Return complete-block baseline paths in target-pair order.

candidate_sources property

candidate_sources: tuple[str, ...]

Return complete-block candidate paths in target-pair order.

orphan_success_count property

orphan_success_count: int

Return successful commands excluded from complete pairs.

complete property

complete: bool

Return whether every requested pair has a complete block.

__post_init__

__post_init__() -> None

Validate and normalize the portable paired snapshot.

Source code in src/benchmatrix/bench_report.py

def __post_init__(self) -> None:
    """Validate and normalize the portable paired snapshot."""
    if not isinstance(self.manifest, str) or not self.manifest:
        raise ValueError("Paired collection manifest must be a non-empty string.")
    if not isinstance(self.created_at, str) or not self.created_at:
        raise ValueError("Paired collection created_at must be a non-empty string.")
    baseline_command = tuple(self.baseline_command)
    candidate_command = tuple(self.candidate_command)
    for name, command in (
        ("baseline_command", baseline_command),
        ("candidate_command", candidate_command),
    ):
        if not command or any(not isinstance(argument, str) or not argument for argument in command):
            raise ValueError(f"Paired collection {name} must contain non-empty strings.")
    for name, cwd in (("baseline_cwd", self.baseline_cwd), ("candidate_cwd", self.candidate_cwd)):
        if not isinstance(cwd, str) or not cwd:
            raise ValueError(f"Paired collection {name} must be a non-empty string.")
    for name, value in (
        ("baseline_commit", self.baseline_commit),
        ("candidate_commit", self.candidate_commit),
        ("baseline_environment_fingerprint", self.baseline_environment_fingerprint),
        ("candidate_environment_fingerprint", self.candidate_environment_fingerprint),
    ):
        if value is not None and (not isinstance(value, str) or not value):
            raise ValueError(f"Paired collection {name} must be a non-empty string or None.")
    if (
        isinstance(self.requested_pairs, bool)
        or not isinstance(self.requested_pairs, int)
        or self.requested_pairs <= 0
    ):
        raise ValueError("Paired collection requested_pairs must be a positive integer.")
    if isinstance(self.random_seed, bool) or not isinstance(self.random_seed, int) or self.random_seed < 0:
        raise ValueError("Paired collection random_seed must be a non-negative integer.")
    if not isinstance(self.automatic_pairs, bool):
        raise TypeError("Paired collection automatic_pairs must be a boolean.")

    expected_cells = tuple(self.expected_cells)
    records = tuple(self.records)
    if len(set(expected_cells)) != len(expected_cells):
        raise ValueError("Paired collection expected_cells must not contain duplicates.")
    for implementation, case, metric in expected_cells:
        if not implementation or not case or metric not in KNOWN_METRICS:
            raise ValueError(f"Invalid paired collection expected cell: {(implementation, case, metric)!r}.")
    if any(not isinstance(record, BenchmarkPairedRunRecord) for record in records):
        raise TypeError("Paired collection records must contain BenchmarkPairedRunRecord values.")
    if tuple(record.index for record in records) != tuple(range(1, len(records) + 1)):
        raise ValueError("Paired collection record indexes must be contiguous and one-based.")
    canonical_paths = tuple(record.path.resolve() for record in records)
    if len(set(canonical_paths)) != len(canonical_paths):
        raise ValueError("Paired collection record paths must be unique.")

    successful_records = tuple(record for record in records if record.status == "succeeded")
    if successful_records and not expected_cells:
        raise ValueError("Paired collection with successful records requires expected cells.")
    if expected_cells and not successful_records:
        raise ValueError("Paired collection without successful records cannot define expected cells.")
    if self.automatic_pairs:
        if expected_cells:
            supercycle = balanced_order_supercycle_length(len(expected_cells))
            minimum_pairs = EvidencePolicy().minimum_runs
            expected_pair_target = ((minimum_pairs + supercycle - 1) // supercycle) * supercycle
            if self.requested_pairs != expected_pair_target:
                raise ValueError(
                    "Automatic paired collection target must be the smallest complete joint-design "
                    "supercycle at or above the evidence minimum."
                )
        elif self.requested_pairs != 6:
            raise ValueError("Automatic paired collection without successful records must retain target 6.")

    schedule = {
        entry.pair_index: entry
        for entry in make_paired_ab_ba_schedule(
            self.requested_pairs,
            random_seed=self.random_seed,
            cell_count=(len(expected_cells) if expected_cells else None),
        )
    }
    block_variants: dict[tuple[int, int], set[str]] = {}
    attempts_by_pair: dict[int, set[int]] = {}
    latest_attempt_by_pair: dict[int, int] = {}
    first_seen_pairs: set[int] = set()
    matrix_anchor_seen = False
    for record in records:
        if record.pair_index > self.requested_pairs:
            raise ValueError("Paired collection record pair_index exceeds requested_pairs.")
        if record.pair_index not in first_seen_pairs:
            expected_pair_index = len(first_seen_pairs) + 1
            if record.pair_index != expected_pair_index:
                raise ValueError("Paired collection target pairs must first appear as a one-based prefix.")
            first_seen_pairs.add(record.pair_index)
        if record.pair_index > 1 and not matrix_anchor_seen:
            raise ValueError("Paired collection cannot start later target pairs before a successful matrix exists.")
        expected = schedule[record.pair_index]
        if record.pair_order != expected.pair_order or record.cell_order_index != expected.cell_order_index:
            raise ValueError("Paired collection record differs from its deterministic schedule.")
        previous_attempt = latest_attempt_by_pair.get(record.pair_index, record.block_attempt)
        if record.block_attempt < previous_attempt:
            raise ValueError("Paired collection block attempts must be chronological per pair.")
        latest_attempt_by_pair[record.pair_index] = record.block_attempt
        key = (record.pair_index, record.block_attempt)
        variants = block_variants.setdefault(key, set())
        if record.variant in variants:
            raise ValueError("Paired collection block attempts cannot repeat a variant.")
        variants.add(record.variant)
        attempts_by_pair.setdefault(record.pair_index, set()).add(record.block_attempt)
        matrix_anchor_seen = matrix_anchor_seen or record.status == "succeeded"
    records_by_block: dict[tuple[int, int], list[BenchmarkPairedRunRecord]] = {}
    for record in records:
        records_by_block.setdefault((record.pair_index, record.block_attempt), []).append(record)
    for (pair_index, _block_attempt), block_records in records_by_block.items():
        scheduled_variants = schedule[pair_index].variants
        observed_variants = tuple(record.variant for record in block_records)
        if observed_variants != scheduled_variants[: len(observed_variants)]:
            raise ValueError("Paired collection block records are not in scheduled AB/BA order.")
        if len(block_records) == 2 and block_records[1].index != block_records[0].index + 1:
            raise ValueError("Paired collection block members must be adjacent records.")
        if len(block_records) == 1 and block_records[0].index < len(records):
            following = records[block_records[0].index]
            if following.pair_index != pair_index or following.block_attempt != block_records[0].block_attempt + 1:
                raise ValueError("Paired collection a partial block must be followed by a retry of the same pair.")
    for pair_index, attempts in attempts_by_pair.items():
        if sorted(attempts) != list(range(1, max(attempts) + 1)):
            raise ValueError("Paired collection block attempts must be contiguous per pair.")
        complete_attempts = sum(
            {
                record.variant
                for record in records
                if record.pair_index == pair_index
                and record.block_attempt == block_attempt
                and record.status == "succeeded"
            }
            == {"baseline", "candidate"}
            for block_attempt in attempts
        )
        if complete_attempts > 1:
            raise ValueError("Paired collection cannot contain two complete blocks for one pair.")
        complete_attempt = next(
            (
                block_attempt
                for block_attempt in attempts
                if {
                    record.variant
                    for record in records
                    if record.pair_index == pair_index
                    and record.block_attempt == block_attempt
                    and record.status == "succeeded"
                }
                == {"baseline", "candidate"}
            ),
            None,
        )
        if complete_attempt is not None and any(attempt > complete_attempt for attempt in attempts):
            raise ValueError("Paired collection cannot retry a pair after a complete block.")

    for variant, commit, fingerprint in (
        (
            "baseline",
            self.baseline_commit,
            self.baseline_environment_fingerprint,
        ),
        (
            "candidate",
            self.candidate_commit,
            self.candidate_environment_fingerprint,
        ),
    ):
        variant_records = tuple(record for record in successful_records if record.variant == variant)
        if not variant_records:
            if commit is not None or fingerprint is not None:
                raise ValueError(f"Paired collection {variant} anchors require a successful record.")
            continue
        if variant_records[0].environment_fingerprint != fingerprint:
            raise ValueError(f"First successful {variant} fingerprint does not match the collection anchor.")
        if any(record.commit != commit for record in variant_records):
            raise ValueError(f"Successful {variant} commits do not match the collection anchor.")

    object.__setattr__(self, "baseline_command", baseline_command)
    object.__setattr__(self, "candidate_command", candidate_command)
    object.__setattr__(self, "expected_cells", expected_cells)
    object.__setattr__(self, "records", records)

from_group classmethod

from_group(
    group: BenchmarkPairedRunGroup,
) -> BenchmarkPairedCollectionSnapshot

Create a portable snapshot from a loaded paired run group.

Source code in src/benchmatrix/bench_report.py

@classmethod
def from_group(cls, group: BenchmarkPairedRunGroup) -> BenchmarkPairedCollectionSnapshot:
    """Create a portable snapshot from a loaded paired run group."""
    return cls(
        manifest=str(group.manifest_path),
        created_at=group.created_at,
        baseline_command=group.baseline_command,
        candidate_command=group.candidate_command,
        baseline_cwd=str(group.baseline_cwd),
        candidate_cwd=str(group.candidate_cwd),
        baseline_commit=group.baseline_commit,
        candidate_commit=group.candidate_commit,
        baseline_environment_fingerprint=group.baseline_environment_fingerprint,
        candidate_environment_fingerprint=group.candidate_environment_fingerprint,
        requested_pairs=group.requested_pairs,
        random_seed=group.random_seed,
        expected_cells=group.expected_cells,
        records=group.records,
        automatic_pairs=group.automatic_pairs,
    )

to_dict

to_dict() -> dict[str, JsonValue]

Return the stable JSON representation of this paired collection.

Source code in src/benchmatrix/bench_report.py

def to_dict(self) -> dict[str, JsonValue]:
    """Return the stable JSON representation of this paired collection."""
    return {
        "manifest": self.manifest,
        "created_at": self.created_at,
        "baseline_command": list(self.baseline_command),
        "candidate_command": list(self.candidate_command),
        "baseline_cwd": self.baseline_cwd,
        "candidate_cwd": self.candidate_cwd,
        "baseline_commit": self.baseline_commit,
        "candidate_commit": self.candidate_commit,
        "baseline_environment_fingerprint": self.baseline_environment_fingerprint,
        "candidate_environment_fingerprint": self.candidate_environment_fingerprint,
        "requested_pairs": self.requested_pairs,
        "random_seed": self.random_seed,
        "automatic_pairs": self.automatic_pairs,
        "attempted_commands": self.attempted_commands,
        "successful_commands": self.successful_commands,
        "failed_commands": self.failed_commands,
        "complete_pairs": self.complete_pair_count,
        "orphan_successes": self.orphan_success_count,
        "complete": self.complete,
        "expected_cells": [
            {
                "implementation_name": implementation,
                "case_name": case,
                "metric_name": metric,
            }
            for implementation, case, metric in self.expected_cells
        ],
        "runs": [_paired_record_dict(record) for record in self.records],
    }

BenchmarkComparisonReport dataclass

One portable, versioned benchmark comparison result.

Source code in src/benchmatrix/bench_report.py

@dataclass(frozen=True, slots=True)
class BenchmarkComparisonReport:
    """One portable, versioned benchmark comparison result."""

    baselines: tuple[str, ...]
    candidates: tuple[str, ...]
    baseline_collections: tuple[BenchmarkCollectionSnapshot, ...]
    candidate_collections: tuple[BenchmarkCollectionSnapshot, ...]
    compatibility: RunCompatibilityReport
    evidence_policy: EvidencePolicy
    regression_policy: RegressionPolicy
    policy_provenance: BenchmarkPolicyProvenance
    comparisons: tuple[BenchmarkComparison, ...]
    threshold_provenance: tuple[BenchmarkThresholdProvenance, ...]
    inference_policy: InferencePolicy = field(
        default_factory=lambda: InferencePolicy(method="legacy_consistency", multiplicity="none")
    )
    design: ComparisonDesign = "independent"
    precision_policy: PrecisionPolicy = field(default_factory=PrecisionPolicy)
    paired_collections: tuple[BenchmarkPairedCollectionSnapshot, ...] = ()

    def __post_init__(self) -> None:
        """Validate and normalize report containers."""
        baselines = tuple(self.baselines)
        candidates = tuple(self.candidates)
        baseline_collections = tuple(self.baseline_collections)
        candidate_collections = tuple(self.candidate_collections)
        paired_collections = tuple(self.paired_collections)
        comparisons = tuple(self.comparisons)
        threshold_provenance = tuple(self.threshold_provenance)
        if not baselines or any(not isinstance(source, str) or not source for source in baselines):
            raise ValueError("Report baselines must contain non-empty source strings.")
        if not candidates or any(not isinstance(source, str) or not source for source in candidates):
            raise ValueError("Report candidates must contain non-empty source strings.")
        if len(comparisons) != len(threshold_provenance):
            raise ValueError("Every reported comparison requires threshold provenance.")
        if any(
            not isinstance(item, BenchmarkCollectionSnapshot)
            for item in (*baseline_collections, *candidate_collections)
        ):
            raise TypeError("Report collections must contain BenchmarkCollectionSnapshot values.")
        if any(not isinstance(item, BenchmarkPairedCollectionSnapshot) for item in paired_collections):
            raise TypeError("Report paired_collections must contain BenchmarkPairedCollectionSnapshot values.")
        if any(not isinstance(item, BenchmarkComparison) for item in comparisons):
            raise TypeError("Report comparisons must contain BenchmarkComparison values.")
        if any(not isinstance(item, BenchmarkThresholdProvenance) for item in threshold_provenance):
            raise TypeError("Report threshold_provenance must contain BenchmarkThresholdProvenance values.")
        if not isinstance(self.compatibility, RunCompatibilityReport):
            raise TypeError("Report compatibility must be a RunCompatibilityReport.")
        if not isinstance(self.evidence_policy, EvidencePolicy):
            raise TypeError("Report evidence_policy must be an EvidencePolicy.")
        if not isinstance(self.regression_policy, RegressionPolicy):
            raise TypeError("Report regression_policy must be a RegressionPolicy.")
        if not isinstance(self.inference_policy, InferencePolicy):
            raise TypeError("Report inference_policy must be an InferencePolicy.")
        if self.design not in {"independent", "paired"}:
            raise ValueError(f"Unsupported report comparison design: {self.design!r}.")
        if not isinstance(self.precision_policy, PrecisionPolicy):
            raise TypeError("Report precision_policy must be a PrecisionPolicy.")
        if not isinstance(self.policy_provenance, BenchmarkPolicyProvenance):
            raise TypeError("Report policy_provenance must be BenchmarkPolicyProvenance.")
        paired_strata_count: int | None = None
        paired_supercycle_multiple: int | None = None
        if self.design == "independent":
            if self.precision_policy.enabled:
                raise ValueError("Independent reports cannot enable paired precision planning.")
            if paired_collections:
                raise ValueError("Independent reports cannot include paired collection provenance.")
        else:
            if baseline_collections or candidate_collections:
                raise ValueError("Paired reports cannot include independent collection provenance.")
            if len(baselines) != len(candidates):
                raise ValueError("Paired reports require equal baseline and candidate source counts.")
            evidence_paths = tuple(Path(source).resolve() for source in (*baselines, *candidates))
            if len(set(evidence_paths)) != len(evidence_paths):
                raise ValueError("Paired report complete-pair sources cannot reuse a file as independent evidence.")
            if paired_collections:
                supercycle_multiples = {
                    balanced_order_supercycle_length(len(collection.expected_cells))
                    for collection in paired_collections
                }
                if len(supercycle_multiples) != 1:
                    raise ValueError("Paired collection provenance has conflicting joint-design supercycles.")
                paired_supercycle_multiple = next(iter(supercycle_multiples))
                paired_strata_count = len(
                    {
                        baseline.pair_order
                        for collection in paired_collections
                        for baseline, _candidate in collection.complete_pair_records
                    }
                )
                expected_baselines = tuple(
                    source for collection in paired_collections for source in collection.baseline_sources
                )
                expected_candidates = tuple(
                    source for collection in paired_collections for source in collection.candidate_sources
                )
                if baselines != expected_baselines or candidates != expected_candidates:
                    raise ValueError("Paired report sources must match complete same-block record paths in pair order.")
        object.__setattr__(self, "baselines", baselines)
        object.__setattr__(self, "candidates", candidates)
        object.__setattr__(self, "baseline_collections", baseline_collections)
        object.__setattr__(self, "candidate_collections", candidate_collections)
        object.__setattr__(self, "paired_collections", paired_collections)
        object.__setattr__(self, "comparisons", comparisons)
        object.__setattr__(self, "threshold_provenance", threshold_provenance)
        for cell, threshold in zip(comparisons, threshold_provenance, strict=True):
            _validate_threshold_provenance(
                cell,
                threshold,
                regression_policy=self.regression_policy,
                policy_provenance=self.policy_provenance,
            )
            _validate_cell_inference(
                cell,
                inference_policy=self.inference_policy,
                environment_compatible=self.compatibility.is_compatible,
                design=self.design,
                expected_pair_count=(len(baselines) if self.design == "paired" else None),
                expected_strata_count=paired_strata_count,
            )
            _validate_cell_precision(
                cell,
                precision_policy=self.precision_policy,
                inference_policy=self.inference_policy,
                evidence_policy=self.evidence_policy,
                design=self.design,
                expected_pair_count=(len(baselines) if self.design == "paired" else None),
                expected_strata_count=paired_strata_count,
                expected_pair_count_multiple=paired_supercycle_multiple,
            )
        inferences = tuple(cell.inference for cell in comparisons if cell.inference is not None)
        if inferences:
            family_sizes = {inference.family_size for inference in inferences}
            if len(family_sizes) != 1 or next(iter(family_sizes)) != len(inferences):
                raise ValueError("Reported inference family size is inconsistent with inferred cells.")
        plans = tuple(cell.precision for cell in comparisons if cell.precision is not None)
        if plans:
            family_sizes = {plan.family_size for plan in plans}
            if len(family_sizes) != 1:
                raise ValueError("Reported precision-plan family sizes are inconsistent across cells.")
            if self.compatibility.is_compatible and next(iter(family_sizes)) != len(plans):
                raise ValueError("Reported precision-plan family size is inconsistent with planned cells.")

    @classmethod
    def from_comparison(
        cls,
        comparison: BenchmarkRunComparison,
        *,
        baselines: Sequence[str | Path],
        candidates: Sequence[str | Path],
        policy_provenance: BenchmarkPolicyProvenance,
        threshold_provenance: Sequence[BenchmarkThresholdProvenance],
        baseline_collections: Sequence[BenchmarkRunGroup] = (),
        candidate_collections: Sequence[BenchmarkRunGroup] = (),
        paired_collections: Sequence[BenchmarkPairedRunGroup] = (),
    ) -> BenchmarkComparisonReport:
        """Create a portable report from a live comparison result."""
        return cls(
            baselines=tuple(str(source) for source in baselines),
            candidates=tuple(str(source) for source in candidates),
            baseline_collections=tuple(BenchmarkCollectionSnapshot.from_group(group) for group in baseline_collections),
            candidate_collections=tuple(
                BenchmarkCollectionSnapshot.from_group(group) for group in candidate_collections
            ),
            paired_collections=tuple(
                BenchmarkPairedCollectionSnapshot.from_group(group) for group in paired_collections
            ),
            compatibility=comparison.compatibility,
            evidence_policy=comparison.evidence_policy,
            regression_policy=comparison.regression_policy,
            policy_provenance=policy_provenance,
            comparisons=comparison.comparisons,
            threshold_provenance=tuple(threshold_provenance),
            inference_policy=comparison.inference_policy,
            design=comparison.design,
            precision_policy=comparison.precision_policy,
        )

    @property
    def producer(self) -> str:
        """Return the comparison report producer identifier."""
        return PRODUCER

    @property
    def kind(self) -> str:
        """Return the comparison report document kind."""
        return COMPARISON_REPORT_KIND

    @property
    def schema_version(self) -> int:
        """Return the comparison report schema version."""
        return COMPARISON_REPORT_SCHEMA_VERSION

    @property
    def baseline(self) -> str:
        """Return the primary baseline source."""
        return self.baselines[0]

    @property
    def candidate(self) -> str:
        """Return the primary candidate source."""
        return self.candidates[0]

    @property
    def improved(self) -> tuple[BenchmarkComparison, ...]:
        """Return cells classified as improvements."""
        return tuple(cell for cell in self.comparisons if cell.regression == "improved")

    @property
    def unchanged(self) -> tuple[BenchmarkComparison, ...]:
        """Return cells classified as unchanged."""
        return tuple(cell for cell in self.comparisons if cell.regression == "unchanged")

    @property
    def regressed(self) -> tuple[BenchmarkComparison, ...]:
        """Return cells classified as regressions."""
        return tuple(cell for cell in self.comparisons if cell.regression == "regressed")

    @property
    def inconclusive(self) -> tuple[BenchmarkComparison, ...]:
        """Return cells with inconclusive evidence."""
        return tuple(cell for cell in self.comparisons if cell.regression == "inconclusive")

    @property
    def not_comparable(self) -> tuple[BenchmarkComparison, ...]:
        """Return cells without a trustworthy comparison."""
        return tuple(cell for cell in self.comparisons if cell.regression == "not_comparable")

    @property
    def comparison_passed(self) -> bool:
        """Return whether the benchmark comparison itself passed."""
        complete = all(cell.status == "matched" for cell in self.comparisons)
        comparable = self.compatibility.is_compatible and complete and not self.not_comparable
        return comparable and not self.regressed and not self.inconclusive

    @property
    def passed(self) -> bool:
        """Return whether comparison and collection lifecycle gates passed."""
        collections = (*self.baseline_collections, *self.candidate_collections, *self.paired_collections)
        return self.comparison_passed and all(collection.complete for collection in collections)

    @property
    def is_comparable(self) -> bool:
        """Return whether all report cells and environments are comparable."""
        complete = all(cell.status == "matched" for cell in self.comparisons)
        return self.compatibility.is_compatible and complete and not self.not_comparable

    @property
    def has_regressions(self) -> bool:
        """Return whether any matrix cell regressed."""
        return bool(self.regressed)

    def to_dict(self) -> dict[str, JsonValue]:
        """Return the complete stable comparison report document."""
        return {
            "producer": self.producer,
            "kind": self.kind,
            "schema_version": self.schema_version,
            "baseline": self.baseline,
            "candidate": self.candidate,
            "baselines": list(self.baselines),
            "candidates": list(self.candidates),
            "baseline_collections": [collection.to_dict() for collection in self.baseline_collections],
            "candidate_collections": [collection.to_dict() for collection in self.candidate_collections],
            "paired_collections": [collection.to_dict() for collection in self.paired_collections],
            "design": self.design,
            "passed": self.passed,
            "comparison_passed": self.comparison_passed,
            "is_comparable": self.is_comparable,
            "has_regressions": self.has_regressions,
            "compatibility": _compatibility_dict(self.compatibility),
            "summary": {
                "improved": len(self.improved),
                "unchanged": len(self.unchanged),
                "regressed": len(self.regressed),
                "inconclusive": len(self.inconclusive),
                "not_comparable": len(self.not_comparable),
            },
            "evidence_policy": _evidence_policy_dict(self.evidence_policy),
            "inference_policy": _inference_policy_dict(self.inference_policy),
            "precision_policy": _precision_policy_dict(self.precision_policy),
            "policy": _policy_dict(self.policy_provenance, self.regression_policy),
            "comparisons": [
                _comparison_dict(cell, threshold)
                for cell, threshold in zip(
                    self.comparisons,
                    self.threshold_provenance,
                    strict=True,
                )
            ],
        }

producer property

producer: str

Return the comparison report producer identifier.

kind property

kind: str

Return the comparison report document kind.

schema_version property

schema_version: int

Return the comparison report schema version.

baseline property

baseline: str

Return the primary baseline source.

candidate property

candidate: str

Return the primary candidate source.

improved property

improved: tuple[BenchmarkComparison, ...]

Return cells classified as improvements.

unchanged property

unchanged: tuple[BenchmarkComparison, ...]

Return cells classified as unchanged.

regressed property

regressed: tuple[BenchmarkComparison, ...]

Return cells classified as regressions.

inconclusive property

inconclusive: tuple[BenchmarkComparison, ...]

Return cells with inconclusive evidence.

not_comparable property

not_comparable: tuple[BenchmarkComparison, ...]

Return cells without a trustworthy comparison.

comparison_passed property

comparison_passed: bool

Return whether the benchmark comparison itself passed.

passed property

passed: bool

Return whether comparison and collection lifecycle gates passed.

is_comparable property

is_comparable: bool

Return whether all report cells and environments are comparable.

has_regressions property

has_regressions: bool

Return whether any matrix cell regressed.

__post_init__

__post_init__() -> None

Validate and normalize report containers.

Source code in src/benchmatrix/bench_report.py

def __post_init__(self) -> None:
    """Validate and normalize report containers."""
    baselines = tuple(self.baselines)
    candidates = tuple(self.candidates)
    baseline_collections = tuple(self.baseline_collections)
    candidate_collections = tuple(self.candidate_collections)
    paired_collections = tuple(self.paired_collections)
    comparisons = tuple(self.comparisons)
    threshold_provenance = tuple(self.threshold_provenance)
    if not baselines or any(not isinstance(source, str) or not source for source in baselines):
        raise ValueError("Report baselines must contain non-empty source strings.")
    if not candidates or any(not isinstance(source, str) or not source for source in candidates):
        raise ValueError("Report candidates must contain non-empty source strings.")
    if len(comparisons) != len(threshold_provenance):
        raise ValueError("Every reported comparison requires threshold provenance.")
    if any(
        not isinstance(item, BenchmarkCollectionSnapshot)
        for item in (*baseline_collections, *candidate_collections)
    ):
        raise TypeError("Report collections must contain BenchmarkCollectionSnapshot values.")
    if any(not isinstance(item, BenchmarkPairedCollectionSnapshot) for item in paired_collections):
        raise TypeError("Report paired_collections must contain BenchmarkPairedCollectionSnapshot values.")
    if any(not isinstance(item, BenchmarkComparison) for item in comparisons):
        raise TypeError("Report comparisons must contain BenchmarkComparison values.")
    if any(not isinstance(item, BenchmarkThresholdProvenance) for item in threshold_provenance):
        raise TypeError("Report threshold_provenance must contain BenchmarkThresholdProvenance values.")
    if not isinstance(self.compatibility, RunCompatibilityReport):
        raise TypeError("Report compatibility must be a RunCompatibilityReport.")
    if not isinstance(self.evidence_policy, EvidencePolicy):
        raise TypeError("Report evidence_policy must be an EvidencePolicy.")
    if not isinstance(self.regression_policy, RegressionPolicy):
        raise TypeError("Report regression_policy must be a RegressionPolicy.")
    if not isinstance(self.inference_policy, InferencePolicy):
        raise TypeError("Report inference_policy must be an InferencePolicy.")
    if self.design not in {"independent", "paired"}:
        raise ValueError(f"Unsupported report comparison design: {self.design!r}.")
    if not isinstance(self.precision_policy, PrecisionPolicy):
        raise TypeError("Report precision_policy must be a PrecisionPolicy.")
    if not isinstance(self.policy_provenance, BenchmarkPolicyProvenance):
        raise TypeError("Report policy_provenance must be BenchmarkPolicyProvenance.")
    paired_strata_count: int | None = None
    paired_supercycle_multiple: int | None = None
    if self.design == "independent":
        if self.precision_policy.enabled:
            raise ValueError("Independent reports cannot enable paired precision planning.")
        if paired_collections:
            raise ValueError("Independent reports cannot include paired collection provenance.")
    else:
        if baseline_collections or candidate_collections:
            raise ValueError("Paired reports cannot include independent collection provenance.")
        if len(baselines) != len(candidates):
            raise ValueError("Paired reports require equal baseline and candidate source counts.")
        evidence_paths = tuple(Path(source).resolve() for source in (*baselines, *candidates))
        if len(set(evidence_paths)) != len(evidence_paths):
            raise ValueError("Paired report complete-pair sources cannot reuse a file as independent evidence.")
        if paired_collections:
            supercycle_multiples = {
                balanced_order_supercycle_length(len(collection.expected_cells))
                for collection in paired_collections
            }
            if len(supercycle_multiples) != 1:
                raise ValueError("Paired collection provenance has conflicting joint-design supercycles.")
            paired_supercycle_multiple = next(iter(supercycle_multiples))
            paired_strata_count = len(
                {
                    baseline.pair_order
                    for collection in paired_collections
                    for baseline, _candidate in collection.complete_pair_records
                }
            )
            expected_baselines = tuple(
                source for collection in paired_collections for source in collection.baseline_sources
            )
            expected_candidates = tuple(
                source for collection in paired_collections for source in collection.candidate_sources
            )
            if baselines != expected_baselines or candidates != expected_candidates:
                raise ValueError("Paired report sources must match complete same-block record paths in pair order.")
    object.__setattr__(self, "baselines", baselines)
    object.__setattr__(self, "candidates", candidates)
    object.__setattr__(self, "baseline_collections", baseline_collections)
    object.__setattr__(self, "candidate_collections", candidate_collections)
    object.__setattr__(self, "paired_collections", paired_collections)
    object.__setattr__(self, "comparisons", comparisons)
    object.__setattr__(self, "threshold_provenance", threshold_provenance)
    for cell, threshold in zip(comparisons, threshold_provenance, strict=True):
        _validate_threshold_provenance(
            cell,
            threshold,
            regression_policy=self.regression_policy,
            policy_provenance=self.policy_provenance,
        )
        _validate_cell_inference(
            cell,
            inference_policy=self.inference_policy,
            environment_compatible=self.compatibility.is_compatible,
            design=self.design,
            expected_pair_count=(len(baselines) if self.design == "paired" else None),
            expected_strata_count=paired_strata_count,
        )
        _validate_cell_precision(
            cell,
            precision_policy=self.precision_policy,
            inference_policy=self.inference_policy,
            evidence_policy=self.evidence_policy,
            design=self.design,
            expected_pair_count=(len(baselines) if self.design == "paired" else None),
            expected_strata_count=paired_strata_count,
            expected_pair_count_multiple=paired_supercycle_multiple,
        )
    inferences = tuple(cell.inference for cell in comparisons if cell.inference is not None)
    if inferences:
        family_sizes = {inference.family_size for inference in inferences}
        if len(family_sizes) != 1 or next(iter(family_sizes)) != len(inferences):
            raise ValueError("Reported inference family size is inconsistent with inferred cells.")
    plans = tuple(cell.precision for cell in comparisons if cell.precision is not None)
    if plans:
        family_sizes = {plan.family_size for plan in plans}
        if len(family_sizes) != 1:
            raise ValueError("Reported precision-plan family sizes are inconsistent across cells.")
        if self.compatibility.is_compatible and next(iter(family_sizes)) != len(plans):
            raise ValueError("Reported precision-plan family size is inconsistent with planned cells.")

from_comparison classmethod

from_comparison(
    comparison: BenchmarkRunComparison,
    *,
    baselines: Sequence[str | Path],
    candidates: Sequence[str | Path],
    policy_provenance: BenchmarkPolicyProvenance,
    threshold_provenance: Sequence[
        BenchmarkThresholdProvenance
    ],
    baseline_collections: Sequence[BenchmarkRunGroup] = (),
    candidate_collections: Sequence[BenchmarkRunGroup] = (),
    paired_collections: Sequence[
        BenchmarkPairedRunGroup
    ] = (),
) -> BenchmarkComparisonReport

Create a portable report from a live comparison result.

Source code in src/benchmatrix/bench_report.py

@classmethod
def from_comparison(
    cls,
    comparison: BenchmarkRunComparison,
    *,
    baselines: Sequence[str | Path],
    candidates: Sequence[str | Path],
    policy_provenance: BenchmarkPolicyProvenance,
    threshold_provenance: Sequence[BenchmarkThresholdProvenance],
    baseline_collections: Sequence[BenchmarkRunGroup] = (),
    candidate_collections: Sequence[BenchmarkRunGroup] = (),
    paired_collections: Sequence[BenchmarkPairedRunGroup] = (),
) -> BenchmarkComparisonReport:
    """Create a portable report from a live comparison result."""
    return cls(
        baselines=tuple(str(source) for source in baselines),
        candidates=tuple(str(source) for source in candidates),
        baseline_collections=tuple(BenchmarkCollectionSnapshot.from_group(group) for group in baseline_collections),
        candidate_collections=tuple(
            BenchmarkCollectionSnapshot.from_group(group) for group in candidate_collections
        ),
        paired_collections=tuple(
            BenchmarkPairedCollectionSnapshot.from_group(group) for group in paired_collections
        ),
        compatibility=comparison.compatibility,
        evidence_policy=comparison.evidence_policy,
        regression_policy=comparison.regression_policy,
        policy_provenance=policy_provenance,
        comparisons=comparison.comparisons,
        threshold_provenance=tuple(threshold_provenance),
        inference_policy=comparison.inference_policy,
        design=comparison.design,
        precision_policy=comparison.precision_policy,
    )

to_dict

to_dict() -> dict[str, JsonValue]

Return the complete stable comparison report document.

Source code in src/benchmatrix/bench_report.py

def to_dict(self) -> dict[str, JsonValue]:
    """Return the complete stable comparison report document."""
    return {
        "producer": self.producer,
        "kind": self.kind,
        "schema_version": self.schema_version,
        "baseline": self.baseline,
        "candidate": self.candidate,
        "baselines": list(self.baselines),
        "candidates": list(self.candidates),
        "baseline_collections": [collection.to_dict() for collection in self.baseline_collections],
        "candidate_collections": [collection.to_dict() for collection in self.candidate_collections],
        "paired_collections": [collection.to_dict() for collection in self.paired_collections],
        "design": self.design,
        "passed": self.passed,
        "comparison_passed": self.comparison_passed,
        "is_comparable": self.is_comparable,
        "has_regressions": self.has_regressions,
        "compatibility": _compatibility_dict(self.compatibility),
        "summary": {
            "improved": len(self.improved),
            "unchanged": len(self.unchanged),
            "regressed": len(self.regressed),
            "inconclusive": len(self.inconclusive),
            "not_comparable": len(self.not_comparable),
        },
        "evidence_policy": _evidence_policy_dict(self.evidence_policy),
        "inference_policy": _inference_policy_dict(self.inference_policy),
        "precision_policy": _precision_policy_dict(self.precision_policy),
        "policy": _policy_dict(self.policy_provenance, self.regression_policy),
        "comparisons": [
            _comparison_dict(cell, threshold)
            for cell, threshold in zip(
                self.comparisons,
                self.threshold_provenance,
                strict=True,
            )
        ],
    }

load_comparison_report

load_comparison_report(
    path: str | Path,
) -> BenchmarkComparisonReport

Load and strictly validate a versioned comparison report.

Parameters:

Name	Type	Description	Default
path	str | Path	JSON report written by benchmatrix compare --format json or :func:write_comparison_report.	required

Returns:

Type	Description
BenchmarkComparisonReport	A portable typed comparison report.

Raises:

Type	Description
BenchmarkJsonError	If the file is unreadable, malformed, unsupported, or inconsistent with the report schema.

Source code in src/benchmatrix/bench_report.py

def load_comparison_report(path: str | Path) -> BenchmarkComparisonReport:
    """Load and strictly validate a versioned comparison report.

    Args:
        path: JSON report written by ``benchmatrix compare --format json`` or
            :func:`write_comparison_report`.

    Returns:
        A portable typed comparison report.

    Raises:
        BenchmarkJsonError: If the file is unreadable, malformed, unsupported,
            or inconsistent with the report schema.
    """
    source = Path(path)
    try:
        payload = cast(object, json.loads(source.read_text(encoding="utf-8")))
    except OSError as exc:
        raise BenchmarkJsonError(f"Could not read benchmark comparison report: {source}") from exc
    except json.JSONDecodeError as exc:
        raise BenchmarkJsonError(f"Invalid JSON in benchmark comparison report: {source}") from exc
    return _parse_report(payload)

write_comparison_report

write_comparison_report(
    report: BenchmarkComparisonReport, path: str | Path
) -> None

Write a comparison report as deterministic strict JSON.

Parameters:

Name	Type	Description	Default
report	BenchmarkComparisonReport	Portable report to serialize.	required
path	str | Path	Destination JSON path.	required

Raises:

Type	Description
TypeError	If report is not a BenchmarkComparisonReport.
OSError	If the destination cannot be written.

Source code in src/benchmatrix/bench_report.py

def write_comparison_report(report: BenchmarkComparisonReport, path: str | Path) -> None:
    """Write a comparison report as deterministic strict JSON.

    Args:
        report: Portable report to serialize.
        path: Destination JSON path.

    Raises:
        TypeError: If ``report`` is not a ``BenchmarkComparisonReport``.
        OSError: If the destination cannot be written.
    """
    if not isinstance(report, BenchmarkComparisonReport):
        raise TypeError("report must be a BenchmarkComparisonReport.")
    destination = Path(path)
    destination.write_text(
        json.dumps(report.to_dict(), allow_nan=False, indent=2, sort_keys=True) + "\n",
        encoding="utf-8",
    )

format_comparison_report_markdown

format_comparison_report_markdown(
    report: BenchmarkComparisonReport,
) -> str

Render a comparison report as deterministic GitHub-flavored Markdown.

Parameters:

Name	Type	Description	Default
report	BenchmarkComparisonReport	Portable comparison report to render.	required

Returns:

Type	Description
str	A complete Markdown document ending with a newline.

Raises:

Type	Description
TypeError	If report is not a BenchmarkComparisonReport.

Source code in src/benchmatrix/bench_report.py

def format_comparison_report_markdown(report: BenchmarkComparisonReport) -> str:
    """Render a comparison report as deterministic GitHub-flavored Markdown.

    Args:
        report: Portable comparison report to render.

    Returns:
        A complete Markdown document ending with a newline.

    Raises:
        TypeError: If ``report`` is not a ``BenchmarkComparisonReport``.
    """
    if not isinstance(report, BenchmarkComparisonReport):
        raise TypeError("report must be a BenchmarkComparisonReport.")

    overall = "PASS" if report.passed else "FAIL"
    comparison = "PASS" if report.comparison_passed else "FAIL"
    compatibility = "compatible" if report.compatibility.is_compatible else "blocked"
    lines = [
        "# Benchmark comparison",
        "",
        f"**Overall:** {overall}  ",
        f"**Comparison decision:** {comparison}  ",
        f"**Environment compatibility:** {compatibility}  ",
        f"**Comparison design:** `{report.design}`  ",
        f"**Report schema:** `{report.kind}` version {report.schema_version}",
        "",
        "## Inputs",
        "",
        "| Side | Sources | Collections |",
        "| --- | --- | ---: |",
        (f"| Baseline | {_markdown_sources(report.baselines)} | " + f"{len(report.baseline_collections)} |"),
        (f"| Candidate | {_markdown_sources(report.candidates)} | " + f"{len(report.candidate_collections)} |"),
        "",
    ]
    collections = (*report.baseline_collections, *report.candidate_collections)
    if collections:
        lines.extend(
            [
                "### Collection lifecycle",
                "",
                "| Manifest | Target | Attempts | Successful | Failed | Retries | Complete |",
                "| --- | ---: | ---: | ---: | ---: | ---: | --- |",
            ]
        )
        for collection in collections:
            lines.append(
                "| "
                + " | ".join(
                    (
                        _markdown_text(collection.manifest),
                        str(collection.requested_runs),
                        str(collection.attempted_runs),
                        str(collection.successful_runs),
                        str(collection.failed_runs),
                        str(collection.retry_attempts),
                        "yes" if collection.complete else "no",
                    )
                )
                + " |"
            )
        lines.append("")
    if report.paired_collections:
        lines.extend(
            [
                "### Paired AB/BA collection lifecycle",
                "",
                (
                    "| Manifest | Pair target | Command attempts | Complete pairs | "
                    + "Failed commands | Orphan successes | Complete |"
                ),
                "| --- | ---: | ---: | ---: | ---: | ---: | --- |",
            ]
        )
        for collection in report.paired_collections:
            lines.append(
                "| "
                + " | ".join(
                    (
                        _markdown_text(collection.manifest),
                        str(collection.requested_pairs),
                        str(collection.attempted_commands),
                        str(collection.complete_pair_count),
                        str(collection.failed_commands),
                        str(collection.orphan_success_count),
                        "yes" if collection.complete else "no",
                    )
                )
                + " |"
            )
        lines.append("")

    lines.extend(
        [
            "## Summary",
            "",
            "| Improved | Unchanged | Regressed | Inconclusive | Not comparable |",
            "| ---: | ---: | ---: | ---: | ---: |",
            (
                f"| {len(report.improved)} | {len(report.unchanged)} | "
                + f"{len(report.regressed)} | {len(report.inconclusive)} | "
                + f"{len(report.not_comparable)} |"
            ),
            "",
            "## Matrix results",
            "",
            (
                "| Implementation | Case | Metric | Result | Baseline | Candidate | "
                + "Improvement | Confidence interval | Observed pairwise range | Threshold | Evidence |"
            ),
            "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |",
        ]
    )
    for cell, threshold in zip(
        report.comparisons,
        report.threshold_provenance,
        strict=True,
    ):
        result = cell.regression if cell.status == "matched" else cell.status
        effect_range = _markdown_effect_range(
            cell.improvement_low_percent,
            cell.improvement_high_percent,
        )
        confidence_interval = _markdown_confidence_interval(cell.inference)
        evidence = _markdown_evidence_state(
            cell.baseline_evidence,
            cell.candidate_evidence,
        )
        lines.append(
            "| "
            + " | ".join(
                (
                    _markdown_text(cell.implementation_name),
                    _markdown_text(cell.case_name),
                    _markdown_text(cell.metric_name),
                    _markdown_text(result),
                    _markdown_number(cell.baseline_value),
                    _markdown_number(cell.candidate_value),
                    _markdown_percent(cell.improvement_percent),
                    confidence_interval,
                    effect_range,
                    (
                        f"{cell.threshold_percent:.2f}% "
                        + f"({_markdown_text(threshold.scope)}/{_markdown_text(threshold.origin)})"
                    ),
                    evidence,
                )
            )
            + " |"
        )
    lines.append("")

    lines.extend(
        [
            "## Environment compatibility",
            "",
            (
                f"{report.compatibility.pairs_checked} environment pair(s) checked; "
                + f"{len(report.compatibility.blocking)} blocking finding(s); "
                + f"{len(report.compatibility.warnings)} warning(s)."
            ),
            "",
        ]
    )
    if report.compatibility.findings:
        lines.extend(
            [
                "| Severity | Field | Baseline | Candidate | Reason |",
                "| --- | --- | --- | --- | --- |",
            ]
        )
        for finding in report.compatibility.findings:
            lines.append(
                "| "
                + " | ".join(
                    (
                        _markdown_text(finding.severity),
                        _markdown_text(finding.field),
                        _markdown_text(_compact_json(finding.baseline_value)),
                        _markdown_text(_compact_json(finding.candidate_value)),
                        _markdown_text(finding.reason),
                    )
                )
                + " |"
            )
        lines.append("")
    else:
        lines.extend(["No environment differences were reported.", ""])

    lines.extend(
        [
            "## Evidence and diagnostics",
            "",
        ]
    )
    for cell in report.comparisons:
        lines.extend(
            [
                (
                    "### "
                    + " / ".join(
                        (
                            _markdown_text(cell.implementation_name),
                            _markdown_text(cell.case_name),
                            _markdown_text(cell.metric_name),
                        )
                    )
                ),
                "",
                f"- Baseline: {_markdown_evidence_detail(cell.baseline_evidence)}",
                f"- Candidate: {_markdown_evidence_detail(cell.candidate_evidence)}",
                f"- Inference: {_markdown_inference_detail(cell.inference)}",
                f"- Precision planning: {_markdown_precision_detail(cell.precision)}",
            ]
        )
        if cell.reason is not None:
            lines.append(f"- Diagnostic: {_markdown_text(cell.reason)}")
        lines.append("")

    policy = report.regression_policy
    provenance = report.policy_provenance
    lines.extend(
        [
            "## Effective policy",
            "",
            f"- Selection: `{provenance.selection}`",
            (
                "- Configuration: "
                + (
                    _markdown_text(provenance.configuration_file)
                    if provenance.configuration_file is not None
                    else "built-in defaults"
                )
            ),
            (
                "- Configured fields: "
                + (
                    ", ".join(f"`{_markdown_code(field)}`" for field in provenance.configured_fields)
                    if provenance.configured_fields
                    else "none"
                )
            ),
            (
                "- CLI overrides: "
                + (
                    ", ".join(f"`{_markdown_code(field)}`" for field in provenance.cli_overrides)
                    if provenance.cli_overrides
                    else "none"
                )
            ),
            f"- Compatibility mode: `{report.compatibility.policy.mode}`",
            f"- Minimum runs: {report.evidence_policy.minimum_runs}",
            f"- Minimum samples per run: {report.evidence_policy.minimum_samples_per_run}",
            f"- Minimum rounds per run: {report.evidence_policy.minimum_rounds_per_run}",
            (
                "- Require raw samples for inference: "
                + ("yes" if report.evidence_policy.require_raw_samples_for_inference else "no")
            ),
            f"- Minimum tail samples per run: {report.evidence_policy.minimum_tail_samples_per_run}",
            (
                "- Require one iteration for tail latency: "
                + ("yes" if report.evidence_policy.require_tail_iterations_one else "no")
            ),
            f"- Inference method: `{report.inference_policy.method}`",
            f"- Confidence level: {report.inference_policy.confidence_level * 100.0:.2f}%",
            f"- Bootstrap resamples: {report.inference_policy.resamples}",
            f"- Random seed: {report.inference_policy.random_seed}",
            f"- Multiplicity correction: `{report.inference_policy.multiplicity}`",
            (
                "- Precision target: disabled"
                if report.precision_policy.target_half_width_percent is None
                else f"- Precision target half-width: {report.precision_policy.target_half_width_percent:.2f}%"
            ),
            f"- Default regression threshold: {policy.default_threshold_percent:.2f}%",
            (
                "- Selector thresholds: "
                + f"{len(policy.by_metric)} metric, "
                + f"{len(policy.by_implementation)} implementation, "
                + f"{len(policy.by_case)} case, {len(policy.by_cell)} exact cell"
            ),
            "",
        ]
    )
    return "\n".join(lines)

write_comparison_report_markdown

write_comparison_report_markdown(
    report: BenchmarkComparisonReport, path: str | Path
) -> None

Write a comparison report as deterministic Markdown.

Parameters:

Name	Type	Description	Default
report	BenchmarkComparisonReport	Portable report to render.	required
path	str | Path	Destination Markdown path.	required

Raises:

Type	Description
TypeError	If report is not a BenchmarkComparisonReport.
OSError	If the destination cannot be written.

Source code in src/benchmatrix/bench_report.py

def write_comparison_report_markdown(
    report: BenchmarkComparisonReport,
    path: str | Path,
) -> None:
    """Write a comparison report as deterministic Markdown.

    Args:
        report: Portable report to render.
        path: Destination Markdown path.

    Raises:
        TypeError: If ``report`` is not a ``BenchmarkComparisonReport``.
        OSError: If the destination cannot be written.
    """
    Path(path).write_text(format_comparison_report_markdown(report), encoding="utf-8")