Bench Compare

benchmatrix.bench_compare

Matrix-aware comparison of parsed benchmark runs.

RunCompatibilityPolicy dataclass

Policy controlling run-environment compatibility checks.

permissive keeps lower-risk differences as warnings, strict promotes every difference or missing environment record to a blocker, and off disables run-level compatibility checks.

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class RunCompatibilityPolicy:
    """Policy controlling run-environment compatibility checks.

    ``permissive`` keeps lower-risk differences as warnings, ``strict``
    promotes every difference or missing environment record to a blocker, and
    ``off`` disables run-level compatibility checks.
    """

    mode: CompatibilityMode = "permissive"

    def __post_init__(self) -> None:
        """Validate the compatibility mode."""
        if self.mode not in {"strict", "permissive", "off"}:
            raise ValueError(f"Unsupported run compatibility mode: {self.mode!r}.")

__post_init__

__post_init__() -> None

Validate the compatibility mode.

Source code in src/benchmatrix/bench_compare.py

def __post_init__(self) -> None:
    """Validate the compatibility mode."""
    if self.mode not in {"strict", "permissive", "off"}:
        raise ValueError(f"Unsupported run compatibility mode: {self.mode!r}.")

RunCompatibilityFinding dataclass

One material difference between two run environments.

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class RunCompatibilityFinding:
    """One material difference between two run environments."""

    field: str
    baseline_value: object | None
    candidate_value: object | None
    severity: CompatibilitySeverity
    reason: str
    baseline_run: str | None = None
    candidate_run: str | None = None

RunCompatibilityReport dataclass

Compatibility findings for the baseline and candidate environments.

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class RunCompatibilityReport:
    """Compatibility findings for the baseline and candidate environments."""

    policy: RunCompatibilityPolicy
    findings: tuple[RunCompatibilityFinding, ...]
    pairs_checked: int = 1

    @property
    def blocking(self) -> tuple[RunCompatibilityFinding, ...]:
        """Return differences that prevent a trustworthy comparison."""
        return tuple(finding for finding in self.findings if finding.severity == "blocking")

    @property
    def warnings(self) -> tuple[RunCompatibilityFinding, ...]:
        """Return non-blocking environment differences."""
        return tuple(finding for finding in self.findings if finding.severity == "warning")

    @property
    def is_compatible(self) -> bool:
        """Return whether no blocking environment differences were found."""
        return not self.blocking

blocking property

blocking: tuple[RunCompatibilityFinding, ...]

Return differences that prevent a trustworthy comparison.

warnings property

warnings: tuple[RunCompatibilityFinding, ...]

Return non-blocking environment differences.

is_compatible property

is_compatible: bool

Return whether no blocking environment differences were found.

EvidencePolicy dataclass

Minimum evidence required for repeated-run classifications.

Parameters:

Name	Type	Description	Default
minimum_runs	int	Minimum files on each side containing a matrix cell.	5
minimum_samples_per_run	int	Minimum raw timing samples required from each observed file.	5
minimum_rounds_per_run	int	Minimum pytest-benchmark rounds required from each observed file.	5
require_rounds	bool	Whether every row must report a positive round count.	True
require_iterations	bool	Whether every row must report a positive iteration count.	True
require_raw_samples_for_inference	bool	Whether every observed row must retain raw per-round durations.	True
minimum_tail_samples_per_run	int	Minimum round-duration observations required from each tail-latency row.	100
require_tail_iterations_one	bool	Whether tail-latency rows must represent individual calls rather than averages of multiple iterations.	True
maximum_cv	float | None	Optional maximum within-run coefficient of variation.	None
maximum_outlier_fraction	float | None	Optional maximum within-run Tukey-outlier fraction.	None

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class EvidencePolicy:
    """Minimum evidence required for repeated-run classifications.

    Args:
        minimum_runs: Minimum files on each side containing a matrix cell.
        minimum_samples_per_run: Minimum raw timing samples required from each
            observed file.
        minimum_rounds_per_run: Minimum pytest-benchmark rounds required from
            each observed file.
        require_rounds: Whether every row must report a positive round count.
        require_iterations: Whether every row must report a positive iteration
            count.
        require_raw_samples_for_inference: Whether every observed row must
            retain raw per-round durations.
        minimum_tail_samples_per_run: Minimum round-duration observations
            required from each tail-latency row.
        require_tail_iterations_one: Whether tail-latency rows must represent
            individual calls rather than averages of multiple iterations.
        maximum_cv: Optional maximum within-run coefficient of variation.
        maximum_outlier_fraction: Optional maximum within-run Tukey-outlier
            fraction.
    """

    minimum_runs: int = 5
    minimum_samples_per_run: int = 5
    minimum_rounds_per_run: int = 5
    require_rounds: bool = True
    require_iterations: bool = True
    require_raw_samples_for_inference: bool = True
    minimum_tail_samples_per_run: int = 100
    require_tail_iterations_one: bool = True
    maximum_cv: float | None = None
    maximum_outlier_fraction: float | None = None

    def __post_init__(self) -> None:
        """Validate evidence thresholds."""
        if isinstance(self.minimum_runs, bool) or not isinstance(self.minimum_runs, int):
            raise TypeError("EvidencePolicy.minimum_runs must be an integer.")
        if self.minimum_runs <= 0:
            raise ValueError("EvidencePolicy.minimum_runs must be a positive integer.")
        if isinstance(self.minimum_samples_per_run, bool) or not isinstance(self.minimum_samples_per_run, int):
            raise TypeError("EvidencePolicy.minimum_samples_per_run must be an integer.")
        if self.minimum_samples_per_run < 0:
            raise ValueError("EvidencePolicy.minimum_samples_per_run must be a non-negative integer.")
        if isinstance(self.minimum_rounds_per_run, bool) or not isinstance(self.minimum_rounds_per_run, int):
            raise TypeError("EvidencePolicy.minimum_rounds_per_run must be an integer.")
        if self.minimum_rounds_per_run < 0:
            raise ValueError("EvidencePolicy.minimum_rounds_per_run must be a non-negative integer.")
        if not isinstance(self.require_rounds, bool):
            raise TypeError("EvidencePolicy.require_rounds must be a boolean.")
        if not isinstance(self.require_iterations, bool):
            raise TypeError("EvidencePolicy.require_iterations must be a boolean.")
        if not isinstance(self.require_raw_samples_for_inference, bool):
            raise TypeError("EvidencePolicy.require_raw_samples_for_inference must be a boolean.")
        if isinstance(self.minimum_tail_samples_per_run, bool) or not isinstance(
            self.minimum_tail_samples_per_run, int
        ):
            raise TypeError("EvidencePolicy.minimum_tail_samples_per_run must be an integer.")
        if self.minimum_tail_samples_per_run < 0:
            raise ValueError("EvidencePolicy.minimum_tail_samples_per_run must be a non-negative integer.")
        if not isinstance(self.require_tail_iterations_one, bool):
            raise TypeError("EvidencePolicy.require_tail_iterations_one must be a boolean.")
        if self.maximum_cv is not None:
            object.__setattr__(
                self,
                "maximum_cv",
                _validate_non_negative_number(
                    self.maximum_cv,
                    field_name="EvidencePolicy.maximum_cv",
                ),
            )
        if self.maximum_outlier_fraction is not None:
            object.__setattr__(
                self,
                "maximum_outlier_fraction",
                _validate_fraction(
                    self.maximum_outlier_fraction,
                    field_name="EvidencePolicy.maximum_outlier_fraction",
                ),
            )

__post_init__

__post_init__() -> None

Validate evidence thresholds.

Source code in src/benchmatrix/bench_compare.py

def __post_init__(self) -> None:
    """Validate evidence thresholds."""
    if isinstance(self.minimum_runs, bool) or not isinstance(self.minimum_runs, int):
        raise TypeError("EvidencePolicy.minimum_runs must be an integer.")
    if self.minimum_runs <= 0:
        raise ValueError("EvidencePolicy.minimum_runs must be a positive integer.")
    if isinstance(self.minimum_samples_per_run, bool) or not isinstance(self.minimum_samples_per_run, int):
        raise TypeError("EvidencePolicy.minimum_samples_per_run must be an integer.")
    if self.minimum_samples_per_run < 0:
        raise ValueError("EvidencePolicy.minimum_samples_per_run must be a non-negative integer.")
    if isinstance(self.minimum_rounds_per_run, bool) or not isinstance(self.minimum_rounds_per_run, int):
        raise TypeError("EvidencePolicy.minimum_rounds_per_run must be an integer.")
    if self.minimum_rounds_per_run < 0:
        raise ValueError("EvidencePolicy.minimum_rounds_per_run must be a non-negative integer.")
    if not isinstance(self.require_rounds, bool):
        raise TypeError("EvidencePolicy.require_rounds must be a boolean.")
    if not isinstance(self.require_iterations, bool):
        raise TypeError("EvidencePolicy.require_iterations must be a boolean.")
    if not isinstance(self.require_raw_samples_for_inference, bool):
        raise TypeError("EvidencePolicy.require_raw_samples_for_inference must be a boolean.")
    if isinstance(self.minimum_tail_samples_per_run, bool) or not isinstance(
        self.minimum_tail_samples_per_run, int
    ):
        raise TypeError("EvidencePolicy.minimum_tail_samples_per_run must be an integer.")
    if self.minimum_tail_samples_per_run < 0:
        raise ValueError("EvidencePolicy.minimum_tail_samples_per_run must be a non-negative integer.")
    if not isinstance(self.require_tail_iterations_one, bool):
        raise TypeError("EvidencePolicy.require_tail_iterations_one must be a boolean.")
    if self.maximum_cv is not None:
        object.__setattr__(
            self,
            "maximum_cv",
            _validate_non_negative_number(
                self.maximum_cv,
                field_name="EvidencePolicy.maximum_cv",
            ),
        )
    if self.maximum_outlier_fraction is not None:
        object.__setattr__(
            self,
            "maximum_outlier_fraction",
            _validate_fraction(
                self.maximum_outlier_fraction,
                field_name="EvidencePolicy.maximum_outlier_fraction",
            ),
        )

BenchmarkEvidence dataclass

Trust diagnostics for one side of a matrix-cell comparison.

Attributes:

Name	Type	Description
provided_run_count	int	Files supplied for this side.
observed_run_count	int	Files containing this matrix cell.
rounds	tuple[int | None, ...]	Positive pytest-benchmark round counts aligned to the files.
iterations	tuple[int | None, ...]	Positive iteration counts aligned to the files.
sample_counts	tuple[int, ...]	Raw timing sample counts aligned to the files.
sample_count	int	Total pooled raw timing samples.
iqr	float | None	Interquartile range of pooled timing samples in seconds. Retained as a descriptive compatibility field; evidence gates use the corresponding per-run diagnostics.
coefficient_of_variation	float | None	Pooled timing-sample population standard deviation divided by the absolute mean.
outlier_count	int | None	Samples outside the pooled 1.5-IQR Tukey fences.
outlier_fraction	float | None	Outlier count divided by total sample count.
adequate	bool	Whether the configured evidence policy was satisfied.
issues	tuple[str, ...]	Human-readable reasons evidence is inadequate.
run_iqrs	tuple[float | None, ...]	Per-run timing-sample interquartile ranges.
run_coefficients_of_variation	tuple[float | None, ...]	Per-run coefficients of variation.
run_outlier_counts	tuple[int | None, ...]	Per-run Tukey-outlier counts.
run_outlier_fractions	tuple[float | None, ...]	Per-run Tukey-outlier fractions.

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class BenchmarkEvidence:
    """Trust diagnostics for one side of a matrix-cell comparison.

    Attributes:
        provided_run_count: Files supplied for this side.
        observed_run_count: Files containing this matrix cell.
        rounds: Positive pytest-benchmark round counts aligned to the files.
        iterations: Positive iteration counts aligned to the files.
        sample_counts: Raw timing sample counts aligned to the files.
        sample_count: Total pooled raw timing samples.
        iqr: Interquartile range of pooled timing samples in seconds. Retained
            as a descriptive compatibility field; evidence gates use the
            corresponding per-run diagnostics.
        coefficient_of_variation: Pooled timing-sample population standard
            deviation divided by the absolute mean.
        outlier_count: Samples outside the pooled 1.5-IQR Tukey fences.
        outlier_fraction: Outlier count divided by total sample count.
        adequate: Whether the configured evidence policy was satisfied.
        issues: Human-readable reasons evidence is inadequate.
        run_iqrs: Per-run timing-sample interquartile ranges.
        run_coefficients_of_variation: Per-run coefficients of variation.
        run_outlier_counts: Per-run Tukey-outlier counts.
        run_outlier_fractions: Per-run Tukey-outlier fractions.
    """

    provided_run_count: int
    observed_run_count: int
    rounds: tuple[int | None, ...]
    iterations: tuple[int | None, ...]
    sample_counts: tuple[int, ...]
    sample_count: int
    iqr: float | None
    coefficient_of_variation: float | None
    outlier_count: int | None
    outlier_fraction: float | None
    adequate: bool
    issues: tuple[str, ...]
    run_iqrs: tuple[float | None, ...] = ()
    run_coefficients_of_variation: tuple[float | None, ...] = ()
    run_outlier_counts: tuple[int | None, ...] = ()
    run_outlier_fractions: tuple[float | None, ...] = ()

InferencePolicy dataclass

Policy controlling run-level statistical inference.

The default method bootstraps complete process-run statistics, applies a BCa interval, and uses a Bonferroni-adjusted simultaneous confidence level across the reported matrix. legacy_consistency preserves the version 1 observed-pairwise-range decision rule and is intentionally non-inferential.

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class InferencePolicy:
    """Policy controlling run-level statistical inference.

    The default method bootstraps complete process-run statistics, applies a
    BCa interval, and uses a Bonferroni-adjusted simultaneous confidence level
    across the reported matrix. ``legacy_consistency`` preserves the version 1
    observed-pairwise-range decision rule and is intentionally non-inferential.
    """

    method: InferenceMethod = "bca_bootstrap"
    confidence_level: float = 0.95
    resamples: int = 50_000
    random_seed: int = 0
    multiplicity: MultiplicityCorrection = "bonferroni"

    def __post_init__(self) -> None:
        """Validate inference controls."""
        if self.method not in {"bca_bootstrap", "legacy_consistency"}:
            raise ValueError(f"Unsupported inference method: {self.method!r}.")
        confidence_level = _validate_fraction(
            self.confidence_level,
            field_name="InferencePolicy.confidence_level",
        )
        if confidence_level in {0.0, 1.0}:
            raise ValueError("InferencePolicy.confidence_level must be between zero and one.")
        if isinstance(self.resamples, bool) or not isinstance(self.resamples, int):
            raise TypeError("InferencePolicy.resamples must be an integer.")
        if self.resamples < 1_000:
            raise ValueError("InferencePolicy.resamples must be at least 1000.")
        if isinstance(self.random_seed, bool) or not isinstance(self.random_seed, int):
            raise TypeError("InferencePolicy.random_seed must be an integer.")
        if self.random_seed < 0:
            raise ValueError("InferencePolicy.random_seed must be non-negative.")
        if self.multiplicity not in {"bonferroni", "none"}:
            raise ValueError(f"Unsupported multiplicity correction: {self.multiplicity!r}.")
        object.__setattr__(self, "confidence_level", confidence_level)

__post_init__

__post_init__() -> None

Validate inference controls.

Source code in src/benchmatrix/bench_compare.py

def __post_init__(self) -> None:
    """Validate inference controls."""
    if self.method not in {"bca_bootstrap", "legacy_consistency"}:
        raise ValueError(f"Unsupported inference method: {self.method!r}.")
    confidence_level = _validate_fraction(
        self.confidence_level,
        field_name="InferencePolicy.confidence_level",
    )
    if confidence_level in {0.0, 1.0}:
        raise ValueError("InferencePolicy.confidence_level must be between zero and one.")
    if isinstance(self.resamples, bool) or not isinstance(self.resamples, int):
        raise TypeError("InferencePolicy.resamples must be an integer.")
    if self.resamples < 1_000:
        raise ValueError("InferencePolicy.resamples must be at least 1000.")
    if isinstance(self.random_seed, bool) or not isinstance(self.random_seed, int):
        raise TypeError("InferencePolicy.random_seed must be an integer.")
    if self.random_seed < 0:
        raise ValueError("InferencePolicy.random_seed must be non-negative.")
    if self.multiplicity not in {"bonferroni", "none"}:
        raise ValueError(f"Unsupported multiplicity correction: {self.multiplicity!r}.")
    object.__setattr__(self, "confidence_level", confidence_level)

PrecisionPolicy dataclass

Optional fixed-design precision target for paired pilot comparisons.

target_half_width_percent=None disables planning. When enabled, each paired matrix cell estimates the pair count for a fresh future collection; the pilot comparison and its pass/fail decision are never changed by the plan.

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class PrecisionPolicy:
    """Optional fixed-design precision target for paired pilot comparisons.

    ``target_half_width_percent=None`` disables planning. When enabled, each
    paired matrix cell estimates the pair count for a fresh future collection;
    the pilot comparison and its pass/fail decision are never changed by the
    plan.
    """

    target_half_width_percent: float | None = None

    def __post_init__(self) -> None:
        """Validate and normalize the optional percentage target."""
        if self.target_half_width_percent is None:
            return
        target = _validate_non_negative_number(
            self.target_half_width_percent,
            field_name="PrecisionPolicy.target_half_width_percent",
        )
        if target == 0.0:
            raise ValueError("PrecisionPolicy.target_half_width_percent must be positive when enabled.")
        object.__setattr__(self, "target_half_width_percent", target)

    @property
    def enabled(self) -> bool:
        """Return whether paired precision planning is requested."""
        return self.target_half_width_percent is not None

enabled property

enabled: bool

Return whether paired precision planning is requested.

__post_init__

__post_init__() -> None

Validate and normalize the optional percentage target.

Source code in src/benchmatrix/bench_compare.py

def __post_init__(self) -> None:
    """Validate and normalize the optional percentage target."""
    if self.target_half_width_percent is None:
        return
    target = _validate_non_negative_number(
        self.target_half_width_percent,
        field_name="PrecisionPolicy.target_half_width_percent",
    )
    if target == 0.0:
        raise ValueError("PrecisionPolicy.target_half_width_percent must be positive when enabled.")
    object.__setattr__(self, "target_half_width_percent", target)

BenchmarkInference dataclass

Statistical inference for one benchmark matrix cell.

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class BenchmarkInference:
    """Statistical inference for one benchmark matrix cell."""

    method: IntervalMethod
    estimand: str
    design: ComparisonDesign
    confidence_level: float
    adjusted_confidence_level: float
    multiplicity: MultiplicityCorrection
    family_size: int
    resamples: int
    random_seed: int
    estimate_percent: float | None
    confidence_low_percent: float | None
    confidence_high_percent: float | None
    warnings: tuple[str, ...] = ()
    issues: tuple[str, ...] = ()
    pair_count: int | None = None
    strata_count: int | None = None

    def __post_init__(self) -> None:
        """Validate and normalize an inference result."""
        if self.method not in {"bca_bootstrap", "percentile_bootstrap"}:
            raise ValueError(f"Unsupported interval method: {self.method!r}.")
        if self.design not in {"independent", "paired"}:
            raise ValueError(f"Unsupported inference design: {self.design!r}.")
        if not isinstance(self.estimand, str) or not self.estimand:
            raise ValueError("BenchmarkInference.estimand must be a non-empty string.")
        confidence_level = _validate_fraction(
            self.confidence_level,
            field_name="BenchmarkInference.confidence_level",
        )
        adjusted_confidence_level = _validate_fraction(
            self.adjusted_confidence_level,
            field_name="BenchmarkInference.adjusted_confidence_level",
        )
        if confidence_level in {0.0, 1.0} or adjusted_confidence_level in {0.0, 1.0}:
            raise ValueError("BenchmarkInference confidence levels must be between zero and one.")
        if adjusted_confidence_level < confidence_level:
            raise ValueError("Adjusted confidence level must not be lower than the nominal confidence level.")
        if self.multiplicity not in {"bonferroni", "none"}:
            raise ValueError(f"Unsupported multiplicity correction: {self.multiplicity!r}.")
        if isinstance(self.family_size, bool) or not isinstance(self.family_size, int) or self.family_size <= 0:
            raise ValueError("BenchmarkInference.family_size must be a positive integer.")
        if isinstance(self.resamples, bool) or not isinstance(self.resamples, int):
            raise TypeError("BenchmarkInference.resamples must be an integer.")
        if self.resamples < 1_000:
            raise ValueError("BenchmarkInference.resamples must be at least 1000.")
        if isinstance(self.random_seed, bool) or not isinstance(self.random_seed, int):
            raise TypeError("BenchmarkInference.random_seed must be an integer.")
        if self.random_seed < 0:
            raise ValueError("BenchmarkInference.random_seed must be non-negative.")
        if self.design == "independent" and (self.pair_count is not None or self.strata_count is not None):
            raise ValueError("Independent inference must not define paired-design counts.")
        if self.design == "paired" and (
            isinstance(self.pair_count, bool) or not isinstance(self.pair_count, int) or self.pair_count <= 0
        ):
            raise ValueError("Paired inference requires a positive integer pair_count.")
        if self.design == "paired" and (
            isinstance(self.strata_count, bool)
            or not isinstance(self.strata_count, int)
            or self.strata_count <= 0
            or self.pair_count is None
            or self.strata_count > self.pair_count
        ):
            raise ValueError("Paired inference requires a valid positive strata_count no greater than pair_count.")
        for field_name, value in (
            ("estimate_percent", self.estimate_percent),
            ("confidence_low_percent", self.confidence_low_percent),
            ("confidence_high_percent", self.confidence_high_percent),
        ):
            if value is not None and not math.isfinite(value):
                raise ValueError(f"BenchmarkInference.{field_name} must be finite or None.")
        present = (
            self.estimate_percent is not None,
            self.confidence_low_percent is not None,
            self.confidence_high_percent is not None,
        )
        if any(present) and not all(present):
            raise ValueError("BenchmarkInference estimate and confidence bounds must be present together.")
        if (
            self.confidence_low_percent is not None
            and self.confidence_high_percent is not None
            and self.confidence_low_percent > self.confidence_high_percent
        ):
            raise ValueError("BenchmarkInference confidence bounds are reversed.")
        warnings = tuple(self.warnings)
        issues = tuple(self.issues)
        if any(not isinstance(item, str) or not item for item in (*warnings, *issues)):
            raise ValueError("BenchmarkInference warnings and issues must contain non-empty strings.")
        object.__setattr__(self, "warnings", warnings)
        object.__setattr__(self, "issues", issues)
        object.__setattr__(self, "confidence_level", confidence_level)
        object.__setattr__(self, "adjusted_confidence_level", adjusted_confidence_level)

    @property
    def adequate(self) -> bool:
        """Return whether a complete confidence interval is available."""
        return not self.issues and self.confidence_low_percent is not None and self.confidence_high_percent is not None

adequate property

adequate: bool

Return whether a complete confidence interval is available.

__post_init__

__post_init__() -> None

Validate and normalize an inference result.

Source code in src/benchmatrix/bench_compare.py

def __post_init__(self) -> None:
    """Validate and normalize an inference result."""
    if self.method not in {"bca_bootstrap", "percentile_bootstrap"}:
        raise ValueError(f"Unsupported interval method: {self.method!r}.")
    if self.design not in {"independent", "paired"}:
        raise ValueError(f"Unsupported inference design: {self.design!r}.")
    if not isinstance(self.estimand, str) or not self.estimand:
        raise ValueError("BenchmarkInference.estimand must be a non-empty string.")
    confidence_level = _validate_fraction(
        self.confidence_level,
        field_name="BenchmarkInference.confidence_level",
    )
    adjusted_confidence_level = _validate_fraction(
        self.adjusted_confidence_level,
        field_name="BenchmarkInference.adjusted_confidence_level",
    )
    if confidence_level in {0.0, 1.0} or adjusted_confidence_level in {0.0, 1.0}:
        raise ValueError("BenchmarkInference confidence levels must be between zero and one.")
    if adjusted_confidence_level < confidence_level:
        raise ValueError("Adjusted confidence level must not be lower than the nominal confidence level.")
    if self.multiplicity not in {"bonferroni", "none"}:
        raise ValueError(f"Unsupported multiplicity correction: {self.multiplicity!r}.")
    if isinstance(self.family_size, bool) or not isinstance(self.family_size, int) or self.family_size <= 0:
        raise ValueError("BenchmarkInference.family_size must be a positive integer.")
    if isinstance(self.resamples, bool) or not isinstance(self.resamples, int):
        raise TypeError("BenchmarkInference.resamples must be an integer.")
    if self.resamples < 1_000:
        raise ValueError("BenchmarkInference.resamples must be at least 1000.")
    if isinstance(self.random_seed, bool) or not isinstance(self.random_seed, int):
        raise TypeError("BenchmarkInference.random_seed must be an integer.")
    if self.random_seed < 0:
        raise ValueError("BenchmarkInference.random_seed must be non-negative.")
    if self.design == "independent" and (self.pair_count is not None or self.strata_count is not None):
        raise ValueError("Independent inference must not define paired-design counts.")
    if self.design == "paired" and (
        isinstance(self.pair_count, bool) or not isinstance(self.pair_count, int) or self.pair_count <= 0
    ):
        raise ValueError("Paired inference requires a positive integer pair_count.")
    if self.design == "paired" and (
        isinstance(self.strata_count, bool)
        or not isinstance(self.strata_count, int)
        or self.strata_count <= 0
        or self.pair_count is None
        or self.strata_count > self.pair_count
    ):
        raise ValueError("Paired inference requires a valid positive strata_count no greater than pair_count.")
    for field_name, value in (
        ("estimate_percent", self.estimate_percent),
        ("confidence_low_percent", self.confidence_low_percent),
        ("confidence_high_percent", self.confidence_high_percent),
    ):
        if value is not None and not math.isfinite(value):
            raise ValueError(f"BenchmarkInference.{field_name} must be finite or None.")
    present = (
        self.estimate_percent is not None,
        self.confidence_low_percent is not None,
        self.confidence_high_percent is not None,
    )
    if any(present) and not all(present):
        raise ValueError("BenchmarkInference estimate and confidence bounds must be present together.")
    if (
        self.confidence_low_percent is not None
        and self.confidence_high_percent is not None
        and self.confidence_low_percent > self.confidence_high_percent
    ):
        raise ValueError("BenchmarkInference confidence bounds are reversed.")
    warnings = tuple(self.warnings)
    issues = tuple(self.issues)
    if any(not isinstance(item, str) or not item for item in (*warnings, *issues)):
        raise ValueError("BenchmarkInference warnings and issues must contain non-empty strings.")
    object.__setattr__(self, "warnings", warnings)
    object.__setattr__(self, "issues", issues)
    object.__setattr__(self, "confidence_level", confidence_level)
    object.__setattr__(self, "adjusted_confidence_level", adjusted_confidence_level)

RegressionPolicy dataclass

Threshold policy for classifying benchmark changes.

Thresholds are percentage points and must be finite and non-negative. More specific mappings override broader ones in this order: exact matrix cell, case, implementation, metric, then the default threshold.

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class RegressionPolicy:
    """Threshold policy for classifying benchmark changes.

    Thresholds are percentage points and must be finite and non-negative. More
    specific mappings override broader ones in this order: exact matrix cell,
    case, implementation, metric, then the default threshold.
    """

    default_threshold_percent: float = 5.0
    by_metric: Mapping[MetricName, float] = field(default_factory=dict)
    by_implementation: Mapping[str, float] = field(default_factory=dict)
    by_case: Mapping[str, float] = field(default_factory=dict)
    by_cell: Mapping[tuple[str, str, MetricName], float] = field(default_factory=dict)

    def __post_init__(self) -> None:
        """Validate thresholds and freeze policy mappings."""
        default_threshold = _validate_threshold(
            self.default_threshold_percent,
            field_name="default_threshold_percent",
        )
        by_metric = {
            _validate_metric_key(metric_name): _validate_threshold(
                threshold,
                field_name=f"by_metric[{metric_name!r}]",
            )
            for metric_name, threshold in self.by_metric.items()
        }
        by_implementation = {
            _validate_selector_name(implementation_name, field_name="implementation"): _validate_threshold(
                threshold,
                field_name=f"by_implementation[{implementation_name!r}]",
            )
            for implementation_name, threshold in self.by_implementation.items()
        }
        by_case = {
            _validate_selector_name(case_name, field_name="case"): _validate_threshold(
                threshold,
                field_name=f"by_case[{case_name!r}]",
            )
            for case_name, threshold in self.by_case.items()
        }
        by_cell = {
            _validate_cell_key(cell): _validate_threshold(
                threshold,
                field_name=f"by_cell[{cell!r}]",
            )
            for cell, threshold in self.by_cell.items()
        }

        object.__setattr__(self, "default_threshold_percent", default_threshold)
        object.__setattr__(self, "by_metric", MappingProxyType(by_metric))
        object.__setattr__(self, "by_implementation", MappingProxyType(by_implementation))
        object.__setattr__(self, "by_case", MappingProxyType(by_case))
        object.__setattr__(self, "by_cell", MappingProxyType(by_cell))

    def threshold_for(
        self,
        implementation_name: str,
        case_name: str,
        metric_name: MetricName,
    ) -> float:
        """Return the effective threshold for one matrix cell."""
        scope = self.threshold_scope_for(implementation_name, case_name, metric_name)
        cell = (implementation_name, case_name, metric_name)
        if scope == "cell":
            return self.by_cell[cell]
        if scope == "case":
            return self.by_case[case_name]
        if scope == "implementation":
            return self.by_implementation[implementation_name]
        if scope == "metric":
            return self.by_metric[metric_name]
        return self.default_threshold_percent

    def threshold_scope_for(
        self,
        implementation_name: str,
        case_name: str,
        metric_name: MetricName,
    ) -> RegressionThresholdScope:
        """Return the selector scope that supplies one cell's threshold."""
        cell = (implementation_name, case_name, metric_name)
        if cell in self.by_cell:
            return "cell"
        if case_name in self.by_case:
            return "case"
        if implementation_name in self.by_implementation:
            return "implementation"
        if metric_name in self.by_metric:
            return "metric"
        return "default"

__post_init__

__post_init__() -> None

Validate thresholds and freeze policy mappings.

Source code in src/benchmatrix/bench_compare.py

def __post_init__(self) -> None:
    """Validate thresholds and freeze policy mappings."""
    default_threshold = _validate_threshold(
        self.default_threshold_percent,
        field_name="default_threshold_percent",
    )
    by_metric = {
        _validate_metric_key(metric_name): _validate_threshold(
            threshold,
            field_name=f"by_metric[{metric_name!r}]",
        )
        for metric_name, threshold in self.by_metric.items()
    }
    by_implementation = {
        _validate_selector_name(implementation_name, field_name="implementation"): _validate_threshold(
            threshold,
            field_name=f"by_implementation[{implementation_name!r}]",
        )
        for implementation_name, threshold in self.by_implementation.items()
    }
    by_case = {
        _validate_selector_name(case_name, field_name="case"): _validate_threshold(
            threshold,
            field_name=f"by_case[{case_name!r}]",
        )
        for case_name, threshold in self.by_case.items()
    }
    by_cell = {
        _validate_cell_key(cell): _validate_threshold(
            threshold,
            field_name=f"by_cell[{cell!r}]",
        )
        for cell, threshold in self.by_cell.items()
    }

    object.__setattr__(self, "default_threshold_percent", default_threshold)
    object.__setattr__(self, "by_metric", MappingProxyType(by_metric))
    object.__setattr__(self, "by_implementation", MappingProxyType(by_implementation))
    object.__setattr__(self, "by_case", MappingProxyType(by_case))
    object.__setattr__(self, "by_cell", MappingProxyType(by_cell))

threshold_for

threshold_for(
    implementation_name: str,
    case_name: str,
    metric_name: MetricName,
) -> float

Return the effective threshold for one matrix cell.

Source code in src/benchmatrix/bench_compare.py

def threshold_for(
    self,
    implementation_name: str,
    case_name: str,
    metric_name: MetricName,
) -> float:
    """Return the effective threshold for one matrix cell."""
    scope = self.threshold_scope_for(implementation_name, case_name, metric_name)
    cell = (implementation_name, case_name, metric_name)
    if scope == "cell":
        return self.by_cell[cell]
    if scope == "case":
        return self.by_case[case_name]
    if scope == "implementation":
        return self.by_implementation[implementation_name]
    if scope == "metric":
        return self.by_metric[metric_name]
    return self.default_threshold_percent

threshold_scope_for

threshold_scope_for(
    implementation_name: str,
    case_name: str,
    metric_name: MetricName,
) -> RegressionThresholdScope

Return the selector scope that supplies one cell's threshold.

Source code in src/benchmatrix/bench_compare.py

def threshold_scope_for(
    self,
    implementation_name: str,
    case_name: str,
    metric_name: MetricName,
) -> RegressionThresholdScope:
    """Return the selector scope that supplies one cell's threshold."""
    cell = (implementation_name, case_name, metric_name)
    if cell in self.by_cell:
        return "cell"
    if case_name in self.by_case:
        return "case"
    if implementation_name in self.by_implementation:
        return "implementation"
    if metric_name in self.by_metric:
        return "metric"
    return "default"

BenchmarkComparison dataclass

Comparison for one implementation, case, and metric matrix cell.

percent_change is the conventional candidate change from baseline, while improvement_percent is direction-aware and therefore positive when the candidate is better.

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class BenchmarkComparison:
    """Comparison for one implementation, case, and metric matrix cell.

    ``percent_change`` is the conventional candidate change from baseline,
    while ``improvement_percent`` is direction-aware and therefore positive
    when the candidate is better.
    """

    implementation_name: str
    case_name: str
    metric_name: MetricName
    statistic: str
    direction: ComparisonDirection
    status: ComparisonStatus
    baseline_value: float | None
    candidate_value: float | None
    ratio: float | None
    percent_change: float | None
    improvement_percent: float | None
    regression: RegressionClassification
    threshold_percent: float
    unit: str
    baseline_evidence: BenchmarkEvidence | None = None
    candidate_evidence: BenchmarkEvidence | None = None
    improvement_low_percent: float | None = None
    improvement_high_percent: float | None = None
    reason: str | None = None
    inference: BenchmarkInference | None = None
    precision: PrecisionPlan | None = None

BenchmarkRunComparison dataclass

Matrix-aware comparison between a baseline and candidate run.

Source code in src/benchmatrix/bench_compare.py

@dataclass(frozen=True, slots=True)
class BenchmarkRunComparison:
    """Matrix-aware comparison between a baseline and candidate run."""

    baseline: BenchmarkRun
    candidate: BenchmarkRun
    compatibility: RunCompatibilityReport
    regression_policy: RegressionPolicy
    comparisons: tuple[BenchmarkComparison, ...]
    baseline_runs: tuple[BenchmarkRun, ...] = ()
    candidate_runs: tuple[BenchmarkRun, ...] = ()
    evidence_policy: EvidencePolicy = field(default_factory=EvidencePolicy)
    inference_policy: InferencePolicy = field(default_factory=InferencePolicy)
    design: ComparisonDesign = "independent"
    precision_policy: PrecisionPolicy = field(default_factory=PrecisionPolicy)

    @property
    def matched(self) -> tuple[BenchmarkComparison, ...]:
        """Return cells that were compared successfully."""
        return tuple(comparison for comparison in self.comparisons if comparison.status == "matched")

    @property
    def missing(self) -> tuple[BenchmarkComparison, ...]:
        """Return cells absent from either input run."""
        return tuple(
            comparison
            for comparison in self.comparisons
            if comparison.status in {"missing_baseline", "missing_candidate"}
        )

    @property
    def incompatible(self) -> tuple[BenchmarkComparison, ...]:
        """Return cells whose measurement context cannot be compared."""
        return tuple(comparison for comparison in self.comparisons if comparison.status == "incompatible")

    @property
    def improved(self) -> tuple[BenchmarkComparison, ...]:
        """Return comparable cells that exceeded their improvement threshold."""
        return tuple(comparison for comparison in self.comparisons if comparison.regression == "improved")

    @property
    def unchanged(self) -> tuple[BenchmarkComparison, ...]:
        """Return comparable cells whose changes stayed within threshold."""
        return tuple(comparison for comparison in self.comparisons if comparison.regression == "unchanged")

    @property
    def regressed(self) -> tuple[BenchmarkComparison, ...]:
        """Return comparable cells that exceeded their regression threshold."""
        return tuple(comparison for comparison in self.comparisons if comparison.regression == "regressed")

    @property
    def inconclusive(self) -> tuple[BenchmarkComparison, ...]:
        """Return matched cells whose evidence cannot support a decision."""
        return tuple(comparison for comparison in self.comparisons if comparison.regression == "inconclusive")

    @property
    def not_comparable(self) -> tuple[BenchmarkComparison, ...]:
        """Return cells without a trustworthy regression classification."""
        return tuple(comparison for comparison in self.comparisons if comparison.regression == "not_comparable")

    @property
    def is_complete(self) -> bool:
        """Return whether every matrix cell was compared successfully."""
        return len(self.matched) == len(self.comparisons)

    @property
    def is_comparable(self) -> bool:
        """Return whether environment and every matrix cell are comparable."""
        return self.compatibility.is_compatible and self.is_complete and not self.not_comparable

    @property
    def has_regressions(self) -> bool:
        """Return whether any comparable matrix cell regressed."""
        return bool(self.regressed)

    @property
    def passed(self) -> bool:
        """Return whether the comparison is trustworthy and regression-free."""
        return self.is_comparable and not self.has_regressions and not self.inconclusive

matched property

matched: tuple[BenchmarkComparison, ...]

Return cells that were compared successfully.

missing property

missing: tuple[BenchmarkComparison, ...]

Return cells absent from either input run.

incompatible property

incompatible: tuple[BenchmarkComparison, ...]

Return cells whose measurement context cannot be compared.

improved property

improved: tuple[BenchmarkComparison, ...]

Return comparable cells that exceeded their improvement threshold.

unchanged property

unchanged: tuple[BenchmarkComparison, ...]

Return comparable cells whose changes stayed within threshold.

regressed property

regressed: tuple[BenchmarkComparison, ...]

Return comparable cells that exceeded their regression threshold.

inconclusive property

inconclusive: tuple[BenchmarkComparison, ...]

Return matched cells whose evidence cannot support a decision.

not_comparable property

not_comparable: tuple[BenchmarkComparison, ...]

Return cells without a trustworthy regression classification.

is_complete property

is_complete: bool

Return whether every matrix cell was compared successfully.

is_comparable property

is_comparable: bool

Return whether environment and every matrix cell are comparable.

has_regressions property

has_regressions: bool

Return whether any comparable matrix cell regressed.

passed property

passed: bool

Return whether the comparison is trustworthy and regression-free.

compare_benchmark_runs

compare_benchmark_runs(
    baseline: BenchmarkRun,
    candidate: BenchmarkRun,
    *,
    compatibility_policy: RunCompatibilityPolicy
    | None = None,
    regression_policy: RegressionPolicy | None = None,
    inference_policy: InferencePolicy | None = None,
    precision_policy: PrecisionPolicy | None = None,
) -> BenchmarkRunComparison

Compare two runs across the union of their benchmark matrix cells.

Comparisons use mean latency for single_call_latency, mean throughput for batch_throughput, and p95 latency for tail_latency. Missing cells and changed case or unit metadata are retained as explicit results rather than silently dropped.

Parameters:

Name	Type	Description	Default
baseline	BenchmarkRun	Reference benchmark run.	required
candidate	BenchmarkRun	Benchmark run being evaluated.	required
compatibility_policy	RunCompatibilityPolicy | None	Environment checks to apply. Defaults to permissive compatibility.	None
regression_policy	RegressionPolicy | None	Thresholds used to classify cell changes. Defaults to a five-percent threshold.	None
inference_policy	InferencePolicy | None	Run-level uncertainty analysis to apply. A single run cannot produce a default bootstrap interval and is therefore inconclusive unless the legacy method is selected explicitly.	None
precision_policy	PrecisionPolicy | None	Optional precision planning. Planning requires an explicitly paired design and is rejected for this single-run API.	None

Returns:

Type	Description
BenchmarkRunComparison	A deterministic comparison across both run matrices.

Source code in src/benchmatrix/bench_compare.py

def compare_benchmark_runs(
    baseline: BenchmarkRun,
    candidate: BenchmarkRun,
    *,
    compatibility_policy: RunCompatibilityPolicy | None = None,
    regression_policy: RegressionPolicy | None = None,
    inference_policy: InferencePolicy | None = None,
    precision_policy: PrecisionPolicy | None = None,
) -> BenchmarkRunComparison:
    """Compare two runs across the union of their benchmark matrix cells.

    Comparisons use mean latency for ``single_call_latency``, mean throughput
    for ``batch_throughput``, and p95 latency for ``tail_latency``. Missing
    cells and changed case or unit metadata are retained as explicit results
    rather than silently dropped.

    Args:
        baseline: Reference benchmark run.
        candidate: Benchmark run being evaluated.
        compatibility_policy: Environment checks to apply. Defaults to
            permissive compatibility.
        regression_policy: Thresholds used to classify cell changes. Defaults
            to a five-percent threshold.
        inference_policy: Run-level uncertainty analysis to apply. A single
            run cannot produce a default bootstrap interval and is therefore
            inconclusive unless the legacy method is selected explicitly.
        precision_policy: Optional precision planning. Planning requires an
            explicitly paired design and is rejected for this single-run API.

    Returns:
        A deterministic comparison across both run matrices.
    """
    return compare_benchmark_run_groups(
        (baseline,),
        (candidate,),
        compatibility_policy=compatibility_policy,
        regression_policy=regression_policy,
        inference_policy=inference_policy,
        precision_policy=precision_policy,
        evidence_policy=EvidencePolicy(
            minimum_runs=1,
            minimum_samples_per_run=0,
            minimum_rounds_per_run=0,
            require_rounds=False,
            require_iterations=False,
            require_raw_samples_for_inference=False,
            minimum_tail_samples_per_run=0,
            require_tail_iterations_one=False,
        ),
    )

compare_benchmark_run_groups

compare_benchmark_run_groups(
    baselines: Sequence[BenchmarkRun],
    candidates: Sequence[BenchmarkRun],
    *,
    compatibility_policy: RunCompatibilityPolicy
    | None = None,
    regression_policy: RegressionPolicy | None = None,
    evidence_policy: EvidencePolicy | None = None,
    inference_policy: InferencePolicy | None = None,
    precision_policy: PrecisionPolicy | None = None,
) -> BenchmarkRunComparison

Compare repeated baseline and candidate runs as two evidence groups.

Each cell uses the median of its per-run metric values. By default, run-level BCa bootstrap intervals quantify uncertainty and a Bonferroni adjustment controls the matrix-wide family-wise error rate. Practical thresholds then distinguish improvements, regressions, equivalence, and inconclusive intervals.

Parameters:

Name	Type	Description	Default
baselines	Sequence[BenchmarkRun]	Repeated reference benchmark runs.	required
candidates	Sequence[BenchmarkRun]	Repeated candidate benchmark runs.	required
compatibility_policy	RunCompatibilityPolicy | None	Environment checks applied across every run.	None
regression_policy	RegressionPolicy | None	Percentage thresholds for classifying changes.	None
evidence_policy	EvidencePolicy | None	Minimum repeated-run and sample evidence.	None
inference_policy	InferencePolicy | None	Statistical inference and multiplicity controls.	None
precision_policy	PrecisionPolicy | None	Optional paired fixed-design planning target. It must remain disabled for independent groups.	None

Returns:

Type	Description
BenchmarkRunComparison	A matrix comparison with per-side trust diagnostics.

Raises:

Type	Description
ValueError	If either run group is empty.
TypeError	If a group contains a value other than BenchmarkRun.

Source code in src/benchmatrix/bench_compare.py

def compare_benchmark_run_groups(
    baselines: Sequence[BenchmarkRun],
    candidates: Sequence[BenchmarkRun],
    *,
    compatibility_policy: RunCompatibilityPolicy | None = None,
    regression_policy: RegressionPolicy | None = None,
    evidence_policy: EvidencePolicy | None = None,
    inference_policy: InferencePolicy | None = None,
    precision_policy: PrecisionPolicy | None = None,
) -> BenchmarkRunComparison:
    """Compare repeated baseline and candidate runs as two evidence groups.

    Each cell uses the median of its per-run metric values. By default,
    run-level BCa bootstrap intervals quantify uncertainty and a Bonferroni
    adjustment controls the matrix-wide family-wise error rate. Practical
    thresholds then distinguish improvements, regressions, equivalence, and
    inconclusive intervals.

    Args:
        baselines: Repeated reference benchmark runs.
        candidates: Repeated candidate benchmark runs.
        compatibility_policy: Environment checks applied across every run.
        regression_policy: Percentage thresholds for classifying changes.
        evidence_policy: Minimum repeated-run and sample evidence.
        inference_policy: Statistical inference and multiplicity controls.
        precision_policy: Optional paired fixed-design planning target. It
            must remain disabled for independent groups.

    Returns:
        A matrix comparison with per-side trust diagnostics.

    Raises:
        ValueError: If either run group is empty.
        TypeError: If a group contains a value other than ``BenchmarkRun``.
    """
    return _compare_benchmark_run_groups(
        baselines,
        candidates,
        compatibility_policy=compatibility_policy,
        regression_policy=regression_policy,
        evidence_policy=evidence_policy,
        inference_policy=inference_policy,
        precision_policy=precision_policy,
        design="independent",
    )

compare_paired_benchmark_run_groups

compare_paired_benchmark_run_groups(
    baselines: Sequence[BenchmarkRun],
    candidates: Sequence[BenchmarkRun],
    *,
    pair_strata: Sequence[str] | None = None,
    precision_pair_count_multiple: int = 2,
    compatibility_policy: RunCompatibilityPolicy
    | None = None,
    regression_policy: RegressionPolicy | None = None,
    evidence_policy: EvidencePolicy | None = None,
    inference_policy: InferencePolicy | None = None,
    precision_policy: PrecisionPolicy | None = None,
) -> BenchmarkRunComparison

Compare explicitly matched baseline/candidate process-run pairs.

The values at each position must come from one adjacent collection block. Complete pairs, rather than individual run files, are the independent experimental units. Pairing is explicit in this API and is never inferred from filenames or timestamps.

Parameters:

Name	Type	Description	Default
baselines	Sequence[BenchmarkRun]	Baseline members in pair order.	required
candidates	Sequence[BenchmarkRun]	Candidate members in the same pair order.	required
pair_strata	Sequence[str] | None	Optional fixed-design stratum label for each pair, such as its recorded AB or BA command orientation. When given, paired resampling preserves the observed count in every stratum.	None
precision_pair_count_multiple	int	Divisibility constraint for a future confirmatory collection. Paired designs default to an even count; manifest-backed collections pass their complete joint design supercycle.	2
compatibility_policy	RunCompatibilityPolicy | None	Environment checks applied across every run.	None
regression_policy	RegressionPolicy | None	Percentage thresholds for classifying changes.	None
evidence_policy	EvidencePolicy | None	Minimum complete-pair and sample evidence.	None
inference_policy	InferencePolicy | None	Statistical inference and multiplicity controls.	None
precision_policy	PrecisionPolicy | None	Optional fixed-design precision target for a fresh future paired collection.	None

Returns:

Type	Description
BenchmarkRunComparison	A paired matrix comparison with per-side trust diagnostics.

Raises:

Type	Description
ValueError	If the sequences are empty or have different lengths.
TypeError	If either sequence contains a non-BenchmarkRun value.

Source code in src/benchmatrix/bench_compare.py

def compare_paired_benchmark_run_groups(
    baselines: Sequence[BenchmarkRun],
    candidates: Sequence[BenchmarkRun],
    *,
    pair_strata: Sequence[str] | None = None,
    precision_pair_count_multiple: int = 2,
    compatibility_policy: RunCompatibilityPolicy | None = None,
    regression_policy: RegressionPolicy | None = None,
    evidence_policy: EvidencePolicy | None = None,
    inference_policy: InferencePolicy | None = None,
    precision_policy: PrecisionPolicy | None = None,
) -> BenchmarkRunComparison:
    """Compare explicitly matched baseline/candidate process-run pairs.

    The values at each position must come from one adjacent collection block.
    Complete pairs, rather than individual run files, are the independent
    experimental units. Pairing is explicit in this API and is never inferred
    from filenames or timestamps.

    Args:
        baselines: Baseline members in pair order.
        candidates: Candidate members in the same pair order.
        pair_strata: Optional fixed-design stratum label for each pair, such
            as its recorded ``AB`` or ``BA`` command orientation. When given,
            paired resampling preserves the observed count in every stratum.
        precision_pair_count_multiple: Divisibility constraint for a future
            confirmatory collection. Paired designs default to an even count;
            manifest-backed collections pass their complete joint design
            supercycle.
        compatibility_policy: Environment checks applied across every run.
        regression_policy: Percentage thresholds for classifying changes.
        evidence_policy: Minimum complete-pair and sample evidence.
        inference_policy: Statistical inference and multiplicity controls.
        precision_policy: Optional fixed-design precision target for a fresh
            future paired collection.

    Returns:
        A paired matrix comparison with per-side trust diagnostics.

    Raises:
        ValueError: If the sequences are empty or have different lengths.
        TypeError: If either sequence contains a non-``BenchmarkRun`` value.
    """
    if len(baselines) != len(candidates):
        raise ValueError("Paired baseline and candidate groups must contain the same number of runs.")
    return _compare_benchmark_run_groups(
        baselines,
        candidates,
        compatibility_policy=compatibility_policy,
        regression_policy=regression_policy,
        evidence_policy=evidence_policy,
        inference_policy=inference_policy,
        precision_policy=precision_policy,
        design="paired",
        pair_strata=pair_strata,
        precision_pair_count_multiple=precision_pair_count_multiple,
    )