Bench Policy

benchmatrix.bench_policy

Load version-controlled benchmark policies from TOML configuration.

BenchmarkPolicyConfig dataclass

Resolved benchmatrix policy configuration.

Attributes:

Name	Type	Description
compatibility	RunCompatibilityPolicy	Run-environment compatibility policy.
evidence	EvidencePolicy	Repeated-run evidence policy.
inference	InferencePolicy	Run-level inference and multiplicity policy.
precision	PrecisionPolicy	Optional fixed-design precision-planning policy.
regression	RegressionPolicy	Regression threshold policy.
source	Path | None	Selected TOML file, or None when using built-in defaults.
configured_fields	frozenset[str]	Explicit tool.benchmatrix field paths.

Source code in src/benchmatrix/bench_policy.py

@dataclass(frozen=True, slots=True)
class BenchmarkPolicyConfig:
    """Resolved benchmatrix policy configuration.

    Attributes:
        compatibility: Run-environment compatibility policy.
        evidence: Repeated-run evidence policy.
        inference: Run-level inference and multiplicity policy.
        precision: Optional fixed-design precision-planning policy.
        regression: Regression threshold policy.
        source: Selected TOML file, or ``None`` when using built-in defaults.
        configured_fields: Explicit ``tool.benchmatrix`` field paths.
    """

    compatibility: RunCompatibilityPolicy
    evidence: EvidencePolicy
    regression: RegressionPolicy
    source: Path | None = None
    configured_fields: frozenset[str] = frozenset()
    inference: InferencePolicy = dataclass_field(default_factory=InferencePolicy)
    precision: PrecisionPolicy = dataclass_field(default_factory=PrecisionPolicy)

    def __post_init__(self) -> None:
        """Normalize the optional source path and configured field set."""
        if not isinstance(self.compatibility, RunCompatibilityPolicy):
            raise TypeError("BenchmarkPolicyConfig.compatibility must be a RunCompatibilityPolicy.")
        if not isinstance(self.evidence, EvidencePolicy):
            raise TypeError("BenchmarkPolicyConfig.evidence must be an EvidencePolicy.")
        if not isinstance(self.inference, InferencePolicy):
            raise TypeError("BenchmarkPolicyConfig.inference must be an InferencePolicy.")
        if not isinstance(self.precision, PrecisionPolicy):
            raise TypeError("BenchmarkPolicyConfig.precision must be a PrecisionPolicy.")
        if not isinstance(self.regression, RegressionPolicy):
            raise TypeError("BenchmarkPolicyConfig.regression must be a RegressionPolicy.")
        fields = frozenset(self.configured_fields)
        if any(not isinstance(field, str) or not field for field in fields):
            raise ValueError("BenchmarkPolicyConfig.configured_fields must contain non-empty strings.")
        if self.source is not None:
            object.__setattr__(self, "source", Path(self.source))
        object.__setattr__(self, "configured_fields", fields)

    @property
    def is_configured(self) -> bool:
        """Return whether a ``tool.benchmatrix`` table was loaded."""
        return self.source is not None

is_configured property

is_configured: bool

Return whether a tool.benchmatrix table was loaded.

__post_init__

__post_init__() -> None

Normalize the optional source path and configured field set.

Source code in src/benchmatrix/bench_policy.py

def __post_init__(self) -> None:
    """Normalize the optional source path and configured field set."""
    if not isinstance(self.compatibility, RunCompatibilityPolicy):
        raise TypeError("BenchmarkPolicyConfig.compatibility must be a RunCompatibilityPolicy.")
    if not isinstance(self.evidence, EvidencePolicy):
        raise TypeError("BenchmarkPolicyConfig.evidence must be an EvidencePolicy.")
    if not isinstance(self.inference, InferencePolicy):
        raise TypeError("BenchmarkPolicyConfig.inference must be an InferencePolicy.")
    if not isinstance(self.precision, PrecisionPolicy):
        raise TypeError("BenchmarkPolicyConfig.precision must be a PrecisionPolicy.")
    if not isinstance(self.regression, RegressionPolicy):
        raise TypeError("BenchmarkPolicyConfig.regression must be a RegressionPolicy.")
    fields = frozenset(self.configured_fields)
    if any(not isinstance(field, str) or not field for field in fields):
        raise ValueError("BenchmarkPolicyConfig.configured_fields must contain non-empty strings.")
    if self.source is not None:
        object.__setattr__(self, "source", Path(self.source))
    object.__setattr__(self, "configured_fields", fields)

default_benchmark_policy

default_benchmark_policy() -> BenchmarkPolicyConfig

Return benchmatrix's built-in comparison policies.

Source code in src/benchmatrix/bench_policy.py

def default_benchmark_policy() -> BenchmarkPolicyConfig:
    """Return benchmatrix's built-in comparison policies."""
    return BenchmarkPolicyConfig(
        compatibility=RunCompatibilityPolicy(),
        evidence=EvidencePolicy(),
        inference=InferencePolicy(),
        precision=PrecisionPolicy(),
        regression=RegressionPolicy(),
    )

load_benchmark_policy

load_benchmark_policy(
    path: str | Path | None = None,
    *,
    search_from: str | Path | None = None,
) -> BenchmarkPolicyConfig

Load tool.benchmatrix policy from TOML.

With an explicit path, the file must contain [tool.benchmatrix]. Otherwise the nearest pyproject.toml at or above search_from is inspected. Discovery stops at the first pyproject; a project without a benchmatrix table uses built-in defaults.

Parameters:

Name	Type	Description	Default
path	str | Path | None	Explicit TOML or pyproject path.	None
search_from	str | Path | None	File or directory from which to discover pyproject.toml. Defaults to the current working directory.	None

Returns:

Type	Description
BenchmarkPolicyConfig	Validated compatibility, evidence, inference, precision, and regression policies.

Raises:

Type	Description
BenchmarkPolicyError	If an explicit file is missing, TOML is invalid, or the benchmatrix configuration does not satisfy its schema.

Source code in src/benchmatrix/bench_policy.py

def load_benchmark_policy(
    path: str | Path | None = None,
    *,
    search_from: str | Path | None = None,
) -> BenchmarkPolicyConfig:
    """Load ``tool.benchmatrix`` policy from TOML.

    With an explicit ``path``, the file must contain ``[tool.benchmatrix]``.
    Otherwise the nearest ``pyproject.toml`` at or above ``search_from`` is
    inspected. Discovery stops at the first pyproject; a project without a
    benchmatrix table uses built-in defaults.

    Args:
        path: Explicit TOML or pyproject path.
        search_from: File or directory from which to discover pyproject.toml.
            Defaults to the current working directory.

    Returns:
        Validated compatibility, evidence, inference, precision, and regression policies.

    Raises:
        BenchmarkPolicyError: If an explicit file is missing, TOML is invalid,
            or the benchmatrix configuration does not satisfy its schema.
    """
    explicit = path is not None
    source = Path(path) if explicit else _discover_pyproject(search_from)
    if source is None:
        return default_benchmark_policy()
    source = source.resolve()

    try:
        with source.open("rb") as stream:
            payload = cast(object, tomllib.load(stream))
    except OSError as exc:
        raise BenchmarkPolicyError(f"Could not read benchmark policy configuration: {source}") from exc
    except tomllib.TOMLDecodeError as exc:
        raise BenchmarkPolicyError(f"Invalid TOML in benchmark policy configuration: {source}") from exc

    root = _mapping(payload, path="root")
    tool = root.get("tool")
    if tool is None:
        if explicit:
            raise BenchmarkPolicyError(f"Configuration does not contain [tool.benchmatrix]: {source}")
        return default_benchmark_policy()
    tool_mapping = _mapping(tool, path="tool")
    raw_config = tool_mapping.get("benchmatrix")
    if raw_config is None:
        if explicit:
            raise BenchmarkPolicyError(f"Configuration does not contain [tool.benchmatrix]: {source}")
        return default_benchmark_policy()

    config = _mapping(raw_config, path="tool.benchmatrix")
    _exact_keys(config, _TOOL_KEYS, path="tool.benchmatrix")
    try:
        compatibility, compatibility_fields = _parse_compatibility(config.get("compatibility"))
        evidence, evidence_fields = _parse_evidence(config.get("evidence"))
        inference, inference_fields = _parse_inference(config.get("inference"))
        precision, precision_fields = _parse_precision(config.get("precision"))
        regression, regression_fields = _parse_regression(config.get("regression"))
        return BenchmarkPolicyConfig(
            compatibility=compatibility,
            evidence=evidence,
            inference=inference,
            precision=precision,
            regression=regression,
            source=source,
            configured_fields=frozenset(
                (
                    *compatibility_fields,
                    *evidence_fields,
                    *inference_fields,
                    *precision_fields,
                    *regression_fields,
                )
            ),
        )
    except BenchmarkPolicyError:
        raise
    except (TypeError, ValueError) as exc:
        raise BenchmarkPolicyError(f"Invalid benchmark policy in {source}: {exc}") from exc