Cli

benchmatrix.cli

Command-line interface for measuring, collecting, and comparing benchmark runs.

build_parser

build_parser() -> argparse.ArgumentParser

Build the benchmatrix command-line parser.

Source code in src/benchmatrix/cli.py

def build_parser() -> argparse.ArgumentParser:
    """Build the benchmatrix command-line parser."""
    parser = argparse.ArgumentParser(
        prog="benchmatrix",
        description="Measure repeatable Python benchmark matrices and detect regressions.",
    )
    parser.add_argument(
        "--version",
        action="version",
        version=f"%(prog)s {__version__}",
    )
    subparsers = parser.add_subparsers(dest="command", required=True)

    collect_parser = subparsers.add_parser(
        "collect",
        help="Collect repeated pytest-benchmark runs.",
        description=(
            "Execute one pytest command sequentially and save validated "
            + "benchmark JSON files in a manifest-backed group."
        ),
    )
    _add_collection_options(collect_parser)
    collect_parser.add_argument(
        "pytest_command",
        nargs=argparse.REMAINDER,
        metavar="...",
        help="Pytest command after '--'; optional when resuming because the manifest command is reused.",
    )

    paired_collect_parser = subparsers.add_parser(
        "collect-paired",
        help="Collect matched baseline/candidate benchmark pairs.",
        description=(
            "Execute adjacent baseline/candidate blocks with balanced matrix ordering. "
            + "Separate the two commands with ':::'."
        ),
    )
    paired_collect_parser.add_argument(
        "--pairs",
        type=_positive_integer,
        default=None,
        metavar="COUNT",
        help=(
            "Complete-pair target; omit to select the smallest complete AB/BA-by-row supercycle with at least 5 pairs."
        ),
    )
    paired_collect_parser.add_argument(
        "--output",
        type=Path,
        required=True,
        metavar="DIR",
        help="New or empty directory, or an existing paired collection directory when resuming.",
    )
    paired_collect_parser.add_argument(
        "--random-seed",
        type=_non_negative_integer,
        default=None,
        metavar="INTEGER",
        help="Deterministic AB/BA orientation and balanced cell-order seed (default: 0).",
    )
    paired_collect_parser.add_argument(
        "--baseline-cwd",
        type=Path,
        default=None,
        metavar="DIR",
        help="Working directory for the baseline command.",
    )
    paired_collect_parser.add_argument(
        "--candidate-cwd",
        type=Path,
        default=None,
        metavar="DIR",
        help="Working directory for the candidate command.",
    )
    paired_collect_parser.add_argument(
        "--resume",
        action="store_true",
        help="Continue missing or interrupted blocks in an existing paired collection.",
    )
    paired_collect_parser.add_argument(
        "--retry-failed",
        action="store_true",
        help="Resume and rerun both members of each still-incomplete block.",
    )
    paired_collect_parser.add_argument(
        "--format",
        choices=("text", "json"),
        default="text",
        help="Collection summary format (default: text).",
    )
    paired_collect_parser.add_argument(
        "paired_commands",
        nargs=argparse.REMAINDER,
        metavar="BASELINE ... ::: CANDIDATE ...",
        help=("Commands after '--', separated by ':::'; optional on resume because manifest commands are reused."),
    )

    measure_parser = subparsers.add_parser(
        "measure",
        help="Measure pytest benchmarks with safe defaults.",
        description=(
            "Run pytest benchmarks sequentially with quiet reporting and isolated "
            + "pytest addopts, then save validated JSON files in a manifest-backed group."
        ),
    )
    _add_collection_options(measure_parser)
    measure_parser.add_argument(
        "--inherit-pytest-addopts",
        action="store_true",
        help="Use configured and PYTEST_ADDOPTS options instead of isolating the benchmark run.",
    )
    measure_parser.add_argument(
        "pytest_arguments",
        nargs=argparse.REMAINDER,
        metavar="TARGET ...",
        help=(
            "One or more pytest targets, followed by optional pytest arguments after '--'; "
            + "targets are optional when resuming because the manifest command is reused."
        ),
    )

    compare_parser = subparsers.add_parser(
        "compare",
        help="Compare a baseline run with a candidate run.",
        description=(
            "Compare matching implementation, case, and metric cells while checking run-environment compatibility."
        ),
    )
    compare_parser.add_argument(
        "baseline",
        type=Path,
        help="Baseline JSON file, run-group manifest, or collection directory.",
    )
    compare_parser.add_argument(
        "candidate",
        nargs="?",
        type=Path,
        help="Candidate JSON file, run-group manifest, or directory; omit with --paired.",
    )
    compare_parser.add_argument(
        "--paired",
        action="store_true",
        help="Treat BASELINE as one paired collection and preserve its matched-pair design.",
    )
    compare_parser.add_argument(
        "--baseline-run",
        action="append",
        default=[],
        type=Path,
        metavar="FILE",
        help="Additional baseline file, manifest, or directory; repeat as needed.",
    )
    compare_parser.add_argument(
        "--candidate-run",
        action="append",
        default=[],
        type=Path,
        metavar="FILE",
        help="Additional candidate file, manifest, or directory; repeat as needed.",
    )
    compare_parser.add_argument(
        "--threshold",
        type=_percentage,
        default=None,
        metavar="PERCENT",
        help="Override the default regression threshold; selector-specific configured thresholds remain active.",
    )
    compare_parser.add_argument(
        "--compatibility",
        choices=("strict", "permissive", "off"),
        default=None,
        help="Override the configured run-environment compatibility mode.",
    )
    compare_parser.add_argument(
        "--format",
        choices=("text", "json", "markdown"),
        default="text",
        help="Output format (default: text).",
    )
    compare_parser.add_argument(
        "--summary",
        action="store_true",
        help="Omit per-cell evidence diagnostics from text output.",
    )
    compare_parser.add_argument(
        "--github-summary",
        action="store_true",
        help="Append the Markdown report to the path in GITHUB_STEP_SUMMARY.",
    )
    compare_parser.add_argument(
        "--fail-on-regression",
        action="store_true",
        help="Exit 1 for regressions, inconclusive evidence, incomplete matrices, or blocked compatibility.",
    )
    compare_parser.add_argument(
        "--minimum-runs",
        type=_positive_integer,
        default=None,
        metavar="COUNT",
        help="Override the configured minimum successful runs per side.",
    )
    compare_parser.add_argument(
        "--minimum-samples",
        type=_non_negative_integer,
        default=None,
        metavar="COUNT",
        help="Override the configured minimum round-duration observations per file and cell.",
    )
    compare_parser.add_argument(
        "--inference-method",
        choices=("bca_bootstrap", "legacy_consistency"),
        default=None,
        help="Override the configured inference method.",
    )
    compare_parser.add_argument(
        "--confidence-level",
        type=_confidence_level,
        default=None,
        metavar="FRACTION",
        help="Override the configured family confidence level, for example 0.95.",
    )
    compare_parser.add_argument(
        "--bootstrap-resamples",
        type=_positive_integer,
        default=None,
        metavar="COUNT",
        help="Override the configured number of deterministic bootstrap resamples.",
    )
    compare_parser.add_argument(
        "--random-seed",
        type=_non_negative_integer,
        default=None,
        metavar="INTEGER",
        help="Override the configured bootstrap random seed.",
    )
    compare_parser.add_argument(
        "--multiplicity",
        choices=("bonferroni", "none"),
        default=None,
        help="Override matrix-wide multiplicity correction; 'none' is exploratory.",
    )
    compare_parser.add_argument(
        "--precision-target",
        type=_positive_percentage,
        default=None,
        metavar="PERCENT",
        help="Override the paired fixed-design multiplicative log-ratio half-width proxy.",
    )
    config_group = compare_parser.add_mutually_exclusive_group()
    config_group.add_argument(
        "--config",
        type=Path,
        metavar="FILE",
        help="Load [tool.benchmatrix] from this TOML file instead of discovering pyproject.toml.",
    )
    config_group.add_argument(
        "--no-config",
        action="store_true",
        help="Disable pyproject.toml discovery and use built-in policies plus CLI overrides.",
    )

    policy_parser = subparsers.add_parser(
        "policy",
        help="Inspect or validate benchmark policy configuration.",
        description="Resolve the same effective benchmark policy used by the compare command.",
    )
    policy_actions = policy_parser.add_subparsers(dest="policy_command", required=True)
    policy_show = policy_actions.add_parser(
        "show",
        help="Display the resolved effective policy.",
    )
    policy_validate = policy_actions.add_parser(
        "validate",
        help="Validate policy configuration without running benchmarks.",
    )
    for action_parser in (policy_show, policy_validate):
        policy_config = action_parser.add_mutually_exclusive_group()
        policy_config.add_argument(
            "--config",
            type=Path,
            metavar="FILE",
            help="Validate and use [tool.benchmatrix] from this explicit TOML file.",
        )
        policy_config.add_argument(
            "--no-config",
            action="store_true",
            help="Inspect built-in defaults without discovering pyproject.toml.",
        )
        action_parser.add_argument(
            "--search-from",
            type=Path,
            metavar="PATH",
            help="Discover the nearest pyproject.toml from this file or directory.",
        )
        action_parser.add_argument(
            "--format",
            choices=("text", "json"),
            default="text",
            help="Output format (default: text).",
        )
    policy_validate.add_argument(
        "--quiet",
        action="store_true",
        help="Emit no output when validation succeeds.",
    )
    return parser

main

main(argv: Sequence[str] | None = None) -> int

Run the benchmatrix command-line interface.

Source code in src/benchmatrix/cli.py

def main(argv: Sequence[str] | None = None) -> int:
    """Run the benchmatrix command-line interface."""
    parser = build_parser()
    args = parser.parse_args(argv)

    if args.command == "collect":
        return _run_collect(args)
    if args.command == "collect-paired":
        return _run_collect_paired(args)
    if args.command == "measure":
        return _run_measure(args)
    if args.command == "compare":
        return _run_compare(args)
    if args.command == "policy":
        return _run_policy(args)

    parser.error(f"Unsupported command: {args.command!r}")
    return _EXIT_USAGE_ERROR