from __future__ import annotations import argparse import json import subprocess import sys from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] RUNS_DIR = ROOT / "runs" @dataclass(frozen=True) class Case: name: str group: str script: str docker: bool = True note: str = "" network: bool = False CASES = [ Case( name="mcp-security-repro-pack", group="core", script="scripts/run_mcp_security_repro_pack.py", note="Minimal MCP validation, storage path, and backend error repros.", ), Case( name="migration-symlink-secret-chain", group="core", script="scripts/run_sibyl_security_round11_migration_symlink_suite.py", note="Migration backup and extraction symlink secret chain.", ), Case( name="hermes-prefetch-boundary", group="core", script="scripts/run_hermes_security_round13_prefetch_boundary_suite.py", note="Prefetch delimiter escape across memory tiers.", ), Case( name="hermes-learning-authority", group="core", script="scripts/run_hermes_security_round14_learning_authority_suite.py", note="Self-learning trust boundary and accepted references.", ), Case( name="hermes-profile-diagnostics", group="core", script="scripts/run_hermes_security_round15_profile_diagnostics_suite.py", note="Status surface controls and active_profile symlink boundary.", ), Case( name="product-memory-update", group="core", script="scripts/run_mcp_memory_product_update_suite.py", note="Corrections, old-new conflict, forget, ownership, priority drift.", ), Case( name="ux-qol-round2", group="core", script="scripts/run_sibyl_ux_qol_round2.py", note="First-run CLI, migration UX, ambiguous memory questions.", ), Case( name="next6-dependent", group="dependent", script="scripts/run_sibyl_next6_tracks.py", docker=False, note="Requires prior Sonnet raw result and its source DB path to exist locally.", ), Case( name="token-economy-dependent", group="dependent", script="scripts/measure_sibyl_token_economy.py", docker=False, network=True, note="Requires DeepSeek credentials and the 42k-write DB source path.", ), Case( name="paid-api-next6", group="paid-api", script="scripts/run_sibyl_paid_api_next6_validation.py", docker=False, network=True, note="Requires Anthropic or DeepSeek credentials and network access.", ), Case( name="paid-api-tracks-456", group="paid-api", script="scripts/run_sibyl_paid_api_tracks_456.py", docker=False, network=True, note="Requires Anthropic or DeepSeek credentials and network access.", ), ] def iso_now() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") def docker_command(case: Case, image: str, tmp_size: str, data_size: str) -> list[str]: network = "bridge" if case.network else "none" return [ "docker", "run", "--rm", "--network", network, "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--pids-limit", "192", "--memory", "768m", "--cpus", "1", "--tmpfs", f"/tmp:rw,nosuid,nodev,size={tmp_size}", "--tmpfs", f"/data:rw,nosuid,nodev,size={data_size},mode=1777", "-v", f"{ROOT}:/work", "-w", "/work", "-e", "SIBYL_SECURITY_DATA_ROOT=/data", image, "python", case.script, ] def local_command(case: Case) -> list[str]: return [sys.executable, case.script] def command_for(case: Case, image: str, tmp_size: str, data_size: str) -> list[str]: if case.docker: return docker_command(case, image=image, tmp_size=tmp_size, data_size=data_size) return local_command(case) def shell_join(parts: list[str]) -> str: return " ".join(json.dumps(part) if any(char.isspace() for char in part) else part for part in parts) def selected_cases(groups: set[str]) -> list[Case]: return [case for case in CASES if case.group in groups] def write_summary(results: list[dict[str, object]]) -> Path: RUNS_DIR.mkdir(parents=True, exist_ok=True) stamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S") path = RUNS_DIR / f"golden-suite-{stamp}.json" path.write_text(json.dumps({"ts": iso_now(), "results": results}, indent=2) + "\n", encoding="utf8") return path def main() -> int: parser = argparse.ArgumentParser(description="Run the Sibyl golden regression suite.") parser.add_argument("--execute", action="store_true", help="Run commands. Without this flag, only print the plan.") parser.add_argument( "--group", action="append", choices=["core", "dependent", "paid-api", "all"], help="Suite group to run. Defaults to core.", ) parser.add_argument("--docker-image", default="sibyl-memory:local", help="Docker image used for Docker-backed cases.") parser.add_argument("--tmp-size", default="256m", help="Docker /tmp tmpfs size.") parser.add_argument("--data-size", default="512m", help="Docker /data tmpfs size.") parser.add_argument("--continue-on-fail", action="store_true", help="Continue after a command exits non-zero.") parser.add_argument("--write-plan", action="store_true", help="Write a JSON plan file even when not executing.") args = parser.parse_args() groups = set(args.group or ["core"]) if "all" in groups: groups = {"core", "dependent", "paid-api"} cases = selected_cases(groups) if not cases: raise SystemExit("No cases selected.") print(f"golden_suite_started_at={iso_now()}") print(f"groups={','.join(sorted(groups))}") print(f"execute={args.execute}") print() results: list[dict[str, object]] = [] for index, case in enumerate(cases, start=1): cmd = command_for(case, image=args.docker_image, tmp_size=args.tmp_size, data_size=args.data_size) print(f"[{index}/{len(cases)}] {case.name}") print(f"group={case.group}") if case.note: print(f"note={case.note}") print(shell_join(cmd)) if not args.execute: print() results.append({"name": case.name, "group": case.group, "executed": False, "command": cmd}) continue completed = subprocess.run(cmd, cwd=ROOT, check=False) result = { "name": case.name, "group": case.group, "executed": True, "returncode": completed.returncode, "command": cmd, } results.append(result) print(f"returncode={completed.returncode}") print() if completed.returncode != 0 and not args.continue_on_fail: summary = write_summary(results) print(f"summary={summary}") return completed.returncode if args.execute or args.write_plan: summary = write_summary(results) print(f"summary={summary}") else: print("summary=not_written_dry_run") return 0 if __name__ == "__main__": raise SystemExit(main())