feat: add compose deliverable, scenario definitions, eval stub, and CI
This commit is contained in:
parent
ef9e102b05
commit
7c3934daee
8 changed files with 148 additions and 0 deletions
23
eval/run_eval.py
Normal file
23
eval/run_eval.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
"""Eval harness entry point.
|
||||
|
||||
Fixture mode (default, key-free): byte-snapshots of deterministic pipeline
|
||||
stages against recorded cassettes. Live mode: property assertions only —
|
||||
LLM output is constrained, not pinned.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
SCENARIOS_FILE = Path(__file__).parent / "scenarios.yaml"
|
||||
|
||||
|
||||
def load_scenarios() -> dict:
|
||||
"""Load the shared scenario definitions."""
|
||||
with SCENARIOS_FILE.open() as handle:
|
||||
return yaml.safe_load(handle)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
scenarios = load_scenarios()
|
||||
print(f"{len(scenarios['scenarios'])} scenarios, {len(scenarios['refinements'])} refinements")
|
||||
Loading…
Add table
Add a link
Reference in a new issue