#!/usr/bin/env python3
"""Original LockedIn Labs synthetic regression exercise, Python 3.11+.
Keep fde-agent-engineering-lab.py beside this file. No network, keys or inference.
Run: python3 fde-agent-evaluation-lab.py test
     python3 fde-agent-evaluation-lab.py run --candidate reference
Teaching mutants operate only on temporary synthetic SQLite databases.
"""
import argparse
import importlib.util
import json
import socket
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

_spec = importlib.util.spec_from_file_location(
    "field_desk_reference", Path(__file__).with_name("fde-agent-engineering-lab.py"))
_reference = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(_reference)
Workflow = _reference.Workflow


class RefuseAll(Workflow):
    """Deliberately broken utility; never a proposed production repair."""
    def start(self, tenant, request, source="P1"):
        raise PermissionError("source_unavailable")


class StaleReview(Workflow):
    """Deliberately omits current-source validation on first acceptance."""
    def review(self, tenant, request, expected_digest, decision):
        run = self.inspect(tenant, request)
        if run["status"] != "awaiting_review":
            return super().review(tenant, request, expected_digest, decision)
        if expected_digest != run["digest"]:
            raise ValueError("proposal_changed")
        if decision not in ("accepted", "rejected"):
            raise ValueError("invalid_decision")
        with self.db:
            self.db.execute("UPDATE runs SET status=? WHERE tenant=? AND request=?",
                            (decision, tenant, request))
        return self.inspect(tenant, request)


CANDIDATES = {"reference": Workflow, "refuse-all": RefuseAll, "stale-review": StaleReview}
# Expectations are specified before execution, not inferred from a candidate's output.
CONTRACT = {
    "authorized-draft": ("utility", {"status": "awaiting_review", "source": "P1",
                                     "version": 1, "draft_matches_current": True}),
    "foreign-source": ("boundary", {"error": "source_unavailable", "runs": 0}),
    "foreign-run": ("boundary", {"error": "run_unavailable"}),
    "changed-source": ("boundary", {"error": "source_changed", "status": "stale"}),
    "withdrawn-source": ("boundary", {"error": "source_changed", "status": "stale"}),
    "restart-replay": ("recovery", {"same_run": True, "runs": 1, "same_decision": True,
                                     "conflict": "decision_conflict"}),
}


def failure_code(call):
    try:
        call()
    except (ValueError, PermissionError) as error:
        return str(error)
    return "none"


def observe(candidate, case):
    """Exercise real domain methods in a fresh temporary database for each case."""
    with tempfile.TemporaryDirectory(prefix="field-desk-eval-") as folder:
        path = str(Path(folder) / "synthetic.sqlite")
        flow = candidate(path)
        try:
            if case == "foreign-source":
                error = failure_code(lambda: flow.start("north", "R1", "P9"))
                return {"error": error, "runs": flow.db.execute("SELECT count(*) FROM runs").fetchone()[0]}
            run = flow.start("north", "R1")
            if case == "authorized-draft":
                source = flow.db.execute("SELECT body FROM sources WHERE tenant='north' AND id='P1'").fetchone()[0]
                return {"status": run["status"], "source": run["source"], "version": run["version"],
                        "draft_matches_current": run["draft"] == "Operator draft: " + source}
            if case == "foreign-run":
                return {"error": failure_code(lambda: flow.inspect("south", "R1"))}
            if case in ("changed-source", "withdrawn-source"):
                with flow.db:
                    if case == "changed-source":
                        flow.db.execute("UPDATE sources SET version=2, body='Revised synthetic procedure.' WHERE tenant='north' AND id='P1'")
                    else:
                        flow.db.execute("UPDATE sources SET active=0 WHERE tenant='north' AND id='P1'")
                error = failure_code(lambda: flow.review("north", "R1", run["digest"], "accepted"))
                return {"error": error, "status": flow.inspect("north", "R1")["status"]}
            if case == "restart-replay":
                flow.close()
                flow = candidate(path)
                repeated = flow.start("north", "R1")
                accepted = flow.review("north", "R1", run["digest"], "accepted")
                again = flow.review("north", "R1", run["digest"], "accepted")
                return {"same_run": repeated == run,
                        "runs": flow.db.execute("SELECT count(*) FROM runs").fetchone()[0],
                        "same_decision": accepted == again,
                        "conflict": failure_code(lambda: flow.review("north", "R1", run["digest"], "rejected"))}
            raise ValueError("unknown_case")
        finally:
            flow.close()


def evaluate(observations):
    """Missing, duplicate and unknown case IDs cannot silently shrink denominators."""
    ids = [row["case"] for row in observations]
    if len(set(ids)) != len(ids) or any(case not in CONTRACT for case in ids):
        raise ValueError("invalid_case_coverage")
    by_id = {row["case"]: row["observed"] for row in observations}
    rows = [{"case": case, "slice": category,
             "state": "unrun" if case not in by_id else "passed" if by_id[case] == expected else "failed",
             **({"observed": by_id[case]} if case in by_id else {})}
            for case, (category, expected) in CONTRACT.items()]
    slices = {category: {"passed": sum(row["state"] == "passed" for row in rows if row["slice"] == category),
                         "planned": sum(row["slice"] == category for row in rows)}
              for category in ("utility", "boundary", "recovery")}
    return {"contract": "field-desk-regression-v1", "planned": len(CONTRACT), "executed": len(ids),
            "slices": slices, "cases": rows,
            "decision": "local_contract_met" if all(row["state"] == "passed" for row in rows) else "blocked",
            "limits": "Synthetic local domain behavior only; not model quality, human acceptance, distributed safety or production readiness."}


def run(candidate, omit=()):
    observations = []
    for case in CONTRACT:
        if case in omit:
            continue
        try:
            observed = observe(candidate, case)
        except Exception as error:
            # Do not publish exception messages that could contain a source body or path.
            observed = {"unexpected_error": type(error).__name__}
        observations.append({"case": case, "observed": observed})
    return evaluate(observations)


class EvaluationTests(unittest.TestCase):
    def test_reference_meets_all_three_slices(self):
        report = run(Workflow)
        self.assertEqual(report["decision"], "local_contract_met")
        self.assertEqual(report["executed"], 6)
        self.assertEqual(report["slices"], {"utility": {"passed": 1, "planned": 1},
                         "boundary": {"passed": 4, "planned": 4}, "recovery": {"passed": 1, "planned": 1}})

    def test_all_refusal_is_not_usefulness(self):
        report = run(RefuseAll)
        self.assertEqual(report["decision"], "blocked")
        self.assertEqual(report["slices"]["utility"]["passed"], 0)
        self.assertEqual(next(row for row in report["cases"] if row["case"] == "foreign-source")["state"], "passed")

    def test_stale_approval_blocks_otherwise_useful_candidate(self):
        report = run(StaleReview)
        self.assertEqual(report["decision"], "blocked")
        self.assertEqual(report["slices"]["utility"]["passed"], 1)
        self.assertEqual({row["case"] for row in report["cases"] if row["state"] == "failed"},
                         {"changed-source", "withdrawn-source"})

    def test_missing_case_stays_in_planned_denominator(self):
        report = run(Workflow, ["withdrawn-source"])
        self.assertEqual(report["decision"], "blocked")
        self.assertEqual((report["planned"], report["executed"]), (6, 5))
        self.assertEqual(report["slices"]["boundary"], {"passed": 3, "planned": 4})
        self.assertEqual(next(row for row in report["cases"] if row["case"] == "withdrawn-source")["state"], "unrun")

    def test_duplicate_or_unknown_case_is_not_evidence(self):
        row = {"case": "authorized-draft", "observed": {}}
        with self.assertRaisesRegex(ValueError, "invalid_case_coverage"):
            evaluate([row, row])
        with self.assertRaisesRegex(ValueError, "invalid_case_coverage"):
            evaluate([{"case": "invented", "observed": {}}])

    def test_unexpected_exception_is_failed_and_redacted(self):
        class Broken(Workflow):
            def start(self, *args):
                raise RuntimeError("private synthetic detail")
        report = run(Broken)
        self.assertEqual(report["decision"], "blocked")
        self.assertNotIn("private synthetic detail", json.dumps(report))
        self.assertTrue(all(row["state"] == "failed" for row in report["cases"]))


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    sub = parser.add_subparsers(dest="command", required=True)
    sub.add_parser("test")
    cli = sub.add_parser("run")
    cli.add_argument("--candidate", choices=CANDIDATES, default="reference")
    cli.add_argument("--omit-case", action="append", choices=CONTRACT, default=[])
    args = parser.parse_args()
    if args.command == "test":
        with patch.object(socket.socket, "connect", side_effect=AssertionError("network_forbidden")):
            suite = unittest.defaultTestLoader.loadTestsFromTestCase(EvaluationTests)
            result = unittest.TextTestRunner(verbosity=2).run(suite)
        raise SystemExit(0 if result.wasSuccessful() else 1)
    report = run(CANDIDATES[args.candidate], args.omit_case)
    print(json.dumps({"candidate": args.candidate, **report}, indent=2))
    raise SystemExit(0 if report["decision"] == "local_contract_met" else 2)