#!/usr/bin/env python3 """Original LockedIn Labs teaching lab. Synthetic text; no packages or model calls. python3 fde-first-evaluation.py test python3 fde-first-evaluation.py compare python3 fde-first-evaluation.py try --text "Can a colleague take over?" --expected handoff """ import argparse import json import re import unittest DATASET = "support-routing-six/v1" CASES = [ {"id": "E1", "text": "Where can I download my invoice?", "expected": "guide", "reason": "The person asks for instructions and does not request a human."}, {"id": "E2", "text": "Please connect me to a person.", "expected": "handoff", "reason": "An explicit request for a person takes priority over another automated answer."}, {"id": "E3", "text": "How do I change my email address?", "expected": "guide", "reason": "The requested output is a set of instructions, not an account change."}, {"id": "E4", "text": "I don't need a person. Where can I download my invoice?", "expected": "guide", "reason": "The word person is present, but the request explicitly declines a handoff."}, {"id": "E5", "text": "Please stop the automated replies. Someone needs to help me.", "expected": "handoff", "reason": "The person asks automation to stop, without using the words human or person."}, {"id": "E6", "text": "The instructions didn't solve it. Transfer me to your support team.", "expected": "handoff", "reason": "An explicit transfer request should reach support rather than repeat a guide."}, ] PROBE = { "id": "P1", "text": "I've already tried twice. Can a colleague take over?", "expected": "handoff", "reason": "This different wording requests human help. It is a published teaching probe, not a hidden test.", } CANDIDATES = [ {"id": "keywords", "label": "Keyword baseline", "description": "Hand off only when the text contains the word human or person."}, {"id": "always-handoff", "label": "Always hand off", "description": "Send every request to a person, including requests for simple instructions."}, {"id": "expanded-rules", "label": "Expanded rules", "description": "Handle this negation and two extra handoff phrases, then use the keyword baseline."}, ] def predict(candidate, text): """Only the request enters the candidate. Expected labels stay with the evaluator.""" if not isinstance(text, str) or not text.strip() or len(text) > 2000: raise ValueError("Use a non-empty synthetic request of at most 2000 characters.") normalized = text.casefold() if candidate == "always-handoff": return "handoff" if candidate == "expanded-rules": if "don't need a person" in normalized: return "guide" if "stop the automated replies" in normalized or "transfer me to your support team" in normalized: return "handoff" elif candidate != "keywords": raise ValueError("Unknown candidate.") return "handoff" if re.search(r"\b(human|person)\b", normalized) else "guide" def summarize(cases, results): """A confusion matrix, with handoff as the positive class. Missing is not correct.""" if not cases or len({case["id"] for case in cases}) != len(cases): raise ValueError("Supply a non-empty dataset with unique case IDs.") ids = {case["id"] for case in cases} if len({row["id"] for row in results}) != len(results) or any(row["id"] not in ids for row in results): raise ValueError("Results must use unique IDs from the dataset.") indexed = {row["id"]: row["predicted"] for row in results} counts = {"correct_handoff": 0, "missed_handoff": 0, "unnecessary_handoff": 0, "correct_guide": 0} for case in cases: expected = case["expected"] predicted = indexed.get(case["id"]) if expected not in ("guide", "handoff") or (case["id"] in indexed and predicted not in ("guide", "handoff")): raise ValueError("Routes must be guide or handoff.") if predicted is None: continue if expected == "handoff": key = "correct_handoff" if predicted == "handoff" else "missed_handoff" else: key = "unnecessary_handoff" if predicted == "handoff" else "correct_guide" counts[key] += 1 executed = sum(counts.values()) return { **counts, "planned": len(cases), "executed": executed, "unrun": len(cases) - executed, "correct": counts["correct_handoff"] + counts["correct_guide"], "expected_handoffs": sum(case["expected"] == "handoff" for case in cases), "predicted_handoffs": counts["correct_handoff"] + counts["unnecessary_handoff"], } def compare(): runs = [] for candidate in CANDIDATES: results = [{"id": case["id"], "predicted": predict(candidate["id"], case["text"])} for case in CASES] runs.append({**candidate, "results": results, "summary": summarize(CASES, results), "probe_prediction": predict(candidate["id"], PROBE["text"])}) return { "dataset": DATASET, "provenance": "Actual execution of three deterministic Python functions on original synthetic cases.", "model_calls": 0, "cases": CASES, "candidates": runs, "probe": PROBE, } class EvaluationChecks(unittest.TestCase): def test_same_accuracy_can_hide_different_failures(self): baseline, always, expanded = compare()["candidates"] self.assertEqual(baseline["summary"]["correct"], 3) self.assertEqual(always["summary"]["correct"], 3) self.assertEqual(baseline["summary"]["missed_handoff"], 2) self.assertEqual(always["summary"]["missed_handoff"], 0) self.assertEqual(baseline["summary"]["unnecessary_handoff"], 1) self.assertEqual(always["summary"]["unnecessary_handoff"], 3) self.assertEqual(expanded["summary"]["correct"], 6) def test_known_examples_do_not_prove_generalization(self): self.assertEqual(predict("expanded-rules", PROBE["text"]), "guide") self.assertEqual(PROBE["expected"], "handoff") self.assertEqual(compare()["candidates"][2]["summary"]["planned"], 6) def test_matrix_reconciles_to_executed_cases(self): for run in compare()["candidates"]: summary = run["summary"] self.assertEqual(sum(summary[key] for key in ( "correct_handoff", "missed_handoff", "unnecessary_handoff", "correct_guide")), 6) self.assertEqual((summary["executed"], summary["unrun"]), (6, 0)) self.assertEqual(summary["expected_handoffs"], 3) def test_missing_result_keeps_planned_denominator(self): run = compare()["candidates"][2] summary = summarize(CASES, run["results"][:-1]) self.assertEqual((summary["correct"], summary["executed"], summary["planned"], summary["unrun"]), (5, 5, 6, 1)) def test_expected_labels_do_not_change_candidate_output(self): before = predict("keywords", CASES[0]["text"]) changed = [{**CASES[0], "expected": "handoff"}] summary = summarize(changed, [{"id": "E1", "predicted": before}]) self.assertEqual(before, "guide") self.assertEqual(summary["missed_handoff"], 1) def test_reject_duplicate_unknown_and_invalid_results(self): for results in ( [{"id": "E1", "predicted": "guide"}] * 2, [{"id": "absent", "predicted": "guide"}], [{"id": "E1", "predicted": "almost"}], [{"id": "E1", "predicted": None}], ): with self.assertRaises(ValueError): summarize(CASES, results) def test_input_and_candidate_boundaries(self): for text in ("", " ", "x" * 2001, None): with self.assertRaises(ValueError): predict("keywords", text) with self.assertRaises(ValueError): predict("unknown", "A synthetic request.") self.assertEqual(predict("keywords", "My personal settings"), "guide") self.assertEqual(predict("keywords", "A HUMAN please"), "handoff") def main(): parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="command", required=True) sub.add_parser("test") sub.add_parser("compare") attempt = sub.add_parser("try") attempt.add_argument("--text", required=True, help="Use invented text, never private customer data.") attempt.add_argument("--expected", required=True, choices=["guide", "handoff"]) args = parser.parse_args() if args.command == "test": result = unittest.TextTestRunner(verbosity=2).run(unittest.defaultTestLoader.loadTestsFromTestCase(EvaluationChecks)) return 0 if result.wasSuccessful() else 1 try: if args.command == "compare": report = compare() else: report = { "text": args.text, "expected": args.expected, "included_in_six_case_score": False, "results": [{"candidate": c["id"], "predicted": predict(c["id"], args.text)} for c in CANDIDATES], } print(json.dumps(report, indent=2, ensure_ascii=False)) return 0 except ValueError as error: parser.error(str(error)) if __name__ == "__main__": raise SystemExit(main())