#!/usr/bin/env python3
"""Original LockedIn Labs exercise. Python 3.11+, standard library only.

Synthetic measurements, not a model benchmark. No network or credentials.
Run: python3 fde-evaluation-science.py
"""
import json
from math import isfinite


def classification(expected, predicted):
    if len(expected) != len(predicted) or any(type(x) is not bool for x in [*expected, *predicted]):
        raise ValueError("Supply equal-length lists of boolean labels.")
    tp = sum(a and b for a, b in zip(expected, predicted))
    fp = sum(not a and b for a, b in zip(expected, predicted))
    fn = sum(a and not b for a, b in zip(expected, predicted))
    tn = sum(not a and not b for a, b in zip(expected, predicted))
    return {"tp": tp, "fp": fp, "fn": fn, "tn": tn,
            "precision": tp / (tp + fp) if tp + fp else None,
            "recall": tp / (tp + fn) if tp + fn else None,
            "accuracy": (tp + tn) / len(expected) if expected else None}


def source_overlap(records):
    """Detect shared source IDs across splits; this does not judge representativeness."""
    sources = {}
    for row in records:
        sources.setdefault(row["source"], set()).add(row["split"])
    return {source: sorted(splits) for source, splits in sources.items() if len(splits) > 1}


def retrieval_metrics(ranked, relevant, forbidden, k):
    if type(k) is not int or k < 1 or len(set(ranked)) != len(ranked):
        raise ValueError("Use positive k and unique ranked document IDs.")
    top = ranked[:k]
    relevant = set(relevant)
    first = next((i for i, doc in enumerate(top, 1) if doc in relevant), None)
    return {"k": k, "returned": top,
            "recall_at_k": len(set(top) & relevant) / len(relevant) if relevant else None,
            "reciprocal_rank_at_k": 1 / first if first else 0,
            "forbidden_returned": sorted(set(top) & set(forbidden))}


def economics(minutes, cost_units, accepted):
    if (type(accepted) is not int or accepted < 0
            or any(not isfinite(x) or x < 0 for x in (minutes, cost_units))):
        raise ValueError("Use finite nonnegative effort/cost and an integer outcome count.")
    return {"minutes": minutes, "cost_units": cost_units, "accepted": accepted,
            "minutes_per_accepted": minutes / accepted if accepted else None,
            "cost_per_accepted": cost_units / accepted if accepted else None}


def main():
    report = {
        "scope": "Synthetic arithmetic; no model invoked and no customer outcome measured.",
        "classification": classification([True] * 8 + [False, True], [True] * 9 + [False]),
        "empty_classification": classification([], []),
        "source_overlap": source_overlap([
            {"source": "handbook-7", "split": "development"},
            {"source": "handbook-7", "split": "test"},
            {"source": "handbook-8", "split": "test"},
        ]),
        "retrieval": retrieval_metrics(["D3", "D1", "D4"], {"D1", "D2"}, {"D3"}, 3),
        "manual": economics(120, 24, 8),
        "assisted": economics(30 + 110, 35, 7),
    }
    print(json.dumps(report, indent=2, allow_nan=False))


if __name__ == "__main__":
    main()