📊 Eval Mínimo para Prompts: La herramienta que faltaba para medir lo que realmente mejora en tu IA

import csv, re, argparse, sys, json
from dataclasses import dataclass, asdict
from typing import Dict, Tuple, List

# -------- utilidades --------

def normalize(s: str) -> str:
    return " ".join((s or "").strip().split()).casefold()

def levenshtein(a: str, b: str) -> int:
    a, b = a or "", b or ""
    la, lb = len(a), len(b)
    if la == 0: return lb
    if lb == 0: return la
    dp = list(range(lb + 1))
    for i in range(1, la + 1):
        prev = dp[0]
        dp[0] = i
        ca = a[i - 1]
        for j in range(1, lb + 1):
            temp = dp[j]
            cb = b[j - 1]
            cost = 0 if ca == cb else 1
            dp[j] = min(
                dp[j] + 1,      # deletion
                dp[j - 1] + 1,  # insertion
                prev + cost     # substitution
            )
            prev = temp
    return dp[lb]

def near_miss(pred: str, expected: str) -> Tuple[float, int]:
    # similitud normalizada 0..1 basada en Levenshtein sobre textos normalizados
    p, e = normalize(pred), normalize(expected)
    if e == "" and p == "":
        return 1.0, 0
    dist = levenshtein(p, e)
    denom = max(len(p), len(e)) or 1
    sim = 1.0 - (dist / denom)
    return sim, dist

# -------- datos --------

@dataclass
class Case:
    id: str
    input: str
    expected: str
    match_type: str
    pattern: str

@dataclass
class Verdict:
    id: str
    match_type: str
    ok: bool
    reason: str
    near_miss: float
    distance: int

# -------- carga y validaciones --------

REQUIRED_GOLD = ["id","input","expected","match_type","pattern"]
REQUIRED_PRED = ["id","output"]

def assert_columns(path: str, required: List[str]):
    with open(path, newline='', encoding='utf-8') as f:
        reader = csv.reader(f)
        try:
            header = next(reader)
        except StopIteration:
            raise ValueError(f"{path}: CSV vacío")
    missing = [c for c in required if c not in header]
    if missing:
        raise ValueError(f"{path}: faltan columnas {missing}. Esperado {required}")

def load_gold(path: str) -> Dict[str, Case]:
    assert_columns(path, REQUIRED_GOLD)
    cases: Dict[str, Case] = {}
    seen = set()
    with open(path, newline='', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        for i, row in enumerate(reader, start=2):
            cid = (row.get("id") or "").strip()
            if not cid:
                raise ValueError(f"{path}: fila {i} sin 'id'")
            if cid in seen:
                raise ValueError(f"{path}: 'id' duplicado '{cid}' (fila {i})")
            seen.add(cid)
            match_type = (row.get("match_type") or "exact").strip().lower()
            expected = (row.get("expected") or "").strip()
            pattern = (row.get("pattern") or "").strip()
            if match_type == "exact" and expected == "":
                raise ValueError(f"{path}: fila {i} exact sin 'expected'")
            if match_type == "regex" and pattern == "":
                raise ValueError(f"{path}: fila {i} regex sin 'pattern'")
            cases[cid] = Case(
                id=cid,
                input=row.get("input",""),
                expected=expected,
                match_type=match_type,
                pattern=pattern
            )
    return cases

def load_pred(path: str) -> Dict[str, str]:
    assert_columns(path, REQUIRED_PRED)
    pred: Dict[str, str] = {}
    seen = set()
    with open(path, newline='', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        for i, row in enumerate(reader, start=2):
            cid = (row.get("id") or "").strip()
            if not cid:
                raise ValueError(f"{path}: fila {i} sin 'id'")
            if cid in seen:
                raise ValueError(f"{path}: 'id' duplicado '{cid}' (fila {i})")
            seen.add(cid)
            pred[cid] = (row.get("output") or "").strip()
    return pred

# -------- evaluación --------

def judge(case: Case, output: str) -> Verdict:
    if case.match_type == "exact":
        ok = normalize(output) == normalize(case.expected)
        reason = "exact" if ok else f"mismatch (expected='{case.expected}', got='{output}')"
        sim, dist = near_miss(output, case.expected)
        return Verdict(case.id, case.match_type, ok, reason, sim, dist)
    elif case.match_type == "regex":
        try:
            ok = re.fullmatch(case.pattern, output or "") is not None
            reason = f"regex:{case.pattern}" if ok else f"no-match (pattern={case.pattern}, got='{output}')"
        except re.error as e:
            ok = False
            reason = f"bad-regex ({e})"
        # near-miss para regex: si hay expected, medir contra expected; si no, 0.0
        exp = case.expected or ""
        sim, dist = near_miss(output, exp) if exp else (0.0, max(len(output or ""), 0))
        return Verdict(case.id, case.match_type, ok, reason, sim, dist)
    else:
        return Verdict(case.id, case.match_type, False, f"unknown match_type '{case.match_type}'", 0.0, 0)

def run_eval(gold_path: str, pred_path: str):
    gold = load_gold(gold_path)
    pred = load_pred(pred_path)

    verdicts: List[Verdict] = []
    for cid, case in gold.items():
        out = pred.get(cid, "")
        verdicts.append(judge(case, out))

    total = len(verdicts)
    correct = sum(1 for v in verdicts if v.ok)
    acc = (correct / total * 100.0) if total else 0.0
    failures = [v for v in verdicts if not v.ok]
    return verdicts, total, correct, acc, failures

def format_table(verdicts: List[Verdict]) -> str:
    lines = []
    lines.append("RESULTADOS POR CASO")
    lines.append("-------------------")
    lines.append(f"{'id':<4} {'tipo':<6} {'ok':<4} {'reason':<44} {'near_miss':<9} {'dist':<4}")
    for v in verdicts:
        lines.append(f"{v.id:<4} {v.match_type:<6} {str(v.ok):<4} {v.reason:<44} {v.near_miss:>1.3f}      {v.distance:<4}")
    return "\n".join(lines)

def format_summary(total: int, correct: int, acc: float) -> str:
    lines = []
    lines.append("\nRESUMEN")
    lines.append("-------")
    lines.append(f"Casos totales: {total}")
    lines.append(f"Correctos    : {correct}")
    lines.append(f"Accuracy     : {acc:.1f}%")
    return "\n".join(lines)

def format_failures(failures: List[Verdict]) -> str:
    if not failures:
        return "\nSIN ERRORES 🎉"
    lines = []
    lines.append("\nFAILURES")
    lines.append("--------")
    for v in failures:
        lines.append(f"- id={v.id} [{v.match_type}] -> {v.reason} (near_miss={v.near_miss:.3f}, dist={v.distance})")
    return "\n".join(lines)

def main():
    ap = argparse.ArgumentParser(description="Eval mínimo para prompts (exact/regex) con errores visibles y near-miss.")
    ap.add_argument("--gold", default="data/gold.csv", help="Ruta a gold.csv")
    ap.add_argument("--pred", default="data/pred.csv", help="Ruta a pred.csv")
    ap.add_argument("--out", default="", help="Guardar salida de texto en archivo")
    ap.add_argument("--json", default="", help="Guardar resultados en JSON")
    ap.add_argument("--strict", action="store_true", help="Salir con código 1 si hay fallos")
    args = ap.parse_args()

    try:
        verdicts, total, correct, acc, failures = run_eval(args.gold, args.pred)
        text = "\n".join([
            format_table(verdicts),
            format_summary(total, correct, acc),
            format_failures(failures),
            ""
        ])
        print(text)

        if args.out:
            with open(args.out, "w", encoding="utf-8") as f:
                f.write(text)

        if args.json:
            with open(args.json, "w", encoding="utf-8") as f:
                json.dump({
                    "total": total,
                    "correct": correct,
                    "accuracy": acc,
                    "failures": [asdict(v) for v in failures],
                    "verdicts": [asdict(v) for v in verdicts],
                }, f, ensure_ascii=False, indent=2)

        if args.strict and failures:
            sys.exit(1)

    except Exception as e:
        msg = f"ERROR FATAL: {e}"
        print(msg, file=sys.stderr)
        if args.out:
            with open(args.out, "w", encoding="utf-8") as f:
                f.write(msg+"\n")
        if args.strict:
            sys.exit(1)
        else:
            sys.exit(0)

if __name__ == "__main__":
    main()

name: Eval Minimo CI

on:
  push:
    branches: [ "main" ]
  workflow_dispatch:

jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Run eval
        run: |
          python -V
          python eval.py --gold data/gold.csv --pred data/pred.csv --out results.txt --json results.json --strict
      - name: Publicar artefactos
        uses: actions/upload-artifact@v4
        with:
          name: eval-report
          path: |
            results.txt
            results.json