"""Generate the paper's LaTeX tables from results/results.md and runs/*/*/timing.json. --table film / icartoon_test (Table 1 style) AP@0.5 per training setting + inference speed (im/s); one row per model plus an indented "+ TTA" row. --table training (Table 2) training seconds for every trained model x dataset setting. Examples (from the repo root): python -m src.eval.collect_results --table film # -> results/table_film.tex python -m src.eval.collect_results --table icartoon_test python -m src.eval.collect_results --table training # -> results/table_training.tex """ from __future__ import annotations import argparse import json from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] SETTINGS = ["wf", "icf", "wf-icf"] # base tag prefix -> (display name, {setting: training run dir or None}) TRAINABLE = [ ("retinaface", "RetinaFace-R50", {"wf": None, # released checkpoint "icf": "runs/retinaface/icf", "wf-icf": "runs/retinaface/wf_icf"}), ("yolo26", "YOLO26-L", {"wf": "runs/ultralytics/yolo26_wf", "icf": "runs/ultralytics/yolo26_icf", "wf-icf": "runs/ultralytics/yolo26_wf_icf"}), ("rtdetr", "RT-DETR-L", {"wf": "runs/ultralytics/rtdetr_wf", "icf": "runs/ultralytics/rtdetr_icf", "wf-icf": "runs/ultralytics/rtdetr_wf_icf"}), ("yunet", "YuNet-N", {"wf": None, # released checkpoint "icf": "runs/yunet/icf", "wf-icf": "runs/yunet/wf_icf"}), ] EVAL_ONLY = [ ("yoloe26l-zeroshot", "YOLOE-26L (text-prompted)"), ] CAPTIONS = {"film": "Performance on film", "icartoon_test": "Performance on iCartoonFace (test)"} def parse_results(path): """results.md -> {tag: {"ap", "ci", "ims", "images"}}""" rows = {} if not path.is_file(): return rows for line in path.read_text().splitlines(): cells = [c.strip() for c in line.strip().strip("|").split("|")] if len(cells) < 7 or cells[0] in ("tag", ""): continue try: # columns: tag|checkpoint|images|AP|95% CI|recall|im/s|protocol # (files without the CI column are also accepted) has_ci = len(cells) >= 8 rows[cells[0]] = { "ap": float(cells[3]), "ci": cells[4] if has_ci and cells[4] != "-" else None, "images": int(cells[2]), "ims": (float(cells[6 if has_ci else 5]) if cells[6 if has_ci else 5] != "-" else None), } except ValueError: continue return rows def load_train_seconds(runs_root, run_dir): if run_dir is None: return None p = Path(runs_root) / run_dir / "timing.json" if not p.is_file(): return None return json.loads(p.read_text())["train_seconds"] def performance_table(rows, eval_set): def ap_cell(tag): r = rows.get(tag) if not r: return "--" if r["ci"]: return f"{r['ap']:.2f} {{\\scriptsize {r['ci']}}}" return f"{r['ap']:.2f}" def ims(base, tta): suffix = "-tta" if tta else "" for setting in ["wf-icf", "icf", "wf", None]: tag = (f"{base}_{setting}_{eval_set}{suffix}" if setting else f"{base}_{eval_set}{suffix}") r = rows.get(tag) if r and r["ims"]: return f"{r['ims']:.1f}" return "--" lines = [ "% generated by src/eval/collect_results.py", "\\begin{table}", "\\centering", "\\begin{tabular}{lrrrr}", "\\toprule", "Model & Train WF & Train ICF & Train WF+ICF & im/s \\\\", "\\midrule", ] for base, name, _ in TRAINABLE: plain = [ap_cell(f"{base}_{s}_{eval_set}") for s in SETTINGS] tta = [ap_cell(f"{base}_{s}_{eval_set}-tta") for s in SETTINGS] lines.append(f"{name} & " + " & ".join(plain) + f" & {ims(base, False)} \\\\") lines.append("\\quad + TTA & " + " & ".join(tta) + f" & {ims(base, True)} \\\\") lines.append("\\midrule") for base, name in EVAL_ONLY: lines.append(f"{name} & \\multicolumn{{3}}{{c}}{{{ap_cell(f'{base}_{eval_set}')}}} " f"& {ims(base, False)} \\\\") lines.append(f"\\quad + TTA & \\multicolumn{{3}}{{c}}" f"{{{ap_cell(f'{base}_{eval_set}-tta')}}} & {ims(base, True)} \\\\") lines += [ "\\bottomrule", "\\end{tabular}", f"\\caption{{{CAPTIONS[eval_set]}. AP@0.5 (95\\% bootstrap CI) by " "training data; inference throughput in images per second on the " f"{eval_set.replace('_', ' ')} set.}}", f"\\label{{tab:{eval_set}}}", "\\end{table}", ] return "\n".join(lines) + "\n" def training_table(runs_root): lines = [ "% generated by src/eval/collect_results.py", "\\begin{table}", "\\centering", "\\begin{tabular}{lrrr}", "\\toprule", "Model & Train WF & Train ICF & Train WF+ICF \\\\", "\\midrule", ] for _, name, run_dirs in TRAINABLE: cells = [] for s in SETTINGS: secs = load_train_seconds(runs_root, run_dirs[s]) if secs is None: cells.append("--" if run_dirs[s] else "\\textit{released}") else: cells.append(f"{secs:,.0f}") lines.append(f"{name} & " + " & ".join(cells) + " \\\\") lines += [ "\\bottomrule", "\\end{tabular}", "\\caption{Training time in seconds per model and training dataset. " "``released'' marks publicly released WIDER Face checkpoints used " "as-is. Hardware and per-epoch details are recorded in each run's " "\\texttt{timing.json}.}", "\\label{tab:training}", "\\end{table}", ] return "\n".join(lines) + "\n" def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--table", default="film", choices=["film", "icartoon_test", "training"]) ap.add_argument("--results", default=str(REPO_ROOT / "results/results.md")) ap.add_argument("--runs-root", default=str(REPO_ROOT), help="directory containing runs/ (default: repo root)") ap.add_argument("--out", default=None, help="default: results/table_.tex") args = ap.parse_args() out = Path(args.out or REPO_ROOT / "results" / f"table_{args.table}.tex") if args.table == "training": table = training_table(args.runs_root) else: table = performance_table(parse_results(Path(args.results)), args.table) out.parent.mkdir(parents=True, exist_ok=True) out.write_text(table) print(table) print(f"% written to {out}") if __name__ == "__main__": main()