537 lines
26 KiB
Python
537 lines
26 KiB
Python
#!/usr/bin/env python3
|
|
"""results.py - turn experiment results into paper-ready LaTeX tables and PDF figures.
|
|
|
|
All commands work on ONE tidy CSV (long format), one row per measured value:
|
|
method,dataset,metric,value[,seed][,step][,<any extra column, e.g. noise, train_size, variant>]
|
|
|
|
python results.py melt wide.csv --id method,dataset,seed[,step] [--metrics acc,f1] --out paper/data/results.csv [--append]
|
|
python results.py tb LOGDIR --method-level 1 [--dataset NAME] --out paper/data/results.csv [--append]
|
|
python results.py check paper/data/results.csv
|
|
python results.py summary paper/data/results.csv [--final last|best:max|best:min] [--out paper/data/summary.csv]
|
|
python results.py table paper/data/results.csv --metrics acc:max[,loss:min] --out paper/tables/main.tex
|
|
[--rows method] [--cols dataset] [--order ours,base1,...] [--highlight ours] [--scale 100] [--digits 1]
|
|
[--where "variant==full"] [--final last] [--caption "..."] [--label tab:main] [--no-std] [--wide]
|
|
python results.py figure line paper/data/results.csv --metric val_acc --x step [--facet dataset] --out paper/figures/curves.pdf
|
|
python results.py figure bars paper/data/results.csv --metric acc [--group dataset] --out paper/figures/bars.pdf
|
|
python results.py figure scatter paper/data/results.csv --x-metric time --y-metric acc [--facet dataset] --out paper/figures/tradeoff.pdf
|
|
common figure options: [--order a,b,c] [--highlight ours] [--width single|double|<inches>] [--height 2.0]
|
|
[--where "dataset=='CIFAR-10'"] [--ylabel "..."] [--xlabel "..."] [--logx] [--logy] [--ylim 60,80] [--xlim 0,50] [--scale 100] [--final last]
|
|
|
|
melt wide table (one column per metric) -> tidy rows. tb TensorBoard scalars -> tidy rows (pip install tensorboard).
|
|
check validates the tidy file and prints the coverage: methods x datasets x metrics, seeds per cell, gaps.
|
|
summary final value per run (last step, or best step), then mean/std/n/95%-CI per method x dataset x metric.
|
|
Writes summary.csv - the ONLY allowed source for numbers quoted in the paper text.
|
|
table booktabs table: mean +- std over seeds, best per column in bold, second underlined, cells that are
|
|
not significantly worse than the best (Welch t-test, p >= 0.05, needs >= 3 seeds) marked with a dagger.
|
|
figure vector PDF (+ PNG preview for view_image) in a consistent, colour-blind-safe style; each method keeps
|
|
the same colour and marker in every figure (paper/figures/style.json). Shaded band = +-1 std over seeds.
|
|
--rename "val_acc=Accuracy,ce=Cross-entropy" sets display names (methods, datasets, metrics, columns) in
|
|
tables and figures without touching the data. Axis labels are plain text: write "Accuracy (%)", not "\\%".
|
|
Every table/figure command also prints the numbers it used, so the text can quote them exactly.
|
|
Requires pandas, numpy, matplotlib (scipy optional for the t-test).
|
|
"""
|
|
import json, math, re, sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
import numpy as np
|
|
import pandas as pd
|
|
except ImportError:
|
|
sys.exit("pip install pandas numpy matplotlib scipy")
|
|
|
|
REQUIRED = ["method", "dataset", "metric", "value"]
|
|
# Okabe-Ito without black/yellow (validated: CVD-separable; markers give the secondary encoding).
|
|
PALETTE = ["#0072B2", "#E69F00", "#009E73", "#D55E00", "#CC79A7", "#56B4E9"]
|
|
MARKERS = ["o", "s", "^", "D", "v", "P", "X", "*"]
|
|
LINESTYLES = ["-", "--", ":", "-."]
|
|
STYLE_FILE = Path("paper/figures/style.json")
|
|
RENAME = {} # display names, from --rename "val_acc=Accuracy,ce=Cross-entropy"
|
|
|
|
|
|
def disp(x):
|
|
return RENAME.get(str(x), str(x))
|
|
|
|
|
|
# ----------------------------------------------------------------- arguments
|
|
def parse(argv):
|
|
pos, opt, i = [], {}, 0
|
|
while i < len(argv):
|
|
a = argv[i]
|
|
if a.startswith("--"):
|
|
if i + 1 < len(argv) and not argv[i + 1].startswith("--"):
|
|
opt[a[2:]] = argv[i + 1]; i += 2
|
|
else:
|
|
opt[a[2:]] = True; i += 1
|
|
else:
|
|
pos.append(a); i += 1
|
|
return pos, opt
|
|
|
|
|
|
def csv_list(s):
|
|
return [x.strip() for x in str(s).split(",") if x.strip()] if s and s is not True else []
|
|
|
|
|
|
# ----------------------------------------------------------------- loading
|
|
def load(path, where=None):
|
|
df = pd.read_csv(path)
|
|
miss = [c for c in REQUIRED if c not in df.columns]
|
|
if miss:
|
|
sys.exit(f"{path}: missing columns {miss}. Required: {REQUIRED} (+ optional seed, step, extras)")
|
|
df["value"] = pd.to_numeric(df["value"], errors="coerce")
|
|
for c in ("method", "dataset", "metric"):
|
|
df[c] = df[c].astype(str)
|
|
if "seed" not in df.columns:
|
|
df["seed"] = 0
|
|
if where:
|
|
df = df.query(where, engine="python")
|
|
if df.empty:
|
|
sys.exit(f"--where {where!r} selects no rows")
|
|
return df
|
|
|
|
|
|
def extras(df):
|
|
return [c for c in df.columns if c not in REQUIRED + ["seed", "step"]]
|
|
|
|
|
|
def final_values(df, final="last"):
|
|
"""One value per run (= method, dataset, metric, seed, extras): last step, or best step."""
|
|
keys = ["method", "dataset", "metric", "seed"] + extras(df)
|
|
df = df.dropna(subset=["value"])
|
|
if "step" not in df.columns or df["step"].isna().all():
|
|
return df.groupby(keys, dropna=False, as_index=False)["value"].mean()
|
|
df = df.sort_values("step")
|
|
if final == "last":
|
|
return df.groupby(keys, dropna=False, as_index=False)["value"].last()
|
|
mode = final.split(":")[-1]
|
|
agg = "max" if mode == "max" else "min"
|
|
return df.groupby(keys, dropna=False, as_index=False)["value"].agg(agg)
|
|
|
|
|
|
def aggregate(fin, by):
|
|
g = fin.groupby(by, dropna=False)["value"]
|
|
s = g.agg(["mean", "std", "count", "min", "max"]).reset_index().rename(columns={"count": "n"})
|
|
s["std"] = s["std"].fillna(0.0)
|
|
tcrit = s["n"].map(lambda n: _t975(n - 1) if n > 1 else float("nan"))
|
|
s["ci95"] = tcrit * s["std"] / np.sqrt(s["n"])
|
|
return s
|
|
|
|
|
|
def _t975(dof):
|
|
try:
|
|
from scipy import stats
|
|
return float(stats.t.ppf(0.975, dof))
|
|
except ImportError:
|
|
table = {1: 12.71, 2: 4.30, 3: 3.18, 4: 2.78, 5: 2.57, 6: 2.45, 7: 2.36, 8: 2.31, 9: 2.26, 10: 2.23}
|
|
return table.get(dof, 1.96)
|
|
|
|
|
|
def welch_p(a, b):
|
|
if len(a) < 3 or len(b) < 3:
|
|
return None
|
|
try:
|
|
from scipy import stats
|
|
return float(stats.ttest_ind(a, b, equal_var=False).pvalue)
|
|
except ImportError:
|
|
return None
|
|
|
|
|
|
def ordered(values, order, highlight=None):
|
|
vals = list(dict.fromkeys(values))
|
|
first = [v for v in order if v in vals]
|
|
rest = sorted(v for v in vals if v not in first)
|
|
out = first + rest
|
|
if highlight and highlight in out and not order:
|
|
out.remove(highlight); out.insert(0, highlight)
|
|
return out
|
|
|
|
|
|
# ----------------------------------------------------------------- melt / tb
|
|
def cmd_melt(pos, opt):
|
|
df = pd.read_csv(pos[0])
|
|
ids = csv_list(opt.get("id"))
|
|
miss = [c for c in ids if c not in df.columns]
|
|
if miss:
|
|
sys.exit(f"--id columns not in file: {miss}; columns are {list(df.columns)}")
|
|
metrics = csv_list(opt.get("metrics")) or [c for c in df.columns if c not in ids and pd.api.types.is_numeric_dtype(df[c])]
|
|
long = df.melt(id_vars=ids, value_vars=metrics, var_name="metric", value_name="value")
|
|
for c in ("method", "dataset"):
|
|
if c not in long.columns:
|
|
if opt.get(c):
|
|
long[c] = opt[c]
|
|
else:
|
|
sys.exit(f"no '{c}' column - add it to --id, or give a constant with --{c} NAME")
|
|
write_tidy(long, opt)
|
|
|
|
|
|
def cmd_tb(pos, opt):
|
|
try:
|
|
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
|
|
except ImportError:
|
|
sys.exit("pip install tensorboard")
|
|
root, level = Path(pos[0]), int(opt.get("method-level", 1))
|
|
rows = []
|
|
for ev in sorted(root.rglob("*tfevents*")):
|
|
run = ev.parent.relative_to(root).parts
|
|
method = run[min(level, len(run)) - 1] if run else root.name
|
|
seed_m = re.search(r"seed[_-]?(\d+)", str(ev.parent))
|
|
acc = EventAccumulator(str(ev.parent)); acc.Reload()
|
|
for tag in acc.Tags().get("scalars", []):
|
|
for e in acc.Scalars(tag):
|
|
rows.append({"method": method, "dataset": opt.get("dataset", "default"), "metric": tag,
|
|
"value": e.value, "seed": int(seed_m.group(1)) if seed_m else 0, "step": e.step,
|
|
"run": "/".join(run)})
|
|
if not rows:
|
|
sys.exit("no scalars found")
|
|
write_tidy(pd.DataFrame(rows).drop_duplicates(), opt)
|
|
|
|
|
|
def write_tidy(df, opt):
|
|
out = Path(opt.get("out", "paper/data/results.csv"))
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
if opt.get("append") and out.exists():
|
|
df = pd.concat([pd.read_csv(out), df], ignore_index=True)
|
|
cols = REQUIRED + [c for c in ("seed", "step") if c in df.columns] + [c for c in df.columns if c not in REQUIRED + ["seed", "step"]]
|
|
df[cols].to_csv(out, index=False)
|
|
print(f"wrote {out}: {len(df)} rows, metrics {sorted(df['metric'].astype(str).unique())[:15]}")
|
|
|
|
|
|
# ----------------------------------------------------------------- check / summary
|
|
def cmd_check(pos, opt):
|
|
df = load(pos[0], opt.get("where"))
|
|
ex = extras(df)
|
|
print(f"{pos[0]}: {len(df)} rows; extra columns: {ex or '-'}; steps: {'yes' if 'step' in df.columns else 'no'}")
|
|
nan = df["value"].isna().sum()
|
|
if nan:
|
|
print(f"WARNING: {nan} non-numeric/NaN values")
|
|
keys = ["method", "dataset", "metric", "seed"] + ex + (["step"] if "step" in df.columns else [])
|
|
dup = df.duplicated(keys).sum()
|
|
if dup:
|
|
print(f"WARNING: {dup} duplicate rows for the same run/step - add a distinguishing column (e.g. variant) or deduplicate")
|
|
for metric, d in df.groupby("metric"):
|
|
seeds = d.groupby(["method", "dataset"])["seed"].nunique().unstack("dataset")
|
|
print(f"\nmetric '{metric}': seeds per method x dataset (blank = missing)")
|
|
print(seeds.fillna(0).astype(int).replace(0, "").to_string())
|
|
n = df.groupby(["method", "dataset", "metric"])["seed"].nunique()
|
|
if (n < 3).any():
|
|
print("\nNOTE: some cells have < 3 seeds - no significance tests there; say so in the paper.")
|
|
methods, datasets = df["method"].unique(), df["dataset"].unique()
|
|
print(f"\n{len(methods)} methods: {', '.join(map(str, methods))}\n{len(datasets)} datasets: {', '.join(map(str, datasets))}")
|
|
|
|
|
|
def cmd_summary(pos, opt):
|
|
df = load(pos[0], opt.get("where"))
|
|
fin = final_values(df, opt.get("final", "last"))
|
|
s = aggregate(fin, ["method", "dataset", "metric"] + extras(df))
|
|
out = Path(opt.get("out", Path(pos[0]).with_name("summary.csv")))
|
|
s.to_csv(out, index=False, float_format="%.6g")
|
|
with pd.option_context("display.max_rows", 200, "display.width", 200):
|
|
print(s.to_string(index=False, float_format=lambda v: f"{v:.4g}"))
|
|
print(f"\nwrote {out} ({len(s)} cells) - quote numbers in the text only from this file or the generated tables")
|
|
|
|
|
|
# ----------------------------------------------------------------- table
|
|
def fmt(v, digits):
|
|
return "--" if v is None or (isinstance(v, float) and math.isnan(v)) else f"{v:.{digits}f}"
|
|
|
|
|
|
def tex_escape(s):
|
|
return re.sub(r"([&%$#_{}])", r"\\\1", str(s))
|
|
|
|
|
|
def arrow(d):
|
|
return r"$\uparrow$" if d == "max" else r"$\downarrow$"
|
|
|
|
|
|
def cmd_table(pos, opt):
|
|
df = load(pos[0], opt.get("where"))
|
|
specs = [m.split(":") for m in csv_list(opt.get("metrics"))]
|
|
if not specs:
|
|
sys.exit("--metrics acc:max,loss:min is required (max = higher is better)")
|
|
for sp in specs:
|
|
if len(sp) != 2 or sp[1] not in ("max", "min"):
|
|
sys.exit(f"bad metric spec {':'.join(sp)} - use name:max or name:min")
|
|
if sp[0] not in set(df["metric"]):
|
|
sys.exit(f"metric '{sp[0]}' not in data; available: {sorted(df['metric'].unique())}")
|
|
rows_by, cols_by = opt.get("rows", "method"), opt.get("cols", "dataset")
|
|
scale, digits = float(opt.get("scale", 1)), int(opt.get("digits", 2))
|
|
show_std = not opt.get("no-std")
|
|
fin = final_values(df[df["metric"].isin([s[0] for s in specs])], opt.get("final", "last"))
|
|
fin["value"] *= scale
|
|
row_vals = ordered(fin[rows_by], csv_list(opt.get("order")), opt.get("highlight"))
|
|
col_vals = ordered(fin[cols_by], csv_list(opt.get("col-order")))
|
|
multi_metric = len(specs) > 1
|
|
columns = [(c, m, d) for c in col_vals for m, d in specs]
|
|
cell, raw = {}, {}
|
|
for (r, c, m), g in fin.groupby([rows_by, cols_by, "metric"]):
|
|
cell[(r, c, m)] = (g["value"].mean(), g["value"].std(ddof=1) if len(g) > 1 else 0.0, len(g))
|
|
raw[(r, c, m)] = g["value"].to_numpy()
|
|
marks = {}
|
|
for c, m, d in columns:
|
|
present = [(r, cell[(r, c, m)][0]) for r in row_vals if (r, c, m) in cell]
|
|
if not present:
|
|
continue
|
|
ranked = sorted(present, key=lambda x: -x[1] if d == "max" else x[1])
|
|
best = ranked[0][0]
|
|
marks[(best, c, m)] = "best"
|
|
if len(ranked) > 1 and ranked[1][1] != ranked[0][1]:
|
|
marks[(ranked[1][0], c, m)] = "second"
|
|
for r, _ in ranked[1:]:
|
|
p = welch_p(raw[(r, c, m)], raw[(best, c, m)])
|
|
if p is not None and p >= 0.05:
|
|
marks[(r, c, m, "tie")] = True
|
|
ncol = len(columns)
|
|
L = [r"% generated by results.py - do not edit by hand; regenerate instead", r"\begin{table*}[t]" if opt.get("wide") else r"\begin{table}[t]",
|
|
r"\centering", r"\caption{" + opt.get("caption", "TODO caption.") + "}", r"\label{" + opt.get("label", "tab:main") + "}",
|
|
r"\resizebox{\linewidth}{!}{%" if ncol > 4 else "", r"\begin{tabular}{l" + "c" * ncol + "}", r"\toprule"]
|
|
if multi_metric and len(col_vals) > 1:
|
|
L.append(" & " + " & ".join(rf"\multicolumn{{{len(specs)}}}{{c}}{{{tex_escape(disp(c))}}}" for c in col_vals) + r" \\")
|
|
L.append("".join(rf"\cmidrule(lr){{{2 + i * len(specs)}-{1 + (i + 1) * len(specs)}}}" for i in range(len(col_vals))))
|
|
L.append(tex_escape(rows_by.capitalize()) + " & " + " & ".join(
|
|
tex_escape(disp(m)) + " " + arrow(d) for _, m, d in columns) + r" \\")
|
|
elif multi_metric:
|
|
L.append(tex_escape(rows_by.capitalize()) + " & " + " & ".join(
|
|
tex_escape(disp(m)) + " " + arrow(d) for _, m, d in columns) + r" \\")
|
|
else:
|
|
m, d = specs[0]
|
|
L.append(tex_escape(rows_by.capitalize()) + " & " + " & ".join(tex_escape(disp(c)) for c, _, _ in columns) + r" \\")
|
|
L.append(r"\midrule")
|
|
printed = []
|
|
for r in row_vals:
|
|
cells = []
|
|
for c, m, d in columns:
|
|
if (r, c, m) not in cell:
|
|
cells.append("--"); continue
|
|
mean, std, n = cell[(r, c, m)]
|
|
txt = fmt(mean, digits)
|
|
mk = marks.get((r, c, m))
|
|
if mk == "best":
|
|
txt = rf"\textbf{{{txt}}}"
|
|
elif mk == "second":
|
|
txt = rf"\underline{{{txt}}}"
|
|
if marks.get((r, c, m, "tie")):
|
|
txt += r"$^\dagger$"
|
|
if show_std and n > 1:
|
|
txt += rf"\,{{\scriptsize$\pm${fmt(std, digits)}}}"
|
|
cells.append(txt)
|
|
printed.append(f"{r:>18s} | {c:>14s} | {m:>10s} | mean {fmt(mean, digits)} std {fmt(std, digits)} n={n}{' BEST' if mk == 'best' else ''}")
|
|
name = tex_escape(disp(r))
|
|
if r == opt.get("highlight"):
|
|
name = rf"{name} (ours)" if "ours" not in disp(r).lower() else name
|
|
L.append(r"\midrule") if row_vals.index(r) == len(row_vals) - 1 else None
|
|
L.append(name + " & " + " & ".join(cells) + r" \\")
|
|
L += [r"\bottomrule", r"\end{tabular}", "}" if ncol > 4 else "", r"\end{table*}" if opt.get("wide") else r"\end{table}"]
|
|
out = Path(opt.get("out", "paper/tables/main.tex"))
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text("\n".join(l for l in L if l != "") + "\n", encoding="utf-8")
|
|
ns = sorted({v[2] for v in cell.values()})
|
|
any_tie = any(len(k) == 4 for k in marks)
|
|
seeds = str(ns[0]) if len(ns) == 1 else f"{ns[0]}--{ns[-1]}"
|
|
note = "Mean $\\pm$ standard deviation over " + seeds + " seeds"
|
|
if opt.get("scale"):
|
|
note += " (values $\\times$" + str(opt["scale"]) + ")"
|
|
note += ". Best in \\textbf{bold}, second \\underline{underlined}."
|
|
if any_tie:
|
|
note += " $^\\dagger$: not significantly worse than the best (Welch's $t$-test, $p \\geq 0.05$)."
|
|
print("\n".join(printed))
|
|
print(f"\nwrote {out} ({len(row_vals)} rows x {ncol} value columns)")
|
|
print(f"caption note to append (edit caption text in the .tex via --caption):\n {note}")
|
|
if max(ns) < 3:
|
|
print("NOTE: < 3 seeds per cell - no significance marks possible; report this as a limitation.")
|
|
|
|
|
|
# ----------------------------------------------------------------- figures
|
|
def mpl():
|
|
import matplotlib
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt
|
|
plt.rcParams.update({
|
|
"font.family": "serif", "font.serif": ["Times New Roman", "Times", "Nimbus Roman", "DejaVu Serif"],
|
|
"mathtext.fontset": "stix", "font.size": 8, "axes.titlesize": 8, "axes.labelsize": 8,
|
|
"legend.fontsize": 7, "xtick.labelsize": 7, "ytick.labelsize": 7, "axes.linewidth": 0.6,
|
|
"axes.spines.top": False, "axes.spines.right": False, "axes.grid": True, "grid.color": "#e3e3e3",
|
|
"grid.linewidth": 0.5, "lines.linewidth": 1.4, "lines.markersize": 4, "legend.frameon": False,
|
|
"savefig.bbox": "tight", "savefig.pad_inches": 0.02, "pdf.fonttype": 42, "ps.fonttype": 42})
|
|
return plt
|
|
|
|
|
|
def method_styles(methods, highlight=None):
|
|
"""Stable colour/marker per method across all figures (persisted in style.json)."""
|
|
try:
|
|
style = json.loads(STYLE_FILE.read_text())
|
|
except (OSError, json.JSONDecodeError):
|
|
style = {}
|
|
used = len(style)
|
|
order = ([highlight] if highlight in methods and highlight not in style else []) + [m for m in methods if m not in style and m != highlight]
|
|
for m in order:
|
|
i = used
|
|
style[m] = {"color": PALETTE[i % len(PALETTE)], "marker": MARKERS[i % len(MARKERS)],
|
|
"ls": LINESTYLES[(i // len(PALETTE)) % len(LINESTYLES)]}
|
|
used += 1
|
|
STYLE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
STYLE_FILE.write_text(json.dumps(style, indent=1))
|
|
if len(methods) > len(PALETTE):
|
|
print(f"NOTE: {len(methods)} methods > {len(PALETTE)} colours - colours repeat with other line styles/markers; "
|
|
"consider fewer methods per figure (--where) or small multiples.")
|
|
return style
|
|
|
|
|
|
def figsize(opt, ncols=1):
|
|
wopt = str(opt.get("width", ""))
|
|
if re.fullmatch(r"\d+(\.\d+)?", wopt):
|
|
w = float(wopt) # inches = the width the figure gets in the PDF (fonts stay 7-8 pt)
|
|
else:
|
|
w = 6.75 if wopt == "double" or ncols > 2 else 3.25
|
|
h = float(opt.get("height", 2.0 if ncols <= 2 else 1.9))
|
|
return w, h
|
|
|
|
|
|
def facets(df, opt):
|
|
f = opt.get("facet")
|
|
if not f:
|
|
return [(None, df)]
|
|
if f not in df.columns:
|
|
sys.exit(f"--facet {f}: no such column")
|
|
return [(v, df[df[f] == v]) for v in ordered(df[f], csv_list(opt.get("facet-order")))]
|
|
|
|
|
|
def save(fig, opt, used):
|
|
out = Path(opt.get("out", "paper/figures/figure.pdf")).with_suffix(".pdf")
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
fig.savefig(out)
|
|
fig.savefig(out.with_suffix(".png"), dpi=200)
|
|
data = out.with_suffix(".data.csv")
|
|
used.to_csv(data, index=False, float_format="%.6g")
|
|
with pd.option_context("display.max_rows", 60, "display.width", 200):
|
|
print(used.to_string(index=False, float_format=lambda v: f"{v:.4g}")[:3000])
|
|
print(f"\nwrote {out} (+ .png preview for view_image, + {data.name} with the plotted numbers)")
|
|
|
|
|
|
def legend(fig, axes, n):
|
|
h, l = axes[0].get_legend_handles_labels()
|
|
if n >= 2:
|
|
fig.legend(h, l, loc="lower center", bbox_to_anchor=(0.5, 1.0), ncol=min(n, 4 if len(axes) == 1 else 6),
|
|
handlelength=2.2, columnspacing=1.0)
|
|
|
|
|
|
def cmd_figure(pos, opt):
|
|
if len(pos) < 2:
|
|
sys.exit("usage: figure line|bars|scatter results.csv ...")
|
|
kind, path = pos[0], pos[1]
|
|
df = load(path, opt.get("where"))
|
|
scale = float(opt.get("scale", 1))
|
|
plt = mpl()
|
|
methods = ordered(df["method"], csv_list(opt.get("order")), opt.get("highlight"))
|
|
style = method_styles(methods, opt.get("highlight"))
|
|
groups = facets(df, opt)
|
|
fig, axes = plt.subplots(1, len(groups), figsize=figsize(opt, len(groups)), squeeze=False)
|
|
axes = list(axes[0])
|
|
used_rows = []
|
|
if kind == "line":
|
|
metric, x = opt.get("metric"), opt.get("x", "step")
|
|
if not metric or metric not in set(df["metric"]):
|
|
sys.exit(f"--metric required; available: {sorted(df['metric'].unique())}")
|
|
if x not in df.columns:
|
|
sys.exit(f"--x {x}: no such column (columns: {list(df.columns)})")
|
|
for ax, (fv, d) in zip(axes, groups):
|
|
d = d[d["metric"] == metric].dropna(subset=["value", x])
|
|
for m in methods:
|
|
dm = d[d["method"] == m]
|
|
if dm.empty:
|
|
continue
|
|
if x != "step" and "step" in dm.columns: # e.g. x = train_size: use the final value per run
|
|
dm = final_values(dm, opt.get("final", "last"))
|
|
s = dm.groupby(x)["value"].agg(["mean", "std", "count"]).reset_index().sort_values(x)
|
|
s[["mean", "std"]] *= scale
|
|
s["std"] = s["std"].fillna(0)
|
|
st = style[m]
|
|
ax.plot(s[x], s["mean"], color=st["color"], ls=st["ls"], marker=st["marker"],
|
|
markevery=max(1, len(s) // 8), label=disp(m), zorder=3 if m == opt.get("highlight") else 2,
|
|
lw=1.8 if m == opt.get("highlight") else 1.3)
|
|
if (s["count"] > 1).any():
|
|
ax.fill_between(s[x], s["mean"] - s["std"], s["mean"] + s["std"], color=st["color"], alpha=0.15, lw=0)
|
|
used_rows.append(s.assign(method=m, facet=fv))
|
|
ax.set_xlabel(opt.get("xlabel", disp(x).replace("_", " ")))
|
|
if fv is not None:
|
|
ax.set_title(disp(fv))
|
|
axes[0].set_ylabel(opt.get("ylabel", disp(metric).replace("_", " ")))
|
|
elif kind == "bars":
|
|
metric = opt.get("metric")
|
|
if not metric or metric not in set(df["metric"]):
|
|
sys.exit(f"--metric required; available: {sorted(df['metric'].unique())}")
|
|
fin = final_values(df[df["metric"] == metric], opt.get("final", "last"))
|
|
fin["value"] *= scale
|
|
group = opt.get("group", "dataset")
|
|
for ax, (fv, d) in zip(axes, groups if opt.get("facet") else [(None, fin)]):
|
|
if opt.get("facet"):
|
|
d = fin[fin[opt["facet"]] == fv]
|
|
gvals = ordered(d[group], csv_list(opt.get("group-order")))
|
|
ms = [m for m in methods if m in set(d["method"])]
|
|
width = 0.8 / max(1, len(ms))
|
|
for j, m in enumerate(ms):
|
|
s = d[d["method"] == m].groupby(group)["value"].agg(["mean", "std", "count"]).reindex(gvals)
|
|
xs = np.arange(len(gvals)) + (j - (len(ms) - 1) / 2) * width
|
|
st = style[m]
|
|
ax.bar(xs, s["mean"], width * 0.92, yerr=s["std"].fillna(0) if (s["count"] > 1).any() else None,
|
|
color=st["color"], label=disp(m), error_kw={"lw": 0.7, "capsize": 1.5, "ecolor": "#333333"}, zorder=2)
|
|
used_rows.append(s.reset_index().assign(method=m, facet=fv))
|
|
ax.set_xticks(range(len(gvals)))
|
|
ax.set_xticklabels([disp(g) for g in gvals], rotation=0 if len(gvals) <= 4 else 30, ha="center" if len(gvals) <= 4 else "right")
|
|
ax.grid(axis="x", visible=False)
|
|
lo = d["value"].min()
|
|
if lo > 0 and not opt.get("zero"):
|
|
ax.set_ylim(bottom=0)
|
|
if fv is not None:
|
|
ax.set_title(disp(fv))
|
|
axes[0].set_ylabel(opt.get("ylabel", disp(metric).replace("_", " ")))
|
|
elif kind == "scatter":
|
|
xm, ym = opt.get("x-metric"), opt.get("y-metric")
|
|
avail = set(df["metric"])
|
|
if xm not in avail or ym not in avail:
|
|
sys.exit(f"--x-metric/--y-metric must be metrics; available: {sorted(avail)}")
|
|
fin = final_values(df[df["metric"].isin([xm, ym])], opt.get("final", "last"))
|
|
for ax, (fv, _) in zip(axes, groups):
|
|
d = fin if fv is None else fin[fin[opt["facet"]] == fv]
|
|
for m in methods:
|
|
dm = d[d["method"] == m]
|
|
if dm.empty:
|
|
continue
|
|
a = dm[dm["metric"] == xm]["value"]; b = dm[dm["metric"] == ym]["value"] * scale
|
|
st = style[m]
|
|
ax.errorbar(a.mean(), b.mean(), xerr=a.std() if len(a) > 1 else None, yerr=b.std() if len(b) > 1 else None,
|
|
fmt=st["marker"], color=st["color"], ms=5, elinewidth=0.7, capsize=1.5, label=disp(m),
|
|
mec="white", mew=0.6, zorder=3)
|
|
used_rows.append(pd.DataFrame([{"method": m, "facet": fv, xm: a.mean(), ym: b.mean(),
|
|
f"{xm}_std": a.std(), f"{ym}_std": b.std(), "n": len(b)}]))
|
|
ax.set_xlabel(opt.get("xlabel", disp(xm).replace("_", " ")))
|
|
if fv is not None:
|
|
ax.set_title(disp(fv))
|
|
axes[0].set_ylabel(opt.get("ylabel", disp(ym).replace("_", " ")))
|
|
else:
|
|
sys.exit("figure kinds: line, bars, scatter")
|
|
for ax in axes:
|
|
if opt.get("logx"): ax.set_xscale("log")
|
|
if opt.get("logy"): ax.set_yscale("log")
|
|
if opt.get("ylim"): ax.set_ylim(*[float(v) for v in opt["ylim"].split(",")])
|
|
if opt.get("xlim"): ax.set_xlim(*[float(v) for v in opt["xlim"].split(",")])
|
|
shown = len({r["method"].iloc[0] for r in used_rows if len(r)})
|
|
legend(fig, axes, shown)
|
|
fig.tight_layout()
|
|
save(fig, opt, pd.concat(used_rows, ignore_index=True) if used_rows else pd.DataFrame())
|
|
|
|
|
|
def main(argv):
|
|
if not argv or argv[0] in ("-h", "--help"):
|
|
print(__doc__); return 0
|
|
cmd, (pos, opt) = argv[0], parse(argv[1:])
|
|
RENAME.update(dict(kv.split("=", 1) for kv in csv_list(opt.get("rename")) if "=" in kv))
|
|
fn = {"melt": cmd_melt, "tb": cmd_tb, "check": cmd_check, "summary": cmd_summary,
|
|
"table": cmd_table, "figure": cmd_figure}.get(cmd)
|
|
if not fn or (not pos and cmd != "figure"):
|
|
print(__doc__); return 2
|
|
fn(pos, opt)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|