From e330f40d795f73893284abc9bf00904961b4a42f Mon Sep 17 00:00:00 2001 From: VasiliyMatr Date: Sun, 22 Feb 2026 23:13:43 +0300 Subject: [PATCH 1/6] [tools] Add script for stats plotting --- tools/bench/Plot.py | 465 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 465 insertions(+) create mode 100644 tools/bench/Plot.py diff --git a/tools/bench/Plot.py b/tools/bench/Plot.py new file mode 100644 index 0000000..22113cc --- /dev/null +++ b/tools/bench/Plot.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +""" +Plot.py — Visualize benchmark metrics data collected into CSV files + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +REGIME: bars +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Compare geomean metrics values across series of experiments for a fixed set of benchmarks. + + CSV data layout: + / <- one directory per series (compiler, config, ...) + .csv <- one CSV per benchmark; rows = repeated runs + + For each CSV, the bar height = geometric mean of the metric column over + all rows. Error bars show standard deviation. Bars for the same + benchmark are grouped side-by-side, one bar per series. + + Output: one SVG per metric, named .svg in output directory + + Example: + python3 Plot.py --regime bars \ + --data llvm/ asmjit/ lightning/ \ + --metrics instructions cycles ipc \ + --out plots/ + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +REGIME: graphs +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Show how dependent metrics scale as a base metric grows (workload scaling). + + CSV data layout: + / <- one directory per series + / <- one sub-directory per benchmark + .csv <- one CSV per workload size; rows = repeated runs + + Each CSV produces one point on the line: + X = geometric mean of --base-metric across all rows + Y = geometric mean of the dependent metric across all rows + + Points are sorted by X and connected with a line. A std shaded band + is drawn around each line. Each output SVG contains one subplot per + benchmark, tiled up to 3 columns wide with a shared legend at the top. + + Output: one SVG per dependent metric, named _vs_.svg in output + directory + + Example: + python plot_metrics.py --regime graphs \ + --data llvm/ asmjit/ lightning/ \ + --base-metric instructions \ + --metrics cycles ipc l1i-cache-misses \ + --out plots/ +""" + +import argparse +import sys +from pathlib import Path + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.ticker as ticker + + +# ── shared helpers ──────────────────────────────────────────────────────────── + +def geomean(values: np.ndarray) -> float: + """Geometric mean, ignoring non-positive and non-finite values.""" + v = values[np.isfinite(values) & (values > 0)] + return float(np.exp(np.mean(np.log(v)))) if len(v) > 0 else float("nan") + + +def auto_scale(max_val: float) -> tuple[int, float]: + """ + Return (exp3, divisor) for a human-readable axis scale. + exp3 is the largest multiple of 3 such that max_val / 10^exp3 >= 1, + clamped to [0, 12] (covers 1 … 1E12). + """ + if not np.isfinite(max_val) or max_val <= 0: + return 0, 1.0 + exp3 = int(np.floor(np.log10(max_val) / 3) * 3) + exp3 = max(0, min(exp3, 12)) + return exp3, float(10 ** exp3) + + +def axis_label(metric: str, exp3: int) -> str: + return f"{metric} [1E{exp3}]" if exp3 > 0 else metric + + +# Muted/earthy palette — desaturated, professional +_PALETTE = [ + "#5B7FA6", # muted steel blue + "#A0674B", # terracotta + "#5A8C6E", # sage green + "#8C7A4B", # warm tan/khaki + "#7B6A8C", # dusty mauve + "#4B7A7A", # muted teal + "#8C5A5A", # dusty rose/brick +] + +def palette(n: int) -> list[str]: + return (_PALETTE * ((n // len(_PALETTE)) + 1))[:n] + + +# ── bars regime ─────────────────────────────────────────────────────────────── + +def bars_load(data_dirs: list[str]) -> dict[str, dict[str, pd.DataFrame]]: + """series -> benchmark -> DataFrame (rows = runs).""" + result: dict[str, dict[str, pd.DataFrame]] = {} + for d in data_dirs: + p = Path(d) + if not p.is_dir(): + print(f"Warning: '{d}' is not a directory, skipping.", file=sys.stderr) + continue + benches: dict[str, pd.DataFrame] = {} + for csv in sorted(p.glob("*.csv")): + try: + benches[csv.stem] = pd.read_csv(csv) + except Exception as e: + print(f"Warning: cannot read '{csv}': {e}", file=sys.stderr) + if benches: + result[p.name] = benches + else: + print(f"Warning: no valid CSV files found in '{d}'.", file=sys.stderr) + return result + + +def bars_aggregate( + series_data: dict[str, dict[str, pd.DataFrame]], + metric: str, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Return (geomeans_df, stdev_df) with index=benchmark, columns=series.""" + benches: set[str] = set() + for bmap in series_data.values(): + for b, df in bmap.items(): + if metric in df.columns: + benches.add(b) + if not benches: + return pd.DataFrame(), pd.DataFrame() + + series_names = list(series_data.keys()) + means_rows, std_rows = [], [] + for bench in sorted(benches): + mr, sr = {"benchmark": bench}, {"benchmark": bench} + for s in series_names: + df = series_data.get(s, {}).get(bench) + if df is not None and metric in df.columns: + v = df[metric].to_numpy(dtype=float) + mr[s] = geomean(v) + sr[s] = float(np.std(v[np.isfinite(v)], ddof=1)) if v.size > 1 else 0.0 + else: + mr[s] = sr[s] = float("nan") + means_rows.append(mr) + std_rows.append(sr) + + return (pd.DataFrame(means_rows).set_index("benchmark"), + pd.DataFrame(std_rows).set_index("benchmark")) + + +def bars_plot( + means: pd.DataFrame, + stdev: pd.DataFrame, + metric: str, + out_dir: Path, +) -> None: + series = means.columns.tolist() + benchmarks = means.index.tolist() + n_s, n_g = len(series), len(benchmarks) + if n_g == 0 or n_s == 0: + print(f" No data to plot for '{metric}', skipping.") + return + + all_vals = means.to_numpy(dtype=float) + max_val = float(np.nanmax(all_vals)) if all_vals.size else 1.0 + exp3, divisor = auto_scale(max_val if np.isfinite(max_val) else 1.0) + + group_width = 0.8 + bar_width = group_width / n_s + x = np.arange(n_g) + colors = palette(n_s) + + fig, ax = plt.subplots(figsize=(max(6, n_g * 1.4 + 2), 5)) + + for i, (s, color) in enumerate(zip(series, colors)): + offsets = x - group_width / 2 + bar_width * (i + 0.5) + vals = means[s].to_numpy(dtype=float) + errs = stdev[s].to_numpy(dtype=float) / divisor if s in stdev.columns else None + bars = ax.bar( + offsets, vals / divisor, width=bar_width * 0.9, + label=s, color=color, zorder=3, + yerr=errs, capsize=3, + error_kw=dict(elinewidth=1, ecolor="black", alpha=0.6, capthick=1), + ) + for rect, v in zip(bars, vals): + if np.isnan(v): + ax.text( + rect.get_x() + rect.get_width() / 2, 0.01, "N/A", + ha="center", va="bottom", fontsize=7, color="grey", + transform=ax.get_xaxis_transform(), + ) + + ax.set_xticks(x) + ax.set_xticklabels(benchmarks, rotation=15, ha="right", fontsize=9) + ax.set_ylabel(axis_label(metric, exp3), fontsize=10) + ax.set_title(metric, fontsize=12, fontweight="bold") + ax.legend(fontsize=9, framealpha=0.7) + ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) + ax.grid(axis="y", linestyle="--", alpha=0.4, zorder=0) + ax.set_axisbelow(True) + ax.spines[["top", "right"]].set_visible(False) + + plt.tight_layout() + out_path = out_dir / f"{metric}.svg" + fig.savefig(out_path, format="svg", bbox_inches="tight") + plt.close(fig) + print(f" Saved: {out_path}") + + +def run_bars(args: argparse.Namespace, out_dir: Path) -> None: + print("Loading data (bars regime)...") + series_data = bars_load(args.data) + if not series_data: + sys.exit("Error: no valid data directories found.") + print(f"Series: {list(series_data.keys())}") + + for metric in args.metrics: + print(f"\nPlotting: {metric}") + means, stdev = bars_aggregate(series_data, metric) + if means.empty: + print(f" No data for '{metric}', skipping.") + continue + bars_plot(means, stdev, metric, out_dir) + + +# ── graphs regime ───────────────────────────────────────────────────────────── + +# series -> benchmark -> [(x_mean, y_mean, y_std), ...] sorted by x +GraphData = dict[str, dict[str, list[tuple[float, float, float]]]] + + +def graphs_load( + data_dirs: list[str], + base_metric: str, + dep_metric: str, +) -> GraphData: + result: GraphData = {} + for d in data_dirs: + p = Path(d) + if not p.is_dir(): + print(f"Warning: '{d}' is not a directory, skipping.", file=sys.stderr) + continue + bench_map: dict[str, list[tuple[float, float, float]]] = {} + for bench_dir in sorted(p.iterdir()): + if not bench_dir.is_dir(): + continue + points: list[tuple[float, float, float]] = [] + for csv in sorted(bench_dir.glob("*.csv")): + try: + df = pd.read_csv(csv) + except Exception as e: + print(f"Warning: cannot read '{csv}': {e}", file=sys.stderr) + continue + if base_metric not in df.columns: + print(f"Warning: base metric '{base_metric}' not in '{csv}', skipping.", file=sys.stderr) + continue + if dep_metric not in df.columns: + print(f"Warning: dep metric '{dep_metric}' not in '{csv}', skipping.", file=sys.stderr) + continue + x = geomean(df[base_metric].to_numpy(dtype=float)) + yv = df[dep_metric].to_numpy(dtype=float) + yv = yv[np.isfinite(yv) & (yv > 0)] + if not np.isfinite(x) or len(yv) == 0: + continue + points.append(( + x, + geomean(yv), + float(np.std(yv, ddof=1)) if len(yv) > 1 else 0.0, + )) + if points: + bench_map[bench_dir.name] = sorted(points, key=lambda t: t[0]) + if bench_map: + result[p.name] = bench_map + return result + + +def graphs_plot( + series_data: GraphData, + base_metric: str, + dep_metric: str, + out_dir: Path, +) -> None: + all_benches: set[str] = set() + for bmap in series_data.values(): + all_benches.update(bmap.keys()) + # Should be non-empty for valid series_data + assert(all_benches) + + benches = sorted(all_benches) + series_names = list(series_data.keys()) + colors = palette(len(series_names)) + + all_x = [p[0] for bmap in series_data.values() for pts in bmap.values() for p in pts] + all_y = [p[1] for bmap in series_data.values() for pts in bmap.values() for p in pts] + x_exp3, x_div = auto_scale(max(all_x, default=1.0)) + y_exp3, y_div = auto_scale(max(all_y, default=1.0)) + + n_benches = len(benches) + ncols = min(3, n_benches) + nrows = int(np.ceil(n_benches / ncols)) + fig, axes = plt.subplots( + nrows, ncols, + figsize=(max(5 * ncols, 7), 4 * nrows + 1), + squeeze=False, + ) + + legend_handles, legend_labels = [], [] + + for idx, bench in enumerate(benches): + row, col = divmod(idx, ncols) + ax = axes[row][col] + + for s_idx, (series, color) in enumerate(zip(series_names, colors)): + pts = series_data.get(series, {}).get(bench) + if not pts: + continue + xs = np.array([p[0] for p in pts]) / x_div + ys = np.array([p[1] for p in pts]) / y_div + std = np.array([p[2] for p in pts]) / y_div + + line, = ax.plot( + xs, ys, marker="o", color=color, + linewidth=1.8, markersize=4, label=series, zorder=3, + ) + ax.fill_between(xs, ys - std, ys + std, + color=color, alpha=0.15, zorder=2) + if idx == 0: + legend_handles.append(line) + legend_labels.append(series) + + ax.set_title(bench, fontsize=10, fontweight="bold") + ax.set_xlabel(axis_label(base_metric, x_exp3), fontsize=8) + ax.set_ylabel(axis_label(dep_metric, y_exp3), fontsize=8) + ax.grid(True, linestyle="--", alpha=0.4, zorder=0) + ax.set_axisbelow(True) + ax.spines[["top", "right"]].set_visible(False) + ax.tick_params(labelsize=8) + ax.xaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) + ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) + + for idx in range(n_benches, nrows * ncols): + r, c = divmod(idx, ncols) + axes[r][c].set_visible(False) + + fig.legend( + legend_handles, legend_labels, + loc="upper center", ncol=min(len(series_names), 5), + fontsize=9, framealpha=0.8, bbox_to_anchor=(0.5, 1.0), + ) + fig.suptitle(f"{dep_metric} vs {base_metric}", + fontsize=13, fontweight="bold", y=1.03) + + plt.tight_layout() + out_path = out_dir / f"{dep_metric}_vs_{base_metric}.svg" + fig.savefig(out_path, format="svg", bbox_inches="tight") + plt.close(fig) + print(f" Saved: {out_path}") + + +def run_graphs(args: argparse.Namespace, out_dir: Path) -> None: + if not args.base_metric: + sys.exit("Error: --base-metric is required for --regime graphs.") + print("Loading data (graphs regime)...") + + for dep_metric in args.metrics: + print(f"\nPlotting: {dep_metric} vs {args.base_metric}") + series_data = graphs_load(args.data, args.base_metric, dep_metric) + if not series_data: + print(" No valid data found, skipping.") + continue + graphs_plot(series_data, args.base_metric, dep_metric, out_dir) + + +# ── CLI ─────────────────────────────────────────────────────────────────────── + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + p.add_argument( + "--regime", required=True, choices=["bars", "graphs"], + help=( + "Plotting regime (required).\n" + "\n" + " bars — Grouped bar chart comparing absolute metric values\n" + " across series for a fixed set of benchmarks.\n" + " Expected layout inside each --data dir:\n" + " /.csv\n" + " Bar height = geomean over repeated runs (CSV rows).\n" + " Error bars = +/-1 standard deviation.\n" + " Bars for the same benchmark are placed side-by-side.\n" + " Output: one SVG per metric, named .svg\n" + "\n" + " graphs — Line chart showing how a dependent metric scales\n" + " with a base metric as workload size grows.\n" + " Expected layout inside each --data dir:\n" + " //.csv\n" + " X = geomean of --base-metric across all rows of a CSV.\n" + " Y = geomean of the dependent metric across same rows.\n" + " Points sorted by X and connected with a line;\n" + " shaded band shows +/-1 std around each line.\n" + " One subplot per benchmark, tiled in the same figure.\n" + " Output: one SVG per metric, named _vs_.svg\n" + ), + ) + p.add_argument( + "--data", nargs="+", required=True, metavar="DIR", + help=( + "One or more series directories. The directory name is used as\n" + "the legend label. See --regime for the expected layout inside\n" + "each directory." + ), + ) + p.add_argument( + "--metrics", nargs="+", required=True, metavar="METRIC", + help=( + "One or more CSV column names to plot.\n" + "bars regime : these are the Y-axis metrics, one chart each.\n" + "graphs regime : these are the dependent (Y-axis) metrics;\n" + " the X axis is controlled by --base-metric." + ), + ) + p.add_argument( + "--base-metric", metavar="METRIC", default=None, + help=( + "Column name to use as the X axis in 'graphs' regime.\n" + "Its geomean across all rows of a CSV represents the workload\n" + "size for that data point. Required for --regime graphs,\n" + "ignored for --regime bars." + ), + ) + p.add_argument( + "--out", default="plots", metavar="DIR", + help="Output directory for SVG files (default: ./plots).", + ) + + return p.parse_args() + + +def main() -> None: + args = parse_args() + out_dir = Path(args.out) + out_dir.mkdir(parents=True, exist_ok=True) + + if args.regime == "bars": + run_bars(args, out_dir) + else: + run_graphs(args, out_dir) + + print("\nDone.") + + +if __name__ == "__main__": + main() From d7f4505597e9826c3f34fae0e8224a1616485145 Mon Sep 17 00:00:00 2001 From: VasiliyMatr Date: Mon, 23 Feb 2026 13:34:35 +0300 Subject: [PATCH 2/6] [tools] Improve axis scaling in Plot tool --- tools/bench/CLAUDE_Plot.md | 114 +++++++++++++++++++++++++++++++++++++ tools/bench/Plot.py | 82 +++++++++++++------------- 2 files changed, 157 insertions(+), 39 deletions(-) create mode 100644 tools/bench/CLAUDE_Plot.md diff --git a/tools/bench/CLAUDE_Plot.md b/tools/bench/CLAUDE_Plot.md new file mode 100644 index 0000000..f933ea0 --- /dev/null +++ b/tools/bench/CLAUDE_Plot.md @@ -0,0 +1,114 @@ +# CLAUDE_Plot.md — Benchmark Metrics Plotter + +## Project overview + +`Plot.py` — a single Python script for visu alising benchmark performance +data collected across multiple series (e.g. JIT compilers, build configs). +Produces SVG charts. Two regimes selectable via `--regime`. + +## Usage + +```bash +# Bars: compare absolute metric values across series +python3 Plot.py \ + --regime bars \ + --data results/llvm results/asmjit results/gcc \ + --metrics cycles ipc l1i-cache-misses \ + --out plots/ + +# Graphs: show how metrics scale with a base metric (workload scaling) +python3 Plot.py \ + --regime graphs \ + --data results/llvm results/asmjit results/gcc \ + --base-metric instructions \ + --metrics cycles ipc \ + --out plots/ +``` + +## CLI arguments + +| Argument | Required | Description | +|---|---|---| +| `--regime` | yes | `bars` or `graphs` | +| `--data` | yes | One or more series directories; dir name = legend label | +| `--metrics` | yes | CSV column names to plot | +| `--base-metric` | graphs only | Column used as X axis (workload proxy) | +| `--out` | no | Output directory for SVGs (default: `./plots`) | +| `--logx` | no | Logarithmic X axis (graphs regime only) | +| `--logy` | no | Logarithmic Y axis (both regimes) | + +## Data layouts + +### `bars` regime + +``` +/ + .csv ← rows = independent repeated runs +``` + +- Benchmarks matched across series by filename (without extension) +- One SVG per metric, named `.svg` + +### `graphs` regime + +``` +/ + / + .csv ← rows = independent repeated runs at that size +``` + +- Benchmarks matched across series by sub-directory name +- One SVG per dependent metric, named `_vs_.svg` + +## CSV format + +Plain CSV with a header row. Column names are metric names. Each data row is +one independent run. Example: + +``` +instructions,cycles,branches,l1i-cache-misses +100,50,10,1 +110,50,11,2 +``` + +## How values are computed + +- **Bar height / line Y point** — geometric mean of all rows in a CSV file +- **Error bars / shaded band** — ±1 standard deviation across rows +- **Line X point** (`graphs` only) — geometric mean of `--base-metric` in that CSV + +Geomean is used rather than arithmetic mean because it is more appropriate for +ratios and hardware counters that span orders of magnitude. Non-positive and +non-finite values are excluded before computing geomean (required since +`log(x)` is undefined for `x ≤ 0`). + +## Axis formatting + +On linear axes, matplotlib's `ScalarFormatter` is used with `useMathText=True` +and `powerlimits=(-3, 3)`, which produces an automatic `×10^N` offset label +when values are large or small. The axis label shows only the bare metric name. + +On log axes (`--logx` / `--logy`), matplotlib's default `LogFormatter` is used, +showing ticks in `10^x` notation. `nonpositive="clip"` is passed to +`set_xscale`/`set_yscale` so that bars starting from 0 and any negative +error-bar lower bounds are silently clipped to a small positive value rather +than crashing or producing invisible elements. + +## Dependencies + +```bash +pip install numpy pandas matplotlib +``` + +## Known issues / notes + +- In `graphs_plot`, the emptiness check on `all_benches` is now an `assert` + rather than a soft print-and-return, reflecting that an empty `series_data` + should be caught upstream in `run_graphs` and never reach this point. +- `--base-metric` is silently ignored when `--regime bars` is used. +- Series with no valid CSV files produce a warning and are skipped rather than + causing a hard error. +- In `graphs` regime, CSV files missing the dependent metric column produce a + warning and are skipped (the base metric missing is also warned). +- Missing benchmark/series combinations in `bars` regime show an "N/A" + annotation on the bar rather than crashing. diff --git a/tools/bench/Plot.py b/tools/bench/Plot.py index 22113cc..41084f3 100644 --- a/tools/bench/Plot.py +++ b/tools/bench/Plot.py @@ -70,23 +70,6 @@ def geomean(values: np.ndarray) -> float: return float(np.exp(np.mean(np.log(v)))) if len(v) > 0 else float("nan") -def auto_scale(max_val: float) -> tuple[int, float]: - """ - Return (exp3, divisor) for a human-readable axis scale. - exp3 is the largest multiple of 3 such that max_val / 10^exp3 >= 1, - clamped to [0, 12] (covers 1 … 1E12). - """ - if not np.isfinite(max_val) or max_val <= 0: - return 0, 1.0 - exp3 = int(np.floor(np.log10(max_val) / 3) * 3) - exp3 = max(0, min(exp3, 12)) - return exp3, float(10 ** exp3) - - -def axis_label(metric: str, exp3: int) -> str: - return f"{metric} [1E{exp3}]" if exp3 > 0 else metric - - # Muted/earthy palette — desaturated, professional _PALETTE = [ "#5B7FA6", # muted steel blue @@ -162,6 +145,7 @@ def bars_plot( stdev: pd.DataFrame, metric: str, out_dir: Path, + logy: bool, ) -> None: series = means.columns.tolist() benchmarks = means.index.tolist() @@ -170,10 +154,6 @@ def bars_plot( print(f" No data to plot for '{metric}', skipping.") return - all_vals = means.to_numpy(dtype=float) - max_val = float(np.nanmax(all_vals)) if all_vals.size else 1.0 - exp3, divisor = auto_scale(max_val if np.isfinite(max_val) else 1.0) - group_width = 0.8 bar_width = group_width / n_s x = np.arange(n_g) @@ -184,9 +164,9 @@ def bars_plot( for i, (s, color) in enumerate(zip(series, colors)): offsets = x - group_width / 2 + bar_width * (i + 0.5) vals = means[s].to_numpy(dtype=float) - errs = stdev[s].to_numpy(dtype=float) / divisor if s in stdev.columns else None + errs = stdev[s].to_numpy(dtype=float) if s in stdev.columns else None bars = ax.bar( - offsets, vals / divisor, width=bar_width * 0.9, + offsets, vals, width=bar_width * 0.9, label=s, color=color, zorder=3, yerr=errs, capsize=3, error_kw=dict(elinewidth=1, ecolor="black", alpha=0.6, capthick=1), @@ -201,10 +181,16 @@ def bars_plot( ax.set_xticks(x) ax.set_xticklabels(benchmarks, rotation=15, ha="right", fontsize=9) - ax.set_ylabel(axis_label(metric, exp3), fontsize=10) + ax.set_ylabel(metric, fontsize=10) ax.set_title(metric, fontsize=12, fontweight="bold") ax.legend(fontsize=9, framealpha=0.7) - ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) + if logy: + ax.set_yscale("log", nonpositive="clip") + else: + fmt = ticker.ScalarFormatter(useMathText=True) + fmt.set_scientific(True) + fmt.set_powerlimits((-3, 3)) + ax.yaxis.set_major_formatter(fmt) ax.grid(axis="y", linestyle="--", alpha=0.4, zorder=0) ax.set_axisbelow(True) ax.spines[["top", "right"]].set_visible(False) @@ -229,7 +215,7 @@ def run_bars(args: argparse.Namespace, out_dir: Path) -> None: if means.empty: print(f" No data for '{metric}', skipping.") continue - bars_plot(means, stdev, metric, out_dir) + bars_plot(means, stdev, metric, out_dir, logy=args.logy) # ── graphs regime ───────────────────────────────────────────────────────────── @@ -288,6 +274,8 @@ def graphs_plot( base_metric: str, dep_metric: str, out_dir: Path, + logx: bool, + logy: bool, ) -> None: all_benches: set[str] = set() for bmap in series_data.values(): @@ -299,11 +287,6 @@ def graphs_plot( series_names = list(series_data.keys()) colors = palette(len(series_names)) - all_x = [p[0] for bmap in series_data.values() for pts in bmap.values() for p in pts] - all_y = [p[1] for bmap in series_data.values() for pts in bmap.values() for p in pts] - x_exp3, x_div = auto_scale(max(all_x, default=1.0)) - y_exp3, y_div = auto_scale(max(all_y, default=1.0)) - n_benches = len(benches) ncols = min(3, n_benches) nrows = int(np.ceil(n_benches / ncols)) @@ -323,9 +306,9 @@ def graphs_plot( pts = series_data.get(series, {}).get(bench) if not pts: continue - xs = np.array([p[0] for p in pts]) / x_div - ys = np.array([p[1] for p in pts]) / y_div - std = np.array([p[2] for p in pts]) / y_div + xs = np.array([p[0] for p in pts]) + ys = np.array([p[1] for p in pts]) + std = np.array([p[2] for p in pts]) line, = ax.plot( xs, ys, marker="o", color=color, @@ -338,14 +321,26 @@ def graphs_plot( legend_labels.append(series) ax.set_title(bench, fontsize=10, fontweight="bold") - ax.set_xlabel(axis_label(base_metric, x_exp3), fontsize=8) - ax.set_ylabel(axis_label(dep_metric, y_exp3), fontsize=8) + ax.set_xlabel(base_metric, fontsize=8) + ax.set_ylabel(dep_metric, fontsize=8) + if logx: + ax.set_xscale("log", nonpositive="clip") + else: + x_fmt = ticker.ScalarFormatter(useMathText=True) + x_fmt.set_scientific(True) + x_fmt.set_powerlimits((-3, 3)) + ax.xaxis.set_major_formatter(x_fmt) + if logy: + ax.set_yscale("log", nonpositive="clip") + else: + y_fmt = ticker.ScalarFormatter(useMathText=True) + y_fmt.set_scientific(True) + y_fmt.set_powerlimits((-3, 3)) + ax.yaxis.set_major_formatter(y_fmt) ax.grid(True, linestyle="--", alpha=0.4, zorder=0) ax.set_axisbelow(True) ax.spines[["top", "right"]].set_visible(False) ax.tick_params(labelsize=8) - ax.xaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) - ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda v, _: f"{v:g}")) for idx in range(n_benches, nrows * ncols): r, c = divmod(idx, ncols) @@ -377,7 +372,8 @@ def run_graphs(args: argparse.Namespace, out_dir: Path) -> None: if not series_data: print(" No valid data found, skipping.") continue - graphs_plot(series_data, args.base_metric, dep_metric, out_dir) + graphs_plot(series_data, args.base_metric, dep_metric, out_dir, + logx=args.logx, logy=args.logy) # ── CLI ─────────────────────────────────────────────────────────────────────── @@ -444,6 +440,14 @@ def parse_args() -> argparse.Namespace: "--out", default="plots", metavar="DIR", help="Output directory for SVG files (default: ./plots).", ) + p.add_argument( + "--logx", action="store_true", + help="Use logarithmic scale for the X axis (graphs regime only).", + ) + p.add_argument( + "--logy", action="store_true", + help="Use logarithmic scale for the Y axis.", + ) return p.parse_args() From a49ccda72d6b7114c77f9558a601dbcb254df89b Mon Sep 17 00:00:00 2001 From: VasiliyMatr Date: Mon, 23 Feb 2026 16:55:49 +0300 Subject: [PATCH 3/6] [third_party] Add perf-cpp Also setup RPATH to link with dependencies shared libraries --- CMakeLists.txt | 4 ++++ cmake/dependencies.cmake | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1553b0e..4788f40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,10 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) PROPERTY VALUE "${CMAKE_BINARY_DIR}/install") endif() +# Add rpath to link with dependencies shared libraries +set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) +set(CMAKE_INSTALL_RPATH "\${ORIGIN}/../lib") + option(PROT_ENABLE_WERROR "Enable -Werror option (CI)" OFF) option(PROT_BUILD_BENCHMARKS "Enable benchmarks build (requires riscv gnu toolchain)" OFF) diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 3c2e89f..72cc85b 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -64,3 +64,16 @@ CPMAddPackage( OPTIONS "BUILD_TESTING OFF" DOWNLOAD_ONLY True ) + +# perf-cpp +CPMAddPackage( + NAME perf-cpp + GITHUB_REPOSITORY jmuehlig/perf-cpp + GIT_TAG v0.12.6 + # Don't add EXCLUDE_FROM_ALL, as perf-cpp will be used as shared library + SYSTEM True + OPTIONS + "BUILD_LIB_SHARED ON" + "CMAKE_INSTALL_LIBRARY_DIR ${CMAKE_INSTALL_PREFIX}/lib" +) +target_include_directories(perf-cpp PUBLIC $) From a3edb22255dd221f8bc051d1f92c3bb6f2815be2 Mon Sep 17 00:00:00 2001 From: VasiliyMatr Date: Mon, 23 Feb 2026 17:03:37 +0300 Subject: [PATCH 4/6] [psim] Add support for basic metrics collection Use perf-cpp for perf counters collection --- tools/sim/CMakeLists.txt | 1 + tools/sim/sim_app.cpp | 167 +++++++++++++++++++++++++++++++++------ 2 files changed, 146 insertions(+), 22 deletions(-) diff --git a/tools/sim/CMakeLists.txt b/tools/sim/CMakeLists.txt index ed73fad..0d7b0ae 100644 --- a/tools/sim/CMakeLists.txt +++ b/tools/sim/CMakeLists.txt @@ -2,6 +2,7 @@ add_executable(prot_sim_app sim_app.cpp) target_link_libraries( prot_sim_app PRIVATE CLI11::CLI11 + perf-cpp PROT::defaults PROT::interpreter PROT::JIT::factory diff --git a/tools/sim/sim_app.cpp b/tools/sim/sim_app.cpp index 2507361..a390bbb 100644 --- a/tools/sim/sim_app.cpp +++ b/tools/sim/sim_app.cpp @@ -1,10 +1,22 @@ #include -#include +#include #include +#include +#include +#include +#include +#include +#include +#include #include +#include #include +#include + +#include +#include #include "prot/elf_loader.hh" #include "prot/hart.hh" @@ -12,12 +24,63 @@ #include "prot/jit/factory.hh" #include "prot/memory.hh" -int main(int argc, const char *argv[]) try { +namespace { + +struct RunConfig { + const std::filesystem::path &elfPath; + prot::isa::Addr stackTop = 0x7fffffff; + const std::string &jitBackend; + const std::vector &perfEvents; + bool dumpHart = false; +}; + +struct RunResult { + uint32_t status = 0; + uint64_t guestIc = 0; + perf::CounterResult perfRes; +}; + +RunResult run(const RunConfig &cfg) { + auto hart = [&] { + prot::ElfLoader loader{cfg.elfPath}; + + std::unique_ptr engine = + !cfg.jitBackend.empty() + ? prot::engine::JitFactory::createEngine(cfg.jitBackend) + : std::make_unique(); + + prot::Hart hart{prot::memory::makePlain(4ULL << 30U), std::move(engine)}; + hart.load(loader); + hart.setSP(cfg.stackTop); + + return hart; + }(); + + auto eventCounter = perf::EventCounter{}; + eventCounter.add(cfg.perfEvents); + + eventCounter.start(); + hart.run(); + eventCounter.stop(); + + if (cfg.dumpHart) { + hart.dump(std::cout); + } + + return RunResult{.status = hart.getExitCode(), + .guestIc = hart.getIcount(), + .perfRes = eventCounter.result()}; +} + +} // namespace +int main(int argc, const char *argv[]) try { std::filesystem::path elfPath; constexpr prot::isa::Addr kDefaultStack = 0x7fffffff; prot::isa::Addr stackTop{}; std::string jitBackend{}; + std::filesystem::path statsCsvPath; + bool dumpHart = false; { CLI::App app{"App for JIT research from ProteusLab team"}; @@ -33,33 +96,93 @@ int main(int argc, const char *argv[]) try { app.add_option("--jit", jitBackend, "Use JIT & set backend") ->check(CLI::IsMember(prot::engine::JitFactory::backends())); + app.add_option( + "--stats", statsCsvPath, + "Path to CSV stats file. Enables stats collection regime if set"); + + app.add_flag("--dump", dumpHart, "Dump hart values on each run"); + CLI11_PARSE(app, argc, argv); } - auto hart = [&] { - prot::ElfLoader loader{elfPath}; + bool collectStats = !statsCsvPath.empty(); - std::unique_ptr engine = - !jitBackend.empty() ? prot::engine::JitFactory::createEngine(jitBackend) - : std::make_unique(); + // Group related counters together + // clang-format off + std::vector> counters { + { + "seconds", + "instructions", + "cycles", + "branches", + "branch-misses", + }, + { + "L1-dcache-loads", + "L1-dcache-load-misses", + "L1-icache-loads", + "L1-icache-load-misses", + "iTLB-load-misses", + "dTLB-load-misses", + }, + }; + // clang-format on - prot::Hart hart{prot::memory::makePlain(4ULL << 30U), std::move(engine)}; - hart.load(loader); - hart.setSP(stackTop); + std::ofstream statsCsv{}; + if (collectStats) { + std::filesystem::create_directories(statsCsvPath.parent_path()); - return hart; - }(); + bool addHeader = !std::filesystem::exists(statsCsvPath); + statsCsv.open(statsCsvPath, std::ios_base::app | std::ios_base::out); - auto start = std::chrono::high_resolution_clock::now(); - hart.run(); - auto end = std::chrono::high_resolution_clock::now(); - hart.dump(std::cout); - std::chrono::duration duration = end - start; - fmt::println("icount: {}", hart.getIcount()); - fmt::println("time: {}s", duration.count()); - fmt::println("threshold: {}", 0); - fmt::println("mips: {}", hart.getIcount() / (duration.count() * 1000000)); - return hart.getExitCode(); + if (addHeader) { + fmt::println(statsCsv, "{}", fmt::join(counters | std::views::join, ",")); + } + } + + auto runNum = 0; + for (const auto &runCounters : + counters | std::views::take(collectStats ? 0xffU : 1U)) { + fmt::println("Run #{}", runNum++); + + auto res = run({ + .elfPath = elfPath, + .stackTop = stackTop, + .jitBackend = jitBackend, + .perfEvents = runCounters, + .dumpHart = dumpHart, + }); + + auto seconds = res.perfRes["seconds"]; + auto cycles = res.perfRes["cycles"]; + auto hostInstrs = res.perfRes["instructions"]; + + if (seconds) { + fmt::println(" time: {:.2f}s", *seconds); + } + + if (seconds && cycles && hostInstrs) { + auto guestIPS = res.guestIc / *seconds; + fmt::println(" MIPS: {:.2f}", guestIPS / 1'000'000); + } + + if (collectStats) { + fmt::print(statsCsv, "{}", + fmt::join(res.perfRes | std::views::values, ",")); + } + + fmt::println(" status = {}", res.status); + if (res.status != 0) { + fmt::println("Run failed, aborting"); + return res.status; + } + } + + if (collectStats) { + statsCsv << "\n"; + } + + return 0; } catch (const std::exception &ex) { fmt::println(std::cerr, "Caught an exception of type {}, message: {}", typeid(ex).name(), ex.what()); From 6426ad8bdfded8ffae375c3e8c9614f3efd2c2a5 Mon Sep 17 00:00:00 2001 From: VasiliyMatr Date: Fri, 27 Feb 2026 22:03:56 +0300 Subject: [PATCH 5/6] [jit] Add translation threshold setter --- src/jit/base/base.cc | 4 ++-- src/jit/base/include/prot/jit/base.hh | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/jit/base/base.cc b/src/jit/base/base.cc index 2d82b20..0326f38 100644 --- a/src/jit/base/base.cc +++ b/src/jit/base/base.cc @@ -38,7 +38,7 @@ void JitEngine::step(CPUState &cpu) { } curAddr += isa::kWordSize; } - } else if (bbIt->second.num_exec >= kExecThreshold) [[likely]] { + } else if (bbIt->second.num_exec >= m_translationThreshold) [[likely]] { auto code = translate(bbIt->second); m_tbCache.insert(pc, code); if (code != nullptr) [[likely]] { @@ -60,7 +60,7 @@ void JitEngine::interpret(CPUState &cpu, BBInfo &info) { auto JitEngine::getBBInfo(isa::Addr pc) const -> const BBInfo * { if (const auto found = m_cacheBB.find(pc); found != m_cacheBB.end()) { - if (found->second.num_exec >= kExecThreshold) { + if (found->second.num_exec >= m_translationThreshold) { return &found->second; } } diff --git a/src/jit/base/include/prot/jit/base.hh b/src/jit/base/include/prot/jit/base.hh index 460a6d9..3208969 100644 --- a/src/jit/base/include/prot/jit/base.hh +++ b/src/jit/base/include/prot/jit/base.hh @@ -3,6 +3,7 @@ #include "prot/interpreter.hh" +#include #include #include #include @@ -11,10 +12,11 @@ namespace prot::engine { using JitFunction = void (*)(CPUState &); class JitEngine : public Interpreter { - static constexpr std::size_t kExecThreshold = 10; - public: void step(CPUState &cpu) override; + void setTranslationThreshold(size_t threshold) { + m_translationThreshold = threshold; + } protected: struct TbCache { @@ -63,6 +65,7 @@ private: private: [[nodiscard]] virtual JitFunction translate(const BBInfo &info) = 0; + size_t m_translationThreshold = 0; TbCache m_tbCache; std::unordered_map m_cacheBB; }; From d305c80c481f5f512fbf2de91012a7e99d519c38 Mon Sep 17 00:00:00 2001 From: VasiliyMatr Date: Fri, 27 Feb 2026 22:05:31 +0300 Subject: [PATCH 6/6] [sim_app] Add jit threshold option & more metrics --- tools/sim/CMakeLists.txt | 1 + tools/sim/sim_app.cpp | 189 ++++++++++++++++++++++++++------------- 2 files changed, 126 insertions(+), 64 deletions(-) diff --git a/tools/sim/CMakeLists.txt b/tools/sim/CMakeLists.txt index 0d7b0ae..41c3c9f 100644 --- a/tools/sim/CMakeLists.txt +++ b/tools/sim/CMakeLists.txt @@ -6,6 +6,7 @@ target_link_libraries( PROT::defaults PROT::interpreter PROT::JIT::factory + PROT::JIT::base PROT::elf_loader PROT::hart fmt::fmt) diff --git a/tools/sim/sim_app.cpp b/tools/sim/sim_app.cpp index a390bbb..6dbaa33 100644 --- a/tools/sim/sim_app.cpp +++ b/tools/sim/sim_app.cpp @@ -1,11 +1,17 @@ #include +#include #include #include #include #include +#include +#include #include +#include +#include #include +#include #include #include @@ -15,39 +21,61 @@ #include #include +#include #include #include #include "prot/elf_loader.hh" #include "prot/hart.hh" #include "prot/interpreter.hh" +#include "prot/jit/base.hh" #include "prot/jit/factory.hh" #include "prot/memory.hh" namespace { struct RunConfig { - const std::filesystem::path &elfPath; - prot::isa::Addr stackTop = 0x7fffffff; - const std::string &jitBackend; - const std::vector &perfEvents; + std::filesystem::path elfPath; + prot::isa::Addr stackTop = 0; + std::string jitBackend; + size_t jitTranslationThreshold = 0; + + std::vector metrics; bool dumpHart = false; }; struct RunResult { uint32_t status = 0; - uint64_t guestIc = 0; - perf::CounterResult perfRes; + struct Metrics { + std::vector> metrics; + + std::optional operator[](std::string_view name) { + if (auto it = std::ranges::find_if( + metrics, [&](const auto &s) { return s == name; }, + &decltype(metrics)::value_type::first); + it != metrics.end()) { + return it->second; + } + + return std::nullopt; + } + } metrics; }; RunResult run(const RunConfig &cfg) { auto hart = [&] { prot::ElfLoader loader{cfg.elfPath}; + bool hasJit = !cfg.jitBackend.empty(); + std::unique_ptr engine = - !cfg.jitBackend.empty() - ? prot::engine::JitFactory::createEngine(cfg.jitBackend) - : std::make_unique(); + hasJit ? prot::engine::JitFactory::createEngine(cfg.jitBackend) + : std::make_unique(); + + if (hasJit) { + auto *jit = dynamic_cast(engine.get()); + jit->setTranslationThreshold(cfg.jitTranslationThreshold); + } prot::Hart hart{prot::memory::makePlain(4ULL << 30U), std::move(engine)}; hart.load(loader); @@ -56,8 +84,23 @@ RunResult run(const RunConfig &cfg) { return hart; }(); + auto perfMetrics = [&]() { + auto rng = cfg.metrics | std::views::filter([](const auto &name) { + const auto &defs = perf::CounterDefinition::global(); + return defs.is_metric(name) || !defs.counter(name).empty() || + defs.is_time_event(name); + }) | + std::views::common; + + return std::vector{rng.begin(), rng.end()}; + }(); + auto eventCounter = perf::EventCounter{}; - eventCounter.add(cfg.perfEvents); + if (!eventCounter.add(perfMetrics, perf::EventCounter::Schedule::Group)) { + throw std::runtime_error( + fmt::format("Failed to add perf events group: {{{}}}", + fmt::join(perfMetrics, ","))); + } eventCounter.start(); hart.run(); @@ -67,66 +110,90 @@ RunResult run(const RunConfig &cfg) { hart.dump(std::cout); } + std::vector> metrics{}; + + for (const auto &metric : cfg.metrics) { + // Dump perf metrics + if (auto perfMetric = eventCounter.result().get(metric)) { + metrics.emplace_back(metric, *perfMetric); + } + // Dump custom metrics + else if (metric == "guest-instructions") { + metrics.emplace_back(metric, hart.getIcount()); + } else if (metric == "jit-threshold") { + metrics.emplace_back(metric, cfg.jitTranslationThreshold); + } + } + return RunResult{.status = hart.getExitCode(), - .guestIc = hart.getIcount(), - .perfRes = eventCounter.result()}; + .metrics = {std::move(metrics)}}; } } // namespace int main(int argc, const char *argv[]) try { - std::filesystem::path elfPath; - constexpr prot::isa::Addr kDefaultStack = 0x7fffffff; - prot::isa::Addr stackTop{}; - std::string jitBackend{}; + constexpr prot::isa::Addr kDefaultStack = 0x7fff'ffff; + constexpr size_t kDefaultJitThreshold = 1'000; + + RunConfig cfg{}; std::filesystem::path statsCsvPath; - bool dumpHart = false; { CLI::App app{"App for JIT research from ProteusLab team"}; - app.add_option("elf", elfPath, "Path to executable ELF file") + app.add_option("elf", cfg.elfPath, "Path to executable ELF file") ->required() ->check(CLI::ExistingFile); - app.add_option("--stack-top", stackTop, "Address of the stack top") + app.add_option("--stack-top", cfg.stackTop, "Address of the stack top") ->default_val(kDefaultStack) ->default_str(fmt::format("{:#x}", kDefaultStack)); - app.add_option("--jit", jitBackend, "Use JIT & set backend") + app.add_option("--jit", cfg.jitBackend, "Use JIT & set backend") ->check(CLI::IsMember(prot::engine::JitFactory::backends())); + app.add_option("--jit-threshold", cfg.jitTranslationThreshold, + "Execution count threshold for code to be JIT-ed") + ->default_val(kDefaultJitThreshold); + app.add_option( "--stats", statsCsvPath, "Path to CSV stats file. Enables stats collection regime if set"); - app.add_flag("--dump", dumpHart, "Dump hart values on each run"); + app.add_flag("--dump", cfg.dumpHart, "Dump hart values on each run"); CLI11_PARSE(app, argc, argv); } bool collectStats = !statsCsvPath.empty(); - // Group related counters together - // clang-format off - std::vector> counters { - { - "seconds", - "instructions", - "cycles", - "branches", - "branch-misses", - }, - { - "L1-dcache-loads", - "L1-dcache-load-misses", - "L1-icache-loads", - "L1-icache-load-misses", - "iTLB-load-misses", - "dTLB-load-misses", - }, - }; - // clang-format on + // Group related metrics together + std::vector> metrics{ + // First group of metrics is always collected + { + // Config info + "jit-threshold", + // Guest metrics + "guest-instructions", + // Perf metrics + "seconds", + "instructions", + "cycles", + "branches", + "branch-misses", + }, + // Cache-related metrics (1) + { + "L1-dcache-loads", + "L1-dcache-load-misses", + "L1-icache-loads", + "L1-icache-load-misses", + }, + // Cache-related metrics (2) + { + "iTLB-load-misses", + "dTLB-load-misses", + }}; std::ofstream statsCsv{}; if (collectStats) { @@ -136,39 +203,31 @@ int main(int argc, const char *argv[]) try { statsCsv.open(statsCsvPath, std::ios_base::app | std::ios_base::out); if (addHeader) { - fmt::println(statsCsv, "{}", fmt::join(counters | std::views::join, ",")); + fmt::println(statsCsv, "{}", fmt::join(metrics | std::views::join, ",")); } } auto runNum = 0; - for (const auto &runCounters : - counters | std::views::take(collectStats ? 0xffU : 1U)) { - fmt::println("Run #{}", runNum++); - - auto res = run({ - .elfPath = elfPath, - .stackTop = stackTop, - .jitBackend = jitBackend, - .perfEvents = runCounters, - .dumpHart = dumpHart, - }); - - auto seconds = res.perfRes["seconds"]; - auto cycles = res.perfRes["cycles"]; - auto hostInstrs = res.perfRes["instructions"]; - - if (seconds) { - fmt::println(" time: {:.2f}s", *seconds); - } + for (const auto &runMetrics : + metrics | std::views::take(collectStats ? 0xffU : 1U)) { + fmt::println("Run #{}", runNum); + + auto runCfg = cfg; + runCfg.metrics = runMetrics; + auto res = run(runCfg); - if (seconds && cycles && hostInstrs) { - auto guestIPS = res.guestIc / *seconds; + if (runNum == 0) { + auto seconds = res.metrics["seconds"].value(); + auto guestInstrs = res.metrics["guest-instructions"].value(); + + fmt::println(" time: {:.2f}s", seconds); + auto guestIPS = guestInstrs / seconds; fmt::println(" MIPS: {:.2f}", guestIPS / 1'000'000); } if (collectStats) { fmt::print(statsCsv, "{}", - fmt::join(res.perfRes | std::views::values, ",")); + fmt::join(res.metrics.metrics | std::views::values, ",")); } fmt::println(" status = {}", res.status); @@ -176,6 +235,8 @@ int main(int argc, const char *argv[]) try { fmt::println("Run failed, aborting"); return res.status; } + + ++runNum; } if (collectStats) {