Skip to content

Helm leaderboards wrt schema v0 2 #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
damian1996 merged 2 commits into from
Feb 12, 2026
Merged

Helm leaderboards wrt schema v0 2 #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9afd31e
Migrated HELM leaderboards into schema v0.2
damian1996 Feb 11, 2026
b775902
Fix naming conventions
damian1996 Feb 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
110 changes: 80 additions & 30 deletions ...18881f8b-b06e-4317-b697-6eadb975077c.json → ...8948bfb0-cc9d-40f7-a02d-d5c9611436d8.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
{
"schema_version": "0.1.0",
"evaluation_id": "helm_capabilities/allenai_olmo-2-0325-32b-instruct/1767657480.2939079",
"retrieved_timestamp": "1767657480.2939079",
"source_data": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
],
"schema_version": "0.2.0",
"evaluation_id": "helm_capabilities/allenai_olmo-2-0325-32b-instruct/1770835969.095764",
"retrieved_timestamp": "1770835969.095764",
"source_metadata": {
"source_name": "helm_capabilities",
"source_type": "documentation",
Expand All @@ -20,6 +17,13 @@
"evaluation_results": [
{
"evaluation_name": "Mean score",
"source_data": {
"dataset_name": "helm_capabilities",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "The mean of the scores from all columns.",
"lower_is_better": false,
Expand All @@ -30,7 +34,6 @@
"score_details": {
"score": 0.475,
"details": {
"description": null,
"tab": "Accuracy",
"Mean score - Efficiency": {
"description": null,
Expand All @@ -39,12 +42,21 @@
}
}
},
"generation_config": {}
"generation_config": {
"additional_details": {}
}
},
{
"evaluation_name": "MMLU-Pro - COT correct",
"evaluation_name": "MMLU-Pro",
"source_data": {
"dataset_name": "MMLU-Pro",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "MMLU-Pro\n\nCOT correct: Fraction of correct answers after chain of thought",
"evaluation_description": "COT correct on MMLU-Pro",
"lower_is_better": false,
"score_type": "continuous",
"min_score": 0.0,
Expand Down Expand Up @@ -88,16 +100,25 @@
}
},
"generation_config": {
"subset": "all",
"use_chain_of_thought": "true",
"use_few_shot": "false",
"num_output_tokens": "2048"
"additional_details": {
"subset": "all",
"use_chain_of_thought": "true",
"use_few_shot": "false",
"num_output_tokens": "2048"
}
}
},
{
"evaluation_name": "GPQA - COT correct",
"evaluation_name": "GPQA",
"source_data": {
"dataset_name": "GPQA",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "GPQA\n\nCOT correct: Fraction of correct answers after chain of thought",
"evaluation_description": "COT correct on GPQA",
"lower_is_better": false,
"score_type": "continuous",
"min_score": 0.0,
Expand Down Expand Up @@ -141,16 +162,25 @@
}
},
"generation_config": {
"subset": "gpqa_main",
"use_chain_of_thought": "true",
"use_few_shot": "false",
"num_output_tokens": "2048"
"additional_details": {
"subset": "gpqa_main",
"use_chain_of_thought": "true",
"use_few_shot": "false",
"num_output_tokens": "2048"
}
}
},
{
"evaluation_name": "IFEval - IFEval Strict Acc",
"evaluation_name": "IFEval",
"source_data": {
"dataset_name": "IFEval",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "IFEval\n\nIFEval strict accuracy: Fraction of instructions in the instance that are correctly followed.",
"evaluation_description": "IFEval Strict Acc on IFEval",
"lower_is_better": false,
"score_type": "continuous",
"min_score": 0.0,
Expand Down Expand Up @@ -194,13 +224,22 @@
}
},
"generation_config": {
"num_output_tokens": "2048"
"additional_details": {
"num_output_tokens": "2048"
}
}
},
{
"evaluation_name": "WildBench - WB Score",
"evaluation_name": "WildBench",
"source_data": {
"dataset_name": "WildBench",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "WildBench\n\nWildBench Score: Score of the AI output judged by GPT-4o, rescaled to be between 0 and 1.",
"evaluation_description": "WB Score on WildBench",
"lower_is_better": false,
"score_type": "continuous",
"min_score": 0.0,
Expand Down Expand Up @@ -244,14 +283,23 @@
}
},
"generation_config": {
"subset": "v2",
"num_output_tokens": "2048"
"additional_details": {
"subset": "v2",
"num_output_tokens": "2048"
}
}
},
{
"evaluation_name": "Omni-MATH - Acc",
"evaluation_name": "Omni-MATH",
"source_data": {
"dataset_name": "Omni-MATH",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "Omni-MATH\n\nOmni-MATH Accuracy: Accuracy of the AI output judged by GPT-4.",
"evaluation_description": "Acc on Omni-MATH",
"lower_is_better": false,
"score_type": "continuous",
"min_score": 0.0,
Expand Down Expand Up @@ -295,7 +343,9 @@
}
},
"generation_config": {
"num_output_tokens": "2048"
"additional_details": {
"num_output_tokens": "2048"
}
}
}
]
Expand Down
110 changes: 80 additions & 30 deletions ...97db1a8d-b7d8-4481-82fb-dc0c6396edac.json → ...7d2d1dba-1b31-47b2-8308-f2668cf36c99.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
{
"schema_version": "0.1.0",
"evaluation_id": "helm_capabilities/allenai_olmo-2-1124-13b-instruct/1767657480.2939079",
"retrieved_timestamp": "1767657480.2939079",
"source_data": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json"
],
"schema_version": "0.2.0",
"evaluation_id": "helm_capabilities/allenai_olmo-2-1124-13b-instruct/1770835969.095764",
"retrieved_timestamp": "1770835969.095764",
"source_metadata": {
"source_name": "helm_capabilities",
"source_type": "documentation",
Expand All @@ -20,6 +17,13 @@
"evaluation_results": [
{
"evaluation_name": "Mean score",
"source_data": {
"dataset_name": "helm_capabilities",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "The mean of the scores from all columns.",
"lower_is_better": false,
Expand All @@ -30,7 +34,6 @@
"score_details": {
"score": 0.44,
"details": {
"description": null,
"tab": "Accuracy",
"Mean score - Efficiency": {
"description": null,
Expand All @@ -39,12 +42,21 @@
}
}
},
"generation_config": {}
"generation_config": {
"additional_details": {}
}
},
{
"evaluation_name": "MMLU-Pro - COT correct",
"evaluation_name": "MMLU-Pro",
"source_data": {
"dataset_name": "MMLU-Pro",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "MMLU-Pro\n\nCOT correct: Fraction of correct answers after chain of thought",
"evaluation_description": "COT correct on MMLU-Pro",
"lower_is_better": false,
"score_type": "continuous",
"min_score": 0.0,
Expand Down Expand Up @@ -88,16 +100,25 @@
}
},
"generation_config": {
"subset": "all",
"use_chain_of_thought": "true",
"use_few_shot": "false",
"num_output_tokens": "2048"
"additional_details": {
"subset": "all",
"use_chain_of_thought": "true",
"use_few_shot": "false",
"num_output_tokens": "2048"
}
}
},
{
"evaluation_name": "GPQA - COT correct",
"evaluation_name": "GPQA",
"source_data": {
"dataset_name": "GPQA",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "GPQA\n\nCOT correct: Fraction of correct answers after chain of thought",
"evaluation_description": "COT correct on GPQA",
"lower_is_better": false,
"score_type": "continuous",
"min_score": 0.0,
Expand Down Expand Up @@ -141,16 +162,25 @@
}
},
"generation_config": {
"subset": "gpqa_main",
"use_chain_of_thought": "true",
"use_few_shot": "false",
"num_output_tokens": "2048"
"additional_details": {
"subset": "gpqa_main",
"use_chain_of_thought": "true",
"use_few_shot": "false",
"num_output_tokens": "2048"
}
}
},
{
"evaluation_name": "IFEval - IFEval Strict Acc",
"evaluation_name": "IFEval",
"source_data": {
"dataset_name": "IFEval",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "IFEval\n\nIFEval strict accuracy: Fraction of instructions in the instance that are correctly followed.",
"evaluation_description": "IFEval Strict Acc on IFEval",
"lower_is_better": false,
"score_type": "continuous",
"min_score": 0.0,
Expand Down Expand Up @@ -194,13 +224,22 @@
}
},
"generation_config": {
"num_output_tokens": "2048"
"additional_details": {
"num_output_tokens": "2048"
}
}
},
{
"evaluation_name": "WildBench - WB Score",
"evaluation_name": "WildBench",
"source_data": {
"dataset_name": "WildBench",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "WildBench\n\nWildBench Score: Score of the AI output judged by GPT-4o, rescaled to be between 0 and 1.",
"evaluation_description": "WB Score on WildBench",
"lower_is_better": false,
"score_type": "continuous",
"min_score": 0.0,
Expand Down Expand Up @@ -244,14 +283,23 @@
}
},
"generation_config": {
"subset": "v2",
"num_output_tokens": "2048"
"additional_details": {
"subset": "v2",
"num_output_tokens": "2048"
}
}
},
{
"evaluation_name": "Omni-MATH - Acc",
"evaluation_name": "Omni-MATH",
"source_data": {
"dataset_name": "Omni-MATH",
"source_type": "url",
"url": [
"https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
]
},
"metric_config": {
"evaluation_description": "Omni-MATH\n\nOmni-MATH Accuracy: Accuracy of the AI output judged by GPT-4.",
"evaluation_description": "Acc on Omni-MATH",
"lower_is_better": false,
"score_type": "continuous",
"min_score": 0.0,
Expand Down Expand Up @@ -295,7 +343,9 @@
}
},
"generation_config": {
"num_output_tokens": "2048"
"additional_details": {
"num_output_tokens": "2048"
}
}
}
]
Expand Down
Loading