Lighteval math (#630)

* doc:model_parallel

* fix:task lighteval/MATH

Author: HERIUN

docs/source/quicktour.mdx

@@ -129,7 +129,7 @@ accelerate).
 - **add_special_tokens** (bool, optional, defaults to True): Whether to add special tokens to the input sequences.
    If `None`, the default value will be set to `True` for seq2seq models (e.g. T5) and
     `False` for causal models.
-- **model_parallel** (bool, optional, defaults to False):
+- **model_parallel** (bool, optional, defaults to None):
     True/False: force to use or not the `accelerate` library to load a large
     model across multiple devices.
     Default: None which corresponds to comparing the number of processes with

examples/nanotron/custom_evaluation_tasks.py

@@ -256,7 +256,7 @@ def __init__(
         self,
         name,
         prompt_function=prompt.math,
-        hf_repo="lighteval/MATH",
+        hf_repo="DigitalLearningGmbH/MATH-lighteval",
         hf_subset=None,
         metric=[Metrics.quasi_exact_match_math],
         hf_avail_splits=None,

src/lighteval/models/transformers/transformers_model.py

@@ -112,7 +112,7 @@ class TransformersModelConfig:
         add_special_tokens (bool, optional, defaults to True): Whether to add special tokens to the input sequences.
            If `None`, the default value will be set to `True` for seq2seq models (e.g. T5) and
             `False` for causal models.
-        model_parallel (bool, optional, defaults to False):
+        model_parallel (bool, optional, defaults to None):
             True/False: force to use or not the `accelerate` library to load a large
             model across multiple devices.
             Default: None which corresponds to comparing the number of processes with

src/lighteval/tasks/default_tasks.py

@@ -9665,9 +9665,9 @@
     name="math:algebra",
     suite=["lighteval", "math"],
     prompt_function=prompt.math,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="algebra",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9681,9 +9681,9 @@
     name="math:counting_and_probability",
     suite=["lighteval", "math"],
     prompt_function=prompt.math,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="counting_and_probability",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9697,9 +9697,9 @@
     name="math:geometry",
     suite=["lighteval", "math"],
     prompt_function=prompt.math,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="geometry",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9713,9 +9713,9 @@
     name="math:intermediate_algebra",
     suite=["lighteval", "math"],
     prompt_function=prompt.math,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="intermediate_algebra",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9729,9 +9729,9 @@
     name="math:number_theory",
     suite=["lighteval", "math"],
     prompt_function=prompt.math,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="number_theory",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9745,9 +9745,9 @@
     name="math:prealgebra",
     suite=["lighteval", "math"],
     prompt_function=prompt.math,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="prealgebra",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9761,9 +9761,9 @@
     name="math:precalculus",
     suite=["lighteval", "math"],
     prompt_function=prompt.math,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="precalculus",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9777,9 +9777,9 @@
     name="math_cot:algebra",
     suite=["lighteval", "math"],
     prompt_function=prompt.math_cot,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="algebra",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9793,9 +9793,9 @@
     name="math_cot:counting_and_probability",
     suite=["lighteval", "math"],
     prompt_function=prompt.math_cot,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="counting_and_probability",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9809,9 +9809,9 @@
     name="math_cot:geometry",
     suite=["lighteval", "math"],
     prompt_function=prompt.math_cot,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="geometry",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9825,9 +9825,9 @@
     name="math_cot:intermediate_algebra",
     suite=["lighteval", "math"],
     prompt_function=prompt.math_cot,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="intermediate_algebra",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9841,9 +9841,9 @@
     name="math_cot:number_theory",
     suite=["lighteval", "math"],
     prompt_function=prompt.math_cot,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="number_theory",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9857,9 +9857,9 @@
     name="math_cot:prealgebra",
     suite=["lighteval", "math"],
     prompt_function=prompt.math_cot,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="prealgebra",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,
@@ -9873,9 +9873,9 @@
     name="math_cot:precalculus",
     suite=["lighteval", "math"],
     prompt_function=prompt.math_cot,
-    hf_repo="lighteval/MATH",
+    hf_repo="DigitalLearningGmbH/MATH-lighteval",
     hf_subset="precalculus",
-    hf_avail_splits=["train", "test", "validation"],
+    hf_avail_splits=["train", "test"],
     evaluation_splits=["test"],
     few_shots_split=None,
     few_shots_select=None,