From ef733fe7cfb6ad98c754153cbb23543354f08855 Mon Sep 17 00:00:00 2001
From: bugdaryan <bugdaryan@gmail.com>
Date: Wed, 8 Jul 2026 13:50:55 +0400
Subject: [PATCH] Add Armenian language evaluation suite

Adds 24 Armenian (hy) tasks backed by the Metric-AI/ArmBench-LLM-data dataset,
custom Armenian metrics (exam, MCQA, MMLU-Pro, NER span, POS, BERTScore),
answer-extraction utilities, and a text-tagging metric.
---
 src/lighteval/metrics/armenian_metrics.py     | 102 ++++----
 src/lighteval/metrics/metrics.py              | 224 ++++-------------
 src/lighteval/metrics/metrics_sample.py       | 198 ++++-----------
 .../metrics/utils/armenian_eval_utils.py      |  68 +++--
 .../models/transformers/transformers_model.py |   4 +-
 .../tasks/multilingual/tasks/armenian.py      | 237 ++++++++----------
 6 files changed, 289 insertions(+), 544 deletions(-)

diff --git a/src/lighteval/metrics/armenian_metrics.py b/src/lighteval/metrics/armenian_metrics.py
index 31b4da088..b89107f3a 100644
--- a/src/lighteval/metrics/armenian_metrics.py
+++ b/src/lighteval/metrics/armenian_metrics.py
@@ -1,46 +1,59 @@
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import json
+import re
+
+import numpy as np
+import torch
+
+from lighteval.metrics.imports.bert_scorer import BERTScorer
 from lighteval.metrics.utils.armenian_eval_utils import (
-    extract_answer_for_numeric_choices,
-    extract_again_for_numeric_choices,
-    extract_final_for_numeric_choices,
-    extract_answer_for_numeric_answer,
+    extract_again_for_letter_choices,
     extract_again_for_numeric_answer,
-    extract_final_for_numeric_answer,
+    extract_again_for_numeric_choices,
     extract_answer_for_letter_choices,
-    extract_again_for_letter_choices,
-    extract_final_for_letter_choices,
-    extract_correct_answers_list,
+    extract_answer_for_numeric_answer,
+    extract_answer_for_numeric_choices,
     extract_correct_answers_dict,
+    extract_correct_answers_list,
+    extract_final_for_letter_choices,
+    extract_final_for_numeric_answer,
+    extract_final_for_numeric_choices,
 )
-from aenum import extend_enum
-import pandas as pd
-import numpy as np
-import torch
-import json
-import re
-import os
-
-from lighteval.metrics.metrics import Metrics
 from lighteval.metrics.utils.metric_utils import (
+    SampleLevelComputation,
     SampleLevelMetric,
     SamplingMethod,
-    SampleLevelComputation,
 )
 from lighteval.models.model_output import ModelResponse
 from lighteval.tasks.lighteval_task import Doc
-from lighteval.metrics.imports.bert_scorer import BERTScorer
 
 
 class NERSpanComputation(SampleLevelComputation):
     def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
-        gold_entities = [
-            (t.lower().strip(), ty.lower().strip())
-            for (t, ty) in doc.specific.get("gold_entities", [])
-        ]
+        gold_entities = [(t.lower().strip(), ty.lower().strip()) for (t, ty) in doc.specific.get("gold_entities", [])]
 
-        if (
-            hasattr(model_response, "text_post_processed")
-            and model_response.text_post_processed
-        ):
+        if hasattr(model_response, "text_post_processed") and model_response.text_post_processed:
             pred_text = model_response.text_post_processed[0]
         elif hasattr(model_response, "text") and model_response.text:
             pred_text = model_response.text[0]
@@ -63,7 +76,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
 
         return correct / len(gold_entities) if gold_entities else 0.0
 
-    def parse_pred(self, pred: str):
+    def parse_pred(self, pred: str):  # noqa: C901
         pred = pred.strip()
         if pred.startswith("```"):
             pred = pred.strip("`").lstrip("json").strip()
@@ -94,9 +107,7 @@ def parse_pred(self, pred: str):
                             entity_text = e["entity"].strip()
                             entity_tag = e["tag"].strip()
                             if entity_text and entity_tag:
-                                entities.append(
-                                    (entity_text.lower(), entity_tag.lower())
-                                )
+                                entities.append((entity_text.lower(), entity_tag.lower()))
                     if entities:
                         break
             except (json.JSONDecodeError, KeyError, TypeError):
@@ -111,9 +122,7 @@ def parse_pred(self, pred: str):
                             entity_text = e["entity"].strip()
                             entity_tag = e["tag"].strip()
                             if entity_text and entity_tag:
-                                entities.append(
-                                    (entity_text.lower(), entity_tag.lower())
-                                )
+                                entities.append((entity_text.lower(), entity_tag.lower()))
             except Exception:
                 parts = [p for p in pred.replace(";", ",").split(",") if ":" in p]
                 for part in parts:
@@ -153,10 +162,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
             if t is not None and ty is not None
         ]
 
-        if (
-            hasattr(model_response, "text_post_processed")
-            and model_response.text_post_processed
-        ):
+        if hasattr(model_response, "text_post_processed") and model_response.text_post_processed:
             pred_text = model_response.text_post_processed[0]
         elif hasattr(model_response, "text") and model_response.text:
             pred_text = model_response.text[0]
@@ -217,13 +223,8 @@ def __init__(self):
     def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
         pred = (
             model_response.text_post_processed[0]
-            if hasattr(model_response, "text_post_processed")
-            and model_response.text_post_processed
-            else (
-                model_response.text[0]
-                if hasattr(model_response, "text") and model_response.text
-                else ""
-            )
+            if hasattr(model_response, "text_post_processed") and model_response.text_post_processed
+            else (model_response.text[0] if hasattr(model_response, "text") and model_response.text else "")
         ).strip()
 
         golds = doc.choices or []
@@ -235,7 +236,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
 
 
 class ArmenianExamComputation(SampleLevelComputation):
-    def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
+    def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):  # noqa: C901
 
         task_type = doc.specific.get("task_type")
         true_answer = doc.specific.get("label")
@@ -255,9 +256,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
         elif task_type == 2:
             extracted_answer = extract_correct_answers_list(text)
             extracted_answer = [str(i) for i in extracted_answer]
-            if extracted_answer is not None and set(true_answer) == set(
-                extracted_answer
-            ):
+            if extracted_answer is not None and set(true_answer) == set(extracted_answer):
                 score = 0.25
 
         elif task_type == 3:
@@ -352,10 +351,7 @@ def compute(self, doc, model_response, **kwargs):
             if extracted_answer is not None:
                 break
 
-        if (
-            extracted_answer is not None
-            and str(extracted_answer).strip() == str(true_answer).strip()
-        ):
+        if extracted_answer is not None and str(extracted_answer).strip() == str(true_answer).strip():
             return 1.0
 
         return 0.0
diff --git a/src/lighteval/metrics/metrics.py b/src/lighteval/metrics/metrics.py
index 835659bbe..7943499a1 100644
--- a/src/lighteval/metrics/metrics.py
+++ b/src/lighteval/metrics/metrics.py
@@ -58,11 +58,7 @@
     StringDistance,
     TextTaggingMetric,
 )
-from lighteval.metrics.normalizations import (
-    bigbench_normalizer,
-    remove_braces,
-    remove_braces_and_strip,
-)
+from lighteval.metrics.normalizations import bigbench_normalizer, remove_braces, remove_braces_and_strip
 from lighteval.metrics.sample_preparator import (
     GenerativePreparator,
     LoglikelihoodPreparator,
@@ -89,36 +85,21 @@
 @scorer(metrics=[accuracy()])
 def math_scorer():
     gold_extraction_target = (ExprExtractionConfig(),)
-    pred_extraction_target = (
-        ExprExtractionConfig(),
-        LatexExtractionConfig(boxed_match_priority=0),
-    )
+    pred_extraction_target = (ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0))
     language = Language.ENGLISH
     fallback_mode = "first_match"
     extraction_mode = "first_match"
     timeout_seconds = 5
 
-    gold_extraction_regexes = get_extraction_regexes_inspect(
-        gold_extraction_target, language, len_choices=1
-    )
-    pred_extraction_regexes = get_extraction_regexes_inspect(
-        pred_extraction_target, language, len_choices=1
-    )
+    gold_extraction_regexes = get_extraction_regexes_inspect(gold_extraction_target, language, len_choices=1)
+    pred_extraction_regexes = get_extraction_regexes_inspect(pred_extraction_target, language, len_choices=1)
 
     async def score(state: TaskState, target: Target):
         extracted_predictions = extract_target_from_pred(
-            state.output.completion,
-            pred_extraction_regexes,
-            fallback_mode,
-            extraction_mode,
-            timeout_seconds,
+            state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
         )
         extracted_gold = extract_target_from_pred(
-            target.text,
-            gold_extraction_regexes,
-            fallback_mode,
-            extraction_mode,
-            timeout_seconds,
+            target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
         )
         return Score(
             # Correct or Incorrect, used by inspect-ai backend
@@ -134,40 +115,24 @@ async def score(state: TaskState, target: Target):
 def multichoice_scorer():
     language = Language.ENGLISH
     gold_extraction_target = (
-        IndicesExtractionConfig(
-            prefix_for_extraction="NativeLetters", try_extract_without_anchor=True
-        ),
+        IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True),
     )
     pred_extraction_target = (
-        IndicesExtractionConfig(
-            prefix_for_extraction="NativeLetters", try_extract_without_anchor=True
-        ),
+        IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True),
     )
     fallback_mode = "first_match"
     extraction_mode = "first_match"
     timeout_seconds = 5
 
-    gold_extraction_regexes = get_extraction_regexes_inspect(
-        gold_extraction_target, language
-    )
-    pred_extraction_regexes = get_extraction_regexes_inspect(
-        pred_extraction_target, language
-    )
+    gold_extraction_regexes = get_extraction_regexes_inspect(gold_extraction_target, language)
+    pred_extraction_regexes = get_extraction_regexes_inspect(pred_extraction_target, language)
 
     async def score(state: TaskState, target: Target):
         extracted_predictions = extract_target_from_pred(
-            state.output.completion,
-            pred_extraction_regexes,
-            fallback_mode,
-            extraction_mode,
-            timeout_seconds,
+            state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
         )
         extracted_gold = extract_target_from_pred(
-            target.text,
-            gold_extraction_regexes,
-            fallback_mode,
-            extraction_mode,
-            timeout_seconds,
+            target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds
         )
         return Score(
             # Correct or Incorrect, used by inspect-ai backend
@@ -199,14 +164,8 @@ class Metrics(Enum):
         sample_level_fn=AvgAtN(
             sample_scoring_function=MultilingualExtractiveMatchMetric(
                 language=Language.ENGLISH,
-                gold_extraction_target=[
-                    ExprExtractionConfig(),
-                    LatexExtractionConfig(),
-                ],
-                pred_extraction_target=[
-                    ExprExtractionConfig(),
-                    LatexExtractionConfig(),
-                ],
+                gold_extraction_target=[ExprExtractionConfig(), LatexExtractionConfig()],
+                pred_extraction_target=[ExprExtractionConfig(), LatexExtractionConfig()],
                 precision=6,
             ),
         ),
@@ -216,20 +175,10 @@ class Metrics(Enum):
     )
     bert_score = SampleLevelMetricGrouping(
         metric_name=["BERTScore-P", "BERTScore-R", "BERTScore-F"],
-        sample_level_fn=BertScore(
-            normalize_gold=remove_braces, normalize_pred=remove_braces_and_strip
-        ),
+        sample_level_fn=BertScore(normalize_gold=remove_braces, normalize_pred=remove_braces_and_strip),
         category=SamplingMethod.GENERATIVE,
-        corpus_level_fn={
-            "BERTScore-P": np.mean,
-            "BERTScore-R": np.mean,
-            "BERTScore-F": np.mean,
-        },
-        higher_is_better={
-            "BERTScore-P": True,
-            "BERTScore-R": True,
-            "BERTScore-F": True,
-        },
+        corpus_level_fn={"BERTScore-P": np.mean, "BERTScore-R": np.mean, "BERTScore-F": np.mean},
+        higher_is_better={"BERTScore-P": True, "BERTScore-R": True, "BERTScore-F": True},
     )
     bits_per_byte = CorpusLevelMetric(
         metric_name="bits_per_byte",
@@ -289,30 +238,13 @@ class Metrics(Enum):
         higher_is_better=True,
     )
     copyright = SampleLevelMetricGrouping(
-        metric_name=[
-            "longest_common_prefix_length",
-            "edit_distance",
-            "edit_similarity",
-        ],
+        metric_name=["longest_common_prefix_length", "edit_distance", "edit_similarity"],
         sample_level_fn=StringDistance(
-            metric_types=[
-                "longest_common_prefix_length",
-                "edit_distance",
-                "edit_similarity",
-            ],
-            strip_prediction=True,
+            metric_types=["longest_common_prefix_length", "edit_distance", "edit_similarity"], strip_prediction=True
         ),
         category=SamplingMethod.GENERATIVE,
-        corpus_level_fn={
-            "longest_common_prefix_length": max,
-            "edit_distance": min,
-            "edit_similarity": max,
-        },
-        higher_is_better={
-            "longest_common_prefix_length": True,
-            "edit_distance": False,
-            "edit_similarity": True,
-        },
+        corpus_level_fn={"longest_common_prefix_length": max, "edit_distance": min, "edit_similarity": max},
+        higher_is_better={"longest_common_prefix_length": True, "edit_distance": False, "edit_similarity": True},
     )
     drop = SampleLevelMetricGrouping(
         metric_name=["em", "f1"],
@@ -336,10 +268,7 @@ class Metrics(Enum):
             precision=5,
             gold_extraction_target=(ExprExtractionConfig(),),
             # Match boxed first before trying other regexes
-            pred_extraction_target=(
-                ExprExtractionConfig(),
-                LatexExtractionConfig(boxed_match_priority=0),
-            ),
+            pred_extraction_target=(ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0)),
             aggregation_function=max,
         ),
         category=SamplingMethod.GENERATIVE,
@@ -347,15 +276,9 @@ class Metrics(Enum):
         higher_is_better=True,
     )
     extractiveness = SampleLevelMetricGrouping(
-        metric_name=[
-            "summarization_coverage",
-            "summarization_density",
-            "summarization_compression",
-        ],
+        metric_name=["summarization_coverage", "summarization_density", "summarization_compression"],
         sample_level_fn=Extractiveness(
-            normalize_input=remove_braces,
-            normalize_pred=remove_braces_and_strip,
-            input_column="text",
+            normalize_input=remove_braces, normalize_pred=remove_braces_and_strip, input_column="text"
         ),
         category=SamplingMethod.GENERATIVE,
         corpus_level_fn={
@@ -370,16 +293,9 @@ class Metrics(Enum):
         },
     )
     extractiveness_de = SampleLevelMetricGrouping(
-        metric_name=[
-            "summarization_coverage",
-            "summarization_density",
-            "summarization_compression",
-        ],
+        metric_name=["summarization_coverage", "summarization_density", "summarization_compression"],
         sample_level_fn=Extractiveness(
-            normalize_input=remove_braces,
-            normalize_pred=remove_braces_and_strip,
-            input_column="text",
-            language="de",
+            normalize_input=remove_braces, normalize_pred=remove_braces_and_strip, input_column="text", language="de"
         ),
         category=SamplingMethod.GENERATIVE,
         corpus_level_fn={
@@ -394,16 +310,9 @@ class Metrics(Enum):
         },
     )
     extractiveness_fr = SampleLevelMetricGrouping(
-        metric_name=[
-            "summarization_coverage",
-            "summarization_density",
-            "summarization_compression",
-        ],
+        metric_name=["summarization_coverage", "summarization_density", "summarization_compression"],
         sample_level_fn=Extractiveness(
-            normalize_input=remove_braces,
-            normalize_pred=remove_braces_and_strip,
-            input_column="text",
-            language="fr",
+            normalize_input=remove_braces, normalize_pred=remove_braces_and_strip, input_column="text", language="fr"
         ),
         category=SamplingMethod.GENERATIVE,
         corpus_level_fn={
@@ -418,16 +327,9 @@ class Metrics(Enum):
         },
     )
     extractiveness_it = SampleLevelMetricGrouping(
-        metric_name=[
-            "summarization_coverage",
-            "summarization_density",
-            "summarization_compression",
-        ],
+        metric_name=["summarization_coverage", "summarization_density", "summarization_compression"],
         sample_level_fn=Extractiveness(
-            normalize_input=remove_braces,
-            normalize_pred=remove_braces_and_strip,
-            input_column="text",
-            language="it",
+            normalize_input=remove_braces, normalize_pred=remove_braces_and_strip, input_column="text", language="it"
         ),
         category=SamplingMethod.GENERATIVE,
         corpus_level_fn={
@@ -465,9 +367,7 @@ class Metrics(Enum):
     faithfulness = SampleLevelMetric(
         metric_name="summac",
         sample_level_fn=Faithfulness(
-            normalize_input=remove_braces,
-            normalize_pred=remove_braces_and_strip,
-            input_column="text",
+            normalize_input=remove_braces, normalize_pred=remove_braces_and_strip, input_column="text"
         ),
         category=SamplingMethod.GENERATIVE,
         corpus_level_fn=np.mean,
@@ -491,10 +391,7 @@ class Metrics(Enum):
                 precision=5,
                 gold_extraction_target=(ExprExtractionConfig(),),
                 # Match boxed first before trying other regexes
-                pred_extraction_target=(
-                    ExprExtractionConfig(),
-                    LatexExtractionConfig(boxed_match_priority=0),
-                ),
+                pred_extraction_target=(ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0)),
                 aggregation_function=max,
             ),
         ),
@@ -513,10 +410,7 @@ class Metrics(Enum):
                 precision=5,
                 gold_extraction_target=(LatexExtractionConfig(),),
                 # Match boxed first before trying other regexes
-                pred_extraction_target=(
-                    ExprExtractionConfig(),
-                    LatexExtractionConfig(boxed_match_priority=0),
-                ),
+                pred_extraction_target=(ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0)),
                 aggregation_function=max,
             ),
         ),
@@ -580,14 +474,8 @@ class Metrics(Enum):
             # Extracting mathematical expressions and latex expressions
             sample_scoring_function=MultilingualExtractiveMatchMetric(
                 language=Language.ENGLISH,
-                gold_extraction_target=[
-                    ExprExtractionConfig(),
-                    LatexExtractionConfig(),
-                ],
-                pred_extraction_target=[
-                    ExprExtractionConfig(),
-                    LatexExtractionConfig(),
-                ],
+                gold_extraction_target=[ExprExtractionConfig(), LatexExtractionConfig()],
+                pred_extraction_target=[ExprExtractionConfig(), LatexExtractionConfig()],
                 precision=6,
             ),
         ),
@@ -600,12 +488,8 @@ class Metrics(Enum):
         sample_level_fn=PassAtK(
             sample_scoring_function=MultilingualExtractiveMatchMetric(
                 language=Language.ENGLISH,
-                gold_extraction_target=[
-                    IndicesExtractionConfig(prefix_for_extraction="NativeLetters")
-                ],
-                pred_extraction_target=[
-                    IndicesExtractionConfig(prefix_for_extraction="NativeLetters")
-                ],
+                gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")],
+                pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")],
                 precision=6,
             ),
         ),
@@ -636,18 +520,8 @@ class Metrics(Enum):
             normalize_pred=bigbench_normalizer,
         ),
         category=SamplingMethod.GENERATIVE,
-        corpus_level_fn={
-            "rouge1": np.mean,
-            "rouge2": np.mean,
-            "rougeL": np.mean,
-            "rougeLsum": np.mean,
-        },
-        higher_is_better={
-            "rouge1": True,
-            "rouge2": True,
-            "rougeL": True,
-            "rougeLsum": True,
-        },
+        corpus_level_fn={"rouge1": np.mean, "rouge2": np.mean, "rougeL": np.mean, "rougeLsum": np.mean},
+        higher_is_better={"rouge1": True, "rouge2": True, "rougeL": True, "rougeLsum": True},
     )
     rouge1 = SampleLevelMetric(
         metric_name="rouge1",
@@ -720,16 +594,10 @@ class Metrics(Enum):
         sample_level_fn=MultilingualExtractiveMatchMetric(
             language=Language.ENGLISH,
             gold_extraction_target=[
-                IndicesExtractionConfig(
-                    prefix_for_extraction="NativeLetters",
-                    try_extract_without_anchor=True,
-                )
+                IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True)
             ],
             pred_extraction_target=[
-                IndicesExtractionConfig(
-                    prefix_for_extraction="NativeLetters",
-                    try_extract_without_anchor=True,
-                )
+                IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True)
             ],
             precision=6,
         ),
@@ -743,16 +611,10 @@ class Metrics(Enum):
             sample_scoring_function=MultilingualExtractiveMatchMetric(
                 language=Language.ENGLISH,
                 gold_extraction_target=[
-                    IndicesExtractionConfig(
-                        prefix_for_extraction="NativeLetters",
-                        try_extract_without_anchor=True,
-                    )
+                    IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True)
                 ],
                 pred_extraction_target=[
-                    IndicesExtractionConfig(
-                        prefix_for_extraction="NativeLetters",
-                        try_extract_without_anchor=True,
-                    )
+                    IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True)
                 ],
                 precision=6,
             ),
diff --git a/src/lighteval/metrics/metrics_sample.py b/src/lighteval/metrics/metrics_sample.py
index b0e3241d1..52c130cca 100644
--- a/src/lighteval/metrics/metrics_sample.py
+++ b/src/lighteval/metrics/metrics_sample.py
@@ -51,10 +51,7 @@
     remove_braces,
     remove_braces_and_strip,
 )
-from lighteval.metrics.utils.judge_utils import (
-    get_judge_prompt_simpleqa,
-    process_judge_response_simpleqa,
-)
+from lighteval.metrics.utils.judge_utils import get_judge_prompt_simpleqa, process_judge_response_simpleqa
 from lighteval.metrics.utils.llm_as_judge import JudgeLM
 from lighteval.models.model_output import ModelResponse
 from lighteval.tasks.requests import Doc
@@ -352,9 +349,7 @@ def compute(
         )
         normalized_probs = np.exp(normalized_log_probs)
 
-        normalized_probs = safe_divide(
-            normalized_probs[gold_ixs], np.sum(normalized_probs)
-        )
+        normalized_probs = safe_divide(normalized_probs[gold_ixs], np.sum(normalized_probs))
         gold_idx_agg_prob = self.aggregation_function(normalized_probs)
         return gold_idx_agg_prob
 
@@ -437,14 +432,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> int:
 
         if self.recall_depth == 1:
             return int(np.argmax(choices_logprobs) in gold_ixs)
-        return int(
-            any(
-                ix in gold_ixs
-                for ix in np.array(choices_logprobs).argsort()[::-1][
-                    : self.recall_depth
-                ]
-            )
-        )
+        return int(any(ix in gold_ixs for ix in np.array(choices_logprobs).argsort()[::-1][: self.recall_depth]))
 
 
 class MRR(SampleLevelComputation):
@@ -473,14 +461,10 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
 
         if self.length_normalization:
             choices_logprobs = [
-                choice_logprob / len(choice)
-                for choice_logprob, choice in zip(choices_logprobs, doc.choices)
+                choice_logprob / len(choice) for choice_logprob, choice in zip(choices_logprobs, doc.choices)
             ]
 
-        ranked_choices = [
-            sorted(choices_logprobs, reverse=True).index(choices_logprobs[gold])
-            for gold in gold_ixs
-        ]
+        ranked_choices = [sorted(choices_logprobs, reverse=True).index(choices_logprobs[gold]) for gold in gold_ixs]
         return 1.0 / (min(ranked_choices) + 1)
 
 
@@ -529,9 +513,7 @@ def __init__(
                 default tokenizer will be used.
         """
         if aggregation_function and bootstrap:
-            logger.warning(
-                "Can't use both bootstrapping and an aggregation function in Rouge. Keeping bootstrap."
-            )
+            logger.warning("Can't use both bootstrapping and an aggregation function in Rouge. Keeping bootstrap.")
         self.aggregation_function = aggregation_function
         if self.aggregation_function is None:
             self.aggregation_function = np.mean
@@ -548,9 +530,7 @@ def __init__(
         self.tokenizer = tokenizer
         self.scorer = None
 
-    def compute(
-        self, doc: Doc, model_response: ModelResponse, **kwargs
-    ) -> float | dict:
+    def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float | dict:
         """Computes the metric(s) over a list of golds and predictions for one single sample.
 
         Args:
@@ -568,9 +548,7 @@ def compute(
         predictions = model_response.final_text
 
         if self.scorer is None:
-            self.scorer = rouge_scorer.RougeScorer(
-                self.methods, tokenizer=self.tokenizer
-            )
+            self.scorer = rouge_scorer.RougeScorer(self.methods, tokenizer=self.tokenizer)
 
         # Normalize
         if self.normalize_gold:
@@ -580,9 +558,7 @@ def compute(
             predictions = [self.normalize_pred(p) for p in predictions]
 
         if self.bootstrap:  # For t5 style rouge score
-            scores = self._rouge_score_with_bootsrap(
-                golds=golds, predictions=predictions
-            )
+            scores = self._rouge_score_with_bootsrap(golds=golds, predictions=predictions)
         elif self.multiple_golds:
             scores = self._rouge_score_multi_golds(golds=golds, preds=predictions)
         else:
@@ -599,9 +575,7 @@ def _rouge_score(self, golds: list[str], preds: list[str]):
                 cur_scores = self.scorer.score(gold, pred)
                 for method in self.methods:
                     scores[method].append(cur_scores[method].fmeasure)
-        return {
-            method: self.aggregation_function(scores[method]) for method in self.methods
-        }
+        return {method: self.aggregation_function(scores[method]) for method in self.methods}
 
     def _rouge_score_multi_golds(self, golds: list[str], preds: list[str]):
         scores = {m: [] for m in self.methods}
@@ -609,9 +583,7 @@ def _rouge_score_multi_golds(self, golds: list[str], preds: list[str]):
             cur_scores = self.scorer.score_multi(golds, pred)
             for method in self.methods:
                 scores[method].append(cur_scores[method].fmeasure)
-        return {
-            method: self.aggregation_function(scores[method]) for method in self.methods
-        }
+        return {method: self.aggregation_function(scores[method]) for method in self.methods}
 
     def _rouge_score_with_bootsrap(self, golds: list[str], predictions: list[str]):
         from rouge_score import scoring
@@ -653,9 +625,7 @@ def __init__(
         self.normalize_gold = normalize_gold
         self.normalize_pred = normalize_pred
 
-    def compute(
-        self, doc: Doc, model_response: ModelResponse, **kwargs
-    ) -> dict[str, float]:
+    def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict[str, float]:
         """Computes the prediction, recall and f1 score using the bert scorer.
 
         Args:
@@ -670,15 +640,10 @@ def compute(
         predictions = model_response.final_text
 
         if self.bert_scorer is None:
-            logger.warning(
-                "The first metric computation step might be a bit longer as we need to download the model."
-            )
+            logger.warning("The first metric computation step might be a bit longer as we need to download the model.")
             # We only initialize on first compute
             self.bert_scorer = BERTScorer(
-                model_type="microsoft/deberta-large-mnli",
-                lang="en",
-                rescale_with_baseline=True,
-                num_layers=9,
+                model_type="microsoft/deberta-large-mnli", lang="en", rescale_with_baseline=True, num_layers=9
             )
         golds = as_list(golds)
         predictions = as_list(predictions)
@@ -690,11 +655,7 @@ def compute(
             predictions = [self.normalize_pred(p) for p in predictions]
 
         p, r, f = self.bert_scorer.score(predictions, golds)
-        return {
-            "BERTScore-P": p[0].item(),
-            "BERTScore-R": r[0].item(),
-            "BERTScore-F": f[0].item(),
-        }
+        return {"BERTScore-P": p[0].item(), "BERTScore-R": r[0].item(), "BERTScore-F": f[0].item()}
 
 
 class Extractiveness(SampleLevelComputation):
@@ -721,9 +682,7 @@ def __init__(
         self.input_column = input_column
         self.language = language
 
-    def compute(
-        self, doc: Doc, model_response: ModelResponse, **kwargs
-    ) -> dict[str, float]:
+    def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict[str, float]:
         """Compute the extractiveness of the predictions.
 
         This method calculates coverage, density, and compression scores for a single
@@ -776,9 +735,7 @@ def __init__(
         self.normalize_pred = normalize_pred
         self.input_column = input_column
 
-    def compute(
-        self, doc: Doc, model_response: ModelResponse, **kwargs
-    ) -> dict[str, float]:
+    def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict[str, float]:
         """Compute the faithfulness of the predictions.
 
         The SummaCZS (Summary Content Zero-Shot) model is used with configurable granularity and model variation.
@@ -822,9 +779,7 @@ def tokenizer(self):
     @property
     def model(self):
         if self._model is None:
-            self._model = AutoModelForSequenceClassification.from_pretrained(
-                "Elron/bleurt-tiny-512"
-            )
+            self._model = AutoModelForSequenceClassification.from_pretrained("Elron/bleurt-tiny-512")
             self._model.eval()
         return self._model
 
@@ -843,9 +798,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
         golds = doc.get_golds()
         if len(predictions) == 1:
             predictions = predictions * len(golds)
-        scores = self.model(**self.tokenizer(golds, predictions, return_tensors="pt"))[
-            0
-        ].squeeze()
+        scores = self.model(**self.tokenizer(golds, predictions, return_tensors="pt"))[0].squeeze()
         return scores.item()
 
 
@@ -888,9 +841,7 @@ def _bleu_score(self, gold: list[str], pred: str):
             float: Score over the current prediction.
         """
         weights = [1 if ix == self.n_gram else 0 for ix in range(1, 5)]
-        return sentence_bleu(
-            [word_tokenize(g) for g in gold], word_tokenize(pred), weights=weights
-        )
+        return sentence_bleu([word_tokenize(g) for g in gold], word_tokenize(pred), weights=weights)
 
 
 class StringDistance(SampleLevelComputation):
@@ -905,11 +856,7 @@ def __init__(
             metric_types (list[str] | str): Can be one or any of `longest_common_prefix_length`, `edit_distance` or `edit_similarity`.
             strip_prediction (bool, optional): Whether to strip the prediction. Defaults to True.
         """
-        allowed_values = [
-            "longest_common_prefix_length",
-            "edit_distance",
-            "edit_similarity",
-        ]
+        allowed_values = ["longest_common_prefix_length", "edit_distance", "edit_similarity"]
         metric_types = as_list(metric_types)
         if any(metric_type not in allowed_values for metric_type in metric_types):
             raise ValueError(
@@ -917,11 +864,7 @@ def __init__(
             )
         self.metric_types = metric_types
         self.strip_prediction = strip_prediction
-        self.sample_aggregations = {
-            "longest_common_prefix_length": max,
-            "edit_distance": min,
-            "edit_similarity": max,
-        }
+        self.sample_aggregations = {"longest_common_prefix_length": max, "edit_distance": min, "edit_similarity": max}
 
     def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
         """Computes all the requested metrics on the golds and prediction.
@@ -954,25 +897,17 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
             truncated_reference = reference[: len(completion)]
 
             completion_tokens = np.array(TreebankWordTokenizer().tokenize(completion))
-            truncated_reference_tokens = np.array(
-                TreebankWordTokenizer().tokenize(truncated_reference)
-            )
+            truncated_reference_tokens = np.array(TreebankWordTokenizer().tokenize(truncated_reference))
 
             if "edit_distance" in self.metric_types:
-                result["edit_distance"].append(
-                    edit_distance(s1=completion_tokens, s2=truncated_reference_tokens)
-                )
+                result["edit_distance"].append(edit_distance(s1=completion_tokens, s2=truncated_reference_tokens))
             if "edit_similarity" in self.metric_types:
                 result["edit_similarity"].append(
-                    self.edit_similarity(
-                        s1=completion_tokens, s2=truncated_reference_tokens
-                    )
+                    self.edit_similarity(s1=completion_tokens, s2=truncated_reference_tokens)
                 )
             if "longest_common_prefix_length" in self.metric_types:
                 result["longest_common_prefix_length"].append(
-                    self.longest_common_prefix_length(
-                        s1=completion_tokens, s2=truncated_reference_tokens
-                    )
+                    self.longest_common_prefix_length(s1=completion_tokens, s2=truncated_reference_tokens)
                 )
 
         final_result = {}
@@ -1005,22 +940,14 @@ def edit_similarity(self, s1, s2):
 
 
 class JudgeLLM(SampleLevelComputation):
-    available_models_openai = [
-        "gpt-3.5-turbo",
-        "gpt-4o",
-        "gpt-4-turbo",
-        "gpt-4",
-        "gpt-4o-2024-08-06",
-    ]
+    available_models_openai = ["gpt-3.5-turbo", "gpt-4o", "gpt-4-turbo", "gpt-4", "gpt-4o-2024-08-06"]
 
     def __init__(
         self,
         judge_model_name: str,
         template: Callable,
         process_judge_response: Callable,
-        judge_backend: Literal[
-            "litellm", "openai", "transformers", "vllm", "tgi", "inference-providers"
-        ],
+        judge_backend: Literal["litellm", "openai", "transformers", "vllm", "tgi", "inference-providers"],
         short_judge_name: str | None = None,
         response_format: BaseModel | None = None,
         url: str | None = None,
@@ -1029,16 +956,12 @@ def __init__(
         max_tokens: int | None = None,
         backend_options: dict | None = None,
     ) -> None:
-        logger.debug(
-            f"Initializing JudgeLLM with backend: {judge_backend}, model: {judge_model_name}"
-        )
+        logger.debug(f"Initializing JudgeLLM with backend: {judge_backend}, model: {judge_model_name}")
 
         match judge_backend:
             case "openai":
                 if judge_model_name not in self.available_models_openai:
-                    raise ValueError(
-                        f"{judge_model_name} not in available models for llm as a judge metric"
-                    )
+                    raise ValueError(f"{judge_model_name} not in available models for llm as a judge metric")
                 api_key = api_key or os.getenv("OPENAI_API_KEY")
                 logger.debug("Using OpenAI backend for llm as a judge metric")
 
@@ -1060,14 +983,10 @@ def __init__(
                 api = HfApi()
                 models = api.list_models(model_name=judge_model_name)
                 if not models:
-                    raise ValueError(
-                        f"{judge_model_name} not found on Hugging Face Hub"
-                    )
+                    raise ValueError(f"{judge_model_name} not found on Hugging Face Hub")
 
             case _:
-                raise ValueError(
-                    f"{judge_backend} is not a valid backend for llm as a judge metric"
-                )
+                raise ValueError(f"{judge_backend} is not a valid backend for llm as a judge metric")
 
         self.short_judge_name = short_judge_name
         self.judge = JudgeLM(
@@ -1098,9 +1017,7 @@ def __init__(self):
             short_judge_name="gpt4o",
         )
 
-    def compute(
-        self, responses: list[ModelResponse], docs: list[Doc], **kwargs
-    ) -> list:
+    def compute(self, responses: list[ModelResponse], docs: list[Doc], **kwargs) -> list:
         """Compute the score of a generative task using a llm as a judge.
         The generative task can be multiturn with 2 turns max, in that case, we
         return scores for turn 1 and 2. Also returns user_prompt and judgement
@@ -1111,9 +1028,7 @@ def compute(
         golds = [formatted_doc.get_golds()[0] for formatted_doc in docs]
         predictions = [response.final_text[0] for response in responses]
 
-        scores, messages, judgements = self.judge.evaluate_answer_batch(
-            questions, predictions, options, golds
-        )
+        scores, messages, judgements = self.judge.evaluate_answer_batch(questions, predictions, options, golds)
 
         metrics = []
         for i in range(len(docs)):
@@ -1149,14 +1064,10 @@ def compute(self, model_response: list[ModelResponse], doc: list[Doc], **kwargs)
         query_context_2 = {"query": questions[1], "context": predictions[0]}
 
         score_turn_1, message_turn_1, judgement_turn_1 = self.judge.evaluate_answer(
-            question=json.dumps(query_context_1, indent=2),
-            answer=predictions[0],
-            gold=golds[0] if golds else None,
+            question=json.dumps(query_context_1, indent=2), answer=predictions[0], gold=golds[0] if golds else None
         )
         score_turn_2, message_turn_2, judgement_turn_2 = self.judge.evaluate_answer(
-            question=json.dumps(query_context_2, indent=2),
-            answer=predictions[1],
-            gold=golds[1] if golds else None,
+            question=json.dumps(query_context_2, indent=2), answer=predictions[1], gold=golds[1] if golds else None
         )
 
         return {
@@ -1179,9 +1090,7 @@ def compute(self, responses: list[ModelResponse], docs: list[Doc], **kwargs):
         golds = [doc.get_golds()[0] for doc in docs]
         predictions = [response.final_text[0] for response in responses]
 
-        scores, messages, judgements = self.judge.evaluate_answer_batch(
-            questions, predictions, options, golds
-        )
+        scores, messages, judgements = self.judge.evaluate_answer_batch(questions, predictions, options, golds)
 
         metrics = []
         for i in range(len(docs)):
@@ -1205,9 +1114,7 @@ def __init__(
         self,
         normalize: Callable | str | None = None,
         strip_strings: bool = False,
-        sample_scoring_function: (
-            Callable[[Doc, ModelResponse], float] | str | None
-        ) = None,
+        sample_scoring_function: Callable[[Doc, ModelResponse], float] | str | None = None,
     ):
         if isinstance(normalize, str):
             import lighteval.metrics.normalizations
@@ -1387,9 +1294,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
         predictions = model_response.final_text
         if self.n is None:
             self.n = len(predictions)
-            logger.warning(
-                "n undefined in the pass@k. We assume it's the same as the sample's number of predictions."
-            )
+            logger.warning("n undefined in the pass@k. We assume it's the same as the sample's number of predictions.")
         elif len(predictions) < self.n:
             logger.warning(f"Number of predictions is less than {self.n} for pass@k.")
 
@@ -1406,9 +1311,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
             new_model_response = ModelResponse(
                 text=[cur_pred],
             )
-            all_scores.append(
-                self.compute_score(doc=new_doc, model_response=new_model_response)
-            )
+            all_scores.append(self.compute_score(doc=new_doc, model_response=new_model_response))
 
         return self.pass_at_k(all_scores)
 
@@ -1556,23 +1459,14 @@ def num_samples(self):
 class TextTaggingMetric(SampleLevelComputation):
     def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
         gold_topics = [
-            g.strip().lower()
-            for g in doc.specific.get("gold_topics", [])
-            if isinstance(g, str) and g.strip()
+            g.strip().lower() for g in doc.specific.get("gold_topics", []) if isinstance(g, str) and g.strip()
         ]
         if not gold_topics:
             return 0.0
-        if (
-            hasattr(model_response, "text_post_processed")
-            and model_response.text_post_processed
-        ):
+        if hasattr(model_response, "text_post_processed") and model_response.text_post_processed:
             pred_text = model_response.text_post_processed[0]
         elif hasattr(model_response, "text") and model_response.text:
-            pred_text = (
-                model_response.text[0]
-                if isinstance(model_response.text, list)
-                else model_response.text
-            )
+            pred_text = model_response.text[0] if isinstance(model_response.text, list) else model_response.text
         else:
             pred_text = str(model_response)
         pred_tags = [p.strip().lower() for p in pred_text.split(",") if p.strip()]
@@ -1604,8 +1498,4 @@ def ngrams(s, n):
         overlap = len(A & B)
         precision = overlap / len(B)
         recall = overlap / len(A)
-        return (
-            2 * precision * recall / (precision + recall)
-            if (precision + recall)
-            else 0.0
-        )
+        return 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
diff --git a/src/lighteval/metrics/utils/armenian_eval_utils.py b/src/lighteval/metrics/utils/armenian_eval_utils.py
index 264944283..a686f5365 100644
--- a/src/lighteval/metrics/utils/armenian_eval_utils.py
+++ b/src/lighteval/metrics/utils/armenian_eval_utils.py
@@ -1,21 +1,46 @@
-import re
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
 import ast
+import re
+
 
 def extract_answer_for_numeric_choices(text):
     pattern = r"answer is \(?([1-7])\)?"
     match = re.search(pattern, text)
     if match:
-        return match.group(1)  
+        return match.group(1)
     else:
-        return None 
-    
+        return None
+
+
 def extract_again_for_numeric_choices(text):
-    match = re.search(r'.*[aA]nswer:\s*([1-7])', text)
+    match = re.search(r".*[aA]nswer:\s*([1-7])", text)
     if match:
         return match.group(1)
     else:
         return None
 
+
 def extract_final_for_numeric_choices(text):
     pattern = r"\b[1-7]\b(?!.*\b[1-7]\b)"
     match = re.search(pattern, text, re.DOTALL)
@@ -24,21 +49,24 @@ def extract_final_for_numeric_choices(text):
     else:
         return None
 
+
 def extract_answer_for_numeric_answer(text):
     pattern = r"answer is \(?(\d+)\)?"
     match = re.search(pattern, text)
     if match:
-        return match.group(1)  
+        return match.group(1)
     else:
         return None
 
+
 def extract_again_for_numeric_answer(text):
-    match = re.search(r'.*[aA]nswer:\s*(\d+)', text)
+    match = re.search(r".*[aA]nswer:\s*(\d+)", text)
     if match:
         return match.group(1)
     else:
         return None
 
+
 def extract_final_for_numeric_answer(text):
     pattern = r"\b(\d+)\b(?!.*\b(\d+)\b)"
     match = re.search(pattern, text, re.DOTALL)
@@ -47,21 +75,24 @@ def extract_final_for_numeric_answer(text):
     else:
         return None
 
+
 def extract_answer_for_letter_choices(text):
     pattern = r"answer is \(?([A-J])\)?"
     match = re.search(pattern, text, re.IGNORECASE)
     if match:
-        return match.group(1)  
+        return match.group(1)
     else:
-        return None 
+        return None
+
 
 def extract_again_for_letter_choices(text):
-    match = re.search(r'.*[aA]nswer:\s*([A-J])', text, re.IGNORECASE)
+    match = re.search(r".*[aA]nswer:\s*([A-J])", text, re.IGNORECASE)
     if match:
         return match.group(1)
     else:
         return None
 
+
 def extract_final_for_letter_choices(text):
     pattern = r"\b[A-J]\b(?!.*\b[A-J]\b)"
     match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
@@ -70,27 +101,30 @@ def extract_final_for_letter_choices(text):
     else:
         return None
 
+
 def extract_correct_answers_list(text):
     pattern = r"\[([^\]]+)\]"
-    
+
     match = re.search(pattern, text)
     if match:
-        extracted_list_str = "[" + match.group(1) + "]"  
+        extracted_list_str = "[" + match.group(1) + "]"
         try:
             extracted_list = ast.literal_eval(extracted_list_str)
             return extracted_list
-        except (ValueError, SyntaxError) as e:
+        except (ValueError, SyntaxError):
             return []
     else:
         return []
 
+
 def extract_correct_answers_dict(text):
-    pattern = r'\{.*?\}'  
+    pattern = r"\{.*?\}"
     match = re.search(pattern, text, re.DOTALL)
     if match:
         try:
-            extracted_dict = ast.literal_eval(match.group())  
-            return list(extracted_dict.values()) 
-        except: return []
+            extracted_dict = ast.literal_eval(match.group())
+            return list(extracted_dict.values())
+        except Exception:
+            return []
     else:
         return []
diff --git a/src/lighteval/models/transformers/transformers_model.py b/src/lighteval/models/transformers/transformers_model.py
index c36138e3b..293a50f0c 100644
--- a/src/lighteval/models/transformers/transformers_model.py
+++ b/src/lighteval/models/transformers/transformers_model.py
@@ -697,9 +697,7 @@ def _padded_greedy_until(
                     # NOTE: we are assuming all items in a batch behave similarly (same
                     # stop_tokens and max_tokens genrated) which is not necessarily
                     # the case! Because of that we only use batch size of 1
-                    stop_tokens = [self.tokenizer.eos_token] + list(
-                        batch[0].stop_sequences
-                    )
+                    stop_tokens = [self.tokenizer.eos_token] + list(batch[0].stop_sequences)
 
                 max_new_tokens = batch[0].generation_size
                 num_samples = batch[0].num_samples
diff --git a/src/lighteval/tasks/multilingual/tasks/armenian.py b/src/lighteval/tasks/multilingual/tasks/armenian.py
index 376f3472d..64aa09ca1 100644
--- a/src/lighteval/tasks/multilingual/tasks/armenian.py
+++ b/src/lighteval/tasks/multilingual/tasks/armenian.py
@@ -1,22 +1,47 @@
-import numpy as np
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
 import json
 import re
-from lighteval.tasks.lighteval_task import LightevalTaskConfig, Doc
-from lighteval.metrics.metrics import Metrics
-from lighteval.models.model_output import ModelResponse
-from lighteval.metrics.utils.metric_utils import (
-    SampleLevelMetric,
-    SamplingMethod,
-    SampleLevelComputation,
-)
+
+import numpy as np
+
 from lighteval.metrics.armenian_metrics import (
-    pos_metric,
-    ner_span_metric,
     armenian_exam_metric,
     armenian_mcqa_metric,
-    bert_score_arm,
     armenian_mmlu_pro_metric,
+    bert_score_arm,
+    ner_span_metric,
+    pos_metric,
+)
+from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.utils.metric_utils import (
+    SampleLevelComputation,
+    SampleLevelMetric,
+    SamplingMethod,
 )
+from lighteval.models.model_output import ModelResponse
+from lighteval.tasks.lighteval_task import Doc, LightevalTaskConfig
+
 
 prompt_language = "hy"
 SIB200_LABEL_MAP = {
@@ -42,14 +67,12 @@
 
 PROMPTS_SIB200 = {
     "instruction": {
-        "hy": (
-            "Տրված տեքստի համար ընտրիր այն թեման, որը լավագույնս նկարագրում է այն։\n"
-        ),
+        "hy": ("Տրված տեքստի համար ընտրիր այն թեման, որը լավագույնս նկարագրում է այն։\n"),
         "en": ("For the given text, choose the topic that best describes it.\n"),
     },
     "query": {
-        "hy": ("Տեքստ\n{text}\n\n" "Հնարավոր թեմաներ՝ {labels}"),
-        "en": ("Text: {text}\n" "Possible topics: {labels}"),
+        "hy": ("Տեքստ\n{text}\n\nՀնարավոր թեմաներ՝ {labels}"),
+        "en": ("Text: {text}\nPossible topics: {labels}"),
     },
 }
 
@@ -59,13 +82,8 @@
         "en": ("Determine the sentiment of the text.\n"),
     },
     "query": {
-        "hy": (
-            "Տեքստ\n{text}\n\n"
-            "Հնարավոր տարբերակներ՝ դրական, բացասական, չեզոք, երկիմաստ"
-        ),
-        "en": (
-            "Text: {text}\n" "Possible options: positive, negative, neutral, ambiguous."
-        ),
+        "hy": ("Տեքստ\n{text}\n\nՀնարավոր տարբերակներ՝ դրական, բացասական, չեզոք, երկիմաստ"),
+        "en": ("Text: {text}\nPossible options: positive, negative, neutral, ambiguous."),
     },
 }
 PROMPTS_URGENCY = {
@@ -74,8 +92,8 @@
         "en": ("Determine the urgency level of the email.\n"),
     },
     "query": {
-        "hy": ("Նամակ\n{text}\n\n" "Տարբերակներ՝ բարձր, միջին, ցածր"),
-        "en": ("Email: {text}\n" "Options: high, medium, low"),
+        "hy": ("Նամակ\n{text}\n\nՏարբերակներ՝ բարձր, միջին, ցածր"),
+        "en": ("Email: {text}\nOptions: high, medium, low"),
     },
 }
 
@@ -141,14 +159,14 @@ def prompt_mmlu_pro(line, task_name=None):
 Possible Choices:""",
     },
     3: {
-        "system": """The following is multiple-option question. Each question has exactly 6 options. For each option, decide whether it is {Ճիշտ է, Սխալ է, Չգիտեմ}. Use the provided context or passage to guide your reasoning if the context exists. 
+        "system": """The following is multiple-option question. Each question has exactly 6 options. For each option, decide whether it is {Ճիշտ է, Սխալ է, Չգիտեմ}. Use the provided context or passage to guide your reasoning if the context exists.
 Think step by step and then output the answer in the format of ['Ճիշտ է', 'Սխալ է', 'Չգիտեմ', 'Ճիշտ է', 'Սխալ է', 'Ճիշտ է'].""",
         "input": """Instruction: {question}
 Context: {context}
 Options:""",
     },
     4: {
-        "system": """You are presented with a matching task. Match each item from Character Items to its corresponding item from Numeric Items. Character Items are: (Ա, Բ, Գ, ...). Numeric Items are: (1, 2, 3, ...). 
+        "system": """You are presented with a matching task. Match each item from Character Items to its corresponding item from Numeric Items. Character Items are: (Ա, Բ, Գ, ...). Numeric Items are: (1, 2, 3, ...).
 Think step by step, considering each item and its possible matches.
 Return your answer as a set of key-value pairs where the keys are the items from Character Items and the values are the corresponding items from Numeric Items.
 Ensure that the output includes only the items listed in Character Items and that the number of keys in your output matches the number of items in Character Items. For example, if Character Items are (Ա, Բ, Գ), and Numeric Items are (1, 2, 3, 4, 5) the output should be {"Ա": 2, "Բ": 1, "Գ": 5}.""",
@@ -171,14 +189,10 @@ def prompt_exam_history(line, task_name=None):
     input_prompt = PROMPTS_EXAM_HISTORY[line["task_type"]]["input"]
 
     if not line["context"]:
-        formatted_input_prompt = input_prompt.format(
-            question=line["question"], context=""
-        )
+        formatted_input_prompt = input_prompt.format(question=line["question"], context="")
         formatted_input_prompt = formatted_input_prompt.replace("\nContext: ", "")
     else:
-        formatted_input_prompt = input_prompt.format(
-            question=line["question"], context=line["context"]
-        )
+        formatted_input_prompt = input_prompt.format(question=line["question"], context=line["context"])
 
     choice_map = "123456789"
     for i in range(len(line["choices"])):
@@ -224,20 +238,16 @@ def prompt_exam_history(line, task_name=None):
 
 
 def prompt_exam_math(line, task_name=None):
-    if not line["task_type"] in PROMPTS_EXAM_MATH:
+    if line["task_type"] not in PROMPTS_EXAM_MATH:
         return None
     system_prompt = PROMPTS_EXAM_MATH[line["task_type"]]["system"]
     input_prompt = PROMPTS_EXAM_MATH[line["task_type"]]["input"]
 
-    formatted_input_prompt = input_prompt.format(
-        task=line["task"], question=line["question"]
-    )
+    formatted_input_prompt = input_prompt.format(task=line["task"], question=line["question"])
     choice_map = "ABCDEFGHIJ"
     if line["task_type"] != 7:
         for i in range(len(line["choices"])):
-            formatted_input_prompt += "{}. {}\n".format(
-                choice_map[i], line["choices"][i]
-            )
+            formatted_input_prompt += "{}. {}\n".format(choice_map[i], line["choices"][i])
 
     formatted_input_prompt += "Output: "
 
@@ -267,7 +277,7 @@ def prompt_exam_math(line, task_name=None):
 Possible Choices:""",
     },
     3: {
-        "system": """The following are multiple-option questions (with answers). Each question has exactly 6 options. For each option, decide whether it is {Ճիշտ է, Սխալ է, Չգիտեմ}. Use the provided context or passage to guide your reasoning if the context exists. 
+        "system": """The following are multiple-option questions (with answers). Each question has exactly 6 options. For each option, decide whether it is {Ճիշտ է, Սխալ է, Չգիտեմ}. Use the provided context or passage to guide your reasoning if the context exists.
 Think step by step and then output the answer in the format of ['Ճիշտ է', 'Սխալ է', 'Չգիտեմ', 'Ճիշտ է', 'Սխալ է', 'Ճիշտ է'].""",
         "input": """Instruction: {question}
 Context: {context}
@@ -281,14 +291,10 @@ def prompt_exam_literature(line, task_name=None):
     input_prompt = PROMPTS_EXAM_LITERATURE[line["task_type"]]["input"]
 
     if not line["context"]:
-        formatted_input_prompt = input_prompt.format(
-            question=line["question"], context=""
-        )
+        formatted_input_prompt = input_prompt.format(question=line["question"], context="")
         formatted_input_prompt = formatted_input_prompt.replace("\nContext: ", "")
     else:
-        formatted_input_prompt = input_prompt.format(
-            question=line["question"], context=line["context"]
-        )
+        formatted_input_prompt = input_prompt.format(question=line["question"], context=line["context"])
 
     choice_map = "123456789"
     for i in range(len(line["choices"])):
@@ -313,13 +319,9 @@ def prompt_sib200(line, task_name=None):
     gold_index = choices.index(gold)
 
     choice_map = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-    formatted_labels = "\n".join(
-        [f"({choice_map[i]}) {opt}" for i, opt in enumerate(choices)]
-    )
+    formatted_labels = "\n".join([f"({choice_map[i]}) {opt}" for i, opt in enumerate(choices)])
 
-    query = PROMPTS_SIB200["query"][prompt_language].format(
-        labels=formatted_labels, text=line["text"]
-    )
+    query = PROMPTS_SIB200["query"][prompt_language].format(labels=formatted_labels, text=line["text"])
     instruction = PROMPTS_SIB200["instruction"][prompt_language]
 
     return Doc(
@@ -378,36 +380,36 @@ def prompt_sentiment(line, task_name=None):
 }
 
 
-def prompt_ms_marco(line, task_name=None):
+def prompt_ms_marco(line, task_name=None):  # noqa: C901
     raw = line["armenian"]
 
     question, context, answers = "", "", ""
     section = None
     buffer = []
 
-    for l in raw.splitlines():
-        if l.startswith("INPUT:"):
+    for raw_line in raw.splitlines():
+        if raw_line.startswith("INPUT:"):
             if section == "ANSWERS":
                 answers = "\n".join(buffer).strip()
             elif section == "CONTEXT":
                 context = "\n".join(buffer).strip()
             elif section == "INPUT":
                 question = "\n".join(buffer).strip()
-            section, buffer = "INPUT", [l.replace("INPUT:", "").strip()]
-        elif l.startswith("CONTEXT:"):
+            section, buffer = "INPUT", [raw_line.replace("INPUT:", "").strip()]
+        elif raw_line.startswith("CONTEXT:"):
             if section == "INPUT":
                 question = "\n".join(buffer).strip()
             elif section == "ANSWERS":
                 answers = "\n".join(buffer).strip()
-            section, buffer = "CONTEXT", [l.replace("CONTEXT:", "").strip()]
-        elif l.startswith("ANSWERS:"):
+            section, buffer = "CONTEXT", [raw_line.replace("CONTEXT:", "").strip()]
+        elif raw_line.startswith("ANSWERS:"):
             if section == "CONTEXT":
                 context = "\n".join(buffer).strip()
             elif section == "INPUT":
                 question = "\n".join(buffer).strip()
-            section, buffer = "ANSWERS", [l.replace("ANSWERS:", "").strip()]
+            section, buffer = "ANSWERS", [raw_line.replace("ANSWERS:", "").strip()]
         else:
-            buffer.append(l.strip())
+            buffer.append(raw_line.strip())
 
     if section == "INPUT":
         question = "\n".join(buffer).strip()
@@ -416,9 +418,7 @@ def prompt_ms_marco(line, task_name=None):
     elif section == "ANSWERS":
         answers = "\n".join(buffer).strip()
 
-    query = PROMPTS_QA_CONTEXT["query"][prompt_language].format(
-        context=context, question=question
-    )
+    query = PROMPTS_QA_CONTEXT["query"][prompt_language].format(context=context, question=question)
     return Doc(
         instruction=PROMPTS_QA_CONTEXT["instruction"][prompt_language],
         task_name=task_name,
@@ -432,9 +432,7 @@ def prompt_ms_marco(line, task_name=None):
 def prompt_squad(line, task_name=None):
     context = line["context"]
     question = line["question"]
-    query = PROMPTS_QA_CONTEXT["query"][prompt_language].format(
-        context=context, question=question
-    )
+    query = PROMPTS_QA_CONTEXT["query"][prompt_language].format(context=context, question=question)
     return Doc(
         task_name=task_name,
         query=query,
@@ -450,8 +448,8 @@ def prompt_squad(line, task_name=None):
         "en": "Choose the correct answer from the given options using the context.\n",
     },
     "query": {
-        "hy": ("Կոնտեքստ\n{context}\n\n" "Հարց\n{question}\n" "Ճիշտ տարբերակ:"),
-        "en": ("Context: {context}\n" "Question: {question}\n" "Correct Option:"),
+        "hy": ("Կոնտեքստ\n{context}\n\nՀարց\n{question}\nՃիշտ տարբերակ:"),
+        "en": ("Context: {context}\nQuestion: {question}\nCorrect Option:"),
     },
 }
 
@@ -461,8 +459,8 @@ def prompt_squad(line, task_name=None):
         "en": "Answer the question based on the dialogue.\n",
     },
     "query": {
-        "hy": ("Երկխոսություն\n {dialogue}\n\n" "Հարց\n{question}\n" "Ճիշտ տարբերակ:"),
-        "en": ("Dialogue:\n{dialogue}\n" "Question: {question}\n" "Correct Option:"),
+        "hy": ("Երկխոսություն\n {dialogue}\n\nՀարց\n{question}\nՃիշտ տարբերակ:"),
+        "en": ("Dialogue:\n{dialogue}\nQuestion: {question}\nCorrect Option:"),
     },
 }
 
@@ -482,9 +480,7 @@ def prompt_belebele(line, task_name=None):
     for i, opt in enumerate(choices):
         question += f"\n({choice_map[i]}) {opt}"
 
-    query = PROMPTS_CONTEXT_MCQA["query"][prompt_language].format(
-        context=passage, question=question
-    )
+    query = PROMPTS_CONTEXT_MCQA["query"][prompt_language].format(context=passage, question=question)
     instruction = PROMPTS_CONTEXT_MCQA["instruction"][prompt_language]
 
     return Doc(
@@ -507,9 +503,7 @@ def prompt_scientific(line, task_name=None):
     for i, opt in enumerate(choices):
         question += f"\n({choice_map[i]}) {opt}"
 
-    query = PROMPTS_CONTEXT_MCQA["query"][prompt_language].format(
-        context=context, question=question
-    )
+    query = PROMPTS_CONTEXT_MCQA["query"][prompt_language].format(context=context, question=question)
     instruction = PROMPTS_CONTEXT_MCQA["instruction"][prompt_language]
 
     return Doc(
@@ -537,9 +531,7 @@ def prompt_syndarin(line, task_name=None):
     for i, opt in enumerate(choices):
         question += f"\n({choice_map[i]}) {opt}"
 
-    query = PROMPTS_CONTEXT_MCQA["query"][prompt_language].format(
-        context=context, question=question
-    )
+    query = PROMPTS_CONTEXT_MCQA["query"][prompt_language].format(context=context, question=question)
     instruction = PROMPTS_CONTEXT_MCQA["instruction"][prompt_language]
 
     return Doc(
@@ -565,9 +557,7 @@ def prompt_dream(line, task_name=None):
     for i, opt in enumerate(choices):
         question += f"\n({choice_map[i]}) {opt}"
 
-    query = PROMPTS_DREAM["query"][prompt_language].format(
-        dialogue=dialogue, question=question
-    )
+    query = PROMPTS_DREAM["query"][prompt_language].format(dialogue=dialogue, question=question)
     instruction = PROMPTS_DREAM["instruction"][prompt_language]
 
     return Doc(
@@ -586,8 +576,8 @@ def prompt_dream(line, task_name=None):
         "en": "Answer the question by choosing the correct option.\n",
     },
     "query": {
-        "hy": "{question}\n" "Ճիշտ տարբերակ:",
-        "en": "{question}\n" "Correct Option:",
+        "hy": "{question}\nՃիշտ տարբերակ:",
+        "en": "{question}\nCorrect Option:",
     },
 }
 
@@ -675,8 +665,8 @@ def prompt_include(line, task_name=None):
         ),
     },
     "query": {
-        "hy": ("Տեքստ\n{text}\n\n" "Հնարավոր թեգեր՝ {tags}"),
-        "en": ("Text: {text}\n" "Possible tags: {tags}\n" ""),
+        "hy": ("Տեքստ\n{text}\n\nՀնարավոր թեգեր՝ {tags}"),
+        "en": ("Text: {text}\nPossible tags: {tags}\n"),
     },
 }
 
@@ -692,18 +682,16 @@ def prompt_include(line, task_name=None):
         ),
     },
     "query": {
-        "hy": ("Տրված բառը՝ '{form}'\n" "Խոսքի մասեր՝ {tags}\n"),
-        "en": ("Given word: '{form}'\n" "Parts of speech: {tags}\n"),
+        "hy": ("Տրված բառը՝ '{form}'\nԽոսքի մասեր՝ {tags}\n"),
+        "en": ("Given word: '{form}'\nParts of speech: {tags}\n"),
     },
 }
 
 
 def prompt_finer(line, task_name=None):
     gold = [(t, ty) for t, ty in line["gold_entities"]]
-    tag_pool = sorted(set(ty for _, ty in gold))
-    query = PROMPTS_NER["query"][prompt_language].format(
-        text=line["text"], tags=", ".join(tag_pool)
-    )
+    tag_pool = sorted({ty for _, ty in gold})
+    query = PROMPTS_NER["query"][prompt_language].format(text=line["text"], tags=", ".join(tag_pool))
     return Doc(
         instruction=PROMPTS_NER["instruction"][prompt_language],
         task_name=task_name,
@@ -750,9 +738,7 @@ def prompt_pioner(line, task_name=None):
 def prompt_ud_armtdp(line, task_name=None):
     word = line["form"]
     gold = [(word, str(line["upos_hy"]))] if line.get("upos_hy") else []
-    query = PROMPTS_UPOS["query"][prompt_language].format(
-        form=word, tags=", ".join(UPOS_TAGS)
-    )
+    query = PROMPTS_UPOS["query"][prompt_language].format(form=word, tags=", ".join(UPOS_TAGS))
     return Doc(
         instruction=PROMPTS_UPOS["instruction"][prompt_language],
         task_name=task_name,
@@ -803,8 +789,8 @@ def prompt_qa(line, task_name=None):
         "en": "Translate the given English text into Armenian and return the translated text only.\n",
     },
     "query": {
-        "hy": "Անգլերեն: {eng}\n\n" "Հայերեն:",
-        "en": "English: {eng}\n\n" "Armenian:",
+        "hy": "Անգլերեն: {eng}\n\nՀայերեն:",
+        "en": "English: {eng}\n\nArmenian:",
     },
 }
 
@@ -825,9 +811,7 @@ def prompt_email_sum(line, task_name=None):
         query=query,
         choices=[line["summary"]],
         gold_index=0,
-        instruction=PROMPTS_EMAIL_SUM["instruction"][prompt_language].format(
-            num_sents=num_sents
-        ),
+        instruction=PROMPTS_EMAIL_SUM["instruction"][prompt_language].format(num_sents=num_sents),
     )
 
 
@@ -839,9 +823,7 @@ def prompt_conv_sum(line, task_name=None):
         query=query,
         choices=[line["summary"]],
         gold_index=0,
-        instruction=PROMPTS_CONV_SUM["instruction"][prompt_language].format(
-            num_sents=num_sents
-        ),
+        instruction=PROMPTS_CONV_SUM["instruction"][prompt_language].format(num_sents=num_sents),
     )
 
 
@@ -867,7 +849,7 @@ def prompt_translation(line, task_name=None):
     )
 
 
-PUNCTUATION_CHARS = set([",", "՝", ":", "։", "`"])
+PUNCTUATION_CHARS = {",", "՝", ":", "։", "`"}
 
 PROMPTS_SPACE_FIX = {
     "instruction": {
@@ -880,8 +862,8 @@ def prompt_translation(line, task_name=None):
         ),
     },
     "query": {
-        "hy": ("Տեքստ: {corrupted}\n" "Պատասխան:"),
-        "en": ("Text: {corrupted}\n" "Answer:"),
+        "hy": ("Տեքստ: {corrupted}\nՊատասխան:"),
+        "en": ("Text: {corrupted}\nAnswer:"),
     },
 }
 
@@ -896,8 +878,8 @@ def prompt_translation(line, task_name=None):
         ),
     },
     "query": {
-        "hy": ("Տեքստ: {corrupted}\n" "Պատասխան:"),
-        "en": ("Text: {corrupted}\n" "Answer:"),
+        "hy": ("Տեքստ: {corrupted}\nՊատասխան:"),
+        "en": ("Text: {corrupted}\nAnswer:"),
     },
 }
 
@@ -909,13 +891,8 @@ def mean_corpus_level(items):
 def extract_fixed_text(model_response):
     raw_text = (
         model_response.text_post_processed[0]
-        if hasattr(model_response, "text_post_processed")
-        and model_response.text_post_processed
-        else (
-            model_response.text[0]
-            if hasattr(model_response, "text") and model_response.text
-            else ""
-        )
+        if hasattr(model_response, "text_post_processed") and model_response.text_post_processed
+        else (model_response.text[0] if hasattr(model_response, "text") and model_response.text else "")
     ).strip()
 
     match = re.search(r"\{.*?\}", raw_text, re.DOTALL)
@@ -973,9 +950,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
                 pred_words.remove(g)
                 matched += 1
                 continue
-            found_idx = next(
-                (i for i, p in enumerate(pred_words) if p.endswith(g)), None
-            )
+            found_idx = next((i for i, p in enumerate(pred_words) if p.endswith(g)), None)
             if found_idx is not None:
                 matched += 1
                 pred_words.pop(found_idx)
@@ -1044,12 +1019,8 @@ def __init__(
         evaluation_splits=None,
         hf_avail_splits=None,
     ):
-        default_eval_splits = (
-            evaluation_splits if evaluation_splits is not None else ["train"]
-        )
-        default_hf_splits = (
-            hf_avail_splits if hf_avail_splits is not None else ["train"]
-        )
+        default_eval_splits = evaluation_splits if evaluation_splits is not None else ["train"]
+        default_hf_splits = hf_avail_splits if hf_avail_splits is not None else ["train"]
 
         super().__init__(
             name=f"armenian:{short_name}",
@@ -1095,12 +1066,8 @@ def __init__(
         [punctuation_accuracy_metric],
         generation=True,
     ),
-    ArmenianEvalTask(
-        "finer", "finer", prompt_finer, [ner_span_metric], generation=True
-    ),
-    ArmenianEvalTask(
-        "pioner", "pioner", prompt_pioner, [ner_span_metric], generation=True
-    ),
+    ArmenianEvalTask("finer", "finer", prompt_finer, [ner_span_metric], generation=True),
+    ArmenianEvalTask("pioner", "pioner", prompt_pioner, [ner_span_metric], generation=True),
     ArmenianEvalTask("pos", "pos", prompt_ud_armtdp, [pos_metric], generation=True),
     ArmenianEvalTask("arak", "simpleqa", prompt_qa, [Metrics.bleu], generation=True),
     ArmenianEvalTask(
@@ -1110,9 +1077,7 @@ def __init__(
         [Metrics.bleu],
         generation=True,
     ),
-    ArmenianEvalTask(
-        "squad", "squad-in-context-qa", prompt_squad, [Metrics.bleu], generation=True
-    ),
+    ArmenianEvalTask("squad", "squad-in-context-qa", prompt_squad, [Metrics.bleu], generation=True),
     ArmenianEvalTask(
         "belebele",
         "belebele-in-context-mcqa",