Skip to content

Add Armenian language evaluation suite #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bugdaryan merged 1 commit into from
Jul 8, 2026
Merged

Add Armenian language evaluation suite #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 49 additions & 53 deletions src/lighteval/metrics/armenian_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,46 +1,59 @@
# MIT License

# Copyright (c) 2024 The HuggingFace Team

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import json
import re

import numpy as np
import torch

from lighteval.metrics.imports.bert_scorer import BERTScorer
from lighteval.metrics.utils.armenian_eval_utils import (
extract_answer_for_numeric_choices,
extract_again_for_numeric_choices,
extract_final_for_numeric_choices,
extract_answer_for_numeric_answer,
extract_again_for_letter_choices,
extract_again_for_numeric_answer,
extract_final_for_numeric_answer,
extract_again_for_numeric_choices,
extract_answer_for_letter_choices,
extract_again_for_letter_choices,
extract_final_for_letter_choices,
extract_correct_answers_list,
extract_answer_for_numeric_answer,
extract_answer_for_numeric_choices,
extract_correct_answers_dict,
extract_correct_answers_list,
extract_final_for_letter_choices,
extract_final_for_numeric_answer,
extract_final_for_numeric_choices,
)
from aenum import extend_enum
import pandas as pd
import numpy as np
import torch
import json
import re
import os

from lighteval.metrics.metrics import Metrics
from lighteval.metrics.utils.metric_utils import (
SampleLevelComputation,
SampleLevelMetric,
SamplingMethod,
SampleLevelComputation,
)
from lighteval.models.model_output import ModelResponse
from lighteval.tasks.lighteval_task import Doc
from lighteval.metrics.imports.bert_scorer import BERTScorer


class NERSpanComputation(SampleLevelComputation):
def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
gold_entities = [
(t.lower().strip(), ty.lower().strip())
for (t, ty) in doc.specific.get("gold_entities", [])
]
gold_entities = [(t.lower().strip(), ty.lower().strip()) for (t, ty) in doc.specific.get("gold_entities", [])]

if (
hasattr(model_response, "text_post_processed")
and model_response.text_post_processed
):
if hasattr(model_response, "text_post_processed") and model_response.text_post_processed:
pred_text = model_response.text_post_processed[0]
elif hasattr(model_response, "text") and model_response.text:
pred_text = model_response.text[0]
Expand All @@ -63,7 +76,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):

return correct / len(gold_entities) if gold_entities else 0.0

def parse_pred(self, pred: str):
def parse_pred(self, pred: str): # noqa: C901
pred = pred.strip()
if pred.startswith("```"):
pred = pred.strip("`").lstrip("json").strip()
Expand Down Expand Up @@ -94,9 +107,7 @@ def parse_pred(self, pred: str):
entity_text = e["entity"].strip()
entity_tag = e["tag"].strip()
if entity_text and entity_tag:
entities.append(
(entity_text.lower(), entity_tag.lower())
)
entities.append((entity_text.lower(), entity_tag.lower()))
if entities:
break
except (json.JSONDecodeError, KeyError, TypeError):
Expand All @@ -111,9 +122,7 @@ def parse_pred(self, pred: str):
entity_text = e["entity"].strip()
entity_tag = e["tag"].strip()
if entity_text and entity_tag:
entities.append(
(entity_text.lower(), entity_tag.lower())
)
entities.append((entity_text.lower(), entity_tag.lower()))
except Exception:
parts = [p for p in pred.replace(";", ",").split(",") if ":" in p]
for part in parts:
Expand Down Expand Up @@ -153,10 +162,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
if t is not None and ty is not None
]

if (
hasattr(model_response, "text_post_processed")
and model_response.text_post_processed
):
if hasattr(model_response, "text_post_processed") and model_response.text_post_processed:
pred_text = model_response.text_post_processed[0]
elif hasattr(model_response, "text") and model_response.text:
pred_text = model_response.text[0]
Expand Down Expand Up @@ -217,13 +223,8 @@ def __init__(self):
def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
pred = (
model_response.text_post_processed[0]
if hasattr(model_response, "text_post_processed")
and model_response.text_post_processed
else (
model_response.text[0]
if hasattr(model_response, "text") and model_response.text
else ""
)
if hasattr(model_response, "text_post_processed") and model_response.text_post_processed
else (model_response.text[0] if hasattr(model_response, "text") and model_response.text else "")
).strip()

golds = doc.choices or []
Expand All @@ -235,7 +236,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):


class ArmenianExamComputation(SampleLevelComputation):
def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
def compute(self, doc: Doc, model_response: ModelResponse, **kwargs): # noqa: C901

task_type = doc.specific.get("task_type")
true_answer = doc.specific.get("label")
Expand All @@ -255,9 +256,7 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs):
elif task_type == 2:
extracted_answer = extract_correct_answers_list(text)
extracted_answer = [str(i) for i in extracted_answer]
if extracted_answer is not None and set(true_answer) == set(
extracted_answer
):
if extracted_answer is not None and set(true_answer) == set(extracted_answer):
score = 0.25

elif task_type == 3:
Expand Down Expand Up @@ -352,10 +351,7 @@ def compute(self, doc, model_response, **kwargs):
if extracted_answer is not None:
break

if (
extracted_answer is not None
and str(extracted_answer).strip() == str(true_answer).strip()
):
if extracted_answer is not None and str(extracted_answer).strip() == str(true_answer).strip():
return 1.0

return 0.0
Expand Down
Loading
Loading