[Python] Language Model Evaluation with Custom Task - Hugging Face Lighteval

I am creating a benchmark to evaluate a language model. First, I generated the dataset that I'm gonna prompt the Language model with. Subsequently, I tried to evaluate any tiny language model just to get a end-to-end small functioning app. I followed the instruction on the Hugging Face Lighteval library docs but I keep getting error when trying to run the evaluation code. given that i am using Python 3.11 / tried 3.10 and the same happens.

error: py .\run_evaluation.py Traceback (most recent call last): File "C:\Users\Mahmoud\Desktop\codings\benchmark-eval-llm\run_evaluation.py", line 48, in main() File "C:\Users\Mahmoud\Desktop\codings\benchmark-eval-llm\run_evaluation.py", line 21, in main pipeline_params = PipelineParameters( ^^^^^^^^^^^^^^^^^^^ TypeError: PipelineParameters.init() got an unexpected keyword argument 'custom_task_directory'

the custom task should be Cross-Lingual Synonym Identification

Here are all the pieces of code with the project structure until now.

-- benchmark-ll

• evaluation/
• -----custom_csi_task.py
• csi_benchmark_advanced.jsonl
• run_evaluation.py

custom_csi_task.py


import numpy as np
from lighteval.tasks.lighteval_task import LightevalTask
from lighteval.tasks.requests import Doc

class CustomCSI(LightevalTask):
def doc_to_text(self, doc: Doc) -> str:
return doc["question"]

def doc_to_target(self, doc: Doc) -> int:
return doc["choices"].index(doc["answer"])

def construct_requests(self, doc: Doc, ctx: str) -> list:
from lighteval.tasks.requests import Request
return [Request(request_type="loglikelihood", args=(ctx, " " + choice)) for choice in doc["choices"]]

def process_results(self, doc: Doc, results: list) -> dict:
prediction_index = np.argmax(results)
ground_truth_index = self.doc_to_target(doc)
return {"acc": 1 if prediction_index == ground_truth_index else 0}

# Register this task via callable
def custom_csi():
return CustomCSI()



run_evaluation.py


import lighteval
from lighteval.logging.evaluation_tracker import EvaluationTracker
from lighteval.models.vllm.vllm_model import VLLMModelConfig
from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters
from lighteval.utils.imports import is_accelerate_available

if is_accelerate_available():
from datetime import timedelta
from accelerate import Accelerator, InitProcessGroupKwargs
accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))])
else:
accelerator = None

def main():
evaluation_tracker = EvaluationTracker(
output_dir="./results",
save_details=True,

)

pipeline_params = PipelineParameters(
launcher_type=ParallelismManager.ACCELERATE,
custom_task_directory='evaluation', # if using a custom task
# Remove the 2 parameters below once your configuration is tested

)

model_config = VLLMModelConfig(
model_name="HuggingFaceH4/zephyr-7b-beta",
dtype="float16",
use_chat_template=True,
)

task = "helm|mmlu|5|1"

pipeline = Pipeline(
# tasks=task,
pipeline_parameters=pipeline_params,
evaluation_tracker=evaluation_tracker,
model_config=model_config,
)

pipeline.evaluate()
pipeline.save_and_push_results()
pipeline.show_results()

if __name__ == "__main__":
main()



csi_benchmark_advanced.jsonl




{"task_id": "CSI-EN-DE-001", "task_type": "cross_lingual_synonym_identification", "source_word": "house", "source_lang": "EN", "target_lang": "DE", "question": "Which word has the same meaning as 'house' in German?", "choices": ["Torbau", "häusliches Arbeitszimmer", "Haus", "Dachboden"], "answer": "Haus"}
{"task_id": "CSI-EN-FR-002", "task_type": "cross_lingual_synonym_identification", "source_word": "house", "source_lang": "EN", "target_lang": "FR", "question": "Which word has the same meaning as 'house' in French?", "choices": ["porche", "galetas", "maison", "bibliothèque"], "answer": "maison"}
{"task_id": "CSI-EN-DE-003", "task_type": "cross_lingual_synonym_identification", "source_word": "water", "source_lang": "EN", "target_lang": "DE", "question": "Which word has the same meaning as 'water' in German?", "choices": ["Flaschenwasser", "Eiswasser", "Trinkwasser", "Sodawasser"], "answer": "Trinkwasser"}
{"task_id": "CSI-EN-FR-004", "task_type": "cross_lingual_synonym_identification", "source_word": "water", "source_lang": "EN", "target_lang": "FR", "question": "Which word has the same meaning as 'water' in French?", "choices": ["eau potable", "eau gazeuse", "eau en bouteille", "l'eau glacée"], "answer": "eau potable"}
{"task_id": "CSI-EN-DE-005", "task_type": "cross_lingual_synonym_identification", "source_word": "sun", "source_lang": "EN", "target_lang": "DE", "question": "Which word has the same meaning as 'sun' in German?", "choices": ["ruhetag", "Sonntag", "Eins", "Weekend"], "answer": "Sonntag"}
{"task_id": "CSI-EN-FR-006", "task_type": "cross_lingual_synonym_identification", "source_word": "sun", "source_lang": "EN", "target_lang": "FR", "question": "Which word has the same meaning as 'sun' in French?", "choices": ["fin de semaine", "un", "jour de repos", "dimanche"], "answer": "dimanche"}
{"task_id": "CSI-EN-DE-007", "task_type": "cross_lingual_synonym_identification", "source_word": "tree", "source_lang": "EN", "target_lang": "DE", "question": "Which word has the same meaning as 'tree' in German?", "choices": ["Kernholz", "Wald", "Splintholz", "Baum"], "answer": "Baum"}
{"task_id": "CSI-EN-FR-008", "task_type": "cross_lingual_synonym_identification", "source_word": "tree", "source_lang": "EN", "target_lang": "FR", "question": "Which word has the same meaning as 'tree' in French?", "choices": ["aubier", "bois", "arbre", "duramen"], "answer": "arbre"}
{"task_id": "CSI-EN-FR-009", "task_type": "cross_lingual_synonym_identification", "source_word": "eat", "source_lang": "EN", "target_lang": "FR", "question": "Which word has the same meaning as 'eat' in French?", "choices": ["essuyer", "consommer", "petit", "occuper"], "answer": "consommer"}

Continue reading...