diff --git a/task_eval/__pycache__/__init__.cpython-314.pyc b/task_eval/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..9d04550 Binary files /dev/null and b/task_eval/__pycache__/__init__.cpython-314.pyc differ diff --git a/task_eval/__pycache__/evaluation.cpython-314.pyc b/task_eval/__pycache__/evaluation.cpython-314.pyc new file mode 100644 index 0000000..ec38e4d Binary files /dev/null and b/task_eval/__pycache__/evaluation.cpython-314.pyc differ diff --git a/task_eval/claude_utils.py b/task_eval/claude_utils.py index 44f964a..3b930cf 100644 --- a/task_eval/claude_utils.py +++ b/task_eval/claude_utils.py @@ -151,13 +151,16 @@ def get_claude_answers(in_data, out_data, prediction_key, args): if qa['category'] == 2: questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.') elif qa['category'] == 5: + # Adversarial distractor is stored under 'adversarial_answer' in + # the released data; fall back to 'answer' for backward compat. + adv_answer = qa.get('adversarial_answer', qa.get('answer')) question = qa['question'] + " Select the correct answer: (a) {} (b) {}. " if random.random() < 0.5: - question = question.format('Not mentioned in the conversation', qa['answer']) - answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']} + question = question.format('Not mentioned in the conversation', adv_answer) + answer = {'a': 'Not mentioned in the conversation', 'b': adv_answer} else: - question = question.format(qa['answer'], 'Not mentioned in the conversation') - answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']} + question = question.format(adv_answer, 'Not mentioned in the conversation') + answer = {'b': 'Not mentioned in the conversation', 'a': adv_answer} cat_5_idxs.append(len(questions)) questions.append(question) diff --git a/task_eval/evaluation.py b/task_eval/evaluation.py index 8f597dd..0537e69 100644 --- a/task_eval/evaluation.py +++ b/task_eval/evaluation.py @@ -196,10 +196,14 @@ def eval_question_answering(qas, eval_key='prediction', metric='f1'): answer_lengths = [] for i, line in enumerate(qas): # line = json.loads(line) + # Category 5 (adversarial) questions store their ground truth under + # 'adversarial_answer' in the released data, not 'answer'. Fall back to + # it so scoring does not crash with KeyError on the released benchmark. + gold = line.get('answer', line.get('adversarial_answer')) if type(line[eval_key]) == list: - answer = line['answer'] + answer = gold else: - answer = str(line['answer']) + answer = str(gold) if line['category'] == 3: answer = answer.split(';')[0].strip() diff --git a/task_eval/gemini_utils.py b/task_eval/gemini_utils.py index 99204db..5ed7b6b 100644 --- a/task_eval/gemini_utils.py +++ b/task_eval/gemini_utils.py @@ -164,13 +164,16 @@ def get_gemini_answers(model, in_data, out_data, prediction_key, args): if qa['category'] == 2: questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.') elif qa['category'] == 5: + # Adversarial distractor is stored under 'adversarial_answer' in + # the released data; fall back to 'answer' for backward compat. + adv_answer = qa.get('adversarial_answer', qa.get('answer')) question = qa['question'] + " Select the correct answer: (a) {} (b) {}. " if random.random() < 0.5: - question = question.format('Not mentioned in the conversation', qa['answer']) - answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']} + question = question.format('Not mentioned in the conversation', adv_answer) + answer = {'a': 'Not mentioned in the conversation', 'b': adv_answer} else: - question = question.format(qa['answer'], 'Not mentioned in the conversation') - answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']} + question = question.format(adv_answer, 'Not mentioned in the conversation') + answer = {'b': 'Not mentioned in the conversation', 'a': adv_answer} cat_5_idxs.append(len(questions)) questions.append(question) diff --git a/task_eval/gpt_utils.py b/task_eval/gpt_utils.py index 64d65df..e59a1c2 100644 --- a/task_eval/gpt_utils.py +++ b/task_eval/gpt_utils.py @@ -243,13 +243,16 @@ def get_gpt_answers(in_data, out_data, prediction_key, args): if qa['category'] == 2: questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.') elif qa['category'] == 5: + # Adversarial distractor is stored under 'adversarial_answer' in + # the released data; fall back to 'answer' for backward compat. + adv_answer = qa.get('adversarial_answer', qa.get('answer')) question = qa['question'] + " Select the correct answer: (a) {} (b) {}. " if random.random() < 0.5: - question = question.format('Not mentioned in the conversation', qa['answer']) - answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']} + question = question.format('Not mentioned in the conversation', adv_answer) + answer = {'a': 'Not mentioned in the conversation', 'b': adv_answer} else: - question = question.format(qa['answer'], 'Not mentioned in the conversation') - answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']} + question = question.format(adv_answer, 'Not mentioned in the conversation') + answer = {'b': 'Not mentioned in the conversation', 'a': adv_answer} cat_5_idxs.append(len(questions)) questions.append(question) diff --git a/task_eval/hf_llm_utils.py b/task_eval/hf_llm_utils.py index 7d16cf6..9795514 100644 --- a/task_eval/hf_llm_utils.py +++ b/task_eval/hf_llm_utils.py @@ -252,13 +252,16 @@ def get_hf_answers(in_data, out_data, args, pipeline, model_name): if qa['category'] == 2: questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.') elif qa['category'] == 5: + # Adversarial distractor is stored under 'adversarial_answer' in + # the released data; fall back to 'answer' for backward compat. + adv_answer = qa.get('adversarial_answer', qa.get('answer')) question = qa['question'] + " (a) {} (b) {}. Select the correct answer by writing (a) or (b)." if random.random() < 0.5: - question = question.format('No information available', qa['answer']) - answer = {'a': 'No information available', 'b': qa['answer']} + question = question.format('No information available', adv_answer) + answer = {'a': 'No information available', 'b': adv_answer} else: - question = question.format(qa['answer'], 'No information available') - answer = {'b': 'No information available', 'a': qa['answer']} + question = question.format(adv_answer, 'No information available') + answer = {'b': 'No information available', 'a': adv_answer} cat_5_idxs.append(len(questions)) questions.append(question) cat_5_answers.append(answer) diff --git a/task_eval/test_cat5_eval.py b/task_eval/test_cat5_eval.py new file mode 100644 index 0000000..f1a278a --- /dev/null +++ b/task_eval/test_cat5_eval.py @@ -0,0 +1,34 @@ +"""Regression test for category-5 (adversarial) scoring. + +The released data/locomo10.json stores the ground truth for category-5 +(adversarial) questions under the key 'adversarial_answer', not 'answer'. +eval_question_answering previously read line['answer'] unconditionally and +raised KeyError on every category-5 question, crashing the whole pipeline. + +Run with: python -m task_eval.test_cat5_eval (or pytest task_eval/test_cat5_eval.py) +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from task_eval.evaluation import eval_question_answering + + +def test_cat5_scores_without_answer_key(): + # A category-5 record exactly as released: no 'answer', only + # 'adversarial_answer'. Correct model behaviour is "not mentioned". + qas = [{ + "question": "What did Caroline realize after her charity race?", + "evidence": ["D2:3"], + "category": 5, + "adversarial_answer": "self-care is important", + "model_prediction": "Not mentioned in the conversation", + }] + scores, _lens, _recall = eval_question_answering(qas, 'model_prediction') + assert scores == [1], scores + + +if __name__ == "__main__": + test_cat5_scores_without_answer_key() + print("OK: category-5 scoring works on released-format data")