From 82306173f26cb1474d79ffdc6010b48e2ff9ab97 Mon Sep 17 00:00:00 2001 From: xkszltl Date: Tue, 26 Aug 2025 19:46:33 -0700 Subject: [PATCH] Fix answer key for category 5. `data/locomo10.json` uses `adversarial_answer` instead of `answer`. --- task_eval/evaluation.py | 2 +- task_eval/gpt_utils.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/task_eval/evaluation.py b/task_eval/evaluation.py index 8f597dd..c93a6e4 100644 --- a/task_eval/evaluation.py +++ b/task_eval/evaluation.py @@ -199,7 +199,7 @@ def eval_question_answering(qas, eval_key='prediction', metric='f1'): if type(line[eval_key]) == list: answer = line['answer'] else: - answer = str(line['answer']) + answer = str(line['answer']) if line['category'] != 5 else 'Not mentioned in the conversation' if line['category'] == 3: answer = answer.split(';')[0].strip() diff --git a/task_eval/gpt_utils.py b/task_eval/gpt_utils.py index 64d65df..0bdf84b 100644 --- a/task_eval/gpt_utils.py +++ b/task_eval/gpt_utils.py @@ -245,11 +245,11 @@ def get_gpt_answers(in_data, out_data, prediction_key, args): elif qa['category'] == 5: question = qa['question'] + " Select the correct answer: (a) {} (b) {}. " if random.random() < 0.5: - question = question.format('Not mentioned in the conversation', qa['answer']) - answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']} + question = question.format('Not mentioned in the conversation', qa['adversarial_answer']) + answer = {'a': 'Not mentioned in the conversation', 'b': qa['adversarial_answer']} else: - question = question.format(qa['answer'], 'Not mentioned in the conversation') - answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']} + question = question.format(qa['adversarial_answer'], 'Not mentioned in the conversation') + answer = {'b': 'Not mentioned in the conversation', 'a': qa['adversarial_answer']} cat_5_idxs.append(len(questions)) questions.append(question)