Fix KeyError on category-5 (adversarial) QA: read ground truth from adversarial_answer

task_eval/claude_utils.py

@@ -151,13 +151,16 @@ def get_claude_answers(in_data, out_data, prediction_key, args):
             if qa['category'] == 2:
                 questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.')
             elif qa['category'] == 5:
+                # Adversarial distractor is stored under 'adversarial_answer' in
+                # the released data; fall back to 'answer' for backward compat.
+                adv_answer = qa.get('adversarial_answer', qa.get('answer'))
                 question = qa['question'] + " Select the correct answer: (a) {} (b) {}. "
                 if random.random() < 0.5:
-                    question = question.format('Not mentioned in the conversation', qa['answer'])
-                    answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']}
+                    question = question.format('Not mentioned in the conversation', adv_answer)
+                    answer = {'a': 'Not mentioned in the conversation', 'b': adv_answer}
                 else:
-                    question = question.format(qa['answer'], 'Not mentioned in the conversation')
-                    answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']}
+                    question = question.format(adv_answer, 'Not mentioned in the conversation')
+                    answer = {'b': 'Not mentioned in the conversation', 'a': adv_answer}
 
                 cat_5_idxs.append(len(questions))
                 questions.append(question)

task_eval/evaluation.py

@@ -196,10 +196,14 @@ def eval_question_answering(qas, eval_key='prediction', metric='f1'):
     answer_lengths = []
     for i, line in enumerate(qas):
         # line = json.loads(line)
+        # Category 5 (adversarial) questions store their ground truth under
+        # 'adversarial_answer' in the released data, not 'answer'. Fall back to
+        # it so scoring does not crash with KeyError on the released benchmark.
+        gold = line.get('answer', line.get('adversarial_answer'))
         if type(line[eval_key]) == list:
-            answer = line['answer']
+            answer = gold
         else:
-            answer = str(line['answer'])
+            answer = str(gold)
         if line['category'] == 3:
             answer = answer.split(';')[0].strip()
 

task_eval/gemini_utils.py

@@ -164,13 +164,16 @@ def get_gemini_answers(model, in_data, out_data, prediction_key, args):
             if qa['category'] == 2:
                 questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.')
             elif qa['category'] == 5:
+                # Adversarial distractor is stored under 'adversarial_answer' in
+                # the released data; fall back to 'answer' for backward compat.
+                adv_answer = qa.get('adversarial_answer', qa.get('answer'))
                 question = qa['question'] + " Select the correct answer: (a) {} (b) {}. "
                 if random.random() < 0.5:
-                    question = question.format('Not mentioned in the conversation', qa['answer'])
-                    answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']}
+                    question = question.format('Not mentioned in the conversation', adv_answer)
+                    answer = {'a': 'Not mentioned in the conversation', 'b': adv_answer}
                 else:
-                    question = question.format(qa['answer'], 'Not mentioned in the conversation')
-                    answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']}
+                    question = question.format(adv_answer, 'Not mentioned in the conversation')
+                    answer = {'b': 'Not mentioned in the conversation', 'a': adv_answer}
 
                 cat_5_idxs.append(len(questions))
                 questions.append(question)

task_eval/gpt_utils.py

@@ -243,13 +243,16 @@ def get_gpt_answers(in_data, out_data, prediction_key, args):
             if qa['category'] == 2:
                 questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.')
             elif qa['category'] == 5:
+                # Adversarial distractor is stored under 'adversarial_answer' in
+                # the released data; fall back to 'answer' for backward compat.
+                adv_answer = qa.get('adversarial_answer', qa.get('answer'))
                 question = qa['question'] + " Select the correct answer: (a) {} (b) {}. "
                 if random.random() < 0.5:
-                    question = question.format('Not mentioned in the conversation', qa['answer'])
-                    answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']}
+                    question = question.format('Not mentioned in the conversation', adv_answer)
+                    answer = {'a': 'Not mentioned in the conversation', 'b': adv_answer}
                 else:
-                    question = question.format(qa['answer'], 'Not mentioned in the conversation')
-                    answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']}
+                    question = question.format(adv_answer, 'Not mentioned in the conversation')
+                    answer = {'b': 'Not mentioned in the conversation', 'a': adv_answer}
 
                 cat_5_idxs.append(len(questions))
                 questions.append(question)

task_eval/hf_llm_utils.py

@@ -252,13 +252,16 @@ def get_hf_answers(in_data, out_data, args, pipeline, model_name):
             if qa['category'] == 2:
                 questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.')
             elif qa['category'] == 5:
+                # Adversarial distractor is stored under 'adversarial_answer' in
+                # the released data; fall back to 'answer' for backward compat.
+                adv_answer = qa.get('adversarial_answer', qa.get('answer'))
                 question = qa['question'] + " (a) {} (b) {}. Select the correct answer by writing (a) or (b)."
                 if random.random() < 0.5:
-                    question = question.format('No information available', qa['answer'])
-                    answer = {'a': 'No information available', 'b': qa['answer']}
+                    question = question.format('No information available', adv_answer)
+                    answer = {'a': 'No information available', 'b': adv_answer}
                 else:
-                    question = question.format(qa['answer'], 'No information available')
-                    answer = {'b': 'No information available', 'a': qa['answer']}
+                    question = question.format(adv_answer, 'No information available')
+                    answer = {'b': 'No information available', 'a': adv_answer}
                 cat_5_idxs.append(len(questions))
                 questions.append(question)
                 cat_5_answers.append(answer)

task_eval/test_cat5_eval.py

@@ -0,0 +1,34 @@
+"""Regression test for category-5 (adversarial) scoring.
+
+The released data/locomo10.json stores the ground truth for category-5
+(adversarial) questions under the key 'adversarial_answer', not 'answer'.
+eval_question_answering previously read line['answer'] unconditionally and
+raised KeyError on every category-5 question, crashing the whole pipeline.
+
+Run with: python -m task_eval.test_cat5_eval   (or pytest task_eval/test_cat5_eval.py)
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from task_eval.evaluation import eval_question_answering
+
+
+def test_cat5_scores_without_answer_key():
+    # A category-5 record exactly as released: no 'answer', only
+    # 'adversarial_answer'. Correct model behaviour is "not mentioned".
+    qas = [{
+        "question": "What did Caroline realize after her charity race?",
+        "evidence": ["D2:3"],
+        "category": 5,
+        "adversarial_answer": "self-care is important",
+        "model_prediction": "Not mentioned in the conversation",
+    }]
+    scores, _lens, _recall = eval_question_answering(qas, 'model_prediction')
+    assert scores == [1], scores
+
+
+if __name__ == "__main__":
+    test_cat5_scores_without_answer_key()
+    print("OK: category-5 scoring works on released-format data")