From 2336f0acf3785e6e05c5106fbdcfdf24870e0bc8 Mon Sep 17 00:00:00 2001 From: "rui.yan" Date: Wed, 23 Apr 2025 06:36:13 +0000 Subject: [PATCH 1/2] fix bug During data processing, the use of map unintentionally introduced extra fields into the ground_truth field of the code data. These additional fields caused incorrect code sandbox selection, leading to unexpected bugs. --- .../data_preprocess/download_and_filter_data_1p5b.py | 8 ++++++++ .../data_preprocess/download_and_filter_data_32b.py | 8 ++++++++ .../data_preprocess/download_and_filter_data_7b.py | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/or1_scripts/data_preprocess/download_and_filter_data_1p5b.py b/or1_scripts/data_preprocess/download_and_filter_data_1p5b.py index 045faa2..5277bda 100644 --- a/or1_scripts/data_preprocess/download_and_filter_data_1p5b.py +++ b/or1_scripts/data_preprocess/download_and_filter_data_1p5b.py @@ -66,6 +66,14 @@ def filter_fn(example): math_data_list = [item for item in data_list if item['ability'] == 'math'] code_data_list = [item for item in data_list if item['ability'] == 'code'] + for i in range(len(code_data_list)): + new_ground_truth = {} + item = code_data_list[i]['reward_model']['ground_truth'] + for key in item: + if item[key]: + new_ground_truth[key] = item[key] + code_data_list[i]['reward_model']['ground_truth'] = new_ground_truth + local_dir = args.local_dir hdfs_dir = args.hdfs_dir os.makedirs(local_dir, exist_ok=True) diff --git a/or1_scripts/data_preprocess/download_and_filter_data_32b.py b/or1_scripts/data_preprocess/download_and_filter_data_32b.py index 4bb3541..96ca143 100644 --- a/or1_scripts/data_preprocess/download_and_filter_data_32b.py +++ b/or1_scripts/data_preprocess/download_and_filter_data_32b.py @@ -66,6 +66,14 @@ def filter_fn(example): math_data_list = [item for item in data_list if item['ability'] == 'math'] code_data_list = [item for item in data_list if item['ability'] == 'code'] + for i in range(len(code_data_list)): + new_ground_truth = {} + item = code_data_list[i]['reward_model']['ground_truth'] + for key in item: + if item[key]: + new_ground_truth[key] = item[key] + code_data_list[i]['reward_model']['ground_truth'] = new_ground_truth + local_dir = args.local_dir hdfs_dir = args.hdfs_dir os.makedirs(local_dir, exist_ok=True) diff --git a/or1_scripts/data_preprocess/download_and_filter_data_7b.py b/or1_scripts/data_preprocess/download_and_filter_data_7b.py index 2de63ff..2ff4c91 100644 --- a/or1_scripts/data_preprocess/download_and_filter_data_7b.py +++ b/or1_scripts/data_preprocess/download_and_filter_data_7b.py @@ -66,6 +66,14 @@ def filter_fn(example): math_data_list = [item for item in data_list if item['ability'] == 'math'] code_data_list = [item for item in data_list if item['ability'] == 'code'] + for i in range(len(code_data_list)): + new_ground_truth = {} + item = code_data_list[i]['reward_model']['ground_truth'] + for key in item: + if item[key]: + new_ground_truth[key] = item[key] + code_data_list[i]['reward_model']['ground_truth'] = new_ground_truth + local_dir = args.local_dir hdfs_dir = args.hdfs_dir os.makedirs(local_dir, exist_ok=True) From 683317313dc135ca3a8c41348db5972c61942552 Mon Sep 17 00:00:00 2001 From: "rui.yan" Date: Wed, 23 Apr 2025 06:51:49 +0000 Subject: [PATCH 2/2] Made `if` condition more explicit for better readability and correctness --- or1_scripts/data_preprocess/download_and_filter_data_1p5b.py | 2 +- or1_scripts/data_preprocess/download_and_filter_data_32b.py | 2 +- or1_scripts/data_preprocess/download_and_filter_data_7b.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/or1_scripts/data_preprocess/download_and_filter_data_1p5b.py b/or1_scripts/data_preprocess/download_and_filter_data_1p5b.py index 5277bda..5ab20d5 100644 --- a/or1_scripts/data_preprocess/download_and_filter_data_1p5b.py +++ b/or1_scripts/data_preprocess/download_and_filter_data_1p5b.py @@ -70,7 +70,7 @@ def filter_fn(example): new_ground_truth = {} item = code_data_list[i]['reward_model']['ground_truth'] for key in item: - if item[key]: + if item[key] is not None: new_ground_truth[key] = item[key] code_data_list[i]['reward_model']['ground_truth'] = new_ground_truth diff --git a/or1_scripts/data_preprocess/download_and_filter_data_32b.py b/or1_scripts/data_preprocess/download_and_filter_data_32b.py index 96ca143..8bcf946 100644 --- a/or1_scripts/data_preprocess/download_and_filter_data_32b.py +++ b/or1_scripts/data_preprocess/download_and_filter_data_32b.py @@ -70,7 +70,7 @@ def filter_fn(example): new_ground_truth = {} item = code_data_list[i]['reward_model']['ground_truth'] for key in item: - if item[key]: + if item[key] is not None: new_ground_truth[key] = item[key] code_data_list[i]['reward_model']['ground_truth'] = new_ground_truth diff --git a/or1_scripts/data_preprocess/download_and_filter_data_7b.py b/or1_scripts/data_preprocess/download_and_filter_data_7b.py index 2ff4c91..6b472c2 100644 --- a/or1_scripts/data_preprocess/download_and_filter_data_7b.py +++ b/or1_scripts/data_preprocess/download_and_filter_data_7b.py @@ -70,7 +70,7 @@ def filter_fn(example): new_ground_truth = {} item = code_data_list[i]['reward_model']['ground_truth'] for key in item: - if item[key]: + if item[key] is not None: new_ground_truth[key] = item[key] code_data_list[i]['reward_model']['ground_truth'] = new_ground_truth