Skip to content

Feature/refcoco/+/g benchmark support #201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zhongzhouTan-coder wants to merge 8 commits into
base: master
Choose a base branch
from
+683 −0
Open

Feature/refcoco/+/g benchmark support #201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
59facd2
[feature] add refcoco support
zhongzhouTan-coder Mar 17, 2026
f003fbd
[feat] save image to local disk instead of storing in share memory
zhongzhouTan-coder Mar 17, 2026
224c731
[feature] add refcoco plus support
zhongzhouTan-coder Mar 17, 2026
8d180d5
[feature] add refcocog support
zhongzhouTan-coder Mar 17, 2026
8aaa9fa
[refactor] use the more general dir name for the saving images
zhongzhouTan-coder Mar 17, 2026
b2f1bac
[feature] add refcoco/+/g base64 support
zhongzhouTan-coder Mar 19, 2026
7d6802e
[refactor] avoid index error and type error to raise to user
zhongzhouTan-coder Mar 20, 2026
771f195
[refactor] remove unused image and question, also move the prompt to …
zhongzhouTan-coder Mar 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions ais_bench/benchmark/configs/datasets/refcoco/refcoco_gen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
from ais_bench.benchmark.openicl.icl_prompt_template import MMPromptTemplate
from ais_bench.benchmark.datasets import RefCOCODataset
from ais_bench.benchmark.datasets.refcoco import refcoco_bbox_postprocess
from ais_bench.benchmark.openicl.icl_evaluator import BBoxIoUEvaluator


refcoco_reader_cfg = dict(
input_columns=['question', 'image'],
output_column='answer'
)

refcoco_infer_cfg = dict(
prompt_template=dict(
type=MMPromptTemplate,
template=dict(
round=[
dict(role='HUMAN', prompt_mm={
'text': {
'type': 'text',
'text': 'Locate every object that matches the description "{question}" in the image. Report bbox coordinates in JSON format.'
},
'image': {'type': 'image_url', 'image_url': {'url': 'file://{image}'}},
})
]
)
),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=GenInferencer),
)

refcoco_eval_cfg = dict(
evaluator=dict(type=BBoxIoUEvaluator, iou_threshold=0.5, coord_scale=1000.0),
pred_postprocessor=dict(type=refcoco_bbox_postprocess),
)

_splits = [
'val',
'test',
'testA',
'testB',
]

refcoco_datasets = [
dict(
abbr='RefCOCO_' + split,
type=RefCOCODataset,
path='ais_bench/datasets/RefCOCO/data',
split=split,
reader_cfg=refcoco_reader_cfg,
infer_cfg=refcoco_infer_cfg,
eval_cfg=refcoco_eval_cfg,
)
for split in _splits
]
57 changes: 57 additions & 0 deletions ais_bench/benchmark/configs/datasets/refcoco/refcoco_gen_base64.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
from ais_bench.benchmark.openicl.icl_prompt_template import MMPromptTemplate
from ais_bench.benchmark.datasets import RefCOCODataset
from ais_bench.benchmark.datasets.refcoco import IMAGE_BASE64_TYPE, refcoco_bbox_postprocess
from ais_bench.benchmark.openicl.icl_evaluator import BBoxIoUEvaluator


refcoco_reader_cfg = dict(
input_columns=['question', 'image'],
output_column='answer'
)

refcoco_infer_cfg = dict(
prompt_template=dict(
type=MMPromptTemplate,
template=dict(
round=[
dict(role='HUMAN', prompt_mm={
'text': {
'type': 'text',
'text': 'Locate every object that matches the description "{question}" in the image. Report bbox coordinates in JSON format.'
},
'image': {'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,{image}'}},
})
]
)
),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=GenInferencer),
)

refcoco_eval_cfg = dict(
evaluator=dict(type=BBoxIoUEvaluator, iou_threshold=0.5, coord_scale=1000.0),
pred_postprocessor=dict(type=refcoco_bbox_postprocess),
)

_splits = [
'val',
'test',
'testA',
'testB',
]

refcoco_datasets = [
dict(
abbr='RefCOCO_base64_' + split,
type=RefCOCODataset,
path='ais_bench/datasets/RefCOCO/data',
split=split,
image_type=IMAGE_BASE64_TYPE,
reader_cfg=refcoco_reader_cfg,
infer_cfg=refcoco_infer_cfg,
eval_cfg=refcoco_eval_cfg,
)
for split in _splits
]
55 changes: 55 additions & 0 deletions ais_bench/benchmark/configs/datasets/refcoco_plus/refcoco_plus_gen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
from ais_bench.benchmark.openicl.icl_prompt_template import MMPromptTemplate
from ais_bench.benchmark.datasets import RefCOCOPlusDataset
from ais_bench.benchmark.datasets.refcoco import refcoco_bbox_postprocess
from ais_bench.benchmark.openicl.icl_evaluator import BBoxIoUEvaluator


refcoco_plus_reader_cfg = dict(
input_columns=['question', 'image'],
output_column='answer'
)

refcoco_plus_infer_cfg = dict(
prompt_template=dict(
type=MMPromptTemplate,
template=dict(
round=[
dict(role='HUMAN', prompt_mm={
'text': {
'type': 'text',
'text': 'Locate every object that matches the description "{question}" in the image. Report bbox coordinates in JSON format.'
},
'image': {'type': 'image_url', 'image_url': {'url': 'file://{image}'}},
})
]
)
),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=GenInferencer),
)

refcoco_plus_eval_cfg = dict(
evaluator=dict(type=BBoxIoUEvaluator, iou_threshold=0.5, coord_scale=1000.0),
pred_postprocessor=dict(type=refcoco_bbox_postprocess),
)

_splits = [
'val',
'testA',
'testB',
]

refcoco_plus_datasets = [
dict(
abbr='RefCOCOPlus_' + split,
type=RefCOCOPlusDataset,
path='ais_bench/datasets/RefCOCOplus/data',
split=split,
reader_cfg=refcoco_plus_reader_cfg,
infer_cfg=refcoco_plus_infer_cfg,
eval_cfg=refcoco_plus_eval_cfg,
)
for split in _splits
]
56 changes: 56 additions & 0 deletions ais_bench/benchmark/configs/datasets/refcoco_plus/refcoco_plus_gen_base64.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
from ais_bench.benchmark.openicl.icl_prompt_template import MMPromptTemplate
from ais_bench.benchmark.datasets import RefCOCOPlusDataset
from ais_bench.benchmark.datasets.refcoco import IMAGE_BASE64_TYPE, refcoco_bbox_postprocess
from ais_bench.benchmark.openicl.icl_evaluator import BBoxIoUEvaluator


refcoco_plus_reader_cfg = dict(
input_columns=['question', 'image'],
output_column='answer'
)

refcoco_plus_infer_cfg = dict(
prompt_template=dict(
type=MMPromptTemplate,
template=dict(
round=[
dict(role='HUMAN', prompt_mm={
'text': {
'type': 'text',
'text': 'Locate every object that matches the description "{question}" in the image. Report bbox coordinates in JSON format.'
},
'image': {'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,{image}'}},
})
]
)
),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=GenInferencer),
)

refcoco_plus_eval_cfg = dict(
evaluator=dict(type=BBoxIoUEvaluator, iou_threshold=0.5, coord_scale=1000.0),
pred_postprocessor=dict(type=refcoco_bbox_postprocess),
)

_splits = [
'val',
'testA',
'testB',
]

refcoco_plus_datasets = [
dict(
abbr='RefCOCOPlus_base64_' + split,
type=RefCOCOPlusDataset,
path='ais_bench/datasets/RefCOCOplus/data',
split=split,
image_type=IMAGE_BASE64_TYPE,
reader_cfg=refcoco_plus_reader_cfg,
infer_cfg=refcoco_plus_infer_cfg,
eval_cfg=refcoco_plus_eval_cfg,
)
for split in _splits
]
54 changes: 54 additions & 0 deletions ais_bench/benchmark/configs/datasets/refcocog/refcocog_gen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
from ais_bench.benchmark.openicl.icl_prompt_template import MMPromptTemplate
from ais_bench.benchmark.datasets import RefCOCOgDataset
from ais_bench.benchmark.datasets.refcoco import refcoco_bbox_postprocess
from ais_bench.benchmark.openicl.icl_evaluator import BBoxIoUEvaluator


refcocog_reader_cfg = dict(
input_columns=['question', 'image'],
output_column='answer'
)

refcocog_infer_cfg = dict(
prompt_template=dict(
type=MMPromptTemplate,
template=dict(
round=[
dict(role='HUMAN', prompt_mm={
'text': {
'type': 'text',
'text': 'Locate every object that matches the description "{question}" in the image. Report bbox coordinates in JSON format.'
},
'image': {'type': 'image_url', 'image_url': {'url': 'file://{image}'}},
})
]
)
),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=GenInferencer),
)

refcocog_eval_cfg = dict(
evaluator=dict(type=BBoxIoUEvaluator, iou_threshold=0.5, coord_scale=1000.0),
pred_postprocessor=dict(type=refcoco_bbox_postprocess),
)

_splits = [
'val',
'test',
]

refcocog_datasets = [
dict(
abbr='RefCOCOg_' + split,
type=RefCOCOgDataset,
path='ais_bench/datasets/RefCOCOg/data',
split=split,
reader_cfg=refcocog_reader_cfg,
infer_cfg=refcocog_infer_cfg,
eval_cfg=refcocog_eval_cfg,
)
for split in _splits
]
55 changes: 55 additions & 0 deletions ais_bench/benchmark/configs/datasets/refcocog/refcocog_gen_base64.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
from ais_bench.benchmark.openicl.icl_prompt_template import MMPromptTemplate
from ais_bench.benchmark.datasets import RefCOCOgDataset
from ais_bench.benchmark.datasets.refcoco import IMAGE_BASE64_TYPE, refcoco_bbox_postprocess
from ais_bench.benchmark.openicl.icl_evaluator import BBoxIoUEvaluator


refcocog_reader_cfg = dict(
input_columns=['question', 'image'],
output_column='answer'
)

refcocog_infer_cfg = dict(
prompt_template=dict(
type=MMPromptTemplate,
template=dict(
round=[
dict(role='HUMAN', prompt_mm={
'text': {
'type': 'text',
'text': 'Locate every object that matches the description "{question}" in the image. Report bbox coordinates in JSON format.'
},
'image': {'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,{image}'}},
})
]
)
),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=GenInferencer),
)

refcocog_eval_cfg = dict(
evaluator=dict(type=BBoxIoUEvaluator, iou_threshold=0.5, coord_scale=1000.0),
pred_postprocessor=dict(type=refcoco_bbox_postprocess),
)

_splits = [
'val',
'test',
]

refcocog_datasets = [
dict(
abbr='RefCOCOg_base64_' + split,
type=RefCOCOgDataset,
path='ais_bench/datasets/RefCOCOg/data',
split=split,
image_type=IMAGE_BASE64_TYPE,
reader_cfg=refcocog_reader_cfg,
infer_cfg=refcocog_infer_cfg,
eval_cfg=refcocog_eval_cfg,
)
for split in _splits
]
1 change: 1 addition & 0 deletions ais_bench/benchmark/datasets/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,4 @@
from ais_bench.benchmark.datasets.mmstar import * # noqa: F401, F403
from ais_bench.benchmark.datasets.dapo_math import * # noqa: F401, F403
from ais_bench.benchmark.datasets.mooncake_trace import * # noqa: F401, F403
from ais_bench.benchmark.datasets.refcoco import * # noqa: F401, F403
9 changes: 9 additions & 0 deletions ais_bench/benchmark/datasets/refcoco/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from ais_bench.benchmark.datasets.refcoco.refcoco import ( # noqa: F401
IMAGE_BASE64_TYPE,
IMAGE_PATH_TYPE,
TEMP_IMAGE_STORE_DIR,
RefCOCODataset,
refcoco_bbox_postprocess,
)
from ais_bench.benchmark.datasets.refcoco.refcoco_g import RefCOCOgDataset # noqa: F401
from ais_bench.benchmark.datasets.refcoco.refcoco_plus import RefCOCOPlusDataset # noqa: F401
Loading
Loading