From b5b31ff9ad144145dd3405becbaa0a9d7079295e Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Thu, 10 Jul 2025 09:14:18 +0800 Subject: [PATCH 1/3] Add Gemini 2.5 support and robust answer key handling Updated Gemini integration to use the new google-genai SDK and support Gemini 2.5 models. Refactored answer extraction logic across evaluation, Claude, GPT, Gemini, and HF utils to handle both 'answer' and 'adversarial_answer' keys, improving robustness to missing fields. Cleaned up requirements.txt, added .gitignore, and removed compiled Python cache files. --- .gitignore | 5 + global_methods.py | 23 +- requirements.txt | 540 ++++++++---------- scripts/evaluate_gemini.sh | 2 +- .../__pycache__/__init__.cpython-310.pyc | Bin 152 -> 0 bytes task_eval/__pycache__/__init__.cpython-39.pyc | Bin 145 -> 0 bytes .../__pycache__/claude_utils.cpython-39.pyc | Bin 5681 -> 0 bytes task_eval/__pycache__/dpr_qa.cpython-39.pyc | Bin 22247 -> 0 bytes .../__pycache__/evaluation.cpython-310.pyc | Bin 9385 -> 0 bytes .../__pycache__/evaluation.cpython-39.pyc | Bin 9338 -> 0 bytes .../evaluation_stats.cpython-310.pyc | Bin 8030 -> 0 bytes .../evaluation_stats.cpython-39.pyc | Bin 5552 -> 0 bytes .../__pycache__/gemini_utils.cpython-39.pyc | Bin 6067 -> 0 bytes .../__pycache__/gpt_utils.cpython-39.pyc | Bin 10139 -> 0 bytes .../__pycache__/hf_llm_utils.cpython-39.pyc | Bin 8763 -> 0 bytes .../__pycache__/rag_utils.cpython-39.pyc | Bin 4956 -> 0 bytes task_eval/claude_utils.py | 18 +- task_eval/evaluate_qa.py | 15 +- task_eval/evaluation.py | 18 +- task_eval/gemini_utils.py | 52 +- task_eval/gpt_utils.py | 18 +- task_eval/hf_llm_utils.py | 18 +- 22 files changed, 344 insertions(+), 365 deletions(-) create mode 100644 .gitignore delete mode 100644 task_eval/__pycache__/__init__.cpython-310.pyc delete mode 100644 task_eval/__pycache__/__init__.cpython-39.pyc delete mode 100644 task_eval/__pycache__/claude_utils.cpython-39.pyc delete mode 100644 task_eval/__pycache__/dpr_qa.cpython-39.pyc delete mode 100644 task_eval/__pycache__/evaluation.cpython-310.pyc delete mode 100644 task_eval/__pycache__/evaluation.cpython-39.pyc delete mode 100644 task_eval/__pycache__/evaluation_stats.cpython-310.pyc delete mode 100644 task_eval/__pycache__/evaluation_stats.cpython-39.pyc delete mode 100644 task_eval/__pycache__/gemini_utils.cpython-39.pyc delete mode 100644 task_eval/__pycache__/gpt_utils.cpython-39.pyc delete mode 100644 task_eval/__pycache__/hf_llm_utils.cpython-39.pyc delete mode 100644 task_eval/__pycache__/rag_utils.cpython-39.pyc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..955c04d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +**/.DS_Store +/.venv +**/__pycache__ +**.pyc + diff --git a/global_methods.py b/global_methods.py index 7ada78d..2b008dd 100644 --- a/global_methods.py +++ b/global_methods.py @@ -5,7 +5,7 @@ import sys import os -import google.generativeai as genai +from google import genai from anthropic import Anthropic @@ -17,9 +17,18 @@ def set_anthropic_key(): pass def set_gemini_key(): + # This is no longer needed with the new SDK + # The client will automatically use the GEMINI_API_KEY or GOOGLE_API_KEY environment variable + pass - # Or use `os.getenv('GOOGLE_API_KEY')` to fetch an environment variable. - genai.configure(api_key=os.environ['GOOGLE_API_KEY']) +def get_gemini_client(): + # Get API key from environment variables + api_key = os.environ.get('GEMINI_API_KEY') or os.environ.get('GOOGLE_API_KEY') + if not api_key: + raise ValueError("Please set GEMINI_API_KEY or GOOGLE_API_KEY environment variable") + + # Create and return the client + return genai.Client(api_key=api_key) def set_openai_key(): openai.api_key = os.environ['OPENAI_API_KEY'] @@ -79,10 +88,12 @@ def run_claude(query, max_new_tokens, model_name): return message.content[0].text -def run_gemini(model, content: str, max_tokens: int = 0): - +def run_gemini(client, model_name: str, content: str, max_tokens: int = 0): try: - response = model.generate_content(content) + response = client.models.generate_content( + model=model_name, + contents=content + ) return response.text except Exception as e: print(f'{type(e).__name__}: {e}') diff --git a/requirements.txt b/requirements.txt index dcda28d..37970ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,316 +1,224 @@ -# This file may be used to create an environment using: -# $ conda create --name --file -# platform: linux-64 -_libgcc_mutex=0.1=main -_openmp_mutex=5.1=1_gnu -accelerate=0.24.1=pypi_0 -aiofiles=23.2.1=pypi_0 -aiohttp=3.8.4=pypi_0 -aiosignal=1.3.1=pypi_0 -altair=5.1.2=pypi_0 -anthropic=0.32.0=pypi_0 -antlr4-python3-runtime=4.9.3=pypi_0 -anyio=3.7.1=pypi_0 -appdirs=1.4.4=pypi_0 -argon2-cffi=23.1.0=pypi_0 -argon2-cffi-bindings=21.2.0=pypi_0 -arrow=1.3.0=pypi_0 -asttokens=2.4.1=pypi_0 -async-lru=2.0.4=pypi_0 -async-timeout=4.0.2=pypi_0 -attrs=22.2.0=pypi_0 -babel=2.13.1=pypi_0 -beautifulsoup4=4.12.2=pypi_0 -bert-score=0.3.13=pypi_0 -blas=1.0=mkl -bleach=6.1.0=pypi_0 -blis=0.7.11=pypi_0 -braceexpand=0.1.7=pypi_0 -brotli-python=1.0.9=py39h6a678d5_7 -bzip2=1.0.8=h7b6447c_0 -ca-certificates=2023.08.22=h06a4308_0 -cachetools=5.4.0=pypi_0 -catalogue=2.0.10=pypi_0 -cchardet=2.1.7=pypi_0 -certifi=2023.7.22=py39h06a4308_0 -cffi=1.15.1=py39h5eee18b_3 -chardet=5.1.0=pypi_0 -charset-normalizer=2.0.4=pyhd3eb1b0_0 -click=8.1.7=pypi_0 -comm=0.1.4=pypi_0 -confection=0.1.3=pypi_0 -contourpy=1.0.7=pypi_0 -cryptography=41.0.3=py39hdda0065_0 -cuda-cudart=11.7.99=0 -cuda-cupti=11.7.101=0 -cuda-libraries=11.7.1=0 -cuda-nvrtc=11.7.99=0 -cuda-nvtx=11.7.91=0 -cuda-runtime=11.7.1=0 -cycler=0.11.0=pypi_0 -cymem=2.0.8=pypi_0 -debugpy=1.8.0=pypi_0 -decorator=5.1.1=pypi_0 -decord=0.6.0=pypi_0 -defusedxml=0.7.1=pypi_0 -diffusers=0.21.4=pypi_0 -distro=1.8.0=pypi_0 -docker-pycreds=0.4.0=pypi_0 -et-xmlfile=1.1.0=pypi_0 -exceptiongroup=1.1.3=pypi_0 -executing=2.0.1=pypi_0 -fastapi=0.104.1=pypi_0 -fastjsonschema=2.18.1=pypi_0 -ffmpeg=4.3=hf484d3e_0 -ffmpy=0.3.1=pypi_0 -filelock=3.9.0=py39h06a4308_0 -fonttools=4.38.0=pypi_0 -fqdn=1.5.1=pypi_0 -freetype=2.12.1=h4a9f257_0 -frozenlist=1.3.3=pypi_0 -fsspec=2023.10.0=pypi_0 -ftfy=6.1.1=pypi_0 -gdown=4.7.1=pypi_0 -giflib=5.2.1=h5eee18b_3 -gitdb=4.0.11=pypi_0 -gitpython=3.1.40=pypi_0 -gmp=6.2.1=h295c915_3 -gmpy2=2.1.2=py39heeb90bb_0 -gnutls=3.6.15=he1e5248_0 -google-ai-generativelanguage=0.6.6=pypi_0 -google-api-core=2.19.1=pypi_0 -google-api-python-client=2.140.0=pypi_0 -google-auth=2.33.0=pypi_0 -google-auth-httplib2=0.2.0=pypi_0 -google-generativeai=0.7.2=pypi_0 -googleapis-common-protos=1.63.2=pypi_0 -gradio=3.24.1=pypi_0 -gradio-client=0.0.8=pypi_0 -grpcio=1.65.4=pypi_0 -grpcio-status=1.62.3=pypi_0 -h11=0.14.0=pypi_0 -httpcore=0.18.0=pypi_0 -httplib2=0.22.0=pypi_0 -httpx=0.25.0=pypi_0 -huggingface-hub=0.17.3=pypi_0 -idna=3.4=py39h06a4308_0 -importlib-metadata=6.8.0=pypi_0 -importlib-resources=5.12.0=pypi_0 -intel-openmp=2023.1.0=hdb19cb5_46305 -iopath=0.1.10=pypi_0 -ipykernel=6.26.0=pypi_0 -ipython=8.17.2=pypi_0 -isoduration=20.11.0=pypi_0 -jedi=0.19.1=pypi_0 -jinja2=3.1.2=py39h06a4308_0 -jiter=0.5.0=pypi_0 -joblib=1.3.2=pypi_0 -jpeg=9e=h5eee18b_1 -json5=0.9.14=pypi_0 -jsonpointer=2.4=pypi_0 -jsonschema=4.19.2=pypi_0 -jsonschema-specifications=2023.7.1=pypi_0 -jupyter-client=8.5.0=pypi_0 -jupyter-core=5.5.0=pypi_0 -jupyter-events=0.8.0=pypi_0 -jupyter-lsp=2.2.0=pypi_0 -jupyter-server=2.9.1=pypi_0 -jupyter-server-terminals=0.4.4=pypi_0 -jupyterlab=4.0.7=pypi_0 -jupyterlab-pygments=0.2.2=pypi_0 -jupyterlab-server=2.25.0=pypi_0 -kiwisolver=1.4.4=pypi_0 -lame=3.100=h7b6447c_0 -langcodes=3.3.0=pypi_0 -lcms2=2.12=h3be6417_0 -ld_impl_linux-64=2.38=h1181459_1 -lerc=3.0=h295c915_0 -libcublas=11.10.3.66=0 -libcufft=10.7.2.124=h4fbf590_0 -libcufile=1.8.0.34=0 -libcurand=10.3.4.52=0 -libcusolver=11.4.0.1=0 -libcusparse=11.7.4.91=0 -libdeflate=1.17=h5eee18b_1 -libffi=3.4.4=h6a678d5_0 -libgcc-ng=11.2.0=h1234567_1 -libgomp=11.2.0=h1234567_1 -libiconv=1.16=h7f8727e_2 -libidn2=2.3.4=h5eee18b_0 -libnpp=11.7.4.75=0 -libnvjpeg=11.8.0.2=0 -libpng=1.6.39=h5eee18b_0 -libstdcxx-ng=11.2.0=h1234567_1 -libtasn1=4.19.0=h5eee18b_0 -libtiff=4.5.1=h6a678d5_0 -libunistring=0.9.10=h27cfd23_0 -libwebp=1.3.2=h11a3e52_0 -libwebp-base=1.3.2=h5eee18b_0 -lightning=2.1.0=pypi_0 -lightning-utilities=0.9.0=pypi_0 -linkify-it-py=2.0.2=pypi_0 -llvmlite=0.41.1=pypi_0 -lz4-c=1.9.4=h6a678d5_0 -markdown-it-py=2.2.0=pypi_0 -markupsafe=2.1.1=py39h7f8727e_0 -matplotlib=3.7.0=pypi_0 -matplotlib-inline=0.1.6=pypi_0 -mdit-py-plugins=0.3.3=pypi_0 -mdurl=0.1.2=pypi_0 -mistune=3.0.2=pypi_0 -mkl=2023.1.0=h213fc3f_46343 -mkl-service=2.4.0=py39h5eee18b_1 -mkl_fft=1.3.8=py39h5eee18b_0 -mkl_random=1.2.4=py39hdb19cb5_0 -mpc=1.1.0=h10f8cd9_1 -mpfr=4.0.2=hb69a4c5_1 -mpmath=1.3.0=py39h06a4308_0 -multidict=6.0.4=pypi_0 -murmurhash=1.0.10=pypi_0 -nbclient=0.8.0=pypi_0 -nbconvert=7.10.0=pypi_0 -nbformat=5.9.2=pypi_0 -ncurses=6.4=h6a678d5_0 -nest-asyncio=1.5.8=pypi_0 -nettle=3.7.3=hbbd107a_1 -networkx=3.1=py39h06a4308_0 -nltk=3.8.1=pypi_0 -notebook=7.0.6=pypi_0 -notebook-shim=0.2.3=pypi_0 -numba=0.58.1=pypi_0 -numpy=1.26.0=py39h5f9d8c6_0 -numpy-base=1.26.0=py39hb5e798b_0 -omegaconf=2.3.0=pypi_0 -open-clip-torch=2.23.0=pypi_0 -openai=0.28.0=pypi_0 -opencv-python=4.8.1.78=pypi_0 -openh264=2.1.1=h4ff587b_0 -openjpeg=2.4.0=h3ad879b_0 -openpyxl=3.1.2=pypi_0 -openssl=3.0.11=h7f8727e_2 -orjson=3.9.10=pypi_0 -overrides=7.4.0=pypi_0 -packaging=23.0=pypi_0 -pandas=2.1.2=pypi_0 -pandocfilters=1.5.0=pypi_0 -parso=0.8.3=pypi_0 -pathtools=0.1.2=pypi_0 -pathy=0.10.3=pypi_0 -peft=0.5.0=pypi_0 -pexpect=4.8.0=pypi_0 -pillow=10.0.1=py39ha6cbd5a_0 -pip=23.3=py39h06a4308_0 -platformdirs=3.11.0=pypi_0 -portalocker=2.8.2=pypi_0 -preshed=3.0.9=pypi_0 -prometheus-client=0.18.0=pypi_0 -prompt-toolkit=3.0.39=pypi_0 -proto-plus=1.24.0=pypi_0 -protobuf=4.25.0=pypi_0 -psutil=5.9.4=pypi_0 -ptyprocess=0.7.0=pypi_0 -pure-eval=0.2.2=pypi_0 -pyasn1=0.6.0=pypi_0 -pyasn1-modules=0.4.0=pypi_0 -pycocoevalcap=1.2=pypi_0 -pycocotools=2.0.6=pypi_0 -pycparser=2.21=pyhd3eb1b0_0 -pydantic=1.10.13=pypi_0 -pydub=0.25.1=pypi_0 -pygments=2.16.1=pypi_0 -pynndescent=0.5.10=pypi_0 -pyopenssl=23.2.0=py39h06a4308_0 -pyparsing=3.0.9=pypi_0 -pysocks=1.7.1=py39h06a4308_0 -python=3.9.18=h955ad1f_0 -python-dateutil=2.8.2=pypi_0 -python-json-logger=2.0.7=pypi_0 -python-multipart=0.0.6=pypi_0 -pytorch=2.0.1=py3.9_cuda11.7_cudnn8.5.0_0 -pytorch-cuda=11.7=h778d358_5 -pytorch-fid=0.3.0=pypi_0 -pytorch-lightning=2.1.0=pypi_0 -pytorch-mutex=1.0=cuda -pytz=2023.3.post1=pypi_0 -pyyaml=6.0=pypi_0 -pyzmq=25.1.1=pypi_0 -readline=8.2=h5eee18b_0 -referencing=0.30.2=pypi_0 -regex=2022.10.31=pypi_0 -requests=2.31.0=py39h06a4308_0 -rfc3339-validator=0.1.4=pypi_0 -rfc3986-validator=0.1.1=pypi_0 -rouge=1.0.1=pypi_0 -rpds-py=0.10.6=pypi_0 -rsa=4.9=pypi_0 -safetensors=0.4.0=pypi_0 -scikit-learn=1.3.2=pypi_0 -scipy=1.11.3=pypi_0 -seaborn=0.13.0=pypi_0 -semantic-version=2.10.0=pypi_0 -send2trash=1.8.2=pypi_0 -sentence-transformers=2.2.2=pypi_0 -sentencepiece=0.1.99=pypi_0 -sentry-sdk=1.34.0=pypi_0 -setproctitle=1.3.3=pypi_0 -setuptools=68.0.0=py39h06a4308_0 -six=1.16.0=pypi_0 -smart-open=6.4.0=pypi_0 -smmap=5.0.1=pypi_0 -sniffio=1.3.0=pypi_0 -soupsieve=2.5=pypi_0 -spacy=3.5.1=pypi_0 -spacy-legacy=3.0.12=pypi_0 -spacy-loggers=1.0.5=pypi_0 -sqlite=3.41.2=h5eee18b_0 -srsly=2.4.8=pypi_0 -stack-data=0.6.3=pypi_0 -starlette=0.27.0=pypi_0 -sympy=1.11.1=py39h06a4308_0 -tbb=2021.10.0=pypi_0 -tenacity=8.2.2=pypi_0 -terminado=0.17.1=pypi_0 -thinc=8.1.12=pypi_0 -threadpoolctl=3.2.0=pypi_0 -tiktoken=0.5.2=pypi_0 -timm=0.6.13=pypi_0 -tinycss2=1.2.1=pypi_0 -tk=8.6.12=h1ccaba5_0 -tokenizers=0.14.1=pypi_0 -tomli=2.0.1=pypi_0 -toolz=0.12.0=pypi_0 -torch-fidelity=0.3.0=pypi_0 -torchaudio=2.0.2=py39_cu117 -torchmetrics=1.2.0=pypi_0 -torchtriton=2.0.0=py39 -torchvision=0.15.2=py39_cu117 -tornado=6.3.3=pypi_0 -tqdm=4.64.1=pypi_0 -traitlets=5.13.0=pypi_0 -transformers=4.35.0=pypi_0 -typer=0.7.0=pypi_0 -types-python-dateutil=2.8.19.14=pypi_0 -typing-extensions=4.8.0=pypi_0 -tzdata=2023.3=pypi_0 -uc-micro-py=1.0.2=pypi_0 -umap-learn=0.5.4=pypi_0 -uri-template=1.3.0=pypi_0 -uritemplate=4.1.1=pypi_0 -urllib3=1.26.18=py39h06a4308_0 -uvicorn=0.23.2=pypi_0 -wandb=0.15.12=pypi_0 -wasabi=1.1.2=pypi_0 -wcwidth=0.2.9=pypi_0 -webcolors=1.13=pypi_0 -webdataset=0.2.48=pypi_0 -webencodings=0.5.1=pypi_0 -websocket-client=1.6.4=pypi_0 -websockets=12.0=pypi_0 -wheel=0.41.2=py39h06a4308_0 -wordcloud=1.9.3=pypi_0 -xformers=0.0.22.post7=py39_cu11.8.0_pyt2.0.1 -xz=5.4.2=h5eee18b_0 -yarl=1.8.2=pypi_0 -zipp=3.14.0=pypi_0 -zlib=1.2.13=h5eee18b_0 -zstd=1.5.5=hc292b87_0 +accelerate==0.24.1 +aiofiles==23.2.1 +aiohttp==3.8.4 +aiosignal==1.3.1 +altair==5.1.2 +anthropic==0.32.0 +antlr4-python3-runtime==4.9.3 +anyio>=4.8.0,<5.0.0 +appdirs==1.4.4 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asttokens==2.4.1 +async-lru==2.0.4 +async-timeout==4.0.2 +attrs==22.2.0 +babel==2.13.1 +beautifulsoup4==4.12.2 +bert-score==0.3.13 +bleach==6.1.0 +braceexpand==0.1.7 +brotli==1.0.9 +cachetools==5.4.0 +cchardet==2.1.7 +certifi==2023.7.22 +cffi==1.15.1 +chardet==5.1.0 +charset-normalizer==2.0.4 +click==8.1.7 +comm==0.1.4 +contourpy==1.0.7 +cryptography==41.0.3 +cycler==0.11.0 +debugpy==1.8.0 +decorator==5.1.1 +defusedxml==0.7.1 +diffusers==0.21.4 +distro==1.8.0 +docker-pycreds==0.4.0 +et-xmlfile==1.1.0 +exceptiongroup==1.1.3 +executing==2.0.1 +fastapi>=0.110.0 +fastjsonschema==2.18.1 +ffmpy==0.3.1 +filelock==3.9.0 +fonttools==4.38.0 +fqdn==1.5.1 +frozenlist==1.3.3 +fsspec==2023.10.0 +ftfy==6.1.1 +gdown==4.7.1 +gitdb==4.0.11 +gitpython==3.1.40 +gmpy2==2.1.2 +google-ai-generativelanguage==0.6.6 +google-api-core==2.19.1 +google-api-python-client==2.140.0 +google-auth==2.33.0 +google-auth-httplib2==0.2.0 +google-genai>=0.2.0 +googleapis-common-protos==1.63.2 +gradio==3.24.1 +gradio-client==0.0.8 +grpcio==1.65.4 +grpcio-status==1.62.3 +h11==0.14.0 +httpcore>=1.0.0,<2.0.0 +httplib2==0.22.0 +httpx>=0.28.1,<1.0.0 +huggingface-hub==0.17.3 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==5.12.0 +iopath==0.1.10 +ipykernel==6.26.0 +ipython==8.17.2 +isoduration==20.11.0 +jedi==0.19.1 +jinja2==3.1.2 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonpointer==2.4 +jsonschema==4.19.2 +jsonschema-specifications==2023.7.1 +jupyter-client==8.5.0 +jupyter-core==5.5.0 +jupyter-events==0.8.0 +jupyter-lsp==2.2.0 +jupyter-server==2.9.1 +jupyter-server-terminals==0.4.4 +jupyterlab==4.0.7 +jupyterlab-pygments==0.2.2 +jupyterlab-server==2.25.0 +kiwisolver==1.4.4 +lightning==2.1.0 +lightning-utilities==0.9.0 +linkify-it-py==2.0.2 +llvmlite==0.41.1 +markdown-it-py==2.2.0 +markupsafe==2.1.1 +matplotlib==3.7.0 +matplotlib-inline==0.1.6 +mdit-py-plugins==0.3.3 +mdurl==0.1.2 +mistune==3.0.2 +mpfr +mpmath==1.3.0 +multidict==6.0.4 +nbclient==0.8.0 +nbconvert==7.10.0 +nbformat==5.9.2 +nest-asyncio==1.5.8 +networkx==3.1 +nltk==3.8.1 +notebook==7.0.6 +notebook-shim==0.2.3 +numba==0.58.1 +numpy==1.26.0 +omegaconf==2.3.0 +open-clip-torch==2.23.0 +openai==0.28.0 +opencv-python==4.8.1.78 +openpyxl==3.1.2 +orjson==3.9.10 +overrides==7.4.0 +packaging==23.0 +pandas==2.1.2 +pandocfilters==1.5.0 +parso==0.8.3 +pathtools==0.1.2 +peft==0.5.0 +pexpect==4.8.0 +pillow==10.0.1 +platformdirs==3.11.0 +portalocker==2.8.2 +prometheus-client==0.18.0 +prompt-toolkit==3.0.39 +proto-plus==1.24.0 +protobuf==4.25.0 +psutil==5.9.4 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyasn1==0.6.0 +pyasn1-modules==0.4.0 +pycocoevalcap==1.2 +pycocotools==2.0.6 +pycparser==2.21 +pydantic>=2.0.0,<3.0.0 +pydub==0.25.1 +pygments==2.16.1 +pynndescent==0.5.10 +pyopenssl==23.2.0 +pyparsing==3.0.9 +pysocks==1.7.1 +python-dateutil==2.8.2 +python-json-logger==2.0.7 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0 +pyzmq==25.1.1 +referencing==0.30.2 +regex==2022.10.31 +requests==2.31.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rouge==1.0.1 +rpds-py==0.10.6 +rsa==4.9 +safetensors==0.4.0 +scikit-learn==1.3.2 +scipy==1.11.3 +seaborn==0.13.0 +semantic-version==2.10.0 +send2trash==1.8.2 +sentence-transformers==2.2.2 +sentencepiece==0.1.99 +sentry-sdk==1.34.0 +setproctitle==1.3.3 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.0 +soupsieve==2.5 +stack-data==0.6.3 +starlette>=0.36.0 +sympy==1.11.1 +tenacity>=8.2.3,<9.0.0 +terminado==0.17.1 +threadpoolctl==3.2.0 +tiktoken==0.5.2 +timm==0.6.13 +tinycss2==1.2.1 +tokenizers==0.14.1 +tomli==2.0.1 +toolz==0.12.0 +torch-fidelity==0.3.0 +torchaudio==2.0.2 +torchmetrics==1.2.0 +tornado==6.3.3 +tqdm==4.64.1 +traitlets==5.13.0 +transformers==4.35.0 +typer==0.7.0 +types-python-dateutil==2.8.19.14 +typing-extensions==4.8.0 +tzdata==2023.3 +uc-micro-py==1.0.2 +umap-learn==0.5.4 +uri-template==1.3.0 +uritemplate==4.1.1 +urllib3==1.26.18 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.9 +webcolors==1.13 +webdataset==0.2.48 +webencodings==0.5.1 +websocket-client==1.6.4 +websockets>=13.0.0,<15.1.0 +wordcloud==1.9.3 +yarl==1.8.2 +zipp==3.14.0 \ No newline at end of file diff --git a/scripts/evaluate_gemini.sh b/scripts/evaluate_gemini.sh index dadc255..7e0082b 100644 --- a/scripts/evaluate_gemini.sh +++ b/scripts/evaluate_gemini.sh @@ -4,4 +4,4 @@ source scripts/env.sh # Evaluate Gemini Pro python3 task_eval/evaluate_qa.py \ --data-file $DATA_FILE_PATH --out-file $OUT_DIR/$QA_OUTPUT_FILE \ - --model gemini-pro-1.0 --batch-size 20 + --model gemini-2.5-flash-lite-preview-06-17 --batch-size 20 diff --git a/task_eval/__pycache__/__init__.cpython-310.pyc b/task_eval/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index b2be7c0ff5c5ff6a56c0bfce16d5f77fc9699766..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 152 zcmd1j<>g`k0uHItR1p0bL?8o3AjbiSi&=m~3PUi1CZpd~vm6}{qte=~jn_pC!l%HRcpg`kf_Z#f(m?cM5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HSeqLg+ZgFvn zequ^xVsS>IenC-wR%&udvA$2fbG~oBeo10+c6@4CVvc@%d}dx|NqoFsLFFwDo80`A O(wtN~ka?ehm;nG=10i4l diff --git a/task_eval/__pycache__/claude_utils.cpython-39.pyc b/task_eval/__pycache__/claude_utils.cpython-39.pyc deleted file mode 100644 index e4e93309b54a7ca89b3b8145479c78eeaa66600a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5681 zcma)AOKc=Z8Sbj?d9`Q8OX#_p=> zs=xmF-}V3P)@rta-yipXvg$l<7=NahqrW0vUd9vs9E~sp^Nl9;+-Du$^H8 zhAc=fi+k*5>1Jt<9biR6mz!nXSDF>w+f5t2=?`^=nnO8EwOPfS<<~m(W?jz}{NYZc z*~r&NnxmqKztShXIVQ@@aZzbb2)j8chMH5N+ME_OQUB0r&WK^rKtC%+#3=e>VoZ#q zpA+o5F+Xt!92iUUEVbTr<8|ueP0>MbtF55b_T4QZ@m3aI6uUuNdf)l*xG~RrwQT8h z6b6Bed)54P-SsxM?m%Wn?-%w(HNw%5<2uoLsA8wPu7JeBv_`TT`hK|M1#8acmW*OA3>KUV8G;=zUU%dzw;ek> zp%Rg^s=|)ml%MJBtjj=dKw0rOVPX=T_sl8|nDorLu0vm=tF>#WM1Ob`psJ6hIS30hVf zBHKKW_D$%~5nh{4cGWJ)O3CaX8N`?%A-Y}==|iJfkq$^%ZcLO6wBZC>ofSyfjhtsT zCLSW2GL%36;uczXifvS0}BELUhK)})9L!2 z1RtsK-p?*d*pTBz5E8gL1b0?sydx#-a3^%SGVJkKGL!IbGjv zO9!^CoX(aXd)Vd(cNf@m)`bhcAS11{`w)NxxB>hC`iixfR^*x&1SH#By+`3;u5B33 z;g4fv;<>CG0AH85Rh(;TxTEH9={DAdYVkAr$AMu$SD(3rUVjNHuFqae| zPTBkOR$3tKcGH3v2ze{D$THG`AG#u%=ZFsQUIKAd`~ySPvFq;QnZS)sN0B(=id{Ea zch7WHcvH6H=*;WkrSOgLOzcJ*EeV~U$-+cyEB5?op}U*b0NR#O)C#w{Tk&b)M2Fs? zGOIG1HMnYEu1%~Cel(^>v;A!hGPd^_u^#ip*kgiiaP_>vc-v6Vf^9H;o!B?24?dGa zS%7+16g1S*Lr_)%Sb=B0xO68IkDIcUJ0$YE=Ckhnys63#^6;*pv=nL9`szzV!mzBYW^cx z(PgA%@jDCz>sx~N3&IrEhq+`fLn8fR%)m9SqS&NX6sC+le%ZLO-!F+`Qkn!TWc91W z5~VK~eej;m!i?yz{1GGv=-2yYoKQ*1qJkBB%D`x7#+Wih6;Ex-=vPENDFf5TNkt4} zwLW7^8o*e=doAk|3vkxvw?qS{jL_K{19pp1VAQKgL7ipB=5II8CKfa>wv-xQdC2&Q z^Xqt`1vG2m;KtK^n0Uf)z9krXrmPt6nG2Bi_t^LOCSkPFQUtq0xJxVKp{>~K$dtA2 z{2M>0(ewhc^RDm3c;gFaQ?~J-_Uv;$ZpC)k>7IWZ6#N5^(K|^h^#WLP3(iaKvT)81 z+pZs-$6x@jMhMoXn_y^~5RXgsNixE<2GjJFTwxUKb|Q zrAXZy)=>s72WxXqMhm@?efh%jl|`pFeh+o)kiTC2($%G}U%7Vu!t$$EmlkbE!Gcvr zX+gQcnoP}(7oc%(&D*JkU2ZytfZiHJ_f{Nk$Ng_7WnNkop6iEe2p(H1cuPeVBAA=h+cT@kt?fwkA1+%R|FR(d62MR=>@X> zJw|_hD@JMuKPk?@Rz+#XI3Ry3_lt>nv!J%);s-2%f4s;0rG7as#br^vM+0Cgd#13( z(D&edx2+o=_bY2g-^Me9r;4W*mlJpkL1sRBo82h(>q-3{mW_h?12b;3&EI6aCGfKT zaKbbG=xv+bW)TMtTm3Q_+8;{l2aFj3-~44VEUMtBw!v>R^?YJuZJ4}N3{T-QfX@ve z8XDr~`RWLKwh>qNs|WDX4Vt&6nD~VEM}p(Do~?>8$mrhsB&{da z{ULBebM^Ko5F?GKA-(p%gh z%%rme2Fj5j_s8RrOq3Gir>Ezmi^cs@IMbhbYkPbHJ_G~wSf`y+>VEMWanJd=zh zHNvn*QW@EwJ;?1e8KPNO89xVF!D&Igl#J#&K3o&Wri^pO<|jCL3ej^6yiAH>e=3<0 z4d}^Ks&@@zit1B`bX3L(qoxKY9usrou>+3&xOiOmj_yGTsGZAbg5^)Y1ALq69V3{w zjQFve$8SD<+uZe;L4V(Xb)ALuHva|L50UI)w?Bm!%=Bk_KRt>inMtaUv#EX!-ltVW2(ECJA?}-7u_wNCv=Q8TVldz1Z zv}Le+IQCB*z(O)BxqoNI4fg*t+usY~$M&DhB!OgVI_JSs-z6TBnNLCgvl-l{q8bGP z{T-h2cd`;~8UIg8FWsZ_p_85xQwNM_J%AO1VjT^WT)r8{xfadM9xRGd!;?!X-*g|k zdMOFfS1;u%`s%GB!^Gt~W8@bsQ_o;YNBeoMD4eHEJP`cVQ<#3Az)qs+J*~4LU2^8> zQ1qy~Gp!!Yiqp#Ax_M3~vHE7cBY;(UqR8QP*EYL&TA+ta)pfAX&IT9Tnp9i z9m*%u^5CB9eMuK4bdypZe)P_C;xI?!=z^09*e-I|TOKMQO8jHFa0j=J(fh1(UHV9Z zbza{Nm7;M*!J>ogkqXXdSEyTX*02G;KYh9P#8MbL9V&Rj09Sfm72n5iF+&2Oj$)4* zLxVAO*~C0X_vwogRw7C3i|$9QuW`60>vKw%628hO5B1h7o{SceOk}Dvwd&N9*p{-* zN9pg!SjzNQf*?*mw{TWHP0-~*{?p^{ow&=MQIxEzV`%QOZ>tIPJre@x@l(!y=id|ueTzvC({a*f%?lv zT7+51K%_-vRwC@ArL9P|fHNI_>q353>*9swOJ7BW6(Z}ZNMuW7s8Z)N>N!oSPD%@r zFJ(8a9Bw>{(R;HAc)U~RvsI)TD zl8|k%sM6uru3ufctY0$zUM2o2gte$k^+*lT%oAwl=hBiFP*UlpL#i858zE(m2XA>&*?^H#wye>u;G<7k%QDMtI8LadbT}8ZM&XHD zX=UKAKhr|KA~hGOxlB|J4Nmq1P8lH0sZ?hP?sEiJ z(DbB5l*h>C(^4KzGT$(XqDC$RDf0OBfWM@p4@bCdppJfs#^5iQ$Jscyc##!(nLnko zU&??dc>^<5KFz9}^Ew-6b!5bKWWo)s)G>n^(QKGDAJcaoJ0sG+fMQNz@+ zzszQ>DqsM~%GNk@EUL0c0X2%5Dlp8kQO?;5tjHGeaGYYZY0RC%TQ+))E24S+G58Ql zhf){c4SjD#=@@F!TUx}2BB^3UR)jdvTJ?O{YNf?4>J)rUTBd0pMp;DAuJjF>s;~Ls zitD#JGF}fwq^-JHi`rgycOe2|h+R=2OAGK7d_3SmH?`lm@NVn%D@(5|e^uSUn))U+ z?@{x0YTlrRz9D3uK=pzq|A=O&Tu{0kFQ0?QZ~5{(mHQDzG5QQK%cJ|V1 diff --git a/task_eval/__pycache__/dpr_qa.cpython-39.pyc b/task_eval/__pycache__/dpr_qa.cpython-39.pyc deleted file mode 100644 index c3c61108248406b86db3aa7c41546cc83923f1e6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22247 zcmd6P3!Ge6dEdQvUUwcl`_yVB%ko&(!(+57S$+UQc(ElLKk)h?)xcmejP~AL&1!aL z_uRX(c9=;3*|kI}?9hZJDW$uHq*Z7rAtaE7rs0tU_(2;|62gU06AVHD+O%mBx@n>R z|99@p&g`yi^Q(WqX683%&OP_s@0@!c-}nE%^PN-b=}8&*-1@bTxtHH=7~kSW=O>Pf z*Wl+>Ov7*t$E+9y{#zAu)+$)0;TkiMrKn@LG1qeAXU&`D<6f$AnoLH#x)3X5`%M-0X}vTM*mhY;~@}z3ZKA&eizOJKLRW z@PC7obFRhz%bXp~b@<=w>~wbFf7IFS?7{z-bBmLAZUBXS&aKXhl_il6# zI5(cMo;KA<_r|lVsd?1Mzx*=$VIpq^(FaTZN&fru&Mf}ZjyqLasQ6BK($7bNo@v)F zy0a%-$0=8*gRFpDrBp3d7VB=YG&woRs`d?CUzuoj+`ZD%?Wf*oRIU9O z+UTIciTcr04xhud>Sf6)x$<*MUW{k#=KS%VY%xU|Oh2d0Z0QNQG!x@fGt za$mLNje4H5ujDM2ypyGUbE-DuPWs-yS$DRk7Ejb_{>gpJUF68y=gg_%d}(ZMF-RR^ z15MUu=Wg4MV*eUHqkaR2_M|kgr`v*U0nQ4+VCsnF3>F$a6i91A3QN zbV|Nk^vknuV5U}5t2cr~*zhZM&MTFjy}2DhYO>;%szu*DoLQ) z3zIETU6^Ix1OtrtwNq}@D=JrJP>@*g+~SS5p33*BK0FjeE47jnL^&A)tL6pKIZR|U zVy#>aq9rx$6;f!QSFEE**T-wy+O?LRe3Im&0%i(L5yU*AmO-^ zHOEy!@4ZJKm^dOAcPdp=dAqQsa|9lcAy<6x;Ro)2@K_K-UDcx63c7_%XWIsGr(CMk zrWGf*%1F7=+-jH!Lku=Eb$H`m*@Syu4uN4sV_VIrIc!DEA+s;K-O8AjIbzyo*6hQ- zx&?WHH2MqUe759Io}45vX?-Yi$YT!U=LHBje3z_-u@rGIh?lKpQ|oEzV_?|_)+IOSjePO?F8Pj+pnLK%j zJ+Ts#X}yvudXqKfdU6$5^CiV`7nqN!9{j5;f_zHl8NY!y;_ev?eT)L-&Q(g2u8cUf zhxf1N{YV9aE~ZM=X*Y;+!sg>a5_6$coGMq41g`D{5ekSXQvL-~XnGvfMr6nwv9i{fIgAb-GSyv3n#{G|VGMFcGsvZ5t(+2{p$Shku- zow0-+fi<;aWLH-iO{-y%IZfoPv58A9vT6rn;58AaJe!Xc(z9S9uEV#f1X1vnN)YiD zc&ATh<*G!)ykp+Npoi(@Rwwa7En(@cec{Y3=l>k2K?uqVa5Nft*KMY{AF)YZwmy7o z@CE&*3K2>{NE`ib7^^-;4M&2!Yjn*u#!1)x1g*BQc<3jkJV&nQYk zyP=hOC+f-3Q84UL@Q`C*$~L)CS-D_5{^FclPmhi&7wp4@jH=tbD1m9*RZoqMvg7$! z-ck>s_G$t_y&u#BJ&a;-jB>E3tB@Y%A=eS`!BWBQM#1ial;>mU7Jaqg<{>UHt-*UC zTx{^TxsvkSAbY!-UYJGuKgdfJWJ-=x#9c{4$N4HqNlqqGw8Z5~slBY149NDVRIf+c zJ@|P82n@>xQ)zv(U@~&QOH7iDb5u%)eisq+2T;o?OHH7~BZRU|b2ienW~0r>Y^)ib zjW=Ux=h#xhiGr`h8PO!sj=nkZe;(Et?^Q&a%^Z@>w7p@Hz06%u&Pdd503Wljk%xflzQ zM;?3_Xl1HA9rWucMctiMEk_2B^iT)|TWMFVPdi4TSFWQMx2~hnzE|G^(cxlRo}!E& zo%GKXG4&{}Mm)IYdQ&wu z>ncyZ38P3o%HTMIH!~BsYA&P7zNIg=jEVx=` zr}R+265XIbtfNGq**pl2l4Lt5)729Wr0zA(V}F21jT!lBZid%YfC{g+oo-$ ztiJVeq}paQqUMmhYg)CG0GU?b!QB@*t)lH|6?sw9>YeK*MG8~$#Z9Qsu6veKk`pQ= z6c7t16lr!&suU*GH`$6Y7n4TbWdd?Yq7cfZlx$%fsZ{fow1`Y$4 z!4!jO1}7Pm5#-Y^FeC=r1&PFyF_CSjtsN9q(p@t$`ebCtG|}VYG%|VQx$DP8bj=uv z%6O5H;URyJ2uEWVmLqm)W4S*DqG?#dcZYopmb2!gRx{F!Rx`f2dNmC_&at4eSu`vx z8|oEKq-r_QX2Q3YV#~(U=2H$dwYZ;fV!lmO2h9%n?FjB1Kh(4_2$N16o%dV7VrIj} zH@?|;g33>VCV&f&s#}01!O4@2Xd?-eMM`wU6!?7xrS?!GY(VllnG3+0rZNAh(CiTv zePkU~8&ngJ%+Y4L5uGy1MkDH6Sh8weJ_&@6k0_8uW+Qa?AKKhrj6^=KP~wcy-`Lw5Rux zPQ;3~83wwIv0aunFB?lE%fzq)6Us+7UmihFkCWRS+#f_H=N9w?;ZzB%IsX{AdlqbQ zK~ce~dNl>`ItLs;qeW7yp=HOr{9}C7-FQgm$;F7OBVI@WTvR7T2ULrsm@WCBBKk4W zot{OUK>01gsgw-L7=X#-Np*@OGG(t=I$bJPN+$q-6V+OAT9q6{V4!Ye@M;D#2!b9W zKvxyD9t;lIIssmK~_h zMbC#48pQQ$D?&y!%Zw4ck>Z>SVo)dN7V|wa<*9K)|0Zw+XD=jIr*|S`DTQpQQt804 zLFxqcsG^5Du)3$q&WuP7Gw$#tzsY$+Y>(h@2@t6pWM}eR@-|>t4 zh7WUvW1ZDn0kp5Dt>f1>6S!wPu|{GT^XRmpwqiDcP19b4`D^Msv?v|_Kr`hen(2BD zbJxafPr_Q1;#`Nd2$mx!0}D|aR-Z9geXPazX^T%+-k$Efp{*y|NTJ-%`Vr)XrRgkY z^Qox%Y$NUTA*FwdF-)sf8dxvt3^Y)xGl-H=BIi~kz7$2Sp)GPXf43i7iVF`rh zq4X<4X)`DdJ6D4zW&La;yVScZ*su{1{*)E|)C>O9OU~5DI@^UmUAcsxk`k_w5`MI% zguak!AF0mthuo;QgBx9$?rx!*16@ny^!qIh_=63*#ha>8f9M@%Cuf@93eIGC2Rk{_ z{8qHfCb9aVO`0(X@)(hM{@Mvq8!yv%QQz`f*Z09~a-pQk$z(zTEum4z^(GZfPih+j z>OATi1~~*lFD+A0&poJ|gi$Nz4s|u}Udy|7d3IX#YGHZx%$s2PLgw5q?|3T6oT!xN zij$=|a*Gx7`V}jm6I)BM>}VkI7-$?*XGo6IRW%s2S)gk2@~0RGR1hXNyvnx%`#yJ3 zK&0kiXPNV-8N7`F8JaLa^>)VI!QgQQPcZn$3}`tFqEPod;a}<;6VEefvnusYT;}^V zBH`Rs#1jQHi@@g=)CuhHJ-uL{eXFf)2kBNC7`?Ekf*q!KL0{WY2X;vtw7Ll;K#7IG z(9QC*EcTx;_@@ls&0r0nX_=-3>8>)p|A4F>8x*D~@U9X3$f5eIsAZWsaH}2G2z1Ed zY#%roI9JHic-7)+ec)^4Zs2N>Y()JsXmx>M+RJSGC{DcRfOLFln2z;0=3Uc79J;;4 zEyQ67ka(nQG^6;%@QXV!@UWZDGjCkqZ$!_The!w3bmniw!fQLcj`Mo{W*^oua3|yd zHc0s35e-0Ga;gv&9Rtp?VTq?%82m6sTYLKDsOlkb#d8L@+c~F4@YD4uKmMfgq&a0d zS*KU*^?ll2f7VprXN|TP>#6uLq#+;6JN_3aE!j-Ll9pPso&IIh8DM$DD9q7J>*t-p z(?A&N@pGmF{S~o0!0pnVb!()M6V`smIA`)%$zN^Yt--Upa#n}@C8KAF7mQPdKI9P-#PL!E^OoYxPCBuyj{CE#5lo3zw^Y1%x^#NOz&?U*T@FRo+WN z4JH#^`->^fb;&(0Q`lebWmSHjF}~epf{DxbF`x!?`8^0;k{YC1HF!y=rC(BNSmo5M zqh5*EC8PA+X)K9IQw6KKJaJq;qCSL7>fbW>Fk1%?w?6<)MjXFE{Jb0jS@jvAK@fwV zliO@f7`@JO(NZ5lwh0*wxER1iV7aFemAU{W&lOMUp~3;F2<3-b^{wx?@>uLxeK}nQ z(w%fy%!4x8L3;;DU1V8yODhG0mL*1*vN=807_CL71xXiJm}_5d5W&z7BC_V;%~i_2 zNUD5By@o_He$B_?gDpNbnEqRM;(t7lSYG>(uOy3FyIv>+%>M(12 zgvCZsJN284wbyBwBD3qam`kQ~w!6)M1P$*H?ivD(GY~-hYSZNagv?fR2;iEwm=Ua* zY=y3!qCkSU83xGiLmEu&pTqN$2~r6^Tyja`7s1c_F3R9i35@9wJTN?45C#z+3m6U- zFxtZ4aS8)K=Ok5AD2m1m4AD@RtBn|dDpceP7KI04YBMeL5;Haw<#Y>(f_e(HGu2PY zvot<_R-Vl`89d(u(Ands8$C;z5Oiv#xJN)|*6#t_rJ_@+cyBKK1NvhmbzfN?Ap0c=}K z15j)S@f*VX_jdx%s#b}oI)SIZX=%7Y5r>*M<`Y!C2V}8(z!`kpZVViuz#D7~>ON>t zEg6Ey&iR{R2-zgdMnnF{%oZ3bh6w*@?BAfwptDR7JR=i=5~nZx;5V24?hihETi#Ol zU=Rcmcg~YZt$qiWtMD>!irTD!R2y0z)7X;CWevbw$8b1Zh2aoN3&Y89nYjUpshP~G zq(NPT!$h!ti3OXKURDI!4HeBci@CSWVy>f;LO3&NDeaVZEgmC0OdVuhX;#upVp}!R zeH;mezQZ-xmffS}S?Ktx>@RAJ{O_506$P)t-na9H;9&w_^)DGve_xA#--a}-7K4Ym z(C|+)$Imdys?G%ou2nygD)b04WOp6a7VXM z3n;9b_u4?V1N{;V7t-2vC)&JTvCXnF&{0CozR$#R8PeStCIRlRE(e-FAP z)c5_!+#j$8aHYNss$HvrWKS(av3CZPwdGIH z=67f|HTd8UU*`J8nwf@!lG7COf%1Nho{GCCE( zdLAjEL@)^(z9-OnVcCR!4`yT&e^a1?W%OMDw3`3XyzQr#dWfi*Ybomtgt1;{P{(Ha zoFQk^1x=+tv?quw7Ifh)F9NH+2+a5h`cPxk{9L8xw`T2$MX0zM(R5B;4OU^wZi4~! zs~mVAWI#Zu{tbg)W6%wGE`JtjFA0u&W2?p*h7No` z2A!XYb6p)G5W_K(=x7^EKM9a0<-nwcLRSPHHiHLCGZPYN&0sc|@;kd@=#s5{E zoEeNbHU-BKxbbN2Y@UKUOB{VhGsI_D9HN`wJr*+$S)+fTm2yHrlJM4)U4B6AfV?mH+sh@X+S4dRRXKjxQx zv_ZV$!hOeqoPf&&7FhF12)>rPg@@a0A`g8~BwBn8$rDosLoF1m?I`ftnD$R$fHfY-4$1{vbNSE;@;cNi zdES6zfHfkN44F_e^oSMYec}7$Io4UM5Npc73=L`HEkA*}qyVq&MtUhJHWJ7mz~`2Y1XdZ=(HRh%$1~?FCnGkJ z(*VO6v5|x|Z_t{88p1|mwQVHdX`o#IvAc3skF=?L0&u?T-iB26!qT#i%G2ij-udU| z-v`+igKV=P5ly(uvK?TthaBtiQ;=;Y^WayJEwpzcsW2UZ`n=gs@ zJJ%}gC82h{MNR!{vIP9PKDgU${dy6I;$1D0FA24$UQ%jLx2XMp#ty&Ksz?2rD~y;H zUQoY{E>Qmt!8+tXxhKeh_8mbEeMAmi2|m}27Sx|2M~D;(u~U>;&>;US64wBP54P%c z<-%TAVZ5jzi3qie=0-T=*ILD3J($YhVNQs{+dBI?D0d)oPw()ggut6fFzHJJ)mrH=$1fWeFuG3nItoJ@FcK%g-ymL)L4xVQ@p>^6DYZbGS zT7XA_%2nJcUp>nrcdf(6(slACx=?~*^%uyizJ(w^_+lXO?=m|V(bVrT5O6FuljnH( zZ3f?AaEZZp8C+)Ymk8hs|2@XP&)^3PUc=y13_i`^_Zj>FgToAtF!)0Tw6Q8Gf$F~@ zD5N{aU?JVvHSFMgVl5gLX7x!v{(l(!Jp=Y%knWfrXE7SQN0F)lF&71Cm+iF7M#VNu zaGWx?ud~i-EA6%jfi$hPDR@P}R%>me!K4MAe+>GQETr|J4cEqxZ|uDXb%2utevbK! ziO`Y|rUS&I5&~)=9+xojjOp0#O*EO3l$4Z&X$dnD_DGnOus6()g+~BsY>GLHjXBMT z0}yt6TQf>GD{(l2jhOqMU`*L0=7!VG@U)4gbb|7o+8vAeoXw56Y}P?uXN2)E|N8*K zqs|rqQE=39wuZ5|bCr(4*#Uc|1Xx)!Hup_AIRYy@C+@UTqwVb)S#-AVtP!9mTqN3c z+TE@b;Nr%0!V?FsSgUI{tksTItpx5}H<3?sOyl!M5&e$dW>VMv9jBk!#l4?Kkwed@L zf3rUHq1S$CJbcIJ#_@WuMa$_xmcLdnpErhA(#`kj@s5W#u_p4I)cZ;VQL5bMYbePi z?I5w$V1x4WiIo(L2e2K`seL7b_|2o(QUH6$;z~BW)4r}?$+mQ$b-$jL+)&s@Z`|K; zap0DYi(B^R)6;K78-26>+_>sv{f5}4y5YV#F7>`7UGVE5?&?&R=FK@g6b zOBiPuY?sC_$!L7_Uyg@E=m&a)T*^y^?~m)=lksv5X-H@4k)wK%5N7=P_@#$Az90It z@k_drUmjOIe1J6#tA&vxt(j%I38t1nZvxrq{uA%~uEg!N*Yt$niaa(Pep(Vd zi?vyp!z6WQJ5?P3-F0%EMlq4=@5bF1E7zUGd*dR}ZIRi}BiALT)i9R~Sx(pqLlRu5 zeQdskp#w`6)H<-5DRDYEW8vT}*uJtpx7Cqb;$iN0iQ|t!H=-6=5W`L^l+ST)?}Zr( zZr7oFjw9cu3*c1qf8@vE@-r-wH4ekp<}j9WMs#c@O{0upfcQ+txk?ZLW+0yHVO^z+ z93;Cg!&+YkWctlNVJX{NrHD+wW&&Qe=o&+qpW1_{p4h@~g-wcEjBqoNo?5(uKVwrm zT;dqqgaBlOC(9!uZS3~_RWg#L&2rbu?jB)X9ehf-k+2xy{K9{Q)5#6tI1-b9H&6?y zkrW}dAWvT-wzaU4>dRXWG7FisJPQ)SZ-u&)EhHo+nfSKq+E#XWPA)9vtjpXEF1=rG8?A0>LRb)mb3~d1cX9Cq>!|Qw6g@#RnnLYKtg&^0vsn`sH+)lXCP_a z!c?tZlFqaSdfy)h8EH9b(i|U_%N>lhrK`G*m(oDe+0qcw9c&knkov9mv$XjxJ|o?} zo3Sg&)vhjr93;ONie%|%X$N^p8HCafEPWEc_NQeXIf@jo(|!c~!x##ewP1+24);aBhXsRH zi@>aEb-yhH#6qzh^g_2f5eXP4Jk_An-rBZ@h&<1N7Ua2zauMpcqIT-#4BGPiVpRP( zi~k0L|H8s$MeMP%)Zee^*wE2&eIrj_f#h_ez_@* z2MN$|HO#$|*2=xCe`&FP{l?mT%d7S=DQt~oU;m5<)Ai5fv)KRdyR)8nS;<-$cNb&Z zD0XbVjPHIMU-gB>x!Q}&P{>F5%ou}x3@D-l^R(K}iyIlNk=L2d&WWMVynFG;8kzm~ z@kEEj#uc}IX0dOL;Rxh3ceCk5D@bizZ*O0#J5XqsypDo=TMz#LE}&CG2Nzu&TDVAL z=;1i30?r1Hi=B^~yQx8oOpJ*3%uP*Lbi!ucIJWCzs)ni1(~+84O@)5kqqR=x$JsCz z=Q0w{OM#}E2p9LMAH#zv)RmK5)OUv4ix8V9#P$x{i}wk)p>@ZmwW|8-#uCbx~J+Z z+EaCRXr__9q44v2z6q;dMRM(_S%`9ot3yaEM2RN#()4{DOVi<0Q=E~y86$|*Z@A*z zTwT>}J$f4-eHJ}X-?465D!m3CR=eBcmU{llRERD@$gQ=aMHOLg!4_i*jt8l8w8t)Ol&_!gJ*4My% zQEu6BQrJ9$4SN#DCK~uK(w!Zf-y4=6CpLvUOaj~du#a=u(De{Y5f_HB9gy}d9L9hH zWM<&o2roy@BlK;jSKT}hWB7chPu+ZPF=m@9j^ zoA&4KFD>rP9oTX|8N_Gt0JD99!JQ1&jBJ}F(natv$MZJiFv#{Xrc>}g8$gT)jo3eqN9diV?niOh znyQaKx@$7Q{NZO4{?qvJ1-&~!4Q`?s))=G%dOs@E5xCIujEahd(*c0)eSPW(R4n+P zC4D%?E+rd0kL2F)j+}YaOv0-HX9+oQtd&zlKF*zKjTCmUCE78m>iDL&B()$p{nMm-+%y3WI zAq@EWx9L4@FYx35p2m)~)hFMGC->tw{{}ye4bTujt*H+GhY|5y$Z};LSMg@AGu-IK zn`KX%*e`l-BL*Lc{WvXUv-m5##qYstyC{FvA*7;pao?9zF%@G4e=FYeDoKADSKDxP zHKu(Z7=v6j`^2G^$80erhxrE%!PyzQ<3?w&ix2L&yPVx?8Hcz4Ti)JI)&Yfd zKYlxef$W)36Bu#z-$~{0d23T2>7=LU6=pKdhsu?5-bwgv=H^Z+u!XVQJyW@A4L7Li z2sDA5cndpAB{oI~|x;m?z+m9)q5M2%$m-U(dT;A|qFvs;(!O1u`bFOV=M3J<;2H)w2G=5}_v}Dz9lRm7#5ph@7+lSO^Bz$8R~hei1+4dA!yV5!<5qfp`q(~sC%JeBLk@a7F6jjWh=Dj}FD8!XRa?T{*nydPAP`&ipmD1~RT z>9*hSP%G{7Ec{#==q?gH$WnS3h?+7O#PyLba;8LKC%n+);4mEABL_-(W1VuPxZsy7 zp6**=(0fQo_;sGK8yLKbfq?xs^H=w<=zAHw40nUL9-5ln3zOF#&p7j?TGh_ZZFVn$ zAxJ{RDH)+^i4>9#-5wrErE7DTg&tw>eg-!oC?vw&gnDgH41E+4h5is=c0|FjvEM_=gqzz;6kC$B4oc(x53tIV-z#}>9tM~twjrN|I!=mC9MliMNv&`7 zS&3zNfS!JOsh>MWma$``ksiho2YBwo^AGz24vsnyx2{1aJ#7-bY2@bgzgNSq9aYOLG^r%y>9+Ra9t1XV0 z`k*73qXZV1qkHkUW{!TaJx51oaem8&CIa-Mv10Di+jJk`7-I$0D-hJP9jBUsY1Ier zi`do0#r-kBwcOag&^>>jKCNcofm?Cr6pjnPQFuIIub#Xg$2*os@2<@iqQ?Nb z3d#GbwZ|&1Gws$>M}Y%zehzP@Z+8SSDmdE0DRc*`A)G<@;@|R;3^P9@XCiep3uk!1 z?f*P^F0f~D03jkck4_E*5xaVjpa~zT>J5CwHO%6dNt>KNdjx!$u_S{-48|DHWfXh0 zs*Kkd%rj67=n;UeDljz&#FMH23~wwm5XK>sUHGTCZ_6ykIbP-JKjXCJn;KRe6ES1; z^=yssFp?;)cmlzY6;0aa0V@?(Mbtd7i#TMiTrCzM({9CM`19otXKU)_iHgv^>#m9X zO*o5cQO>3CPgcq&6v}Fyw^pvg4C1RtkV29?7^_&6bE;tH(d1=B|_YJdUxDWK9-%X)J{N=oOS(^HEf}Ykt*kj zJ{oYSeVF1HQSpL4B7;mP*t*dzIg7%RzA7X<7n+Bv79w8NDMpGW$9r15Qg4D z#ieR(c&va8j=L4$z7Kqsyq0{~#@!{-99|LRn~9k+YzoV_EpCv}It+kTmSqy31z R*B-U|QzNNE_AM#%{{jMb4ygbD diff --git a/task_eval/__pycache__/evaluation.cpython-310.pyc b/task_eval/__pycache__/evaluation.cpython-310.pyc deleted file mode 100644 index 71590f0cf5eb214af5d3a1b840ab61f365d59d3b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9385 zcmcIq+jkqsecst!0E-JLillh4WgAXx$MA)Uo732}6-A0Jwo@5)OxaG>lF=n*2|@xE z(Cm^D>4ocO5({vG?;C!M~u$31<}L*4J2#f2d0#4iQU z&d$!|H?zO%H@{_PYN}x1_tDA+>;LhTVf;HE_Wx`=oW&Jpg<(iTik9J~ci{^CY`Uht zTdsw>*|OUiHzU$9SvQ9^tL3!wZeF+9twOu#7Ii(-Dz&HFDPj0iyM{Y0v+l814Vjb9 z9YZ?)%&z4gmw7+q&n_G5rYy+f9owDrkMA1m0`0SCE6J%l8FyYz%VT#8_k^60$8lef zvvLmiML91|;9igmauN3>xg?iyUzQKahjD*Mo|KQ^{;+&hK8E{AS(cCE{)nu|Z_6j{ z2=`I>q76n>V5`O>YTU#XW`*2t`Mv@e1jYXHw8Fpy0)~WjWsheFS9a--DIVM)eFh7)n>cf z@~?Dm_(5~WS39%sU+sQ)xrd_4;*H+5XLqLF{{U5&&*yvB>X6>($2#SD9#^=6A~LQE zDPrTgiOX6y7uaIs$}F;$U4&e}Kwm8C;4-i-+Jv_t@_RzNw)fCwf5d>Qh5E<>YHz0Jb!NWVv>8O`uh1dFI-IWZ@hE) z%IlY3ucED{X>>u?4*k~pJwwgnwIBT8RN#frhoL;>$!#y(@J@AA=el2y!c%R(-BH_X zoldlID)Pb`HGj)%ono=+Ma@pI(%nv+TCEv0qgsu`LI;JNnHDpm^uIex2Sl_o9D1ov znGb(dijgGF;6Vy$-Z3Brqc38yCoUMfqG!dXT7WjN$I>l!Y_A&9KG;&pR3`*&f+Ck& z#TCw=Xn^!Jy9d(0G;f-#DH}8|?wN_X^4&zV^;25RVMbCo-GYvj`DY#>8Pc3FcAh)< zfB0Cdjy&1?R4L_rf zv%!x1+fkC+4C-F!%OuzKqWXp(CRxp%@W93sXUMgam>iZ+EC^dn3)`Ch$`*&UXgD0c z=Tap{1q{5lU#lgBTCLrYn=RH$wc5>1ua)+wB8VzdR(yH&Z5GtmeFf#Laww*8g^UM| zVPhYTIbG20NmlvMrV8rh@9;-`=Opb#eu_#A?Oir{@DtL+-D(&;_>kDzg%)3X_^R2< z^s;eQTIkEgIcaYisvO&WAv1eI7&oU`kIdMNbFtX9hHdaS*_~HceNWXl%FzZ?p|Mf+ zS}oLl_>pc|cIqK2@GgG2xz6V8@`ktN!x3$lJGueY5V{ktB)LH?vFp+8P*G{Bmnv40 z_k&G%X)p4Vtk>=OflLY;URX6is!ih0vE)eGo2$&FLLfz*PaiAY$`6tcE`0 z3lmQU`22j&N_kGNfnICSYelBCB8&4-LMBV_AB)DWi8-@9J4R0ouafpH8lOUfH%#>h z!a$@(%)9xWn^Q8QWwCOeZ$egke-R7o|hKmSyW=9uO0rL*#k&ou3)AAUeSmEZHdKt9?yiSfO#KbMX=BcPzZ~5UH7^7cy+T5v(xHC}kaTqu-Yl0<1 zNsKlt5d**$#DF~_EwieAfK?z1`+BDt09yne0_<<+$#1lg2~JTT$gNcpqRXR+?BP+-Xy8d6!foF+bRgq4bO(cwR!B$eJofdA zYa9V6m{WeY1#PKh5)rB&lFE-*{CG(3paIMrq{)#wYM1E2ZO3x#c}vkQCL$%@h~m+P zPuB(yOja`niSQ_pwMcCu zJdQ)VmdTeDG-ZHd+PZYMUZyN%FLMT>fq&El;@yq%brUO4#Dg*9JA>D%7KK&?oS zk2oo2O+_V{c;BIPXrW)y!EYd(=@Kk^+aUcrC7~LWJxyP z8=7<@LJZL_GFIRv3}9^67{5vaB@6qr0P0q1*C}K3{e73oF&*peCpd&<6b7t zNE-m+8OehRp5~1x!_f~Lz3k{2^YEOLfQ9o$oWqrkbBhMroR~-5+|exO`yXI=dRp4Y z@_RVXbVD5Xpfy9ji(3aYMIRhK0yRae*g3iWLaH@K@qNNiQD-Q8?nFrzT97_#8>7cb z;r=n(xsF!w1}gad=MV3bRwbLNJIz~B9*n~6B+KKbAFAKyko;Kn-Q#2XucapocW%ET z3>uRx&TZ{Z;5tCJwGc-IH;CCFc4e3vz~>=wW4zR2ReoPqwfk8}}J8 zFMzDN=$%j}Zk>GLJv@sdRVNK}fas}Ah@I)VRRH`9+InULucxgpqpc38bP!LohgE<+ zK{KAAR5Sh_(=LFU-C@ua-J>M^1;_IcyR)$V!cj6-ub}ggpbxL7=vGE+HI7>AFNSMP zkYjwK@14V?5ruXdQ=bog4)}G)xjXPVsWw;42?|5a(-i2f#%_mb;-K51ga#Yc>I3g_ z6;0`mNkf~!FIdqg@c7Y;0kei@c~v7xp}`4&NpY-+q1q&!3933Ae+$l2N3O&Sx{2NP zy#Ng(N3J|h)5fwjBG5sN{z!+ad8pfxsn|B5Zd5g*g!3HuF$$_%%V>+9DGYcU973hl zH=%GDcrS}Vm;NbLE@E!3FBvm z%&!`^41~MFvLTDhP|8ZFnuaeZX;x>qrMe<4}JznyG2fAQDrx`fRSV_z1;>QZVO2 zzn~TQ?@@^iC@;|@@$zCS1`gV*QN*Z*LU4csX8Gy6P zI7Rhx5yESlQ&HbVKEQktW`RJRi<~&u&of->T+4g}l}DQn(lWPe_ME5?75k;w-h&a~ zIUhT5ew~=ksU||H-@`_=x~uP_wquv$xSV7=n^AW&x=;33rqcA0-9bhr$t&NJM0g0x z*Fyv#Wb`~4rpm*Eyn2=egI($^s9}Lrx7ctS1ybMHX0VPVaGDm#XkzZnI8jrpGY_9k zr}=^!k_nAy1IuY#$xW({Sp+E1=m&?CuE-28vc(LH|A0~YxRy1Ml{1kK69rt;f@pu< zoHb{VZBTSlTCFEiHq&UOK&gP?Poa)Hg##&Rae|$r1R0WK83+_Vg|s2!LXeHJX&q>*fEpmT%woiuxX{nZ9M(Xht?%q%Y)YY5 zln5m4Wmq;bgI8(@pBY#e>7>_sJc5#L8lCJ zW1fx&&9b-UHCx_V%io#87oIZnerWcko`hU9bAZTxyN8NY8 z>i?iguX+;f`mk62$+9vnclb}Xqom=C`RudO~I z^=I^RwB7aTV#5fhmPlzXD8+v5&w1cZojSp&25~xDlVrL`5JxH|kwO?Dtr#*9z)yc2 zFH$$yo=w$H{RyXz1HrWVj17Ow;+HJ$qHuF?+BKYb-2y9v)6GN%M282LryE^UpL6bu zEJne>Eu<4`;DP!xc2VJgSM_3=sTv}YsmbEk=-j`eGrWvq-eTZL|8Q(VlD}k@#4G~G zJktJvFbE?h8#S|JE};jZ#kPTU%jN=t&U4~9-L`~KQxNLcAUI)A1vF4wG;H`cRK786 z`2JC_0g!=FB>g~>mDvXTs##XfV(QHfKnoqUnXg;ljQkL?_zm9GHG>Xx1)myynAv8x zi_h)sW7u}BZK4dM#QfB=l{64ZwRRT+v?{AlSvvxG98|YLs&3D7I5kaaAgGl`E5ulN zOus!=n$$40qReVVS!XUze+YtKLdMq!p8&N)WR&p`5K-x>i%N;}JJ4I~fg(G` zK8k4zW@_;R8N%k7EMW;7Hq}322oB`>9K`Kc9@k%8Qhj_44eBKn_+Gr-g^no312=>0 zO?O+nzoc02v|9KN1szi?j@SN`Xxq&Pt?0%IccU23)H)kyE?=y^dgZm+mDk?Bxcb_g zmoEUJukn8z2GAV;;?UBc98*L19w+LP+*_=zu^9VpeLCXlC563oNQ#=NUn^C2ddq!^ z@0?D*OP>LMjF9qm4CZ2?-ui;e!zqs6KxY@{=bUL|(dV5x$HD!WlXIq=OkvTvP&)a4 D324+n diff --git a/task_eval/__pycache__/evaluation.cpython-39.pyc b/task_eval/__pycache__/evaluation.cpython-39.pyc deleted file mode 100644 index 05d612882177f8ee68299887fc6e48c6775e2063..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9338 zcmcgy+jAS&dEaw(u~=M4Q4}SLvMj@??FhC)S!o-mw#Jd7yDLGeD96oO3cSQQ03iXp z(Afno!i&~5nOaV()|p-(IxU#awAD-A+NXA=(?6k)`_P$AA6n(9GtKlt51szLvj9kt zRHCPVvuDr!@?C!4cfO@JHI+B;`^PJPU;p=44C6nju>EJDa0X8}%?v{rg0&1ceKVKo zGIzOto34pBZ&~e(n_+1m+s&fRY&q?mn;X{U-8|~7R-s*Vi@Gk;nrctGSU^s%8}2?~ zyEB&zkrmE{Aso4X-E?PVMjlu&MD8z`$cw^;<<803b)(MY0bN%VQyUrgpqLi>HVpTW zm=XK&o)@#?0N#hioH&SgK^zkEcrS>`-Y<%hcp2|w zqAY$%yt2XEm&B{$HPn>ESH*F>Ulwf1D8K%MC{-!5#JbuHBfPWcg03G)g={9Q1xg|_ zbt6!yT8d=5E!C3@hSX@>_MeHu89d=@NTflO1yva+%3VvC!or#vkrTGaVmG#MuzEh( zx72KRTJlzKPx{TCRK3{`Z+AYq+DB5MoL?LsVno$;?yEo(F#)p{<$~CU#);mhz?H@2qNsZ;wk<>G^&Tc~PLk zw@a14m!&|JWV;isl|mVnjt72gtrQKn9+nr!V|%&duZHEA5+Y;0U9VLwV1lb@~4Hhy&PL$Nv1NvVUGD|u_Zj= z43Y*WzT@;U@elcZ4AAG-$zz`I#Wxey-l|e`s3d>71>q*Dzk3Yj|G{JQPV79~h?9$_ zTS3ihh3_IW8n$aobY8us!^r3@&Cr$x#wO0ZX>8kCnJ1Zs3c8&n(+WP8N*%_Ikf7;?$eIXp*jH)PAb4BKaUnvR%LC zg;FHhwine_WSH2RG~td-C(ejsFJf@0L6{w47MtPIY`5%;IsQO3@00ps`Z&S}o)yfTL3qK`lfEa3V|1 zI#sWgR=iaSH@jxj;Y4uekiM7&v34dP^(|!Um$YrgRS(P?Z4lyys^$P=9Rt`qb0UtD{Y%Mz7G!GGn3O`+>IYk`o$i@YQ^-t1$x7kAayG0`PFp;i#4%M;5Iywjsucm|o+ z7-)#!iuV@y?-(^fk`X7yn-xHQkOj!UNp2OdD6)g#g>^S*`tYPWt!A|M z?mmY8$!gf~YO-2yKDasF9s60k+c9d_?ue9xjUl0aWJK&PUuR!{Z3FXjIGq9-wc(J~ zt;0TJ$5utXgPs%fkU*Gtei=_lrUiy~mJH8BW0ff|_B+t9`*59?jK{2^$qr^5LdS&H zRvyEbln-jHu8=85Xgf*nbVK^`K}Ws&0gBq(^}hLYnb3hr+TzT33MiuqPb{%heN76~ zFG^h-^(~rzOCHWsQE#CzRp=UEG6$zq8HNBkJm_M zcFy_h=ou1p7`(_F%Q5Lu*HE6YkF=#R(i}$txJ*7Ay)}AAK%D{KL>+Q0&D{eErKe73 zRD>$(3=-%;zN4gQ*2w8hisTOYq8h0#jN*q;%Ie3>Xr;OgFR9h^r8W;`OOsI%*%GtuJxDC*bsGkO@J3yyYIY-G zHxqvG)Xqcam!*o*(<*dKy>Qq9haKgLgk<8pBkquHhNNzhp>>+PZYS2>_B!dbx!aQ- z!vYP{p5%0&MaDy7E1&cwiOi2j>fDH6J@i4w!lixIPtYk<;(6H)x2fpZbH=z^=}a^{ zncB|#p+|X}Cef;CViIx^Mz-i9xGoQ7KhHwp+uX3mH*)cXaaZ;#6uhjLcH6`V8TEFL?~ z&Ksz6V!F>}_a?Z;hDm~__o8hCe~i0JC&Zl&GBct&`*??(Xp;XnPMwncX#G?wGkei} z0;Nb|i1p926XnqYf|5zq)T@6=^$U~`@-^___ut=oLMxQ*R0Yl;uZT-zliP_+*G(C! zU!g9!Ep>4BZ(%|;y$|kd z8PVsruyX3N;(l_7&A_{&Tfz+f=1kQ_>x7We)`?p54$mM-g-p9W@a;*&U`f-PD}yUE ztn2d#NT@Y2qcsu!>Cn2`y9I-ALbHQ>>1W>QU_>)y+&Uw}$~t>-wyuToto^NATo za;s;U)FARgc0v|$a=1|~X26~gQI+nP=FlSiL&}f{_g>zcSRml&-mmVUAFMd7{g}Ch zEr|^kA+?6B$yv*$ws{HnJBihno{tKRoMZwN8MAah;dpopxkq?XBgf_#E>cuCA(SML z;k}_4Lg5^8B(5==oT$ozi{CD^U)4)DNIanweg>*(7#?*Z|Q5N5>`iGBcn@d z6>X^xO5T+g2{+%6Ln|L9`R@=aUQjBa@XMBf=y7BNU0ZuUyP__|?F9H^bWHD$z%-j3 zg}=ZdK!k59M%#o4{bKwET2{G~XuCe?FZ_t=weB5Ktv!K?6i$7Q5S z^pT$sHi|zH+MJ_Ke~BbApcLevLRHo|`KY+{=;#2hsb(7EiGk5~-s@)~b6~}Y<)n@P zqC8NQ0XYTs0&aBuYy^{^#&*<4h)eLdpl)y;vXK*K2RU*fbj;^?l#fl+X~#ii*LmNG z@=;+>jIB*57s|QViF0-GGM!4olsbitYBf^7id@es#c?UIgKpI6Mo$Qz%Tqd#l~@5H zJ4sGSPms?ESK)35^M{zCC&EEPE)vAt?WSKx^f?VrWb|b2 z%vQjtTB8pD$#fdGs3NG*h*q$i_H=2JN_!ypkVB>PcL`{L;>!qAlUe>7^a_t6F}TeR zaSK+LVnr0|fQ%wyG>d4V`aQI36_^MeO`(<;s?G3EVN4K|(}6hwHsP^R&ZeHz>fd)cdo`Z6{Lc9f0tac-Rxv;F$8HH+3C>sDu5>*ZmqgVvvjT%0E{SvA6c75PzK z{Tq!E1uZ)NtMginuv4tOjy0mfXDoI^0pBi)|755zt(vw?A=R_)O6Xgb>WIY($lXYIy07s$t8;vx`Z>L!CU%^jk;0^-^B#@z;=o`A=ORw&(``a+5VD)=qZC7B z>J;}mm|7XF9*I*!l(P}2wI?JLPuV-Lt#uL}GN0;GB(T+_Qh1T*^bY$$3F1!iI=Xt6 zyj8E+@|Ih&H-%3-C5r!{S_(Y~F=>hgA_nd{aUy})2nn0P6N1{#IWb7ZG1R2F1 zN1lwHb22Q#hM|y2P$Wx|g5n!mb?(Dkrfm&FQWS}No=+0vyWMM;Pg~k!=~n^)2qOsq z{F$^tgl|Thw4SoJQ}`jOAvCX1V{`%GauvqT31vvG_E6Ese$;=Bw!Mo$M0)C`HDB*! zPKS+1;tZ0<@%l_@*HP2sk1zs}0+1f?aZj$PFKB^DQvHOgwEcMFlYZ$p&X#Z;Kx8{C z4Zmqg@lFX1Uai(uUTv*3ym~yKsrM_2WZ)ymT1OI`g%R#2k*gslys=C~IyC6IbYsyu^^i934NAt4>*mveRh&-!fSQOl&;VL0CDxx%jwqkZKSPrK z360^mkr*a{ar!rlAbuWzTx7G{F_8`*grRv6L1~L0;U#_;SEm#FgswY`k?OB7%LMNy zOz=OW^|?)OYcCTF^NX;|@S}?@>;~{p-EQE!`j3GA_#ETt8Xq+|s=Q?fzH9#&j^pBlE=aP_-Xae|W1 zDIs?!6&bBFW064|tGy*ETe_6YYVoL3q)mU^0ml3kbA+!T*=eeaoD4Re?I!zg(QktH z^sSEc{v4k0ACQn`){YmVGC-cx@nW#qd}PNdRCVVTIL`;>*!iMQY&Jz>zVpWFDr z0%xJ6TsqSw~!RbEEh~=;qY$MicJEMn#_}U4HZGp1hpX zycA8Z6*Ri|lCVZGq#5{2;V}#|csOvW}xi@8Rdirp$=K=hYa22)B#Co6{oed8Xk>&0vJ`2hKe{)i4{lQ&-!|uG56n#? YMn2~pa2&k%Iaz1Q$vE@*x16K@59k5R_W%F@ diff --git a/task_eval/__pycache__/evaluation_stats.cpython-310.pyc b/task_eval/__pycache__/evaluation_stats.cpython-310.pyc deleted file mode 100644 index 28755b390b738674c6c197df85f35817ff4b6466..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8030 zcmcIpTXP&o6`r1%y=X7GUv?!~Uf;b*EXBEkOA!HI0t6g_15V^9o6+`Yz0%&)(~_fQ z7Lv$RoGKs*p{U|bDhjIn1g}u}A5=~8Kvj6a&s-iw#io)}_of-(&IIC#${*%o!Ro3TqJoSoEHRcUQ zZAQhcM&={n2f4{3H`#oY5Aitf0T<)LJi(LT;&u;Dt?1%uJI>R*m-q30KCnr>*cEl> zNA6%u?Q))Zrd;LK+ssRHd&yd>NM0_=qE4;B6~~htIgi`f0pYBrm$jB5VH!6!^$Xg< zYRlxNYmREJw$8-hYLBb=<*ZiHV8}#w4dh>KMVh^Cgh#R(kB({JVp(lWOjtxBO-udNp#Uy|3oo-%*JXXoQ&e4(~h^=O<* zy)>Wpp9NW8v1?Ad=*!>M#3+2;IX_dgoHLHYXDq&9!T6bms4v?^>C9B^YF%tB*6Z^6 zjI^AUg1v54W{9j=rn7#!b)8M3 zG|BYOGI~FYqc(+Sg6aP<(;|Zop1ySUQjz+HR~AeUZUeXT2`FNgas4J%`+Xg};q$F1 zcr)N*;3EMa2Okai9`Lb%Pe`_a!N5YybKAnfz``MS$Q{UNtzkEzEF9j)!k)WZx-Gk^t074^*djkGA>@jcxpV-tt7-(fATOLD8hM(l8evTL0RlM3v z{5I7~?uvF6^W^;H2CS#wYq+_F)A#h{Vz{xxCmqCBaR1W;p|{&@-9Mt$Np&|FgOC@W ztytAXZp|H~e)Og@wdNtKW>q$tcEKtZr)MkmqE&I`!1$5}4lBmdT}%+6lksjsB4xU~lht*yG`_J2Ut6Z?DIK>^jgL(yL}Jb+AKX3PKOV zi4&m{;xKXdG)&3nAhu_o!$7&;sXG~t`h_)}dT5&Rp(amXwY+%Il6DDiTM;3L$-(fV zpr+TL3alKM5)!k8REr_?p)8VZ>v-z){77Hbv1^PScG}Z+SRR>ROxTTzRkUyGo<9A! z$5y-IK<%K!Y^Cf-%5UbLf^roJIjwp4fEWv9y|}$z=EzKJl|*^5g0QjADhHcB9hyEw zrgLq@5O{0D3FLU&*eN&}4Q~Dv&rKFM5)rg=JM5eqLru+PIAWA*QVgKwwk{sU!(;Yc zUWqfP{u*xO&h-8hIz;UkK-7dzeZEzoJPCbW*53IHm!OFi&wyY- z`j;#ruiK6+1j(jvx;kCR>gRDQt1rN>3rgRee_)>+II-d^dI-v2JLBv(9hXSrDKeKsPjqrtjpvGWfxNg; zukmuNl*7@yCamH{u3XF6R`Gf++yLitcj#0=${p+o3EsqT9r<8)vd6&fwbD+ zGQ0etzO-A^;?k*w*RQsa#Oo43$7Qa53n53Z$&b}u=}gpZBZ#FHkF8^!Qem=9gLl@* zOUx=zMf=5udbLzj2kw`)tT*WIk8ZQ*csuXHg)%FrqTZ(eAuwririX_hvR>?~s`ogJ5`?DDq`*#%!^^!N(HHD#6Z+Bh2 zBnF{cyhLP}24Q7!gr0rvBf1=!~Q2+k$+j4^B^ScZ+VL1sp$n68hq zaR9rc>?G?A_&92&nE7^;nM{8>4xo3M^(Ok6smGPRs6M42U=H=|5%fWJ{4? zDea~&Ad_9_YxPQF%LE=|%h9bEj+EYQUFG|IyXrN!cPq}5JhiO{?frzb+z8O*fK13F zFcQ8?bfL+?)(~>|A;N?TarFgg(sYLun!Je2yxj^MWQHgZAB12n+6DqN2@W}W%C7;M z98_rXq8k;z2b$cagAa9~$wi>aG<|sL)~3*8TA|4iw8T5(U{;2K5l3ZjfF{R)CVSoC z0IQ7vF{rT!L2dQBBRtbalc8^<4gT(x==RT-0VXCiIVT5rc8`Tj4k$D^zQ@ARz(OBQ z`W6B$DZER#Wdvw4<0h4bnSCrw+}+Y`8PMcnh-+3C@q0!N@(Ds}e6W?pJr1ljq;TaC zyn-fLlXB3Va3`^n6Y!aknc{%;CV|{0eO&2|`?dI5HST8JNj$T_npdFvkm@rI`D`E$ z{H7vt)YTOR{m_p>*N@3V3WFZPOYo3uC=7}xGA&?AjN$W#!ACK(e^(eZ$B!rss;tI1 zz6`J+Mn4u{P~RiMpn*rqs!4@G;aA5pSp5n59hYM=0}PrCFeplxm!Q|DI4b)Ejv)mGn@8K{ zWvL{-Bhm3YP|T^6=b-${_v53`5AiN>G*6wR zuanm)WUpj%kl14u0Wvkc_xX+Bmo&`2-v}O5)5i}@)34f^$cFHu9E>n0dFPOK3Jvvy z@N%08yaRnamS;Puf`dHXcBUzz%QK+B8h37#<@LM|#l%mbp=*BIu;$mKp2iy2n#b=I zP$9q+n-FeB)IcgN5=Tk(J>ju9Mk2?FoF?)Jkgg$rKZlEFrYC?g?&O&{Ydb~)aGvaqq{2NtW2DxVl zwK12UT7Qo^y-ei$MAUM=Lfj9C{E*01BCmqvClp>0CGcL1paU-pF;C6ch`dhZIyIWh z^>R(9k2o2sq@@rGMEZ!lL8L(BO(GVNMIuEaoCu*uu|%XyWSPhckqVJ2NdB-Fr+yC&j2J+FhsmX zJ&q9BAkrlABO)%57RY?e2gfTRv);iN;>RS_IR@UQB89crfA_Zk8ETz(LE1ULLKee1 z;fWmn4BCS&423Q9qedHB=)19n{@3^^6DXsf9ap+C`hDVz_sPO|p-{ETwL+nJc6x?p zex?evR0c4_-)=tEno%D$3Tj!F%J|MaL#JP1)oMO<|6NwC0$^ujO*+ks|4*+OfPQ>$ zQ&3JFkj;ToL!NnZ`sp)rO)S>WoPB1cIlQN;uyi)hF6RwF|D;{Ml-F@5RIK|U{wAt@ zOr9*(D;4|)NFQ(f9if8!9TEsiBOy8LhQkUYZX8xNw@8S?O8f^1aYBe=U%6n38=N8}_*HfH_%oXYE#2zs zS5>cG{oboruRiQjsX*|w{`|Z3Kh+5NtI8aI^2mGxU%-)Y!l_Fd@t-!R&a(!?pSecI zXc$!08LrvMG%}zo+~k?Zw2|ebJje5(a!#HPZ7}hnlj8+G%!|Cl%RI|RKFNZgzDw$3 zhv5V(wd;q=1?a$LG=oO69jGFv#tbDF6cV;h9i3HRt+|C9)%GtH1h3E z%UW-{POtbu@W2*MP`eV`D>RDt-FDY%+T9S%F%WIrYV*1gWrXduoXBt;FUmSz)8|e- zADP?^?I;)LMPoGdL)*2QzPD|Gu@yduhT8mQOg8cgl;v%8A|=-KTa8luw?coz@q$4% z=6_1WD0F#td)c#t>p{SmZN6gz>vC8451nQhEPvO(QP6XW$EdV-f-qvyzrQW zaEs88aEk+a7xMch%!H$IlqBV4S=u)bNPk3*+$DSL-eSKZOR`cW{ZU!g|FJM5$CMk% z%r5JXhuJVEM|nv(=PqRL(R+XY5$j**Pds6Af|si#L*UvYZ$M2ro+&4`Ay@oL=8kIR zgfy#gWW|9QJo=cRZmdcaQbnzjFdvs-0*r5%X#bha?osiMES`{J@C#GuEta&$Hb2{= zy_?|>pExNK_9fxa$7G%INz^2=Rd2A&q#Wl{Vc`jKn1<6+BrM2DMZ*SDawzVvO2Xm& zqMYK@S#m&6XsDz?>$4okwTyZA$OeNhkHZq5IfIafrH?5P&YnRyogf6-0mA77VO3V; z)HLbO$YG6e<{X5D(>2GDy+^zXCF*^jZ2taz;;|L{!!nYC2}qtAZbyznFIf9!VhfW=)cZ8x|LB4&PZ&*B0^o$gUB z210c!CXlG`D<+Ygk){H&)}sgl!88fVP}=wcN`HZ_UrM`v_S%b;J*&P@08cA7&}$&P z39b{5YW9EC>oPjfCKXeP`dpX;-ZL=IxX~A|=#{kRFGbqbFIC#h18GXL^ikVNHBa<_ zC^tkjBKOLDY~eUo4@rN?Hcd<+8Vv<@r|UY{Md>kr$|AP3645vpyMdldg;l*@^*i0$ zb;#2L(YyRP4mMzK{tv*!wCY1oshI(FG{Y{U>{^N^Y)5@YPMwKaw2kQI zv6+(C$!!N=Q`zb3;CQi}2JBJZ*=}>}RUN%uj&hc85SiTP!(U3_UyTQL!w_bO!QIDB z{s3_Wi;ls~pW?quV`ZLZgo;eh=q=rcHqRQy+-Q4YlnJ_SJA9g|=A2SZD93#})#+SR zdbN+GfpZ4tXYj!=z&%~6r(--Hy-h-$}9abD3x_vgb$n`w31`R zk=ts_-&VY85=8W_me&3og2%ZEXs`J1%_{Vi7+^Zja6d>voZIi0YB;f?`Wm2tg~~4G z!*lKP&U2v+LWr7-hv2mUqK1e(3z5PRV8mr;J zb>9&+f4#Q31&lZw1vT5_wWcp{>g)O*Z+op8R_ZNbH+O1nujbgz2eovD->TI!hYF|m zjzbmr4%O0dsAhf(pMH9m*8F4g*ZnQ@08yzF94^Jzz}Xx7J|YPw-JnApN=H&I9MiVE zFlbCSw>oO?pb=U3cW@q!!_A4YRVY-X(PKLgLdW1qoL_LPmsGMRFl3d?k~kJ08LMvS z;38f}n}}}1zQy$5s=ALE^@^OMz}#?l0(~+T$}6HF2d;*}4IMG8@aT=yU?ez@34q;&_x*2Yd&7e5=!~ zFE;YF=jm&VDA!JiyuTI3MTHJ-DO}65J5E&SIH71aEnpU9jZWCDMgdNiTBZ~mH_cW~ zWG&oQj2*LR%2hsI=C{OcB{~NL-7Pp$3AsjLumK1}RaIx>>c{|gF``OnTP!HnkwKtM zE=Z0Fk~@@A5@^y3lq8$f9!G=Amqspj>y4Dy7I)E&_It6aIxHdr;HE6F)Mg^0>v4QB z#7)Kano8bOiC%asr}rWXaf1aW=2<-oJs z*KrMX{l?NtZ}eo|TE9`Jq5%kXx>jdvZ`Fsy8gk;BD$#4Do-V^p-*s_~r2?{^V)0b^ w5L^no@>B4sYS{C1UclCL$D|Wiv{+wJnx=#*xm21?%_jH zvL3Rls;mC`-+$Hrw^OZJ8h(?5->k4NYubO%%kf_xFK^)qe}RT;oOQG&^?HYO^`_1= zS6essO|H2)S9kM!Y`w5v*kc2%DClCdsQOZ~qTa&w$lc=dZ)^AxY~I{FFT;3McK`6wSlKgHRiHamU* zUbTf;Ce06=XpQ=4lXuZuV#{x>x?Rur`n9yb)Dyv_*XOQyZ{4hG2VcO({_m}uPUvzw z@a<^LwVmZ)+qK()AGr^s*X$kPMVR#QAMUuq_EM;oKy>Y3#dhp)EfA62TN6NHV_G3u z2|Ar%$MaY1%`G>KyuhEcZ=?uzylBmKA3E*G-U$Q`?G+Jp)uxQ5y|d=}?lx%kf-v-! zJMNrieLX`nZ-0{5i;85*(7e_E@E;10@lB#VhpT^x*;!?p_IHqjztADU!l8~*MWo~) z6*S>G?KM);^J#~qOUGVucTz>n*$g4@e(wBx#PyPd%9xk0bv zDkmTjK1t^6B^dSueB^Anq222^ZP$h?3%k43i9Bqxxw8xGndsbsTriF@(g_4$0bUQc zhj}8Ure$~4^L=vX%$mpH;hk;x$>ERAkOH1tc*3(-iM4&4Bm=VfnnNMSAwk&h) zV9SqWLAc7C(j@nnxlZ8l5U1R3&y^+xgUrEN-B4O#^iRe-kLFl+@|y{+i;^k;6BlE+-L$r-p*N`aKc7LFfclW<6x6~qP-YBRj%PN%a6mC2nIqZq zfq~%@C1I`kY>q(9qNuHZL_q|L% zx1pD$5HZM(OFc5E<@rf%Pe%akCq-U}ixYr_#{L|ey!b^efd4;UFQCJnbB~iV9JY=Z|;*Oli;H7BBw@G5N^+_?L;b zswHJS6+BfuwWt^)h6pl^+(+!=iKHIakFady#J@4^5!?J*DqCEi){+J;MkOD;ZL>!# zG=amyCD@27`{lSkU`+EHQH59H2CqURNcuPbfH5MM*VT1-GcE&Ct@6esuJ2y;~o1bzm8H2nvaPmai;BHjsQ~M1*O?He%2s7Dm zz`*%9?vv4Ye8ASnl2d#pITekkmX1%w1)NMcoA%6PBF@JXfS%x|f%(+7_DC1aXkveo zpHZB_Lc9R_h9F<6(pif)9h#WnFHCCD6yzO@q&^7Q0iF4xZcr3I1;Tk_XsUe~I6jY}m(KzIoe;4Dz7qw*ShzHPlfxpP@fsXzqeopnDQ$6T|$FnIXh=b_|z_R(B2bw=+ zYSD}9_WDbYjNSe)N%Hk8wBCT`E2MdN@oy0mGs)@xZ;xY%XW}X}Ylwe^#M9BacxDgL z8rR?tpNnS(6uEkGCO(tNcRH2Z7yYY%J%rz5;P>T>-}6I$PwoMy*HZ5IEAWd8$}iXv zj{TPh@Q>6-PVW3jBWbYzztjDFKYDThe5whgxYL;oruZqz5YK!D{x7C*pNZ>-Db+(N z|3ki^{Nex8(!U(>`3$93_|$+AuUT51o*kKfQ;E|SR{R0{NXH#|aY5>v&NJ6CB`fM$ zrkqAy%N5iLxR|H4?DA#e4J@hCe^wW-V=Nhe>M{^i&Hs$R{s~S06}lR-yfe#RpLeUu z%JEu8mWJ2Q>w@w*b>|*%jAr$Ph5qO}sLYWp8U}3QmICl^4tO4RsC1( zMYn@QTh(UmKnNPA{LI_PI_Vq1#bxT|>{V>Q@893-zqAlUc9)9Qz{l;MeaJX2y`W4H z`h5wVIG2v8OD8BnKVdq`a)}>hx-%Fk(~ToaQQ%Ql73(Yd#GwiM`SamCl8)4hhO(l{ zktExEo+H4Z9!>3-+(6i z`fK(H*2xsl^*XvFhbdpmD3b;<_(QLIIt5W)SBI}CR%b7W3&g=GG}z}xa)eCWT3otu zf2sAs{d@0!uq5;J@yLl}Q8=qDN{!?gO!L()N^~-h%kKyQqmp{Xmk5!y>_qLgR_OIz zS)!6&eUw6-*z2LllewOt+%u1yj0as=*b3biP|Nz)Z)7A}H*YN6dKcwm;Q3;LP{Qj` zSq>AlN)$v+Cwo&#k+@D!rpj@1VaIiQvUG^w+hZDAe>ZCaYCarJna7K+;>%DdDoP($G*%R|Z@YWVsPX~w1a3c#&uKAv%Fr^ zU(QlxlTGLi%v5xw%zBOKc*aq1S5(p6z*-G6D9nw9VKLn>QF!K&N>`9fPq1RH!Yu5a z*DXNRj434BGiC))fMrF~Vm3h_A-D8V%v6Bm3>(#Tb`_=lJRTi%EH;I?*YK8(?&@L> z9M4W8oKScadiccK@sDi#7 zYK^99tDRsOUf*@2wSb4pho$xYjSpMz-C1~N>0LSY!JYdpIQ`PSuiaT_-TcPwckV2S zb!-(I)NpE=)F@AvXk*bh6T?jv1Yb5-Q^>Mv;fSOtO4i%*_!-lYXcp_;K9BfP@q`!SAhlkqd(@?qWu&9E7JYZ3Mks| z+!;!gET;yFQkpaOea|`f+;h)4ckF!LknsDRU;NelU%x0x|4NzeUly5{@c3T>Fp0@^ z$)dMXmm7+u$dV&1shb*;oRp(D=?!@)vy|D8w@@Qm>y|FwSt~2vhGmF%kJW>>TF*6d zRxW9qxAG{{>V-zH)ho(U^PzZNR!Q{jw+2|6Wth&g%wRo8^VLNh?Nu;2t0r!nU+R&y8pn)EAKs!2sjs(t4Q zz%Qo&UOED>*$W{3izb!zRuB4~ojx0^@;PriB{JK6^m=CM*|_xVh3A%A`-?6vw~BjP zz3@oM^l~e|r%fv>#y|Va#pQ<=K~wARjBB>!)6Fui%xHvL792WE4TV-OR?Vvu>SUia3(0CQ#4@q?P@PMKHZ0oH24 zqUqeVtAV-Zaps%z+-rzV$(-ieqT@PuFjv#_{n}jJnKF#Gl3^~HAGgihV#*kyOGfLx zuO%SCn@BqVtFIl|{z8_d#|#eqe47aKJ3NXR5l;uIzzN5$E)tUtp4%s0+UC5o7BgbX zG~N}=7ONnbC7Q&&JM3Qau}-X3rKxScPR!y?Jd@)FC?GD{HP;ujcDQf4s}M zd8=5hgPS{UweHb)rl7#7I}P;ppG+)vnSG#ewEpIr193BJK!fPtX>QIr!J6YhWY;{i z>3Ge$BLs-3|F}J6&O#{mv=8iM(9*2iRma3O;AUgB9@NmyWcGSYBrONC!4E8~Sl~Sa z00rzAY#A(VAUHhdEYw_=2u~!TyFDbk3K`q^sYq#fuH*4v2MDE2sMsyJEsK)spQxxn_9D-MM09JEknHK^77spfi=N4u)y`xS4sxf(o2)E59G#gGkI zmveGXF%;g9lBz=G$&cF5X(ib8AE*hWC3#a35G+{Iw!oATu!1Qg5*Q~frI@;`@J~dG zw163u>P!p4R+bu(=9NKd3vBj3cuivIxNK0`kiRDf*-azpd3+z1*`Vc7?g3hYpZ*BzK?foY(x3scsqx+84T4hM^?tW zuroCLZK%MsbU^}lN6$;L_tdAW5M`UJ>fM?x&V@DI2Kn=+QI|eT1L34gy!PY)L^YFj%Wxii4C`0kFvhqfL^QASZngOXx`D{yT*&jJk#|8oYplz5K8}~ zA1Qn;O0$|>_ZC`3j2|bHVfQFA+No(gxAFL}d!z-ZsSi)JWd_;4BXNz%c*|T1kSA4! zeX0`OKT=k-+tLFKgv7{4S<<+bpw@6Axxxo9)&nI{YtUaQzge#Z$SG6jBY9cqp%|3V zDl#tDfqT_!G_Smk{J-OoTF0AEdNy||IHW7wUYNRE_o{Z?zk)=kb(HJY(+`QS+XEe5 z64@PuwvF9V!5FYK%FWel%}Uj7(lHG(nhp*YzO9?SU1O8xF%E-J;-tD-IdODCS8^Pu zt)s6vK|qH^*D8=Hani@3PdxXd3=kvb+?smbaTkI`Uz|ic7H51C$89?c*sz1@B6Pm7 zJ1QR|49Wz~0z}H%T-4*u`3}D$%$n9A5ZRSa7Pxip=5Cw#K-?X(GhMg5w2nWXqKy7zZo3Okq&8|UfPFVg+fCH6DAnX7yK3EWc}mR3oq*TK=!HNp z+D(kcRvXR8c!$if8=QN*T!=KAWBoOVo^2Hf`tUU%%k#CnaH^1>LrIiL1Omb$nFVAG0HgLFEXT|VopO|aLC-I!xXCJw&b2kT4`wY9!rT< zGmo^=LbMe&Q+kG7{n)WmlT*hFnn|vK*<~EqP$8NbbdWZkbWBmU;^Xo08DTsFZHl2M*oT#n>jf3m?6lA*)TOz=2;%F?) z2&opQ#(_$P%u1i6UnrSkn-~Zs6M~5yMI&^iSiT2|j!|pu?gKe0cJ@_g`^9TaXOB=i zzfTELI<4ch*rZF69tRydLLYC-lLw-CfG#B2k5LrUmSz49rrJ@1{yu>d7=7Ns=)g&y zgcsuEAbX#`PvAEQi0ls4Uv%aGDnhNCM?oYX?CS`+#?e5qf8Yb9EqojXV>48S68av< zE#;Carg@+WEgvf_I=c>{@{cUH7m<-sMev@RVa=4h|$Q@pfd%&BIDO-lwTw8>j073sx`|!`~#$|oH(s2 zci={OoWDlZX>T=vpJWwi0*AqCqO8A@PecsvTU_4}QYe3J(E!G7iPe`h+C43-d^}~+ zDCVzNW?@tH?vvynVMPBJfTWb5;m4uzOXRXae+|%E$*JSe`Qyr{!acOA(#hWW<&dC| zA~C?f1|7!aw#Jk;95pyX8cRX12puo9N+yt@zsU(wg%gyHouK>jhv~M?GUV`(Mu$6d z5n4=t9@;LH;&Qk(^rw&xgiG|(b`~=kVU}f4V?frjtVcLJIXFCUuG$71D$+lH1XrjR zwS^IBP{N1?@_Bj(a2hvLtcV%5 z+gOGt(@MX5b@m1n!&6jSxPEiyog24qU!DEd&6!KaY}w%7psp_wxJuv}0XnL&@!TpI zb;cW4->bZSW9F6FSEH0Tyrcd%Z``W9b+s~k^V>IOD%akA`IQ^9kugU$bj7c=oN|tT zn}+Cz`GhD)9N(-2-m>HRkuJuC1t0HJV`nGbv3g(!cHFT@^Pv{_4B>N=0I5k!Lj>5h z3>r5cZXbuEzDIcar|2h~D=9IwN-(Ml*m|Qdy2kM?!iYE}Ps)a<*HNa)dWNr~-=2*) zfdY!B2xEdlNXVE76mdkuh`OgUmA2r9%qN)UYM&s=k0V%va0}xATT+7=1v6m-2AFTX zCLL5WuZT!WDo7(C}YPFQvqAL_ZG!fYt~0H(zK)Q3UaK-{GV zPYzEWPXVzOY;?AgxvSP+O2zy~56+2M@%H5F>*SLjIdeOEJ z<`7}&6%iB-F_J=9WN5>R6#d}$!XDHX8KNwtKt!w0k{10U0zvJmb^t1(FX-Fs+d_0k zD2{&&`!;npi25PHuf9&24S%k*ONiH?KCWeh;OB1r2-S!A%_8Um4{26`N1sR)fKf?c zQrN|g@&bGCOW-xR@+BxV5bK&j>_Uf$hPV~2-zSK{U_X=io~wM)TxSPCJN*e%5W(&phU za#6_0&QU4e%4NddxS!^~84m2PV<(5C%hJl1z;_fpLX56G#Io(-aG0HfL}>i8*5^nY z&r9v$V1hjr4kxiBK?!jsI5Hr0(eu?ldQR`5N8_da^Z;j*mD!ms1@EWXqOm~KeHCS;fHRi|eju}{c46mG~M z62d1aP}~PX3(Ky{+j$^!juyfaL7EKX__*A0L;i`1cAYP}aY(M`4syQAnx*2F72x8s zrM!S=dYNZvhzkG*qA7fc$A_n4V}|Ub9hZ#lJJE$p_Rk$W`AMX#Y$rT(L4 z8rdJXk!v}o>mXDZc(~m%FNm=0t*ft`rPvSdIy!?$Gu;UyVhTc% zduF&GJGXfVpNU&hF!L7raN<7F^ok3}PTaFjWXljrVx9K8m# zP^(y@a^hx^W5b^zAPijrb|fqhheb#jC#zamun$e7TAAd|9nsqO(||pA5x<{Q%z2AO zra}3~1ok3eb`Y5TW;q>Y$Z}jj{IRraLRPpVY*evXqRguAR3PP1PvA6~BpIvRiLz^U zEr`SGddFDjvzV=A%yNX1!IN{({6A6fe20RT^1(4qt{^8fRCqAk@<%9rs?{$dpeEg@ z?x4ddp5Ia-NG)a3JPg@GbC#e*HMiEKkFH_bhkSEX&5M^)NtvcdjgMW((iGfk7 z<}wEXZt?kmAFm)jaFIl;qdZ|{;QCOAEYcCHh*O2#AgKtB3I%=#eGtZo?@Rm&ePrNU zIX|#DMY?f~6LfNkgCs&ENxUKsk@!|`;;V^Xh|@*cj+ch9YWNUgmC?o5UEea|=&MNQ zyCboF?3G<05=b%f?-E!e@CJa@+Zi#zimy`DG(eOVGKzbeMBhfT!-W!|$ct4>s7BN# zu8$>7Y;962k%+Ez9aXFsED+@tO%)LoB|49>e5BY8VYY z6{K@2jVkK3(}?RBB7#4Gc@@+lz9W|uMV^+^@+CY9#y8|)luaTRr(RR|7oekj6q^Fq zBN@7Rsn_PXIDVc(O_aumD-H((DWg}!4NRqiD;C_2;p>*mDCWOQ;8_AfPl$c;2h_f> zQ1|BSdZpn6iyk6fB+FL+&KCz$6nqt59Edv?{weiIVV^YDd6kA+As~ePGfMp-f!`zW zLjolG@y>jYQXT=?TY{u6wXvh0BmFY8!fM^Q0{tQR`1(ttFBwJ^g!Qp36dvmY`IAeK d_$owJ{xhv>ik8-Tw5+D;dHqGbPao5d{|_Fr=0E@d diff --git a/task_eval/__pycache__/hf_llm_utils.cpython-39.pyc b/task_eval/__pycache__/hf_llm_utils.cpython-39.pyc deleted file mode 100644 index 23a440cbecf29562c557000c86476c584a9c3433..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8763 zcmd5>U2GiJb)K1>-JPBN<&P**mgSLR$!1}a`m^P#EG>$nZPg?#iBlSI`>G`U8&!IL{ixi)=bw=!iAhYZ zNoC5Fnp{`PiY(dET4GmalAW{_JGCRPrPtCs@*a8wv{u$co+)QUu9tP>iCVUvEoTLc zTFce*<$MgEE9V4Up?UwMS}mrt<)Hu#}beu|x7LFlWZ9I(`P&w_T@Y^U(xX z0T>Qa>%7P8+N&O)v6{YBo0|{&Z#scL?XsKOf$h(D?ut`&q>>~#?@uAQ@eC5_zhxvH zNgC}7Q?;5^w=VDNQ9gF815n{z%h)wrbbw*4VYmm;;1$#}Bwwqw1y0FR$%U-ZaVJ2R;jel;Ff7;KvbnM1aWP`WtJSM_UetAP54}9)NHpJtZo~=?FP2HYy)^YLciq%tA_o+TE;lSVytkl zE+#xeoUyfPyEce58lLYvm6|=S>u<$`TsPk9=+-sd+AqDx-X&sAtkJfXR}F9F3)bpC zvDmK)W(i)3l?Vu`0$H69S;m?VCNXE3m}T*8S_8MEiNFUmV2Oc+vmz!vV5{4#SFlja zH=bL!x1SRXB*eWmQ}3o#YudjQMp^^JyGeiQ-P;R`OZSb*$;EdTr>69GyryA+WXo8! zYmJp=?FEC|e#2f4P!@EpdM*wFsPS=8EEk)&>iM>@=2Z*_%kqq-%W%Tb#UA5q@Ae8* z@pRU3h|Zu?k{wQL)> zz1*lbYk`Ad2D7$-J!UbpaOrUV#DPA703_gw?WN?T#_h!iBylH8NqiMG0z^p0%DtK$rqPd;2CpumNeg0y6rQjcGpmp#Tx`7AfVgP{kH|9k z>G%Jbe*f~yPyg{B==UG5Ji@h*KFNfIy7j<>;57vrKg=~O7L`qhg*n^v4{I6bnZ8A` zFH8nr!(0!ufn9IVd`%v=HNq0N>ByM4M>R9B9|YkbuDuzz`ZjI>iAhcbgoT)18oDf*uOsE zTK<^tvk8lBTe!Uw*swJSAb(=cn}J0!5m^4ZX+suHtge`~THS00PR$>0Y=>Fibj?^H zz6)f&fkaYtc{ph(imWJ_tm2PS_y0esJf8;4uilC0CqUL{hL0oT6O>#;5}tT%c7A^P zaDSsS!RglWE0kQNDl+4juE_WZ z7S2ynGK}QaC4QR9Fr_7KP;!=%bCi6A5)v?c49Sy+Yh-S2dVczfIrIATW0K?lalnLS z{$0%DKaV83g*mzOxgx8drxIUy8(#(NGSS%mkuIX|i2i}*k>H5Q9hu1;B}g!ZC3cis z(*3&~9Itj_NNUTQ5D^oY?3NA1E+3>vL2>kI6;H+mU7()^<`KkZY0jt zP$f$-RBN8aLe*>7ZkXhjTeU}%p=PCzLFJq=q72;BM!W46?npb`NGM^wVKmr|3 zjp;Yx$#BQ^e~6)e0g0p(An^)vP9Bs?GJhK_%jBPRe|;#Bh8NuZ7<><;wi4<09l4_h z#9NsuA4+XFw#2a=nf}oI>Ta@~Wa z0?%%0PvGg~f^?v@b31a7p_IW=pgsIZzOQ%kOmFA0i_#|Q{6owx>bUUqPk&p`a{tp# z0kag_1(rom4!c(%n*lprz*od>7g?!Yz)AS`c9Hd=7vf?_x`uYti83Q zxem-ThK2lrvY}#-%;(T0&dsP2D&rSJnLU1r^vN2`dGg?RR-EOWv+sO@-7bE zo=nGXM~>H`;!;Jkg_)kaX7aBC7gm>W+#pQ)jhYjTDjYE%DU{bATP%ME&3}iFmnpeT z32~^U8NTJP7mRZVZNTcrS0HK$=NTMcsST18=Z3%7z9=l$_k>elqq>~(|x z7U3w&yt(l9?A^ubrQ2^VT-TRcX+7#5Erba;#bF9&9-<<+`(R+0@a-V%qcP^<()8V> z*iaJ+KUCoi`ur^-AyIc+B-VpapP&AkIXAoT+S2P`vQF`pNabGv*l3@S8T>bBh$be2 zrO&UUT0XSKVw1KIODErSt-1{pH*#$v^SE1!VEH5=%mrRx!FDCX+7Gj!$n5Tg>cjBG znD`*Tu=V*8#_)SIP$k1R)DO++$2Wv5n;+xziBc(1NSuPSDkx;D(Z63l4GDG%rPHzo zDW)YZ$QMu=6cCzn8o8!uim;H57%4@dOXCyL?NeMA94{5pU{Dw=iF>ne_Btgk7NM!63`v0D!>=XUZLRv@tKMV55^s7PP0~y*S!doCNwhOyOHpouw zDZ8aT37TVworGR1?DnzYJ()Bc(ad~#LUa~l1`SAq2M!`86mbab@keBcwQd;CTcZ@Xc)mi}xUp#0;Gl{4!m`IX zm19g5#J~_XBZ_DYqHY*7UIiBXK6|A#yx{dlWejW6a%vWWePJrX0GTdZplHjmnXB>8i*iLb*;O zOgp~cKBAIEOXk%h zFLIhjk&(?wD24+IUT~XY7j+8MvRTeO9;Uf1f=>L~1ZNRk5)mx~_lw|)OIEHGY90a) zR1*4yzeG^FP)I_d@CG#$_xz=PhA}SAj*vE)Tycm`9Q8{L0 zq)FZ*Qs_kU-=X9&mlgq&Uf%yJKz!nl1eZUj=*qbRWipf~$tkoA;w#8Xf(&Caj|)&N zC8dPCAnS52Q9{@Ny-0^p7oVak1&kgR@H!M!LAfdqCsGRkBg}NUq=~tbqE0s25RxU(}~Wy@dKltPlO;qTY}CE*n7o3>%~tx)W&ol%1d& z)Q9ZC1^7T_L*!)bLTL$cFRdjZn7PMA6yAsj-4 z7lJH1739|P>?uH~Za>x8Y0+|uJ^fJasO$_mY^1!{GjVAlC=gx&Wxy>vi!tXOrl7L> zqn^0!D;S@2hYq3R_Gcd^;ieSVO6)w;N&=(LvI}TSxr1?Qg!?Re4j58c&GWPxsB?Km zf!#d1z-b#>!*x5b#)Pz)n4|ZovCH(zv%11*hg&1Zbzc zb8PeC_$9y&5E$Oyq~l+hE?r)4ojZ1XcfJUb%=>`@o(8#-Ut+{%fKp7k)nE0zYR#VL zd4l6D>d+5oEuW`2-|R+pe&)fO_cpiZHfGjuz4ONHm1Xx1yLE4MW?^%7$*Z+a-1AA# zTpg=80Vx$cy&$5B2TDX`VY2ca;9Y_~$&uHMz^Qo^fZgK)ITye3qaDS*nEgB7$?;&!=jrG|K>X$2Id zrr5z{1WNHp3ZFk4cpM>EF>^Qo2-kGFLRmzu!vTTRX+<$?#02ni8tTm6be3(iZZ*O_ zJU{vY;>mRnYakZ`+xzFBk?&HnhXl*{Qz{>AZ57Gc89R)S1B?_3|Ml)TfdSY=tH7{} zu2yaie8dBzkoj;uME7^m6Y6i%Lsg_pimS~ko*b$x(4^*SvqG8#^|D3`qaEPCNAr-P z=D$nH1xn6Sl0rh82iAn?73^AY=_O7U2wV}UG@s(&cxn==XEYtDq3@x)tl_{!droU8 z>$FJ`%OWC4S3Ryl@{a2W6(W{%2_oD7Lx54-D)ps4D3OG!sAHlu3%p(f{XKIu(`vS4$nV|00z-fRL4+YniAfFld_0e*n5h&%t@z_QP}o zukkgf!pXb~RjP$4yfE6Fyvu-xnkk}5rWvLhcnZTa5^2oxn8{sc_vZhZ a8c?%pQq}QSkt=FKJB9yKT2^~OyYqjxqEv(c diff --git a/task_eval/__pycache__/rag_utils.cpython-39.pyc b/task_eval/__pycache__/rag_utils.cpython-39.pyc deleted file mode 100644 index 6921693f42b6342227b9055e8d7c7a7e6dff4ef5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4956 zcma)ATXP&o74Dvy-JQK^b+K-C62+CT#Qz5Tjx-}!ppz6s9FH zMONfK;`LeKi~L8dJ|_yIi1)lGi89^`VnNJ^Dtep|XT+?SLupa)cUf)m4!)Lc*0}NB z3iU3%^}cA=T(htf>7dh%lCT*>?X5_NX0l@{YO39f4$X=@NP@<0sCT+L_^_#WgXl)s z?#7WaMU)ykXotG7+i+=`(Lq?1@P#lJ8H{9sTPAh6?wed#h<<6U(4IQezAi5sL8#3Gp4sHgJfrp>n_-~H(=I5o3bb1rV~jLqI5N8aYf6-^4ltW_X$U*5?)c9wiMdXdLLu`e!K4!MrK8qc91&u!b~v7#c-k%&^vTTt1Fc zgT7+?5LD1=?MnZJLdpb!Q@$avvY54bAE3~8+HT)%dws8+>1Q}gGlQ(~5a!v`8I}gQ zW7f~5xq*L-fJ(DzZX4@|tqg2^3|m>)%J=7g?H&)rb{=u4fFCQyx)~Jq;F?T{_%K9Sk%!C{Yuj4 zh)9|_sNOz^lx}vCvF_|?-4)ubW)A~aPuZBX^`6{DT40m+HZcEsL*EFnku=S^!bZi> zda}R16{^S-8l6PTW^_H0w;fad;eqZ*cYkO};rX1);}@?3imNOR`7!iR^!?27ks}3O;Xd~r->IH1L!0HED_=nG zd-A9V#UzjZiON^XBX5#N?pMg8U!BT>0y6NcWYC{Zjid~u3@X_1D2+0REKakifGqm^ zD2s|GvZ%BvpTHVz7Re`3>Rp(|5~3S%W|QUtAuiWx@I?|&lX!;2H%L57;+rH0b!$cW zJe405X7#zrMAJy9Omz2$!uQG6eeqDW@nG{Ju>CKilcE?t9Sz=nk>S}WvGMQ&^f$Rt zSlz=R=+v2Th`#`OQ^x}AqDF`u@vBbX?R!Z{^J#5|^)tfhXSG8-?3l@K3pa6u*UxEZ zkU3_b@lUUErn5RHGTJ9Lk!Fv;8!n^e+OvHhp_j$R{T>W}r#}9&OYE1<3KKbEUAJ6O zz%k|mu=A;x=E3jsg92&=z8J7#EZ2GX76O~(D zznFU4tjW^i=lm1iFX@@IBr@PDT+D#?IRD3f)!I)UY=2Nq>BzWemewp7C)Or1!}m5A z)^MH_hr;N~XRYN=hN~OQ>3QUSp@61;K-Y$`(@87v;N z!O}5trnyb|BDVL=GDI&+2>0Df#%**DhKWHbVw|qN0vN9V)(&J6;Ph4<37*sqhdedk zOu7d;XbN@bzj)*$7-aL~wkv-`1_~O?wGq-8Bwry#CDgD&bx>BdH_RSNoZ7O*C24~( zqdT&(E5Ab{N=+4n*TZHUZpD$wC7obLhC&iv$psSg)T>OKA(BDoK;w&Wl0-UefZ2sz zus30xBr%0;d`p^WQQx3M5pBUsH2O; z$d{;;Nx%XRYh|0|L@O~bQ5`_I$xkxcRtGz20m@a)su3 zm8=)09LePAytVdCZn(Zvys1Lhi0P1Rm{0^vgaZ|)B;t@I06+@13h07DgBPAh}TF}@kIxfYTt z7LsNFNizgSX+=D2A?d;Pd$9kAwf~>{>{mxNt7NlP9RbkH2>^Pq)yH6U4OShwJY3me zR?nsW7>*K8&)?)H!DxRKVC1N0PJ+??Dm=0PIGQ1E^fNfGGsL!Qk8Voh`*$d%dn>O- zT7y`wjSi(Xn?z73x;l$UHF^aVDMc{yxauuT%K>g*QJ|ac$ly#<-6-6PWY1qyVN+aK zJ71q^blPBcO5y54Oz|#1Z+HQ;HKvOPX^h`&@7Pc~c3v!I5&STb7_eYCV^|?vYw#lr z=$2p5%ykl5BtjDR(VImAwedVtaJz-BioD7!tStlDO8Bcfp5yQ}0NHtGnO6a3RbFxc zY9qL#+8FSvfII>|0G?ZM<%h6U<1#mVy=Y&XwIy6=5A9*6cjM-k1oAOSjEKNqMN*?( zOXCMNSc1T0yNDC=KZTzxQFpqr54VrbD^uBtJ6mBKv?IOSf#d9^5Z1~p3A)M;_l+pJ zJqj%x+Ydb99qXra=wkfmao!xn(F=4fR;v*H2<5A|2bB1NSD5v){+fTj@Z|ph`6uUs diff --git a/task_eval/claude_utils.py b/task_eval/claude_utils.py index 44f964a..f14628d 100644 --- a/task_eval/claude_utils.py +++ b/task_eval/claude_utils.py @@ -151,13 +151,23 @@ def get_claude_answers(in_data, out_data, prediction_key, args): if qa['category'] == 2: questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.') elif qa['category'] == 5: + # Check for both 'answer' and 'adversarial_answer' keys + answer_text = None + if 'answer' in qa: + answer_text = qa['answer'] + elif 'adversarial_answer' in qa: + answer_text = qa['adversarial_answer'] + else: + print(f"Warning: Missing 'answer' or 'adversarial_answer' key in QA item: {qa}") + continue + question = qa['question'] + " Select the correct answer: (a) {} (b) {}. " if random.random() < 0.5: - question = question.format('Not mentioned in the conversation', qa['answer']) - answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']} + question = question.format('Not mentioned in the conversation', answer_text) + answer = {'a': 'Not mentioned in the conversation', 'b': answer_text} else: - question = question.format(qa['answer'], 'Not mentioned in the conversation') - answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']} + question = question.format(answer_text, 'Not mentioned in the conversation') + answer = {'b': 'Not mentioned in the conversation', 'a': answer_text} cat_5_idxs.append(len(questions)) questions.append(question) diff --git a/task_eval/evaluate_qa.py b/task_eval/evaluate_qa.py index c3e888c..229848f 100644 --- a/task_eval/evaluate_qa.py +++ b/task_eval/evaluate_qa.py @@ -5,7 +5,7 @@ import os, json from tqdm import tqdm import argparse -from global_methods import set_openai_key, set_anthropic_key, set_gemini_key +from global_methods import set_openai_key, set_anthropic_key, set_gemini_key, get_gemini_client from task_eval.evaluation import eval_question_answering from task_eval.evaluation_stats import analyze_aggr_acc from task_eval.gpt_utils import get_gpt_answers @@ -14,7 +14,6 @@ from task_eval.hf_llm_utils import init_hf_model, get_hf_answers import numpy as np -import google.generativeai as genai def parse_args(): @@ -50,12 +49,12 @@ def main(): set_anthropic_key() elif 'gemini' in args.model: - # set openai API key - set_gemini_key() + # Get Gemini client + gemini_client = get_gemini_client() + # Map old model names to new ones if needed if args.model == "gemini-pro-1.0": - model_name = "models/gemini-1.0-pro-latest" - - gemini_model = genai.GenerativeModel(model_name) + print("Warning: gemini-pro-1.0 is deprecated. Using gemini-2.5-pro instead.") + args.model = "gemini-2.5-pro" elif any([model_name in args.model for model_name in ['gemma', 'llama', 'mistral']]): hf_pipeline, hf_model_name = init_hf_model(args) @@ -89,7 +88,7 @@ def main(): elif 'claude' in args.model: answers = get_claude_answers(data, out_data, prediction_key, args) elif 'gemini' in args.model: - answers = get_gemini_answers(gemini_model, data, out_data, prediction_key, args) + answers = get_gemini_answers(gemini_client, data, out_data, prediction_key, args) elif any([model_name in args.model for model_name in ['gemma', 'llama', 'mistral']]): answers = get_hf_answers(data, out_data, args, hf_pipeline, hf_model_name) else: diff --git a/task_eval/evaluation.py b/task_eval/evaluation.py index 8f597dd..57e0199 100644 --- a/task_eval/evaluation.py +++ b/task_eval/evaluation.py @@ -197,9 +197,23 @@ def eval_question_answering(qas, eval_key='prediction', metric='f1'): for i, line in enumerate(qas): # line = json.loads(line) if type(line[eval_key]) == list: - answer = line['answer'] + # Check for both 'answer' and 'adversarial_answer' keys + if 'answer' in line: + answer = line['answer'] + elif 'adversarial_answer' in line: + answer = line['adversarial_answer'] + else: + print(f"Warning: Missing answer key in evaluation line: {line}") + continue else: - answer = str(line['answer']) + # Check for both 'answer' and 'adversarial_answer' keys + if 'answer' in line: + answer = str(line['answer']) + elif 'adversarial_answer' in line: + answer = str(line['adversarial_answer']) + else: + print(f"Warning: Missing answer key in evaluation line: {line}") + continue if line['category'] == 3: answer = answer.split(';')[0].strip() diff --git a/task_eval/gemini_utils.py b/task_eval/gemini_utils.py index 99204db..29e905f 100644 --- a/task_eval/gemini_utils.py +++ b/task_eval/gemini_utils.py @@ -6,10 +6,10 @@ import os, json from tqdm import tqdm import time -from global_methods import run_gemini +from global_methods import run_gemini, get_gemini_client -MAX_LENGTH={'gemini-pro-1.0': 1000000} +MAX_LENGTH={'gemini-2.0-flash': 1000000, 'gemini-2.5-flash': 1000000, 'gemini-2.5-pro': 1000000} PER_QA_TOKEN_BUDGET = 50 QA_PROMPT = """ @@ -87,7 +87,7 @@ def get_cat_5_answer(model_prediction, answer_key): else: return model_prediction -def get_input_context(data, num_question_tokens, model, args): +def get_input_context(data, num_question_tokens, client, args): query_conv = '' min_session = -1 @@ -103,9 +103,9 @@ def get_input_context(data, num_question_tokens, model, args): turn += ' and shared %s.' % dialog["blip_caption"] turn += '\n' - # num_tokens = model.count_tokens('DATE: ' + data['session_%s_date_time' % i] + '\n' + 'CONVERSATION:\n' + turn).total_tokens + # num_tokens = client.models.count_tokens(model=args.model, contents='DATE: ' + data['session_%s_date_time' % i] + '\n' + 'CONVERSATION:\n' + turn).total_tokens - # if (num_tokens + model.count_tokens(query_conv).total_tokens + num_question_tokens) < (MAX_LENGTH[args.model]-(PER_QA_TOKEN_BUDGET*(args.batch_size))): # 20 tokens assigned for answers + # if (num_tokens + client.models.count_tokens(model=args.model, contents=query_conv).total_tokens + num_question_tokens) < (MAX_LENGTH[args.model]-(PER_QA_TOKEN_BUDGET*(args.batch_size))): # 20 tokens assigned for answers # query_conv = turn + query_conv # else: # min_session = i @@ -119,14 +119,14 @@ def get_input_context(data, num_question_tokens, model, args): break # if min_session == -1: - # print("Saved %s tokens in query conversation from full conversation" % model.count_tokens(query_conv).total_tokens) + # print("Saved %s tokens in query conversation from full conversation" % client.models.count_tokens(model=args.model, contents=query_conv).total_tokens) # else: - # print("Saved %s conv. tokens + %s question tokens in query from %s out of %s sessions" % (model.count_tokens(query_conv).total_tokens, num_question_tokens, max_session-min_session, max_session)) + # print("Saved %s conv. tokens + %s question tokens in query from %s out of %s sessions" % (client.models.count_tokens(model=args.model, contents=query_conv).total_tokens, num_question_tokens, max_session-min_session, max_session)) return query_conv -def get_gemini_answers(model, in_data, out_data, prediction_key, args): +def get_gemini_answers(client, in_data, out_data, prediction_key, args): assert len(in_data['qa']) == len(out_data['qa']), (len(in_data['qa']), len(out_data['qa'])) @@ -134,7 +134,7 @@ def get_gemini_answers(model, in_data, out_data, prediction_key, args): # start instruction prompt speakers_names = list(set([d['speaker'] for d in in_data['conversation']['session_1']])) start_prompt = CONV_START_PROMPT.format(speakers_names[0], speakers_names[1]) - # start_tokens = model.count_tokens(start_prompt).total_tokens + # start_tokens = client.models.count_tokens(model=args.model, contents=start_prompt).total_tokens start_tokens = 100 if args.rag_mode: @@ -164,13 +164,25 @@ def get_gemini_answers(model, in_data, out_data, prediction_key, args): if qa['category'] == 2: questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.') elif qa['category'] == 5: + # Check for both 'answer' and 'adversarial_answer' keys + answer_text = None + if 'answer' in qa: + answer_text = qa['answer'] + elif 'adversarial_answer' in qa: + answer_text = qa['adversarial_answer'] + else: + print(f"Warning: Missing 'answer' or 'adversarial_answer' key in QA item: {qa}") + print(f"Available keys: {list(qa.keys())}") + # Skip this question if no answer is available + continue + question = qa['question'] + " Select the correct answer: (a) {} (b) {}. " if random.random() < 0.5: - question = question.format('Not mentioned in the conversation', qa['answer']) - answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']} + question = question.format('Not mentioned in the conversation', answer_text) + answer = {'a': 'Not mentioned in the conversation', 'b': answer_text} else: - question = question.format(qa['answer'], 'Not mentioned in the conversation') - answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']} + question = question.format(answer_text, 'Not mentioned in the conversation') + answer = {'b': 'Not mentioned in the conversation', 'a': answer_text} cat_5_idxs.append(len(questions)) questions.append(question) @@ -190,22 +202,22 @@ def get_gemini_answers(model, in_data, out_data, prediction_key, args): raise NotImplementedError else: question_prompt = QA_PROMPT_BATCH + "\n".join(["%s: %s" % (k, q) for k, q in enumerate(questions)]) - num_question_tokens = model.count_tokens(question_prompt).total_tokens + num_question_tokens = client.models.count_tokens(model=args.model, contents=question_prompt).total_tokens num_question_tokens = 200 - query_conv = get_input_context(in_data['conversation'], num_question_tokens + start_tokens, model, args) + query_conv = get_input_context(in_data['conversation'], num_question_tokens + start_tokens, client, args) query_conv = start_prompt + query_conv - # print("%s tokens in query" % model.count_tokens(query_conv).total_tokens) + # print("%s tokens in query" % client.models.count_tokens(model=args.model, contents=query_conv).total_tokens) - if 'pro-1.0' in args.model: + if 'pro' in args.model: time.sleep(30) if args.batch_size == 1: query = query_conv + '\n\n' + QA_PROMPT.format(questions[0]) if len(cat_5_idxs) == 0 else query_conv + '\n\n' + QA_PROMPT_CAT_5.format(questions[0]) - answer = run_gemini(model, query) + answer = run_gemini(client, args.model, query) if len(cat_5_idxs) > 0: answer = get_cat_5_answer(answer, cat_5_answers[0]) @@ -225,9 +237,9 @@ def get_gemini_answers(model, in_data, out_data, prediction_key, args): try: trials += 1 # print("Trial %s" % trials) - # print("Sending query of %s tokens" % model.count_tokens(query).total_tokens) + # print("Sending query of %s tokens" % client.models.count_tokens(model=args.model, contents=query).total_tokens) # print("Trying with answer token budget = %s per question" % PER_QA_TOKEN_BUDGET) - answer = run_gemini(model, query) + answer = run_gemini(client, args.model, query) answer = answer.replace('\\"', "'").replace('json','').replace('`','').strip() # try: diff --git a/task_eval/gpt_utils.py b/task_eval/gpt_utils.py index 64d65df..2aadc33 100644 --- a/task_eval/gpt_utils.py +++ b/task_eval/gpt_utils.py @@ -243,13 +243,23 @@ def get_gpt_answers(in_data, out_data, prediction_key, args): if qa['category'] == 2: questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.') elif qa['category'] == 5: + # Check for both 'answer' and 'adversarial_answer' keys + answer_text = None + if 'answer' in qa: + answer_text = qa['answer'] + elif 'adversarial_answer' in qa: + answer_text = qa['adversarial_answer'] + else: + print(f"Warning: Missing 'answer' or 'adversarial_answer' key in QA item: {qa}") + continue + question = qa['question'] + " Select the correct answer: (a) {} (b) {}. " if random.random() < 0.5: - question = question.format('Not mentioned in the conversation', qa['answer']) - answer = {'a': 'Not mentioned in the conversation', 'b': qa['answer']} + question = question.format('Not mentioned in the conversation', answer_text) + answer = {'a': 'Not mentioned in the conversation', 'b': answer_text} else: - question = question.format(qa['answer'], 'Not mentioned in the conversation') - answer = {'b': 'Not mentioned in the conversation', 'a': qa['answer']} + question = question.format(answer_text, 'Not mentioned in the conversation') + answer = {'b': 'Not mentioned in the conversation', 'a': answer_text} cat_5_idxs.append(len(questions)) questions.append(question) diff --git a/task_eval/hf_llm_utils.py b/task_eval/hf_llm_utils.py index 7d16cf6..3f2afbf 100644 --- a/task_eval/hf_llm_utils.py +++ b/task_eval/hf_llm_utils.py @@ -252,13 +252,23 @@ def get_hf_answers(in_data, out_data, args, pipeline, model_name): if qa['category'] == 2: questions.append(qa['question'] + ' Use DATE of CONVERSATION to answer with an approximate date.') elif qa['category'] == 5: + # Check for both 'answer' and 'adversarial_answer' keys + answer_text = None + if 'answer' in qa: + answer_text = qa['answer'] + elif 'adversarial_answer' in qa: + answer_text = qa['adversarial_answer'] + else: + print(f"Warning: Missing 'answer' or 'adversarial_answer' key in QA item: {qa}") + continue + question = qa['question'] + " (a) {} (b) {}. Select the correct answer by writing (a) or (b)." if random.random() < 0.5: - question = question.format('No information available', qa['answer']) - answer = {'a': 'No information available', 'b': qa['answer']} + question = question.format('No information available', answer_text) + answer = {'a': 'No information available', 'b': answer_text} else: - question = question.format(qa['answer'], 'No information available') - answer = {'b': 'No information available', 'a': qa['answer']} + question = question.format(answer_text, 'No information available') + answer = {'b': 'No information available', 'a': answer_text} cat_5_idxs.append(len(questions)) questions.append(question) cat_5_answers.append(answer) From 095638dc618129d65848a75b9f46cf6282367c54 Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Thu, 10 Jul 2025 09:20:11 +0800 Subject: [PATCH 2/3] Add category mapping and stats to QA evaluation scripts Introduced a CATEGORY_MAPPING dictionary in evaluate_qa.py, evaluation.py, and evaluation_stats.py to map category numbers to descriptive names. Enhanced output and result dictionaries to include category names and detailed statistics, improving interpretability of QA evaluation results and reporting. --- task_eval/evaluate_qa.py | 13 ++++++++++ task_eval/evaluation.py | 9 +++++++ task_eval/evaluation_stats.py | 47 ++++++++++++++++++++++++++++++----- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/task_eval/evaluate_qa.py b/task_eval/evaluate_qa.py index 229848f..e0c34b9 100644 --- a/task_eval/evaluate_qa.py +++ b/task_eval/evaluate_qa.py @@ -15,6 +15,15 @@ import numpy as np +# Category mapping for QA evaluation +CATEGORY_MAPPING = { + 1: "Multi-hop", + 2: "Temporal", + 3: "Open-domain", + 4: "Single-hop", + 5: "Adversarial" +} + def parse_args(): parser = argparse.ArgumentParser() @@ -100,6 +109,10 @@ def main(): answers['qa'][i][model_key + '_f1'] = round(exact_matches[i], 3) if args.use_rag and len(recall) > 0: answers['qa'][i][model_key + '_recall'] = round(recall[i], 3) + + # Add category name to output + category_num = answers['qa'][i].get('category', 0) + answers['qa'][i]['category_name'] = CATEGORY_MAPPING.get(category_num, f"Unknown-{category_num}") out_samples[data['sample_id']] = answers diff --git a/task_eval/evaluation.py b/task_eval/evaluation.py index 57e0199..9c8450a 100644 --- a/task_eval/evaluation.py +++ b/task_eval/evaluation.py @@ -12,6 +12,15 @@ LENGTH_THRESHOLD = 5 +# Category mapping for QA evaluation +CATEGORY_MAPPING = { + 1: "Multi-hop", + 2: "Temporal", + 3: "Open-domain", + 4: "Single-hop", + 5: "Adversarial" +} + class SimpleTokenizer(object): ALPHA_NUM = r'[\p{L}\p{N}\p{M}]+' NON_WS = r'[^\p{Z}\p{C}]' diff --git a/task_eval/evaluation_stats.py b/task_eval/evaluation_stats.py index 3283edf..c8af2e3 100644 --- a/task_eval/evaluation_stats.py +++ b/task_eval/evaluation_stats.py @@ -3,6 +3,15 @@ from tqdm import tqdm from collections import defaultdict +# Category mapping for QA evaluation +CATEGORY_MAPPING = { + 1: "Multi-hop", + 2: "Temporal", + 3: "Open-domain", + 4: "Single-hop", + 5: "Adversarial" +} + def get_conversation_lengths(data, encoder=None): @@ -92,20 +101,25 @@ def analyze_aggr_acc(ann_file, in_file, out_file, model_name, metric_key, encode print("Total number of questions and corresponding accuracy in each category: ") + print("Category | Name | Count | Correct | Accuracy") + print("-" * 55) total_k = 0 total_v = 0 # for k, v in total_counts.items(): keys = [4, 1, 2, 3, 5] for k in keys: v = total_counts[k] + category_name = CATEGORY_MAPPING.get(k, f"Unknown-{k}") if float(v) == 0.0: - print("No questions found in category %s" % k) + print(f"No questions found in category {k} ({category_name})") else: - print(k, v, acc_counts[k], round(float(acc_counts[k])/v, 3)) + accuracy = round(float(acc_counts[k])/v, 3) + print(f"{k:8} | {category_name:11} | {v:5} | {acc_counts[k]:7.1f} | {accuracy:8.3f}") total_v += acc_counts[k] total_k += v - print("Overall accuracy: ", round(float(total_v)/total_k, 3)) + print("-" * 55) + print(f"Overall accuracy: {round(float(total_v)/total_k, 3):.3f}") # print("Total number of questions and corresponding accuracy by memory") # keys = list(memory_counts_og.keys()) @@ -123,19 +137,40 @@ def analyze_aggr_acc(ann_file, in_file, out_file, model_name, metric_key, encode results_dict[model_name] = {} results_dict[model_name]['category_counts'] = total_counts results_dict[model_name]['cum_accuracy_by_category'] = acc_counts + + # Add category names mapping to output + results_dict[model_name]['category_mapping'] = CATEGORY_MAPPING + + # Add category statistics with names + category_stats = {} + for k in [4, 1, 2, 3, 5]: + if k in total_counts and total_counts[k] > 0: + category_stats[k] = { + 'name': CATEGORY_MAPPING.get(k, f"Unknown-{k}"), + 'count': total_counts[k], + 'correct': acc_counts[k], + 'accuracy': round(float(acc_counts[k])/total_counts[k], 3) + } + results_dict[model_name]['category_statistics'] = category_stats if rag: results_dict[model_name]['recall_by_category'] = {k: v/total_counts[k] for k, v in recall_by_category.items()} print("Category and corresponding recall accuracy in each category: ") + print("Category | Name | Recall") + print("-" * 35) # for k, v in recall_by_category.items(): keys = [4, 1, 2, 3, 5] for k in keys: v = recall_by_category[k] + category_name = CATEGORY_MAPPING.get(k, f"Unknown-{k}") if float(total_counts[k]) == 0.0: - print("No questions found in category %s" % k) + print(f"No questions found in category {k} ({category_name})") else: - print(k, round(float(v)/total_counts[k], 3)) - print("Overall recall accuracy: ", sum(list(recall_by_category.values()))/sum(list(total_counts.values()))) + recall_acc = round(float(v)/total_counts[k], 3) + print(f"{k:8} | {category_name:11} | {recall_acc:6.3f}") + print("-" * 35) + overall_recall = sum(list(recall_by_category.values()))/sum(list(total_counts.values())) + print(f"Overall recall accuracy: {overall_recall:.3f}") else: results_dict[model_name]['category_counts_by_memory'] = memory_counts_og results_dict[model_name]['cum_accuracy_by_category_by_memory'] = memory_counts From 8bcbd89328da54b2de3dea2ac6d37b032dd15c35 Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Thu, 10 Jul 2025 09:34:28 +0800 Subject: [PATCH 3/3] Ensure answer values are strings in process_output Added checks to convert non-string answer values to strings in the process_output function. This prevents errors when processing outputs that may not be strings. --- task_eval/gemini_utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/task_eval/gemini_utils.py b/task_eval/gemini_utils.py index 29e905f..d6b1625 100644 --- a/task_eval/gemini_utils.py +++ b/task_eval/gemini_utils.py @@ -53,6 +53,10 @@ def process_ouput(text): if v is None: answers[k] = "" continue + # Convert to string if it's not already a string + if not isinstance(v, str): + v = str(v) + answers[k] = v if v.startswith('{') and v.endswith('}'): try: answers[k] = json.loads(v)['answer'] @@ -64,6 +68,11 @@ def process_ouput(text): for k, v in enumerate(answers): if v is None: answers[k] = "" + continue + # Convert to string if it's not already a string + if not isinstance(v, str): + v = str(v) + answers[k] = v if v.startswith('{') and v.endswith('}'): try: answers[k] = json.loads(v)['answer']