From a435c0c89adcf1fadf13c6de1e26870433e1b04e Mon Sep 17 00:00:00 2001 From: Priyanshu6968 Date: Sun, 21 Jun 2026 21:11:08 +0530 Subject: [PATCH] fix 502: portable paths and zero-division guard in llm_simple_qa example Signed-off-by: Priyanshu6968 --- examples/llm_simple_qa/README.md | 4 ++-- examples/llm_simple_qa/benchmarkingjob.yaml | 6 +++--- examples/llm_simple_qa/testalgorithms/gen/basemodel.py | 4 ++-- .../llm_simple_qa/testalgorithms/gen/gen_algorithm.yaml | 2 +- examples/llm_simple_qa/testalgorithms/gen/op_eval.py | 2 +- examples/llm_simple_qa/testenv/acc.py | 3 +++ examples/llm_simple_qa/testenv/testenv.yaml | 6 +++--- 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/examples/llm_simple_qa/README.md b/examples/llm_simple_qa/README.md index dbaf845a2..d18da1c79 100644 --- a/examples/llm_simple_qa/README.md +++ b/examples/llm_simple_qa/README.md @@ -70,7 +70,7 @@ Replace the file in `yourpath/anaconda3/envs/ianvs/lib/python3.x/site-packages/s Run the following command: -`ianvs -f examples/llm/singletask_learning_bench/simple_qa/benchmarkingjob.yaml` +`ianvs -f examples/llm_simple_qa/benchmarkingjob.yaml` ## OpenCompass Evaluation @@ -80,5 +80,5 @@ Run the following command: ### Run Evaluation -`python run_op.py examples/llm/singletask_learning_bench/simple_qa/testalgorithms/gen/op_eval.py` +`python run_op.py examples/llm_simple_qa/testalgorithms/gen/op_eval.py` diff --git a/examples/llm_simple_qa/benchmarkingjob.yaml b/examples/llm_simple_qa/benchmarkingjob.yaml index 78961e523..3c6082b57 100644 --- a/examples/llm_simple_qa/benchmarkingjob.yaml +++ b/examples/llm_simple_qa/benchmarkingjob.yaml @@ -2,11 +2,11 @@ benchmarkingjob: # job name of bechmarking; string type; name: "benchmarkingjob" # the url address of job workspace that will reserve the output of tests; string type; - workspace: "/home/icyfeather/project/ianvs/workspace" + workspace: "./workspace-llm_simple_qa" # the url address of test environment configuration file; string type; # the file format supports yaml/yml; - testenv: "./examples/llm/singletask_learning_bench/simple_qa/testenv/testenv.yaml" + testenv: "./examples/llm_simple_qa/testenv/testenv.yaml" # the configuration of test object test_object: @@ -19,7 +19,7 @@ benchmarkingjob: - name: "simple_qa_singletask_learning" # the url address of test algorithm configuration file; string type; # the file format supports yaml/yml; - url: "./examples/llm/singletask_learning_bench/simple_qa/testalgorithms/gen/gen_algorithm.yaml" + url: "./examples/llm_simple_qa/testalgorithms/gen/gen_algorithm.yaml" # the configuration of ranking leaderboard rank: diff --git a/examples/llm_simple_qa/testalgorithms/gen/basemodel.py b/examples/llm_simple_qa/testalgorithms/gen/basemodel.py index fdeedc988..82de66605 100644 --- a/examples/llm_simple_qa/testalgorithms/gen/basemodel.py +++ b/examples/llm_simple_qa/testalgorithms/gen/basemodel.py @@ -41,11 +41,11 @@ class BaseModel: def __init__(self, **kwargs): self.model = AutoModelForCausalLM.from_pretrained( - "/home/icyfeather/models/Qwen2-0.5B-Instruct", + "Qwen/Qwen2-0.5B-Instruct", torch_dtype="auto", device_map="auto" ) - self.tokenizer = AutoTokenizer.from_pretrained("/home/icyfeather/models/Qwen2-0.5B-Instruct") + self.tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct") def train(self, train_data, valid_data=None, **kwargs): print("BaseModel doesn't need to train") diff --git a/examples/llm_simple_qa/testalgorithms/gen/gen_algorithm.yaml b/examples/llm_simple_qa/testalgorithms/gen/gen_algorithm.yaml index 6536ceb99..1e52a99c6 100644 --- a/examples/llm_simple_qa/testalgorithms/gen/gen_algorithm.yaml +++ b/examples/llm_simple_qa/testalgorithms/gen/gen_algorithm.yaml @@ -15,4 +15,4 @@ algorithm: # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; name: "gen" # the url address of python module; string type; - url: "./examples/llm/singletask_learning_bench/simple_qa/testalgorithms/gen/basemodel.py" \ No newline at end of file + url: "./examples/llm_simple_qa/testalgorithms/gen/basemodel.py" \ No newline at end of file diff --git a/examples/llm_simple_qa/testalgorithms/gen/op_eval.py b/examples/llm_simple_qa/testalgorithms/gen/op_eval.py index dc6d9c046..c39c8bf50 100644 --- a/examples/llm_simple_qa/testalgorithms/gen/op_eval.py +++ b/examples/llm_simple_qa/testalgorithms/gen/op_eval.py @@ -12,7 +12,7 @@ dict( type=HuggingFacewithChatTemplate, abbr='qwen1.5-1.8b-chat-hf', - path='/home/icyfeather/models/Qwen1.5-1.8B-Chat', + path='Qwen/Qwen1.5-1.8B-Chat', max_out_len=1024, batch_size=2, run_cfg=dict(num_gpus=1), diff --git a/examples/llm_simple_qa/testenv/acc.py b/examples/llm_simple_qa/testenv/acc.py index beccdadfd..f80efb34e 100644 --- a/examples/llm_simple_qa/testenv/acc.py +++ b/examples/llm_simple_qa/testenv/acc.py @@ -29,6 +29,9 @@ def get_last_letter(input_string): @ClassFactory.register(ClassType.GENERAL, alias="acc") def acc(y_true, y_pred): + if not y_pred or len(y_pred) != len(y_true): + return 0 + y_pred = [get_last_letter(pred) for pred in y_pred] print(y_true) print(y_pred) diff --git a/examples/llm_simple_qa/testenv/testenv.yaml b/examples/llm_simple_qa/testenv/testenv.yaml index 0bc7239f3..4353ddabc 100644 --- a/examples/llm_simple_qa/testenv/testenv.yaml +++ b/examples/llm_simple_qa/testenv/testenv.yaml @@ -2,13 +2,13 @@ testenv: # dataset configuration dataset: # the url address of train dataset index; string type; - train_data: "/home/icyfeather/Projects/ianvs/dataset/llm_simple_qa/train_data/data.jsonl" + train_data: "./dataset/llm_simple_qa/train_data/data.jsonl" # the url address of test dataset index; string type; - test_data: "/home/icyfeather/Projects/ianvs/dataset/llm_simple_qa/test_data/data.jsonl" + test_data: "./dataset/llm_simple_qa/test_data/data.jsonl" # metrics configuration for test case's evaluation; list type; metrics: # metric name; string type; - name: "acc" # the url address of python file - url: "./examples/llm/singletask_learning_bench/simple_qa/testenv/acc.py" + url: "./examples/llm_simple_qa/testenv/acc.py"