From 6727093b460345c2043ce1f34bde5194bb9ae1b2 Mon Sep 17 00:00:00 2001 From: qiuhan Date: Thu, 6 Jun 2024 01:23:11 +0800 Subject: [PATCH] delete inrelavant tasks --- README.md | 15 +- lmms_eval/tasks/ai2d/ai2d.yaml | 36 - lmms_eval/tasks/ai2d/upload_ai2d.py | 117 ---- lmms_eval/tasks/ai2d/utils.py | 27 - lmms_eval/tasks/chartqa/chartqa.yaml | 34 - lmms_eval/tasks/chartqa/upload_chartqa.py | 129 ---- lmms_eval/tasks/chartqa/utils.py | 62 -- lmms_eval/tasks/cmmmu/_cmmmu.yaml | 4 - .../tasks/cmmmu/_default_template_cmmmu_yaml | 8 - lmms_eval/tasks/cmmmu/cmmmu_test.yaml | 12 - lmms_eval/tasks/cmmmu/cmmmu_val.yaml | 15 - lmms_eval/tasks/cmmmu/utils.py | 421 ------------ lmms_eval/tasks/coco_cap/coco2014_cap.yaml | 4 - .../tasks/coco_cap/coco2014_cap_test.yaml | 24 - .../tasks/coco_cap/coco2014_cap_val.yaml | 45 -- lmms_eval/tasks/coco_cap/coco2017_cap.yaml | 4 - .../tasks/coco_cap/coco2017_cap_test.yaml | 24 - .../tasks/coco_cap/coco2017_cap_val.yaml | 45 -- lmms_eval/tasks/coco_cap/coco_cap.yaml | 6 - lmms_eval/tasks/coco_cap/utils.py | 157 ----- lmms_eval/tasks/conbench/conbench.yaml | 24 - lmms_eval/tasks/conbench/utils.py | 100 --- .../docvqa/_default_template_docvqa_yaml | 19 - lmms_eval/tasks/docvqa/docvqa.yaml | 4 - lmms_eval/tasks/docvqa/docvqa_test.yaml | 8 - lmms_eval/tasks/docvqa/docvqa_val.yaml | 7 - lmms_eval/tasks/docvqa/utils.py | 32 - lmms_eval/tasks/ferret/ferret.yaml | 39 -- lmms_eval/tasks/ferret/rule.json | 5 - lmms_eval/tasks/ferret/utils.py | 206 ------ lmms_eval/tasks/flickr30k/flickr30k.yaml | 3 - lmms_eval/tasks/flickr30k/flickr30k_test.yaml | 44 -- lmms_eval/tasks/flickr30k/utils.py | 141 ---- lmms_eval/tasks/gqa/gqa.yaml | 32 - lmms_eval/tasks/gqa/utils.py | 23 - .../tasks/hallusion_bench/evaluate_hb.py | 129 ---- .../hallusion_bench_image.yaml | 41 -- lmms_eval/tasks/hallusion_bench/utils.py | 306 --------- .../iconqa/_default_template_docvqa_yaml | 22 - lmms_eval/tasks/iconqa/iconqa.yaml | 4 - lmms_eval/tasks/iconqa/iconqa_test.yaml | 3 - lmms_eval/tasks/iconqa/iconqa_val.yaml | 3 - lmms_eval/tasks/iconqa/utils.py | 57 -- .../infovqa/_default_template_infovqa_yaml | 15 - lmms_eval/tasks/infovqa/infovqa.yaml | 4 - lmms_eval/tasks/infovqa/infovqa_test.yaml | 10 - lmms_eval/tasks/infovqa/infovqa_val.yaml | 8 - lmms_eval/tasks/infovqa/utils.py | 33 - .../llava-bench-coco/llava-bench-coco.yaml | 38 -- lmms_eval/tasks/llava-bench-coco/rule.json | 11 - lmms_eval/tasks/llava-bench-coco/utils.py | 200 ------ .../llava-in-the-wild/llava-in-the-wild.yaml | 39 -- lmms_eval/tasks/llava-in-the-wild/rule.json | 11 - lmms_eval/tasks/llava-in-the-wild/utils.py | 197 ------ .../longhallqa/conversation_binary_v1.yaml | 34 - .../longhallqa/conversation_choice_v1.yaml | 20 - .../conversation_choice_v1_ppl.yaml | 15 - .../conversation_completion_v1.yaml | 20 - .../conversation_completion_v1_ppl.yaml | 15 - .../longhallqa/description_binary_v1.yaml | 34 - .../longhallqa/description_choice_v1.yaml | 20 - .../longhallqa/description_choice_v1_ppl.yaml | 15 - .../longhallqa/description_completion_v1.yaml | 20 - .../description_completion_v1_ppl.yaml | 15 - .../tasks/longhallqa/object_binary_v1.yaml | 34 - lmms_eval/tasks/mathverse/mathverse.yaml | 14 - lmms_eval/tasks/mathverse/mathverse_evals.py | 306 --------- .../tasks/mathverse/mathverse_testmini.yaml | 34 - .../mathverse_testmini_text_dominant.yaml | 34 - .../mathverse_testmini_text_lite.yaml | 34 - .../mathverse_testmini_text_only.yaml | 34 - .../mathverse_testmini_vision_dominant.yaml | 34 - .../mathverse_testmini_vision_intensive.yaml | 34 - .../mathverse_testmini_vision_only.yaml | 34 - lmms_eval/tasks/mathverse/utils.py | 94 --- lmms_eval/tasks/mathvista/mathvista.yaml | 8 - lmms_eval/tasks/mathvista/mathvista_evals.py | 456 ------------- lmms_eval/tasks/mathvista/mathvista_test.yaml | 29 - .../tasks/mathvista/mathvista_testmini.yaml | 31 - lmms_eval/tasks/mathvista/utils.py | 121 ---- .../mmbench/_default_template_mmbench_cn_yaml | 22 - .../mmbench/_default_template_mmbench_en_yaml | 25 - lmms_eval/tasks/mmbench/cc_utils.py | 109 --- lmms_eval/tasks/mmbench/cn_utils.py | 127 ---- lmms_eval/tasks/mmbench/en_utils.py | 126 ---- lmms_eval/tasks/mmbench/mmbench.yaml | 11 - lmms_eval/tasks/mmbench/mmbench_cc.yaml | 34 - lmms_eval/tasks/mmbench/mmbench_cn.yaml | 9 - lmms_eval/tasks/mmbench/mmbench_cn_dev.yaml | 10 - lmms_eval/tasks/mmbench/mmbench_cn_test.yaml | 7 - lmms_eval/tasks/mmbench/mmbench_en.yaml | 8 - lmms_eval/tasks/mmbench/mmbench_en_dev.yaml | 10 - lmms_eval/tasks/mmbench/mmbench_en_test.yaml | 7 - lmms_eval/tasks/mmbench/mmbench_evals.py | 323 --------- lmms_eval/tasks/mme/mme.yaml | 37 - lmms_eval/tasks/mme/utils.py | 120 ---- lmms_eval/tasks/mmmu/mmmu.yaml | 4 - lmms_eval/tasks/mmmu/mmmu_group_img.yaml | 4 - lmms_eval/tasks/mmmu/mmmu_group_img_test.yaml | 19 - lmms_eval/tasks/mmmu/mmmu_group_img_val.yaml | 21 - lmms_eval/tasks/mmmu/mmmu_test.yaml | 19 - lmms_eval/tasks/mmmu/mmmu_val.yaml | 21 - lmms_eval/tasks/mmmu/utils.py | 469 ------------- lmms_eval/tasks/mmmu/utils_group_img.py | 589 ---------------- .../tasks/mmupd/_default_template_mmupd_yaml | 18 - lmms_eval/tasks/mmupd/mmaad_base.yaml | 12 - lmms_eval/tasks/mmupd/mmaad_instruction.yaml | 12 - lmms_eval/tasks/mmupd/mmaad_option.yaml | 12 - lmms_eval/tasks/mmupd/mmiasd_base.yaml | 12 - lmms_eval/tasks/mmupd/mmiasd_instruction.yaml | 12 - lmms_eval/tasks/mmupd/mmiasd_option.yaml | 12 - lmms_eval/tasks/mmupd/mmivqd_base.yaml | 12 - lmms_eval/tasks/mmupd/mmivqd_instruction.yaml | 12 - lmms_eval/tasks/mmupd/mmivqd_option.yaml | 12 - lmms_eval/tasks/mmupd/mmupd.yaml | 15 - lmms_eval/tasks/mmupd/mmupd_base.yaml | 10 - lmms_eval/tasks/mmupd/mmupd_evals.py | 630 ------------------ lmms_eval/tasks/mmupd/mmupd_instruction.yaml | 9 - lmms_eval/tasks/mmupd/mmupd_option.yaml | 9 - lmms_eval/tasks/mmupd/utils.py | 168 ----- lmms_eval/tasks/mmvet/mmvet.yaml | 29 - lmms_eval/tasks/mmvet/utils.py | 213 ------ lmms_eval/tasks/multidocvqa/multidocvqa.yaml | 4 - .../tasks/multidocvqa/multidocvqa_test.yaml | 20 - .../tasks/multidocvqa/multidocvqa_val.yaml | 23 - lmms_eval/tasks/multidocvqa/utils.py | 116 ---- .../README.md | 1 - .../_default_template.yaml | 35 - .../arabic_llava_in_the_wild.yaml | 6 - .../bengali_llava_in_the_wild.yaml | 6 - .../chinese_llava_in_the_wild.yaml | 6 - .../french_llava_in_the_wild.yaml | 6 - .../hindi_llava_in_the_wild.yaml | 6 - .../japanese_llava_in_the_wild.yaml | 6 - .../rule.json | 11 - .../russian_llava_in_the_wild.yaml | 6 - .../spanish_llava_in_the_wild.yaml | 6 - .../urdu_llava_in_the_wild.yaml | 6 - .../utils.py | 197 ------ .../nocaps/_default_template_nocaps_yaml | 3 - lmms_eval/tasks/nocaps/nocaps.yaml | 4 - lmms_eval/tasks/nocaps/nocaps_test.yaml | 25 - lmms_eval/tasks/nocaps/nocaps_val.yaml | 46 -- lmms_eval/tasks/nocaps/utils.py | 153 ----- lmms_eval/tasks/ocrbench/ocrbench.yaml | 22 - lmms_eval/tasks/ocrbench/upload_ocrbench.py | 94 --- lmms_eval/tasks/ocrbench/utils.py | 103 --- .../tasks/ok_vqa/_default_template_vqa_yaml | 24 - lmms_eval/tasks/ok_vqa/_generate_config.py | 25 - lmms_eval/tasks/ok_vqa/_ok_vqa.yaml | 3 - lmms_eval/tasks/ok_vqa/ok_vqa_val2014.yaml | 4 - lmms_eval/tasks/ok_vqa/utils.py | 70 -- lmms_eval/tasks/olympiadbench/cn_utils.py | 72 -- lmms_eval/tasks/olympiadbench/en_utils.py | 74 -- .../tasks/olympiadbench/olympiadbench.yaml | 6 - .../olympiadbench/olympiadbench_evals.py | 355 ---------- .../olympiadbench/olympiadbench_test_cn.yaml | 25 - .../olympiadbench/olympiadbench_test_en.yaml | 25 - lmms_eval/tasks/pope/pope.yaml | 34 - lmms_eval/tasks/pope/pope_adv.yaml | 35 - lmms_eval/tasks/pope/pope_full.yaml | 5 - lmms_eval/tasks/pope/pope_pop.yaml | 35 - lmms_eval/tasks/pope/pope_random.yaml | 35 - lmms_eval/tasks/pope/utils.py | 87 --- lmms_eval/tasks/realworldqa/realworldqa.yaml | 43 -- lmms_eval/tasks/realworldqa/utils.py | 117 ---- .../refcoco+/_default_template_bbox_rec_yaml | 34 - .../refcoco+/_default_template_bbox_yaml | 36 - .../tasks/refcoco+/_default_template_seg_yaml | 36 - lmms_eval/tasks/refcoco+/_generate_config.py | 26 - lmms_eval/tasks/refcoco+/_refcoco.yaml | 8 - .../refcoco+/refcoco+_bbox_rec_testA.yaml | 4 - .../refcoco+/refcoco+_bbox_rec_testB.yaml | 4 - .../tasks/refcoco+/refcoco+_bbox_rec_val.yaml | 4 - .../tasks/refcoco+/refcoco+_bbox_testA.yaml | 4 - .../tasks/refcoco+/refcoco+_bbox_testB.yaml | 4 - .../tasks/refcoco+/refcoco+_bbox_val.yaml | 4 - .../tasks/refcoco+/refcoco+_seg_testA.yaml | 4 - .../tasks/refcoco+/refcoco+_seg_testB.yaml | 4 - .../tasks/refcoco+/refcoco+_seg_val.yaml | 4 - lmms_eval/tasks/refcoco+/utils.py | 135 ---- lmms_eval/tasks/refcoco+/utils_rec.py | 221 ------ .../refcoco/_default_template_bbox_rec_yaml | 34 - .../tasks/refcoco/_default_template_bbox_yaml | 36 - .../tasks/refcoco/_default_template_seg_yaml | 36 - lmms_eval/tasks/refcoco/_generate_config.py | 26 - lmms_eval/tasks/refcoco/_refcoco.yaml | 10 - .../tasks/refcoco/refcoco_bbox_rec_test.yaml | 4 - .../tasks/refcoco/refcoco_bbox_rec_testA.yaml | 4 - .../tasks/refcoco/refcoco_bbox_rec_testB.yaml | 4 - .../tasks/refcoco/refcoco_bbox_rec_val.yaml | 4 - .../tasks/refcoco/refcoco_bbox_test.yaml | 4 - .../tasks/refcoco/refcoco_bbox_testA.yaml | 4 - .../tasks/refcoco/refcoco_bbox_testB.yaml | 4 - lmms_eval/tasks/refcoco/refcoco_bbox_val.yaml | 4 - lmms_eval/tasks/refcoco/refcoco_seg_test.yaml | 4 - .../tasks/refcoco/refcoco_seg_testA.yaml | 4 - .../tasks/refcoco/refcoco_seg_testB.yaml | 4 - lmms_eval/tasks/refcoco/refcoco_seg_val.yaml | 4 - lmms_eval/tasks/refcoco/utils.py | 135 ---- lmms_eval/tasks/refcoco/utils_rec.py | 221 ------ .../refcocog/_default_template_bbox_rec_yaml | 34 - .../refcocog/_default_template_bbox_yaml | 36 - .../tasks/refcocog/_default_template_seg_yaml | 36 - lmms_eval/tasks/refcocog/_generate_config.py | 26 - lmms_eval/tasks/refcocog/_refcoco.yaml | 6 - .../refcocog/refcocog_bbox_rec_test.yaml | 4 - .../tasks/refcocog/refcocog_bbox_rec_val.yaml | 4 - .../tasks/refcocog/refcocog_bbox_test.yaml | 4 - .../tasks/refcocog/refcocog_bbox_val.yaml | 4 - .../tasks/refcocog/refcocog_seg_test.yaml | 4 - .../tasks/refcocog/refcocog_seg_val.yaml | 4 - lmms_eval/tasks/refcocog/utils.py | 135 ---- lmms_eval/tasks/refcocog/utils_rec.py | 221 ------ lmms_eval/tasks/scienceqa/scienceqa.yaml | 36 - lmms_eval/tasks/scienceqa/scienceqa_full.yaml | 4 - lmms_eval/tasks/scienceqa/scienceqa_img.yaml | 39 -- lmms_eval/tasks/scienceqa/utils.py | 44 -- lmms_eval/tasks/screenspot/README.md | 54 -- .../screenspot/_default_template_rec_yaml | 33 - .../screenspot/_default_template_reg_yaml | 15 - lmms_eval/tasks/screenspot/_screenspot.yaml | 4 - .../tasks/screenspot/screenspot_rec_test.yaml | 4 - .../tasks/screenspot/screenspot_reg_test.yaml | 4 - lmms_eval/tasks/screenspot/utils.py | 126 ---- lmms_eval/tasks/screenspot/utils_rec.py | 215 ------ lmms_eval/tasks/seedbench/seedbench.yaml | 28 - lmms_eval/tasks/seedbench/seedbench_ppl.yaml | 15 - lmms_eval/tasks/seedbench/utils.py | 60 -- lmms_eval/tasks/seedbench_2/seedbench_2.yaml | 49 -- lmms_eval/tasks/seedbench_2/utils.py | 58 -- lmms_eval/tasks/stvqa/stvqa.yaml | 20 - lmms_eval/tasks/stvqa/utils.py | 28 - .../textcaps/_default_template_textcaps_yaml | 3 - lmms_eval/tasks/textcaps/textcaps.yaml | 4 - lmms_eval/tasks/textcaps/textcaps_test.yaml | 25 - lmms_eval/tasks/textcaps/textcaps_train.yaml | 48 -- lmms_eval/tasks/textcaps/textcaps_val.yaml | 46 -- lmms_eval/tasks/textcaps/utils.py | 150 ----- .../textvqa/_default_template_textvqa_yaml | 17 - lmms_eval/tasks/textvqa/_textvqa.yaml | 4 - lmms_eval/tasks/textvqa/textvqa_test.yaml | 7 - lmms_eval/tasks/textvqa/textvqa_val.yaml | 12 - lmms_eval/tasks/textvqa/utils.py | 68 -- .../vizwiz_vqa/_default_template_vqa_yaml | 15 - .../tasks/vizwiz_vqa/_generate_config.py | 25 - lmms_eval/tasks/vizwiz_vqa/_vizwiz_vqa.yaml | 4 - lmms_eval/tasks/vizwiz_vqa/utils.py | 70 -- .../tasks/vizwiz_vqa/vizwiz_vqa_test.yaml | 14 - .../tasks/vizwiz_vqa/vizwiz_vqa_val.yaml | 13 - .../tasks/vqav2/_default_template_vqav2_yaml | 15 - lmms_eval/tasks/vqav2/_vqav2.yaml | 4 - lmms_eval/tasks/vqav2/utils.py | 89 --- lmms_eval/tasks/vqav2/vqav2_test.yaml | 8 - lmms_eval/tasks/vqav2/vqav2_val.yaml | 10 - lmms_eval/tasks/websrc/README.md | 51 -- lmms_eval/tasks/websrc/utils.py | 160 ----- lmms_eval/tasks/websrc/websrc.yaml | 4 - lmms_eval/tasks/websrc/websrc_test.yaml | 19 - lmms_eval/tasks/websrc/websrc_val.yaml | 19 - 260 files changed, 14 insertions(+), 13193 deletions(-) delete mode 100644 lmms_eval/tasks/ai2d/ai2d.yaml delete mode 100644 lmms_eval/tasks/ai2d/upload_ai2d.py delete mode 100644 lmms_eval/tasks/ai2d/utils.py delete mode 100644 lmms_eval/tasks/chartqa/chartqa.yaml delete mode 100644 lmms_eval/tasks/chartqa/upload_chartqa.py delete mode 100644 lmms_eval/tasks/chartqa/utils.py delete mode 100644 lmms_eval/tasks/cmmmu/_cmmmu.yaml delete mode 100644 lmms_eval/tasks/cmmmu/_default_template_cmmmu_yaml delete mode 100644 lmms_eval/tasks/cmmmu/cmmmu_test.yaml delete mode 100644 lmms_eval/tasks/cmmmu/cmmmu_val.yaml delete mode 100644 lmms_eval/tasks/cmmmu/utils.py delete mode 100644 lmms_eval/tasks/coco_cap/coco2014_cap.yaml delete mode 100644 lmms_eval/tasks/coco_cap/coco2014_cap_test.yaml delete mode 100644 lmms_eval/tasks/coco_cap/coco2014_cap_val.yaml delete mode 100644 lmms_eval/tasks/coco_cap/coco2017_cap.yaml delete mode 100644 lmms_eval/tasks/coco_cap/coco2017_cap_test.yaml delete mode 100644 lmms_eval/tasks/coco_cap/coco2017_cap_val.yaml delete mode 100644 lmms_eval/tasks/coco_cap/coco_cap.yaml delete mode 100644 lmms_eval/tasks/coco_cap/utils.py delete mode 100644 lmms_eval/tasks/conbench/conbench.yaml delete mode 100644 lmms_eval/tasks/conbench/utils.py delete mode 100644 lmms_eval/tasks/docvqa/_default_template_docvqa_yaml delete mode 100644 lmms_eval/tasks/docvqa/docvqa.yaml delete mode 100644 lmms_eval/tasks/docvqa/docvqa_test.yaml delete mode 100644 lmms_eval/tasks/docvqa/docvqa_val.yaml delete mode 100644 lmms_eval/tasks/docvqa/utils.py delete mode 100644 lmms_eval/tasks/ferret/ferret.yaml delete mode 100644 lmms_eval/tasks/ferret/rule.json delete mode 100644 lmms_eval/tasks/ferret/utils.py delete mode 100644 lmms_eval/tasks/flickr30k/flickr30k.yaml delete mode 100644 lmms_eval/tasks/flickr30k/flickr30k_test.yaml delete mode 100644 lmms_eval/tasks/flickr30k/utils.py delete mode 100644 lmms_eval/tasks/gqa/gqa.yaml delete mode 100644 lmms_eval/tasks/gqa/utils.py delete mode 100644 lmms_eval/tasks/hallusion_bench/evaluate_hb.py delete mode 100644 lmms_eval/tasks/hallusion_bench/hallusion_bench_image.yaml delete mode 100644 lmms_eval/tasks/hallusion_bench/utils.py delete mode 100644 lmms_eval/tasks/iconqa/_default_template_docvqa_yaml delete mode 100644 lmms_eval/tasks/iconqa/iconqa.yaml delete mode 100644 lmms_eval/tasks/iconqa/iconqa_test.yaml delete mode 100644 lmms_eval/tasks/iconqa/iconqa_val.yaml delete mode 100644 lmms_eval/tasks/iconqa/utils.py delete mode 100644 lmms_eval/tasks/infovqa/_default_template_infovqa_yaml delete mode 100644 lmms_eval/tasks/infovqa/infovqa.yaml delete mode 100644 lmms_eval/tasks/infovqa/infovqa_test.yaml delete mode 100644 lmms_eval/tasks/infovqa/infovqa_val.yaml delete mode 100644 lmms_eval/tasks/infovqa/utils.py delete mode 100644 lmms_eval/tasks/llava-bench-coco/llava-bench-coco.yaml delete mode 100644 lmms_eval/tasks/llava-bench-coco/rule.json delete mode 100644 lmms_eval/tasks/llava-bench-coco/utils.py delete mode 100644 lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml delete mode 100644 lmms_eval/tasks/llava-in-the-wild/rule.json delete mode 100644 lmms_eval/tasks/llava-in-the-wild/utils.py delete mode 100644 lmms_eval/tasks/longhallqa/conversation_binary_v1.yaml delete mode 100644 lmms_eval/tasks/longhallqa/conversation_choice_v1.yaml delete mode 100644 lmms_eval/tasks/longhallqa/conversation_choice_v1_ppl.yaml delete mode 100644 lmms_eval/tasks/longhallqa/conversation_completion_v1.yaml delete mode 100644 lmms_eval/tasks/longhallqa/conversation_completion_v1_ppl.yaml delete mode 100644 lmms_eval/tasks/longhallqa/description_binary_v1.yaml delete mode 100644 lmms_eval/tasks/longhallqa/description_choice_v1.yaml delete mode 100644 lmms_eval/tasks/longhallqa/description_choice_v1_ppl.yaml delete mode 100644 lmms_eval/tasks/longhallqa/description_completion_v1.yaml delete mode 100644 lmms_eval/tasks/longhallqa/description_completion_v1_ppl.yaml delete mode 100644 lmms_eval/tasks/longhallqa/object_binary_v1.yaml delete mode 100644 lmms_eval/tasks/mathverse/mathverse.yaml delete mode 100644 lmms_eval/tasks/mathverse/mathverse_evals.py delete mode 100644 lmms_eval/tasks/mathverse/mathverse_testmini.yaml delete mode 100644 lmms_eval/tasks/mathverse/mathverse_testmini_text_dominant.yaml delete mode 100644 lmms_eval/tasks/mathverse/mathverse_testmini_text_lite.yaml delete mode 100644 lmms_eval/tasks/mathverse/mathverse_testmini_text_only.yaml delete mode 100644 lmms_eval/tasks/mathverse/mathverse_testmini_vision_dominant.yaml delete mode 100644 lmms_eval/tasks/mathverse/mathverse_testmini_vision_intensive.yaml delete mode 100644 lmms_eval/tasks/mathverse/mathverse_testmini_vision_only.yaml delete mode 100644 lmms_eval/tasks/mathverse/utils.py delete mode 100644 lmms_eval/tasks/mathvista/mathvista.yaml delete mode 100644 lmms_eval/tasks/mathvista/mathvista_evals.py delete mode 100644 lmms_eval/tasks/mathvista/mathvista_test.yaml delete mode 100644 lmms_eval/tasks/mathvista/mathvista_testmini.yaml delete mode 100644 lmms_eval/tasks/mathvista/utils.py delete mode 100644 lmms_eval/tasks/mmbench/_default_template_mmbench_cn_yaml delete mode 100644 lmms_eval/tasks/mmbench/_default_template_mmbench_en_yaml delete mode 100644 lmms_eval/tasks/mmbench/cc_utils.py delete mode 100644 lmms_eval/tasks/mmbench/cn_utils.py delete mode 100644 lmms_eval/tasks/mmbench/en_utils.py delete mode 100644 lmms_eval/tasks/mmbench/mmbench.yaml delete mode 100644 lmms_eval/tasks/mmbench/mmbench_cc.yaml delete mode 100644 lmms_eval/tasks/mmbench/mmbench_cn.yaml delete mode 100644 lmms_eval/tasks/mmbench/mmbench_cn_dev.yaml delete mode 100644 lmms_eval/tasks/mmbench/mmbench_cn_test.yaml delete mode 100644 lmms_eval/tasks/mmbench/mmbench_en.yaml delete mode 100644 lmms_eval/tasks/mmbench/mmbench_en_dev.yaml delete mode 100644 lmms_eval/tasks/mmbench/mmbench_en_test.yaml delete mode 100644 lmms_eval/tasks/mmbench/mmbench_evals.py delete mode 100644 lmms_eval/tasks/mme/mme.yaml delete mode 100644 lmms_eval/tasks/mme/utils.py delete mode 100644 lmms_eval/tasks/mmmu/mmmu.yaml delete mode 100644 lmms_eval/tasks/mmmu/mmmu_group_img.yaml delete mode 100644 lmms_eval/tasks/mmmu/mmmu_group_img_test.yaml delete mode 100644 lmms_eval/tasks/mmmu/mmmu_group_img_val.yaml delete mode 100644 lmms_eval/tasks/mmmu/mmmu_test.yaml delete mode 100644 lmms_eval/tasks/mmmu/mmmu_val.yaml delete mode 100644 lmms_eval/tasks/mmmu/utils.py delete mode 100644 lmms_eval/tasks/mmmu/utils_group_img.py delete mode 100644 lmms_eval/tasks/mmupd/_default_template_mmupd_yaml delete mode 100644 lmms_eval/tasks/mmupd/mmaad_base.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmaad_instruction.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmaad_option.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmiasd_base.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmiasd_instruction.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmiasd_option.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmivqd_base.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmivqd_instruction.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmivqd_option.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmupd.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmupd_base.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmupd_evals.py delete mode 100644 lmms_eval/tasks/mmupd/mmupd_instruction.yaml delete mode 100644 lmms_eval/tasks/mmupd/mmupd_option.yaml delete mode 100644 lmms_eval/tasks/mmupd/utils.py delete mode 100644 lmms_eval/tasks/mmvet/mmvet.yaml delete mode 100644 lmms_eval/tasks/mmvet/utils.py delete mode 100644 lmms_eval/tasks/multidocvqa/multidocvqa.yaml delete mode 100644 lmms_eval/tasks/multidocvqa/multidocvqa_test.yaml delete mode 100644 lmms_eval/tasks/multidocvqa/multidocvqa_val.yaml delete mode 100644 lmms_eval/tasks/multidocvqa/utils.py delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/README.md delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/_default_template.yaml delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/arabic_llava_in_the_wild.yaml delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/bengali_llava_in_the_wild.yaml delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/chinese_llava_in_the_wild.yaml delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/french_llava_in_the_wild.yaml delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/hindi_llava_in_the_wild.yaml delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/japanese_llava_in_the_wild.yaml delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/rule.json delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/russian_llava_in_the_wild.yaml delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/spanish_llava_in_the_wild.yaml delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/urdu_llava_in_the_wild.yaml delete mode 100644 lmms_eval/tasks/multilingual-llava-bench-in-the-wild/utils.py delete mode 100644 lmms_eval/tasks/nocaps/_default_template_nocaps_yaml delete mode 100644 lmms_eval/tasks/nocaps/nocaps.yaml delete mode 100644 lmms_eval/tasks/nocaps/nocaps_test.yaml delete mode 100644 lmms_eval/tasks/nocaps/nocaps_val.yaml delete mode 100644 lmms_eval/tasks/nocaps/utils.py delete mode 100644 lmms_eval/tasks/ocrbench/ocrbench.yaml delete mode 100644 lmms_eval/tasks/ocrbench/upload_ocrbench.py delete mode 100644 lmms_eval/tasks/ocrbench/utils.py delete mode 100644 lmms_eval/tasks/ok_vqa/_default_template_vqa_yaml delete mode 100644 lmms_eval/tasks/ok_vqa/_generate_config.py delete mode 100644 lmms_eval/tasks/ok_vqa/_ok_vqa.yaml delete mode 100644 lmms_eval/tasks/ok_vqa/ok_vqa_val2014.yaml delete mode 100644 lmms_eval/tasks/ok_vqa/utils.py delete mode 100644 lmms_eval/tasks/olympiadbench/cn_utils.py delete mode 100644 lmms_eval/tasks/olympiadbench/en_utils.py delete mode 100644 lmms_eval/tasks/olympiadbench/olympiadbench.yaml delete mode 100644 lmms_eval/tasks/olympiadbench/olympiadbench_evals.py delete mode 100644 lmms_eval/tasks/olympiadbench/olympiadbench_test_cn.yaml delete mode 100644 lmms_eval/tasks/olympiadbench/olympiadbench_test_en.yaml delete mode 100644 lmms_eval/tasks/pope/pope.yaml delete mode 100644 lmms_eval/tasks/pope/pope_adv.yaml delete mode 100644 lmms_eval/tasks/pope/pope_full.yaml delete mode 100644 lmms_eval/tasks/pope/pope_pop.yaml delete mode 100644 lmms_eval/tasks/pope/pope_random.yaml delete mode 100644 lmms_eval/tasks/pope/utils.py delete mode 100644 lmms_eval/tasks/realworldqa/realworldqa.yaml delete mode 100644 lmms_eval/tasks/realworldqa/utils.py delete mode 100644 lmms_eval/tasks/refcoco+/_default_template_bbox_rec_yaml delete mode 100644 lmms_eval/tasks/refcoco+/_default_template_bbox_yaml delete mode 100644 lmms_eval/tasks/refcoco+/_default_template_seg_yaml delete mode 100644 lmms_eval/tasks/refcoco+/_generate_config.py delete mode 100644 lmms_eval/tasks/refcoco+/_refcoco.yaml delete mode 100644 lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_testA.yaml delete mode 100644 lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_testB.yaml delete mode 100644 lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_val.yaml delete mode 100644 lmms_eval/tasks/refcoco+/refcoco+_bbox_testA.yaml delete mode 100644 lmms_eval/tasks/refcoco+/refcoco+_bbox_testB.yaml delete mode 100644 lmms_eval/tasks/refcoco+/refcoco+_bbox_val.yaml delete mode 100644 lmms_eval/tasks/refcoco+/refcoco+_seg_testA.yaml delete mode 100644 lmms_eval/tasks/refcoco+/refcoco+_seg_testB.yaml delete mode 100644 lmms_eval/tasks/refcoco+/refcoco+_seg_val.yaml delete mode 100644 lmms_eval/tasks/refcoco+/utils.py delete mode 100644 lmms_eval/tasks/refcoco+/utils_rec.py delete mode 100644 lmms_eval/tasks/refcoco/_default_template_bbox_rec_yaml delete mode 100644 lmms_eval/tasks/refcoco/_default_template_bbox_yaml delete mode 100644 lmms_eval/tasks/refcoco/_default_template_seg_yaml delete mode 100644 lmms_eval/tasks/refcoco/_generate_config.py delete mode 100644 lmms_eval/tasks/refcoco/_refcoco.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_bbox_rec_test.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_bbox_rec_testA.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_bbox_rec_testB.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_bbox_rec_val.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_bbox_test.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_bbox_testA.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_bbox_testB.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_bbox_val.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_seg_test.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_seg_testA.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_seg_testB.yaml delete mode 100644 lmms_eval/tasks/refcoco/refcoco_seg_val.yaml delete mode 100644 lmms_eval/tasks/refcoco/utils.py delete mode 100644 lmms_eval/tasks/refcoco/utils_rec.py delete mode 100644 lmms_eval/tasks/refcocog/_default_template_bbox_rec_yaml delete mode 100644 lmms_eval/tasks/refcocog/_default_template_bbox_yaml delete mode 100644 lmms_eval/tasks/refcocog/_default_template_seg_yaml delete mode 100644 lmms_eval/tasks/refcocog/_generate_config.py delete mode 100644 lmms_eval/tasks/refcocog/_refcoco.yaml delete mode 100644 lmms_eval/tasks/refcocog/refcocog_bbox_rec_test.yaml delete mode 100644 lmms_eval/tasks/refcocog/refcocog_bbox_rec_val.yaml delete mode 100644 lmms_eval/tasks/refcocog/refcocog_bbox_test.yaml delete mode 100644 lmms_eval/tasks/refcocog/refcocog_bbox_val.yaml delete mode 100644 lmms_eval/tasks/refcocog/refcocog_seg_test.yaml delete mode 100644 lmms_eval/tasks/refcocog/refcocog_seg_val.yaml delete mode 100644 lmms_eval/tasks/refcocog/utils.py delete mode 100644 lmms_eval/tasks/refcocog/utils_rec.py delete mode 100644 lmms_eval/tasks/scienceqa/scienceqa.yaml delete mode 100644 lmms_eval/tasks/scienceqa/scienceqa_full.yaml delete mode 100644 lmms_eval/tasks/scienceqa/scienceqa_img.yaml delete mode 100644 lmms_eval/tasks/scienceqa/utils.py delete mode 100644 lmms_eval/tasks/screenspot/README.md delete mode 100644 lmms_eval/tasks/screenspot/_default_template_rec_yaml delete mode 100644 lmms_eval/tasks/screenspot/_default_template_reg_yaml delete mode 100644 lmms_eval/tasks/screenspot/_screenspot.yaml delete mode 100644 lmms_eval/tasks/screenspot/screenspot_rec_test.yaml delete mode 100644 lmms_eval/tasks/screenspot/screenspot_reg_test.yaml delete mode 100644 lmms_eval/tasks/screenspot/utils.py delete mode 100644 lmms_eval/tasks/screenspot/utils_rec.py delete mode 100644 lmms_eval/tasks/seedbench/seedbench.yaml delete mode 100644 lmms_eval/tasks/seedbench/seedbench_ppl.yaml delete mode 100644 lmms_eval/tasks/seedbench/utils.py delete mode 100644 lmms_eval/tasks/seedbench_2/seedbench_2.yaml delete mode 100644 lmms_eval/tasks/seedbench_2/utils.py delete mode 100644 lmms_eval/tasks/stvqa/stvqa.yaml delete mode 100644 lmms_eval/tasks/stvqa/utils.py delete mode 100644 lmms_eval/tasks/textcaps/_default_template_textcaps_yaml delete mode 100644 lmms_eval/tasks/textcaps/textcaps.yaml delete mode 100644 lmms_eval/tasks/textcaps/textcaps_test.yaml delete mode 100644 lmms_eval/tasks/textcaps/textcaps_train.yaml delete mode 100644 lmms_eval/tasks/textcaps/textcaps_val.yaml delete mode 100644 lmms_eval/tasks/textcaps/utils.py delete mode 100644 lmms_eval/tasks/textvqa/_default_template_textvqa_yaml delete mode 100644 lmms_eval/tasks/textvqa/_textvqa.yaml delete mode 100644 lmms_eval/tasks/textvqa/textvqa_test.yaml delete mode 100644 lmms_eval/tasks/textvqa/textvqa_val.yaml delete mode 100644 lmms_eval/tasks/textvqa/utils.py delete mode 100644 lmms_eval/tasks/vizwiz_vqa/_default_template_vqa_yaml delete mode 100644 lmms_eval/tasks/vizwiz_vqa/_generate_config.py delete mode 100644 lmms_eval/tasks/vizwiz_vqa/_vizwiz_vqa.yaml delete mode 100644 lmms_eval/tasks/vizwiz_vqa/utils.py delete mode 100644 lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_test.yaml delete mode 100644 lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_val.yaml delete mode 100644 lmms_eval/tasks/vqav2/_default_template_vqav2_yaml delete mode 100644 lmms_eval/tasks/vqav2/_vqav2.yaml delete mode 100644 lmms_eval/tasks/vqav2/utils.py delete mode 100644 lmms_eval/tasks/vqav2/vqav2_test.yaml delete mode 100644 lmms_eval/tasks/vqav2/vqav2_val.yaml delete mode 100644 lmms_eval/tasks/websrc/README.md delete mode 100644 lmms_eval/tasks/websrc/utils.py delete mode 100644 lmms_eval/tasks/websrc/websrc.yaml delete mode 100644 lmms_eval/tasks/websrc/websrc_test.yaml delete mode 100644 lmms_eval/tasks/websrc/websrc_val.yaml diff --git a/README.md b/README.md index d8e03ae..2dbf0cb 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ accelerate launch --num_processes=1 -m lmms_eval \ accelerate launch --num_processes=1 -m lmms_eval \ --model llava \ --model_args pretrained="liuhaotian/llava-v1.5-7b" \ ---tasks lhqa_complete_conversation,lhqa_complete_conversation_ppl,lhqa_complete_description,lhqa_complete_description_ppl \ +--tasks lhqa_complete_description,lhqa_complete_description_ppl,lhqa_complete_conversation,lhqa_complete_conversation_ppl \ --batch_size 1 \ --log_samples \ --log_samples_suffix llava_v15_7b_lhqa_completion \ @@ -41,6 +41,19 @@ accelerate launch --num_processes=1 -m lmms_eval \ | LLaVA-1.5-13b | llava | "liuhaotian/llava-v1.5-13b" | | LLaVA-1.6-7b | llava | "liuhaotian/llava-v1.6-mistral-7b,conv_template=mistral_instruct" | +### The evaluated sub-items in LongHalQA are: + + +| Hallucinaiton Discrimination | Hallucination Completion | +|:---------------------------------|:-------------------------------| +| lhqa_discrim_object_binary | lhqa_complete_description | +| lhqa_discrim_description_binary | lhqa_complete_description_ppl | +| lhqa_discrim_conversation_binary | lhqa_complete_conversation | +| lhqa_discrim_description_choice | lhqa_complete_conversation_ppl | +| lhqa_discrim_description_ppl | | +| lhqa_discrim_conversation_choice | | +| lhqa_discrim_conversation_ppl | | + # Acknowledgement diff --git a/lmms_eval/tasks/ai2d/ai2d.yaml b/lmms_eval/tasks/ai2d/ai2d.yaml deleted file mode 100644 index e032a3c..0000000 --- a/lmms_eval/tasks/ai2d/ai2d.yaml +++ /dev/null @@ -1,36 +0,0 @@ -dataset_path: lmms-lab/ai2d -task: "ai2d" -dataset_kwargs: - token: True -test_split: test -output_type: generate_until -doc_to_visual: !function utils.ai2d_doc_to_visual -doc_to_text: !function utils.ai2d_doc_to_text -doc_to_target: !function utils.ai2d_doc_to_target -generation_kwargs: - max_new_tokens: 16 - temperature: 0 - do_sample: False -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true -metadata: - - version: 0.0 - -model_specific_prompt_kwargs: - default: - prompt_format: mcq - pre_prompt: "" - post_prompt: "\nAnswer with the option's letter from the given choices directly." - # qwen formulate ai2d as question answering instead of mcq - qwen_vl: - prompt_format: qa - pre_prompt: "" - post_prompt: " Answer:" - -model_specific_target_kwargs: - default: "mcq" - qwen_vl: "qa" \ No newline at end of file diff --git a/lmms_eval/tasks/ai2d/upload_ai2d.py b/lmms_eval/tasks/ai2d/upload_ai2d.py deleted file mode 100644 index df0225e..0000000 --- a/lmms_eval/tasks/ai2d/upload_ai2d.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import json -import os - -import datasets -from PIL import Image - -# Find for instance the citation on arxiv or on the dataset repo/website -_CITATION = """https://allenai.org/data/diagrams""" -_DESCRIPTION = "AI2D is a dataset of illustrative diagrams for research on diagram understanding and associated question answering." - - -def get_builder_config(VERSION): - builder_config = [ - datasets.BuilderConfig( - name=f"ai2d", - version=VERSION, - description=f"ai2d", - ) - ] - return builder_config - - -dataset_features = { - "question": datasets.Value("string"), - "options": datasets.features.Sequence(datasets.Value("string")), - "answer": datasets.Value("string"), - "image": datasets.Image(), -} - - -class AI2D(datasets.GeneratorBasedBuilder): - VERSION = datasets.Version("1.0.0") - - BUILDER_CONFIGS = get_builder_config(VERSION) - - def _info(self): - features = datasets.Features(dataset_features) - return datasets.DatasetInfo( - # This is the description that will appear on the datasets page. - description=_DESCRIPTION, - # This defines the different columns of the dataset and their types - features=features, # Here we define them above because they are different between the two configurations - # If there's a common (input, target) tuple from the features, uncomment supervised_keys line below and - # specify them. They'll be used if as_supervised=True in builder.as_dataset. - # supervised_keys=("sentence", "label"), - # Homepage of the dataset for documentation - # Citation for the dataset - citation=_CITATION, - ) - - def _split_generators(self, dl_manager): - # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name - - # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS - # It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files. - # By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive - image_path = "/path/to/ai2d/images" - annotation_path = "/path/to/ai2d/questions" - # wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/evaluation/ai2diagram/test.jsonl - test_annotation_path = "/path/to/Qwen-VL/data/ai2diagram/test.jsonl" - return [ - datasets.SplitGenerator( - name=datasets.Split.TEST, - gen_kwargs={"annotation": annotation_path, "images": image_path, "test_annotation": test_annotation_path}, - ), - ] - - # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` - def _generate_examples(self, annotation, images, test_annotation): - # open test_annotation jsonl (note; not a json) - with open(test_annotation, encoding="utf-8") as f: - test_annotation = [json.loads(line) for line in f] - test_qn_ids = {x["question_id"] for x in test_annotation} - index = -1 - # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example. - for sub_annotation in os.listdir(annotation): - sub_annotation = os.path.join(annotation, sub_annotation) - with open(sub_annotation, encoding="utf-8") as f: - data = json.load(f) - image = data["imageName"] - image_path = os.path.join(images, image) - for question in data["questions"]: - if data["questions"]["questionId"] in test_qn_ids: - index += 1 - options = data["questions"][question]["answerTexts"] - answer = data["questions"][question]["correctAnswer"] - - now_data = {} - now_data["image"] = Image.open(image_path) - now_data["question"] = question - now_data["answer"] = answer - now_data["options"] = options - yield index, now_data - - -if __name__ == "__main__": - from datasets import load_dataset - - data = load_dataset( - "/path/to/lmms-eval/lmms_eval/tasks/ai2d/upload_ai2d.py", - ) - data.push_to_hub("lmms-lab/ai2d", private=True) diff --git a/lmms_eval/tasks/ai2d/utils.py b/lmms_eval/tasks/ai2d/utils.py deleted file mode 100644 index 0549fba..0000000 --- a/lmms_eval/tasks/ai2d/utils.py +++ /dev/null @@ -1,27 +0,0 @@ -def ai2d_doc_to_text(doc, model_specific_prompt_kwargs=None): - question, choices = doc["question"], doc["options"] - len_choices = len(choices) - post_prompt = model_specific_prompt_kwargs["post_prompt"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if model_specific_prompt_kwargs["prompt_format"] == "mcq": - options = [chr(ord("A") + i) for i in range(len_choices)] - choices_str = "\n".join([f"{option}. {choice}" for option, choice in zip(options, choices)]) - return f"{pre_prompt}{question}\n{choices_str}{post_prompt}" - elif model_specific_prompt_kwargs["prompt_format"] == "qa": - options = "\n".join(choices) - return f"{pre_prompt}{question}{options}{post_prompt}" - else: - raise ValueError(f"Unknown prompt format: {model_specific_prompt_kwargs['prompt_format']}") - - -def ai2d_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def ai2d_doc_to_target(doc, model_specific_target_kwargs): - if model_specific_target_kwargs == "mcq": - len_choices = len(doc["options"]) - options = [chr(ord("A") + i) for i in range(len_choices)] - return options[int(doc["answer"])] - elif model_specific_target_kwargs == "qa": - return doc["options"][int(doc["answer"])] diff --git a/lmms_eval/tasks/chartqa/chartqa.yaml b/lmms_eval/tasks/chartqa/chartqa.yaml deleted file mode 100644 index e8b7e82..0000000 --- a/lmms_eval/tasks/chartqa/chartqa.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: lmms-lab/ChartQA -dataset_kwargs: - token: True -task: "chartqa" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.chartqa_doc_to_visual -doc_to_text: !function utils.chartqa_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 16 - temperature: 0 - do_sample: False -process_results: !function utils.chartqa_process_results -metric_list: - - metric: relaxed_overall - aggregation: mean - higher_is_better: true - - metric: relaxed_human_split - aggregation: mean - higher_is_better: true - - metric: relaxed_augmented_split - aggregation: mean - higher_is_better: true -metadata: - - version: 0.0 -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer the question with a single word." - qwen_vl: - pre_prompt: "" - post_prompt: " Answer:" - diff --git a/lmms_eval/tasks/chartqa/upload_chartqa.py b/lmms_eval/tasks/chartqa/upload_chartqa.py deleted file mode 100644 index 14de734..0000000 --- a/lmms_eval/tasks/chartqa/upload_chartqa.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import json -import os -from PIL import Image -import datasets - - -# Find for instance the citation on arxiv or on the dataset repo/website -_CITATION = """\ -@inproceedings{masry-etal-2022-chartqa, - title = "{C}hart{QA}: A Benchmark for Question Answering about Charts with Visual and Logical Reasoning", - author = "Masry, Ahmed and - Long, Do and - Tan, Jia Qing and - Joty, Shafiq and - Hoque, Enamul", - booktitle = "Findings of the Association for Computational Linguistics: ACL 2022", - month = may, - year = "2022", - address = "Dublin, Ireland", - publisher = "Association for Computational Linguistics", - url = "https://aclanthology.org/2022.findings-acl.177", - doi = "10.18653/v1/2022.findings-acl.177", - pages = "2263--2279", -} -""" -_DESCRIPTION = "A largescale benchmark covering 9.6K human-written questions as well as 23.1K questions generated from human-written chart summaries." - - -def get_builder_config(VERSION): - builder_config = [ - datasets.BuilderConfig( - name=f"ChartQA", - version=VERSION, - description=f"ChartQA", - ) - ] - return builder_config - - -dataset_features = { - "type": datasets.Value("string"), - "question": datasets.Value("string"), - "answer": datasets.Value("string"), - "image": datasets.Image(), -} - - -class ChartQA(datasets.GeneratorBasedBuilder): - VERSION = datasets.Version("1.0.0") - - BUILDER_CONFIGS = get_builder_config(VERSION) - - def _info(self): - features = datasets.Features(dataset_features) - return datasets.DatasetInfo( - # This is the description that will appear on the datasets page. - description=_DESCRIPTION, - # This defines the different columns of the dataset and their types - features=features, # Here we define them above because they are different between the two configurations - # If there's a common (input, target) tuple from the features, uncomment supervised_keys line below and - # specify them. They'll be used if as_supervised=True in builder.as_dataset. - # supervised_keys=("sentence", "label"), - # Homepage of the dataset for documentation - # Citation for the dataset - citation=_CITATION, - ) - - def _split_generators(self, dl_manager): - # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name - - # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS - # It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files. - # By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive - image_path = "/home/yhzhang/ChartQA/ChartQA Dataset/test/png/" - human_annotation_path = "/home/yhzhang/ChartQA/ChartQA Dataset/test/test_human.json" - augmented_annotation_path = "/home/yhzhang/ChartQA/ChartQA Dataset/test/test_augmented.json" - return [ - datasets.SplitGenerator( - name=datasets.Split.TEST, - gen_kwargs={ - "human_annotation": human_annotation_path, - "augmented_annotation": augmented_annotation_path, - "images": image_path, - }, - ), - ] - - # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` - def _generate_examples(self, human_annotation, augmented_annotation, images): - # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example. - with open(human_annotation, encoding="utf-8") as f: - human_data = json.load(f) - with open(augmented_annotation, encoding="utf-8") as f: - augmented_data = json.load(f) - index = -1 - for data in [human_data, augmented_data]: - for row in data["data"]: - index += 1 - image_path = os.path.join(images, row["imgname"]) - now_data = {} - now_data["type"] = "human_test" if data == human_data else "augmented_test" - now_data["image"] = Image.open(image_path) - now_data["question"] = row["query"] - now_data["answer"] = row["label"] - yield index, now_data - - -if __name__ == "__main__": - from datasets import load_dataset - - data = load_dataset( - "/home/yhzhang/lmms-eval/lmms_eval/tasks/chartqa/upload_chartqa.py", - ) - data.push_to_hub("lmms-lab/chartqa", private=True) diff --git a/lmms_eval/tasks/chartqa/utils.py b/lmms_eval/tasks/chartqa/utils.py deleted file mode 100644 index 99de098..0000000 --- a/lmms_eval/tasks/chartqa/utils.py +++ /dev/null @@ -1,62 +0,0 @@ -def chartqa_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def chartqa_doc_to_text(doc, model_specific_prompt_kwargs): - question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] - return f"{pre_prompt}{question}{post_prompt}" - - -def chartqa_process_results(doc, results): - pred = results[0] - type = doc["type"] - score = relaxed_correctness(pred, doc["answer"]) - score = 1.0 if score else 0.0 - return_dict = {"relaxed_overall": score} - if type == "human_test": - return_dict["relaxed_human_split"] = score - else: - return_dict["relaxed_augmented_split"] = score - return return_dict - - -def relaxed_correctness(prediction, target, max_relative_change: float = 0.05) -> bool: - """Calculates relaxed correctness. - - The correctness tolerates certain error ratio defined by max_relative_change. - See https://arxiv.org/pdf/2203.10244.pdf, end of section 5.1: - “Following Methani et al. (2020), we use a relaxed accuracy measure for the - numeric answers to allow a minor inaccuracy that may result from the automatic - data extraction process. We consider an answer to be correct if it is within - 5% of the gold answer. For non-numeric answers, we still need an exact match - to consider an answer to be correct.” - - This funcion is taken from https://github.com/QwenLM/Qwen-VL/blob/34b4c0ee7b07726371b960911f249fe61b362ca3/eval_mm/evaluate_vqa.py#L113 - Args: - target: List of target string. - prediction: List of predicted string. - max_relative_change: Maximum relative change. - - Returns: - Whether the prediction was correct given the specified tolerance. - """ - - def _to_float(text: str): - try: - if text.endswith("%"): - # Convert percentages to floats. - return float(text.rstrip("%")) / 100.0 - else: - return float(text) - except ValueError: - return None - - prediction_float = _to_float(prediction) - target_float = _to_float(target) - if prediction_float is not None and target_float: - relative_change = abs(prediction_float - target_float) / abs(target_float) - return relative_change <= max_relative_change - else: - return prediction.lower() == target.lower() diff --git a/lmms_eval/tasks/cmmmu/_cmmmu.yaml b/lmms_eval/tasks/cmmmu/_cmmmu.yaml deleted file mode 100644 index 52a7666..0000000 --- a/lmms_eval/tasks/cmmmu/_cmmmu.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: cmmmu -task: -- cmmmu_val -- cmmmu_test diff --git a/lmms_eval/tasks/cmmmu/_default_template_cmmmu_yaml b/lmms_eval/tasks/cmmmu/_default_template_cmmmu_yaml deleted file mode 100644 index 4e769a8..0000000 --- a/lmms_eval/tasks/cmmmu/_default_template_cmmmu_yaml +++ /dev/null @@ -1,8 +0,0 @@ -dataset_path: lmms-lab/CMMMU -output_type: generate_until -doc_to_visual: !function utils.cmmmu_doc_to_visual -doc_to_text: !function utils.cmmmu_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 16 - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/cmmmu/cmmmu_test.yaml b/lmms_eval/tasks/cmmmu/cmmmu_test.yaml deleted file mode 100644 index b94854d..0000000 --- a/lmms_eval/tasks/cmmmu/cmmmu_test.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: "cmmmu_test" -test_split: test -# The return value of process_results will be used by metrics -process_results: !function utils.cmmmu_process_test_results_for_submission -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: submission - aggregation: !function utils.cmmmu_test_aggregate_results_for_submission - higher_is_better: false -metadata: - - version: 0.0 -include: _default_template_cmmmu_yaml diff --git a/lmms_eval/tasks/cmmmu/cmmmu_val.yaml b/lmms_eval/tasks/cmmmu/cmmmu_val.yaml deleted file mode 100644 index e503b3d..0000000 --- a/lmms_eval/tasks/cmmmu/cmmmu_val.yaml +++ /dev/null @@ -1,15 +0,0 @@ -task: "cmmmu_val" -test_split: val -# The return value of process_results will be used by metrics -process_results: !function utils.cmmmu_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -generation_kwargs: - max_new_tokens: 16 - image_aspect_ratio: original -metric_list: - - metric: cmmmu_acc - aggregation: !function utils.cmmmu_aggregate_results - higher_is_better: true -metadata: - - version: 0.0 -include: _default_template_cmmmu_yaml diff --git a/lmms_eval/tasks/cmmmu/utils.py b/lmms_eval/tasks/cmmmu/utils.py deleted file mode 100644 index 7f50a80..0000000 --- a/lmms_eval/tasks/cmmmu/utils.py +++ /dev/null @@ -1,421 +0,0 @@ -from collections import defaultdict -import re -import random -import os -import json -import logging -from collections import Counter -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -eval_logger = logging.getLogger("lmms-eval") - -PROMPT = { - "task_instructions": [ - "请回答以下多项选择题，并选出正确选项。这些题目可能包括单选和多选题型。如果所提供的信息不足以确定一个明确的答案，那么请根据可用的数据和你的判断来选择最可能正确的选项。", - "请回答以下判断题，并根据题目描述和所给的信息来判断问题中陈述的对错。如果信息不完整或不足以作出绝对判断，请运用你的逻辑推理和现有信息来做出最可能的判断。", - "请回答以下填空题，并根据题目的要求和所提供的信息来给出最恰当的答案。如果信息不足以确切回答，那么请依据现有的数据和你的推理能力来填写最合理的答案。", - ], - "multi_choice_example_format": ["问题：{}\n选项：\n{}\n正确答案：\n"], - "T/F_example_format": ["问题：{}\n正确答案：\n"], - "short_ans_example_format": ["问题：{}\n正确答案：\n"], -} - - -def construct_prompt(sample): - question = sample["question"] - task_instructions = PROMPT["task_instructions"] - - if sample["type"] == "选择": - formatted_options = "" - start_chr = "A" - for i in range(1, 5): - formatted_options += f"({start_chr}) {sample[f'option{i}']}\n" - start_chr = chr(ord(start_chr) + 1) - - current_example_template = PROMPT["multi_choice_example_format"][0] - current_example = current_example_template.format(question, formatted_options) - final_input_prompt = task_instructions[0] + "\n\n" + current_example - - elif sample["type"] == "判断": - current_example_template = PROMPT["T/F_example_format"][0] - current_example = current_example_template.format(question) - final_input_prompt = task_instructions[1] + "\n\n" + current_example - - else: # For fill in the blanks questions. - current_example_template = PROMPT["short_ans_example_format"][0] - current_example = current_example_template.format(question) - final_input_prompt = task_instructions[2] + "\n\n" + current_example - - for i in range(1, 6): - final_input_prompt = final_input_prompt.replace(f'', f"<图片 {i}>") - - return final_input_prompt - - -def cmmmu_doc_to_text(doc): - return construct_prompt(doc) - - -def cmmmu_doc_to_visual(doc): - prompt = construct_prompt(doc) - image_tokens = re.findall(r"<图片 \d+>", prompt) - # Remove <> and swap space as _ - image_tokens = [image_token.strip("<>").replace(" ", "_").replace("图片", "image") for image_token in image_tokens] - visual = [doc[image_token].convert("RGB") for image_token in image_tokens] - return visual - - -def cmmmu_process_results(doc, results): - pred = results[0] - if doc["type"] == "选择": - index2ans, all_choices = get_multi_choice_info([doc[f"option{i}"] for i in range(1, 5)]) - parsed_pred = get_multi_choice_prediction(pred, all_choices, index2ans) - elif doc["type"] == "判断": - parsed_pred = get_TF_prediction(pred) - else: - parsed_pred = get_fill_blank_prediction(pred, doc["answer"]) - return {"cmmmu_acc": {"id": doc["id"], "subdomain": doc["subcategory"], "question_type": doc["type"], "answer": doc["answer"], "parsed_pred": parsed_pred}} - - -def cmmmu_aggregate_results(results): - evaluation_result = {} - subset_to_eval_samples = defaultdict(list) - for result in results: - subset_to_eval_samples[result["subdomain"]].append(result) - for subset, sub_eval_samples in subset_to_eval_samples.items(): - metric_dict = eval_cmmmu(sub_eval_samples) - evaluation_result[subset] = metric_dict - - printable_results = {} - for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items(): - in_domain_cat_results = {} - for cat_name in in_domain_cats: - if cat_name in evaluation_result.keys(): - in_domain_cat_results[cat_name] = evaluation_result[cat_name] - else: - pass - in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results) - in_domain_data_num = sum([cat_results["entries_num"] for cat_results in in_domain_cat_results.values()]) - printable_results["Overall-" + domain] = { - "num": int(in_domain_data_num), - "acc": round(in_domain_ins_acc, 3), - } - # add sub category - for cat_name, cat_results in in_domain_cat_results.items(): - printable_results[cat_name] = { - "num": int(cat_results["entries_num"]), - "acc": round(cat_results["acc"], 3), - } - all_ins_acc = calculate_ins_level_acc(evaluation_result) - printable_results["Overall"] = { - "num": sum([cat_results["entries_num"] for cat_results in evaluation_result.values()]), - "acc": round(all_ins_acc, 3), - } - print(printable_results) - return printable_results["Overall"]["acc"] - - -def cmmmu_process_test_results_for_submission(doc, results): - response = results[0] - return {"submission": {"id": doc["id"], "type": doc["type"], "response": response}} - - -def cmmmu_test_aggregate_results_for_submission(results, args): - file = generate_submission_file("cmmmu_test_for_submission.jsonl", args) - with open(file, "w", encoding="utf8") as f: - for result in results: - json.dump(result, f, ensure_ascii=False) - f.write("\n") - eval_logger.info(f"Submission file saved to {file}") - - -################## -# Helper functions -################## - -DOMAIN_CAT2SUB_CAT = { - "艺术与设计": ["艺术", "艺术理论", "设计", "音乐"], - "商业": ["会计", "经济", "金融", "管理", "营销"], - "科学": ["生物", "化学", "地理", "数学", "物理"], - "健康与医学": ["基础医学", "临床医学", "诊断学与实验室医学", "制药", "公共卫生"], - "人文社会科学": ["历史", "文献学", "社会学", "心理学"], - "技术与工程": ["农业", "建筑学", "计算机科学", "电子学", "能源和电力", "材料", "机械工程"], -} - - -def eval_cmmmu(entries): - correct_cnt = 0 - for entry in entries: - parsed_pred = entry.get("parsed_pred", "") - correct = False - if entry.get("question_type") == "选择": - if parsed_pred == entry["answer"]: - correct_cnt += 1 - correct = True - - elif entry.get("question_type") == "填空": - norm_answers = normalize_str(entry["answer"], entry["answer"]) - - for pred in parsed_pred: - # already normalized - if isinstance(pred, str): # if it's a string, then find if ans in the pred_i - for norm_ans in norm_answers: - # only see if the string answer in the string pred - # print(norm_ans, pred) - if isinstance(norm_ans, str) and norm_ans in pred: - if not correct: - correct_cnt += 1 - correct = True - break - else: # it's a number - if pred in norm_answers: - if not correct: - correct_cnt += 1 - correct = True - break - - else: - positive_keywords = ["正确", "对", "准确", "肯定", "对的"] - negative_keywords = ["不对", "错误", "不正确", "不准确", "不合适", "否定", "错的", "错"] - ambiguous_keywords = ["对错", "是否正确", "否正确", "或者", "是否", "正确性", "对不"] - - def judge_similarity(pred_list, positive_keywords, negative_keywords): - positive_count = 0 - negative_count = 0 - - for pred in pred_list: - if any(pos_word in pred for pos_word in positive_keywords): - positive_count += 1 - elif any(neg_word in pred for neg_word in negative_keywords): - negative_count += 1 - - if positive_count > negative_count: - return "对" - elif negative_count > positive_count: - return "错" - else: - return random.choice(["对", "错"]) - - answer = entry["answer"] - parsed_pred = [word for word in parsed_pred if not any(ambiguous in word for ambiguous in ambiguous_keywords)] - result = judge_similarity(parsed_pred, positive_keywords, negative_keywords) - if result == answer: - correct_cnt += 1 - correct = True - if correct: - entry["judge"] = "正确" - else: - entry["judge"] = "错误" - - if len(entries) == 0: - print("entries_num == 0, please check your file") - results_count = {"correct_num": 0, "entries_num": 0, "acc": 0} - else: - results_count = {"correct_num": correct_cnt, "entries_num": len(entries), "acc": correct_cnt / len(entries)} - - return results_count - - -def get_multi_choice_prediction(response, all_choices, index2ans): - for char in [",", ".", "!", "?", ";", ":", "'"]: - response = response.strip(char) - response = " " + response + " " # add space to avoid partial match - - candidates = [] - - for choice in all_choices: # (A) (B) (C) (D) - # Add the choice to candidates each time it appears in the response - candidates.extend([choice for _ in range(response.count(f"({choice})"))]) - - if len(candidates) == 0: - for choice in all_choices: # A B C D - # Similarly, add the choice for each occurrence - candidates.extend([choice for _ in range(response.count(f"{choice}"))]) - - if len(candidates) == 0 and len(response.split()) >= 1: - for index, ans in index2ans.items(): - # Add index for each occurrence of ans in response - candidates.extend([index for _ in range(response.count(ans))]) - - # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example - if len(candidates) == 0 and len(response.split()) >= 1: - for index, ans in index2ans.items(): - if ans in response: - candidates.append(index) - index_ans = False # it's content ans. - - if len(candidates) == 0: # still not get answer, randomly choose one. - return random.choice(all_choices) - # return '' - else: - # Count the occurrence of each candidate - candidate_counts = Counter(candidates) - - # Select the most frequent candidates - max_count = max(candidate_counts.values()) - most_frequent_candidates = [c for c in all_choices if candidate_counts.get(c, 0) == max_count] - - # Combine the most frequent candidates in ABCD order - return "".join(most_frequent_candidates) - - -def extract_numbers(string): - # Pattern for numbers with Chinese commas - pattern_commas = r"-?\d{1,3}(?:，\d{3})+" - # Pattern for scientific notation - pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" - # Pattern for simple numbers without Chinese commas - pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+)(?![eE][+-]?\d+)(?!，\d)" - - # Extract numbers with Chinese commas - numbers_with_commas = re.findall(pattern_commas, string) - # Extract numbers in scientific notation - numbers_scientific = re.findall(pattern_scientific, string) - # Extract simple numbers without Chinese commas - numbers_simple = re.findall(pattern_simple, string) - - # Combine all extracted numbers - all_numbers = numbers_with_commas + numbers_scientific + numbers_simple - return all_numbers - - -def check_is_number(string): - try: - float(string.replace(",", "")) - return True - except ValueError: - # check if there's comma inside - return False - - -def count_letters(string): - return sum(c.isalpha() and "a" <= c <= "z" or "A" <= c <= "Z" for c in string) - - -def normalize_str(string, answer): - # check if characters in the string - - # if number, numerize it. - if string == None: - return [string] - string = string.strip() - - is_number = check_is_number(string) - - if is_number: - string = string.replace(",", "") - string = float(string) - # leave 2 decimal - string = round(string, 2) - return [string] - else: # it's likely to be a string - if len(string) > len(answer) + 20 or count_letters(string) > count_letters(answer) + 2: - return [] - return [string] - - -def get_fill_blank_prediction(response, answer): - """get the prediction from the generated response, - return a list of predicted strings or numbers""" - - def get_key_subresponses(response): - key_responses = [] - response = response.strip("。").strip() - sub_responses = re.split(r"。|\n", response) - indicators_of_keys = ["是", "为", "所以", "等于", "方案", "选择", "正确答案", "因此", "最后", "答案", "结果"] - key_responses = [] - for index, resp in enumerate(sub_responses): - # if last one, accept it's an equation (the entire response can be just one sentence with equation) - if index == len(sub_responses) - 1: - indicators_of_keys.extend(["="]) - shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) - for indicator in indicators_of_keys: - if indicator in resp: - if not shortest_key_response: - shortest_key_response = resp.split(indicator)[-1].strip() - else: - if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response): - shortest_key_response = resp.split(indicator)[-1].strip() - - if shortest_key_response: - # and it's not trivial - if shortest_key_response.strip() not in [":", ",", ".", "!", "?", ";", ":", "'"]: - key_responses.append(shortest_key_response) - if len(key_responses) == 0: # did not found any - return [response] - return key_responses - - key_responses = get_key_subresponses(response) - - pred_list = key_responses.copy() # keep the original string response - for resp in key_responses: - pred_list.extend(extract_numbers(resp)) - - tmp_pred_list = [] - for i in range(len(pred_list)): - tmp_pred_list.extend(normalize_str(pred_list[i], answer)) - pred_list = tmp_pred_list - - # remove duplicates - pred_list = list(set(pred_list)) - - return pred_list - - -def get_TF_prediction(response): - """get the prediction from the generated response, - return a list of predicted strings or numbers""" - - def get_key_subresponses(response): - key_responses = [] - response = response.strip("。").strip() - sub_responses = re.split(r"。|\n", response) - indicators_of_keys = ["是", "为", "所以", "判断", "陈述", "说法", "表达", "答案", "结果"] - key_responses = [] - for index, resp in enumerate(sub_responses): - shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) - for indicator in indicators_of_keys: - if indicator in resp: - if not shortest_key_response: - shortest_key_response = resp.split(indicator)[-1].strip() - else: - if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response): - shortest_key_response = resp.split(indicator)[-1].strip() - - if shortest_key_response: - # and it's not trivial - if shortest_key_response.strip() not in [":", ",", ".", "!", "?", ";", ":", "'"]: - key_responses.append(shortest_key_response) - if len(key_responses) == 0: # did not found any - return [response] - return key_responses - - key_responses = get_key_subresponses(response) - - pred_list = key_responses.copy() # keep the original string response - # remove duplicates - pred_list = list(set(pred_list)) - - return pred_list - - -def get_multi_choice_info(options): - start_chr = "A" - all_choices = [] - index2ans = {} - for i, option in enumerate(options): - index2ans[chr(ord(start_chr) + i)] = option - all_choices.append(chr(ord(start_chr) + i)) - - return index2ans, all_choices - - -def calculate_ins_level_acc(results): - correct_sum = 0 - entries_sum = 0 - for cat_results in results.values(): - correct_sum += cat_results["correct_num"] - entries_sum += cat_results["entries_num"] - if entries_sum == 0: - return 0 - return correct_sum / entries_sum diff --git a/lmms_eval/tasks/coco_cap/coco2014_cap.yaml b/lmms_eval/tasks/coco_cap/coco2014_cap.yaml deleted file mode 100644 index 042fd9e..0000000 --- a/lmms_eval/tasks/coco_cap/coco2014_cap.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group : coco2014_cap -task: - - coco2014_cap_val - - coco2014_cap_test \ No newline at end of file diff --git a/lmms_eval/tasks/coco_cap/coco2014_cap_test.yaml b/lmms_eval/tasks/coco_cap/coco2014_cap_test.yaml deleted file mode 100644 index e080450..0000000 --- a/lmms_eval/tasks/coco_cap/coco2014_cap_test.yaml +++ /dev/null @@ -1,24 +0,0 @@ -dataset_path: lmms-lab/COCO-Caption -dataset_kwargs: - token: True -task : "coco2014_cap_test" -group : "coco_caption" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.coco_doc_to_visual -doc_to_text: "Provide a one-sentence caption for the provided image." -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.coco_test_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: coco_passthrough - aggregation : !function utils.coco_test_aggregation_result - higher_is_better : true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/coco_cap/coco2014_cap_val.yaml b/lmms_eval/tasks/coco_cap/coco2014_cap_val.yaml deleted file mode 100644 index 57b01a7..0000000 --- a/lmms_eval/tasks/coco_cap/coco2014_cap_val.yaml +++ /dev/null @@ -1,45 +0,0 @@ -dataset_path: lmms-lab/COCO-Caption -dataset_kwargs: - token: True -task: "coco2014_cap_val" -group : "coco_caption" -test_split: val -output_type: generate_until -doc_to_visual: !function utils.coco_doc_to_visual -doc_to_text: "Provide a one-sentence caption for the provided image." -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 64 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.coco_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: coco_Bleu_4 - aggregation : !function utils.coco_bleu4 - higher_is_better : true - - metric: coco_Bleu_3 - aggregation : !function utils.coco_bleu3 - higher_is_better : true - - metric: coco_Bleu_2 - aggregation : !function utils.coco_bleu2 - higher_is_better : true - - metric: coco_Bleu_1 - aggregation : !function utils.coco_bleu1 - higher_is_better : true - - metric: coco_METEOR - aggregation : !function utils.coco_meteor - higher_is_better : true - - metric: coco_ROUGE_L - aggregation : !function utils.coco_rougel - higher_is_better : true - - metric: coco_CIDEr - aggregation : !function utils.coco_cider - higher_is_better : true - #- metric: coco_SPICE - # aggregation : !function utils.coco_spice - # higher_is_better : true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/coco_cap/coco2017_cap.yaml b/lmms_eval/tasks/coco_cap/coco2017_cap.yaml deleted file mode 100644 index d9b4c7d..0000000 --- a/lmms_eval/tasks/coco_cap/coco2017_cap.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group : coco2017_cap -task: - - coco2017_cap_val - - coco2017_cap_test \ No newline at end of file diff --git a/lmms_eval/tasks/coco_cap/coco2017_cap_test.yaml b/lmms_eval/tasks/coco_cap/coco2017_cap_test.yaml deleted file mode 100644 index ecd7375..0000000 --- a/lmms_eval/tasks/coco_cap/coco2017_cap_test.yaml +++ /dev/null @@ -1,24 +0,0 @@ -dataset_path: lmms-lab/COCO-Caption2017 -dataset_kwargs: - token: True -task : "coco2017_cap_test" -group : "coco_caption2017" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.coco_doc_to_visual -doc_to_text: !function utils.coco_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.coco_test_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: coco_passthrough - aggregation : !function utils.coco_test_aggregation_result - higher_is_better : true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/coco_cap/coco2017_cap_val.yaml b/lmms_eval/tasks/coco_cap/coco2017_cap_val.yaml deleted file mode 100644 index b0d9d4a..0000000 --- a/lmms_eval/tasks/coco_cap/coco2017_cap_val.yaml +++ /dev/null @@ -1,45 +0,0 @@ -dataset_path: lmms-lab/COCO-Caption2017 -dataset_kwargs: - token: True -task: "coco2017_cap_val" -group : "coco_caption2017" -test_split: val -output_type: generate_until -doc_to_visual: !function utils.coco_doc_to_visual -doc_to_text: !function utils.coco_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 64 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.coco_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: coco_Bleu_4 - aggregation : !function utils.coco_bleu4 - higher_is_better : true - - metric: coco_Bleu_3 - aggregation : !function utils.coco_bleu3 - higher_is_better : true - - metric: coco_Bleu_2 - aggregation : !function utils.coco_bleu2 - higher_is_better : true - - metric: coco_Bleu_1 - aggregation : !function utils.coco_bleu1 - higher_is_better : true - - metric: coco_METEOR - aggregation : !function utils.coco_meteor - higher_is_better : true - - metric: coco_ROUGE_L - aggregation : !function utils.coco_rougel - higher_is_better : true - - metric: coco_CIDEr - aggregation : !function utils.coco_cider - higher_is_better : true - #- metric: coco_SPICE - # aggregation : !function utils.coco_spice - # higher_is_better : true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/coco_cap/coco_cap.yaml b/lmms_eval/tasks/coco_cap/coco_cap.yaml deleted file mode 100644 index e3bff59..0000000 --- a/lmms_eval/tasks/coco_cap/coco_cap.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group : coco_cap -task: - - coco2014_cap_val - - coco2014_cap_test - - coco2017_cap_val - - coco2017_cap_test diff --git a/lmms_eval/tasks/coco_cap/utils.py b/lmms_eval/tasks/coco_cap/utils.py deleted file mode 100644 index 0102dbe..0000000 --- a/lmms_eval/tasks/coco_cap/utils.py +++ /dev/null @@ -1,157 +0,0 @@ -import os -import json -from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice -from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer -from pycocotools.coco import COCO - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -import logging - -eval_logger = logging.getLogger("lmms-eval") - -dir_name = os.path.dirname(os.path.abspath(__file__)) - -COCO_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"] # , "SPICE"] - - -def coco_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def coco_doc_to_text(doc): - return f"Provide a one-sentence caption for the provided image." - - -def coco_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name, value: metric value - """ - pred = result[0] if len(result) > 0 else "" - question_id = doc["question_id"] - # The question id in our dataset is the image file itself - image_id = int(question_id.split("_")[-1].split(".")[0]) - id = doc["id"] - - data_dict = {"answer": doc["answer"], "pred": pred, "image_id": image_id, "id": id} - - return {f"coco_{metric}": data_dict for metric in COCO_METRICS} - - -def coco_aggregation_result(results, metric, args): - scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr")]#, (Spice(), "SPICE")] - scorers_dict = {s[1]: s for s in scorers} - - stored_results = [] - # In order to make the coco eval tools to successfully create index - # We need at least two dict in the dataset - # 'annotation' and 'images' - # 'annotation' exactly reproduce the original annotation - # 'images' however only need the image id which is contained in the file name - dataset = {"annotations": [], "images": []} - idx = 0 - for result in results: - stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]}) - for a in result["answer"]: - dataset["annotations"].append({"image_id": int(result["image_id"]), "caption": a, "id": idx}) - idx += 1 - dataset["images"].append({"id": result["image_id"]}) - - coco = COCO() - # Manually create index here - coco.dataset = dataset - coco.createIndex() - - coco_result = coco.loadRes(stored_results) - coco_eval = COCOEvalCap(coco, coco_result) - - imgIds = coco_eval.params["image_id"] - gts = {} - res = {} - for imgId in imgIds: - gts[imgId] = coco_eval.coco.imgToAnns[imgId] - res[imgId] = coco_eval.cocoRes.imgToAnns[imgId] - - eval_logger.info("tokenization...") - tokenizer = PTBTokenizer() - gts = tokenizer.tokenize(gts) - res = tokenizer.tokenize(res) - - eval_logger.info(f"Computing {metric} scores...") - - score, scores = scorers_dict[metric][0].compute_score(gts, res) - # When metric is one of the Bleu, score will be a list - if type(score) == list: - n = int(metric.split("_")[-1]) - score = score[n - 1] - - path = generate_submission_file("coco_captions_val2014_alg_results.json", args) - if not os.path.exists(path): - eval_logger.info("Storing prediction that can be submitted to the server ...") - with open(path, "w") as f: - json.dump(stored_results, f, indent=4) - - return score - - -def coco_bleu4(results, args): - return coco_aggregation_result(results, "Bleu_4", args) - - -def coco_bleu3(results, args): - return coco_aggregation_result(results, "Bleu_3", args) - - -def coco_bleu2(results, args): - return coco_aggregation_result(results, "Bleu_2", args) - - -def coco_bleu1(results, args): - return coco_aggregation_result(results, "Bleu_1", args) - - -def coco_meteor(results, args): - return coco_aggregation_result(results, "METEOR", args) - - -def coco_rougel(results, args): - return coco_aggregation_result(results, "ROUGE_L", args) - - -def coco_cider(results, args): - return coco_aggregation_result(results, "CIDEr", args) - - -def coco_spice(results, args): - return coco_aggregation_result(results, "SPICE", args) - - -def coco_test_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case coco_passthrough), value: metric value - """ - question_id = doc["question_id"] - # The question id in our dataset is the image file itself - image_id = int(question_id.split("_")[-1].split(".")[0]) - return {"coco_passthrough": {"pred": result, "image_id": image_id}} - - -def coco_test_aggregation_result(results, args): - stored_results = [] - for result in results: - stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]}) - - path = generate_submission_file("coco_captions_test2014_alg_results.json", args) - eval_logger.info("Storing prediction that can be submitted to the server ...") - with open(path, "w") as f: - json.dump(stored_results, f, indent=4) - - eval_logger.info(f"Your test result has been stored in to {path}. Make sure you also have the val result stored to submit to the server on https://codalab.lisn.upsaclay.fr/competitions/7404#participate.") diff --git a/lmms_eval/tasks/conbench/conbench.yaml b/lmms_eval/tasks/conbench/conbench.yaml deleted file mode 100644 index 3739282..0000000 --- a/lmms_eval/tasks/conbench/conbench.yaml +++ /dev/null @@ -1,24 +0,0 @@ -dataset_path: ConBench/ConBench_D -dataset_kwargs: - token: True -task: "ConBench" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.conbench_doc_to_visual -doc_to_text: !function utils.conbench_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 1024 - temperature: 0.2 - top_p: 0 - num_beams: 1 - do_sample: True -# The return value of process_results will be used by metrics -process_results: !function utils.conbench_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: ConScore_D - aggregation: !function utils.conbench_aggregate_results - higher_is_better: true -metadata: - - version: 0.0 diff --git a/lmms_eval/tasks/conbench/utils.py b/lmms_eval/tasks/conbench/utils.py deleted file mode 100644 index f80f855..0000000 --- a/lmms_eval/tasks/conbench/utils.py +++ /dev/null @@ -1,100 +0,0 @@ -from collections import defaultdict -import os -from anls import anls_score - -import logging - -eval_logger = logging.getLogger("lmms-eval") - -dir_name = os.path.dirname(os.path.abspath(__file__)) - -# 19 classes -eval_type_dict = { - "Sensation": ["count","color", "scene", "poster", "attribute_recognition", "ocr", "position"], - "Cognition": ["calculation", "code", "translation", "math", "cross_instance_reason", "attribute_reason"], - "Knowledge": ["celebrity", "chemistry", "physics", "biology", "landmark", "artwork"] -} - - -def conbench_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def conbench_doc_to_text(doc): - question = doc["question"].strip() - return question - - -def parse_pred_ans_NY(pred_ans): - pred_label = None - if pred_ans in ["yes", "no"]: - pred_label = pred_ans - else: - prefix_pred_ans = pred_ans[:4] - - if "yes" in prefix_pred_ans: - pred_label = "yes" - elif "no" in prefix_pred_ans: - pred_label = "no" - else: - pred_label = "other" - return pred_label - - -def parse_pred_ans_choice(pred_ans): - return pred_ans.replace(" ", "")[0] - - -def conbench_process_results(doc, results): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case mme score), value: metric value - """ - pred = results[0] - pred = pred.replace('\n', "").lower() - # parser - if doc["question_field"] == "N/Y": - pred_ans = parse_pred_ans_NY(pred) - elif doc["question_field"] == "Choices": - pred_ans = parse_pred_ans_choice(pred) - else: - pred_ans = pred - - gt_ans = doc["answer"].lower() - - # score - score = 1 if (doc["question_field"] == "Q/A" and anls_score(prediction=pred_ans, gold_labels=[gt_ans], threshold=0.95) >= 0.4) \ - or (gt_ans == pred_ans) \ - else 0 - # Note: the key name here is very important. It decides which aggregation function will receive the results - # We note down the question id/category to help us aggregate the results later - return {"ConScore_D":{"image_id": doc["image_id"], "question_field": doc["question_field"], "score": score}} - - -def conbench_aggregate_results(results): - """ - Args: - results: a list of values returned by process_results - Returns: - A score - """ - summary = defaultdict(dict) - for result in results: - image_id = result["image_id"] - score = result["score"] - if image_id not in summary.keys(): - summary[image_id] = 0 - summary[image_id] += score - - cnt_con = 0 - for image_id, score in summary.items(): - if score == 3: - cnt_con += 1 - - print("Consistency Cases are ", cnt_con) - cnt_con = cnt_con / (len(results) / 3) - eval_logger.info(f"ConScore_D: {cnt_con:.2f}") - return cnt_con \ No newline at end of file diff --git a/lmms_eval/tasks/docvqa/_default_template_docvqa_yaml b/lmms_eval/tasks/docvqa/_default_template_docvqa_yaml deleted file mode 100644 index 6e0cab6..0000000 --- a/lmms_eval/tasks/docvqa/_default_template_docvqa_yaml +++ /dev/null @@ -1,19 +0,0 @@ -dataset_path: lmms-lab/DocVQA -dataset_name: DocVQA -dataset_kwargs: - token: True -output_type: generate_until -doc_to_visual: !function utils.docvqa_doc_to_visual -doc_to_text: !function utils.docvqa_doc_to_text -doc_to_target: "answers" -generation_kwargs: - max_new_tokens: 32 - temperature: 0 - do_sample: False -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer the question using a single word or phrase." - qwen_vl: - pre_prompt: "" - post_prompt: " Answer:" diff --git a/lmms_eval/tasks/docvqa/docvqa.yaml b/lmms_eval/tasks/docvqa/docvqa.yaml deleted file mode 100644 index af179eb..0000000 --- a/lmms_eval/tasks/docvqa/docvqa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: docvqa -task: -- docvqa_val -- docvqa_test \ No newline at end of file diff --git a/lmms_eval/tasks/docvqa/docvqa_test.yaml b/lmms_eval/tasks/docvqa/docvqa_test.yaml deleted file mode 100644 index 9d4ec49..0000000 --- a/lmms_eval/tasks/docvqa/docvqa_test.yaml +++ /dev/null @@ -1,8 +0,0 @@ -task: "docvqa_test" -test_split: test -process_results: !function utils.docvqa_test_process_results -metric_list: - - metric: submission - aggregation: !function utils.docvqa_test_aggregate_results - higher_is_better: true -include: _default_template_docvqa_yaml diff --git a/lmms_eval/tasks/docvqa/docvqa_val.yaml b/lmms_eval/tasks/docvqa/docvqa_val.yaml deleted file mode 100644 index 4b13ac5..0000000 --- a/lmms_eval/tasks/docvqa/docvqa_val.yaml +++ /dev/null @@ -1,7 +0,0 @@ -task: "docvqa_val" -test_split: validation -metric_list: - - metric: anls - aggregation: mean - higher_is_better: true -include: _default_template_docvqa_yaml diff --git a/lmms_eval/tasks/docvqa/utils.py b/lmms_eval/tasks/docvqa/utils.py deleted file mode 100644 index 0a785d2..0000000 --- a/lmms_eval/tasks/docvqa/utils.py +++ /dev/null @@ -1,32 +0,0 @@ -import json -import os -import logging - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -logger = logging.getLogger("lmms-eval") - - -def docvqa_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def docvqa_doc_to_text(doc, model_specific_prompt_kwargs): - question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] - return f"{pre_prompt}{question}{post_prompt}" - - -def docvqa_test_process_results(doc, results): - pred = results[0] - questionId = doc["questionId"] - return {"anls": {"questionId": int(questionId), "answer": pred}, "submission": {"questionId": int(questionId), "answer": pred}} - - -def docvqa_test_aggregate_results(results, args): - # save results as json - path = generate_submission_file("docvqa_test_for_submission.json", args) - with open(path, "w") as f: - json.dump(results, f) - logger.info(f"Results saved to {path}") diff --git a/lmms_eval/tasks/ferret/ferret.yaml b/lmms_eval/tasks/ferret/ferret.yaml deleted file mode 100644 index 517649e..0000000 --- a/lmms_eval/tasks/ferret/ferret.yaml +++ /dev/null @@ -1,39 +0,0 @@ -dataset_path: lmms-lab/Ferret-Bench -dataset_kwargs: - token: True -task: "ferret" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.ferret_doc_to_visual -doc_to_text: !function utils.ferret_doc_to_text -doc_to_target: "gpt_answer" -generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.ferret_process_results -metric_list: - - metric: gpt_eval_ferret_all - aggregation: !function utils.ferret_all_aggregation - higher_is_better: true - - metric: gpt_eval_ferret_refer_desc - aggregation: !function utils.ferret_refer_desc_aggregation - higher_is_better: true - - metric: gpt_eval_ferret_refer_reason - aggregation: !function utils.ferret_refer_reason_aggregation - higher_is_better: true - - metric: gpt_eval_ferret_ground_conv - aggregation: !function utils.ferret_ground_conv_aggregation - higher_is_better: true -metadata: - version: 0.0 - gpt_eval_model_name: "gpt-4-0314" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/ferret/rule.json b/lmms_eval/tasks/ferret/rule.json deleted file mode 100644 index 7294372..0000000 --- a/lmms_eval/tasks/ferret/rule.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "refer_desc": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question about specific region of an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image. In addition, specific object locations within the image are given, along with detailed coordinates. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. Also, the relationships between pairs of objects are provided, in the format of object -> relationship -> subject, where the object/subject are indexed by object id from previous object lists as well as the object names. Also, several region description are given, each describing a box region of image, with detailed coordinates. \nPlease rate the spatial correspondence, helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "refer_reason": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question about specific region of an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image. In addition, specific object locations within the image are given, along with detailed coordinates. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. Also, the relationships between pairs of objects are provided, in the format of object -> relationship -> subject, where the object/subject are indexed by object id from previous object lists as well as the object names. Also, several region description are given, each describing a box region of image, with detailed coordinates. \nPlease rate the spatial correspondence, helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "ground_conv": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question that requires model to predict the coordinates of relevant object. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image. In addition, specific object locations within the image are given, along with detailed coordinates. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. Also, the relationships between pairs of objects are provided, in the format of object -> relationship -> subject, where the object/subject are indexed by object id from previous object lists as well as the object names. Also, several region description are given, each describing a box region of image, with detailed coordinates. \nPlease rate the predicted coordinates, helpfulness, relevance, accuracy, level of details of their responses. Specifically, pay your attention to the precision of the coordinates and whether it matches the object. Small deviation (<20% of ground-truth box width or height) of coordinates is allowed and shouldn't be punished. More than that, the degree of deviation should be reflected in scoring too. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."} -} \ No newline at end of file diff --git a/lmms_eval/tasks/ferret/utils.py b/lmms_eval/tasks/ferret/utils.py deleted file mode 100644 index 7e86ea4..0000000 --- a/lmms_eval/tasks/ferret/utils.py +++ /dev/null @@ -1,206 +0,0 @@ -import json -import logging -import os -import requests -import numpy as np -import openai -from openai import OpenAI -import time -import yaml -from pathlib import Path -from copy import deepcopy - -eval_logger = logging.getLogger("lmms-eval") -NUM_SECONDS_TO_SLEEP = 0.5 - -FERRET_W_METRICS = ["gpt_eval_ferret_refer_desc", "gpt_eval_ferret_refer_reason", "gpt_eval_ferret_ground_conv"] - -rule_dict = json.load(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "rule.json"), "r")) - -with open(Path(__file__).parent / "ferret.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] - -API_TYPE = os.getenv("API_TYPE", "openai") - -if API_TYPE == "openai": - API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") - API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json", - } -elif API_TYPE == "azure": - API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") - API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") - headers = { - "api-key": API_KEY, - "Content-Type": "application/json", - } - - -def get_eval(content: str, max_tokens: int, retries: int = 3): - global headers - - messages = [ - { - "role": "system", - "content": "You are a helpful and precise assistant for checking the quality of the answer.", - }, - {"role": "user", "content": content}, - ] - - payload = { - "model": GPT_EVAL_MODEL_NAME, - "messages": messages, - "temperature": 0.2, - "max_tokens": max_tokens, - } - - for attempt in range(retries): - try: - response = requests.post(API_URL, headers=headers, json=payload) - response.raise_for_status() - response_data = response.json() - - content = response_data["choices"][0]["message"]["content"].strip() - if content != "": - return content, response_data["model"] - break # If successful, break out of the loop - - except Exception as e: - eval_logger.info(f"Attempt {attempt + 1} failed with error: {e}") - if attempt < retries - 1: # If we have retries left, sleep and then continue to next attempt - time.sleep(NUM_SECONDS_TO_SLEEP) - else: # If this was the last attempt, log and return empty - eval_logger.error(f"All {retries} attempts failed. Last error message: {e}") - return "", "" - return "", "" - - -def parse_score(review): - try: - score_pair = review.split("\n")[0] - score_pair = score_pair.replace(",", " ") - sp = score_pair.split(" ") - if len(sp) == 2: - return [float(sp[0]), float(sp[1])] - else: - eval_logger.debug(f"Can not split: {review}. Returning [-1, -1]") - return [-1, -1] - except Exception as e: - eval_logger.debug(f"Error: {e}. Returning [-1, -1]") - return [-1, -1] - - -def ferret_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def ferret_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") - question = f"{pre_prompt}{doc['question']}{post_prompt}" - return question - - -def ferret_process_results(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case coco_bleu), value: metric value - """ - try: - question = doc.get("question", "") - ans1 = doc.get("gpt_answer", "") - ans2 = result[0] if result else "" - context = doc.get("context", []) - context = "\n".join(context) if isinstance(context, list) else context - category = doc.get("category", "") - rule = rule_dict.get(category, {}) - prompt = rule.get("prompt", "") - role = rule.get("role", "user") - content = f"[Context]\n{context}\n\n" f"[Question]\n{question}\n\n" f"[{role} 1]\n{ans1}\n\n[End of {role} 1]\n\n" f"[{role} 2]\n{ans2}\n\n[End of {role} 2]\n\n" f"[System]\n{prompt}\n\n" - review, model_name = get_eval(content, 1024) - scores = parse_score(review) - except Exception as e: - eval_logger.error(f"Error for Question ID: {doc.get('question_id', 'Unknown')}: {e}") - review = "Failed to Get a Proper Review." - model_name = "Failed Request" - scores = [-1, -1] - - metric = f"gpt_eval_ferret_{doc.get('category', 'all')}" - category_review_dict = { - "question": question, - "ans1": ans1, - "ans2": ans2, - "context": context, - "category": category, - "review": review, - "scores": scores, - "eval_model": model_name, - } - - non_category_review_dict = deepcopy(category_review_dict) - non_category_review_dict["scores"] = [-999, -999] - - data_dict = {} - for m in FERRET_W_METRICS: - if m == metric: - data_dict[m] = category_review_dict - else: - data_dict[m] = non_category_review_dict - data_dict["gpt_eval_ferret_all"] = category_review_dict - - # return {"gpt_eval_ferret_all": review_dict} - return data_dict - - -def ferret_refer_desc_aggregation(results): - return ferret_aggregation(results, "refer_desc") - - -def ferret_refer_reason_aggregation(results): - return ferret_aggregation(results, "refer_reason") - - -def ferret_ground_conv_aggregation(results): - return ferret_aggregation(results, "ground_conv") - - -def ferret_all_aggregation(results): - return ferret_aggregation(results, "all") - - -def ferret_aggregation(results, category): - try: - scores = [] - for result in results: - if -999 in result["scores"]: - continue - scores.append(result["scores"]) - - stats = np.asarray(scores).mean(0).tolist() - stats = [round(x, 3) for x in stats] - # gpt4_score_percentage = stats[0] * 10 - # model_score_percentage = stats[1] * 10 - # eval_logger.info(f"Category: {category}") - # eval_logger.info(f"GPT4 Score: {gpt4_score_percentage:.1f}%") - # eval_logger.info(f"Model Score: {model_score_percentage:.1f}%") - # eval_logger.info("=========================") - return round(stats[1] / stats[0] * 100, 1) - except Exception as e: - eval_logger.info(f"Error in ferret_aggregation: {e}, and in category: {category}") - return None diff --git a/lmms_eval/tasks/flickr30k/flickr30k.yaml b/lmms_eval/tasks/flickr30k/flickr30k.yaml deleted file mode 100644 index 94f63bc..0000000 --- a/lmms_eval/tasks/flickr30k/flickr30k.yaml +++ /dev/null @@ -1,3 +0,0 @@ -group: flickr30k -task: -- flickr30k_test \ No newline at end of file diff --git a/lmms_eval/tasks/flickr30k/flickr30k_test.yaml b/lmms_eval/tasks/flickr30k/flickr30k_test.yaml deleted file mode 100644 index 737d9ff..0000000 --- a/lmms_eval/tasks/flickr30k/flickr30k_test.yaml +++ /dev/null @@ -1,44 +0,0 @@ -dataset_path: lmms-lab/flickr30k -dataset_kwargs: - token: True -task : "flickr30k_test" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.flickr_doc_to_visual -doc_to_text: !function utils.flickr_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 64 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.flickr_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: flickr_Bleu_4 - aggregation : !function utils.flickr_bleu4 - higher_is_better : true - - metric: flickr_Bleu_3 - aggregation : !function utils.flickr_bleu3 - higher_is_better : true - - metric: flickr_Bleu_2 - aggregation : !function utils.flickr_bleu2 - higher_is_better : true - - metric: flickr_Bleu_1 - aggregation : !function utils.flickr_bleu1 - higher_is_better : true - - metric: flickr_METEOR - aggregation : !function utils.flickr_meteor - higher_is_better : true - - metric: flickr_ROUGE_L - aggregation : !function utils.flickr_rougel - higher_is_better : true - - metric: flickr_CIDEr - aggregation : !function utils.flickr_cider - higher_is_better : true - #- metric: flickr_SPICE - # aggregation : !function utils.flickr_spice - # higher_is_better : true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/flickr30k/utils.py b/lmms_eval/tasks/flickr30k/utils.py deleted file mode 100644 index f5d5c14..0000000 --- a/lmms_eval/tasks/flickr30k/utils.py +++ /dev/null @@ -1,141 +0,0 @@ -import os -import json -from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice -from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer -from pycocotools.coco import COCO -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file -import datetime - -import logging - -eval_logger = logging.getLogger("lmms-eval") - -dir_name = os.path.dirname(os.path.abspath(__file__)) - -FLICKR_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"] # , "SPICE"] - - -def flickr_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def flickr_doc_to_text(doc): - # question = "Please carefully observe the image and come up with a caption for the image" - return f"Provide a one-sentence caption for the provided image." - - -def flickr_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name, value: metric value - """ - pred = result[0] if len(result) > 0 else "" - image_id = int(doc["img_id"]) - - data_dict = {"answer": doc["caption"], "pred": pred, "image_id": image_id} - - return {f"flickr_{metric}": data_dict for metric in FLICKR_METRICS} - - -def flickr_aggregation_result(results, metric, args): - scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr")]#, (Spice(), "SPICE")] - scorers_dict = {s[1]: s for s in scorers} - - stored_results = [] - # In order to make the coco eval tools to successfully create index - # We need at least two dict in the dataset - # 'annotation' and 'images' - # 'annotation' exactly reproduce the original annotation - # 'images' however only need the image id which is contained in the file name - dataset = {"annotations": [], "images": []} - idx = 0 - for result in results: - stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]}) - for a in result["answer"]: - dataset["annotations"].append({"image_id": int(result["image_id"]), "caption": a, "id": idx}) - idx += 1 - dataset["images"].append({"id": int(result["image_id"])}) - - coco = COCO() - # Manually create index here - coco.dataset = dataset - coco.createIndex() - - flickr_result = coco.loadRes(stored_results) - flickr_eval = COCOEvalCap(coco, flickr_result) - - imgIds = flickr_eval.params["image_id"] - gts = {} - res = {} - for imgId in imgIds: - gts[imgId] = flickr_eval.coco.imgToAnns[imgId] - res[imgId] = flickr_eval.cocoRes.imgToAnns[imgId] - - eval_logger.info("tokenization...") - tokenizer = PTBTokenizer() - gts = tokenizer.tokenize(gts) - res = tokenizer.tokenize(res) - - eval_logger.info(f"Computing {metric} scores...") - - score, scores = scorers_dict[metric][0].compute_score(gts, res) - # When metric is one of the Bleu, score will be a list - if type(score) == list: - n = int(metric.split("_")[-1]) - score = score[n - 1] - - path = generate_submission_file(f"flickr30k_captions_val2014_alg_results_{metric}.json", args) - - eval_logger.info("Storing prediction that can be submitted to the server ...") - with open(path, "w") as f: - json.dump(stored_results, f, indent=4) - - return score - - -def flickr_bleu4(results, args): - return flickr_aggregation_result(results, "Bleu_4", args) - - -def flickr_bleu3(results, args): - return flickr_aggregation_result(results, "Bleu_3", args) - - -def flickr_bleu2(results, args): - return flickr_aggregation_result(results, "Bleu_2", args) - - -def flickr_bleu1(results, args): - return flickr_aggregation_result(results, "Bleu_1", args) - - -def flickr_meteor(results, args): - return flickr_aggregation_result(results, "METEOR", args) - - -def flickr_rougel(results, args): - return flickr_aggregation_result(results, "ROUGE_L", args) - - -def flickr_cider(results, args): - return flickr_aggregation_result(results, "CIDEr", args) - - -def flickr_spice(results, args): - return flickr_aggregation_result(results, "SPICE", args) - - -def flickr_test_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case flickr_passthrough), value: metric value - """ - # The question id in our dataset is the image file itself - image_id = doc["img_id"] - return {"flickr_passthrough": {"pred": result, "image_id": image_id}} diff --git a/lmms_eval/tasks/gqa/gqa.yaml b/lmms_eval/tasks/gqa/gqa.yaml deleted file mode 100644 index 404e5a0..0000000 --- a/lmms_eval/tasks/gqa/gqa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -dataset_path: lmms-lab/GQA -dataset_name: testdev_balanced_instructions -dataset_kwargs: - token: True -task: "gqa" -test_split: testdev -output_type: generate_until -doc_to_visual: !function utils.gqa_doc_to_visual -doc_to_text: !function utils.gqa_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 16 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true -metadata: - - version: 0.0 - -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer the question using a single word or phrase." - qwen_vl: - pre_prompt: "" - post_prompt: " Answer:" \ No newline at end of file diff --git a/lmms_eval/tasks/gqa/utils.py b/lmms_eval/tasks/gqa/utils.py deleted file mode 100644 index 5d74d6f..0000000 --- a/lmms_eval/tasks/gqa/utils.py +++ /dev/null @@ -1,23 +0,0 @@ -from datasets import load_dataset - -GQA_RAW_IMAGE_DATASET = None -GQA_ID2IMAGE = None - - -def gqa_doc_to_visual(doc): - global GQA_RAW_IMAGE_DATASET - global GQA_ID2IMAGE - if GQA_RAW_IMAGE_DATASET is None: - GQA_RAW_IMAGE_DATASET = load_dataset("lmms-lab/GQA", "testdev_balanced_images", split="testdev", token=True) - GQA_ID2IMAGE = {} - for row in GQA_RAW_IMAGE_DATASET: - GQA_ID2IMAGE[row["id"]] = row["image"].convert("RGB") - image = GQA_ID2IMAGE[doc["imageId"]] - return [image] - - -def gqa_doc_to_text(doc, model_specific_prompt_kwargs): - question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] - return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/hallusion_bench/evaluate_hb.py b/lmms_eval/tasks/hallusion_bench/evaluate_hb.py deleted file mode 100644 index 87c6551..0000000 --- a/lmms_eval/tasks/hallusion_bench/evaluate_hb.py +++ /dev/null @@ -1,129 +0,0 @@ -import os -import json -import logging -from tqdm import tqdm - -from lmms_eval.tasks.hallusion_bench.utils import evaluate_by_chatgpt, check_same_by_chatgpt, assign_correctness, get_eval_all, get_eval_fig, get_eval_pair_all - -cur_dir = os.path.dirname(os.path.abspath(__file__)) -output_entry = "model_prediction" -correctness_entry = "gpt4v_output_gpt_check" - -metric = ["aAcc", "fAcc", "qAcc"] - -eval_logger = logging.getLogger("lmms-eval") - - -def hb_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") - return f"{pre_prompt}{doc['question']}{post_prompt}" - - -def hb_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def hb_process_results(doc, result): - sample = doc - # doc.pop("image") - sample["model_prediction"] = result[0] - return {k: sample for k in metric} - - -def hb_aggregation_result(results, metric, args): - data_vd = [] - data_vs = [] - for data in tqdm(results, desc="Split vd and vs"): - if data["category"] == "VD": - data_vd.append(data) - if data["category"] == "VS": - data_vs.append(data) - eval_logger.info("Do gpt eval vd ...") - path = os.path.join(args.output_path, "gpt_response") - os.makedirs(path, exist_ok=True) - save_json_path_vd = f"{path}/hallusion_output_vd_model.json" - save_json_path_vs = f"{path}/hallusion_output_vs_model.json" - data_vd = evaluate_by_chatgpt(data_vd, output_entry=output_entry, correctness_entry=correctness_entry, load_json=True, save_json_path=save_json_path_vd) - # data_vd = check_same_by_chatgpt(data_vd, output_entry=output_entry, load_json=True, save_json_path=save_json_path_vd) - data_vd = assign_correctness(data_vd, correctness_entry=correctness_entry) - eval_logger.info("Do gpt eval vs") - data_vs = evaluate_by_chatgpt(data_vs, output_entry=output_entry, correctness_entry=correctness_entry, load_json=True, save_json_path=save_json_path_vs) - # data_vs = check_same_by_chatgpt(data_vs, output_entry=output_entry, load_json=True, save_json_path=save_json_path_vs) - data_vs = assign_correctness(data_vs, correctness_entry=correctness_entry) - results = data_vs + data_vd - - if metric == "aAcc": - all_data = get_eval_all(results, model_correctness_entry=correctness_entry) - return round(100 * all_data["correct"] / all_data["total"], 4) - elif metric == "fAcc": - fig_all = get_eval_fig(results) - return round(100 * fig_all["correct"] / fig_all["total"], 4) - elif metric == "qAcc": - all_data = get_eval_pair_all(results, model_correctness_entry=correctness_entry) - return round(100 * all_data["correct"] / all_data["total"], 4) - - -def hb_aggregation_result_qAcc(results, args): - return hb_aggregation_result(results, "qAcc", args) - - -def hb_aggregation_result_fAcc(results, args): - return hb_aggregation_result(results, "fAcc", args) - - -def hb_aggregation_result_aAcc(results, args): - return hb_aggregation_result(results, "aAcc", args) - - -def hb_aggregation_result_intern(results, metric): - scores = [] - for result in results: - ans = "1" if result["model_prediction"].lower().find("yes") != -1 else "0" - scores.append(ans == result["gt_answer"]) - result["answer"] = ans - - if metric == "aAcc": - return sum(scores) / len(scores) - elif metric == "qAcc": - qlist = {} - for r in results: - key = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["question_id"])]) - try: - qlist[key].append(r["answer"] == r["gt_answer"]) - except: - qlist[key] = [r["answer"] == r["gt_answer"]] - out = [] - for q, v in qlist.items(): - out.append(min(v)) - - return sum(out) / len(out) - elif metric == "fAcc": - qlist = {} - for r in results: - key = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["figure_id"])]) - try: - qlist[key].append(r["answer"] == r["gt_answer"]) - except: - qlist[key] = [r["answer"] == r["gt_answer"]] - out = [] - for q, v in qlist.items(): - out.append(min(v)) - return sum(out) / len(out) - - -def hb_aggregation_result_qAcc_intern(results): - eval_logger.info("Calculating qAcc ...") - return hb_aggregation_result_intern(results, "qAcc") - - -def hb_aggregation_result_fAcc_intern(results): - eval_logger.info("Calculating fAcc ...") - return hb_aggregation_result_intern(results, "fAcc") - - -def hb_aggregation_result_aAcc_intern(results): - eval_logger.info("Calculating aAcc ...") - return hb_aggregation_result_intern(results, "aAcc") diff --git a/lmms_eval/tasks/hallusion_bench/hallusion_bench_image.yaml b/lmms_eval/tasks/hallusion_bench/hallusion_bench_image.yaml deleted file mode 100644 index 39a5be4..0000000 --- a/lmms_eval/tasks/hallusion_bench/hallusion_bench_image.yaml +++ /dev/null @@ -1,41 +0,0 @@ -dataset_path: lmms-lab/HallusionBench -dataset_kwargs: - token: True -task: "hallusion_bench_image" -test_split: image -output_type: generate_until -doc_to_visual: !function evaluate_hb.hb_doc_to_visual -doc_to_text: !function evaluate_hb.hb_doc_to_text -doc_to_target: "gt_answer_details" -process_results: !function evaluate_hb.hb_process_results -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -metric_list: - - metric: aAcc - aggregation: !function evaluate_hb.hb_aggregation_result_aAcc - higher_is_better: true - - metric: qAcc - aggregation: !function evaluate_hb.hb_aggregation_result_qAcc - higher_is_better: true - - metric: fAcc - aggregation: !function evaluate_hb.hb_aggregation_result_fAcc - higher_is_better: true - # - metric: aAcc - # aggregation: !function evaluate_hb.hb_aggregation_result_aAcc_intern - # higher_is_better: true - # - metric: qAcc - # aggregation: !function evaluate_hb.hb_aggregation_result_qAcc_intern - # higher_is_better: true - # - metric: fAcc - # aggregation: !function evaluate_hb.hb_aggregation_result_fAcc_intern - # higher_is_better: true -metadata: - - version: 0.0 diff --git a/lmms_eval/tasks/hallusion_bench/utils.py b/lmms_eval/tasks/hallusion_bench/utils.py deleted file mode 100644 index 1d8dfda..0000000 --- a/lmms_eval/tasks/hallusion_bench/utils.py +++ /dev/null @@ -1,306 +0,0 @@ -import csv -import json -from tqdm import tqdm -import numpy as np -import os -import time -import openai -import threading -import requests -import logging - -API_TYPE = os.getenv("API_TYPE", "openai") - -if API_TYPE == "openai": - API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") - API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json", - } -elif API_TYPE == "azure": - API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") - API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") - headers = { - "api-key": API_KEY, - "Content-Type": "application/json", - } - -eval_logger = logging.getLogger("lmms-eval") - - -def evaluate_by_chatgpt(data, output_entry, correctness_entry, gpt_model="gpt-4", load_json=False, save_json_path="./hallusion_output.json", retries=3): - if load_json and os.path.exists(save_json_path): - with open(save_json_path, "r") as f: - output = json.load(f) - else: - output = [] - for sample in tqdm(data[len(output) :], desc="Eval by GPT"): - prompt = "Imagine you are an intelligent teacher. Thoroughly read the question, reference answer and the prediction answer to ensure a clear understanding of the information provided. Assess the correctness of the predictions. " - prompt += 'If the prediction answer does not conflict with the reference answer, please generate “correct”. If the prediction answer conflict with the reference answer, please generate “incorrect”. If the prediction answer is unclear about the answer, please generate "unclear". \n\n Question:' - prompt += sample["question"] - prompt += "\nReference answer: " - prompt += sample["gt_answer_details"] - prompt += "\nPrediction answer:" - prompt += sample[output_entry] - prompt += "\nOutput:" - - # https://github.com/openai/openai-python/issues/322#issuecomment-1767841683 - for attempt in range(retries): - try: - messages = [{"role": "user", "content": prompt}] - payload = { - "messages": messages, - "max_tokens": 16, - } - # set model when using openai api_key. Azure api_key does not need model since the endpoint fixed the model. - if API_TYPE == "openai": - payload["model"] = gpt_model - response = requests.post(API_URL, headers=headers, json=payload, timeout=30) - response.raise_for_status() - response = response.json() - break - except Exception as e: - eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}") - if attempt < retries - 1: # If we have retries left, sleep and then continue to next attempt - time.sleep(5) - else: # If this was the last attempt, log and return empty - eval_logger.error(f"All {retries} attempts failed. Last error message: {str(e)}") - try: - output_text = response["choices"][0]["message"]["content"] - except Exception as e: - eval_logger.info(f"Get error {str(e)} when extracting response") - output_text = "unclear" - - if "incorrect" in output_text.lower(): - gpt_correctness = "0" - - elif "correct" in output_text.lower(): - gpt_correctness = "1" - else: - gpt_correctness = "2" - - sample[correctness_entry] = gpt_correctness - sample["gpt_answer"] = prompt + output_text - - output.append(sample) - - with open(save_json_path, "w") as f: - json.dump(output, f, indent=4) - - return output - - -def check_same_by_chatgpt(data, output_entry, gpt_model="gpt-4", load_json=False, save_json_path="./hallusion_output.json", retries=3): - orig_response = {} - - for r in data: - if str(r["figure_id"]) == "0": - key = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["question_id"])]) - orig_response[key] = r[output_entry] - - for sample in tqdm(data, desc="Check same by GPT"): - if "same" not in sample.keys(): - key = "_".join([sample["category"], sample["subcategory"], str(sample["set_id"]), str(sample["question_id"])]) - response2 = orig_response[key] - - prompt = "Imagine you are an intelligent teacher. Thoroughly read the two responses to two different questions. Assess the consistency of the information provided within those two responses. " - prompt += "You do not know the specific questions, but you can asssess the consistency among the two responses by checking for logical conflicts if both responses are correct. " - prompt += 'If response1 does not conflict with response2, please generate “same”. Otherwise, generate "different". \n\n response1:' - prompt += sample[output_entry] - prompt += "\nresponse2: " - prompt += response2 - prompt += "\nOutput:" - - # https://github.com/openai/openai-python/issues/322#issuecomment-1767841683 - for attempt in range(retries): - try: - headers = { - "api-key": API_KEY, - "Content-Type": "application/json", - } - - messages = [{"role": "user", "content": prompt}] - - payload = { - "model": gpt_model, - "messages": messages, - "max_tokens": 16, - } - response = requests.post(API_URL, headers=headers, json=payload) - response.raise_for_status() - response = response.json() - - break - except Exception as e: - eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}") - if attempt < retries - 1: # If we have retries left, sleep and then continue to next attempt - time.sleep(5) - else: # If this was the last attempt, log and return empty - eval_logger.error(f"All {retries} attempts failed. Last error message: {str(e)}") - - try: - output_text = response["choices"][0]["message"]["content"] - except Exception as e: - eval_logger.info(f"Get error {str(e)} when extracting response") - output_text = "different" - - gpt_same = "0" - - if "same" in output_text.lower(): - gpt_same = "1" - - elif "different" in output_text.lower(): - gpt_same = "0" - - sample["same"] = gpt_same - - with open(save_json_path, "w") as f: - json.dump(data, f, indent=4) - - return data - - -def assign_correctness(data_arr, correctness_entry): - for r in data_arr: - assert int(r[correctness_entry]) == 0 or int(r[correctness_entry]) == 1 or int(r[correctness_entry]) == 2 - if r["category"] == "VS" and int(r["figure_id"]) == 0: # if there is no visual supplement and the model does not know, count it as correct - r["correct"] = 1 if int(r[correctness_entry]) == 1 or int(r[correctness_entry]) == 2 else 0 - else: - r["correct"] = 1 if int(r[correctness_entry]) == 1 else 0 - return data_arr - - -def get_eval_fig(data): # per figure - eval_fig_dict = dict() - - for r in data: - if r["category"] == "VS" and str(r["figure_id"]) == "0": # no figure - continue - name = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["figure_id"])]) - if name in eval_fig_dict: - c, t = eval_fig_dict[name] - eval_fig_dict[name] = (c + r["correct"], t + 1) - else: - eval_fig_dict[name] = (r["correct"], 1) - - eval_fig_stat = {} - eval_fig_stat["note"] = "all accuracy per image (consistency test)" - eval_fig_stat["total"] = len(eval_fig_dict.keys()) - eval_fig_stat["correct"] = 0 - eval_fig_stat["wrong"] = 0 - eval_fig_stat["inconsistent"] = 0 - eval_fig_stat["score"] = 0 - - for v in eval_fig_dict.values(): - if v[0] == v[1]: - eval_fig_stat["correct"] += 1 - elif v[0] == 0: - eval_fig_stat["wrong"] += 1 - else: - eval_fig_stat["inconsistent"] += 1 - eval_fig_stat["score"] += v[0] / v[1] - - eval_fig_stat["score"] = eval_fig_stat["score"] / eval_fig_stat["total"] - return eval_fig_stat - - -def get_eval_all(data, model_correctness_entry): # per question - eval_all_dict = dict() - eval_all_stat = {} - eval_all_stat["LH"] = 0 - eval_all_stat["VI"] = 0 - eval_all_stat["Mix"] = 0 - - for r in data: - name = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["figure_id"]), str(r["question_id"])]) - assert name not in eval_all_dict - - eval_all_dict[name] = r["correct"] - - if str(r["category"]) == "VD": # VD - if str(r["figure_id"]) == "0": - if str(r[model_correctness_entry]) == "0" or str(r[model_correctness_entry]) == "2": - eval_all_stat["VI"] += 1 - else: - if str(r[model_correctness_entry]) == "0": - eval_all_stat["Mix"] += 1 - elif str(r[model_correctness_entry]) == "2": - eval_all_stat["VI"] += 1 - else: # VS - if str(r["visual_input"]) == "0": # no visual - if str(r[model_correctness_entry]) == "0": - eval_all_stat["LH"] += 1 - else: # original visual or modified visual (isual_input == 1 or 2) - if str(r[model_correctness_entry]) == "0": - eval_all_stat["Mix"] += 1 - elif str(r[model_correctness_entry]) == "2": - eval_all_stat["VI"] += 1 - - eval_all_stat["note"] = "all accuracy per question" - eval_all_stat["total"] = len(eval_all_dict.keys()) - eval_all_stat["correct"] = np.count_nonzero(list(eval_all_dict.values())) - eval_all_stat["wrong"] = eval_all_stat["total"] - eval_all_stat["correct"] - - return eval_all_stat - - -def get_eval_pair_all(data, model_correctness_entry): # per question pair - orig_correctness = dict() - counter = 0 - lh_counter = 0 - vi_counter = 0 - both_counter = 0 - - for r in data: - if str(r["figure_id"]) == "0": - key = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["question_id"])]) - orig_correctness[key] = r[model_correctness_entry] - - get_eval_pair_dict = dict() - - for r in data: - name = "_".join([r["category"], r["subcategory"], str(r["set_id"]), str(r["question_id"])]) - if name in get_eval_pair_dict: - c, t = get_eval_pair_dict[name] - get_eval_pair_dict[name] = (c + r["correct"], t + 1) - else: - get_eval_pair_dict[name] = (r["correct"], 1) - counter += 1 - - eval_all_pair_stat = {} - eval_all_pair_stat["note"] = "all accuracy per question pair" - eval_all_pair_stat["total"] = len(get_eval_pair_dict.keys()) - eval_all_pair_stat["total_q"] = counter - eval_all_pair_stat["correct"] = 0 - eval_all_pair_stat["wrong"] = 0 - eval_all_pair_stat["LH"] = 0 - eval_all_pair_stat["VI"] = 0 - eval_all_pair_stat["Mix"] = 0 - - eval_all_pair_stat["LH_cg"] = lh_counter - eval_all_pair_stat["VI_cg"] = vi_counter - eval_all_pair_stat["Mix_cg"] = both_counter - - # for v in get_eval_pair_dict.values(): - # if v[0] == v[1]: - # eval_all_pair_stat["correct"] += 1 - # else: - # eval_all_pair_stat["wrong"] += 1 - - # for v in get_analysis_pair_dict.values(): - # if v[0] > 0 and v[1] > 0: - # eval_all_pair_stat["Mix"] += 1 - # elif v[0] > 0: - # eval_all_pair_stat["LH"] += 1 - # elif v[1] > 0: - # eval_all_pair_stat["VI"] += 1 - - for k in get_eval_pair_dict.keys(): - v = get_eval_pair_dict[k] - if v[0] == v[1]: - eval_all_pair_stat["correct"] += 1 - else: - eval_all_pair_stat["wrong"] += 1 - - return eval_all_pair_stat diff --git a/lmms_eval/tasks/iconqa/_default_template_docvqa_yaml b/lmms_eval/tasks/iconqa/_default_template_docvqa_yaml deleted file mode 100644 index 6648788..0000000 --- a/lmms_eval/tasks/iconqa/_default_template_docvqa_yaml +++ /dev/null @@ -1,22 +0,0 @@ -dataset_path: lmms-lab/ICON-QA -dataset_kwargs: - token: True -output_type: generate_until -doc_to_visual: !function utils.doc_to_visual -doc_to_text: !function utils.doc_to_text -doc_to_target: "answers" -# process_results: !function utils.test_process_results -generation_kwargs: - max_new_tokens: 32 - temperature: 0 - do_sample: False -model_specific_prompt_kwargs: - default: - pre_prompt: "" - statement: "Given a set of images and a question, please provide the answer to the question.\n" - options_statement: "Question: {question}.\nOptions:\n{options}\nPlease answer with the option letter from the given choices directly." - freeform_statement: "Question: {question}.\nPlease answer the question using a single word or phrase." -metric_list: - - metric: anls - aggregation: mean - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/iconqa/iconqa.yaml b/lmms_eval/tasks/iconqa/iconqa.yaml deleted file mode 100644 index 7c5caea..0000000 --- a/lmms_eval/tasks/iconqa/iconqa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: iconqa -task: -- iconqa_val -- iconqa_test diff --git a/lmms_eval/tasks/iconqa/iconqa_test.yaml b/lmms_eval/tasks/iconqa/iconqa_test.yaml deleted file mode 100644 index 84cfc6f..0000000 --- a/lmms_eval/tasks/iconqa/iconqa_test.yaml +++ /dev/null @@ -1,3 +0,0 @@ -task: "iconqa_test" -test_split: test -include: _default_template_docvqa_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/iconqa/iconqa_val.yaml b/lmms_eval/tasks/iconqa/iconqa_val.yaml deleted file mode 100644 index 73e0300..0000000 --- a/lmms_eval/tasks/iconqa/iconqa_val.yaml +++ /dev/null @@ -1,3 +0,0 @@ -task: "iconqa_val" -test_split: val -include: _default_template_docvqa_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/iconqa/utils.py b/lmms_eval/tasks/iconqa/utils.py deleted file mode 100644 index de99e80..0000000 --- a/lmms_eval/tasks/iconqa/utils.py +++ /dev/null @@ -1,57 +0,0 @@ -import json -import os - - -def options_to_str(options_prompt): - option_prompt_str = "" - for i, option in enumerate(options_prompt): - option_choice = chr(ord("A") + i) - option_prompt_str += f"{option_choice}. {option}\n" - - option_prompt_str = option_prompt_str.rstrip("\n") - return option_prompt_str - - -def doc_to_visual(doc): - image_list = [] - if "query_image" in doc: - image_list.append(doc["query_image"].convert("RGB")) - for i in range(5): - id = f"choice_image_{i}" - if id in doc and doc[id] is not None: - image_list.append(doc[id].convert("RGB")) - assert len(image_list) < 6, "Maximum 5 images allowed for ICON-QA" - return image_list - - -def doc_to_text(doc, model_specific_prompt_kwargs): - question = doc["question"] - ques_type = doc["ques_type"] - options_prompt = [] - - if ques_type == "choose_img": - options_prompt.append("The first image.") - options_prompt.append("The second image.") - - options_str = options_to_str(options_prompt) - full_prompt = f"{model_specific_prompt_kwargs['pre_prompt']}{model_specific_prompt_kwargs['statement']}{model_specific_prompt_kwargs['options_statement'].format(question=question, options=options_str)}" - - elif ques_type == "choose_txt": - choices = doc["choices"].split(",") - for i, choice in enumerate(choices): - options_prompt.append(f"{choice}") - - options_str = options_to_str(options_prompt) - full_prompt = f"{model_specific_prompt_kwargs['pre_prompt']}{model_specific_prompt_kwargs['statement']}{model_specific_prompt_kwargs['options_statement'].format(question=question, options=options_str)}" - - elif ques_type == "fill_in_blank": - full_prompt = f"{model_specific_prompt_kwargs['pre_prompt']}{model_specific_prompt_kwargs['statement']}{model_specific_prompt_kwargs['freeform_statement'].format(question=question)}" - - return full_prompt - - -def test_process_results(doc, results): - pred = results[0] - questionId = doc["question_id"] - answer = doc["answer"] - return {"anls": {"questionId": int(questionId), "answer": answer, "pred_answer": pred}} diff --git a/lmms_eval/tasks/infovqa/_default_template_infovqa_yaml b/lmms_eval/tasks/infovqa/_default_template_infovqa_yaml deleted file mode 100644 index 22d69ed..0000000 --- a/lmms_eval/tasks/infovqa/_default_template_infovqa_yaml +++ /dev/null @@ -1,15 +0,0 @@ -dataset_path: lmms-lab/DocVQA -dataset_name: InfographicVQA -dataset_kwargs: - token: True -doc_to_target: "answers" -doc_to_visual: !function utils.infovqa_doc_to_visual -doc_to_text: !function utils.infovqa_doc_to_text -generation_kwargs: - max_new_tokens: 32 - temperature: 0 - do_sample: False -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer the question using a single word or phrase." \ No newline at end of file diff --git a/lmms_eval/tasks/infovqa/infovqa.yaml b/lmms_eval/tasks/infovqa/infovqa.yaml deleted file mode 100644 index 9e3b7f4..0000000 --- a/lmms_eval/tasks/infovqa/infovqa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: infovqa -task: -- infovqa_val -- infovqa_test diff --git a/lmms_eval/tasks/infovqa/infovqa_test.yaml b/lmms_eval/tasks/infovqa/infovqa_test.yaml deleted file mode 100644 index 0ca3932..0000000 --- a/lmms_eval/tasks/infovqa/infovqa_test.yaml +++ /dev/null @@ -1,10 +0,0 @@ -task: "infovqa_test" -test_split: test -output_type: generate_until -process_results: !function utils.infovqa_test_process_results -metric_list: - - metric: submission - aggregation: !function utils.infovqa_test_aggregate_results - higher_is_better: true -include: _default_template_infovqa_yaml - \ No newline at end of file diff --git a/lmms_eval/tasks/infovqa/infovqa_val.yaml b/lmms_eval/tasks/infovqa/infovqa_val.yaml deleted file mode 100644 index 5adc20b..0000000 --- a/lmms_eval/tasks/infovqa/infovqa_val.yaml +++ /dev/null @@ -1,8 +0,0 @@ -task: "infovqa_val" -test_split: validation -output_type: generate_until -metric_list: - - metric: anls - aggregation: mean - higher_is_better: true -include: _default_template_infovqa_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/infovqa/utils.py b/lmms_eval/tasks/infovqa/utils.py deleted file mode 100644 index a4b1ab5..0000000 --- a/lmms_eval/tasks/infovqa/utils.py +++ /dev/null @@ -1,33 +0,0 @@ -import json -import os -import logging - - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -lmms_logger = logging.getLogger("lmms-eval") - - -def infovqa_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def infovqa_doc_to_text(doc, model_specific_prompt_kwargs): - question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] - return f"{pre_prompt}{question}{post_prompt}" - - -def infovqa_test_process_results(doc, results): - pred = results[0] - questionId = doc["questionId"] - return {"submission": {"questionId": int(questionId), "answer": pred}} - - -def infovqa_test_aggregate_results(results, args): - # save results as json - file = generate_submission_file("infovqa_test_for_submission.json", args) - with open(file, "w") as f: - json.dump(results, f) - lmms_logger.info(f"Results saved to {file}") diff --git a/lmms_eval/tasks/llava-bench-coco/llava-bench-coco.yaml b/lmms_eval/tasks/llava-bench-coco/llava-bench-coco.yaml deleted file mode 100644 index 0f7d335..0000000 --- a/lmms_eval/tasks/llava-bench-coco/llava-bench-coco.yaml +++ /dev/null @@ -1,38 +0,0 @@ -dataset_path: lmms-lab/llava-bench-coco -dataset_kwargs: - token: True -task: "llava_bench_coco" -test_split: train -output_type: generate_until -doc_to_visual: !function utils.llava_doc_to_visual -doc_to_text: !function utils.llava_doc_to_text -doc_to_target: "gpt_answer" -generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 -process_results: !function utils.llava_process_results -metric_list: - - metric: gpt_eval_llava_all - aggregation: !function utils.llava_all_aggregation - higher_is_better: true - - metric: gpt_eval_llava_conv - aggregation: !function utils.llava_conv_aggregation - higher_is_better: true - - metric: gpt_eval_llava_detail - aggregation: !function utils.llava_detail_aggregation - higher_is_better: true - - metric: gpt_eval_llava_complex - aggregation: !function utils.llava_complex_aggregation - higher_is_better: true -metadata: - version: 0.0 - gpt_eval_model_name: "gpt-4-0314" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/llava-bench-coco/rule.json b/lmms_eval/tasks/llava-bench-coco/rule.json deleted file mode 100644 index 26c7f4e..0000000 --- a/lmms_eval/tasks/llava-bench-coco/rule.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "coding": {"role": "Assistant", "prompt": "Your task is to evaluate the coding abilities of the above two assistants. They have been asked to implement a program to solve a given problem. Please review their code submissions, paying close attention to their problem-solving approach, code structure, readability, and the inclusion of helpful comments.\n\nPlease ensure that the assistants' submissions:\n\n1. Correctly implement the given problem statement.\n2. Contain accurate and efficient code.\n3. Include clear and concise comments that explain the code's logic and functionality.\n4. Adhere to proper coding standards and best practices.\n\nOnce you have carefully reviewed both submissions, provide detailed feedback on their strengths and weaknesses, along with any suggestions for improvement. You should first output a single line containing two scores on the scale of 1-10 (1: no code/no sense; 10: perfect) for Assistant 1 and 2, respectively. Then give extra comments starting from the next line."}, - "math": {"role": "Assistant", "prompt": "We would like to request your feedback on the mathematical proficiency of two AI assistants regarding the given user question.\nFirstly, please solve the problem independently, without referring to the answers provided by Assistant 1 and Assistant 2.\nAfterward, please examine the problem-solving process of Assistant 1 and Assistant 2 step-by-step to ensure their correctness, identifying any incorrect steps if present. Your evaluation should take into account not only the answer but also the problem-solving steps.\nFinally, please output a Python tuple containing two numerical scores for Assistant 1 and Assistant 2, ranging from 1 to 10, respectively. If applicable, explain the reasons for any variations in their scores and determine which assistant performed better."}, - "default": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above.\nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "conv": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "detail": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "complex": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "llava_bench_conv": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "llava_bench_detail": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "llava_bench_complex": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."} -} \ No newline at end of file diff --git a/lmms_eval/tasks/llava-bench-coco/utils.py b/lmms_eval/tasks/llava-bench-coco/utils.py deleted file mode 100644 index 8858637..0000000 --- a/lmms_eval/tasks/llava-bench-coco/utils.py +++ /dev/null @@ -1,200 +0,0 @@ -import json -import logging -import os -import requests -import numpy as np -import openai -from openai import OpenAI -import time -import yaml -from pathlib import Path -from copy import deepcopy - -eval_logger = logging.getLogger("lmms-eval") -NUM_SECONDS_TO_SLEEP = 0.5 - -LLAVA_W_METRICS = ["gpt_eval_llava_conv", "gpt_eval_llava_detail", "gpt_eval_llava_complex"] - -rule_dict = json.load(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "rule.json"), "r")) - -with open(Path(__file__).parent / "llava-bench-coco.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] - -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] - -API_TYPE = os.getenv("API_TYPE", "openai") - -if API_TYPE == "openai": - API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") - API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json", - } -elif API_TYPE == "azure": - API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") - API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") - headers = { - "api-key": API_KEY, - "Content-Type": "application/json", - } - - -def get_eval(content: str, max_tokens: int, retries: int = 3): - global headers - - messages = [ - { - "role": "system", - "content": "You are a helpful and precise assistant for checking the quality of the answer.", - }, - {"role": "user", "content": content}, - ] - - payload = { - "model": GPT_EVAL_MODEL_NAME, - "messages": messages, - "temperature": 0.2, - "max_tokens": max_tokens, - } - - for attempt in range(retries): - try: - response = requests.post(API_URL, headers=headers, json=payload, timeout=60) - response.raise_for_status() - response_data = response.json() - - content = response_data["choices"][0]["message"]["content"].strip() - if content != "": - return content, response_data["model"] - break # If successful, break out of the loop - - except Exception as e: - eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}") - if attempt < retries - 1: # If we have retries left, sleep and then continue to next attempt - time.sleep(NUM_SECONDS_TO_SLEEP) - else: # If this was the last attempt, log and return empty - eval_logger.error(f"All {retries} attempts failed. Last error message: {str(e)}") - return "", "" - return "", "" - - -def parse_score(review): - try: - score_pair = review.split("\n")[0] - score_pair = score_pair.replace(",", " ") - sp = score_pair.split(" ") - if len(sp) == 2: - return [float(sp[0]), float(sp[1])] - else: - eval_logger.debug("error", review) - return [-1, -1] - except Exception as e: - eval_logger.debug(e) - eval_logger.debug("error", review) - return [-1, -1] - - -def llava_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def llava_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") - return f"{pre_prompt}{doc['question']}{post_prompt}" - - -def llava_process_results(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case coco_bleu), value: metric value - """ - try: - question = doc.get("question", "") - ans1 = doc.get("answer", "") - ans2 = result[0] if result else "" - captions = doc.get("caption", []) - context = "\n".join(captions) if isinstance(captions, list) else captions - category = "llava_bench_" + doc.get("category", "") - rule = rule_dict.get(category, {}) - prompt = rule.get("prompt", "") - role = rule.get("role", "user") - content = f"[Context]\n{context}\n\n" f"[Question]\n{question}\n\n" f"[{role} 1]\n{ans1}\n\n[End of {role} 1]\n\n" f"[{role} 2]\n{ans2}\n\n[End of {role} 2]\n\n" f"[System]\n{prompt}\n\n" - - review, model_name = get_eval(content, 1024) - scores = parse_score(review) - except Exception as e: - eval_logger.error(f"Error for Question ID: {doc.get('question_id', 'Unknown')}: {e}") - review = "Failed to Get a Proper Review." - model_name = "Failed Request" - scores = [-1, -1] - - metric = f"gpt_eval_llava_{doc.get('category', 'unknown')}" - category_review_dict = {"question": question, "ans1": ans1, "ans2": ans2, "context": context, "category": category, "review": review, "scores": scores, "eval_model": model_name, "content": content} - - non_category_review_dict = deepcopy(category_review_dict) - non_category_review_dict["scores"] = [-999, -999] - - data_dict = {} - for m in LLAVA_W_METRICS: - if m == metric: - data_dict[m] = category_review_dict - else: - data_dict[m] = non_category_review_dict - data_dict["gpt_eval_llava_all"] = category_review_dict - - # return {"gpt_eval_llava_all": review_dict} - return data_dict - - -def llava_conv_aggregation(results): - return llava_aggregation(results, "conv") - - -def llava_complex_aggregation(results): - return llava_aggregation(results, "complex") - - -def llava_detail_aggregation(results): - return llava_aggregation(results, "detail") - - -def llava_all_aggregation(results): - return llava_aggregation(results, "all") - - -def llava_aggregation(results, category): - try: - scores = [] - for result in results: - if -999 in result["scores"]: - continue - scores.append(result["scores"]) - - stats = np.asarray(scores).mean(0).tolist() - stats = [round(x, 3) for x in stats] - # gpt4_score_percentage = stats[0] * 10 - # model_score_percentage = stats[1] * 10 - # eval_logger.info(f"Category: {category}") - # eval_logger.info(f"GPT4 Score: {gpt4_score_percentage:.1f}%") - # eval_logger.info(f"Model Score: {model_score_percentage:.1f}%") - # eval_logger.info("=========================") - return round(stats[1] / stats[0] * 100, 1) - except Exception as e: - eval_logger.error(f"Error in llava_aggregation: {e}") - return None diff --git a/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml b/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml deleted file mode 100644 index 5352027..0000000 --- a/lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml +++ /dev/null @@ -1,39 +0,0 @@ -dataset_path: lmms-lab/llava-bench-in-the-wild -dataset_kwargs: - token: True -task: "llava_in_the_wild" -test_split: train -output_type: generate_until -doc_to_visual: !function utils.llava_doc_to_visual -doc_to_text: !function utils.llava_doc_to_text -doc_to_target: "gpt_answer" -generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.llava_process_results -metric_list: - - metric: gpt_eval_llava_all - aggregation: !function utils.llava_all_aggregation - higher_is_better: true - - metric: gpt_eval_llava_conv - aggregation: !function utils.llava_conv_aggregation - higher_is_better: true - - metric: gpt_eval_llava_detail - aggregation: !function utils.llava_detail_aggregation - higher_is_better: true - - metric: gpt_eval_llava_complex - aggregation: !function utils.llava_complex_aggregation - higher_is_better: true -metadata: - version: 0.0 - gpt_eval_model_name: "gpt-4-0613" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "" diff --git a/lmms_eval/tasks/llava-in-the-wild/rule.json b/lmms_eval/tasks/llava-in-the-wild/rule.json deleted file mode 100644 index 26c7f4e..0000000 --- a/lmms_eval/tasks/llava-in-the-wild/rule.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "coding": {"role": "Assistant", "prompt": "Your task is to evaluate the coding abilities of the above two assistants. They have been asked to implement a program to solve a given problem. Please review their code submissions, paying close attention to their problem-solving approach, code structure, readability, and the inclusion of helpful comments.\n\nPlease ensure that the assistants' submissions:\n\n1. Correctly implement the given problem statement.\n2. Contain accurate and efficient code.\n3. Include clear and concise comments that explain the code's logic and functionality.\n4. Adhere to proper coding standards and best practices.\n\nOnce you have carefully reviewed both submissions, provide detailed feedback on their strengths and weaknesses, along with any suggestions for improvement. You should first output a single line containing two scores on the scale of 1-10 (1: no code/no sense; 10: perfect) for Assistant 1 and 2, respectively. Then give extra comments starting from the next line."}, - "math": {"role": "Assistant", "prompt": "We would like to request your feedback on the mathematical proficiency of two AI assistants regarding the given user question.\nFirstly, please solve the problem independently, without referring to the answers provided by Assistant 1 and Assistant 2.\nAfterward, please examine the problem-solving process of Assistant 1 and Assistant 2 step-by-step to ensure their correctness, identifying any incorrect steps if present. Your evaluation should take into account not only the answer but also the problem-solving steps.\nFinally, please output a Python tuple containing two numerical scores for Assistant 1 and Assistant 2, ranging from 1 to 10, respectively. If applicable, explain the reasons for any variations in their scores and determine which assistant performed better."}, - "default": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above.\nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "conv": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "detail": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "complex": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "llava_bench_conv": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "llava_bench_detail": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "llava_bench_complex": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."} -} \ No newline at end of file diff --git a/lmms_eval/tasks/llava-in-the-wild/utils.py b/lmms_eval/tasks/llava-in-the-wild/utils.py deleted file mode 100644 index ac86ee9..0000000 --- a/lmms_eval/tasks/llava-in-the-wild/utils.py +++ /dev/null @@ -1,197 +0,0 @@ -import json -import logging -import os -import requests -import numpy as np -import openai -from openai import OpenAI -import time -import yaml -from pathlib import Path -from copy import deepcopy - -eval_logger = logging.getLogger("lmms-eval") -NUM_SECONDS_TO_SLEEP = 5 - -LLAVA_W_METRICS = ["gpt_eval_llava_conv", "gpt_eval_llava_detail", "gpt_eval_llava_complex"] - -rule_dict = json.load(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "rule.json"), "r")) - -with open(Path(__file__).parent / "llava-in-the-wild.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] - -API_TYPE = os.getenv("API_TYPE", "openai") - -if API_TYPE == "openai": - API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") - API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json", - } -elif API_TYPE == "azure": - API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") - API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") - headers = { - "api-key": API_KEY, - "Content-Type": "application/json", - } - - -def get_eval(content: str, max_tokens: int, retries: int = 5): - global headers - - messages = [ - { - "role": "system", - "content": "You are a helpful and precise assistant for checking the quality of the answer.", - }, - {"role": "user", "content": content}, - ] - - payload = { - "model": GPT_EVAL_MODEL_NAME, - "messages": messages, - "temperature": 0.2, - "max_tokens": max_tokens, - } - - for attempt in range(retries): - try: - response = requests.post(API_URL, headers=headers, json=payload, timeout=60) - response.raise_for_status() - response_data = response.json() - - content = response_data["choices"][0]["message"]["content"].strip() - if content != "": - return content, response_data["model"] - break # If successful, break out of the loop - - except Exception as e: - eval_logger.info(f"Attempt {attempt + 1} failed with error: {e}") - if attempt < retries: # If we have retries left, sleep and then continue to next attempt - time.sleep(NUM_SECONDS_TO_SLEEP) - else: # If this was the last attempt, log and return empty - eval_logger.error(f"All {retries} attempts failed. Last error message: {e}") - return "", "" - return "", "" - - -def parse_score(review): - try: - score_pair = review.split("\n")[0] - score_pair = score_pair.replace(",", " ") - sp = score_pair.split(" ") - if len(sp) == 2: - return [float(sp[0]), float(sp[1])] - else: - eval_logger.debug(f"Can not split: {review}. Returning [-1, -1]") - return [-1, -1] - except Exception as e: - eval_logger.debug(f"Error: {e}. Returning [-1, -1]") - return [-1, -1] - - -def llava_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def llava_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") - return f"{pre_prompt}{doc['question']}{post_prompt}" - - -def llava_process_results(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case coco_bleu), value: metric value - """ - try: - question = doc.get("question", "") - ans1 = doc.get("gpt_answer", "") - ans2 = result[0] if result else "" - captions = doc.get("caption", []) - context = "\n".join(captions) if isinstance(captions, list) else captions - category = "llava_bench_" + doc.get("category", "") - rule = rule_dict.get(category, {}) - prompt = rule.get("prompt", "") - role = rule.get("role", "user") - content = f"[Context]\n{context}\n\n" f"[Question]\n{question}\n\n" f"[{role} 1]\n{ans1}\n\n[End of {role} 1]\n\n" f"[{role} 2]\n{ans2}\n\n[End of {role} 2]\n\n" f"[System]\n{prompt}\n\n" - - review, model_name = get_eval(content, 1024) - scores = parse_score(review) - except Exception as e: - eval_logger.error(f"Error for Question ID: {doc.get('question_id', 'Unknown')}: {e}") - review = "Failed to Get a Proper Review." - model_name = "Failed Request" - scores = [-1, -1] - - metric = f"gpt_eval_llava_{doc.get('category', 'all')}" - category_review_dict = {"question": question, "ans1": ans1, "ans2": ans2, "context": context, "category": category, "review": review, "scores": scores, "eval_model": model_name, "content": content} - - non_category_review_dict = deepcopy(category_review_dict) - non_category_review_dict["scores"] = [-999, -999] - - data_dict = {} - for m in LLAVA_W_METRICS: - if m == metric: - data_dict[m] = category_review_dict - else: - data_dict[m] = non_category_review_dict - data_dict["gpt_eval_llava_all"] = category_review_dict - - # return {"gpt_eval_llava_all": review_dict} - return data_dict - - -def llava_conv_aggregation(results): - return llava_aggregation(results, "conv") - - -def llava_complex_aggregation(results): - return llava_aggregation(results, "complex") - - -def llava_detail_aggregation(results): - return llava_aggregation(results, "detail") - - -def llava_all_aggregation(results): - return llava_aggregation(results, "all") - - -def llava_aggregation(results, category): - try: - scores = [] - for result in results: - if -999 in result["scores"]: - continue - scores.append(result["scores"]) - - stats = np.asarray(scores).mean(0).tolist() - stats = [round(x, 3) for x in stats] - # gpt4_score_percentage = stats[0] * 10 - # model_score_percentage = stats[1] * 10 - # eval_logger.info(f"Category: {category}") - # eval_logger.info(f"GPT4 Score: {gpt4_score_percentage:.1f}%") - # eval_logger.info(f"Model Score: {model_score_percentage:.1f}%") - # eval_logger.info("=========================") - return round(stats[1] / stats[0] * 100, 1) - except Exception as e: - eval_logger.info(f"Error in llava_aggregation: {e}, and in category: {category}") - return None diff --git a/lmms_eval/tasks/longhallqa/conversation_binary_v1.yaml b/lmms_eval/tasks/longhallqa/conversation_binary_v1.yaml deleted file mode 100644 index 4c4eadf..0000000 --- a/lmms_eval/tasks/longhallqa/conversation_binary_v1.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: QHQK/conversation_hall_binary_v1 -dataset_kwargs: - token: True -task: "conversation_binary_v1" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_binary -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.lhqa_process_results -metric_list: - - metric: lhqa_accuracy - aggregation: !function utils.lhqa_aggregate_accuracy - higher_is_better: true - - metric: lhqa_precision - aggregation: !function utils.lhqa_aggregate_precision - higher_is_better: true - - metric: lhqa_recall - aggregation: !function utils.lhqa_aggregate_recall - higher_is_better: true - - metric: lhqa_f1_score - aggregation: !function utils.lhqa_aggregate_f1_score - higher_is_better: true - - metric: lhqa_yes_ratio - aggregation: !function utils.lhqa_aggregate_yes_ratio - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/longhallqa/conversation_choice_v1.yaml b/lmms_eval/tasks/longhallqa/conversation_choice_v1.yaml deleted file mode 100644 index 9783e68..0000000 --- a/lmms_eval/tasks/longhallqa/conversation_choice_v1.yaml +++ /dev/null @@ -1,20 +0,0 @@ -dataset_path: QHQK/conversation_hall_choice_v2 -dataset_kwargs: - token: True -task: "conversation_choice_v1" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_choice -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original -process_results: !function utils.lhqa_process_result_choice -metric_list: - - metric: lhqa_acc - aggregation: !function utils.lhqa_aggregate_choice_acc - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/longhallqa/conversation_choice_v1_ppl.yaml b/lmms_eval/tasks/longhallqa/conversation_choice_v1_ppl.yaml deleted file mode 100644 index 654f222..0000000 --- a/lmms_eval/tasks/longhallqa/conversation_choice_v1_ppl.yaml +++ /dev/null @@ -1,15 +0,0 @@ -dataset_path: QHQK/conversation_hall_choice_v2 -dataset_kwargs: - token: True -task: "conversation_choice_v1_ppl" -test_split: test -output_type: multiple_choice -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_ppl -doc_to_choice : !function utils.lhqa_doc_to_choice -doc_to_target: !function utils.lhqa_doc_to_mc_target - -metric_list: - - metric: acc -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/longhallqa/conversation_completion_v1.yaml b/lmms_eval/tasks/longhallqa/conversation_completion_v1.yaml deleted file mode 100644 index c4cd333..0000000 --- a/lmms_eval/tasks/longhallqa/conversation_completion_v1.yaml +++ /dev/null @@ -1,20 +0,0 @@ -dataset_path: QHQK/conversation_hall_completion_v2 -dataset_kwargs: - token: True -task: "conversation_completion_v1" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_choice -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original -process_results: !function utils.lhqa_process_result_choice -metric_list: - - metric: lhqa_acc - aggregation: !function utils.lhqa_aggregate_choice_acc - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/longhallqa/conversation_completion_v1_ppl.yaml b/lmms_eval/tasks/longhallqa/conversation_completion_v1_ppl.yaml deleted file mode 100644 index fa96075..0000000 --- a/lmms_eval/tasks/longhallqa/conversation_completion_v1_ppl.yaml +++ /dev/null @@ -1,15 +0,0 @@ -dataset_path: QHQK/conversation_hall_completion_v2 -dataset_kwargs: - token: True -task: "conversation_completion_v1_ppl" -test_split: test -output_type: multiple_choice -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_completion -doc_to_choice : !function utils.lhqa_doc_to_choice -doc_to_target: !function utils.lhqa_doc_to_mc_target - -metric_list: - - metric: acc -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/longhallqa/description_binary_v1.yaml b/lmms_eval/tasks/longhallqa/description_binary_v1.yaml deleted file mode 100644 index 1229472..0000000 --- a/lmms_eval/tasks/longhallqa/description_binary_v1.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: QHQK/description_hall_binary_v1 -dataset_kwargs: - token: True -task: "description_binary_v1" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_binary -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.lhqa_process_results -metric_list: - - metric: lhqa_accuracy - aggregation: !function utils.lhqa_aggregate_accuracy - higher_is_better: true - - metric: lhqa_precision - aggregation: !function utils.lhqa_aggregate_precision - higher_is_better: true - - metric: lhqa_recall - aggregation: !function utils.lhqa_aggregate_recall - higher_is_better: true - - metric: lhqa_f1_score - aggregation: !function utils.lhqa_aggregate_f1_score - higher_is_better: true - - metric: lhqa_yes_ratio - aggregation: !function utils.lhqa_aggregate_yes_ratio - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/longhallqa/description_choice_v1.yaml b/lmms_eval/tasks/longhallqa/description_choice_v1.yaml deleted file mode 100644 index 6348a0e..0000000 --- a/lmms_eval/tasks/longhallqa/description_choice_v1.yaml +++ /dev/null @@ -1,20 +0,0 @@ -dataset_path: QHQK/description_hall_choice_v2 -dataset_kwargs: - token: True -task: "description_choice_v1" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_choice -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original -process_results: !function utils.lhqa_process_result_choice -metric_list: - - metric: lhqa_acc - aggregation: !function utils.lhqa_aggregate_choice_acc - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/longhallqa/description_choice_v1_ppl.yaml b/lmms_eval/tasks/longhallqa/description_choice_v1_ppl.yaml deleted file mode 100644 index 12a6105..0000000 --- a/lmms_eval/tasks/longhallqa/description_choice_v1_ppl.yaml +++ /dev/null @@ -1,15 +0,0 @@ -dataset_path: QHQK/description_hall_choice_v2 -dataset_kwargs: - token: True -task: "description_choice_v1_ppl" -test_split: test -output_type: multiple_choice -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_ppl -doc_to_choice : !function utils.lhqa_doc_to_choice -doc_to_target: !function utils.lhqa_doc_to_mc_target - -metric_list: - - metric: acc -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/longhallqa/description_completion_v1.yaml b/lmms_eval/tasks/longhallqa/description_completion_v1.yaml deleted file mode 100644 index 6e22346..0000000 --- a/lmms_eval/tasks/longhallqa/description_completion_v1.yaml +++ /dev/null @@ -1,20 +0,0 @@ -dataset_path: QHQK/description_hall_completion_v2 -dataset_kwargs: - token: True -task: "description_completion_v1" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_choice -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original -process_results: !function utils.lhqa_process_result_choice -metric_list: - - metric: lhqa_acc - aggregation: !function utils.lhqa_aggregate_choice_acc - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/longhallqa/description_completion_v1_ppl.yaml b/lmms_eval/tasks/longhallqa/description_completion_v1_ppl.yaml deleted file mode 100644 index 51a3b05..0000000 --- a/lmms_eval/tasks/longhallqa/description_completion_v1_ppl.yaml +++ /dev/null @@ -1,15 +0,0 @@ -dataset_path: QHQK/description_hall_completion_v2 -dataset_kwargs: - token: True -task: "description_completion_v1_ppl" -test_split: test -output_type: multiple_choice -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_completion -doc_to_choice : !function utils.lhqa_doc_to_choice -doc_to_target: !function utils.lhqa_doc_to_mc_target - -metric_list: - - metric: acc -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/longhallqa/object_binary_v1.yaml b/lmms_eval/tasks/longhallqa/object_binary_v1.yaml deleted file mode 100644 index 1b9fb7a..0000000 --- a/lmms_eval/tasks/longhallqa/object_binary_v1.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: QHQK/object_hall_binary_v1 -dataset_kwargs: - token: True -task: "object_binary_v1" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.lhqa_doc_to_visual -doc_to_text: !function utils.lhqa_doc_to_text_binary -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.lhqa_process_results -metric_list: - - metric: lhqa_accuracy - aggregation: !function utils.lhqa_aggregate_accuracy - higher_is_better: true - - metric: lhqa_precision - aggregation: !function utils.lhqa_aggregate_precision - higher_is_better: true - - metric: lhqa_recall - aggregation: !function utils.lhqa_aggregate_recall - higher_is_better: true - - metric: lhqa_f1_score - aggregation: !function utils.lhqa_aggregate_f1_score - higher_is_better: true - - metric: lhqa_yes_ratio - aggregation: !function utils.lhqa_aggregate_yes_ratio - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/mathverse.yaml b/lmms_eval/tasks/mathverse/mathverse.yaml deleted file mode 100644 index 4df3fc6..0000000 --- a/lmms_eval/tasks/mathverse/mathverse.yaml +++ /dev/null @@ -1,14 +0,0 @@ -group: mathverse -task: - - mathverse_testmini - - mathverse_testmini_text_only - - mathverse_testmini_text_lite - - mathverse_testmini_text_dominant - - mathverse_testmini_vision_intensive - - mathverse_testmini_vision_dominant - - mathverse_testmini_vision_only -metadata: - version: 0.0 - gpt_eval_model_name: "gpt-3.5-turbo" - trunk_response: 30 - quick_match: false \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/mathverse_evals.py b/lmms_eval/tasks/mathverse/mathverse_evals.py deleted file mode 100644 index 5b0b6f5..0000000 --- a/lmms_eval/tasks/mathverse/mathverse_evals.py +++ /dev/null @@ -1,306 +0,0 @@ -import time -import requests -import logging -from tqdm import tqdm -import pandas as pd - -eval_logger = logging.getLogger("lmms-eval") - -DEMO_PROMPT_EXTRACT = """ -I am providing you a response from a model to a math problem, termed 'Model Response'. You should extract the answer from the response as 'Extracted Answer'. Directly output the extracted answer with no explanation. - -1. -Model response: 'Rounded to two decimal places, the perimeter of the sector is approximately:\n\n(-2, 1)' -Extracted Answer: (-2, 1) - -2. -Model response: 'at those points.\n\nTherefore, the correct option that represents the meaning of the intersection points of the graphs is:\n\nD. They give the solutions to the equation $f(t)=g(t)$.",' -Extracted Answer: D - -3. -Model response: ' at 1 (there's a closed circle at y = 1), the range in interval notation is \\((-4, 1]\\).\n\nFinal values:\nDomain: \\((-3, 3]\\)\nRange: \\((-4, 1]\\)' -Extracted Answer: Domain: \\((-3, 3]\\)\nRange: \\((-4, 1]\\) - -4. -Model response: 'As it stands, I cannot provide the correct option letter because there isn't enough information to solve for 'y'.' -Extracted Answer: null - -5. -Model response: 'Given that AB = 17.6 meters, we can now substitute into the equation:\n\nd = 17.6 / cos(38\u00b0)\n\nTherefore, to one decimal place, the distance d between Ned and Bart is approximately 22.3 meters.' -Extracted answer: 22.3 - -6. -Model response: have all the coefficients for the quadratic function:\n\\( f(x) = ax^2 + bx + c \\)\n\\( f(x) = -1x^2 - 2x + 1 \\)\n\nTherefore, the equation for the graphed function \\( f \\) is:\n\\( f(x) = -x^2 - 2x + 1 \\)"' -Extracted answer: f(x) = -x^2 - 2x + 1 - -7. -""" - -DEMO_PROMPT_SCORE = """ -Below are two answers to a math question. Question is [Question], [Standard Answer] is the standard answer to the question, and [Model_answer] is the answer extracted from a model's output to this question. Determine whether these two answers are consistent. -Please note that only when the [Model_answer] completely matches the [Standard Answer] means they are consistent. For non-multiple-choice questions, if the meaning is expressed in the same way, it is also considered consistent, for example, 0.5m and 50cm. -If they are consistent, Judement is 1; if they are different, Judement is 0. - -[Question]: Write the set of numbers represented on the number line in interval notation. -[Standard Answer]: (-2,1] -[Model_answer] : Extracted Answer: \\((-2, 1)\\) -Judgement: 0 - -[Question]: As shown in the figure, circle O has a radius 1.0, if angle BAC = 60.0, then the length of BC is ()\nChoices:\nA:2\nB:2\u221a{{3}}\nC:\u221a{{3}}\nD:2\u221a{{2}} -[Standard Answer]: C -[Model_answer] : B:2\u221a{{3}} -Judgement: 0 - -[Question]: Find the domain and range of the function f using interval notation. -[Standard Answer]: domain: [-4, 0) and range: (-3, 1] -[Model_answer] : Range: \\((-4, 1]\\) -Judgement: 0 - -[Question]: As shown in the figure, circle O has a radius 1.0, if angle BAC = 60.0, then the length of BC is ()\nChoices:\nA:2\nB:2\u221a{{3}}\nC:\u221a{{3}}\nD:2\u221a{{2}} -[Standard Answer]: C -[Model_answer] : null -Judgement: 0 - -[Question]: Given the graph of the ellipse that intersects with x-axis at 9 and -9 and with y-axis at 3 and -3, determine its equation.A. \\frac{{x^2}}{{81}} + \\frac{{y^2}}{{9}} = 1 B. Can not determine.\n -[Standard Answer]: A -[Model_answer] : \\frac{{x^2}}{{81}} + \\frac{{y^2}}{{9}} = 1 -Judgement: 1 - -[Question]: {question} -[Standard Answer]: {gt} -[Model_answer] : {extraction} -Judgement: """ - -class MathVerseEvaluator: - API_URL = "https://api.openai.com/v1/chat/completions" - - def __init__(self, api_key, gpt_model="gpt-3.5-turbo"): - self.api_key = api_key - self.gpt_model = gpt_model - - def _post_request(self, payload): - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json", - } - response = requests.post(self.API_URL, headers=headers, json=payload, timeout=30) - response.raise_for_status() - return response.json() - - def get_chat_response(self, prompt, temperature=0, max_tokens=256, n=1, patience=10000000, sleep_time=0): - messages = [ - {"role": "user", "content": prompt}, - ] - payload = {"model": self.gpt_model, "messages": messages, "temperature": temperature, "max_tokens": max_tokens, "n": n} - - while patience > 0: - patience -= 1 - try: - response = self._post_request(payload) - if n == 1: - prediction = response["choices"][0]["message"]["content"].strip() - if prediction and prediction != "": - return prediction - else: - prediction = [choice["message"]["content"].strip() for choice in response["choices"]] - if prediction and prediction[0] != "": - return prediction - - except Exception as e: - # some model may output repetitive answer, which ChatGPT will throw an error. - if 'repetitive patterns' in str(e): - print(str(e)) - print("Continue with empty answer") - return "" - # some answer may contain some sensitive words, like 'test' - if 'sensitive' in str(e) or '400' in str(e): - print(str(e)) - print("Continue with empty answer") - return "0" - - if "Rate limit" not in str(e): - eval_logger.error(e) - - if "Please reduce the length of the messages" in str(e): - eval_logger.error("!!Reduce prompt size") - # reduce input prompt and keep the tail - new_size = int(len(prompt) * 0.9) - new_start = len(prompt) - new_size - prompt = prompt[new_start:] - payload["messages"] = [ - {"role": "user", "content": prompt}, - ] - - if sleep_time > 0: - time.sleep(sleep_time) - return "" - - def verify_extraction(self, extraction): - extraction = extraction.strip() - if not extraction: - return False - return True - - def create_extract_prompt(self, demo_prompt, response): - demo_prompt = demo_prompt.strip() - test_prompt = f"Model response: '{response}'\nExtracted Answer: " - full_prompt = f"{demo_prompt}\n\n{test_prompt}" - return full_prompt - - def create_match_prompt(self, demo_prompt, question, answer, extraction): - demo_prompt = demo_prompt.strip() - full_prompt = demo_prompt.format(question=question, gt=answer, extraction=extraction) - return full_prompt - - def extract_answer(self, response): - - if not response: - return "" - - # general extraction - try: - full_prompt = self.create_extract_prompt(DEMO_PROMPT_EXTRACT, response) - extraction = self.get_chat_response(full_prompt, temperature=0, max_tokens=256, n=1) - return extraction - except Exception as e: - eval_logger.error(e) - eval_logger.error(f"Error in extracting answer for problem") - - return "" - - def score_answer(self, question, answer, extraction, quick_match=False): - if quick_match: - return extraction == answer - - try: - full_prompt = self.create_match_prompt(DEMO_PROMPT_SCORE, question, answer, extraction) - while True: - extraction = self.get_chat_response(full_prompt, temperature=0, max_tokens=8, n=1) - judgement = extraction.replace("Judgement:", "").strip() - if judgement.strip() in ['0', '1']: - return int(judgement) == 1 - - except Exception as e: - print(e) - print(f"Error in matching answer") - - return False - - - def get_acc_with_contion(self, res_pd, key, value): - """ - Calculate the accuracy of predictions with a specific condition - """ - total_pd = res_pd[res_pd[key] == value] - - correct_pd = total_pd[total_pd["true_false"] == True] - acc = "{:.2f}".format(len(correct_pd) / len(total_pd) * 100) if len(total_pd) > 0 else "0.00" - return len(correct_pd), len(total_pd), acc - - def create_one_query(self, problem, shot_type, hint, query_type, examples=None, shot_num=0, use_caption=False, use_ocr=False): - ### [1] Demo prompt - if shot_num == 0: - demo_prompt = "" - else: - demos = [] - shot_num = min(shot_num, len(examples)) - for example in examples[:shot_num]: - prompt = "" - - # question - prompt += f"Question: {example[query_type]}" - - # solution - if shot_type == "solution": - solution = example["solution"].strip() - prompt += "\n" + f"Solution: {solution}" - - # step-by-step - if shot_type == "step-by-step": - solution = example["solution"].strip() - prompt += "\n" + f"{solution}" - - # direct - if shot_type == "direct": - solution = example["solution"].strip() - prompt += "\n" + f"{solution}" - - demos.append(prompt) - - demo_prompt = "\n\n".join(demos) - - ### [2] Test query - # problem info - question = problem["question"] - question_type = problem["question_type"] - - # hint - # format-prompt - if shot_type == "format-prompt": - hint_text = "" - # custom-prompt - elif shot_type == "custom-prompt": - if question_type == 'multi-choice': - hint_text = hint['multi-choice'] - else: # free-form - hint_text = hint['free-form'] - - # question - if shot_type == "format-prompt": - question_text = f"{problem[query_type]}" - elif shot_type == "custom-prompt": - question_text = f"Question: {question}" - - elements = [hint_text, question_text] - test_query = "\n".join([e for e in elements if e != ""]) - - ### [3] Final query - query = demo_prompt + "\n\n" + test_query - query = query.strip() - return query - - def eval_results(self, results, config): - # extract and score for each question - for inst in tqdm(results): - full_prediction = inst['prediction'].strip() - problem = { - "question_type": inst["question_type"], - "answer": inst["answer"] if "answer" in inst else None, - "question_for_eval": inst["question_for_eval"], - } - if config['metadata'].get('trunk_response', -1) > 0: - prediction = ' '.join(full_prediction.split(' ')[-config['metadata']['trunk_response']:]) - else: - prediction = full_prediction - extraction = self.extract_answer(prediction) - # set test set answer to None - true_false = self.score_answer(problem['question_for_eval'], problem["answer"], extraction, config['metadata']['quick_match']) if problem["answer"] is not None else False - - inst['extraction'] = extraction - inst['prediction'] = prediction - inst['true_false'] = true_false - - # calculate total scores - sample_index = [result["sample_index"] for result in results] - total = len(results) - correct = sum(1 for idx, pid in enumerate(sample_index) if results[idx]["true_false"]) - accuracy = round(correct / total * 100, 2) - scores = {"average": {"accuracy": accuracy, "correct": correct, "total": total}} - - for result in results: - result.update(result.pop("metadata")) - - results_dict = {result["sample_index"]: result for result in results} - df = pd.DataFrame(results_dict).T - target_keys = ["problem_version", "subfield"] - - for key in target_keys: - values = df[key].unique() - scores[key] = {} - for value in values: - correct, total, acc = self.get_acc_with_contion(df, key, value) - if total > 0: - scores[key][value] = {"accuracy": acc, "correct": correct, "total": total} - scores[key] = dict(sorted(scores[key].items(), key=lambda item: float(item[1]["accuracy"]), reverse=True)) - - return results_dict, scores \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini.yaml deleted file mode 100644 index 62cd8e1..0000000 --- a/lmms_eval/tasks/mathverse/mathverse_testmini.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: CaraJ/MathVerse-lmmseval -dataset_name: testmini -dataset_kwargs: - token: False -task: "mathverse_testmini" -test_split: testmini -output_type: generate_until -doc_to_visual: !function utils.mathverse_doc_to_visual -doc_to_text: !function utils.mathverse_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.mathverse_process_results -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mathverse_aggregate_results_eval - higher_is_better: true - - metric: submission - aggregation: !function utils.mathverse_aggregate_results_submission - higher_is_better: true - -model_specific_prompt_kwargs: - default: - shot_type: "format-prompt" # can also be "custom-prompt" - query_type: "query_wo" # now only support query_wo -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_text_dominant.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_text_dominant.yaml deleted file mode 100644 index 7ff121e..0000000 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_text_dominant.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: CaraJ/MathVerse-lmmseval -dataset_name: testmini_version_split -dataset_kwargs: - token: False -task: "mathverse_testmini_text_dominant" -test_split: text_dominant -output_type: generate_until -doc_to_visual: !function utils.mathverse_doc_to_visual -doc_to_text: !function utils.mathverse_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.mathverse_process_results -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mathverse_aggregate_results_eval - higher_is_better: true - - metric: submission - aggregation: !function utils.mathverse_aggregate_results_submission - higher_is_better: true - -model_specific_prompt_kwargs: - default: - shot_type: "format-prompt" # can also be "custom-prompt" - query_type: "query_wo" # now only support query_wo -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_text_lite.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_text_lite.yaml deleted file mode 100644 index 2137e35..0000000 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_text_lite.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: CaraJ/MathVerse-lmmseval -dataset_name: testmini_version_split -dataset_kwargs: - token: False -task: "mathverse_testmini_text_lite" -test_split: text_lite -output_type: generate_until -doc_to_visual: !function utils.mathverse_doc_to_visual -doc_to_text: !function utils.mathverse_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.mathverse_process_results -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mathverse_aggregate_results_eval - higher_is_better: true - - metric: submission - aggregation: !function utils.mathverse_aggregate_results_submission - higher_is_better: true - -model_specific_prompt_kwargs: - default: - shot_type: "format-prompt" # can also be "custom-prompt" - query_type: "query_wo" # now only support query_wo -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_text_only.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_text_only.yaml deleted file mode 100644 index 42ceb5e..0000000 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_text_only.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: CaraJ/MathVerse-lmmseval -dataset_name: testmini_text_only -dataset_kwargs: - token: False -task: "mathverse_testmini_text_only" -test_split: text_only -output_type: generate_until -doc_to_visual: !function utils.mathverse_doc_to_visual -doc_to_text: !function utils.mathverse_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.mathverse_process_results -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mathverse_aggregate_results_eval - higher_is_better: true - - metric: submission - aggregation: !function utils.mathverse_aggregate_results_submission - higher_is_better: true - -model_specific_prompt_kwargs: - default: - shot_type: "format-prompt" # can also be "custom-prompt" - query_type: "query_wo" # now only support query_wo -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_dominant.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_vision_dominant.yaml deleted file mode 100644 index fe66a40..0000000 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_dominant.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: CaraJ/MathVerse-lmmseval -dataset_name: testmini_version_split -dataset_kwargs: - token: False -task: "mathverse_testmini_vision_dominant" -test_split: vision_dominant -output_type: generate_until -doc_to_visual: !function utils.mathverse_doc_to_visual -doc_to_text: !function utils.mathverse_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.mathverse_process_results -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mathverse_aggregate_results_eval - higher_is_better: true - - metric: submission - aggregation: !function utils.mathverse_aggregate_results_submission - higher_is_better: true - -model_specific_prompt_kwargs: - default: - shot_type: "format-prompt" # can also be "custom-prompt" - query_type: "query_wo" # now only support query_wo -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_intensive.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_vision_intensive.yaml deleted file mode 100644 index cb92299..0000000 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_intensive.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: CaraJ/MathVerse-lmmseval -dataset_name: testmini_version_split -dataset_kwargs: - token: False -task: "mathverse_testmini_vision_intensive" -test_split: vision_intensive -output_type: generate_until -doc_to_visual: !function utils.mathverse_doc_to_visual -doc_to_text: !function utils.mathverse_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.mathverse_process_results -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mathverse_aggregate_results_eval - higher_is_better: true - - metric: submission - aggregation: !function utils.mathverse_aggregate_results_submission - higher_is_better: true - -model_specific_prompt_kwargs: - default: - shot_type: "format-prompt" # can also be "custom-prompt" - query_type: "query_wo" # now only support query_wo -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_only.yaml b/lmms_eval/tasks/mathverse/mathverse_testmini_vision_only.yaml deleted file mode 100644 index 72df736..0000000 --- a/lmms_eval/tasks/mathverse/mathverse_testmini_vision_only.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: CaraJ/MathVerse-lmmseval -dataset_name: testmini_version_split -dataset_kwargs: - token: False -task: "mathverse_testmini_vision_only" -test_split: vision_only -output_type: generate_until -doc_to_visual: !function utils.mathverse_doc_to_visual -doc_to_text: !function utils.mathverse_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.mathverse_process_results -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mathverse_aggregate_results_eval - higher_is_better: true - - metric: submission - aggregation: !function utils.mathverse_aggregate_results_submission - higher_is_better: true - -model_specific_prompt_kwargs: - default: - shot_type: "format-prompt" # can also be "custom-prompt" - query_type: "query_wo" # now only support query_wo -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/mathverse/utils.py b/lmms_eval/tasks/mathverse/utils.py deleted file mode 100644 index 1b6d8db..0000000 --- a/lmms_eval/tasks/mathverse/utils.py +++ /dev/null @@ -1,94 +0,0 @@ -import logging -import yaml -import os -from pathlib import Path -import pandas as pd -import json - -eval_logger = logging.getLogger("lmms-eval") -from lmms_eval.tasks.mathverse.mathverse_evals import MathVerseEvaluator -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -with open(Path(__file__).parent / "mathverse.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -mathverse_evaluator = MathVerseEvaluator(api_key=os.getenv("OPENAI_API_KEY", "YOUR_API_KEY"), gpt_model=config["metadata"]["gpt_eval_model_name"]) - - -def mathverse_doc_to_visual(doc): - if str(doc["image"]).strip() == '': - return [] - return [doc["image"].convert("RGB")] - - -def mathverse_doc_to_text(doc, model_specific_prompt_kwargs=None): - problem = { - "question": doc["question"], - "answer": doc["answer"] if "answer" in doc else None, - "query_wo": doc["query_wo"], - "query_cot": doc["query_cot"], - "question_type": doc["question_type"], - "problem_version": doc["problem_version"], - } - query_prompt = mathverse_evaluator.create_one_query(problem, examples=None, shot_num=0, shot_type=model_specific_prompt_kwargs["shot_type"], hint=model_specific_prompt_kwargs.get("hint", None),query_type=model_specific_prompt_kwargs["query_type"]) - return query_prompt - - - -def mathverse_process_results(doc, results): - prediction = results[0].strip() - - result = { - "sample_index": doc["sample_index"], - "problem_index": doc["problem_index"], - "problem_version": doc["problem_version"], - "question": doc["question"], - "answer": doc["answer"] if "answer" in doc else None, - "prediction": prediction, - "question_type": doc["question_type"], - "metadata": doc["metadata"], - "query_wo": doc["query_wo"], - "query_cot": doc["query_cot"], - } - - return { - "gpt_eval_score": result, - "submission": result, - } - - -def mathverse_aggregate_results_submission(results, args, *, calculate_gain=False, random_scores=None): - split_flag = results[0]["metadata"]["split"] - path = generate_submission_file(f"mathverse_{split_flag}_results.json", args) - with open(path, "w") as f: - json.dump(results, f, indent=4) - - eval_logger.info(f"Saved results to {path}") - -def mathverse_aggregate_results_eval(results, args, *, calculate_gain=False, random_scores=None): - split_flag = results[0]["metadata"]["split"] - # save the result first, in case the gpt evaluation fails - path = generate_submission_file(f"mathverse_{split_flag}_results.json", args) - with open(path, "w") as f: - json.dump(results, f, indent=4) - # gpt evaluation - results_dict, scores = mathverse_evaluator.eval_results(results, config) - # save results - path = generate_submission_file(f"mathverse_{split_flag}_results.json", args) - with open(path, "w") as f: - json.dump(results_dict, f, indent=4) - # save scores - path = generate_submission_file(f"mathverse_{split_flag}_scores.json", args) - with open(path, "w") as f: - json.dump(scores, f, indent=4) - eval_logger.info(f"Saved scores to {path}") - if scores["average"]["accuracy"] == 0: - return None - return scores["average"]["accuracy"] diff --git a/lmms_eval/tasks/mathvista/mathvista.yaml b/lmms_eval/tasks/mathvista/mathvista.yaml deleted file mode 100644 index b108ca5..0000000 --- a/lmms_eval/tasks/mathvista/mathvista.yaml +++ /dev/null @@ -1,8 +0,0 @@ -group: mathvista -task: - - mathvista_testmini - - mathvista_test -metadata: - version: 0.0 - gpt_eval_model_name: "gpt-4-0613" - quick_extract: false \ No newline at end of file diff --git a/lmms_eval/tasks/mathvista/mathvista_evals.py b/lmms_eval/tasks/mathvista/mathvista_evals.py deleted file mode 100644 index d40f609..0000000 --- a/lmms_eval/tasks/mathvista/mathvista_evals.py +++ /dev/null @@ -1,456 +0,0 @@ -import time -import requests -import re -from Levenshtein import distance -import logging - -eval_logger = logging.getLogger("lmms-eval") -DEMO_PROMPT = """ -Please read the following example. Then extract the answer from the model response and type it at the end of the prompt. - -Hint: Please answer the question requiring an integer answer and provide the final value, e.g., 1, 2, 3, at the end. -Question: Which number is missing? - -Model response: The number missing in the sequence is 14. - -Extracted answer: 14 - -Hint: Please answer the question requiring a floating-point number with one decimal place and provide the final value, e.g., 1.2, 1.3, 1.4, at the end. -Question: What is the fraction of females facing the camera? - -Model response: The fraction of females facing the camera is 0.6, which means that six out of ten females in the group are facing the camera. - -Extracted answer: 0.6 - -Hint: Please answer the question requiring a floating-point number with two decimal places and provide the final value, e.g., 1.23, 1.34, 1.45, at the end. -Question: How much money does Luca need to buy a sour apple candy and a butterscotch candy? (Unit: $) - -Model response: Luca needs $1.45 to buy a sour apple candy and a butterscotch candy. - -Extracted answer: 1.45 - -Hint: Please answer the question requiring a Python list as an answer and provide the final list, e.g., [1, 2, 3], [1.2, 1.3, 1.4], at the end. -Question: Between which two years does the line graph saw its maximum peak? - -Model response: The line graph saw its maximum peak between 2007 and 2008. - -Extracted answer: [2007, 2008] - -Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end. -Question: What fraction of the shape is blue?\nChoices:\n(A) 3/11\n(B) 8/11\n(C) 6/11\n(D) 3/5 - -Model response: The correct answer is (B) 8/11. - -Extracted answer: B -""" - - -class MathVistaEvaluator: - API_URL = "https://api.openai.com/v1/chat/completions" - - def __init__(self, api_key, gpt_model="gpt-3.5-turbo", quick_extract=False): - self.api_key = api_key - self.gpt_model = gpt_model - self.quick_extract = quick_extract - - def _post_request(self, payload): - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json", - } - response = requests.post(self.API_URL, headers=headers, json=payload, timeout=30) - response.raise_for_status() - return response.json() - - def get_chat_response(self, prompt, temperature=0, max_tokens=256, n=1, patience=10000000, sleep_time=0): - messages = [ - {"role": "user", "content": prompt}, - ] - payload = {"model": self.gpt_model, "messages": messages, "temperature": temperature, "max_tokens": max_tokens, "n": n} - - while patience > 0: - patience -= 1 - try: - response = self._post_request(payload) - if n == 1: - prediction = response["choices"][0]["message"]["content"].strip() - if prediction and prediction != "": - return prediction - else: - prediction = [choice["message"]["content"].strip() for choice in response["choices"]] - if prediction and prediction[0] != "": - return prediction - - except Exception as e: - if "Rate limit" not in str(e): - eval_logger.error(e) - - if "Please reduce the length of the messages" in str(e): - eval_logger.error("!!Reduce prompt size") - # reduce input prompt and keep the tail - new_size = int(len(prompt) * 0.9) - new_start = len(prompt) - new_size - prompt = prompt[new_start:] - payload["messages"] = [ - {"role": "user", "content": prompt}, - ] - - if sleep_time > 0: - time.sleep(sleep_time) - return "" - - def verify_extraction(self, extraction): - extraction = extraction.strip() - if not extraction: - return False - return True - - def create_test_prompt(self, demo_prompt, query, response): - demo_prompt = demo_prompt.strip() - test_prompt = f"{query}\n\n{response}" - full_prompt = f"{demo_prompt}\n\n{test_prompt}\n\nExtracted answer: " - return full_prompt - - def extract_answer(self, response, problem, quick_extract=False): - question_type = problem["question_type"] - answer_type = problem["answer_type"] - choices = problem.get("choices", []) - query = problem["query"] - - if not response: - return "" - - if question_type == "multi_choice" and response in choices: - return response - - if answer_type == "integer": - try: - extraction = int(response) - return str(extraction) - except ValueError: - pass - - if answer_type == "float": - try: - extraction = str(float(response)) - return extraction - except ValueError: - pass - - # quick extraction - if quick_extract: - eval_logger.info("Quickly extracting answer...") - # The answer is "text". -> "text" - try: - result = re.search(r'The answer is "(.*)"\.', response) - if result: - extraction = result.group(1) - return extraction - except re.error: - pass - - # general extraction - try: - full_prompt = self.create_test_prompt(DEMO_PROMPT, query, response) - extraction = self.get_chat_response(full_prompt, temperature=0, max_tokens=256, n=1) - return extraction - except Exception as e: - eval_logger.error(e) - eval_logger.error(f"Error in extracting answer for problem") - - return "" - - def get_most_similar(self, prediction, choices): - """ - Use the Levenshtein distance (or edit distance) to determine which of the choices is most similar to the given prediction - """ - distances = [distance(prediction, choice) for choice in choices] - ind = distances.index(min(distances)) - return choices[ind] - - def normalize_extracted_answer(self, extraction, choices, question_type, answer_type, precision): - """ - Normalize the extracted answer to match the answer type - """ - if question_type == "multi_choice": - # make sure the extraction is a string - if isinstance(extraction, str): - extraction = extraction.strip() - else: - try: - extraction = str(extraction) - except: - extraction = "" - - # extract "A" from "(A) text" - letter = re.findall(r"\(([a-zA-Z])\)", extraction) - if len(letter) > 0: - extraction = letter[0].upper() - - options = [chr(ord("A") + i) for i in range(len(choices))] - - if extraction in options: - # convert option letter to text, e.g. "A" -> "text" - ind = options.index(extraction) - extraction = choices[ind] - else: - # select the most similar option - extraction = self.get_most_similar(extraction, choices) - assert extraction in choices - - elif answer_type == "integer": - try: - extraction = str(int(float(extraction))) - except: - extraction = None - - elif answer_type == "float": - try: - extraction = str(round(float(extraction), precision)) - except: - extraction = None - - elif answer_type == "list": - try: - extraction = str(extraction) - except: - extraction = None - - return extraction - - def safe_equal(self, prediction, answer): - """ - Check if the prediction is equal to the answer, even if they are of different types - """ - try: - if str(prediction).strip() == str(answer).strip(): - return True - return False - except Exception as e: - eval_logger.info(e) - return False - - def get_acc_with_contion(self, res_pd, key, value): - """ - Calculate the accuracy of predictions with a specific condition - """ - if key == "skills": - total_pd = res_pd[res_pd[key].apply(lambda x: value in x)] - else: - total_pd = res_pd[res_pd[key] == value] - - correct_pd = total_pd[total_pd["true_false"] == True] - acc = "{:.2f}".format(len(correct_pd) / len(total_pd) * 100) if len(total_pd) > 0 else "0.00" - return len(correct_pd), len(total_pd), acc - - def create_one_query(self, problem, shot_type, examples=None, shot_num=0, use_caption=False, use_ocr=False): - ### [1] Demo prompt - if shot_num == 0: - demo_prompt = "" - else: - demos = [] - shot_num = min(shot_num, len(examples)) - for example in examples[:shot_num]: - prompt = "" - - # question - prompt += f"Question: {example['question']}" - - # choices - if "choices" in example: - texts = ["Choices:"] - for i, choice in enumerate(example["choices"]): - texts.append(f"({chr(ord('A')+i)}) {choice}") - prompt += "\n" + "\n".join(texts) - - # caption - if use_caption: - caption = example["caption"] if "caption" in example else "" - if caption != "": - prompt += "\n" + f"Image description: {caption}" - - # ocr - if use_ocr: - ocr = example["ocr"] if "ocr" in example else "" - if ocr != "": - prompt += "\n" + f"Image detected text: {ocr}" - - # solution - if shot_type == "solution": - solution = example["solution"].strip() - prompt += "\n" + f"Solution: {solution}" - - # step-by-step - if shot_type == "step-by-step": - solution = example["solution"].strip() - prompt += "\n" + f"{solution}" - - # think-step-by-step - if shot_type == "think-step-by-step": - solution = example["solution"].strip() - prompt += "\n" + f"{solution}" - - # direct - if shot_type == "direct": - solution = example["solution"].strip() - prompt += "\n" + f"{solution}" - - # code - if shot_type == "code": - code = example["code"].strip() - prompt += "\n" + f"Python code: {code}" - - demos.append(prompt) - - demo_prompt = "\n\n".join(demos) - - ### [2] Test query - # problem info - question = problem["question"] - unit = problem["unit"] - choices = problem["choices"] - caption = problem["caption"] - ocr = problem["ocr"] - precision = problem["precision"] - question_type = problem["question_type"] - answer_type = problem["answer_type"] - - # hint - if shot_type == "solution": - if question_type == "multi_choice": - assert answer_type == "text" - hint_text = f"Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end." - else: - assert answer_type in ["integer", "float", "list"] - if answer_type == "integer": - hint_text = f"Hint: Please answer the question requiring an integer answer and provide the final value, e.g., 1, 2, 3, at the end." - - elif answer_type == "float" and precision == 1: - hint_text = f"Hint: Please answer the question requiring a floating-point number with one decimal place and provide the final value, e.g., 1.2, 1.3, 1.4, at the end." - - elif answer_type == "float" and precision == 2: - hint_text = f"Hint: Please answer the question requiring a floating-point number with two decimal places and provide the final value, e.g., 1.23, 1.34, 1.45, at the end." - - elif answer_type == "list": - hint_text = f"Hint: Please answer the question requiring a Python list as an answer and provide the final list, e.g., [1, 2, 3], [1.2, 1.3, 1.4], at the end." - # step-by-step - elif shot_type == "format-prompt": - if question_type == "multi_choice": - assert answer_type == "text" - hint_text = f"Answer with the option's letter from the given choices directly." - else: - if answer_type == "integer": - hint_text = f"Answer the question using a single integer number." - - elif answer_type == "float" and precision == 1: - hint_text = f"Answer the question using a single floating-point number with one decimal place." - - elif answer_type == "float" and precision == 2: - hint_text = f"Answer the question using a single floating-point number with two decimal places." - - elif answer_type == "list": - hint_text = f"Answer the question using a Python list." - # step-by-step - elif shot_type == "step-by-step": - if question_type == "multi_choice": - assert answer_type == "text" - hint_text = f"Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end." - else: - assert answer_type in ["integer", "float", "list"] - if answer_type == "integer": - hint_text = f"Hint: Please answer the question requiring an integer answer and provide the final value, e.g., 1, 2, 3, at the end." - - elif answer_type == "float" and precision == 1: - hint_text = f"Hint: Please answer the question requiring a floating-point number with one decimal place and provide the final value, e.g., 1.2, 1.3, 1.4, at the end." - - elif answer_type == "float" and precision == 2: - hint_text = f"Hint: Please answer the question requiring a floating-point number with two decimal places and provide the final value, e.g., 1.23, 1.34, 1.45, at the end." - - elif answer_type == "list": - hint_text = f"Hint: Please answer the question requiring a Python list as an answer and provide the final list, e.g., [1, 2, 3], [1.2, 1.3, 1.4], at the end." - # step-by-step - elif shot_type == "reason-first": - if question_type == "multi_choice": - assert answer_type == "text" - hint_text = f"First perform reasoning, then finally select the question from the choices in the following format: Answer: xxx." - else: - assert answer_type in ["integer", "float", "list"] - if answer_type == "integer": - hint_text = f"First perform reasoning, then finally answer the question requiring an integer answer and provide the final value, e.g., 1, 2, 3, at the end in the following format: Answer: xxx." - - elif answer_type == "float" and precision == 1: - hint_text = ( - f"First perform reasoning, then finally answer the question requiring a floating-point number with one decimal place and provide the final value, e.g., 1.2, 1.3, 1.4, at the end in the following format: Answer: xxx." - ) - - elif answer_type == "float" and precision == 2: - hint_text = f"First perform reasoning, then finally answer the question requiring a floating-point number with two decimal places and provide the final value, e.g., 1.23, 1.34, 1.45, at the end in the following format: Answer: xxx." - - elif answer_type == "list": - hint_text = f"First perform reasoning, then finally answer the question requiring a Python list as an answer and provide the final list, e.g., [1, 2, 3], [1.2, 1.3, 1.4], at the end in the following format: Answer: xxx." - elif shot_type == "direct": - hint_text = "" - else: - assert shot_type == "code" - hint_text = "Hint: Please generate a python code to solve the problem" - - # question - if shot_type == "format-prompt": - question_text = f"{question}" - else: - question_text = f"Question: {question}" - if unit: - question_text += f" (Unit: {unit})" - - # choices - if choices: - if shot_type == "format-prompt": - texts = [] - for i, choice in enumerate(choices): - texts.append(f"{chr(ord('A')+i)}. {choice}") - choices_text = "\n".join(texts) - else: - # choices: (A) 1.2 (B) 1.3 (C) 1.4 (D) 1.5 - texts = ["Choices:"] - for i, choice in enumerate(choices): - texts.append(f"({chr(ord('A')+i)}) {choice}") - choices_text = "\n".join(texts) - else: - choices_text = "" - - # caption - caption_text = "" - if use_caption and caption != "": - caption_text = f"Image description: {caption}" - - # ocr - ocr_text = "" - if use_ocr and ocr != "": - ocr_text = f"Image detected text: {ocr}" - - # prompt - if shot_type == "solution": - prompt = "Solution: " - elif shot_type == "format-prompt": - prompt = "" - elif shot_type == "step-by-step": - prompt = "" - elif shot_type == "reason-first": - prompt = "" - elif shot_type == "direct": - prompt = "" - else: - assert shot_type == "code" - prompt = "Python code: " - - if shot_type == "reason-first": - elements = [hint_text, question_text, choices_text, caption_text, ocr_text, prompt] - test_query = "\n".join([e for e in elements if e != ""]) - else: - elements = [question_text, choices_text, caption_text, ocr_text, hint_text, prompt] - test_query = "\n".join([e for e in elements if e != ""]) - - ### [3] Final query - query = demo_prompt + "\n\n" + test_query - query = query.strip() - return query diff --git a/lmms_eval/tasks/mathvista/mathvista_test.yaml b/lmms_eval/tasks/mathvista/mathvista_test.yaml deleted file mode 100644 index 171e06d..0000000 --- a/lmms_eval/tasks/mathvista/mathvista_test.yaml +++ /dev/null @@ -1,29 +0,0 @@ -dataset_path: AI4Math/MathVista -dataset_kwargs: - token: True -task: "mathvista_test" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.mathvista_doc_to_visual -doc_to_text: !function utils.mathvista_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.mathvista_process_results -metric_list: - - metric: submission - aggregation: !function utils.mathvista_aggregate_results - higher_is_better: true - -model_specific_prompt_kwargs: - default: - shot_type: "format-prompt" # can be "reason-first", "solution", "step-by-step" -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/mathvista/mathvista_testmini.yaml b/lmms_eval/tasks/mathvista/mathvista_testmini.yaml deleted file mode 100644 index bb6a938..0000000 --- a/lmms_eval/tasks/mathvista/mathvista_testmini.yaml +++ /dev/null @@ -1,31 +0,0 @@ -dataset_path: AI4Math/MathVista -dataset_kwargs: - token: True -task: "mathvista_testmini" -test_split: testmini -output_type: generate_until -doc_to_visual: !function utils.mathvista_doc_to_visual -doc_to_text: !function utils.mathvista_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.mathvista_process_results -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mathvista_aggregate_results - higher_is_better: true - -model_specific_prompt_kwargs: - default: - shot_type: "format-prompt" # can be "reason-first", "solution", "step-by-step" - phi3v: - shot_type: "solution" -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/mathvista/utils.py b/lmms_eval/tasks/mathvista/utils.py deleted file mode 100644 index 620e3f2..0000000 --- a/lmms_eval/tasks/mathvista/utils.py +++ /dev/null @@ -1,121 +0,0 @@ -import logging -import yaml -import os -from pathlib import Path -import pandas as pd -import json - -eval_logger = logging.getLogger("lmms-eval") -from lmms_eval.tasks.mathvista.mathvista_evals import MathVistaEvaluator -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -with open(Path(__file__).parent / "mathvista.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -mathvista_evaluator = MathVistaEvaluator(api_key=os.getenv("OPENAI_API_KEY", "YOUR_API_KEY"), gpt_model=config["metadata"]["gpt_eval_model_name"]) - - -def mathvista_doc_to_visual(doc): - return [doc["decoded_image"].convert("RGB")] - - -def mathvista_doc_to_text(doc, model_specific_prompt_kwargs=None): - problem = { - "question_type": doc["question_type"], - "answer_type": doc["answer_type"], - "question": doc["question"], - "unit": doc["unit"] if "unit" in doc else "", - "caption": doc["caption"] if "caption" in doc else "", - "ocr": doc["ocr"] if "ocr" in doc else "", - "choices": doc["choices"], - "answer": doc["answer"] if "answer" in doc else None, - "precision": doc["precision"] if "precision" in doc else 0, - } - query_prompt = mathvista_evaluator.create_one_query(problem, examples=None, shot_num=0, shot_type=model_specific_prompt_kwargs["shot_type"]) - return query_prompt - - -def mathvista_process_results(doc, results): - prediction = results[0].strip() - problem = { - "question_type": doc["question_type"], - "answer_type": doc["answer_type"], - "query": doc["query"], - "choices": doc["choices"], - "answer": doc["answer"] if "answer" in doc else None, - "precision": doc["precision"] if "precision" in doc else 0, - } - extraction = mathvista_evaluator.extract_answer(prediction, problem, config["metadata"]["quick_extract"]) - - prediction = mathvista_evaluator.normalize_extracted_answer(extraction, problem["choices"], problem["question_type"], problem["answer_type"], problem["precision"]) - # set test set answer to None - true_false = mathvista_evaluator.safe_equal(prediction, problem["answer"]) if problem["answer"] is not None else False - - result = { - "question_id": doc["pid"], - "query": doc["query"], - "choices": doc["choices"], - "answer": doc["answer"] if "answer" in doc else None, - "extraction": extraction, - "prediction": prediction, - "true_false": true_false, - "question_type": doc["question_type"], - "answer_type": doc["answer_type"], - "precision": doc["precision"] if "precision" in doc else 0, - "metadata": doc["metadata"], - } - - return { - "gpt_eval_score": result, - "submission": result, - } - - -def mathvista_aggregate_results(results, args, *, calculate_gain=False, random_scores=None): - split_flag = results[0]["metadata"]["split"] - full_pids = [result["question_id"] for result in results] - total = len(results) - correct = sum(1 for idx, pid in enumerate(full_pids) if results[idx]["true_false"]) - accuracy = round(correct / total * 100, 2) - scores = {"average": {"accuracy": accuracy, "correct": correct, "total": total}} - - for result in results: - result.update(result.pop("metadata")) - - results_dict = {result["question_id"]: result for result in results} - df = pd.DataFrame(results_dict).T - target_keys = ["question_type", "answer_type", "language", "source", "category", "task", "context", "grade", "skills"] - - for key in target_keys: - values = df[key].explode().unique() if key == "skills" else df[key].unique() - scores[key] = {} - for value in values: - correct, total, acc = mathvista_evaluator.get_acc_with_contion(df, key, value) - if total > 0: - scores[key][value] = {"accuracy": acc, "correct": correct, "total": total} - scores[key] = dict(sorted(scores[key].items(), key=lambda item: float(item[1]["accuracy"]), reverse=True)) - - if calculate_gain: - for key in scores: - if key == "average": - gain = round(float(scores[key]["accuracy"]) - float(random_scores[key]["accuracy"]), 2) - scores[key]["acc_gain"] = gain - else: - for sub_key in scores[key]: - gain = round(float(scores[key][sub_key]["accuracy"]) - float(random_scores[key][sub_key]["accuracy"]), 2) - scores[key][sub_key]["acc_gain"] = gain - - path = generate_submission_file(f"mathvista_{split_flag}_scores.json", args) - with open(path, "w") as f: - json.dump(results_dict, f, indent=4) - eval_logger.info(f"Saved results to {path}") - if scores["average"]["accuracy"] == 0: - return None - return scores["average"]["accuracy"] diff --git a/lmms_eval/tasks/mmbench/_default_template_mmbench_cn_yaml b/lmms_eval/tasks/mmbench/_default_template_mmbench_cn_yaml deleted file mode 100644 index 8109462..0000000 --- a/lmms_eval/tasks/mmbench/_default_template_mmbench_cn_yaml +++ /dev/null @@ -1,22 +0,0 @@ -dataset_path: lmms-lab/MMBench -dataset_kwargs: - token: True -doc_to_target: "answer" -dataset_name: "cn" -output_type: generate_until -doc_to_visual: !function cn_utils.mmbench_doc_to_visual -doc_to_text: !function cn_utils.mmbench_doc_to_text -generation_kwargs: - max_new_tokens: 256 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function cn_utils.mmbench_process_results -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\n请直接使用所提供的选项字母作为答案回答。" -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original diff --git a/lmms_eval/tasks/mmbench/_default_template_mmbench_en_yaml b/lmms_eval/tasks/mmbench/_default_template_mmbench_en_yaml deleted file mode 100644 index ab2b882..0000000 --- a/lmms_eval/tasks/mmbench/_default_template_mmbench_en_yaml +++ /dev/null @@ -1,25 +0,0 @@ -dataset_path: lmms-lab/MMBench -dataset_kwargs: - token: True -doc_to_target: "answer" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer with the option's letter from the given choices directly." -doc_to_visual: !function en_utils.mmbench_doc_to_visual -doc_to_text: !function en_utils.mmbench_doc_to_text -doc_to_target: "answer" -process_results: !function en_utils.mmbench_process_results -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -output_type: generate_until -dataset_name: "en" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false diff --git a/lmms_eval/tasks/mmbench/cc_utils.py b/lmms_eval/tasks/mmbench/cc_utils.py deleted file mode 100644 index abb24ab..0000000 --- a/lmms_eval/tasks/mmbench/cc_utils.py +++ /dev/null @@ -1,109 +0,0 @@ -import logging -import yaml -import os -from pathlib import Path -import pandas as pd -import json - -eval_logger = logging.getLogger("lmms-eval") -from lmms_eval.tasks.mmbench.mmbench_evals import MMBench_Evaluator -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -with open(Path(__file__).parent / "mmbench.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] -API_TYPE = os.getenv("API_TYPE", "openai") - -if API_TYPE == "openai": - API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") - API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") -elif API_TYPE == "azure": - API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") - API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") - - -mmbench_evaluator = MMBench_Evaluator(sys_prompt=config["metadata"]["sys_prompt"], API_KEY=API_KEY, API_URL=API_URL, model_version=GPT_EVAL_MODEL_NAME) - - -def mmbench_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def mmbench_cn_cc_doc_to_text(doc, model_specific_prompt_kwargs=None): - option_candidate = ["A", "B", "C", "D", "E"] - options_prompt, options_dict = mmbench_evaluator.create_options_prompt(doc, option_candidate) - - data = { - # "img": doc["image"], - "question": doc["question"], - "answer": doc.get("answer", None), - "options": options_prompt, - "category": doc["category"], - "options_dict": options_dict, - "index": doc["index"], - "source": doc["source"], - } - - query_prompt = f"{data['question']} {data['options']}" - - if model_specific_prompt_kwargs: - query_prompt = f"{query_prompt}\n{model_specific_prompt_kwargs['post_prompt']}" - - return query_prompt - - -def mmbench_cn_cc_process_results(doc, results): - model_response = results[0].strip() - data = { - "gpt_eval_score": { - "index": doc["index"], - "question": doc["question"], - "answer": doc["answer"], - "prediction": model_response, - "source": doc["source"], - "category": doc["category"], - }, - "submission": { - "index": doc["index"], - "question": doc["question"], - "answer": doc["answer"], - "prediction": model_response, - "source": doc["source"], - "category": doc["category"], - }, - } - option_candidate = ["A", "B", "C", "D", "E"] - for c in option_candidate: - data["submission"][c] = doc.get(c, "nan") - data["gpt_eval_score"][c] = doc.get(c, "nan") - return data - - -def mmbench_cn_cc_aggregate_dev_results_eval(results, args): - print(f"============= MMBench-CN(CC) Detailed Results =============") - overall_acc, category_acc, l2_category_acc = mmbench_evaluator.eval_result(results, eval_method="openai") - file = generate_submission_file("mmbench_cn_cc_results.json", args) - details_info = { - "overall_acc": overall_acc, - "category_acc": category_acc, - "l2_category_acc": l2_category_acc, - } - with open(file, "w") as f: - json.dump(details_info, f) - return overall_acc * 100 - - -def mmbench_cn_cc_aggregate_results(results, args): - df = pd.DataFrame(results) - file = generate_submission_file("mmbench_cn_cc_results.xlsx", args) - with pd.ExcelWriter(file) as writer: - df.to_excel(writer, index=False) - eval_logger.info(f"Saved results to {file}") diff --git a/lmms_eval/tasks/mmbench/cn_utils.py b/lmms_eval/tasks/mmbench/cn_utils.py deleted file mode 100644 index 39a55f7..0000000 --- a/lmms_eval/tasks/mmbench/cn_utils.py +++ /dev/null @@ -1,127 +0,0 @@ -import logging -import yaml -import os -from pathlib import Path -import pandas as pd -import json -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -eval_logger = logging.getLogger("lmms-eval") -from lmms_eval.tasks.mmbench.mmbench_evals import MMBench_Evaluator -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -with open(Path(__file__).parent / "mmbench.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] -API_TYPE = os.getenv("API_TYPE", "openai") - -if API_TYPE == "openai": - API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") - API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") -elif API_TYPE == "azure": - API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") - API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") - - -mmbench_evaluator = MMBench_Evaluator(sys_prompt=config["metadata"]["sys_prompt"], API_KEY=API_KEY, API_URL=API_URL, model_version=GPT_EVAL_MODEL_NAME) - - -def mmbench_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None): - option_candidate = ["A", "B", "C", "D", "E"] - options_prompt, options_dict = mmbench_evaluator.create_options_prompt(doc, option_candidate) - - data = { - # "img": doc["image"], - "question": doc["question"], - "answer": doc.get("answer", None), - "options": options_prompt, - "category": doc["category"], - "L2-category": doc["L2-category"], - "options_dict": options_dict, - "index": doc["index"], - "hint": doc["hint"], - "source": doc["source"], - "split": doc["split"], - } - - query_prompt = f"{data['hint']} {data['question']} {data['options']}" if pd.notna(data["hint"]) else f"{data['question']} {data['options']}" - - if model_specific_prompt_kwargs: - query_prompt = f"{query_prompt}\n{model_specific_prompt_kwargs['post_prompt']}" - - return query_prompt - - -def mmbench_process_results(doc, results): - model_response = results[0].strip() - data = { - "gpt_eval_score": { - "index": doc["index"], - "question": doc["question"], - "answer": doc["answer"], - "prediction": model_response, - "hint": doc["hint"], - "source": doc["source"], - "split": doc["split"], - "category": doc["category"], - "L2-category": doc["L2-category"], - }, - "submission": { - "index": doc["index"], - "question": doc["question"], - "answer": doc["answer"], - "prediction": model_response, - "hint": doc["hint"], - "source": doc["source"], - "split": doc["split"], - "category": doc["category"], - "L2-category": doc["L2-category"], - }, - } - option_candidate = ["A", "B", "C", "D", "E"] - for c in option_candidate: - data["submission"][c] = doc.get(c, "nan") - data["gpt_eval_score"][c] = doc.get(c, "nan") - return data - - -def mmbench_aggregate_dev_results_eval(results, args): - print(f"============= MMBench-CN(Dev) Detailed Results =============") - overall_acc, category_acc, l2_category_acc = mmbench_evaluator.eval_result(results, eval_method="openai") - file = generate_submission_file("mmbench_cn_dev_results.json", args) - details_info = { - "overall_acc": overall_acc, - "category_acc": category_acc, - "l2_category_acc": l2_category_acc, - } - with open(file, "w") as f: - json.dump(details_info, f) - return overall_acc * 100 - - -def mmbench_aggregate_dev_results(results, args): - df = pd.DataFrame(results) - excel_write_path = generate_submission_file("mmbench_cn_dev_results.xlsx", args) - with pd.ExcelWriter(excel_write_path) as writer: - df.to_excel(writer, index=False) - eval_logger.info(f"Saved results to {excel_write_path}") - - -def mmbench_aggregate_test_results(results, args): - df = pd.DataFrame(results) - excel_write_path = generate_submission_file("mmbench_cn_test_results.xlsx", args) - with pd.ExcelWriter(excel_write_path) as writer: - df.to_excel(writer, index=False) - eval_logger.info(f"Saved results to {excel_write_path}") diff --git a/lmms_eval/tasks/mmbench/en_utils.py b/lmms_eval/tasks/mmbench/en_utils.py deleted file mode 100644 index 1ddccbb..0000000 --- a/lmms_eval/tasks/mmbench/en_utils.py +++ /dev/null @@ -1,126 +0,0 @@ -import logging -import yaml -import os -from pathlib import Path -import pandas as pd -import json - -eval_logger = logging.getLogger("lmms-eval") -from lmms_eval.tasks.mmbench.mmbench_evals import MMBench_Evaluator -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -with open(Path(__file__).parent / "mmbench.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] -API_TYPE = os.getenv("API_TYPE", "openai") - -if API_TYPE == "openai": - API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") - API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") -elif API_TYPE == "azure": - API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") - API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") - - -mmbench_evaluator = MMBench_Evaluator(sys_prompt=config["metadata"]["sys_prompt"], API_KEY=API_KEY, API_URL=API_URL, model_version=GPT_EVAL_MODEL_NAME) - - -def mmbench_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None): - option_candidate = ["A", "B", "C", "D", "E"] - options_prompt, options_dict = mmbench_evaluator.create_options_prompt(doc, option_candidate) - - data = { - # "img": doc["image"], - "question": doc["question"], - "answer": doc.get("answer", None), - "options": options_prompt, - "category": doc["category"], - "L2-category": doc["L2-category"], - "options_dict": options_dict, - "index": doc["index"], - "hint": doc["hint"], - "source": doc["source"], - "split": doc["split"], - } - - query_prompt = f"{data['hint']} {data['question']} {data['options']}" if pd.notna(data["hint"]) and data["hint"] != "nan" else f"{data['question']} {data['options']}" - - if model_specific_prompt_kwargs: - query_prompt = f"{query_prompt}\n{model_specific_prompt_kwargs['post_prompt']}" - - return query_prompt - - -def mmbench_process_results(doc, results): - model_response = results[0].strip() - data = { - "gpt_eval_score": { - "index": doc["index"], - "question": doc["question"], - "answer": doc["answer"], - "prediction": model_response, - "hint": doc["hint"], - "source": doc["source"], - "split": doc["split"], - "category": doc["category"], - "L2-category": doc["L2-category"], - }, - "submission": { - "index": doc["index"], - "question": doc["question"], - "answer": doc["answer"], - "prediction": model_response, - "hint": doc["hint"], - "source": doc["source"], - "split": doc["split"], - "category": doc["category"], - "L2-category": doc["L2-category"], - }, - } - option_candidate = ["A", "B", "C", "D", "E"] - for c in option_candidate: - data["submission"][c] = doc.get(c, "nan") - data["gpt_eval_score"][c] = doc.get(c, "nan") - return data - - -def mmbench_aggregate_dev_results_eval(results, args): - print(f"============= MMBench-EN(Dev) Detailed Results =============") - overall_acc, category_acc, l2_category_acc = mmbench_evaluator.eval_result(results, eval_method="openai") - file = generate_submission_file("mmbench_en_dev_results.json", args) - details_info = { - "overall_acc": overall_acc, - "category_acc": category_acc, - "l2_category_acc": l2_category_acc, - } - with open(file, "w") as f: - json.dump(details_info, f) - return overall_acc * 100 - - -def mmbench_aggregate_dev_results_submission(results, args): - df = pd.DataFrame(results) - excel_write_path = generate_submission_file("mmbench_en_dev_results.xlsx", args) - with pd.ExcelWriter(excel_write_path) as writer: - df.to_excel(writer, index=False) - eval_logger.info(f"Saved results to {excel_write_path}") - - -def mmbench_aggregate_test_results(results, args): - df = pd.DataFrame(results) - excel_write_path = generate_submission_file("mmbench_en_test_results.xlsx", args) - with pd.ExcelWriter(excel_write_path) as writer: - df.to_excel(writer, index=False) - eval_logger.info(f"Saved results to {excel_write_path}") diff --git a/lmms_eval/tasks/mmbench/mmbench.yaml b/lmms_eval/tasks/mmbench/mmbench.yaml deleted file mode 100644 index 821065e..0000000 --- a/lmms_eval/tasks/mmbench/mmbench.yaml +++ /dev/null @@ -1,11 +0,0 @@ -group: mmbench -task: - - mmbench_en_dev - - mmbench_en_test - - mmbench_cn_dev - - mmbench_cn_test - - mmbench_cn_cc -metadata: - version: 0.0 - sys_prompt: "There are several options:" - gpt_eval_model_name: "gpt-3.5-turbo-0613" \ No newline at end of file diff --git a/lmms_eval/tasks/mmbench/mmbench_cc.yaml b/lmms_eval/tasks/mmbench/mmbench_cc.yaml deleted file mode 100644 index 4a0d589..0000000 --- a/lmms_eval/tasks/mmbench/mmbench_cc.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: lmms-lab/MMBench -dataset_name: cc -dataset_kwargs: - token: True -task: "mmbench_cn_cc" -test_split: test -output_type: generate_until -doc_to_visual: !function cc_utils.mmbench_doc_to_visual -doc_to_text: !function cc_utils.mmbench_cn_cc_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 256 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function cc_utils.mmbench_cn_cc_process_results -metric_list: - - metric: gpt_eval_score - aggregation: !function cc_utils.mmbench_cn_cc_aggregate_dev_results_eval - higher_is_better: true - - metric: submission - aggregation: !function cc_utils.mmbench_cn_cc_aggregate_results -metadata: - version: 0.0 - gpt_eval_model_name: "gpt-3.5-turbo-0613" - -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\n请直接使用所提供的选项字母作为答案回答。" -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original \ No newline at end of file diff --git a/lmms_eval/tasks/mmbench/mmbench_cn.yaml b/lmms_eval/tasks/mmbench/mmbench_cn.yaml deleted file mode 100644 index 9da764c..0000000 --- a/lmms_eval/tasks/mmbench/mmbench_cn.yaml +++ /dev/null @@ -1,9 +0,0 @@ -group: mmbench_cn -task: - - mmbench_cn_dev - - mmbench_cn_test - - mmbench_cn_cc -metadata: - version: 0.0 - gpt_eval_model_name: "gpt-3.5-turbo-0613" - sys_prompt: "有如下几个选项：" \ No newline at end of file diff --git a/lmms_eval/tasks/mmbench/mmbench_cn_dev.yaml b/lmms_eval/tasks/mmbench/mmbench_cn_dev.yaml deleted file mode 100644 index b3aaa54..0000000 --- a/lmms_eval/tasks/mmbench/mmbench_cn_dev.yaml +++ /dev/null @@ -1,10 +0,0 @@ -task: "mmbench_cn_dev" -test_split: "dev" -metric_list: - - metric: gpt_eval_score - aggregation: !function cn_utils.mmbench_aggregate_dev_results_eval - higher_is_better: true - - metric: submission - higher_is_better: true - aggregation: !function cn_utils.mmbench_aggregate_dev_results -include: _default_template_mmbench_cn_yaml diff --git a/lmms_eval/tasks/mmbench/mmbench_cn_test.yaml b/lmms_eval/tasks/mmbench/mmbench_cn_test.yaml deleted file mode 100644 index b86f092..0000000 --- a/lmms_eval/tasks/mmbench/mmbench_cn_test.yaml +++ /dev/null @@ -1,7 +0,0 @@ -task: mmbench_cn_test -test_split: test -metric_list: - - metric: submission - aggregation: !function cn_utils.mmbench_aggregate_test_results - higher_is_better: true -include: _default_template_mmbench_cn_yaml diff --git a/lmms_eval/tasks/mmbench/mmbench_en.yaml b/lmms_eval/tasks/mmbench/mmbench_en.yaml deleted file mode 100644 index 0a5c558..0000000 --- a/lmms_eval/tasks/mmbench/mmbench_en.yaml +++ /dev/null @@ -1,8 +0,0 @@ -group: mmbench_en -task: - - mmbench_en_dev - - mmbench_en_test -metadata: - version: 0.0 - sys_prompt: "There are several options:" - gpt_eval_model_name: "gpt-3.5-turbo-0613" diff --git a/lmms_eval/tasks/mmbench/mmbench_en_dev.yaml b/lmms_eval/tasks/mmbench/mmbench_en_dev.yaml deleted file mode 100644 index f9e6bab..0000000 --- a/lmms_eval/tasks/mmbench/mmbench_en_dev.yaml +++ /dev/null @@ -1,10 +0,0 @@ -task: "mmbench_en_dev" -test_split: dev -include: _default_template_mmbench_en_yaml -metric_list: - - metric: gpt_eval_score - aggregation: !function en_utils.mmbench_aggregate_dev_results_eval - higher_is_better: true - - metric: submission - aggregation: !function en_utils.mmbench_aggregate_dev_results_submission - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmbench/mmbench_en_test.yaml b/lmms_eval/tasks/mmbench/mmbench_en_test.yaml deleted file mode 100644 index 5acf404..0000000 --- a/lmms_eval/tasks/mmbench/mmbench_en_test.yaml +++ /dev/null @@ -1,7 +0,0 @@ -task: "mmbench_en_test" -test_split: test -include: _default_template_mmbench_en_yaml -metric_list: - - metric: submission - aggregation: !function en_utils.mmbench_aggregate_test_results - higher_is_better: true diff --git a/lmms_eval/tasks/mmbench/mmbench_evals.py b/lmms_eval/tasks/mmbench/mmbench_evals.py deleted file mode 100644 index 7868157..0000000 --- a/lmms_eval/tasks/mmbench/mmbench_evals.py +++ /dev/null @@ -1,323 +0,0 @@ -import os.path as osp -import time -import random as rd -import string -from collections import defaultdict -import requests - -import math -import numpy as np -import pandas as pd -from tqdm import tqdm - -import logging - -eval_logger = logging.getLogger("lmms-eval") - - -class MMBench_Evaluator: - def __init__(self, sys_prompt="There are several options:", API_KEY="", API_URL="", model_version="gpt-3.5-turbo-0613"): - self.sys_prompt = sys_prompt - self.model_version = model_version - self.API_KEY = API_KEY - self.API_URL = API_URL - - def create_options_prompt(self, row_data, option_candidate): - available_keys = set(row_data.keys()) & set(option_candidate) - options = {cand: row_data[cand] for cand in available_keys if row_data[cand]} - sorted_options = dict(sorted(options.items())) - options_prompt = f"{self.sys_prompt}\n" - for key, item in sorted_options.items(): - if pd.notna(item) and item != "nan": - options_prompt += f"{key}. {item}\n" - return options_prompt.rstrip("\n"), sorted_options - - # Prompt Building - def build_option_str(self, option_list): - chars = string.ascii_uppercase - s = "There are several options: \n" - for c, opt in zip(chars, option_list): - if not pd.isna(opt): - s += f"{c}. {opt}\n" - else: - return s - return s - - def extract_options(self, item): - options = [] - for c in "ABCD": - if c in item and not pd.isna(item[c]): - options.append(item[c]) - else: - return options - return options - - def build_choices(self, item): - ret = {} - for ch in "ABCD": - if not pd.isna(item[ch]): - ret[ch] = item[ch] - return ret - - def build_prompt(self, question, options, prediction): - tmpl = ( - "You are an AI assistant who will help me to match an answer " - "with several options of a single-choice question. " - "You are provided with a question, several options, and an answer, " - "and you need to find which option is most similar to the answer. " - "If the meaning of all options are significantly different " - "from the answer, output E. " - "Your should output a single uppercase character in A, B, C, D " - "(if they are valid options), and E. \n" - "Example 1: \n" - "Question: What is the main object in image?\nOptions: A. teddy bear " - "B. rabbit C. cat D. dog\nAnswer: a cute teddy bear\nYour output: A\n" - "Example 2: \n" - "Question: What is the main object in image?\nOptions: A. teddy bear " - "B. rabbit C. cat D. dog\nAnswer: Spider\nYour output: E\n" - "Example 3: \n" - "Question: {}?\nOptions: {}\nAnswer: {}\nYour output: " - ) - return tmpl.format(question, options, prediction) - - # Prefetch Answers - def can_infer_option(self, answer, num_choice=5): - choices = string.ascii_uppercase[:num_choice] - if "Failed to obtain answer via API" in answer: - return False - - def count(splits, choices="ABCD", prefix="", suffix=""): - cnt = 0 - for c in choices: - if prefix + c + suffix in splits: - cnt += 1 - return cnt - - splits = [x.strip() for x in answer.split()] - if count(splits, choices) == 1: - for ch in choices: - if "A" in splits and len(splits) > 3: - eval_logger.info(f"A might be a quantifier in the string: {answer}.") - break - if ch in splits: - return ch - tups = [("", "."), ("", ","), ("", ":"), ("", ")"), ("", ")."), ("(", ")"), ("(", ")."), (":", ""), (":", ","), (":", "."), (":", ")"), (":", ").")] - for tup in tups: - if count(splits, choices, prefix=tup[0], suffix=tup[1]) == 1: - for ch in choices: - if tup[0] + ch + tup[1] in splits: - return ch - return False - - def can_infer_text(self, answer, choices): - answer = answer.lower() - assert isinstance(choices, dict) - for k in choices: - assert k in "ABCD" - choices[k] = str(choices[k]).lower() - cands = [] - for k in choices: - if choices[k] in answer: - cands.append(k) - if len(cands) == 1: - return cands[0] - return False - - def can_infer(self, answer, choices): - copt = self.can_infer_option(answer) - return copt if copt else self.can_infer_text(answer, choices) - - def prefetch_answer(self, item): - choices = self.build_choices(item) - return self.can_infer(item["prediction"], choices) - - def _post_request(self, payload): - headers = { - "Authorization": f"Bearer {self.API_KEY}", - "Content-Type": "application/json", - } - response = requests.post(self.API_URL, headers=headers, json=payload, timeout=30) - response.raise_for_status() - return response.json() - - def get_chat_response(self, prompt, temperature=0, max_tokens=256, n=1, patience=5, sleep_time=3): - messages = [ - {"role": "user", "content": prompt}, - ] - payload = {"model": self.model_version, "messages": messages, "temperature": temperature, "max_tokens": max_tokens, "n": n} - - while patience > 0: - patience -= 1 - try: - response = self._post_request(payload) - if n == 1: - prediction = response["choices"][0]["message"]["content"].strip() - if prediction and prediction != "": - return prediction - else: - prediction = [choice["message"]["content"].strip() for choice in response["choices"]] - if prediction and prediction[0] != "": - return prediction - - except Exception as e: - eval_logger.info(f"Attempt {patience + 1} failed with error: {e}") - if sleep_time > 0: - time.sleep(sleep_time) - - return "Failed to obtain answer via API" - - def extract_answer_from_item(self, item): - options = self.extract_options(item) - option_str = self.build_option_str(options) - - prompt = self.build_prompt(item["question"], option_str, item["prediction"]) - retry = 3 - choices = self.build_choices(item) - - ret = self.can_infer(item["prediction"], choices) - if ret: - return ret, item["prediction"] - - while retry: - ans = self.get_chat_response(prompt) - if "Failed to obtain answer via API" in ans: - msg = "GPT API failed to answer. " - eval_logger.info(msg) - retry -= 1 - else: - ret = self.can_infer(ans, choices) - if ret: - return ret, ans - else: - eval_logger.info(f'GPT output includes 0 / >1 letter in "ABCD": {ans}') - retry -= 1 - - if retry == 0: - num_options = sum([ch in item for ch in "ABCD"]) - if num_options >= 2: - chars = string.ascii_uppercase[:num_options] - chars = chars + "E" - num_options += 1 - tmp = rd.randint(0, num_options - 1) - return chars[tmp], "Failed to predict, thus randomly generate one. " - - # Extract answer from multiple rolling records - def eval_sub_data(self, sub_data, answer_map): - lt = len(sub_data) - GT, PRED = [], [] - for i in range(lt): - item = sub_data.iloc[i] - idx = item["index"] - GT.append(answer_map[idx]) - PRED.append(self.prefetch_answer(item)) - if PRED[-1] and (GT[-1] != PRED[-1]): - return 0 - - for i in range(lt): - if PRED[i]: - continue - else: - ret, _ = self.extract_answer_from_item(sub_data.iloc[i]) - PRED[i] = ret - if PRED[i] != GT[i]: - return 0 - return 1 - - def calculate_hit_rates(self, data): - overall_hit_rate = data["hit"].mean() - - category_hit_rate = {} - if "category" in data.columns: - # Category-based hit rate - category_hit_rate = data.groupby("category")["hit"].mean().to_dict() - - # l2-category based hit rate - l2_category_hit_rate = {} - if "l2-category" in data.columns: - l2_category_hit_rate = data.groupby("l2-category")["hit"].mean().to_dict() - - return overall_hit_rate, category_hit_rate, l2_category_hit_rate - - # Evaluate Results - def eval_result(self, results, eval_method): - rd.seed(2680) - assert eval_method == "openai" - # Set a large retry number to avoid failure - # model = OpenAI('gpt-3.5-turbo-0613', retry=99) - - # double_log(f'Evaluating {eval_file}', fout) - - # result_file = eval_file.replace('.xlsx', f'_{eval_method}_result.pkl') - result = {} - # if osp.exists(result_file): - # result = load(result_file) - - # data = load(eval_file) - data = pd.DataFrame(results) - data = data.sort_values(by="index") - data["prediction"] = [str(x) for x in data["prediction"]] - for k in data.keys(): - data[k.lower() if k not in "ABCD" else k] = data.pop(k) - - # meta = load(meta_file) - - data_main = data[data["index"] < int(1e6)] - - data_main["hit"] = 0 - cate_map = {i: c for i, c in zip(data["index"], data["category"])} - answer_map = {i: c for i, c in zip(data["index"], data["answer"])} - if "l2-category" in data.columns: - l2_cate_map = {i: c for i, c in zip(data["index"], data["l2-category"])} - - lt = len(data_main) - hit, tot = 0, 0 - - for i in range(lt): - # Dealing with the normal part - item_main = data_main.iloc[i] - idx = item_main["index"] - - if idx in result: - correct = result[idx] - assert correct in [0, 1] - hit += correct - tot += 1 - continue - - sub_data = data[data["index"] % int(1e6) == idx] - ret = self.eval_sub_data(sub_data, answer_map) - result[idx] = ret - hit += ret - tot += 1 - - data_main.loc[data_main["index"] == idx, "hit"] = ret - # if (i + 1) % 100 == 0: - # eval_logger.info(f"Evaluating: {i + 1}/{lt}, Acc: {hit / tot * 100: .2f}%. ") - - indices = data_main["index"] - data_main = data_main.set_index("index") - data_main["category"] = [cate_map[i] if not math.isnan(i) else "uncategorized" for i in indices] - if "l2-category" in data_main.columns: - data_main["l2-category"] = [l2_cate_map[i] if not math.isnan(i) else "uncategorized" for i in indices] - - overall_hit_rate, category_hit_rate, l2_category_hit_rate = self.calculate_hit_rates(data_main) - - if "category" in data_main.columns: - print(f"Category Acc. (dev):") - for category_key in category_hit_rate: - if category_key == "split": - continue - - category_percentage = category_hit_rate[category_key] * 100 - print(f"\t{category_key}: {category_percentage:.3f}") - - if "l2-category" in data_main.columns: - print(f"L2-category Acc. (dev):") - for l2_category_key in l2_category_hit_rate: - if l2_category_key == "split": - continue - - l2_category_percentage = l2_category_hit_rate[l2_category_key] * 100 - print(f"\t{l2_category_key}: {l2_category_percentage:.3f}") - - return overall_hit_rate, category_hit_rate, l2_category_hit_rate diff --git a/lmms_eval/tasks/mme/mme.yaml b/lmms_eval/tasks/mme/mme.yaml deleted file mode 100644 index 504e6dd..0000000 --- a/lmms_eval/tasks/mme/mme.yaml +++ /dev/null @@ -1,37 +0,0 @@ -dataset_path: lmms-lab/MME -dataset_kwargs: - token: True -task: "mme" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.mme_doc_to_visual -doc_to_text: !function utils.mme_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 16 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -# The return value of process_results will be used by metrics -process_results: !function utils.mme_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: mme_percetion_score - aggregation: !function utils.mme_aggregate_results - higher_is_better: true - - metric: mme_cognition_score - aggregation: !function utils.mme_aggregate_results - higher_is_better: true -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer the question using a single word or phrase." - qwen_vl: - pre_prompt: "" - post_prompt: " Answer:" - otterhd: - pre_prompt: "" - post_prompt: " Answer:" -metadata: - - version: 0.0 diff --git a/lmms_eval/tasks/mme/utils.py b/lmms_eval/tasks/mme/utils.py deleted file mode 100644 index b001b2c..0000000 --- a/lmms_eval/tasks/mme/utils.py +++ /dev/null @@ -1,120 +0,0 @@ -from collections import defaultdict -import os -import datetime -import json -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -import logging - -eval_logger = logging.getLogger("lmms-eval") - -dir_name = os.path.dirname(os.path.abspath(__file__)) - -eval_type_dict = { - "Perception": [ - "existence", - "count", - "position", - "color", - "posters", - "celebrity", - "scene", - "landmark", - "artwork", - "OCR", - ], - "Cognition": [ - "commonsense_reasoning", - "numerical_calculation", - "text_translation", - "code_reasoning", - ], -} - - -replace_prompt = " Please answer yes or no." - - -def mme_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def mme_doc_to_text(doc, model_specific_prompt_kwargs=None): - question = doc["question"].strip() - if "pre_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["pre_prompt"] != "": - question = question.replace(replace_prompt, "") - question = f"{model_specific_prompt_kwargs['pre_prompt']}{question}" - if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"] != "": - question = question.replace(replace_prompt, "") - question = f"{question}{model_specific_prompt_kwargs['post_prompt']}" - return question - - -def parse_pred_ans(pred_ans): - """Brought from Otter Eval""" - pred_ans = pred_ans.lower().strip().replace(".", "") - pred_label = None - if pred_ans in ["yes", "no"]: - pred_label = pred_ans - else: - prefix_pred_ans = pred_ans[:4] - if "yes" in prefix_pred_ans: - pred_label = "yes" - elif "no" in prefix_pred_ans: - pred_label = "no" - else: - pred_label = "other" - return pred_label - - -def mme_process_results(doc, results): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case mme score), value: metric value - """ - pred = results[0] - pred_ans = parse_pred_ans(pred) - gt_ans = doc["answer"].lower().strip().replace(".", "") - assert gt_ans in ["yes", "no"] - assert pred_ans in ["yes", "no", "other"] - score = 1.0 if pred_ans == gt_ans else 0.0 - category = doc["category"] - key_name = "mme_percetion_score" if category in eval_type_dict["Perception"] else "mme_cognition_score" - # Note: the key name here is very important. It decides which aggregation function will receive the results - # We note down the question id/category to help us aggregate the results later - return {key_name: {"question_id": doc["question_id"], "category": category, "score": score}} - - -def mme_aggregate_results(results): - """ - Args: - results: a list of values returned by process_results - Returns: - A score - """ - category2score = defaultdict(dict) - for result in results: - question_id = result["question_id"] - score = result["score"] - category = result["category"] - if question_id not in category2score[category]: - category2score[category][question_id] = [] - category2score[category][question_id].append(score) - category2avg_score = {} - for category, question2scores in category2score.items(): - total_score = 0 - for question_id, scores in question2scores.items(): - assert len(scores) == 2 - acc = sum(scores) / len(scores) * 100.0 - acc_plus = (sum(scores) == 2) * 100.0 - score = acc_plus + acc - total_score += score - avg_score = total_score / len(question2scores) - category2avg_score[category] = avg_score - for category, avg_score in category2avg_score.items(): - eval_logger.info(f"{category}: {avg_score:.2f}") - total_score = sum(category2avg_score.values()) - return total_score diff --git a/lmms_eval/tasks/mmmu/mmmu.yaml b/lmms_eval/tasks/mmmu/mmmu.yaml deleted file mode 100644 index 935d03b..0000000 --- a/lmms_eval/tasks/mmmu/mmmu.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: mmmu -task: -- mmmu_val -- mmmu_test diff --git a/lmms_eval/tasks/mmmu/mmmu_group_img.yaml b/lmms_eval/tasks/mmmu/mmmu_group_img.yaml deleted file mode 100644 index 8d8713a..0000000 --- a/lmms_eval/tasks/mmmu/mmmu_group_img.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: mmmu_group_img -task: -- mmmu_val_group_img -- mmmu_test_group_img diff --git a/lmms_eval/tasks/mmmu/mmmu_group_img_test.yaml b/lmms_eval/tasks/mmmu/mmmu_group_img_test.yaml deleted file mode 100644 index 509336c..0000000 --- a/lmms_eval/tasks/mmmu/mmmu_group_img_test.yaml +++ /dev/null @@ -1,19 +0,0 @@ -dataset_path: lmms-lab/MMMU -task: "mmmu_test_group_img" -test_split: test -output_type: generate_until -doc_to_visual: !function utils_group_img.mmmu_doc_to_visual -doc_to_text: !function utils_group_img.mmmu_doc_to_text -doc_to_target: "answer" -# The return value of process_results will be used by metrics -process_results: !function utils_group_img.mmmu_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -generation_kwargs: - max_new_tokens: 16 - image_aspect_ratio: original -metric_list: - - metric: submission - aggregation: !function utils_group_img.mmmu_test_aggregate_results_for_submission - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu/mmmu_group_img_val.yaml b/lmms_eval/tasks/mmmu/mmmu_group_img_val.yaml deleted file mode 100644 index c9f9239..0000000 --- a/lmms_eval/tasks/mmmu/mmmu_group_img_val.yaml +++ /dev/null @@ -1,21 +0,0 @@ -dataset_path: lmms-lab/MMMU -task: "mmmu_val_group_img" -test_split: validation -output_type: generate_until -doc_to_visual: !function utils_group_img.mmmu_doc_to_visual -doc_to_text: !function utils_group_img.mmmu_doc_to_text -doc_to_target: "answer" -# The return value of process_results will be used by metrics -process_results: !function utils_group_img.mmmu_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -generation_kwargs: - max_new_tokens: 16 -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -metric_list: - - metric: mmmu_acc - aggregation: !function utils_group_img.mmmu_aggregate_results - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu/mmmu_test.yaml b/lmms_eval/tasks/mmmu/mmmu_test.yaml deleted file mode 100644 index 8f1a1f2..0000000 --- a/lmms_eval/tasks/mmmu/mmmu_test.yaml +++ /dev/null @@ -1,19 +0,0 @@ -dataset_path: lmms-lab/MMMU -task: "mmmu_test" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.mmmu_doc_to_visual -doc_to_text: !function utils.mmmu_doc_to_text -doc_to_target: "answer" -# The return value of process_results will be used by metrics -process_results: !function utils.mmmu_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -generation_kwargs: - max_new_tokens: 16 - image_aspect_ratio: original -metric_list: - - metric: submission - aggregation: !function utils.mmmu_test_aggregate_results_for_submission - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu/mmmu_val.yaml b/lmms_eval/tasks/mmmu/mmmu_val.yaml deleted file mode 100644 index 49ab0dd..0000000 --- a/lmms_eval/tasks/mmmu/mmmu_val.yaml +++ /dev/null @@ -1,21 +0,0 @@ -dataset_path: lmms-lab/MMMU -task: "mmmu_val" -test_split: validation -output_type: generate_until -doc_to_visual: !function utils.mmmu_doc_to_visual -doc_to_text: !function utils.mmmu_doc_to_text -doc_to_target: "answer" -# The return value of process_results will be used by metrics -process_results: !function utils.mmmu_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -generation_kwargs: - max_new_tokens: 16 -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -metric_list: - - metric: mmmu_acc - aggregation: !function utils.mmmu_aggregate_results - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/mmmu/utils.py b/lmms_eval/tasks/mmmu/utils.py deleted file mode 100644 index bc68c00..0000000 --- a/lmms_eval/tasks/mmmu/utils.py +++ /dev/null @@ -1,469 +0,0 @@ -from collections import defaultdict -import re -import ast -import random -import numpy as np -import os -import json -import logging - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -lmms_logger = logging.getLogger("lmms-eval") - -MULTI_CHOICE_PROMPT = "Answer with the option letter from the given choices directly." -OPEN_ENDED_PROMPT = "Answer the question using a single word or phrase." - - -def replace_images_tokens(input_string): - for i in range(1, 8): - question_text = f"" - query_text = "" - if question_text in input_string: - input_string = input_string.replace(question_text, query_text) - return input_string - - -def parse_options(options): - option_letters = [chr(ord("A") + i) for i in range(len(options))] - choices_str = "\n".join([f"{option_letter}. {option}" for option_letter, option in zip(option_letters, options)]) - return choices_str - - -def construct_prompt(doc): - question = doc["question"] - if doc["question_type"] == "multiple-choice": - # Weirdly, data["options"] is a string in MMMU Huggingface dataset - parsed_options = parse_options(ast.literal_eval(doc["options"])) - # parsed_options already prepends a newline so no need to add space here - question = f"{question}\n{parsed_options}\n{MULTI_CHOICE_PROMPT}" - else: - question = f"{question}\n{OPEN_ENDED_PROMPT}" - return question - - -def mmmu_doc_to_text(doc): - question = construct_prompt(doc) - return replace_images_tokens(question) - - -def mmmu_doc_to_visual(doc): - prompt = construct_prompt(doc) - image_tokens = re.findall(r"", prompt) - # Remove <> and swap space as _ - image_tokens = [image_token.strip("<>").replace(" ", "_") for image_token in image_tokens] - visual = [doc[image_token].convert("RGB") for image_token in image_tokens] - return visual - - -def mmmu_process_results(doc, results): - pred = results[0] - if doc["question_type"] == "multiple-choice": - index2ans, all_choices = get_multi_choice_info(ast.literal_eval(doc["options"])) - parsed_pred = parse_multi_choice_response(pred, all_choices, index2ans) - else: - parsed_pred = parse_open_response(pred) - id = doc["id"] - mmmu_acc = {"id": id, "subdomain": extract_subset_name(doc["id"]), "question_type": doc["question_type"], "answer": doc["answer"], "parsed_pred": parsed_pred} - return { - "mmmu_acc": mmmu_acc, - "submission": { - id: pred, - }, - } - - -def extract_subset_name(input_string): - # Define a regex pattern to match "validation_" at the beginning and "_" at the end - split = input_string.split("_")[0] - pattern = re.compile(rf"^{split}_(.+?)_\d+$") - match = pattern.search(input_string) - if match: - return match.group(1) - else: - raise ValueError(f'No match found in "{input_string}"') - - -def mmmu_test_aggregate_results_for_submission(results, args): - path = generate_submission_file("mmmu_test_for_submission.json", args) - with open(path, "w") as f: - json.dump(results, f) - lmms_logger.info(f"Results saved to {path}.") - - -def mmmu_aggregate_results(results): - evaluation_result = {} - subset_to_eval_samples = defaultdict(list) - for result in results: - subset_to_eval_samples[result["subdomain"]].append(result) - for subset, sub_eval_samples in subset_to_eval_samples.items(): - judge_dict, metric_dict = evaluate_mmmu(sub_eval_samples) - metric_dict.update({"num_example": len(sub_eval_samples)}) - evaluation_result[subset] = metric_dict - printable_results = {} - for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items(): - in_domain_cat_results = {} - for cat_name in in_domain_cats: - if cat_name in evaluation_result.keys(): - in_domain_cat_results[cat_name] = evaluation_result[cat_name] - else: - pass - in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results) - in_domain_data_num = sum([cat_results["num_example"] for cat_results in in_domain_cat_results.values()]) - printable_results["Overall-" + domain] = { - "num": int(in_domain_data_num), - "acc": round(in_domain_ins_acc, 3), - } - # add sub category - for cat_name, cat_results in in_domain_cat_results.items(): - printable_results[cat_name] = { - "num": int(cat_results["num_example"]), - "acc": round(cat_results["acc"], 3), - } - all_ins_acc = calculate_ins_level_acc(evaluation_result) - printable_results["Overall"] = { - "num": sum([cat_results["num_example"] for cat_results in evaluation_result.values()]), - "acc": round(all_ins_acc, 3), - } - print(printable_results) - return printable_results["Overall"]["acc"] - - -################## -# Helper functions written by official MMMU repo. -################## - - -def calculate_ins_level_acc(results): - """Calculate the instruction level accuracy for given Subject results - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L246 - """ - acc = 0 - ins_num = 0 - for cat_results in results.values(): - acc += cat_results["acc"] * cat_results["num_example"] - ins_num += cat_results["num_example"] - if ins_num == 0: - return 0 - return acc / ins_num - - -DOMAIN_CAT2SUB_CAT = { - "Art and Design": ["Art", "Art_Theory", "Design", "Music"], - "Business": ["Accounting", "Economics", "Finance", "Manage", "Marketing"], - "Science": [ - "Biology", - "Chemistry", - "Geography", - "Math", - "Physics", - ], - "Health and Medicine": [ - "Basic_Medical_Science", - "Clinical_Medicine", - "Diagnostics_and_Laboratory_Medicine", - "Pharmacy", - "Public_Health", - ], - "Humanities and Social Science": [ - "History", - "Literature", - "Sociology", - "Psychology", - ], - "Tech and Engineering": [ - "Agriculture", - "Architecture_and_Engineering", - "Computer_Science", - "Electronics", - "Energy_and_Power", - "Materials", - "Mechanical_Engineering", - ], -} - - -def eval_multi_choice(gold_i, pred_i): - """ - Evaluate a multiple choice instance. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L175 - """ - correct = False - # only they are exactly the same, we consider it as correct - if isinstance(gold_i, list): - for answer in gold_i: - if answer == pred_i: - correct = True - break - else: # gold_i is a string - if gold_i == pred_i: - correct = True - return correct - - -def eval_open(gold_i, pred_i): - """ - Evaluate an open question instance - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L191 - """ - correct = False - if isinstance(gold_i, list): - # use float to avoid trivial matches - norm_answers = [] - for answer in gold_i: - norm_answers.extend(normalize_str(answer)) - else: - norm_answers = normalize_str(gold_i) - for pred in pred_i: # pred is already normalized in parse response phase - if isinstance(pred, str): # if it's a string, then find if ans in the pred_i - for norm_ans in norm_answers: - # only see if the string answer in the string pred - if isinstance(norm_ans, str) and norm_ans in pred: - if not correct: - correct = True - break - else: # it's a float number - if pred in norm_answers: - if not correct: - correct = True - break - return correct - - -def evaluate_mmmu(samples): - """ - Batch evaluation for multiple choice and open questions. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L219 - """ - pred_correct = 0 - judge_dict = dict() - for sample in samples: - gold_i = sample["answer"] - pred_i = sample["parsed_pred"] - if sample["question_type"] == "multiple-choice": - correct = eval_multi_choice(gold_i, pred_i) - else: # open question - correct = eval_open(gold_i, pred_i) - - if correct: - judge_dict[sample["id"]] = "Correct" - pred_correct += 1 - else: - judge_dict[sample["id"]] = "Wrong" - - if len(samples) == 0: - return {"acc": 0} - return judge_dict, {"acc": pred_correct / len(samples)} - - -def parse_multi_choice_response(response, all_choices, index2ans): - """ - Parse the prediction from the generated response. - Return the predicted index e.g., A, B, C, D. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L10 - """ - for char in [",", ".", "!", "?", ";", ":", "'"]: - response = response.strip(char) - response = " " + response + " " # add space to avoid partial match - - index_ans = True - ans_with_brack = False - candidates = [] - for choice in all_choices: # e.g., (A) (B) (C) (D) - if f"({choice})" in response: - candidates.append(choice) - ans_with_brack = True - - if len(candidates) == 0: - for choice in all_choices: # e.g., A B C D - if f"{choice} " in response: - candidates.append(choice) - - if len(candidates) == 0: - for choice in all_choices: # e.g., A. B. C. D. - if f"{choice}." in response: - candidates.append(choice) - - # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example - if len(candidates) == 0 and len(response.split()) > 5: - for index, ans in index2ans.items(): - if ans.lower() in response.lower(): - candidates.append(index) - index_ans = False # it's content ans. - - if len(candidates) == 0: # still not get answer, randomly choose one. - pred_index = random.choice(all_choices) - elif len(candidates) > 1: - start_indexes = [] - if index_ans: - if ans_with_brack: - for can in candidates: - index = response.rfind(f"({can})") - start_indexes.append(index) # -1 will be ignored anyway - # start_indexes = [generated_response.index(f'({can})') for can in candidates] - else: - for can in candidates: - index = response.rfind(f" {can} ") - start_indexes.append(index) - else: - for can in candidates: - index = response.lower().rfind(index2ans[can].lower()) - start_indexes.append(index) - # get the last one - pred_index = candidates[np.argmax(start_indexes)] - else: # if only one candidate, use it. - pred_index = candidates[0] - - return pred_index - - -def extract_numbers(string): - """ - Exact all forms of numbers from a string with regex. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L100 - """ - # Pattern for numbers with commas - pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" - # Pattern for scientific notation - pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" - # Pattern for simple numbers without commas - pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" - - # Extract numbers with commas - numbers_with_commas = re.findall(pattern_commas, string) - # Extract numbers in scientific notation - numbers_scientific = re.findall(pattern_scientific, string) - # Extract simple numbers without commas - numbers_simple = re.findall(pattern_simple, string) - - # Combine all extracted numbersz - all_numbers = numbers_with_commas + numbers_scientific + numbers_simple - return all_numbers - - -def check_is_number(string): - """ - Check if the given string a number. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L65 - """ - try: - float(string.replace(",", "")) - return True - except ValueError: - # check if there's comma inside - return False - - -def normalize_str(string): - """ - Normalize the str to lower case and make them float numbers if possible. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L76 - """ - # check if characters in the string - - # if number, numerize it. - string = string.strip() - - is_number = check_is_number(string) - - if is_number: - string = string.replace(",", "") - string = float(string) - # leave 2 decimal - string = round(string, 2) - return [string] - else: # it's likely to be a string - # lower it - string = string.lower() - if len(string) == 1: - return [" " + string, string + " "] # avoid trivial matches - return [string] - - -def parse_open_response(response): - """ - Parse the prediction from the generated response. - Return a list of predicted strings or numbers. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L122 - """ - - # content = content.strip("\n").strip(".").strip(" ") - def get_key_subresponses(response): - key_responses = [] - response = response.strip().strip(".").lower() - sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) - indicators_of_keys = [ - "could be ", - "so ", - "is ", - "thus ", - "therefore ", - "final ", - "answer ", - "result ", - ] - key_responses = [] - for index, resp in enumerate(sub_responses): - # if last one, accept it's an equation (the entire response can be just one sentence with equation) - if index == len(sub_responses) - 1: - indicators_of_keys.extend(["="]) - shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) - for indicator in indicators_of_keys: - if indicator in resp: - if not shortest_key_response: - shortest_key_response = resp.split(indicator)[-1].strip() - else: - if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response): - shortest_key_response = resp.split(indicator)[-1].strip() - # key_responses.append(resp.split(indicator)[1].strip()) - - if shortest_key_response: - # and it's not trivial - if shortest_key_response.strip() not in [ - ":", - ",", - ".", - "!", - "?", - ";", - ":", - "'", - ]: - key_responses.append(shortest_key_response) - if len(key_responses) == 0: # did not found any - return [response] - return key_responses - - # pdb.set_trace() - key_responses = get_key_subresponses(response) - - pred_list = key_responses.copy() # keep the original string response - for resp in key_responses: - pred_list.extend(extract_numbers(resp)) - - tmp_pred_list = [] - for i in range(len(pred_list)): - tmp_pred_list.extend(normalize_str(pred_list[i])) - pred_list = tmp_pred_list - - # remove duplicates - pred_list = list(set(pred_list)) - - return pred_list - - -def get_multi_choice_info(options): - """ - Given the list of options for multiple choice question - Return the index2ans and all_choices - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/data_utils.py#L54 - """ - - start_chr = "A" - all_choices = [] - index2ans = {} - for i, option in enumerate(options): - index2ans[chr(ord(start_chr) + i)] = option - all_choices.append(chr(ord(start_chr) + i)) - - return index2ans, all_choices diff --git a/lmms_eval/tasks/mmmu/utils_group_img.py b/lmms_eval/tasks/mmmu/utils_group_img.py deleted file mode 100644 index e3e475b..0000000 --- a/lmms_eval/tasks/mmmu/utils_group_img.py +++ /dev/null @@ -1,589 +0,0 @@ -from collections import defaultdict -import re -import ast -import random -import numpy as np -import os -import json -import logging -import matplotlib.font_manager as fm - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -from PIL import Image, ImageDraw, ImageFont -import numpy as np - - -def add_order_label(image, label, font_family="DejaVu Sans", font_size=40): - # Create a drawing context - draw = ImageDraw.Draw(image) - - # Define font for the label - font_path = fm.findfont(fm.FontProperties(family=font_family)) - font = ImageFont.truetype(font_path, font_size) - - # Calculate text size and position - text_width = text_height = font_size - label_background_margin = 10 - label_background_size = (text_width + 2 * label_background_margin, text_height + 2 * label_background_margin) - - # Draw a solid white square for the label background - label_background_position = (0, 0) # Top-left corner - draw.rectangle([label_background_position, (label_background_position[0] + label_background_size[0], label_background_position[1] + label_background_size[1])], fill="white") - - # Add the label text in black over the white square - label_position = (label_background_margin, label_background_margin) - draw.text(label_position, label, font=font, fill="black") - - return image - - -def resize_image_height(image, fixed_size): - # Resize image, maintaining aspect ratio - width, height = image.size - new_size = int(width * fixed_size / height), fixed_size - return image.resize(new_size, Image.Resampling.LANCZOS) - - -def concatenate_images_horizontal(image_list): - # Concatenate images horizontally - widths, heights = zip(*(i.size for i in image_list)) - total_width = sum(widths) - max_height = max(heights) - assert all(height == max_height for height in heights) - new_im = Image.new("RGB", (total_width, max_height)) - x_offset = 0 - for im in image_list: - new_im.paste(im, (x_offset, 0)) - x_offset += im.size[0] - return new_im - - -def resize_image_width(image, fixed_size): - # Resize image, maintaining aspect ratio - width, height = image.size - new_size = fixed_size, int(height * fixed_size / width) - return image.resize(new_size, Image.Resampling.LANCZOS) - - -def concatenate_images_vertical(image_list): - # Concatenate images horizontally - widths, heights = zip(*(i.size for i in image_list)) - total_height = sum(heights) - max_width = max(widths) - assert all(width == max_width for width in widths) - new_im = Image.new("RGB", (max_width, total_height)) - y_offset = 0 - for im in image_list: - new_im.paste(im, (0, y_offset)) - y_offset += im.size[1] - return new_im - - -def process_images_horizontal(original_images, size): - images = [] - for i, img in enumerate(original_images): - # Resize image - img_resized = resize_image_height(img, fixed_size=size) - - # Add order label - img_labeled = add_order_label(img_resized, f"[{i+1}]") - - # Append to list - images.append(img_labeled) - - # Concatenate all images - return concatenate_images_horizontal(images) - - -def process_images_vertical(original_images, size): - images = [] - for i, img in enumerate(original_images): - # Resize image - img_resized = resize_image_width(img, fixed_size=size) - - # Add order label - img_labeled = add_order_label(img_resized, f"[{i+1}]") - - # Append to list - images.append(img_labeled) - - # Concatenate all images - return concatenate_images_vertical(images) - - -def process_images(images, size=672): - concat_horizontal = process_images_horizontal(images, size) - concat_vertical = process_images_vertical(images, size) - - hw, hh = concat_horizontal.size - vw, vh = concat_vertical.size - - ha = hw / hh - va = vh / vw - - if ha > va: - return concat_vertical - else: - return concat_horizontal - - -lmms_logger = logging.getLogger("lmms-eval") - -MULTI_CHOICE_PROMPT = "Answer with the option letter from the given choices directly." -OPEN_ENDED_PROMPT = "Answer the question using a single word or phrase." - - -def replace_images_tokens(input_string): - # for i in range(1, 8): - # question_text = f"" - # query_text = "" - # if question_text in input_string: - # input_string = input_string.replace(question_text, query_text) - return input_string - - -def parse_options(options): - option_letters = [chr(ord("A") + i) for i in range(len(options))] - choices_str = "\n".join([f"{option_letter}. {option}" for option_letter, option in zip(option_letters, options)]) - return choices_str - - -def construct_prompt(doc): - question = doc["question"] - if doc["question_type"] == "multiple-choice": - # Weirdly, data["options"] is a string in MMMU Huggingface dataset - parsed_options = parse_options(ast.literal_eval(doc["options"])) - # parsed_options already prepends a newline so no need to add space here - question = f"{question}\n{parsed_options}\n{MULTI_CHOICE_PROMPT}" - else: - question = f"{question}\n{OPEN_ENDED_PROMPT}" - return question - - -def mmmu_doc_to_text(doc): - question = construct_prompt(doc) - return replace_images_tokens(question) - - -def mmmu_doc_to_visual(doc): - prompt = construct_prompt(doc) - image_tokens = re.findall(r"", prompt) - # Remove <> and swap space as _ - image_tokens = sorted(list(set([image_token.strip("<>").replace(" ", "_") for image_token in image_tokens]))) - visual = [doc[image_token].convert("RGB") for image_token in image_tokens] - visual = process_images(visual) - return [visual] - - -def mmmu_process_results(doc, results): - pred = results[0] - if doc["question_type"] == "multiple-choice": - index2ans, all_choices = get_multi_choice_info(ast.literal_eval(doc["options"])) - parsed_pred = parse_multi_choice_response(pred, all_choices, index2ans) - else: - parsed_pred = parse_open_response(pred) - id = doc["id"] - mmmu_acc = {"id": id, "subdomain": extract_subset_name(doc["id"]), "question_type": doc["question_type"], "answer": doc["answer"], "parsed_pred": parsed_pred} - return { - "mmmu_acc": mmmu_acc, - "submission": { - id: pred, - }, - } - - -def extract_subset_name(input_string): - # Define a regex pattern to match "validation_" at the beginning and "_" at the end - split = input_string.split("_")[0] - pattern = re.compile(rf"^{split}_(.+?)_\d+$") - match = pattern.search(input_string) - if match: - return match.group(1) - else: - raise ValueError(f'No match found in "{input_string}"') - - -def mmmu_test_aggregate_results_for_submission(results, args): - path = generate_submission_file("mmmu_test_for_submission.json", args) - with open(path, "w") as f: - json.dump(results, f) - lmms_logger.info(f"Results saved to {path}.") - - -def mmmu_aggregate_results(results): - evaluation_result = {} - subset_to_eval_samples = defaultdict(list) - for result in results: - subset_to_eval_samples[result["subdomain"]].append(result) - for subset, sub_eval_samples in subset_to_eval_samples.items(): - judge_dict, metric_dict = evaluate_mmmu(sub_eval_samples) - metric_dict.update({"num_example": len(sub_eval_samples)}) - evaluation_result[subset] = metric_dict - printable_results = {} - for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items(): - in_domain_cat_results = {} - for cat_name in in_domain_cats: - if cat_name in evaluation_result.keys(): - in_domain_cat_results[cat_name] = evaluation_result[cat_name] - else: - pass - in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results) - in_domain_data_num = sum([cat_results["num_example"] for cat_results in in_domain_cat_results.values()]) - printable_results["Overall-" + domain] = { - "num": int(in_domain_data_num), - "acc": round(in_domain_ins_acc, 3), - } - # add sub category - for cat_name, cat_results in in_domain_cat_results.items(): - printable_results[cat_name] = { - "num": int(cat_results["num_example"]), - "acc": round(cat_results["acc"], 3), - } - all_ins_acc = calculate_ins_level_acc(evaluation_result) - printable_results["Overall"] = { - "num": sum([cat_results["num_example"] for cat_results in evaluation_result.values()]), - "acc": round(all_ins_acc, 3), - } - print(printable_results) - return printable_results["Overall"]["acc"] - - -################## -# Helper functions written by official MMMU repo. -################## - - -def calculate_ins_level_acc(results): - """Calculate the instruction level accuracy for given Subject results - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L246 - """ - acc = 0 - ins_num = 0 - for cat_results in results.values(): - acc += cat_results["acc"] * cat_results["num_example"] - ins_num += cat_results["num_example"] - if ins_num == 0: - return 0 - return acc / ins_num - - -DOMAIN_CAT2SUB_CAT = { - "Art and Design": ["Art", "Art_Theory", "Design", "Music"], - "Business": ["Accounting", "Economics", "Finance", "Manage", "Marketing"], - "Science": [ - "Biology", - "Chemistry", - "Geography", - "Math", - "Physics", - ], - "Health and Medicine": [ - "Basic_Medical_Science", - "Clinical_Medicine", - "Diagnostics_and_Laboratory_Medicine", - "Pharmacy", - "Public_Health", - ], - "Humanities and Social Science": [ - "History", - "Literature", - "Sociology", - "Psychology", - ], - "Tech and Engineering": [ - "Agriculture", - "Architecture_and_Engineering", - "Computer_Science", - "Electronics", - "Energy_and_Power", - "Materials", - "Mechanical_Engineering", - ], -} - - -def eval_multi_choice(gold_i, pred_i): - """ - Evaluate a multiple choice instance. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L175 - """ - correct = False - # only they are exactly the same, we consider it as correct - if isinstance(gold_i, list): - for answer in gold_i: - if answer == pred_i: - correct = True - break - else: # gold_i is a string - if gold_i == pred_i: - correct = True - return correct - - -def eval_open(gold_i, pred_i): - """ - Evaluate an open question instance - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L191 - """ - correct = False - if isinstance(gold_i, list): - # use float to avoid trivial matches - norm_answers = [] - for answer in gold_i: - norm_answers.extend(normalize_str(answer)) - else: - norm_answers = normalize_str(gold_i) - for pred in pred_i: # pred is already normalized in parse response phase - if isinstance(pred, str): # if it's a string, then find if ans in the pred_i - for norm_ans in norm_answers: - # only see if the string answer in the string pred - if isinstance(norm_ans, str) and norm_ans in pred: - if not correct: - correct = True - break - else: # it's a float number - if pred in norm_answers: - if not correct: - correct = True - break - return correct - - -def evaluate_mmmu(samples): - """ - Batch evaluation for multiple choice and open questions. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L219 - """ - pred_correct = 0 - judge_dict = dict() - for sample in samples: - gold_i = sample["answer"] - pred_i = sample["parsed_pred"] - if sample["question_type"] == "multiple-choice": - correct = eval_multi_choice(gold_i, pred_i) - else: # open question - correct = eval_open(gold_i, pred_i) - - if correct: - judge_dict[sample["id"]] = "Correct" - pred_correct += 1 - else: - judge_dict[sample["id"]] = "Wrong" - - if len(samples) == 0: - return {"acc": 0} - return judge_dict, {"acc": pred_correct / len(samples)} - - -def parse_multi_choice_response(response, all_choices, index2ans): - """ - Parse the prediction from the generated response. - Return the predicted index e.g., A, B, C, D. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L10 - """ - for char in [",", ".", "!", "?", ";", ":", "'"]: - response = response.strip(char) - response = " " + response + " " # add space to avoid partial match - - index_ans = True - ans_with_brack = False - candidates = [] - for choice in all_choices: # e.g., (A) (B) (C) (D) - if f"({choice})" in response: - candidates.append(choice) - ans_with_brack = True - - if len(candidates) == 0: - for choice in all_choices: # e.g., A B C D - if f"{choice} " in response: - candidates.append(choice) - - if len(candidates) == 0: - for choice in all_choices: # e.g., A. B. C. D. - if f"{choice}." in response: - candidates.append(choice) - - # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example - if len(candidates) == 0 and len(response.split()) > 5: - for index, ans in index2ans.items(): - if ans.lower() in response.lower(): - candidates.append(index) - index_ans = False # it's content ans. - - if len(candidates) == 0: # still not get answer, randomly choose one. - pred_index = random.choice(all_choices) - elif len(candidates) > 1: - start_indexes = [] - if index_ans: - if ans_with_brack: - for can in candidates: - index = response.rfind(f"({can})") - start_indexes.append(index) # -1 will be ignored anyway - # start_indexes = [generated_response.index(f'({can})') for can in candidates] - else: - for can in candidates: - index = response.rfind(f" {can} ") - start_indexes.append(index) - else: - for can in candidates: - index = response.lower().rfind(index2ans[can].lower()) - start_indexes.append(index) - # get the last one - pred_index = candidates[np.argmax(start_indexes)] - else: # if only one candidate, use it. - pred_index = candidates[0] - - return pred_index - - -def extract_numbers(string): - """ - Exact all forms of numbers from a string with regex. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L100 - """ - # Pattern for numbers with commas - pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" - # Pattern for scientific notation - pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" - # Pattern for simple numbers without commas - pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" - - # Extract numbers with commas - numbers_with_commas = re.findall(pattern_commas, string) - # Extract numbers in scientific notation - numbers_scientific = re.findall(pattern_scientific, string) - # Extract simple numbers without commas - numbers_simple = re.findall(pattern_simple, string) - - # Combine all extracted numbersz - all_numbers = numbers_with_commas + numbers_scientific + numbers_simple - return all_numbers - - -def check_is_number(string): - """ - Check if the given string a number. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L65 - """ - try: - float(string.replace(",", "")) - return True - except ValueError: - # check if there's comma inside - return False - - -def normalize_str(string): - """ - Normalize the str to lower case and make them float numbers if possible. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L76 - """ - # check if characters in the string - - # if number, numerize it. - string = string.strip() - - is_number = check_is_number(string) - - if is_number: - string = string.replace(",", "") - string = float(string) - # leave 2 decimal - string = round(string, 2) - return [string] - else: # it's likely to be a string - # lower it - string = string.lower() - if len(string) == 1: - return [" " + string, string + " "] # avoid trivial matches - return [string] - - -def parse_open_response(response): - """ - Parse the prediction from the generated response. - Return a list of predicted strings or numbers. - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py#L122 - """ - - # content = content.strip("\n").strip(".").strip(" ") - def get_key_subresponses(response): - key_responses = [] - response = response.strip().strip(".").lower() - sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) - indicators_of_keys = [ - "could be ", - "so ", - "is ", - "thus ", - "therefore ", - "final ", - "answer ", - "result ", - ] - key_responses = [] - for index, resp in enumerate(sub_responses): - # if last one, accept it's an equation (the entire response can be just one sentence with equation) - if index == len(sub_responses) - 1: - indicators_of_keys.extend(["="]) - shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) - for indicator in indicators_of_keys: - if indicator in resp: - if not shortest_key_response: - shortest_key_response = resp.split(indicator)[-1].strip() - else: - if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response): - shortest_key_response = resp.split(indicator)[-1].strip() - # key_responses.append(resp.split(indicator)[1].strip()) - - if shortest_key_response: - # and it's not trivial - if shortest_key_response.strip() not in [ - ":", - ",", - ".", - "!", - "?", - ";", - ":", - "'", - ]: - key_responses.append(shortest_key_response) - if len(key_responses) == 0: # did not found any - return [response] - return key_responses - - # pdb.set_trace() - key_responses = get_key_subresponses(response) - - pred_list = key_responses.copy() # keep the original string response - for resp in key_responses: - pred_list.extend(extract_numbers(resp)) - - tmp_pred_list = [] - for i in range(len(pred_list)): - tmp_pred_list.extend(normalize_str(pred_list[i])) - pred_list = tmp_pred_list - - # remove duplicates - pred_list = list(set(pred_list)) - - return pred_list - - -def get_multi_choice_info(options): - """ - Given the list of options for multiple choice question - Return the index2ans and all_choices - https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/data_utils.py#L54 - """ - - start_chr = "A" - all_choices = [] - index2ans = {} - for i, option in enumerate(options): - index2ans[chr(ord(start_chr) + i)] = option - all_choices.append(chr(ord(start_chr) + i)) - - return index2ans, all_choices diff --git a/lmms_eval/tasks/mmupd/_default_template_mmupd_yaml b/lmms_eval/tasks/mmupd/_default_template_mmupd_yaml deleted file mode 100644 index 7aa8d81..0000000 --- a/lmms_eval/tasks/mmupd/_default_template_mmupd_yaml +++ /dev/null @@ -1,18 +0,0 @@ -dataset_path: MM-UPD/MM-UPD -doc_to_target: "answer" -doc_to_visual: !function utils.mmupd_doc_to_visual -doc_to_text: !function utils.mmupd_doc_to_text -doc_to_target: "answer" -process_results: !function utils.mmupd_process_results -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -output_type: generate_until -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false diff --git a/lmms_eval/tasks/mmupd/mmaad_base.yaml b/lmms_eval/tasks/mmupd/mmaad_base.yaml deleted file mode 100644 index 9a66b3e..0000000 --- a/lmms_eval/tasks/mmupd/mmaad_base.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: "mmaad_base" -test_split: test -dataset_name: mmaad_base -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\n" -include: _default_template_mmupd_yaml -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mmaad_base - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmaad_instruction.yaml b/lmms_eval/tasks/mmupd/mmaad_instruction.yaml deleted file mode 100644 index 30a3bac..0000000 --- a/lmms_eval/tasks/mmupd/mmaad_instruction.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: "mmaad_instruction" -test_split: test -dataset_name: mmaad_base -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"." -include: _default_template_mmupd_yaml -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mmaad_instruction - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmaad_option.yaml b/lmms_eval/tasks/mmupd/mmaad_option.yaml deleted file mode 100644 index f110b82..0000000 --- a/lmms_eval/tasks/mmupd/mmaad_option.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: "mmaad_option" -test_split: test -dataset_name: mmaad_option -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer with the option's letter from the given choices directly." -include: _default_template_mmupd_yaml -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mmaad_option - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmiasd_base.yaml b/lmms_eval/tasks/mmupd/mmiasd_base.yaml deleted file mode 100644 index 6a9159f..0000000 --- a/lmms_eval/tasks/mmupd/mmiasd_base.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: "mmiasd_base" -test_split: test -dataset_name: mmiasd_base -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\n" -include: _default_template_mmupd_yaml -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mmiasd_base - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml b/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml deleted file mode 100644 index d0ac3a0..0000000 --- a/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: "mmiasd_instruction" -test_split: test -dataset_name: mmiasd_base -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"." -include: _default_template_mmupd_yaml -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mmiasd_instruction - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmiasd_option.yaml b/lmms_eval/tasks/mmupd/mmiasd_option.yaml deleted file mode 100644 index a1a1aea..0000000 --- a/lmms_eval/tasks/mmupd/mmiasd_option.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: "mmiasd_option" -test_split: test -dataset_name: mmiasd_option -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer with the option's letter from the given choices directly." -include: _default_template_mmupd_yaml -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mmiasd_option - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmivqd_base.yaml b/lmms_eval/tasks/mmupd/mmivqd_base.yaml deleted file mode 100644 index 82853fd..0000000 --- a/lmms_eval/tasks/mmupd/mmivqd_base.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: "mmivqd_base" -test_split: test -dataset_name: mmivqd_base -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\n" -include: _default_template_mmupd_yaml -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mmivqd_base - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml b/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml deleted file mode 100644 index a024022..0000000 --- a/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: "mmivqd_instruction" -test_split: test -dataset_name: mmivqd_base -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nIf the given image is irrelevant to the question, answer \"F. The image and question are irrelevant.\"." -include: _default_template_mmupd_yaml -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mmivqd_instruction - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmivqd_option.yaml b/lmms_eval/tasks/mmupd/mmivqd_option.yaml deleted file mode 100644 index 1363c34..0000000 --- a/lmms_eval/tasks/mmupd/mmivqd_option.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: "mmivqd_option" -test_split: test -dataset_name: mmivqd_option -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer with the option's letter from the given choices directly." -include: _default_template_mmupd_yaml -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mmivqd_option - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmupd.yaml b/lmms_eval/tasks/mmupd/mmupd.yaml deleted file mode 100644 index f40416a..0000000 --- a/lmms_eval/tasks/mmupd/mmupd.yaml +++ /dev/null @@ -1,15 +0,0 @@ -group: mmupd -task: - - mmaad_base - - mmaad_option - - mmaad_instruction - - mmiasd_base - - mmiasd_option - - mmiasd_instruction - - mmivqd_base - - mmivqd_option - - mmivqd_instruction -metadata: - version: 0.0 - sys_prompt: "" - gpt_eval_model_name: "gpt-3.5-turbo-0613" \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmupd_base.yaml b/lmms_eval/tasks/mmupd/mmupd_base.yaml deleted file mode 100644 index 988ad2e..0000000 --- a/lmms_eval/tasks/mmupd/mmupd_base.yaml +++ /dev/null @@ -1,10 +0,0 @@ -group: mmupd_base -task: - - mmaad_base - - mmiasd_base - - mmivqd_base -metadata: - version: 0.0 - sys_prompt: "" - gpt_eval_model_name: "gpt-3.5-turbo-0613" - \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmupd_evals.py b/lmms_eval/tasks/mmupd/mmupd_evals.py deleted file mode 100644 index 4878763..0000000 --- a/lmms_eval/tasks/mmupd/mmupd_evals.py +++ /dev/null @@ -1,630 +0,0 @@ -import time -import random as rd -import string -from collections import defaultdict -import requests -import math -import numpy as np -import pandas as pd -import pickle -import logging -import json - -eval_logger = logging.getLogger("lmms-eval") - - -def dump(data, f): - - def dump_pkl(data, pth): - pickle.dump(data, open(pth, 'wb')) - - def dump_json(data, pth): - json.dump(data, open(pth, 'w')) - - def dump_jsonl(data, f): - lines = [json.dumps(x, ensure_ascii=False) for x in data] - with open(f, 'w', encoding='utf8') as fout: - fout.write('\n'.join(lines)) - - def dump_xlsx(data, f): - data.to_excel(f, index=False) - - def dump_csv(data, f): - data.to_csv(f, index=False) - - def dump_tsv(data, f): - data.to_csv(f, sep='\t', index=False) - - handlers = dict(pkl=dump_pkl, - json=dump_json, - jsonl=dump_jsonl, - xlsx=dump_xlsx, - csv=dump_csv, - tsv=dump_tsv) - suffix = f.split('.')[-1] - return handlers[suffix](data, f) - - -def load(f): - """ - Loads data from various file formats. - - Parameters: - - file_path: Path to the file to be loaded. - - Returns: - - Loaded data. - """ - def load_pkl(pth): - return pickle.load(open(pth, 'rb')) - - def load_json(pth): - return json.load(open(pth, 'r', encoding='utf-8')) - - def load_jsonl(f): - lines = open(f, encoding='utf-8').readlines() - lines = [x.strip() for x in lines] - if lines[-1] == '': - lines = lines[:-1] - data = [json.loads(x) for x in lines] - return data - - def load_xlsx(f): - df = pd.read_excel(f) - df = df.dropna(subset=['prediction']) - - return df - - def load_csv(f): - return pd.read_csv(f) - - def load_tsv(f): - return pd.read_csv(f, sep='\t') - - handlers = dict(pkl=load_pkl, - json=load_json, - jsonl=load_jsonl, - xlsx=load_xlsx, - csv=load_csv, - tsv=load_tsv) - suffix = f.split('.')[-1] - return handlers[suffix](f) - - -class MMUPD_Evaluator: - def __init__(self, sys_prompt="There are several options:", API_KEY="", API_URL="", model_version="gpt-3.5-turbo-0613"): - self.sys_prompt = sys_prompt - self.model_version = model_version - self.API_KEY = API_KEY - self.API_URL = API_URL - - def create_options_prompt(self, row_data, option_candidate): - available_keys = set(row_data.keys()) & set(option_candidate) - options = {cand: row_data[cand] for cand in available_keys if row_data[cand]} - sorted_options = dict(sorted(options.items())) - options_prompt = f"{self.sys_prompt}\n" - for key, item in sorted_options.items(): - if pd.notna(item) and item != "nan": - options_prompt += f"{key}. {item}\n" - return options_prompt.rstrip("\n"), sorted_options - - def is_none(self, value): - if value is None: - return True - if pd.isna(value): - return True - if type(value) is float and math.isnan(value): - return True - if type(value) is str and value.lower() == 'nan': - return True - if type(value) is str and value.lower() == 'none': - return True - return False - - # Prompt Building - def build_option_str(self, option_list): - chars = string.ascii_uppercase - s = "There are several options: \n" - for c, opt in zip(chars, option_list): - if not pd.isna(opt): - s += f"{c}. {opt}\n" - else: - return s - return s - - def extract_options(self, item): - options = [] - for c in 'ABCDE': - try: - if self.is_none(item[c]) is False: - options.append(item[c]) - except KeyError: - continue - return options - - def build_choices(self, item): - ret = {} - for ch in 'ABCDE': - try: - if self.is_none(item[ch]) is False: - ret[ch] = item[ch] - except KeyError: - continue - return ret - - def build_option_str_w_gt(self, option_list, gt_text, eval_type, question_type, upd_type): - chars = string.ascii_uppercase - s = 'There are several options: \n' - valid_option = [] - answer_option = [] - - for c, opt in zip(chars, option_list): - if self.is_none(opt) is False: - s += f'{c}. {opt}\n' - valid_option.append(c) - if upd_type == "aad": - if eval_type == "aad": - gt_option = "" - for option in ['C', 'D', 'E', 'F']: - if option not in valid_option: - gt_option = option - break - none_option_mapping = {'C': 'D', 'D': 'E', 'E': 'F', 'F': 'G'} - s += f'{gt_option}. {gt_text}\n' - none_option = none_option_mapping[gt_option] - s += f'{none_option}. The correct answer is No answer, None of the above, \ - all provided options are incorrect, or I cannot answer.\n' - valid_option.append(gt_option) - valid_option.append(none_option) - answer_option = [gt_option, none_option] - if question_type == 'inst': - s += f'F. None of the above\n' - valid_option.append("F") - answer_option = [gt_option, none_option, "F"] - - if eval_type == "standard": - none_option = "" - for option in ['C', 'D', 'E', 'F']: - if option not in valid_option: - none_option = option - break - s += f'{none_option}. The correct answer is No answer, None of the above, \ - all provided options are incorrect, or I cannot answer.\n' - valid_option.append(none_option) - if question_type == 'inst': - s += 'F. None of the above\n' - valid_option.append("F") - elif upd_type == "iasd": - if eval_type == "iasd": - gt_option = "" - for option in ['C', 'D', 'E', 'F']: - if option not in valid_option: - gt_option = option - break - - s += f'{gt_option}. {gt_text}\n' - valid_option.append(gt_option) - - if question_type == 'inst': - if gt_option == 'E': - s += f'F. None of the above\n' - valid_option.append('F') - s += 'G. The correct answer is No answer, None of the above, all provided options are irrelevant or incorrect, or I cannot answer.\n' - valid_option.append('G') - answer_option = [gt_option, 'F', 'G'] - else: - none_option_mapping = {'C': 'D', 'D': 'E'} - none_option = none_option_mapping[gt_option] - s += f'{none_option}. The correct answer is No answer, None of the above, all provided options are irrelevant or incorrect, or I cannot answer.\n' - valid_option.append(none_option) - s += f'F. None of the above\n' - valid_option.append('F') - answer_option = [gt_option, none_option, 'F'] - else: - none_option_mapping = {'C': 'D', 'D': 'E', 'E': 'F', 'F': 'G'} - none_option = none_option_mapping[gt_option] - s += f'{none_option}. The correct answer is No answer, None of the above, all provided options are irrelevant or incorrect, or I cannot answer.\n' - valid_option.append(none_option) - answer_option = [gt_option, none_option] - - if eval_type == "standard": - none_option = "" - for option in ['C', 'D', 'E', 'F']: - if option not in valid_option: - none_option = option - break - s += f'{none_option}. The correct answer is No answer, None of the above, \ - all provided options are irrelevant or incorrect, or I cannot answer.\n' - valid_option.append(none_option) - if question_type == 'inst': - s += f'F. None of the above\n' - valid_option.append("F") - elif upd_type == "ivqd": - if eval_type == "ivqd": - none_option = "" - for option in ['C', 'D', 'E', 'F']: - if option not in valid_option: - none_option = option - break - s += f'{none_option}. The correct answer is that The image is incompatible with the question, or I cannot answer.\n' - valid_option.append(none_option) - answer_option = [none_option] - if question_type == 'inst': - s += f'F. The image and question are irrelevant.\n' - valid_option.append("F") - answer_option = [none_option, "F"] - - if eval_type == "standard": - none_option = "" - for option in ['C', 'D', 'E', 'F']: - if option not in valid_option: - none_option = option - break - s += f'{none_option}. The correct answer is that The image is incompatible with the question, or I cannot answer.\n' - valid_option.append(none_option) - if question_type == 'inst': - s += f'F. The image and question are irrelevant.\n' - valid_option.append("F") - return s, valid_option, answer_option - - def build_prompt(self, question, options, prediction, upd_type): - if upd_type == "aad": - tmpl = ( - 'You are an AI assistant who will help me to match an answer ' - 'with several options of a single-choice question. ' - 'You are provided with a question, several options, and an answer, ' - 'and you need to find which option is most similar to the answer. ' - 'If the meaning of all options are significantly different ' - 'from the answer, output H. ' - 'Your should output a single uppercase character in A, B, C, D, E, F, G ' - '(if they are valid options), and H. \n' - 'Example 1: \n' - 'Question: What is the main object in image?\nOptions: A. teddy bear ' - 'B. rabbit C. cat D. dog E. no answer \nAnswer: a cute teddy bear\nYour output: A\n' - 'Example 2: \n' - 'Question: What is the main object in image?\nOptions: A. teddy bear ' - 'B. rabbit C. cat D. dog E. None of the above \nAnswer: no answer \nYour output: E\n' - 'Example 3: \n' - 'Question: What is the main object in image?\nOptions: A. teddy bear ' - 'B. rabbit C. cat D. dog E. None of the above \nAnswer: fish \nYour output: H\n' - 'Example 4: \n' - 'Question: {}?\nOptions: {}\nAnswer: {}\nYour output: ') - elif upd_type == "iasd": - tmpl = ( - 'You are an AI assistant who will help me to match an answer ' - 'with several options of a single-choice question. ' - 'You are provided with a question, several options, and an answer, ' - 'and you need to find which option is most similar to the answer. ' - 'If the meaning of all options are significantly different ' - 'from the answer, output H. ' - 'Your should output a single uppercase character in A, B, C, D, E, F, G ' - '(if they are valid options), and H. \n' - 'Example 1: \n' - 'Question: What is the main object in image?\nOptions: A. teddy bear ' - 'B. rabbit C. cat D. dog E. no answer \nAnswer: a cute teddy bear\nYour output: A\n' - 'Example 2: \n' - 'Question: What is the main object in image?\nOptions: A. teddy bear ' - 'B. rabbit C. cat D. dog E. None of the above \nAnswer: no answer \nYour output: E\n' - 'Example 3: \n' - 'Question: What is the main object in image?\nOptions: A. teddy bear ' - 'B. rabbit C. cat D. dog E. None of the above \nAnswer: fish \nYour output: H\n' - 'Example 4: \n' - 'Question: {}?\nOptions: {}\nAnswer: {}\nYour output: ') - elif upd_type == "ivqd": - tmpl = ( - 'You are an AI assistant who will help me to match an answer ' - 'with several options of a single-choice question. ' - 'You are provided with a question, several options, and an answer, ' - 'and you need to find which option is most similar to the answer. ' - 'If the meaning of all options are significantly different ' - 'from the answer, output H. ' - 'Your should output a single uppercase character in A, B, C, D, E, F, G ' - '(if they are valid options), and H. \n' - 'Example 1: \n' - 'Question: What is the main object in image?\nOptions: A. teddy bear ' - 'B. rabbit C. cat D. dog E. The image and question are irrelevant \nAnswer: a cute teddy bear\nYour output: A\n' - 'Example 2: \n' - 'Question: What is the main object in image?\nOptions: A. teddy bear ' - 'B. rabbit C. cat D. dog E. The image and question are irrelevant \nAnswer: The updloaded image and question are incompatible. \nYour output: E\n' - 'Example 3: \n' - 'Question: What is the main object in image?\nOptions: A. teddy bear ' - 'B. rabbit C. cat D. dog E. The image and question are irrelevant \nAnswer: fish \nYour output: H\n' - 'Example 4: \n' - 'Question: {}?\nOptions: {}\nAnswer: {}\nYour output: ') - return tmpl.format(question, options, prediction) - - # Prefetch Answers - def can_infer_option(self, answer, option_dict, question_type=None, valid_option=None): - if valid_option is None: - valid_option = list(option_dict.keys()) - if question_type == 'inst': - valid_option.append("F") - - if 'Failed to obtain answer via API' in answer: - return False - - answer = answer.strip() - - ch_cand_list = [] - - punctuations = [".", ")", ","] - if "A" in valid_option: - characters = ["B", "C", "D", "E", "F", "G"] - combinations = [char + punct for char in characters for punct in punctuations] - start_patterns = ["A)", "A.", "A,", "(A)"] - if answer == "A" or (any(answer.startswith(pattern) for pattern in start_patterns) and all(x not in answer for x in combinations)): - ch_cand_list.append("A") - if "B" in valid_option: - characters = ["A", "C", "D", "E", "F", "G"] - combinations = [char + punct for char in characters for punct in punctuations] - start_patterns = ["B)", "B.", "B,", "(B)"] - if answer == "B" or (any(answer.startswith(pattern) for pattern in start_patterns) and all(x not in answer for x in combinations)): - ch_cand_list.append("B") - if "C" in valid_option: - characters = ["A", "B", "D", "E", "F", "G"] - combinations = [char + punct for char in characters for punct in punctuations] - start_patterns = ["C)", "C.", "C,", "(C)"] - if answer == "C" or (any(answer.startswith(pattern) for pattern in start_patterns) and all(x not in answer for x in combinations)): - ch_cand_list.append("C") - if "D" in valid_option: - characters = ["A", "B", "C", "E", "F", "G"] - combinations = [char + punct for char in characters for punct in punctuations] - start_patterns = ["D)", "D.", "D,", "(D)"] - if answer == "D" or (any(answer.startswith(pattern) for pattern in start_patterns) and all(x not in answer for x in combinations)): - ch_cand_list.append("D") - if "E" in valid_option: - characters = ["A", "B", "C", "D", "F", "G"] - combinations = [char + punct for char in characters for punct in punctuations] - start_patterns = ["E)", "E.", "E,", "(E)"] - if answer == "E" or (any(answer.startswith(pattern) for pattern in start_patterns) and all(x not in answer for x in combinations)): - ch_cand_list.append("E") - if "F" in valid_option: - characters = ["A", "B", "C", "D", "E", "G"] - combinations = [char + punct for char in characters for punct in punctuations] - start_patterns = ["F)", "F.", "F,", "(F)"] - if answer == "F" or (any(answer.startswith(pattern) for pattern in start_patterns) and all(x not in answer for x in combinations)): - ch_cand_list.append("F") - if "G" in valid_option: - characters = ["A", "B", "C", "D", "E", "F"] - combinations = [char + punct for char in characters for punct in punctuations] - - start_patterns = ["G)", "G.", "G,", "(G)"] - if answer == "G" or (any(answer.startswith(pattern) for pattern in start_patterns) and all(x not in answer for x in combinations)): - ch_cand_list.append("G") - - if len(ch_cand_list) == 1: - return ch_cand_list[0] - - return False - - def can_infer(self, answer, choices, question_type=None, valid_option=None): - copt = self.can_infer_option(answer, choices, question_type, valid_option=valid_option) - return copt if copt else False - - def prefetch_answer(self, item, question_type): - choices = self.build_choices(item) - return self.can_infer(item["prediction"], choices, question_type=question_type) - - def _post_request(self, payload): - headers = { - "Authorization": f"Bearer {self.API_KEY}", - "Content-Type": "application/json", - } - response = requests.post(self.API_URL, headers=headers, json=payload, timeout=30) - response.raise_for_status() - return response.json() - - def get_chat_response(self, prompt, temperature=0, max_tokens=256, n=1, patience=5, sleep_time=3): - messages = [ - {"role": "user", "content": prompt}, - ] - payload = {"model": self.model_version, "messages": messages, "temperature": temperature, "max_tokens": max_tokens, "n": n} - - while patience > 0: - patience -= 1 - try: - response = self._post_request(payload) - if n == 1: - prediction = response["choices"][0]["message"]["content"].strip() - if prediction and prediction != "": - return prediction - else: - prediction = [choice["message"]["content"].strip() for choice in response["choices"]] - if prediction and prediction[0] != "": - return prediction - - except Exception as e: - eval_logger.info(f"Attempt {patience + 1} failed with error: {e}") - if sleep_time > 0: - time.sleep(sleep_time) - - return "Failed to obtain answer via API" - - def extract_answer_from_item(self, item, gt_text, eval_type, question_type, upd_type): - options = self.extract_options(item) - option_str, valid_option, answer_option = self.build_option_str_w_gt(options, gt_text, eval_type, question_type=question_type, upd_type=upd_type) - - prompt = self.build_prompt(item['question'], option_str, item['prediction'], upd_type=upd_type) - retry = 3 - choices = self.build_choices(item) - - ret = self.can_infer(item['prediction'], choices, valid_option=valid_option) - if ret: - return ret, item['prediction'], answer_option - - while retry: - ans = self.get_chat_response(prompt, temperature=0.7) - if 'Failed to obtain answer via API' in ans: - msg = 'GPT API failed to answer. ' - eval_logger.info(msg) - retry -= 1 - else: - ret = self.can_infer(ans, choices, valid_option=valid_option) - if ret: - return ret, ans, answer_option - else: - eval_logger.info(f'GPT output includes 0 / >1 letter in "{valid_option}": {ans}') - retry -= 1 - - if retry == 0: - return 'H', 'Failed to predict. ', answer_option - - def eval_sub_data(self, sub_data, answer_map, gt_text_map, question_type, eval_type, upd_type): - lt = len(sub_data) - GT, PRED = [], [] - - for i in range(lt): - item = sub_data.iloc[i] - idx = item['index'] - GT.append(answer_map[idx]) - PRED.append(self.prefetch_answer(item, question_type)) - if PRED[-1] and (GT[-1] != PRED[-1]): - return 0 - - for i in range(lt): - if PRED[i]: - continue - else: - item = sub_data.iloc[i] - idx = item['index'] - gt_text = gt_text_map[idx] if gt_text_map is not None else None - ret, _, answer_option = self.extract_answer_from_item(sub_data.iloc[i], gt_text, eval_type, question_type=question_type, upd_type=upd_type) - PRED[i] = ret - if eval_type == "standard": - if PRED[i] != GT[i]: - return 0 - else: - if GT[i] == "F": - if PRED[i] not in answer_option: - return 0 - else: - if PRED[i] != GT[i] and PRED[i] not in answer_option: - return 0 - return 1 - - def calculate_hit_rates(self, data): - overall_hit_rate = data["hit"].mean() - - category_hit_rate = {} - if "category" in data.columns: - # Category-based hit rate - category_hit_rate = data.groupby("category")["hit"].mean().to_dict() - - return overall_hit_rate, category_hit_rate - - # Evaluate Results - def eval_result(self, results, eval_method, upd_type, question_type, eval_type): - """ - Parameters: - - args: Arguments. - - results: Results to evaluate. - - eval_method: The evaluation method. either "openai". - - upd_type: The type of UPD. either "aad", "iasd", or "ivqd". - - question_type: The type of question. either "base", "option", or "inst". - - eval_type: The type of evaluation. either "standard", "aad", "iasd", "ivqd". - """ - - rd.seed(2680) - assert eval_method == "openai" - - result = {} - data = pd.DataFrame(results) - - if eval_type == "standard": - data = data[data["type"] == "standard"] - else: - data = data[data["type"] == "upd"] - - data = data.sort_values(by="index") - - data["prediction"] = [str(x) for x in data["prediction"]] - for k in data.keys(): - data[k.lower() if k not in "ABCDE" else k] = data.pop(k) - - # meta = load(meta_file) - - data_main = data[data["index"] < int(1e6)] - - data_main["hit"] = 0 - cate_map = {i: c for i, c in zip(data["index"], data["category"])} - answer_map = {i: c for i, c in zip(data["index"], data["answer"])} - - gt_text_map = {i: c for i, c in zip(data['index'], data['masked_answer'])} - - lt = len(data_main) - hit, tot = 0, 0 - - for i in range(lt): - # Dealing with the normal part - item_main = data_main.iloc[i] - idx = item_main["index"] - - if idx in result: - correct = result[idx] - assert correct in [0, 1] - hit += correct - tot += 1 - continue - - sub_data = data[data["index"] % int(1e6) == idx] - - ret = self.eval_sub_data(sub_data, answer_map, gt_text_map,\ - question_type=question_type, eval_type=eval_type, upd_type=upd_type) - result[idx] = ret - hit += ret - tot += 1 - - data_main.loc[data_main["index"] == idx, "hit"] = ret - - indices = data_main["index"] - - data_main["category"] = [cate_map[i] if not math.isnan(i) else "uncategorized" for i in indices] - - overall_hit_rate, category_hit_rate = self.calculate_hit_rates(data_main) - - return overall_hit_rate, category_hit_rate, data_main - - def report_acc(self, df, groupd='category'): - assert 'split' in df - assert groupd in [None, 'category', 'l2-category'] - - res = defaultdict(list) - res['split'] = ['test'] - if groupd is None: - res['overall'] = [ - np.mean(df['hit']), - ] - return pd.DataFrame(res) - - elif groupd in df: - abilities = list(set(df[groupd])) - abilities.sort() - for ab in abilities: - sub_df = df[df[groupd] == ab] - res[ab] = [ - np.mean(sub_df['hit']), - ] - return pd.DataFrame(res) - - def calculate_dual_acc(self, standard_df, upd_df): - # dual results - dual_df = pd.merge(standard_df, upd_df, on='index', - suffixes=('_standard', '_upd')) - dual_df['hit'] = dual_df.apply(lambda row: 1 if row['hit_standard'] == 1 and row['hit_upd'] == 1 else 0, axis=1) - dual_df['split'] = dual_df['split_standard'] - dual_df['category'] = dual_df['category_standard'] - - # Evaluate dual results - overall_hit_rate, category_hit_rate = self.calculate_hit_rates(dual_df) - print("Overall Dual Acc.", overall_hit_rate) - - if "category" in dual_df.columns: - print("Category Dual Acc.:") - for category_key in category_hit_rate: - if category_key == "split": - continue - - category_percentage = category_hit_rate[category_key] * 100 - print(f"\t{category_key}: {category_percentage:.3f}") - - return overall_hit_rate, category_hit_rate, dual_df diff --git a/lmms_eval/tasks/mmupd/mmupd_instruction.yaml b/lmms_eval/tasks/mmupd/mmupd_instruction.yaml deleted file mode 100644 index 3ca2444..0000000 --- a/lmms_eval/tasks/mmupd/mmupd_instruction.yaml +++ /dev/null @@ -1,9 +0,0 @@ -group: mmupd_instruction -task: - - mmaad_instruction - - mmiasd_instruction - - mmivqd_instruction -metadata: - version: 0.0 - sys_prompt: "" - gpt_eval_model_name: "gpt-3.5-turbo-0613" \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/mmupd_option.yaml b/lmms_eval/tasks/mmupd/mmupd_option.yaml deleted file mode 100644 index 6393fdc..0000000 --- a/lmms_eval/tasks/mmupd/mmupd_option.yaml +++ /dev/null @@ -1,9 +0,0 @@ -group: mmupd_option -task: - - mmaad_option - - mmiasd_option - - mmivqd_option -metadata: - version: 0.0 - sys_prompt: "" - gpt_eval_model_name: "gpt-3.5-turbo-0613" \ No newline at end of file diff --git a/lmms_eval/tasks/mmupd/utils.py b/lmms_eval/tasks/mmupd/utils.py deleted file mode 100644 index a2f237d..0000000 --- a/lmms_eval/tasks/mmupd/utils.py +++ /dev/null @@ -1,168 +0,0 @@ -import logging -import yaml -import os -from pathlib import Path -import pandas as pd -import json -from PIL import Image -from io import BytesIO -import base64 -from lmms_eval.tasks.mmupd.mmupd_evals import MMUPD_Evaluator -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -eval_logger = logging.getLogger("lmms-eval") - -with open(Path(__file__).parent / "mmupd.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] -API_TYPE = os.getenv("API_TYPE", "openai") - -if API_TYPE == "openai": - API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") - API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") -elif API_TYPE == "azure": - API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") - API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") - - -mmupd_evaluator = MMUPD_Evaluator(sys_prompt=config["metadata"]["sys_prompt"], API_KEY=API_KEY, API_URL=API_URL, model_version=GPT_EVAL_MODEL_NAME) - - -def mmupd_doc_to_visual(doc): - return [Image.open(BytesIO(base64.b64decode(doc["image"])))] - - -def mmupd_doc_to_text(doc, model_specific_prompt_kwargs=None): - option_candidate = ["A", "B", "C", "D", "E"] - options_prompt, options_dict = mmupd_evaluator.create_options_prompt(doc, option_candidate) - - data = { - # "img": doc["image"], - "question": doc["question"], - "answer": doc.get("answer", None), - "options": options_prompt, - "category": doc["category"], - "options_dict": options_dict, - "index": doc["index"], - "hint": doc["hint"], - "source": doc["source"], - "split": doc["split"], - } - - query_prompt = f"{data['hint']}\n{data['question']}{data['options']}" if pd.notna(data["hint"]) and data["hint"] != "nan" else f"{data['question']}{data['options']}" - - if model_specific_prompt_kwargs: - query_prompt = f"{query_prompt}{model_specific_prompt_kwargs['post_prompt']}" - - return query_prompt - - -def mmupd_process_results(doc, results): - model_response = results[0].strip() - data = { - "gpt_eval_score": { - "index": doc["index"], - "question": doc["question"], - "answer": doc["answer"], - "prediction": model_response, - "hint": doc["hint"], - "source": doc["source"], - "split": doc["split"], - "category": doc["category"], - "type": doc["type"], - "masked_answer": doc["masked_answer"] - }, - "submission": { - "index": doc["index"], - "question": doc["question"], - "answer": doc["answer"], - "prediction": model_response, - "hint": doc["hint"], - "source": doc["source"], - "split": doc["split"], - "category": doc["category"], - "type": doc["type"], - "masked_answer": doc["masked_answer"] - }, - } - option_candidate = ["A", "B", "C", "D", "E"] - for c in option_candidate: - data["submission"][c] = doc.get(c, "nan") - data["gpt_eval_score"][c] = doc.get(c, "nan") - return data - - -def mmaad_base(results, args): - return mmupd_results_eval(results, args, upd_type="aad", question_type="base") - - -def mmaad_option(results, args): - return mmupd_results_eval(results, args, upd_type="aad", question_type="option") - - -def mmaad_instruction(results, args): - return mmupd_results_eval(results, args, upd_type="aad", question_type="inst") - - -def mmiasd_base(results, args): - return mmupd_results_eval(results, args, upd_type="iasd", question_type="base") - - -def mmiasd_option(results, args): - return mmupd_results_eval(results, args, upd_type="iasd", question_type="option") - - -def mmiasd_instruction(results, args): - return mmupd_results_eval(results, args, upd_type="iasd", question_type="inst") - - -def mmivqd_base(results, args): - return mmupd_results_eval(results, args, upd_type="ivqd", question_type="base") - - -def mmivqd_option(results, args): - return mmupd_results_eval(results, args, upd_type="ivqd", question_type="option") - - -def mmivqd_instruction(results, args): - return mmupd_results_eval(results, args, upd_type="ivqd", question_type="inst") - - -def mmupd_results_eval(results, args, upd_type, question_type): - - print("============= MMUPD Bench Detailed Results =============") - - overall_acc_standard, category_acc_standard, standard_results_df = mmupd_evaluator.eval_result(results, eval_method="openai", upd_type=upd_type, question_type=question_type, eval_type="standard") - overall_acc_upd, category_acc_upd, upd_results_df = mmupd_evaluator.eval_result(results, eval_method="openai", upd_type=upd_type, question_type=question_type, eval_type=upd_type) - - overall_acc_dual, category_acc_dual, dual_results_df = mmupd_evaluator.calculate_dual_acc(standard_results_df, upd_results_df) - - file_json = generate_submission_file(f"mmupd_{upd_type}_{question_type}_dual_results.json", args) - - details_info = { - "overall_acc_dual": overall_acc_dual, - "category_acc_dual": category_acc_dual, - "overall_acc_standard": overall_acc_standard, - "category_acc_standard": category_acc_standard, - "overall_acc_upd": overall_acc_upd, - "category_acc_upd": category_acc_upd, - } - - with open(file_json, "w") as f: - json.dump(details_info, f) - - file_excel = generate_submission_file(f"mmupd_{upd_type}_{question_type}_dual_results_detail.xlsx", args) - dual_results_df.to_excel(file_excel, index=False) - - file_json = generate_submission_file(f"mmupd_{upd_type}_{question_type}_dual_results_detail.json", args) - dual_results_df.to_json(file_json, orient="records", indent=2) # for huggingface leaderboard submission - - return overall_acc_dual * 100 diff --git a/lmms_eval/tasks/mmvet/mmvet.yaml b/lmms_eval/tasks/mmvet/mmvet.yaml deleted file mode 100644 index c29c873..0000000 --- a/lmms_eval/tasks/mmvet/mmvet.yaml +++ /dev/null @@ -1,29 +0,0 @@ -dataset_path: lmms-lab/MMVet -dataset_kwargs: - token: True -task: "mmvet" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.mmvet_doc_to_visual -doc_to_text: !function utils.doc_to_text # Such that {{question}} will be replaced by doc["question"] -doc_to_target: "{{answer}}" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.mmvet_process_results # apply gpt eval here -metric_list: - - metric: gpt_eval_score - aggregation: !function utils.mmvet_aggregate_results - higher_is_better: true -metadata: - version: 0.0 - gpt_eval_model_name: "gpt-4" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "" diff --git a/lmms_eval/tasks/mmvet/utils.py b/lmms_eval/tasks/mmvet/utils.py deleted file mode 100644 index 5caaba4..0000000 --- a/lmms_eval/tasks/mmvet/utils.py +++ /dev/null @@ -1,213 +0,0 @@ -import os -import requests -import time -import logging -import pandas as pd -import yaml -from pathlib import Path - -eval_logger = logging.getLogger("lmms-eval") - -with open(Path(__file__).parent / "mmvet.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") -API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] -MM_VET_PROMPT = """Compare the ground truth and prediction from AI models, to give a correctness score for the prediction. in the ground truth means it is totally right only when all elements in the ground truth are present in the prediction, and means it is totally right when any one element in the ground truth is present in the prediction. The correctness score is 0.0 (totally wrong), 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, or 1.0 (totally right). Just complete the last space of the correctness score. -gpt_query_prompt | Ground truth | Prediction | Correctness ---- | --- | --- | --- -What is x in the equation? | -1 -5 | x = 3 | 0.0 -What is x in the equation? | -1 -5 | x = -1 | 0.5 -What is x in the equation? | -1 -5 | x = -5 | 0.5 -What is x in the equation? | -1 -5 | x = -5 or 5 | 0.5 -What is x in the equation? | -1 -5 | x = -1 or x = -5 | 1.0 -Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme talks about Iceland and Greenland. It's pointing out that despite their names, Iceland is not very icy and Greenland isn't very green. | 0.4 -Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme is using humor to point out the misleading nature of Iceland's and Greenland's names. Iceland, despite its name, has lush green landscapes while Greenland is mostly covered in ice and snow. The text 'This is why I have trust issues' is a playful way to suggest that these contradictions can lead to distrust or confusion. The humor in this meme is derived from the unexpected contrast between the names of the countries and their actual physical characteristics. | 1.0 -""" - - -def get_chat_response(prompt, model=GPT_EVAL_MODEL_NAME, temperature=0.0, max_tokens=128, patience=3, sleep_time=5): - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json", - } - - messages = [ - {"role": "user", "content": prompt}, - ] - - payload = { - "model": model, - "messages": messages, - "temperature": temperature, - "max_tokens": max_tokens, - } - - while patience > 0: - patience -= 1 - try: - response = requests.post( - API_URL, - headers=headers, - json=payload, - timeout=60, - ) - response.raise_for_status() - response_data = response.json() - - content = response_data["choices"][0]["message"]["content"].strip() - if content != "": - return content, response_data["model"] - - except Exception as e: - eval_logger.error(f"Error: {e}") - if "Rate limit" in str(e): - eval_logger.info("Sleeping due to rate limit...") - time.sleep(sleep_time) - eval_logger.info(f"Retrying...Patience left: {patience}") - - return "", "" - - -def mmvet_doc_to_visual(doc): - if doc["image"] is None: - return [] - return [doc["image"].convert("RGB")] - - -def mmvet_process_results(doc, results): - # get pred and ground truth here - pred = results[0] - question = doc["question"] - answer = doc["answer"] - gpt_query_prompt = f"{MM_VET_PROMPT}\n{question} | {answer.replace('', ' ').replace('', ' ')} | {pred} |" - grade_sample_run_complete = False - temperature = 0.0 - - while not grade_sample_run_complete: - content, model_name = get_chat_response( - gpt_query_prompt, - temperature=temperature, - ) - if content: - try: - content = content.split(" ")[0].strip() - score = float(content) - if 0.0 <= score <= 1.0: - grade_sample_run_complete = True - except ValueError: - time.sleep(5) - temperature += 0.5 - eval_logger.info(f"Sleep 5 secs, {doc['question_id']} try again with increased temperature {temperature}.") - content, model_name = get_chat_response( - gpt_query_prompt, - temperature=temperature, - ) - if temperature >= 2: # Assuming a max temperature threshold - score = 0.0 - grade_sample_run_complete = True - eval_logger.info(f"Reach to max trials, {doc['question_id']} failed to get a score.") - else: - score = 0.0 - grade_sample_run_complete = True - eval_logger.info(f"{doc['question_id']} failed to get a score.") - - return { - f"gpt_eval_score": { - "question_id": doc["question_id"], - "question": doc["question"], - "gt_answer": doc["answer"], - "capabilities": doc["capability"], - "pred_answer": pred, - "score": score, - "eval_model": model_name, - } - } - - -cap_columns = pd.DataFrame(["rec", "ocr", "know", "gen", "spat", "math"]) -cap_details_columns = pd.DataFrame( - [ - "rec_gen_know", - "rec", - "spat_ocr", - "spat_math_ocr", - "rec_spat", - "ocr", - "math_ocr", - "rec_know", - "rec_spat_gen_ocr", - "rec_gen_ocr_know", - "rec_spat_ocr", - "rec_ocr", - "know_spat_ocr", - "rec_spat_know", - "spat_gen_ocr", - "rec_spat_math_ocr", - ] -) - - -def mmvet_aggregate_results(results): - """ - Args: - results: a list of values returned by process_results - Returns: - A score - """ - # Calculate the overall score - overall_score = sum([result["score"] for result in results]) / len(results) * 100 - eval_logger.info(f"Overall Score: {overall_score:.2f}") - - # Initialize dictionaries to store scores for each capability and detail - cap_scores = {cap: 0 for cap in cap_columns.squeeze().tolist()} - cap_details_scores = {detail: 0 for detail in cap_details_columns.squeeze().tolist()} - - # Count the number of results for each capability and detail - cap_counts = {cap: 0 for cap in cap_scores} - cap_details_counts = {detail: 0 for detail in cap_details_scores} - - # Aggregate scores for each capability and detail - for result in results: - for cap in cap_scores: - if cap in result["capabilities"]: - cap_scores[cap] += result["score"] - cap_counts[cap] += 1 - for detail in cap_details_scores: - detail_set = set(detail.split("_")) - result_detail_set = set(result["capabilities"].split(",")) - if detail_set == result_detail_set: - cap_details_scores[detail] += result["score"] - cap_details_counts[detail] += 1 - - # Calculate the average score for each capability - for cap in cap_scores: - if cap_counts[cap] > 0: - cap_scores[cap] = cap_scores[cap] / cap_counts[cap] * 100 - eval_logger.info(f"Score for {cap}: {cap_scores[cap]:.2f}") - - # Calculate the average score for each detailed capability - for detail in cap_details_scores: - if cap_details_counts[detail] > 0: - cap_details_scores[detail] = cap_details_scores[detail] / cap_details_counts[detail] * 100 - eval_logger.info(f"Score for {detail}: {cap_details_scores[detail]:.2f}") - - return overall_score - - -def doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - return doc["question"] - question = doc["question"] - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") - - return f"{pre_prompt}{question}{post_prompt}" diff --git a/lmms_eval/tasks/multidocvqa/multidocvqa.yaml b/lmms_eval/tasks/multidocvqa/multidocvqa.yaml deleted file mode 100644 index 3572b69..0000000 --- a/lmms_eval/tasks/multidocvqa/multidocvqa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: multidocvqa -task: -- multidocvqa_val -- multidocvqa_test diff --git a/lmms_eval/tasks/multidocvqa/multidocvqa_test.yaml b/lmms_eval/tasks/multidocvqa/multidocvqa_test.yaml deleted file mode 100644 index d33c5a5..0000000 --- a/lmms_eval/tasks/multidocvqa/multidocvqa_test.yaml +++ /dev/null @@ -1,20 +0,0 @@ -dataset_path: lmms-lab/MP-DocVQA -task: "multidocvqa_test" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.multidocvqa_doc_to_visual -doc_to_text: !function utils.multidocvqa_doc_to_text -doc_to_target: "answers" -generation_kwargs: - max_new_tokens: 32 - temperature: 0 - do_sample: False -process_results: !function utils.multidocvqa_process_test_results_for_submission -metric_list: - - metric: submission - aggregation: !function utils.multidocvqa_test_aggregate_results_for_submission -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer the question using a single word or phrase." - \ No newline at end of file diff --git a/lmms_eval/tasks/multidocvqa/multidocvqa_val.yaml b/lmms_eval/tasks/multidocvqa/multidocvqa_val.yaml deleted file mode 100644 index b4f5238..0000000 --- a/lmms_eval/tasks/multidocvqa/multidocvqa_val.yaml +++ /dev/null @@ -1,23 +0,0 @@ -dataset_path: lmms-lab/MP-DocVQA -task: "multidocvqa_val" -test_split: val -output_type: generate_until -doc_to_visual: !function utils.multidocvqa_doc_to_visual -doc_to_text: !function utils.multidocvqa_doc_to_text -doc_to_target: "answers" -generation_kwargs: - max_new_tokens: 32 - temperature: 0 - do_sample: False -process_results: !function utils.multidocvqa_process_results -metric_list: - - metric: anls - aggregation: !function utils.multidocvqa_aggregate_results_anls - higher_is_better: true - - metric: accuracy - aggregation: !function utils.multidocvqa_aggregate_results_accuracy - higher_is_better: true -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer the question using a single word or phrase." diff --git a/lmms_eval/tasks/multidocvqa/utils.py b/lmms_eval/tasks/multidocvqa/utils.py deleted file mode 100644 index 10fd85e..0000000 --- a/lmms_eval/tasks/multidocvqa/utils.py +++ /dev/null @@ -1,116 +0,0 @@ -import os -import re -import ast -import json -import logging -from lmms_eval.api.metrics import levenshtein_distance -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -lmms_logger = logging.getLogger("lmms-eval") - - -def multidocvqa_doc_to_text(doc, model_specific_prompt_kwargs): - question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] - - return f"{pre_prompt}{question}{post_prompt}" - - -def multidocvqa_doc_to_visual(doc): - return [doc[f"image_{i}"].convert("RGB") for i in range(1, 21) if doc[f"image_{i}"] is not None] - - -def multidocvqa_process_results(doc, results): - pred_answer = results[0] - answer = ast.literal_eval(doc["answers"]) - - return {"anls": {"questionId": int(doc["questionId"]), "answer": answer, "pred_answer": pred_answer}, "accuracy": {"questionId": int(doc["questionId"]), "answer": answer, "pred_answer": pred_answer}} - - -def multidocvqa_aggregate_results_anls(results): - keys = {k for result in results for k in result} - results = {key: [result.get(key, None) for result in results] for key in keys} - evaluator = Evaluator(case_sensitive=False) - metric = evaluator.get_metrics(results["answer"], results["pred_answer"]) - - return sum(metric["anls"]) / len(metric["anls"]) - - -def multidocvqa_aggregate_results_accuracy(results): - keys = {k for result in results for k in result} - results = {key: [result.get(key, None) for result in results] for key in keys} - evaluator = Evaluator(case_sensitive=False) - metric = evaluator.get_metrics(results["answer"], results["pred_answer"]) - - return sum(metric["accuracy"]) / len(metric["accuracy"]) - - -def multidocvqa_process_test_results_for_submission(doc, results): - answer = results[0] - return {"submission": {"questionId": int(doc["questionId"]), "answer": answer, "answer_page": None}} - - -def multidocvqa_test_aggregate_results_for_submission(results, args): - path = generate_submission_file("multidocvqa_test_for_submission.json", args) - with open(path, "w") as f: - json.dump(results, f) - lmms_logger.info(f"Results saved to {path}.") - - -################## -# Helper functions -################## - - -class Evaluator: - def __init__(self, case_sensitive=False): - self.case_sensitive = case_sensitive - self.get_edit_distance = levenshtein_distance - self.anls_threshold = 0.5 - - def get_metrics(self, gt_answers, preds): - batch_accuracy = [] - batch_anls = [] - for batch_idx in range(len(preds)): - gt = [self._preprocess_str(gt_elm) for gt_elm in gt_answers[batch_idx]] - pred = self._preprocess_str(preds[batch_idx]) - - batch_accuracy.append(self._calculate_accuracy(gt, pred)) - batch_anls.append(self._calculate_anls(gt, pred)) - - return {"accuracy": batch_accuracy, "anls": batch_anls} - - def _preprocess_str(self, string): - if not self.case_sensitive: - string = string.lower() - - return string.strip() - - def _calculate_accuracy(self, gt, pred): - if pred == "none": - return 0 - - for gt_elm in gt: - if gt_elm == pred: - return 1 - - return 0 - - def _calculate_anls(self, gt, pred): - if len(pred) == 0: - return 0 - - if pred == "none": - return 0 - - answers_similarity = [1 - self.get_edit_distance(gt_elm, pred) / max(len(gt_elm), len(pred)) for gt_elm in gt] - max_similarity = max(answers_similarity) - - anls = max_similarity if max_similarity >= self.anls_threshold else 0 - return anls - - -if __name__ == "__main__": - print("-----------------") - multidocvqa_aggregate_results_anls([{"questionId": 1, "answer": ["answer"], "pred_answer": "pred_answer"}, {"questionId": 2, "answer": ["nswer"], "pred_answer": "nswer"}]) diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/README.md b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/README.md deleted file mode 100644 index 8b13789..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/README.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/_default_template.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/_default_template.yaml deleted file mode 100644 index 3ae9260..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/_default_template.yaml +++ /dev/null @@ -1,35 +0,0 @@ -test_split: train -output_type: generate_until -doc_to_visual: !function utils.llava_doc_to_visual -doc_to_text: !function utils.llava_doc_to_text -doc_to_target: "gpt_answer" -generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.llava_process_results -metric_list: - - metric: gpt_eval_llava_all - aggregation: !function utils.llava_all_aggregation - higher_is_better: true - - metric: gpt_eval_llava_conv - aggregation: !function utils.llava_conv_aggregation - higher_is_better: true - - metric: gpt_eval_llava_detail - aggregation: !function utils.llava_detail_aggregation - higher_is_better: true - - metric: gpt_eval_llava_complex - aggregation: !function utils.llava_complex_aggregation - higher_is_better: true -metadata: - version: 0.0 - gpt_eval_model_name: "gpt-4-0613" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "" \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/arabic_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/arabic_llava_in_the_wild.yaml deleted file mode 100644 index 8e36630..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/arabic_llava_in_the_wild.yaml +++ /dev/null @@ -1,6 +0,0 @@ -dataset_path: "gagan3012/multilingual-llava-bench" -dataset_kwargs: - config: arabic - token: True -task: "llava_in_the_wild_arabic" -include: _default_template.yaml \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/bengali_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/bengali_llava_in_the_wild.yaml deleted file mode 100644 index cabe38a..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/bengali_llava_in_the_wild.yaml +++ /dev/null @@ -1,6 +0,0 @@ -dataset_path: "gagan3012/multilingual-llava-bench" -dataset_kwargs: - config: bengali - token: True -task: "llava_in_the_wild_bengali" -include: _default_template.yaml \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/chinese_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/chinese_llava_in_the_wild.yaml deleted file mode 100644 index 3abda8a..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/chinese_llava_in_the_wild.yaml +++ /dev/null @@ -1,6 +0,0 @@ -dataset_path: "gagan3012/multilingual-llava-bench" -dataset_kwargs: - config: chinese - token: True -task: "llava_in_the_wild_chinese" -include: _default_template.yaml \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/french_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/french_llava_in_the_wild.yaml deleted file mode 100644 index cd30f2c..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/french_llava_in_the_wild.yaml +++ /dev/null @@ -1,6 +0,0 @@ -dataset_path: "gagan3012/multilingual-llava-bench" -dataset_kwargs: - config: french - token: True -task: "llava_in_the_wild_french" -include: _default_template.yaml \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/hindi_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/hindi_llava_in_the_wild.yaml deleted file mode 100644 index 45dc0d9..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/hindi_llava_in_the_wild.yaml +++ /dev/null @@ -1,6 +0,0 @@ -dataset_path: "gagan3012/multilingual-llava-bench" -dataset_kwargs: - config: hindi - token: True -task: "llava_in_the_wild_hindi" -include: _default_template.yaml \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/japanese_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/japanese_llava_in_the_wild.yaml deleted file mode 100644 index ac4083a..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/japanese_llava_in_the_wild.yaml +++ /dev/null @@ -1,6 +0,0 @@ -dataset_path: "gagan3012/multilingual-llava-bench" -dataset_kwargs: - config: japanese - token: True -task: "llava_in_the_wild_japanese" -include: _default_template.yaml \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/rule.json b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/rule.json deleted file mode 100644 index 26c7f4e..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/rule.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "coding": {"role": "Assistant", "prompt": "Your task is to evaluate the coding abilities of the above two assistants. They have been asked to implement a program to solve a given problem. Please review their code submissions, paying close attention to their problem-solving approach, code structure, readability, and the inclusion of helpful comments.\n\nPlease ensure that the assistants' submissions:\n\n1. Correctly implement the given problem statement.\n2. Contain accurate and efficient code.\n3. Include clear and concise comments that explain the code's logic and functionality.\n4. Adhere to proper coding standards and best practices.\n\nOnce you have carefully reviewed both submissions, provide detailed feedback on their strengths and weaknesses, along with any suggestions for improvement. You should first output a single line containing two scores on the scale of 1-10 (1: no code/no sense; 10: perfect) for Assistant 1 and 2, respectively. Then give extra comments starting from the next line."}, - "math": {"role": "Assistant", "prompt": "We would like to request your feedback on the mathematical proficiency of two AI assistants regarding the given user question.\nFirstly, please solve the problem independently, without referring to the answers provided by Assistant 1 and Assistant 2.\nAfterward, please examine the problem-solving process of Assistant 1 and Assistant 2 step-by-step to ensure their correctness, identifying any incorrect steps if present. Your evaluation should take into account not only the answer but also the problem-solving steps.\nFinally, please output a Python tuple containing two numerical scores for Assistant 1 and Assistant 2, ranging from 1 to 10, respectively. If applicable, explain the reasons for any variations in their scores and determine which assistant performed better."}, - "default": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above.\nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "conv": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "detail": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "complex": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with five descriptive sentences describing the same image and the bounding box coordinates of each object in the scene. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "llava_bench_conv": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "llava_bench_detail": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."}, - "llava_bench_complex": {"role": "Assistant", "prompt": "We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above. The user asks the question on observing an image. For your reference, the visual content in the image is represented with a few sentences describing the image. \nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.\nIn the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."} -} \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/russian_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/russian_llava_in_the_wild.yaml deleted file mode 100644 index 31e29ed..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/russian_llava_in_the_wild.yaml +++ /dev/null @@ -1,6 +0,0 @@ -dataset_path: "gagan3012/multilingual-llava-bench" -dataset_kwargs: - config: russian - token: True -task: "llava_in_the_wild_russian" -include: _default_template.yaml \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/spanish_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/spanish_llava_in_the_wild.yaml deleted file mode 100644 index ac614bd..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/spanish_llava_in_the_wild.yaml +++ /dev/null @@ -1,6 +0,0 @@ -dataset_path: "gagan3012/multilingual-llava-bench" -dataset_kwargs: - config: spanish - token: True -task: "llava_in_the_wild_spanish" -include: _default_template.yaml \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/urdu_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/urdu_llava_in_the_wild.yaml deleted file mode 100644 index c30dda1..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/urdu_llava_in_the_wild.yaml +++ /dev/null @@ -1,6 +0,0 @@ -dataset_path: "gagan3012/multilingual-llava-bench" -dataset_kwargs: - config: urdu - token: True -task: "llava_in_the_wild_urdu" -include: _default_template.yaml \ No newline at end of file diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/utils.py b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/utils.py deleted file mode 100644 index 6666de4..0000000 --- a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/utils.py +++ /dev/null @@ -1,197 +0,0 @@ -import json -import logging -import os -import requests -import numpy as np -import openai -from openai import OpenAI -import time -import yaml -from pathlib import Path -from copy import deepcopy - -eval_logger = logging.getLogger("lmms-eval") -NUM_SECONDS_TO_SLEEP = 5 - -LLAVA_W_METRICS = ["gpt_eval_llava_conv", "gpt_eval_llava_detail", "gpt_eval_llava_complex"] - -rule_dict = json.load(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "rule.json"), "r")) - -with open(Path(__file__).parent / "_default_template.yaml", "r") as f: - raw_data = f.readlines() - safe_data = [] - for i, line in enumerate(raw_data): - # remove function definition since yaml load cannot handle it - if "!function" not in line: - safe_data.append(line) - - config = yaml.safe_load("".join(safe_data)) - -GPT_EVAL_MODEL_NAME = config["metadata"]["gpt_eval_model_name"] - -API_TYPE = os.getenv("API_TYPE", "openai") - -if API_TYPE == "openai": - API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions") - API_KEY = os.getenv("OPENAI_API_KEY", "YOUR_API_KEY") - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json", - } -elif API_TYPE == "azure": - API_URL = os.getenv("AZURE_ENDPOINT", "https://api.cognitive.microsoft.com/sts/v1.0/issueToken") - API_KEY = os.getenv("AZURE_API_KEY", "YOUR_API_KEY") - headers = { - "api-key": API_KEY, - "Content-Type": "application/json", - } - - -def get_eval(content: str, max_tokens: int, retries: int = 5): - global headers - - messages = [ - { - "role": "system", - "content": "You are a helpful and precise assistant for checking the quality of the answer.", - }, - {"role": "user", "content": content}, - ] - - payload = { - "model": GPT_EVAL_MODEL_NAME, - "messages": messages, - "temperature": 0.2, - "max_tokens": max_tokens, - } - - for attempt in range(retries): - try: - response = requests.post(API_URL, headers=headers, json=payload, timeout=60) - response.raise_for_status() - response_data = response.json() - - content = response_data["choices"][0]["message"]["content"].strip() - if content != "": - return content, response_data["model"] - break # If successful, break out of the loop - - except Exception as e: - eval_logger.info(f"Attempt {attempt + 1} failed with error: {e}") - if attempt < retries: # If we have retries left, sleep and then continue to next attempt - time.sleep(NUM_SECONDS_TO_SLEEP) - else: # If this was the last attempt, log and return empty - eval_logger.error(f"All {retries} attempts failed. Last error message: {e}") - return "", "" - return "", "" - - -def parse_score(review): - try: - score_pair = review.split("\n")[0] - score_pair = score_pair.replace(",", " ") - sp = score_pair.split(" ") - if len(sp) == 2: - return [float(sp[0]), float(sp[1])] - else: - eval_logger.debug(f"Can not split: {review}. Returning [-1, -1]") - return [-1, -1] - except Exception as e: - eval_logger.debug(f"Error: {e}. Returning [-1, -1]") - return [-1, -1] - - -def llava_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def llava_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = model_specific_prompt_kwargs.get("pre_prompt", "") - post_prompt = model_specific_prompt_kwargs.get("post_prompt", "") - return f"{pre_prompt}{doc['question']}{post_prompt}" - - -def llava_process_results(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case coco_bleu), value: metric value - """ - try: - question = doc.get("question", "") - ans1 = doc.get("gpt_answer", "") - ans2 = result[0] if result else "" - captions = doc.get("caption", []) - context = "\n".join(captions) if isinstance(captions, list) else captions - category = "llava_bench_" + doc.get("category", "") - rule = rule_dict.get(category, {}) - prompt = rule.get("prompt", "") - role = rule.get("role", "user") - content = f"[Context]\n{context}\n\n" f"[Question]\n{question}\n\n" f"[{role} 1]\n{ans1}\n\n[End of {role} 1]\n\n" f"[{role} 2]\n{ans2}\n\n[End of {role} 2]\n\n" f"[System]\n{prompt}\n\n" - - review, model_name = get_eval(content, 1024) - scores = parse_score(review) - except Exception as e: - eval_logger.error(f"Error for Question ID: {doc.get('question_id', 'Unknown')}: {e}") - review = "Failed to Get a Proper Review." - model_name = "Failed Request" - scores = [-1, -1] - - metric = f"gpt_eval_llava_{doc.get('category', 'all')}" - category_review_dict = {"question": question, "ans1": ans1, "ans2": ans2, "context": context, "category": category, "review": review, "scores": scores, "eval_model": model_name, "content": content} - - non_category_review_dict = deepcopy(category_review_dict) - non_category_review_dict["scores"] = [-999, -999] - - data_dict = {} - for m in LLAVA_W_METRICS: - if m == metric: - data_dict[m] = category_review_dict - else: - data_dict[m] = non_category_review_dict - data_dict["gpt_eval_llava_all"] = category_review_dict - - # return {"gpt_eval_llava_all": review_dict} - return data_dict - - -def llava_conv_aggregation(results): - return llava_aggregation(results, "conv") - - -def llava_complex_aggregation(results): - return llava_aggregation(results, "complex") - - -def llava_detail_aggregation(results): - return llava_aggregation(results, "detail") - - -def llava_all_aggregation(results): - return llava_aggregation(results, "all") - - -def llava_aggregation(results, category): - try: - scores = [] - for result in results: - if -999 in result["scores"]: - continue - scores.append(result["scores"]) - - stats = np.asarray(scores).mean(0).tolist() - stats = [round(x, 3) for x in stats] - # gpt4_score_percentage = stats[0] * 10 - # model_score_percentage = stats[1] * 10 - # eval_logger.info(f"Category: {category}") - # eval_logger.info(f"GPT4 Score: {gpt4_score_percentage:.1f}%") - # eval_logger.info(f"Model Score: {model_score_percentage:.1f}%") - # eval_logger.info("=========================") - return round(stats[1] / stats[0] * 100, 1) - except Exception as e: - eval_logger.info(f"Error in llava_aggregation: {e}, and in category: {category}") - return None diff --git a/lmms_eval/tasks/nocaps/_default_template_nocaps_yaml b/lmms_eval/tasks/nocaps/_default_template_nocaps_yaml deleted file mode 100644 index 12fdbbc..0000000 --- a/lmms_eval/tasks/nocaps/_default_template_nocaps_yaml +++ /dev/null @@ -1,3 +0,0 @@ -model_specific_prompt_kwargs: - default: - prompt: "Provide a one-sentence caption for the provided image." \ No newline at end of file diff --git a/lmms_eval/tasks/nocaps/nocaps.yaml b/lmms_eval/tasks/nocaps/nocaps.yaml deleted file mode 100644 index 579b7ae..0000000 --- a/lmms_eval/tasks/nocaps/nocaps.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group : nocaps -task: - - nocaps_test - - nocaps_val \ No newline at end of file diff --git a/lmms_eval/tasks/nocaps/nocaps_test.yaml b/lmms_eval/tasks/nocaps/nocaps_test.yaml deleted file mode 100644 index 9f21ce2..0000000 --- a/lmms_eval/tasks/nocaps/nocaps_test.yaml +++ /dev/null @@ -1,25 +0,0 @@ -dataset_path: lmms-lab/NoCaps -dataset_kwargs: - token: True -task : "nocaps_test" -group : "nocaps_caption" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.nocaps_doc_to_visual -doc_to_text: !function utils.nocaps_doc_to_text -doc_to_target: "annotations_captions" -generation_kwargs: - max_new_tokens: 64 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.nocaps_test_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: nocaps_passthrough - aggregation : !function utils.nocaps_test_aggregation_result - higher_is_better : true -metadata: - - version: 0.0 -include: _default_template_nocaps_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/nocaps/nocaps_val.yaml b/lmms_eval/tasks/nocaps/nocaps_val.yaml deleted file mode 100644 index 048066a..0000000 --- a/lmms_eval/tasks/nocaps/nocaps_val.yaml +++ /dev/null @@ -1,46 +0,0 @@ -dataset_path: lmms-lab/NoCaps -dataset_kwargs: - token: True -task: "nocaps_val" -group : "nocaps_caption" -test_split: validation -output_type: generate_until -doc_to_visual: !function utils.nocaps_doc_to_visual -doc_to_text: !function utils.nocaps_doc_to_text -doc_to_target: "annotations_captions" -generation_kwargs: - max_new_tokens: 64 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.nocaps_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: nocaps_Bleu_4 - aggregation : !function utils.nocaps_bleu4 - higher_is_better : true - - metric: nocaps_Bleu_3 - aggregation : !function utils.nocaps_bleu3 - higher_is_better : true - - metric: nocaps_Bleu_2 - aggregation : !function utils.nocaps_bleu2 - higher_is_better : true - - metric: nocaps_Bleu_1 - aggregation : !function utils.nocaps_bleu1 - higher_is_better : true - - metric: nocaps_METEOR - aggregation : !function utils.nocaps_meteor - higher_is_better : true - - metric: nocaps_ROUGE_L - aggregation : !function utils.nocaps_rougel - higher_is_better : true - - metric: nocaps_CIDEr - aggregation : !function utils.nocaps_cider - higher_is_better : true - #- metric: nocaps_SPICE - # aggregation : !function utils.nocaps_spice - # higher_is_better : true -metadata: - - version: 0.0 -include: _default_template_nocaps_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/nocaps/utils.py b/lmms_eval/tasks/nocaps/utils.py deleted file mode 100644 index f645b1c..0000000 --- a/lmms_eval/tasks/nocaps/utils.py +++ /dev/null @@ -1,153 +0,0 @@ -import os -import json -from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice -from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer -from pycocotools.coco import COCO - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -import logging - -eval_logger = logging.getLogger("lmms-eval") - -dir_name = os.path.dirname(os.path.abspath(__file__)) - -NOCAPS_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"] # , "SPICE"] - - -def nocaps_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def nocaps_doc_to_text(doc, model_specific_prompt_kwargs=None): - # question = "Please carefully observe the image and come up with a caption for the image" - return model_specific_prompt_kwargs["prompt"] - - -def nocaps_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name, value: metric value - """ - pred = result[0] - # The question id in our dataset is the image file itself - image_id = doc["image_id"] - - data_dict = {"answer": doc["annotations_captions"], "pred": pred, "image_id": image_id} - - return {f"nocaps_{metric}": data_dict for metric in NOCAPS_METRICS} - - -def nocaps_aggregation_result(results, metric, args=None): - scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr")]#, (Spice(), "SPICE")] - scorers_dict = {s[1]: s for s in scorers} - - stored_results = [] - # In order to make the coco eval tools to successfully create index - # We need at least two dict in the dataset - # 'annotation' and 'images' - # 'annotation' exactly reproduce the original annotation - # 'images' however only need the image id which is contained in the file name - dataset = {"annotations": [], "images": []} - idx = 0 - for result in results: - stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]}) - for a in result["answer"]: - dataset["annotations"].append({"image_id": int(result["image_id"]), "caption": a, "id": idx}) - idx += 1 - dataset["images"].append({"id": result["image_id"]}) - - coco = COCO() - # Manually create index here - coco.dataset = dataset - coco.createIndex() - - nocaps_result = coco.loadRes(stored_results) - nocaps_eval = COCOEvalCap(coco, nocaps_result) - - imgIds = nocaps_eval.params["image_id"] - gts = {} - res = {} - for imgId in imgIds: - gts[imgId] = nocaps_eval.coco.imgToAnns[imgId] - res[imgId] = nocaps_eval.cocoRes.imgToAnns[imgId] - - eval_logger.info("tokenization...") - tokenizer = PTBTokenizer() - gts = tokenizer.tokenize(gts) - res = tokenizer.tokenize(res) - - eval_logger.info(f"Computing {metric} scores...") - - score, scores = scorers_dict[metric][0].compute_score(gts, res) - # When metric is one of the Bleu, score will be a list - if type(score) == list: - n = int(metric.split("_")[-1]) - score = score[n - 1] - - path = generate_submission_file(f"nocaps_val_{metric}_scores.json", args) - eval_logger.info("Storing prediction that can be submitted to the server ...") - with open(path, "w") as f: - json.dump(stored_results, f, indent=4) - eval_logger.info(f"Your result has been saved to {path}.") - - return score - - -def nocaps_bleu4(results, args=None): - return nocaps_aggregation_result(results, "Bleu_4", args) - - -def nocaps_bleu3(results, args=None): - return nocaps_aggregation_result(results, "Bleu_3", args) - - -def nocaps_bleu2(results, args=None): - return nocaps_aggregation_result(results, "Bleu_2", args) - - -def nocaps_bleu1(results, args=None): - return nocaps_aggregation_result(results, "Bleu_1", args) - - -def nocaps_meteor(results, args=None): - return nocaps_aggregation_result(results, "METEOR", args) - - -def nocaps_rougel(results, args=None): - return nocaps_aggregation_result(results, "ROUGE_L", args) - - -def nocaps_cider(results, args=None): - return nocaps_aggregation_result(results, "CIDEr", args) - - -def nocaps_spice(results, args=None): - return nocaps_aggregation_result(results, "SPICE", args) - - -def nocaps_test_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case nocaps_passthrough), value: metric value - """ - return {"nocaps_passthrough": {"pred": result[0], "image_id": doc["image_id"]}} - - -def nocaps_test_aggregation_result(results, args=None): - stored_results = [] - for result in results: - stored_results.append({"image_id": int(result["image_id"]), "caption": result["pred"]}) - - path = generate_submission_file("nocaps_captions_nocaps_test_alg_results.json", args) - eval_logger.info("Storing prediction that can be submitted to the server ...") - with open(path, "w") as f: - json.dump(stored_results, f, indent=4) - - eval_logger.info(f"Your test result has been stored in {path}. Make sure you also have the val result stored to submit to the server on https://codalab.lisn.upsaclay.fr/competitions/7404#participate.") diff --git a/lmms_eval/tasks/ocrbench/ocrbench.yaml b/lmms_eval/tasks/ocrbench/ocrbench.yaml deleted file mode 100644 index 7957e7b..0000000 --- a/lmms_eval/tasks/ocrbench/ocrbench.yaml +++ /dev/null @@ -1,22 +0,0 @@ -dataset_path: echo840/OCRBench -dataset_kwargs: - token: True -task: "ocrbench" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.ocrbench_doc_to_visual -doc_to_text: !function utils.ocrbench_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.ocrbench_process_results -metric_list: - - metric: ocrbench_accuracy - aggregation: !function utils.ocrbench_aggregate_accuracy - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/ocrbench/upload_ocrbench.py b/lmms_eval/tasks/ocrbench/upload_ocrbench.py deleted file mode 100644 index 528ab15..0000000 --- a/lmms_eval/tasks/ocrbench/upload_ocrbench.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import json - -import datasets -from PIL import Image as PIL_Image -import json -from uuid import uuid4 -from datasets import Dataset, Features -import pandas as pd -from tqdm import tqdm -import io - -# Find for instance the citation on arxiv or on the dataset repo/website -_CITATION = """https://arxiv.org/abs/2305.07895""" -_DESCRIPTION = "OCRBench is a comprehensive evaluation benchmark designed to assess the OCR capabilities of Large Multimodal Models." - - -def image2byte(image): - img_bytes = io.BytesIO() - image.save(img_bytes, format="JPEG") - image_bytes = img_bytes.getvalue() - return image_bytes - - -def get_builder_config(VERSION): - builder_config = [ - datasets.BuilderConfig( - name=f"ocrbench", - version=VERSION, - description=f"ocrbench", - ) - ] - return builder_config - - -ocrbench_json = "pathto/OCRBench/OCRBench.json" -img_dir = "pathto/OCRBench_Images/" - -dataset_features = Features( - { - "dataset": datasets.Value("string"), - "question": datasets.Value("string"), - "question_type": datasets.Value("string"), - "answer": datasets.features.Sequence(datasets.Value("string")), - "image": datasets.Image(), - } -) - -df_items = { - "dataset": [], - "question": [], - "question_type": [], - "answer": [], - "image": [], -} -# img_feature = datasets.Image(decode=False) -with open(ocrbench_json, "r") as f: - data = json.load(f) -for i in tqdm(range(len(data))): - dataset_name = data[i]["dataset_name"] - image_path = img_dir + data[i]["image_path"] - question = data[i]["question"] - answers = data[i]["answers"] - question_type = data[i]["type"] - if type(answers) == str: - answers = [answers] - img = PIL_Image.open(image_path).convert("RGB") - byte_data = image2byte(img) - image = {"bytes": byte_data, "path": ""} - df_items["image"].append(image) - df_items["question"].append(str(question)) - df_items["answer"].append(answers) - df_items["question_type"].append(str(question_type)) - df_items["dataset"].append(str(dataset_name)) - -df_items = pd.DataFrame(df_items) -df_items.head() -dataset = Dataset.from_pandas(df_items, features=dataset_features) -hub_dataset_path = "echo840/OCRBench" -dataset.push_to_hub(repo_id=hub_dataset_path, split="test") diff --git a/lmms_eval/tasks/ocrbench/utils.py b/lmms_eval/tasks/ocrbench/utils.py deleted file mode 100644 index c8c8c65..0000000 --- a/lmms_eval/tasks/ocrbench/utils.py +++ /dev/null @@ -1,103 +0,0 @@ -import logging - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -logger = logging.getLogger("lmms-eval") - -# Add the following functions to your existing utils.py file -OCRBench_score = { - "Regular Text Recognition": 0, - "Irregular Text Recognition": 0, - "Artistic Text Recognition": 0, - "Handwriting Recognition": 0, - "Digit String Recognition": 0, - "Non-Semantic Text Recognition": 0, - "Scene Text-centric VQA": 0, - "Doc-oriented VQA": 0, - "Key Information Extraction": 0, - "Handwritten Mathematical Expression Recognition": 0, -} - - -def ocrbench_doc_to_visual(doc): - # Assuming the 'doc' dictionary has a key 'image' with image data - return [doc["image"].convert("RGB")] - - -def ocrbench_doc_to_text(doc): - # Assuming the 'doc' dictionary has a key 'question' with the question text - question = doc["question"].strip() - return f"{question}" - - -def ocrbench_process_results(doc, results): - pred = results[0].lower().strip() - gt_ans = doc["answer"] - dataset_name = doc["dataset"] - - score = 0 - if dataset_name == "HME100k": - if type(gt_ans) == list: - for j in range(len(gt_ans)): - answer = gt_ans[j].strip().replace("\n", " ").replace(" ", "") - predict = pred.strip().replace("\n", " ").replace(" ", "") - if answer in predict: - score = 1 - else: - answer = gt_ans.strip().replace("\n", " ").replace(" ", "") - predict = pred.strip().replace("\n", " ").replace(" ", "") - if answer in predict: - score = 1 - else: - if type(gt_ans) == list: - for j in range(len(gt_ans)): - answer = gt_ans[j].lower().strip().replace("\n", " ") - predict = pred.lower().strip().replace("\n", " ") - if answer in predict: - score = 1 - else: - answer = gt_ans.lower().strip().replace("\n", " ") - predict = pred.lower().strip().replace("\n", " ") - if answer in predict: - score = 1 - return { - "ocrbench_accuracy": {"question_type": doc["question_type"], "score": score, "prediction": pred, "ground_truth": gt_ans}, - } - - -def ocrbench_aggregate_accuracy(results, args): - for result in results: - OCRBench_score[result["question_type"]] += result["score"] - recognition_score = ( - OCRBench_score["Regular Text Recognition"] - + OCRBench_score["Irregular Text Recognition"] - + OCRBench_score["Artistic Text Recognition"] - + OCRBench_score["Handwriting Recognition"] - + OCRBench_score["Digit String Recognition"] - + OCRBench_score["Non-Semantic Text Recognition"] - ) - Final_score = recognition_score + OCRBench_score["Scene Text-centric VQA"] + OCRBench_score["Doc-oriented VQA"] + OCRBench_score["Key Information Extraction"] + OCRBench_score["Handwritten Mathematical Expression Recognition"] - file_name = generate_submission_file("ocrbench_results.txt", args, subpath="results") - with open(file_name, "w") as f: - print("######################### OCRBench #############################", file=f) - print(f"Text Recognition(Total 300): {recognition_score}", file=f) - print("---------------- Details of Recognition Score ------------------", file=f) - print(f"Regular Text Recognition(Total 50): {OCRBench_score['Regular Text Recognition']}", file=f) - print(f"Irregular Text Recognition(Total 50): {OCRBench_score['Irregular Text Recognition']}", file=f) - print(f"Artistic Text Recognition(Total 50): {OCRBench_score['Artistic Text Recognition']}", file=f) - print(f"Handwriting Recognition(Total 50): {OCRBench_score['Handwriting Recognition']}", file=f) - print(f"Digit String Recognition(Total 50): {OCRBench_score['Digit String Recognition']}", file=f) - print(f"Non-Semantic Text Recognition(Total 50): {OCRBench_score['Non-Semantic Text Recognition']}", file=f) - print("----------------------------------------------------------------", file=f) - print(f"Scene Text-centric VQA(Total 200): {OCRBench_score['Scene Text-centric VQA']}", file=f) - print("----------------------------------------------------------------", file=f) - print(f"Doc-oriented VQA(Total 200): {OCRBench_score['Doc-oriented VQA']}", file=f) - print("----------------------------------------------------------------", file=f) - print(f"Key Information Extraction(Total 200): {OCRBench_score['Key Information Extraction']}", file=f) - print("----------------------------------------------------------------") - print(f"Handwritten Mathematical Expression Recognition(Total 100): {OCRBench_score['Handwritten Mathematical Expression Recognition']}", file=f) - print("--------------------- Final Score ------------------------------", file=f) - print(f"Final Score(Total 1000): {Final_score}", file=f) - logger.info(f"OCR Bench results saved to {file_name}") - # return {"Final Score":Final_score,"Text Recognition":recognition_score,'Scene Text-centric VQA':OCRBench_score['Scene Text-centric VQA'],'Doc-oriented VQA':OCRBench_score['Doc-oriented VQA'],'Key Information Extraction':OCRBench_score['Key Information Extraction'],'Handwritten Mathematical Expression Recognition':OCRBench_score['Handwritten Mathematical Expression Recognition']} - return Final_score diff --git a/lmms_eval/tasks/ok_vqa/_default_template_vqa_yaml b/lmms_eval/tasks/ok_vqa/_default_template_vqa_yaml deleted file mode 100644 index 8d74eb1..0000000 --- a/lmms_eval/tasks/ok_vqa/_default_template_vqa_yaml +++ /dev/null @@ -1,24 +0,0 @@ -dataset_path: lmms-lab/OK-VQA -output_type: generate_until -doc_to_visual: !function utils.ok_vqa_doc_to_visual -doc_to_text: !function utils.ok_vqa_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - - metric: submission - aggregation: !function utils.ok_vqa_aggreate_submissions - higher_is_better: true -process_results: !function utils.ok_vqa_process_results -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nWhen the provided information is insufficient, respond with 'Unanswerable'.\nAnswer the question using a single word or phrase." -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/ok_vqa/_generate_config.py b/lmms_eval/tasks/ok_vqa/_generate_config.py deleted file mode 100644 index a98f2f4..0000000 --- a/lmms_eval/tasks/ok_vqa/_generate_config.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import yaml - -splits = ["val2014"] -tasks = ["vqa"] - -if __name__ == "__main__": - dump_tasks = [] - for task in tasks: - for split in splits: - yaml_dict = {"group": f"ok_vqa", "task": f"ok_vqa_{split}", "include": f"_default_template_{task}_yaml", "test_split": split} - if split == "train": - yaml_dict.pop("group") - else: - dump_tasks.append(f"ok_vqa_{split}") - - save_path = f"./ok_vqa_{split}.yaml" - print(f"Saving to {save_path}") - with open(save_path, "w") as f: - yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False) - - group_dict = {"group": "ok_vqa", "task": dump_tasks} - - with open("./_ok_vqa.yaml", "w") as f: - yaml.dump(group_dict, f, default_flow_style=False, indent=4) diff --git a/lmms_eval/tasks/ok_vqa/_ok_vqa.yaml b/lmms_eval/tasks/ok_vqa/_ok_vqa.yaml deleted file mode 100644 index 3b6376f..0000000 --- a/lmms_eval/tasks/ok_vqa/_ok_vqa.yaml +++ /dev/null @@ -1,3 +0,0 @@ -group: ok_vqa -task: -- ok_vqa_val2014 \ No newline at end of file diff --git a/lmms_eval/tasks/ok_vqa/ok_vqa_val2014.yaml b/lmms_eval/tasks/ok_vqa/ok_vqa_val2014.yaml deleted file mode 100644 index 6bb1d74..0000000 --- a/lmms_eval/tasks/ok_vqa/ok_vqa_val2014.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: ok_vqa -task: ok_vqa_val2014 -test_split: val2014 -include: _default_template_vqa_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/ok_vqa/utils.py b/lmms_eval/tasks/ok_vqa/utils.py deleted file mode 100644 index 52faa11..0000000 --- a/lmms_eval/tasks/ok_vqa/utils.py +++ /dev/null @@ -1,70 +0,0 @@ -import re -import os -import json -import yaml -import pathlib -import logging -import datetime -import statistics - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file -from lmms_eval.tasks._task_utils.vqa_eval_metric import EvalAIAnswerProcessor - -eval_logger = logging.getLogger("lmms-eval") - - -def ok_vqa_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def ok_vqa_process_results(doc, result): - eval_ai_processor = EvalAIAnswerProcessor() - assert len(result) == 1, f"The result should be a list of length 1, but got {len(result)}." - resAns = eval_ai_processor(result[0]) - accuracy = 0 - - if "answers" in doc and doc["answers"] is not None: - gtAcc = [] - - for i in range(len(doc["answers"])): - doc["answers"][i] = eval_ai_processor(doc["answers"][i]) - - for i in range(len(doc["answers"])): - otherGTAns = [doc["answers"][j] for j in range(len(doc["answers"])) if i != j] - matchingAns = [item for item in otherGTAns if item == resAns] - acc = min(1, float(len(matchingAns)) / 3) - gtAcc.append(acc) - if gtAcc: - accuracy = statistics.mean(gtAcc) - else: - accuracy = 0 - - return { - "exact_match": accuracy, - "submission": { - "image": f"{doc['question_id']}.jpg", - "answer": resAns, - }, - } - - -def ok_vqa_doc_to_text(doc, model_specific_prompt_kwargs=None): - question = doc["question"] - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = "" - post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] - return f"{pre_prompt}{question}{post_prompt}" - - -def ok_vqa_aggreate_submissions(results, args): - now_date_time = datetime.datetime.now().strftime("%Y-%m%d-%H%M-%S") - file = f"ok_vqa-test-submission-{now_date_time}.json" - path = generate_submission_file(file, args) - with open(path, "w") as f: - json.dump(results, f) - print(f"Submission file saved to {path}") diff --git a/lmms_eval/tasks/olympiadbench/cn_utils.py b/lmms_eval/tasks/olympiadbench/cn_utils.py deleted file mode 100644 index 5151186..0000000 --- a/lmms_eval/tasks/olympiadbench/cn_utils.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import json -import datetime -from lmms_eval.tasks.olympiadbench.olympiadbench_evals import OlympiadBenchEvaluator -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -import logging - -eval_logger = logging.getLogger("lmms-eval") -dir_name = os.path.dirname(os.path.abspath(__file__)) - -try: - olympiadbench_evaluator = OlympiadBenchEvaluator() -except: - pass - - -def olympiadbench_doc_to_visual(doc): - return [image.convert("RGB") for image in doc["images"]] - - -def olympiadbench_doc_to_text(doc): - question = doc["question"] - subject = doc["subfield"] - mul_ans = doc["is_multiple_answer"] - if mul_ans is None: - mul_ans = False - ans_type = doc["answer_type"] - if ans_type == "Need_human_evaluate": - ans_type = "proof based" - - pre_prompt = f"以下是中国{subject}竞赛中的解答题。\n" - - post_prompt = "" - if not mul_ans: - post_prompt += f"答案类型为{ans_type}。\n" - else: - post_prompt += f"题目有多个答案，答案类型均为{ans_type}。\n" - post_prompt += "请根据题目的要求和所提供的信息计算得出答案。解答过程和结果中使用的变量和公式请使用LaTeX格式表示。请在最后以" - if not mul_ans: - post_prompt += '"所以最终答案是\\boxed{答案}。"\n' - else: - post_prompt += '"所以最终答案是\\boxed{用英⽂逗号连接的多个答案}。"\n' - - final_question = pre_prompt + question + "\n" + post_prompt - return final_question - - -def olympiadbench_process_results(doc, results): - precision = doc["error"] - is_proving = "TP" in doc["source"] - if precision is None: - precision = 0 - prediction = results[0].strip() - - if is_proving: - return {"submission": prediction} - else: - prediction = prediction.split("所以最终答案是")[-1] - prediction = prediction.replace('"', "").replace("\n", "").replace(" ", "").strip(".").strip("。") - accuracy = olympiadbench_evaluator.judge(prediction, doc["final_answer"][0], precision) - accuracy = int(accuracy) - return {"exact_match": accuracy} - - -def olympiadbench_aggregate_results(results, args): - now_date_time = datetime.datetime.now().strftime("%Y-%m%d-%H%M-%S") - submission_file_name = f"olympiadbench-test-cn-submission-{now_date_time}.json" - path = generate_submission_file(submission_file_name, args) - with open(path, "w") as f: - json.dump(results, f, ensure_ascii=False) - print(f"Submission file saved to {path}") \ No newline at end of file diff --git a/lmms_eval/tasks/olympiadbench/en_utils.py b/lmms_eval/tasks/olympiadbench/en_utils.py deleted file mode 100644 index e98c029..0000000 --- a/lmms_eval/tasks/olympiadbench/en_utils.py +++ /dev/null @@ -1,74 +0,0 @@ -import os -import json -import datetime -from lmms_eval.tasks.olympiadbench.olympiadbench_evals import OlympiadBenchEvaluator -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -import logging - -eval_logger = logging.getLogger("lmms-eval") -dir_name = os.path.dirname(os.path.abspath(__file__)) - -try: - olympiadbench_evaluator = OlympiadBenchEvaluator() -except: - pass - - -def olympiadbench_doc_to_visual(doc): - return [image.convert("RGB") for image in doc["images"]] - - -def olympiadbench_doc_to_text(doc): - question = doc["question"] - subject = doc["subfield"] - mul_ans = doc["is_multiple_answer"] - if mul_ans is None: - mul_ans = False - ans_type = doc["answer_type"] - if ans_type == "Need_human_evaluate": - ans_type = "proof based" - - pre_prompt = f"The following is a question from an International {subject} competition.\n" - - post_prompt = "" - if not mul_ans: - post_prompt += f"The answer of the question should be {ans_type}.\n" - else: - post_prompt += f"The question has multiple answers, each of them should be {ans_type}.\n" - post_prompt += ( - "Please calculate the answer according to the given requirements and the information provided. Please use LaTeX format to represent the variables and formulas used in the solution process and results. Please end your solution with " - ) - if not mul_ans: - post_prompt += '"So the final answer is \\boxed{answer}."\n' - else: - post_prompt += "So the final answer is \\boxed{multiple answers connected with commas}.\n" - - final_question = pre_prompt + question + "\n" + post_prompt - return final_question - - -def olympiadbench_process_results(doc, results): - precision = doc["error"] - is_proving = "TP" in doc["source"] - if precision is None: - precision = 0 - prediction = results[0].strip() - - if is_proving: - return {"submission": prediction} - else: - prediction = prediction.split("final answer is")[-1] - prediction = prediction.replace('"', "").replace("\n", "").replace(" ", "").strip(".").strip("。") - accuracy = olympiadbench_evaluator.judge(prediction, doc["final_answer"][0], precision) - accuracy = int(accuracy) - return {"exact_match": accuracy} - - -def olympiadbench_aggregate_results(results, args): - now_date_time = datetime.datetime.now().strftime("%Y-%m%d-%H%M-%S") - submission_file_name = f"olympiadbench-test-en-submission-{now_date_time}.json" - path = generate_submission_file(submission_file_name, args) - with open(path, "w") as f: - json.dump(results, f, ensure_ascii=False) - print(f"Submission file saved to {path}") \ No newline at end of file diff --git a/lmms_eval/tasks/olympiadbench/olympiadbench.yaml b/lmms_eval/tasks/olympiadbench/olympiadbench.yaml deleted file mode 100644 index 1580b15..0000000 --- a/lmms_eval/tasks/olympiadbench/olympiadbench.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: olympiadbench -task: -- olympiadbench_test_en -- olympiadbench_test_cn -metadata: - - version: 0.0 diff --git a/lmms_eval/tasks/olympiadbench/olympiadbench_evals.py b/lmms_eval/tasks/olympiadbench/olympiadbench_evals.py deleted file mode 100644 index 2e2de5f..0000000 --- a/lmms_eval/tasks/olympiadbench/olympiadbench_evals.py +++ /dev/null @@ -1,355 +0,0 @@ -import re -import sympy as sp - -import logging - -eval_logger = logging.getLogger("lmms-eval") - -try: - from sympy import simplify, Eq, sympify, Pow - from sympy.parsing.latex import parse_latex -except ImportError as e: - eval_logger.debug("Please install sympy package by running 'pip install sympy' if you want to use OlympiadBenchEvaluator.") -import math - -# how to use -# scorer = OlympiadBenchEvaluator() -# exp1 = "10^{10^{10^{10}}}" -# exp2 = "10^{10}" -# precision = 1e-4 -# res = scorer.judge(exp1, exp2, precision) - - -class OlympiadBenchEvaluator: - def __init__(self): - # Map of special symbols to their replacements - self.special_signal_map = { - "\\left": "", - "\\right": "", - "∶": ":", - "，": ",", - "$": "", - "\\approx": "=", - "\\simeq": "=", - "\\sim": "=", - "^\\prime": "'", - "^{\\prime}": "'", - "^\\circ": "", - "%": "", - } - self.pi = parse_latex("\\pi") - self.precision = 1e-8 # Default precision for comparison - - def split_by_comma(self, expr: str): - # Splits expressions by commas outside of brackets - in_bracket_num = 0 - splitted_expr = [] - start_idx = 0 - for i, char in enumerate(expr): - if char in ["(", "["]: - in_bracket_num += 1 - elif char in [")", "]"]: - in_bracket_num -= 1 - elif char == "," and in_bracket_num == 0: - splitted_expr.append(expr[start_idx:i].strip()) - start_idx = i + 1 - - if start_idx < len(expr): - splitted_expr.append(expr[start_idx:].strip()) - - return splitted_expr - - def trans_plus_minus_sign(self, expr_list: list): - # Translates plus-minus signs into separate expressions - new_expr_list = [] - for expr in expr_list: - if "\\pm" in expr: - new_expr_list.append(expr.replace("\\pm", "+")) - new_expr_list.append(expr.replace("\\pm", "-")) - else: - new_expr_list.append(expr) - - return new_expr_list - - def judge(self, expression1, expression2, precision=1e-8): - # Judge if two expressions are equal (expression1 is considered as the Ground Truth) - # Default precision is a list for supporting multiple expressions - precision = precision if isinstance(precision, list) else [precision] - - try: - expression1, expression2 = self.preprocess(expression1, expression2) - except: - return False - if expression1 == expression2: - # print("Exactly equal") - return True - - # Remove Chinese characters from the string, as answers like "yes" or "no" in Chinese have been considered - expression1 = re.sub(r"[\u4e00-\u9fff]+", "", expression1) - expression2 = re.sub(r"[\u4e00-\u9fff]+", "", expression2) - - expression1 = self.split_by_comma(expression1) - expression2 = self.split_by_comma(expression2) - - temp_list1 = self.trans_plus_minus_sign(expression1) - temp_list2 = self.trans_plus_minus_sign(expression2) - - # Set up a list for allowed errors - if len(precision) <= 1: - precision = precision * len(temp_list1) - if len(temp_list1) != len(temp_list2): - return False - - # Check if elements in both lists can be paired and are equal - idx = -1 - while len(temp_list1) != 0: - idx = (idx + 1) % len(temp_list1) - - item1 = temp_list1[idx] - self.precision = precision[idx] - - for item2 in temp_list2: - if self.is_equal(item1, item2): - temp_list1.remove(item1) - temp_list2.remove(item2) - precision.remove(self.precision) - break - else: - # If no match was found, return False - return False - - # If all elements are matched, return True - return True - - def is_interval(self, expr): - # Checks if an expression is an interval - return expr.startswith(("(", "[")) and expr.endswith((")", "]")) - - def sympy_sub_pi(self, expression_sympy): - # Replaces the symbol for pi in sympy expressions with its numerical value - return expression_sympy.subs(self.pi, math.pi) - - def is_equal(self, expression1, expression2): - # Default first expression is ground truth. Check if expressions are equal in different aspects - if expression1 == expression2 and expression1 != "" and expression2 != "": - # print("Equivalent natively") - return True - - # First check if both are intervals - if self.is_interval(expression1) and self.is_interval(expression2): - try: - if self.interval_equal(expression1, expression2): - # print("Interval equivalent") - return True - except: - return False - - # Then check for numerical equality - try: - if self.numerical_equal(expression1, expression2): - # print("Numerically equivalent") - return True - except: - pass - # Then check if expressions are mathematically equal - try: - if self.expression_equal(expression1, expression2) and not ("=" in expression1 and "=" in expression2): - # print("Expression equivalent") - return True - except: - pass - # Lastly, check for equation equality - try: - if self.equation_equal(expression1, expression2): - # print("Equation equivalent") - return True - except: - pass - return False - - def numerical_equal(self, expression1: str, expression2: str, include_percentage: bool = True): - # Check if two numerical values are equal within an allowed error range - # Includes possible percentage cases - reference = float(expression1) - prediction = float(expression2) - if include_percentage: - gt_result = [reference / 100, reference, reference * 100] - else: - gt_result = [reference] - for item in gt_result: - if abs(item - prediction) <= self.precision * 1.01: - return True - return False - - def expression_equal(self, exp1, exp2): - # Check if two expressions are mathematically equivalent - # Extract expression and use sympy for equivalence checking - def extract_expression(expression): - if "=" in expression: - expression = expression.split("=")[1] - return expression.strip() - - exp1 = extract_expression(exp1) - exp2 = extract_expression(exp2) - - expr1_sym = sympify(parse_latex(exp1)) - expr2_sym = sympify(parse_latex(exp2)) - - if expr1_sym == expr2_sym: - return True - else: - expr1_sym = self.sympy_sub_pi(expr1_sym) - expr2_sym = self.sympy_sub_pi(expr2_sym) - - if (expr1_sym.has(sp.Symbol) and not expr2_sym.has(sp.Symbol)) or (not expr1_sym.has(sp.Symbol) and expr2_sym.has(sp.Symbol)): - return False - elif not expr1_sym.has(sp.Symbol) and not expr2_sym.has(sp.Symbol): - try: - if not (self.can_compute_power(expr1_sym) and self.can_compute_power(expr2_sym)): - print(f'These two numbers cannot be calculated by the current computer for: "{str(expr1_sym)}" and "{str(expr2_sym)}"') - return False - - if abs(expr1_sym.evalf() - expr2_sym.evalf()) <= self.precision * 1.01: - return True - else: - return False - except: - return False - else: - try: - simplified_expr = simplify(expr1_sym - expr2_sym) - - num_value = simplified_expr.evalf() - return abs(num_value) < 1e-3 - except: - return False - - def equation_equal(self, expression1, expression2): - # Check if two equations are mathematically equivalent - # Simplify equations and use sympy for equivalence checking - def simplify_equation(latex_eq): - lhs, rhs = latex_eq.split("=") - - lhs_expr = parse_latex(lhs) - rhs_expr = parse_latex(rhs) - - equation = Eq(lhs_expr, rhs_expr) - - simplified_eq = simplify(equation.lhs - equation.rhs) - - return simplified_eq - - expr1_sym = simplify_equation(expression1) - expr2_sym = simplify_equation(expression2) - - division_result_1 = simplify(expr1_sym / expr2_sym) - division_result_2 = simplify(expr2_sym / expr1_sym) - - if (division_result_1.is_Integer and division_result_1 != 0) or (division_result_2.is_Integer and division_result_2 != 0): - return True - else: - return False - - def interval_equal(self, expression1, expression2): - # Check if two intervals are mathematically equivalent - def compare_two_interval(inter1, inter2): - if inter1[0] != inter2[0] or inter1[-1] != inter2[-1]: - return False - - inter1 = inter1.strip("[]()") - inter2 = inter2.strip("[]()") - - items_1 = inter1.split(",") - items_2 = inter2.split(",") - - for item_1, item_2 in zip(items_1, items_2): - if not self.expression_equal(item_1, item_2): - return False - return True - - interval1 = expression1 - interval2 = expression2 - - if interval1 == interval2: - return True - else: - inter_list1 = interval1.split("\\cup") - inter_list2 = interval2.split("\\cup") - if len(inter_list1) != len(inter_list2): - return False - else: - for inter1, inter2 in zip(inter_list1, inter_list2): - if not compare_two_interval(inter1, inter2): - return False - return True - - def preprocess(self, expression1, expression2): - # Preprocess expressions to extract and replace special symbols - def extract_boxed_content(latex_str): - boxed_matches = re.finditer(r"\\boxed{", latex_str) - results = "" - - for match in boxed_matches: - start_index = match.end() - end_index = start_index - stack = 1 - - while stack > 0 and end_index < len(latex_str): - if latex_str[end_index] == "{": - stack += 1 - elif latex_str[end_index] == "}": - stack -= 1 - end_index += 1 - - if stack == 0: - content = latex_str[start_index : end_index - 1] - results += content + "," - else: - raise ValueError("Mismatched braces in LaTeX string.") - - if results == "": - last_line_ans = latex_str.strip().split("\n")[-1] - dollar_pattern = r"\$(.*?)\$" - answers = re.findall(dollar_pattern, last_line_ans) - - if answers: - for ans in answers: - results += ans + "," - else: - results = latex_str - - return results - - def sepcial_symbol_replace(expression): - if "\\in " in expression: - expression = expression.split("\\in ")[1] - - for signal in self.special_signal_map: - expression = expression.replace(signal, self.special_signal_map[signal]) - - expression = expression.strip("\n$,.:;^_=+`!@#$%^&*~，。") - - pattern = r"\\(?:mathrm|mathbf)\{~?([^}]*)\}" - expression = re.sub(pattern, r"\1", expression) - - return expression - - exp1, exp2 = extract_boxed_content(expression1), extract_boxed_content(expression2) - exp1, exp2 = sepcial_symbol_replace(exp1), sepcial_symbol_replace(exp2) - - return exp1, exp2 - - def can_compute_power(self, expr): - # Checks if a power expression can be computed - if isinstance(expr, Pow): - base, exp = expr.as_base_exp() - if base.is_number and exp.is_number: - MAX_EXP = 1000 # Adjust based on computing environment - if abs(exp.evalf()) > MAX_EXP: - return False - else: - return True - else: - return False - else: - return True # Not a power expression, can compute \ No newline at end of file diff --git a/lmms_eval/tasks/olympiadbench/olympiadbench_test_cn.yaml b/lmms_eval/tasks/olympiadbench/olympiadbench_test_cn.yaml deleted file mode 100644 index 574d0c1..0000000 --- a/lmms_eval/tasks/olympiadbench/olympiadbench_test_cn.yaml +++ /dev/null @@ -1,25 +0,0 @@ -dataset_path: lmms-lab/OlympiadBench -dataset_kwargs: - token: True -task : "olympiadbench_test_cn" -test_split: test_cn -output_type: generate_until -doc_to_visual: !function cn_utils.olympiadbench_doc_to_visual -doc_to_text: !function cn_utils.olympiadbench_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function cn_utils.olympiadbench_process_results -metric_list: - - metric: submission - aggregation: !function cn_utils.olympiadbench_aggregate_results - higher_is_better: true - - metric: exact_match - aggregation: mean - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/olympiadbench/olympiadbench_test_en.yaml b/lmms_eval/tasks/olympiadbench/olympiadbench_test_en.yaml deleted file mode 100644 index 6d293fb..0000000 --- a/lmms_eval/tasks/olympiadbench/olympiadbench_test_en.yaml +++ /dev/null @@ -1,25 +0,0 @@ -dataset_path: lmms-lab/OlympiadBench -dataset_kwargs: - token: True -task : "olympiadbench_test_en" -test_split: test_en -output_type: generate_until -doc_to_visual: !function en_utils.olympiadbench_doc_to_visual -doc_to_text: !function en_utils.olympiadbench_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function en_utils.olympiadbench_process_results -metric_list: - - metric: submission - aggregation: !function en_utils.olympiadbench_aggregate_results - higher_is_better: true - - metric: exact_match - aggregation: mean - higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/pope/pope.yaml b/lmms_eval/tasks/pope/pope.yaml deleted file mode 100644 index 703fe3d..0000000 --- a/lmms_eval/tasks/pope/pope.yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: lmms-lab/POPE -dataset_kwargs: - token: True -task: "pope" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.pope_doc_to_visual -doc_to_text: !function utils.pope_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.pope_process_results -metric_list: - - metric: pope_accuracy - aggregation: !function utils.pope_aggregate_accuracy - higher_is_better: true - - metric: pope_precision - aggregation: !function utils.pope_aggregate_precision - higher_is_better: true - - metric: pope_recall - aggregation: !function utils.pope_aggregate_recall - higher_is_better: true - - metric: pope_f1_score - aggregation: !function utils.pope_aggregate_f1_score - higher_is_better: true - - metric: pope_yes_ratio - aggregation: !function utils.pope_aggregate_yes_ratio - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/pope/pope_adv.yaml b/lmms_eval/tasks/pope/pope_adv.yaml deleted file mode 100644 index 9a18572..0000000 --- a/lmms_eval/tasks/pope/pope_adv.yaml +++ /dev/null @@ -1,35 +0,0 @@ -dataset_path: lmms-lab/POPE -dataset_name: Full -dataset_kwargs: - token: True -task: "pope_adv" -test_split: adversarial -output_type: generate_until -doc_to_visual: !function utils.pope_doc_to_visual -doc_to_text: !function utils.pope_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.pope_process_results -metric_list: - - metric: pope_accuracy - aggregation: !function utils.pope_aggregate_accuracy - higher_is_better: true - - metric: pope_precision - aggregation: !function utils.pope_aggregate_precision - higher_is_better: true - - metric: pope_recall - aggregation: !function utils.pope_aggregate_recall - higher_is_better: true - - metric: pope_f1_score - aggregation: !function utils.pope_aggregate_f1_score - higher_is_better: true - - metric: pope_yes_ratio - aggregation: !function utils.pope_aggregate_yes_ratio - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/pope/pope_full.yaml b/lmms_eval/tasks/pope/pope_full.yaml deleted file mode 100644 index d43c4f3..0000000 --- a/lmms_eval/tasks/pope/pope_full.yaml +++ /dev/null @@ -1,5 +0,0 @@ -group : pope_full -task: - - pope_adv - - pope_pop - - pope_random \ No newline at end of file diff --git a/lmms_eval/tasks/pope/pope_pop.yaml b/lmms_eval/tasks/pope/pope_pop.yaml deleted file mode 100644 index 1f5cf65..0000000 --- a/lmms_eval/tasks/pope/pope_pop.yaml +++ /dev/null @@ -1,35 +0,0 @@ -dataset_path: lmms-lab/POPE -dataset_name: Full -dataset_kwargs: - token: True -task: "pope_pop" -test_split: popular -output_type: generate_until -doc_to_visual: !function utils.pope_doc_to_visual -doc_to_text: !function utils.pope_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.pope_process_results -metric_list: - - metric: pope_accuracy - aggregation: !function utils.pope_aggregate_accuracy - higher_is_better: true - - metric: pope_precision - aggregation: !function utils.pope_aggregate_precision - higher_is_better: true - - metric: pope_recall - aggregation: !function utils.pope_aggregate_recall - higher_is_better: true - - metric: pope_f1_score - aggregation: !function utils.pope_aggregate_f1_score - higher_is_better: true - - metric: pope_yes_ratio - aggregation: !function utils.pope_aggregate_yes_ratio - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/pope/pope_random.yaml b/lmms_eval/tasks/pope/pope_random.yaml deleted file mode 100644 index c635f2e..0000000 --- a/lmms_eval/tasks/pope/pope_random.yaml +++ /dev/null @@ -1,35 +0,0 @@ -dataset_path: lmms-lab/POPE -dataset_name: Full -dataset_kwargs: - token: True -task: "pope_random" -test_split: random -output_type: generate_until -doc_to_visual: !function utils.pope_doc_to_visual -doc_to_text: !function utils.pope_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 128 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.pope_process_results -metric_list: - - metric: pope_accuracy - aggregation: !function utils.pope_aggregate_accuracy - higher_is_better: true - - metric: pope_precision - aggregation: !function utils.pope_aggregate_precision - higher_is_better: true - - metric: pope_recall - aggregation: !function utils.pope_aggregate_recall - higher_is_better: true - - metric: pope_f1_score - aggregation: !function utils.pope_aggregate_f1_score - higher_is_better: true - - metric: pope_yes_ratio - aggregation: !function utils.pope_aggregate_yes_ratio - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/pope/utils.py b/lmms_eval/tasks/pope/utils.py deleted file mode 100644 index 8d0752f..0000000 --- a/lmms_eval/tasks/pope/utils.py +++ /dev/null @@ -1,87 +0,0 @@ -# Add the following functions to your existing utils.py file - - -def pope_doc_to_visual(doc): - # Assuming the 'doc' dictionary has a key 'image' with image data - return [doc["image"].convert("RGB")] - - -def pope_doc_to_text(doc): - # Assuming the 'doc' dictionary has a key 'question' with the question text - question = doc["question"].strip() - return f"{question}\nAnswer the question using a single word or phrase." - - -def pope_process_results(doc, results): - pred = results[0].lower().strip() - gt_ans = doc["answer"].lower().strip() - assert gt_ans in ["yes", "no"] - score = 1.0 if pred == gt_ans else 0.0 - return { - "pope_accuracy": {"question_id": doc["question_id"], "score": score, "prediction": pred, "ground_truth": gt_ans}, - "pope_precision": {"question_id": doc["question_id"], "score": score, "prediction": pred, "ground_truth": gt_ans}, - "pope_recall": {"question_id": doc["question_id"], "score": score, "prediction": pred, "ground_truth": gt_ans}, - "pope_f1_score": {"question_id": doc["question_id"], "score": score, "prediction": pred, "ground_truth": gt_ans}, - "pope_yes_ratio": {"question_id": doc["question_id"], "score": score, "prediction": pred, "ground_truth": gt_ans}, - } - - -def pope_aggregate_accuracy(results): - total_score = 0 - for result in results: - total_score += result["score"] - avg_score = total_score / len(results) - return avg_score - - -def pope_aggregate_precision(results): - true_positives = 0 - false_positives = 0 - for result in results: - pred = result["prediction"] - gt = result["ground_truth"] - if gt == "yes" and pred == "yes": - true_positives += 1 - elif gt == "no" and pred == "yes": - false_positives += 1 - precision = true_positives / (true_positives + false_positives) if (true_positives + false_positives) > 0 else 0 - return precision - - -def pope_aggregate_recall(results): - true_positives = 0 - false_negatives = 0 - for result in results: - pred = result["prediction"] - gt = result["ground_truth"] - if gt == "yes" and pred == "yes": - true_positives += 1 - elif gt == "yes" and pred == "no": - false_negatives += 1 - recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) > 0 else 0 - return recall - - -def pope_aggregate_f1_score(results): - precision = pope_aggregate_precision(results) - recall = pope_aggregate_recall(results) - f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 - return f1_score - - -def pope_aggregate_yes_ratio(results): - yes_count = 0 - no_count = 0 - for result in results: - # gt = result["ground_truth"] - # if gt == "yes": - # yes_count += 1 - # elif gt == "no": - # no_count += 1 - pred = result["prediction"] - if pred == "yes": - yes_count += 1 - elif pred == "no": - no_count += 1 - yes_ratio = yes_count / (yes_count + no_count) if (yes_count + no_count) > 0 else 0 - return yes_ratio diff --git a/lmms_eval/tasks/realworldqa/realworldqa.yaml b/lmms_eval/tasks/realworldqa/realworldqa.yaml deleted file mode 100644 index f832ffb..0000000 --- a/lmms_eval/tasks/realworldqa/realworldqa.yaml +++ /dev/null @@ -1,43 +0,0 @@ -dataset_path: lmms-lab/RealWorldQA -dataset_kwargs: - token: True -task: "realworldqa" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.realworldqa_doc_to_visual -doc_to_text: !function utils.realworldqa_doc_to_text -doc_to_target: "answer" - -generation_kwargs: - max_new_tokens: 16 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false - -filter_list: - - name: "flexible-extract" - filter: - - function: !function utils.NumberWordsToDigitsFilter - - function: !function utils.MultiChoiceRegexFilter - group_select: 0 - ignore_case: true - ignore_punctuation: true - regex_pattern: "(\\([A-Z]\\))" - -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "" - gpt4v: - pre_prompt: "" - post_prompt: "" -metadata: - - version: 0.0 diff --git a/lmms_eval/tasks/realworldqa/utils.py b/lmms_eval/tasks/realworldqa/utils.py deleted file mode 100644 index 9ed645e..0000000 --- a/lmms_eval/tasks/realworldqa/utils.py +++ /dev/null @@ -1,117 +0,0 @@ -from lmms_eval.filters.extraction import ExtendedRegexFilter -from lmms_eval.filters.transformation import MapFilter -import re - -REPLACE_PROMPT = "Please answer directly with only the letter of the correct option and nothing else." - - -def realworldqa_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def realworldqa_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = "" - post_prompt = "" - question = doc["question"].strip() - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["post_prompt"]: - question = question.replace(REPLACE_PROMPT, "") - post_prompt = model_specific_prompt_kwargs["post_prompt"] - return f"{pre_prompt}{question}{post_prompt}" - - -# number_words_to_digits = { -# "zero": "0", "one": "1", "two": "2", "three": "3", "four": "4", -# "five": "5", "six": "6", "seven": "7", "eight": "8", "nine": "9", -# "ten": "10" -# } - - -def realworldqa_process_results(doc, results): - pred = results[0].lower().strip().rstrip(".") - gt_ans = doc["answer"].lower().strip() - - print(f"Prediction: {pred}, Ground Truth: {gt_ans}") - # assert gt_ans in ["a", "b", "c", "d"] - score = 1.0 if pred == gt_ans else 0.0 - return { - "exact_match": score, - } - - -class NumberWordsToDigitsFilter(MapFilter): - def __init__(self) -> None: - mapping_dict = {"zero": "0", "one": "1", "two": "2", "three": "3", "four": "4", "five": "5", "six": "6", "seven": "7", "eight": "8", "nine": "9", "ten": "10"} - super().__init__(mapping_dict, default_value=None) - - def apply(self, resps, docs): - def filter_set(inst): - return [self.mapping_dict.get(resp.lower(), resp) for resp in inst] - - return [filter_set(resp) for resp in resps] - - -class MultiChoiceRegexFilter(ExtendedRegexFilter): - def __init__(self, *args, **kwargs): - """ - regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure - - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response. - - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices. - group_select: Selects the (group_select)th match from the findall result. - ignore_case: Ignores the case during step 1 matching - ignore_punctuation: Remove the punctuation during step 1 matching - regexes_to_ignore: Remove these regexes during step 1 matching - """ - super().__init__(*args, **kwargs) - - def apply(self, resps, docs): - # here, we assume we have a list, in which each element is - # a list of model responses for some particular input/target pair. - # so we process each of these (same input/target response sets) - # independently (and keep them a list.) - - filtered_resps = [] - - for r, doc in zip(resps, docs): - fallback_regexes = [] - choice_to_alpha = {} - next_alpha = "A" - - without_paren_fallback_regexes = [] - without_paren_to_target = {} - - # Regex to extract multiple choice options from the question - multiple_choices_regex = re.compile(r"\b([A-Z])\.\s+([^\n]*)") - matches = multiple_choices_regex.findall(doc["question"]) - - # Build regex patterns and mappings for each choice - for m in matches: - choice_text = m[1].strip() - fallback_regexes.append(f"{re.escape(choice_text)}") - choice_to_alpha[choice_text] = next_alpha - - next_alpha = chr(ord(next_alpha) + 1) - - # Compile regex to match any of the extracted choices - fallback_regex = re.compile("|".join(fallback_regexes)) - - # Process each response - filtered = [] - for resp in r: - # Remove any punctuation and extra spaces - cleaned_resp = re.sub(r"[^\w\s]", "", resp).strip() - # Try to match cleaned response with the choice text - match = fallback_regex.search(cleaned_resp) - if match and match.group() in choice_to_alpha: - # Map the matched choice text back to its corresponding letter - filtered.append(choice_to_alpha[match.group()]) - else: - # If no match, return the cleaned response - filtered.append(cleaned_resp) - - filtered_resps.append(filtered[0]) - - return filtered_resps diff --git a/lmms_eval/tasks/refcoco+/_default_template_bbox_rec_yaml b/lmms_eval/tasks/refcoco+/_default_template_bbox_rec_yaml deleted file mode 100644 index c369bae..0000000 --- a/lmms_eval/tasks/refcoco+/_default_template_bbox_rec_yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: lmms-lab/RefCOCOPlus -output_type: generate_until -process_docs: !function utils_rec.refcoco_bbox_rec_preprocess_dataset -doc_to_visual: !function utils_rec.refcoco_bbox_rec_doc_to_visual -doc_to_text: !function utils_rec.refcoco_bbox_rec_doc_to_text -doc_to_target: "bbox" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils_rec.refcoco_bbox_rec_process_result -metric_list: - - metric: refcoco_IoU - aggregation : !function utils_rec.refcoco_bbox_rec_iou - higher_is_better : true - - metric: refcoco_ACC@0.1 - aggregation : !function utils_rec.refcoco_bbox_rec_acc01 - higher_is_better : true - - metric: refcoco_ACC@0.3 - aggregation : !function utils_rec.refcoco_bbox_rec_acc03 - higher_is_better : true - - metric: refcoco_ACC@0.5 - aggregation : !function utils_rec.refcoco_bbox_rec_acc05 - higher_is_better : true - - metric: refcoco_ACC@0.7 - aggregation : !function utils_rec.refcoco_bbox_rec_acc07 - higher_is_better : true - - metric: refcoco_ACC@0.9 - aggregation : !function utils_rec.refcoco_bbox_rec_acc09 - higher_is_better : true - - metric: refcoco_Center_ACC - aggregation : !function utils_rec.refcoco_bbox_rec_center_acc - higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/refcoco+/_default_template_bbox_yaml b/lmms_eval/tasks/refcoco+/_default_template_bbox_yaml deleted file mode 100644 index 82fb991..0000000 --- a/lmms_eval/tasks/refcoco+/_default_template_bbox_yaml +++ /dev/null @@ -1,36 +0,0 @@ -dataset_path: lmms-lab/RefCOCOplus -output_type: generate_until -doc_to_visual: !function utils.refcoco_bbox_doc_to_visual -doc_to_text: !function utils.refcoco_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils.refcoco_process_result -metric_list: - - metric: refcoco_Bleu_4 - aggregation : !function utils.refcoco_bleu4 - higher_is_better : true - - metric: refcoco_Bleu_3 - aggregation : !function utils.refcoco_bleu3 - higher_is_better : true - - metric: refcoco_Bleu_2 - aggregation : !function utils.refcoco_bleu2 - higher_is_better : true - - metric: refcoco_Bleu_1 - aggregation : !function utils.refcoco_bleu1 - higher_is_better : true - - metric: refcoco_METEOR - aggregation : !function utils.refcoco_meteor - higher_is_better : true - - metric: refcoco_ROUGE_L - aggregation : !function utils.refcoco_rougel - higher_is_better : true - - metric: refcoco_CIDEr - aggregation : !function utils.refcoco_cider - higher_is_better : true - #- metric: refcoco_SPICE - # aggregation : !function utils.refcoco_spice - # higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/refcoco+/_default_template_seg_yaml b/lmms_eval/tasks/refcoco+/_default_template_seg_yaml deleted file mode 100644 index 2c38d51..0000000 --- a/lmms_eval/tasks/refcoco+/_default_template_seg_yaml +++ /dev/null @@ -1,36 +0,0 @@ -dataset_path: lmms-lab/RefCOCOplus -output_type: generate_until -doc_to_visual: !function utils.refcoco_seg_doc_to_visual -doc_to_text: !function utils.refcoco_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils.refcoco_process_result -metric_list: - - metric: refcoco_Bleu_4 - aggregation : !function utils.refcoco_bleu4 - higher_is_better : true - - metric: refcoco_Bleu_3 - aggregation : !function utils.refcoco_bleu3 - higher_is_better : true - - metric: refcoco_Bleu_2 - aggregation : !function utils.refcoco_bleu2 - higher_is_better : true - - metric: refcoco_Bleu_1 - aggregation : !function utils.refcoco_bleu1 - higher_is_better : true - - metric: refcoco_METEOR - aggregation : !function utils.refcoco_meteor - higher_is_better : true - - metric: refcoco_ROUGE_L - aggregation : !function utils.refcoco_rougel - higher_is_better : true - - metric: refcoco_CIDEr - aggregation : !function utils.refcoco_cider - higher_is_better : true - #- metric: refcoco_SPICE - # aggregation : !function utils.refcoco_spice - # higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/refcoco+/_generate_config.py b/lmms_eval/tasks/refcoco+/_generate_config.py deleted file mode 100644 index 0079a2d..0000000 --- a/lmms_eval/tasks/refcoco+/_generate_config.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import yaml - -# splits = ["train", "val", "testA", "testB"] -splits = ["val", "testA", "testB"] -tasks = ["seg", "bbox"] - -if __name__ == "__main__": - dump_tasks = [] - for task in tasks: - for split in splits: - yaml_dict = {"group": f"refcoco+_{task}", "task": f"refcoco+_{task}_{split}", "include": f"_default_template_{task}_yaml", "test_split": split} - if split == "train": - yaml_dict.pop("group") - else: - dump_tasks.append(f"refcoco_{task}_{split}") - - save_path = f"./refcoco+_{task}_{split}.yaml" - print(f"Saving to {save_path}") - with open(save_path, "w") as f: - yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False) - - group_dict = {"group": "refcoco+", "task": dump_tasks} - - with open("./_refcoco.yaml", "w") as f: - yaml.dump(group_dict, f, default_flow_style=False, indent=4) diff --git a/lmms_eval/tasks/refcoco+/_refcoco.yaml b/lmms_eval/tasks/refcoco+/_refcoco.yaml deleted file mode 100644 index 606fd0c..0000000 --- a/lmms_eval/tasks/refcoco+/_refcoco.yaml +++ /dev/null @@ -1,8 +0,0 @@ -group: refcoco+ -task: -- refcoco+_seg_val -- refcoco+_seg_testA -- refcoco+_seg_testB -- refcoco+_bbox_val -- refcoco+_bbox_testA -- refcoco+_bbox_testB diff --git a/lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_testA.yaml b/lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_testA.yaml deleted file mode 100644 index 0ebb6c0..0000000 --- a/lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_testA.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco+_bbox_rec -task: refcoco+_bbox_rec_testA -include: _default_template_bbox_rec_yaml -test_split: testA diff --git a/lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_testB.yaml b/lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_testB.yaml deleted file mode 100644 index b347bce..0000000 --- a/lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_testB.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco+_bbox_rec -task: refcoco+_bbox_rec_testB -include: _default_template_bbox_rec_yaml -test_split: testB diff --git a/lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_val.yaml b/lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_val.yaml deleted file mode 100644 index 890f588..0000000 --- a/lmms_eval/tasks/refcoco+/refcoco+_bbox_rec_val.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco+_bbox_rec -task: refcoco+_bbox_rec_val -include: _default_template_bbox_rec_yaml -test_split: val diff --git a/lmms_eval/tasks/refcoco+/refcoco+_bbox_testA.yaml b/lmms_eval/tasks/refcoco+/refcoco+_bbox_testA.yaml deleted file mode 100644 index f32e285..0000000 --- a/lmms_eval/tasks/refcoco+/refcoco+_bbox_testA.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco+_bbox -task: refcoco+_bbox_testA -include: _default_template_bbox_yaml -test_split: testA diff --git a/lmms_eval/tasks/refcoco+/refcoco+_bbox_testB.yaml b/lmms_eval/tasks/refcoco+/refcoco+_bbox_testB.yaml deleted file mode 100644 index 3722398..0000000 --- a/lmms_eval/tasks/refcoco+/refcoco+_bbox_testB.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco+_bbox -task: refcoco+_bbox_testB -include: _default_template_bbox_yaml -test_split: testB diff --git a/lmms_eval/tasks/refcoco+/refcoco+_bbox_val.yaml b/lmms_eval/tasks/refcoco+/refcoco+_bbox_val.yaml deleted file mode 100644 index 9ca6638..0000000 --- a/lmms_eval/tasks/refcoco+/refcoco+_bbox_val.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco+_bbox -task: refcoco+_bbox_val -include: _default_template_bbox_yaml -test_split: val diff --git a/lmms_eval/tasks/refcoco+/refcoco+_seg_testA.yaml b/lmms_eval/tasks/refcoco+/refcoco+_seg_testA.yaml deleted file mode 100644 index 1514ad5..0000000 --- a/lmms_eval/tasks/refcoco+/refcoco+_seg_testA.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco+_seg -task: refcoco+_seg_testA -include: _default_template_seg_yaml -test_split: testA diff --git a/lmms_eval/tasks/refcoco+/refcoco+_seg_testB.yaml b/lmms_eval/tasks/refcoco+/refcoco+_seg_testB.yaml deleted file mode 100644 index 8352025..0000000 --- a/lmms_eval/tasks/refcoco+/refcoco+_seg_testB.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco+_seg -task: refcoco+_seg_testB -include: _default_template_seg_yaml -test_split: testB diff --git a/lmms_eval/tasks/refcoco+/refcoco+_seg_val.yaml b/lmms_eval/tasks/refcoco+/refcoco+_seg_val.yaml deleted file mode 100644 index d7e3493..0000000 --- a/lmms_eval/tasks/refcoco+/refcoco+_seg_val.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco+_seg -task: refcoco+_seg_val -include: _default_template_seg_yaml -test_split: val diff --git a/lmms_eval/tasks/refcoco+/utils.py b/lmms_eval/tasks/refcoco+/utils.py deleted file mode 100644 index 4feb71c..0000000 --- a/lmms_eval/tasks/refcoco+/utils.py +++ /dev/null @@ -1,135 +0,0 @@ -from PIL import ImageDraw -from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice -from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer -from pycocotools.coco import COCO - -COCO_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"] # , "SPICE"] - -import logging - -eval_logger = logging.getLogger("lmms-eval") - - -def refcoco_bbox_doc_to_visual(doc): - bbox = doc["bbox"] - image = doc["image"].convert("RGB") - draw = ImageDraw.Draw(image) - # Origin format (top x, top y, width, height) - bbox_xy = [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]] - draw.rectangle(bbox_xy, outline="red") - return [image.convert("RGB")] - - -def refcoco_seg_doc_to_visual(doc): - seg = doc["segmentation"] - image = doc["image"].convert("RGB") - draw = ImageDraw.Draw(image) - draw.polygon(seg) - return [image.convert("RGB")] - - -def refcoco_doc_to_text(doc): - # question = doc["question"] - # return f"{question}\nAnswer the question using a single word or phrase." - return "Provide a short description for this region." - - -def refcoco_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case coco_bleu), value: metric value - """ - pred = result[0] if len(result) > 0 else "" - ann_id = doc["question_id"] - data_dict = {"answer": doc["answer"], "pred": pred, "ann_id": ann_id} - return {f"refcoco_{metric}": data_dict for metric in COCO_METRICS} - - -def refcoco_aggregation_result(results, metric): - scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr")]#, (Spice(), "SPICE")] - scorers_dict = {s[1]: s for s in scorers} - - stored_results = [] - # In order to make the coco eval tools to successfully create index - # We need at least two dict in the dataset - # 'annotation' and 'images' - # 'annotation' exactly reproduce the original annotation - # 'images' however only need the image id which is contained in the file name - dataset = {"annotations": [], "images": []} - idx = 0 - ann_id = 0 - for result in results: - stored_results.append({"image_id": idx, "caption": result["pred"]}) - for s in result["answer"]: - dataset["annotations"].append({"image_id": idx, "caption": s, "id": ann_id}) - ann_id += 1 - - dataset["images"].append({"id": idx}) - idx += 1 - - coco = COCO() - # Manually create index here - coco.dataset = dataset - coco.createIndex() - - coco_result = coco.loadRes(stored_results) - coco_eval = COCOEvalCap(coco, coco_result) - - imgIds = coco_eval.params["image_id"] - gts = {} - res = {} - for imgId in imgIds: - gts[imgId] = coco_eval.coco.imgToAnns[imgId] - res[imgId] = coco_eval.cocoRes.imgToAnns[imgId] - - eval_logger.info("tokenization...") - tokenizer = PTBTokenizer() - gts = tokenizer.tokenize(gts) - res = tokenizer.tokenize(res) - - eval_logger.info(f"Computing {metric} scores...") - - score, scores = scorers_dict[metric][0].compute_score(gts, res) - # coco_eval.setEval(score, metric) - - # When metric is one of the Bleu, score will be a list - if type(score) == list: - n = int(metric.split("_")[-1]) - score = score[n - 1] - - return score - - -def refcoco_bleu4(results): - return refcoco_aggregation_result(results, "Bleu_4") - - -def refcoco_bleu3(results): - return refcoco_aggregation_result(results, "Bleu_3") - - -def refcoco_bleu2(results): - return refcoco_aggregation_result(results, "Bleu_2") - - -def refcoco_bleu1(results): - return refcoco_aggregation_result(results, "Bleu_1") - - -def refcoco_meteor(results): - return refcoco_aggregation_result(results, "METEOR") - - -def refcoco_rougel(results): - return refcoco_aggregation_result(results, "ROUGE_L") - - -def refcoco_cider(results): - return refcoco_aggregation_result(results, "CIDEr") - - -def refcoco_spice(results): - return refcoco_aggregation_result(results, "SPICE") diff --git a/lmms_eval/tasks/refcoco+/utils_rec.py b/lmms_eval/tasks/refcoco+/utils_rec.py deleted file mode 100644 index ec91bf9..0000000 --- a/lmms_eval/tasks/refcoco+/utils_rec.py +++ /dev/null @@ -1,221 +0,0 @@ -import re -import logging -from datasets import Dataset - -eval_logger = logging.getLogger("lmms-eval") - -COCO_REC_METRICS = ["IoU", "ACC@0.1", "ACC@0.3", "ACC@0.5", "ACC@0.7", "ACC@0.9", "Center_ACC"] - - -def refcoco_bbox_rec_preprocess_dataset(dataset: Dataset): - # PIL image stored in dataset['image'] - # add `image_width` and `image_height` to the dataset - dataset = dataset.map(lambda x: {"image_width": x["image"].width, "image_height": x["image"].height}) - - # Original bbox format (top x, top y, width, height) - # Convert to (top-left x, top-left y, bottom-right x, bottom-right y) - # Normalize the bounding box coordinates to be between 0 and 1 - # using the image width and height - dataset = dataset.map( - lambda x: {"bbox": [x["bbox"][0] / x["image_width"], - x["bbox"][1] / x["image_height"], - (x["bbox"][0] + x["bbox"][2]) / x["image_width"], - (x["bbox"][1] + x["bbox"][3]) / x["image_height"]]} - ) - - # currently, the dataset has `answer` as a list of strings - # each answer should be its own row - # we will explode the dataset to have one row per answer - # duplicate the other columns - def explode_answers(example): - answers = example.pop('answer') - return [{'answer': answer, **example} for answer in answers] - - # Apply the function to each element, collecting the results - exploded_rows = [] - for example in dataset: - exploded_rows.extend(explode_answers(example)) - - # Create a new dataset from the exploded rows - new_dataset = Dataset.from_list(exploded_rows) - print(f"Exploded dataset from {len(dataset)} to {len(new_dataset)} rows") - - return new_dataset - - -def refcoco_bbox_rec_doc_to_visual(doc): - # Image is presented as is - image = doc["image"].convert("RGB") - return [image.convert("RGB")] - - -def refcoco_bbox_rec_doc_to_text(doc): - assert isinstance(doc['answer'], str), "Answer must be a string" - return "Bounding box coordinates are specified in the format (top-left x, top-left y, bottom-right x, bottom-right y). All values are floating point numbers bounded between 0 and 1. Please provide the bounding box coordinate of the region this sentence describes: " + doc['answer'] - - -def parse_float_sequence_within(input_str): - """ - Extract the first sequence of four floating-point numbers within square brackets from a string. - - Args: - input_str (str): A string that may contain a sequence of four floats within square brackets. - - Returns: - list: A list of four floats if the pattern is found, or a list of four zeros if the pattern is not found. - """ - # Define the regex pattern to find the first instance of four floats within square brackets - pattern = r'\[\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?)\s*\]' - - # Use re.search to find the first match of the pattern in the input string - match = re.search(pattern, input_str) - - # If a match is found, convert the captured groups into a list of floats - if match: - return [float(match.group(i)) for i in range(1, 5)] - - # If the input does not contain the pattern, return the null float sequence - return [0, 0, 0, 0] - - -def refcoco_bbox_rec_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name, value: metric value - """ - pred = result[0] if len(result) > 0 else "" - pred = parse_float_sequence_within(pred) - ann_id = doc["question_id"] - data_dict = {"answer": doc["answer"], "pred": pred, "ann_id": ann_id, 'bbox': doc['bbox']} - return {f"refcoco_{metric}": data_dict for metric in COCO_REC_METRICS} - - -def compute_iou(box1, box2): - """ - Compute the Intersection over Union (IoU) of two bounding boxes. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - Returns: - - float: IoU of box1 and box2. - """ - # Determine the coordinates of the intersection rectangle - x_left = max(box1[0], box2[0]) - y_top = max(box1[1], box2[1]) - x_right = min(box1[2], box2[2]) - y_bottom = min(box1[3], box2[3]) - - # Compute the area of intersection - intersection_area = max(0, x_right - x_left) * max(0, y_bottom - y_top) - - # Compute the area of both bounding boxes - box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1]) - box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1]) - - # Compute the area of the union - union_area = box1_area + box2_area - intersection_area - - # Compute the Intersection over Union - iou = intersection_area / union_area - - return iou - - -def compute_accuracy(box1, box2, threshold=0.5): - """ - Compute the accuracy of two bounding boxes based on a specified threshold. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - threshold (float): Threshold for the IoU to consider the prediction correct. - - Returns: - - float: Accuracy of the prediction based on the IoU threshold. - """ - iou = compute_iou(box1, box2) - return iou >= threshold - - -def compute_center_accuracy(box1, box2): - """ - Compute if the center point of box 2 is within box 1. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - Returns: - - bool: True if the center point of box 2 is within box 1, False otherwise. - """ - # Compute the center point of box 2 - center_x = (box2[0] + box2[2]) / 2 - center_y = (box2[1] + box2[3]) / 2 - - # Check if the center point is within box 1 - return box1[0] <= center_x <= box1[2] and box1[1] <= center_y <= box1[3] - - -def refcoco_bbox_rec_aggregation_result(results, metric): - """ - Aggregate the results of the RefCOCO evaluation task using the specified metric. - - Args: - - results (list of dict): List of result dictionaries. - - metric (str): Metric to use for aggregation. - - Returns: - - dict: Dictionary containing the aggregated results for the specified metric. - """ - scorers = { - 'IoU': compute_iou, - 'ACC@0.1': lambda x, y: compute_accuracy(x, y, 0.1), - 'ACC@0.3': lambda x, y: compute_accuracy(x, y, 0.3), - 'ACC@0.5': lambda x, y: compute_accuracy(x, y, 0.5), - 'ACC@0.7': lambda x, y: compute_accuracy(x, y, 0.7), - 'ACC@0.9': lambda x, y: compute_accuracy(x, y, 0.9), - 'Center_ACC': compute_center_accuracy - } - results_dict = {metric: []} - for result in results: - # Extract the ground truth and predicted bounding boxes - gt_bbox = result['bbox'] - pred_bbox = result['pred'] - # Compute the specified metric between the ground truth and predicted bounding boxes - score = scorers[metric](gt_bbox, pred_bbox) - results_dict[metric].append(score) - results_dict[metric] = sum(results_dict[metric]) / len(results_dict[metric]) - print(f"Aggregated {metric} score: {results_dict[metric]}") - return results_dict[metric] - - -def refcoco_bbox_rec_iou(results): - return refcoco_bbox_rec_aggregation_result(results, "IoU") - - -def refcoco_bbox_rec_acc01(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.1") - -def refcoco_bbox_rec_acc03(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.3") - - -def refcoco_bbox_rec_acc05(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.5") - - -def refcoco_bbox_rec_acc07(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.7") - - -def refcoco_bbox_rec_acc09(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.9") - - -def refcoco_bbox_rec_center_acc(results): - return refcoco_bbox_rec_aggregation_result(results, "Center_ACC") diff --git a/lmms_eval/tasks/refcoco/_default_template_bbox_rec_yaml b/lmms_eval/tasks/refcoco/_default_template_bbox_rec_yaml deleted file mode 100644 index 3b5ca10..0000000 --- a/lmms_eval/tasks/refcoco/_default_template_bbox_rec_yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: lmms-lab/RefCOCO -output_type: generate_until -process_docs: !function utils_rec.refcoco_bbox_rec_preprocess_dataset -doc_to_visual: !function utils_rec.refcoco_bbox_rec_doc_to_visual -doc_to_text: !function utils_rec.refcoco_bbox_rec_doc_to_text -doc_to_target: "bbox" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils_rec.refcoco_bbox_rec_process_result -metric_list: - - metric: refcoco_IoU - aggregation : !function utils_rec.refcoco_bbox_rec_iou - higher_is_better : true - - metric: refcoco_ACC@0.1 - aggregation : !function utils_rec.refcoco_bbox_rec_acc01 - higher_is_better : true - - metric: refcoco_ACC@0.3 - aggregation : !function utils_rec.refcoco_bbox_rec_acc03 - higher_is_better : true - - metric: refcoco_ACC@0.5 - aggregation : !function utils_rec.refcoco_bbox_rec_acc05 - higher_is_better : true - - metric: refcoco_ACC@0.7 - aggregation : !function utils_rec.refcoco_bbox_rec_acc07 - higher_is_better : true - - metric: refcoco_ACC@0.9 - aggregation : !function utils_rec.refcoco_bbox_rec_acc09 - higher_is_better : true - - metric: refcoco_Center_ACC - aggregation : !function utils_rec.refcoco_bbox_rec_center_acc - higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/refcoco/_default_template_bbox_yaml b/lmms_eval/tasks/refcoco/_default_template_bbox_yaml deleted file mode 100644 index b5cc6b0..0000000 --- a/lmms_eval/tasks/refcoco/_default_template_bbox_yaml +++ /dev/null @@ -1,36 +0,0 @@ -dataset_path: lmms-lab/RefCOCO -output_type: generate_until -doc_to_visual: !function utils.refcoco_bbox_doc_to_visual -doc_to_text: !function utils.refcoco_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils.refcoco_process_result -metric_list: - - metric: refcoco_Bleu_4 - aggregation : !function utils.refcoco_bleu4 - higher_is_better : true - - metric: refcoco_Bleu_3 - aggregation : !function utils.refcoco_bleu3 - higher_is_better : true - - metric: refcoco_Bleu_2 - aggregation : !function utils.refcoco_bleu2 - higher_is_better : true - - metric: refcoco_Bleu_1 - aggregation : !function utils.refcoco_bleu1 - higher_is_better : true - - metric: refcoco_METEOR - aggregation : !function utils.refcoco_meteor - higher_is_better : true - - metric: refcoco_ROUGE_L - aggregation : !function utils.refcoco_rougel - higher_is_better : true - - metric: refcoco_CIDEr - aggregation : !function utils.refcoco_cider - higher_is_better : true - #- metric: refcoco_SPICE - # aggregation : !function utils.refcoco_spice - # higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/refcoco/_default_template_seg_yaml b/lmms_eval/tasks/refcoco/_default_template_seg_yaml deleted file mode 100644 index 8545654..0000000 --- a/lmms_eval/tasks/refcoco/_default_template_seg_yaml +++ /dev/null @@ -1,36 +0,0 @@ -dataset_path: lmms-lab/RefCOCO -output_type: generate_until -doc_to_visual: !function utils.refcoco_seg_doc_to_visual -doc_to_text: !function utils.refcoco_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils.refcoco_process_result -metric_list: - - metric: refcoco_Bleu_4 - aggregation : !function utils.refcoco_bleu4 - higher_is_better : true - - metric: refcoco_Bleu_3 - aggregation : !function utils.refcoco_bleu3 - higher_is_better : true - - metric: refcoco_Bleu_2 - aggregation : !function utils.refcoco_bleu2 - higher_is_better : true - - metric: refcoco_Bleu_1 - aggregation : !function utils.refcoco_bleu1 - higher_is_better : true - - metric: refcoco_METEOR - aggregation : !function utils.refcoco_meteor - higher_is_better : true - - metric: refcoco_ROUGE_L - aggregation : !function utils.refcoco_rougel - higher_is_better : true - - metric: refcoco_CIDEr - aggregation : !function utils.refcoco_cider - higher_is_better : true - #- metric: refcoco_SPICE - # aggregation : !function utils.refcoco_spice - # higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/refcoco/_generate_config.py b/lmms_eval/tasks/refcoco/_generate_config.py deleted file mode 100644 index 8228102..0000000 --- a/lmms_eval/tasks/refcoco/_generate_config.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import yaml - -# splits = ["train", "test", "val", "testA", "testB"] -splits = ["test", "val", "testA", "testB"] -tasks = ["seg", "bbox"] - -if __name__ == "__main__": - dump_tasks = [] - for task in tasks: - for split in splits: - yaml_dict = {"group": f"refcoco_{task}", "task": f"refcoco_{task}_{split}", "test_split": split, "include": f"_default_template_{task}_yaml"} - if split == "train": - yaml_dict.pop("group") - else: - dump_tasks.append(f"refcoco_{task}_{split}") - - save_path = f"./refcoco_{task}_{split}.yaml" - print(f"Saving to {save_path}") - with open(save_path, "w") as f: - yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False) - - group_dict = {"group": "refcoco", "task": dump_tasks} - - with open("./_refcoco.yaml", "w") as f: - yaml.dump(group_dict, f, default_flow_style=False, indent=4) diff --git a/lmms_eval/tasks/refcoco/_refcoco.yaml b/lmms_eval/tasks/refcoco/_refcoco.yaml deleted file mode 100644 index 97f8c05..0000000 --- a/lmms_eval/tasks/refcoco/_refcoco.yaml +++ /dev/null @@ -1,10 +0,0 @@ -group: refcoco -task: -- refcoco_seg_test -- refcoco_seg_val -- refcoco_seg_testA -- refcoco_seg_testB -- refcoco_bbox_test -- refcoco_bbox_val -- refcoco_bbox_testA -- refcoco_bbox_testB diff --git a/lmms_eval/tasks/refcoco/refcoco_bbox_rec_test.yaml b/lmms_eval/tasks/refcoco/refcoco_bbox_rec_test.yaml deleted file mode 100644 index 896ed4a..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_bbox_rec_test.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_bbox_rec -task: refcoco_bbox_rec_test -test_split: test -include: _default_template_bbox_rec_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_bbox_rec_testA.yaml b/lmms_eval/tasks/refcoco/refcoco_bbox_rec_testA.yaml deleted file mode 100644 index 191268a..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_bbox_rec_testA.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_bbox_rec -task: refcoco_bbox_rec_testA -test_split: testA -include: _default_template_bbox_rec_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_bbox_rec_testB.yaml b/lmms_eval/tasks/refcoco/refcoco_bbox_rec_testB.yaml deleted file mode 100644 index 39b2907..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_bbox_rec_testB.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_bbox_rec -task: refcoco_bbox_rec_testB -test_split: testB -include: _default_template_bbox_rec_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_bbox_rec_val.yaml b/lmms_eval/tasks/refcoco/refcoco_bbox_rec_val.yaml deleted file mode 100644 index f5da6c5..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_bbox_rec_val.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_bbox_rec -task: refcoco_bbox_rec_val -test_split: val -include: _default_template_bbox_rec_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_bbox_test.yaml b/lmms_eval/tasks/refcoco/refcoco_bbox_test.yaml deleted file mode 100644 index 1a5bc5a..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_bbox_test.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_bbox -task: refcoco_bbox_test -test_split: test -include: _default_template_bbox_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_bbox_testA.yaml b/lmms_eval/tasks/refcoco/refcoco_bbox_testA.yaml deleted file mode 100644 index 31ba931..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_bbox_testA.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_bbox -task: refcoco_bbox_testA -test_split: testA -include: _default_template_bbox_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_bbox_testB.yaml b/lmms_eval/tasks/refcoco/refcoco_bbox_testB.yaml deleted file mode 100644 index 7d6b70b..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_bbox_testB.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_bbox -task: refcoco_bbox_testB -test_split: testB -include: _default_template_bbox_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_bbox_val.yaml b/lmms_eval/tasks/refcoco/refcoco_bbox_val.yaml deleted file mode 100644 index 527b259..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_bbox_val.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_bbox -task: refcoco_bbox_val -test_split: val -include: _default_template_bbox_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_seg_test.yaml b/lmms_eval/tasks/refcoco/refcoco_seg_test.yaml deleted file mode 100644 index 80c24b2..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_seg_test.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_seg -task: refcoco_seg_test -test_split: test -include: _default_template_seg_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_seg_testA.yaml b/lmms_eval/tasks/refcoco/refcoco_seg_testA.yaml deleted file mode 100644 index 8a490bc..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_seg_testA.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_seg -task: refcoco_seg_testA -test_split: testA -include: _default_template_seg_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_seg_testB.yaml b/lmms_eval/tasks/refcoco/refcoco_seg_testB.yaml deleted file mode 100644 index 429227a..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_seg_testB.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_seg -task: refcoco_seg_testB -test_split: testB -include: _default_template_seg_yaml diff --git a/lmms_eval/tasks/refcoco/refcoco_seg_val.yaml b/lmms_eval/tasks/refcoco/refcoco_seg_val.yaml deleted file mode 100644 index 8e3e538..0000000 --- a/lmms_eval/tasks/refcoco/refcoco_seg_val.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcoco_seg -task: refcoco_seg_val -test_split: val -include: _default_template_seg_yaml diff --git a/lmms_eval/tasks/refcoco/utils.py b/lmms_eval/tasks/refcoco/utils.py deleted file mode 100644 index 4feb71c..0000000 --- a/lmms_eval/tasks/refcoco/utils.py +++ /dev/null @@ -1,135 +0,0 @@ -from PIL import ImageDraw -from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice -from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer -from pycocotools.coco import COCO - -COCO_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"] # , "SPICE"] - -import logging - -eval_logger = logging.getLogger("lmms-eval") - - -def refcoco_bbox_doc_to_visual(doc): - bbox = doc["bbox"] - image = doc["image"].convert("RGB") - draw = ImageDraw.Draw(image) - # Origin format (top x, top y, width, height) - bbox_xy = [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]] - draw.rectangle(bbox_xy, outline="red") - return [image.convert("RGB")] - - -def refcoco_seg_doc_to_visual(doc): - seg = doc["segmentation"] - image = doc["image"].convert("RGB") - draw = ImageDraw.Draw(image) - draw.polygon(seg) - return [image.convert("RGB")] - - -def refcoco_doc_to_text(doc): - # question = doc["question"] - # return f"{question}\nAnswer the question using a single word or phrase." - return "Provide a short description for this region." - - -def refcoco_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case coco_bleu), value: metric value - """ - pred = result[0] if len(result) > 0 else "" - ann_id = doc["question_id"] - data_dict = {"answer": doc["answer"], "pred": pred, "ann_id": ann_id} - return {f"refcoco_{metric}": data_dict for metric in COCO_METRICS} - - -def refcoco_aggregation_result(results, metric): - scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr")]#, (Spice(), "SPICE")] - scorers_dict = {s[1]: s for s in scorers} - - stored_results = [] - # In order to make the coco eval tools to successfully create index - # We need at least two dict in the dataset - # 'annotation' and 'images' - # 'annotation' exactly reproduce the original annotation - # 'images' however only need the image id which is contained in the file name - dataset = {"annotations": [], "images": []} - idx = 0 - ann_id = 0 - for result in results: - stored_results.append({"image_id": idx, "caption": result["pred"]}) - for s in result["answer"]: - dataset["annotations"].append({"image_id": idx, "caption": s, "id": ann_id}) - ann_id += 1 - - dataset["images"].append({"id": idx}) - idx += 1 - - coco = COCO() - # Manually create index here - coco.dataset = dataset - coco.createIndex() - - coco_result = coco.loadRes(stored_results) - coco_eval = COCOEvalCap(coco, coco_result) - - imgIds = coco_eval.params["image_id"] - gts = {} - res = {} - for imgId in imgIds: - gts[imgId] = coco_eval.coco.imgToAnns[imgId] - res[imgId] = coco_eval.cocoRes.imgToAnns[imgId] - - eval_logger.info("tokenization...") - tokenizer = PTBTokenizer() - gts = tokenizer.tokenize(gts) - res = tokenizer.tokenize(res) - - eval_logger.info(f"Computing {metric} scores...") - - score, scores = scorers_dict[metric][0].compute_score(gts, res) - # coco_eval.setEval(score, metric) - - # When metric is one of the Bleu, score will be a list - if type(score) == list: - n = int(metric.split("_")[-1]) - score = score[n - 1] - - return score - - -def refcoco_bleu4(results): - return refcoco_aggregation_result(results, "Bleu_4") - - -def refcoco_bleu3(results): - return refcoco_aggregation_result(results, "Bleu_3") - - -def refcoco_bleu2(results): - return refcoco_aggregation_result(results, "Bleu_2") - - -def refcoco_bleu1(results): - return refcoco_aggregation_result(results, "Bleu_1") - - -def refcoco_meteor(results): - return refcoco_aggregation_result(results, "METEOR") - - -def refcoco_rougel(results): - return refcoco_aggregation_result(results, "ROUGE_L") - - -def refcoco_cider(results): - return refcoco_aggregation_result(results, "CIDEr") - - -def refcoco_spice(results): - return refcoco_aggregation_result(results, "SPICE") diff --git a/lmms_eval/tasks/refcoco/utils_rec.py b/lmms_eval/tasks/refcoco/utils_rec.py deleted file mode 100644 index ec91bf9..0000000 --- a/lmms_eval/tasks/refcoco/utils_rec.py +++ /dev/null @@ -1,221 +0,0 @@ -import re -import logging -from datasets import Dataset - -eval_logger = logging.getLogger("lmms-eval") - -COCO_REC_METRICS = ["IoU", "ACC@0.1", "ACC@0.3", "ACC@0.5", "ACC@0.7", "ACC@0.9", "Center_ACC"] - - -def refcoco_bbox_rec_preprocess_dataset(dataset: Dataset): - # PIL image stored in dataset['image'] - # add `image_width` and `image_height` to the dataset - dataset = dataset.map(lambda x: {"image_width": x["image"].width, "image_height": x["image"].height}) - - # Original bbox format (top x, top y, width, height) - # Convert to (top-left x, top-left y, bottom-right x, bottom-right y) - # Normalize the bounding box coordinates to be between 0 and 1 - # using the image width and height - dataset = dataset.map( - lambda x: {"bbox": [x["bbox"][0] / x["image_width"], - x["bbox"][1] / x["image_height"], - (x["bbox"][0] + x["bbox"][2]) / x["image_width"], - (x["bbox"][1] + x["bbox"][3]) / x["image_height"]]} - ) - - # currently, the dataset has `answer` as a list of strings - # each answer should be its own row - # we will explode the dataset to have one row per answer - # duplicate the other columns - def explode_answers(example): - answers = example.pop('answer') - return [{'answer': answer, **example} for answer in answers] - - # Apply the function to each element, collecting the results - exploded_rows = [] - for example in dataset: - exploded_rows.extend(explode_answers(example)) - - # Create a new dataset from the exploded rows - new_dataset = Dataset.from_list(exploded_rows) - print(f"Exploded dataset from {len(dataset)} to {len(new_dataset)} rows") - - return new_dataset - - -def refcoco_bbox_rec_doc_to_visual(doc): - # Image is presented as is - image = doc["image"].convert("RGB") - return [image.convert("RGB")] - - -def refcoco_bbox_rec_doc_to_text(doc): - assert isinstance(doc['answer'], str), "Answer must be a string" - return "Bounding box coordinates are specified in the format (top-left x, top-left y, bottom-right x, bottom-right y). All values are floating point numbers bounded between 0 and 1. Please provide the bounding box coordinate of the region this sentence describes: " + doc['answer'] - - -def parse_float_sequence_within(input_str): - """ - Extract the first sequence of four floating-point numbers within square brackets from a string. - - Args: - input_str (str): A string that may contain a sequence of four floats within square brackets. - - Returns: - list: A list of four floats if the pattern is found, or a list of four zeros if the pattern is not found. - """ - # Define the regex pattern to find the first instance of four floats within square brackets - pattern = r'\[\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?)\s*\]' - - # Use re.search to find the first match of the pattern in the input string - match = re.search(pattern, input_str) - - # If a match is found, convert the captured groups into a list of floats - if match: - return [float(match.group(i)) for i in range(1, 5)] - - # If the input does not contain the pattern, return the null float sequence - return [0, 0, 0, 0] - - -def refcoco_bbox_rec_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name, value: metric value - """ - pred = result[0] if len(result) > 0 else "" - pred = parse_float_sequence_within(pred) - ann_id = doc["question_id"] - data_dict = {"answer": doc["answer"], "pred": pred, "ann_id": ann_id, 'bbox': doc['bbox']} - return {f"refcoco_{metric}": data_dict for metric in COCO_REC_METRICS} - - -def compute_iou(box1, box2): - """ - Compute the Intersection over Union (IoU) of two bounding boxes. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - Returns: - - float: IoU of box1 and box2. - """ - # Determine the coordinates of the intersection rectangle - x_left = max(box1[0], box2[0]) - y_top = max(box1[1], box2[1]) - x_right = min(box1[2], box2[2]) - y_bottom = min(box1[3], box2[3]) - - # Compute the area of intersection - intersection_area = max(0, x_right - x_left) * max(0, y_bottom - y_top) - - # Compute the area of both bounding boxes - box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1]) - box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1]) - - # Compute the area of the union - union_area = box1_area + box2_area - intersection_area - - # Compute the Intersection over Union - iou = intersection_area / union_area - - return iou - - -def compute_accuracy(box1, box2, threshold=0.5): - """ - Compute the accuracy of two bounding boxes based on a specified threshold. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - threshold (float): Threshold for the IoU to consider the prediction correct. - - Returns: - - float: Accuracy of the prediction based on the IoU threshold. - """ - iou = compute_iou(box1, box2) - return iou >= threshold - - -def compute_center_accuracy(box1, box2): - """ - Compute if the center point of box 2 is within box 1. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - Returns: - - bool: True if the center point of box 2 is within box 1, False otherwise. - """ - # Compute the center point of box 2 - center_x = (box2[0] + box2[2]) / 2 - center_y = (box2[1] + box2[3]) / 2 - - # Check if the center point is within box 1 - return box1[0] <= center_x <= box1[2] and box1[1] <= center_y <= box1[3] - - -def refcoco_bbox_rec_aggregation_result(results, metric): - """ - Aggregate the results of the RefCOCO evaluation task using the specified metric. - - Args: - - results (list of dict): List of result dictionaries. - - metric (str): Metric to use for aggregation. - - Returns: - - dict: Dictionary containing the aggregated results for the specified metric. - """ - scorers = { - 'IoU': compute_iou, - 'ACC@0.1': lambda x, y: compute_accuracy(x, y, 0.1), - 'ACC@0.3': lambda x, y: compute_accuracy(x, y, 0.3), - 'ACC@0.5': lambda x, y: compute_accuracy(x, y, 0.5), - 'ACC@0.7': lambda x, y: compute_accuracy(x, y, 0.7), - 'ACC@0.9': lambda x, y: compute_accuracy(x, y, 0.9), - 'Center_ACC': compute_center_accuracy - } - results_dict = {metric: []} - for result in results: - # Extract the ground truth and predicted bounding boxes - gt_bbox = result['bbox'] - pred_bbox = result['pred'] - # Compute the specified metric between the ground truth and predicted bounding boxes - score = scorers[metric](gt_bbox, pred_bbox) - results_dict[metric].append(score) - results_dict[metric] = sum(results_dict[metric]) / len(results_dict[metric]) - print(f"Aggregated {metric} score: {results_dict[metric]}") - return results_dict[metric] - - -def refcoco_bbox_rec_iou(results): - return refcoco_bbox_rec_aggregation_result(results, "IoU") - - -def refcoco_bbox_rec_acc01(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.1") - -def refcoco_bbox_rec_acc03(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.3") - - -def refcoco_bbox_rec_acc05(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.5") - - -def refcoco_bbox_rec_acc07(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.7") - - -def refcoco_bbox_rec_acc09(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.9") - - -def refcoco_bbox_rec_center_acc(results): - return refcoco_bbox_rec_aggregation_result(results, "Center_ACC") diff --git a/lmms_eval/tasks/refcocog/_default_template_bbox_rec_yaml b/lmms_eval/tasks/refcocog/_default_template_bbox_rec_yaml deleted file mode 100644 index ce95812..0000000 --- a/lmms_eval/tasks/refcocog/_default_template_bbox_rec_yaml +++ /dev/null @@ -1,34 +0,0 @@ -dataset_path: lmms-lab/RefCOCOg -output_type: generate_until -process_docs: !function utils_rec.refcoco_bbox_rec_preprocess_dataset -doc_to_visual: !function utils_rec.refcoco_bbox_rec_doc_to_visual -doc_to_text: !function utils_rec.refcoco_bbox_rec_doc_to_text -doc_to_target: "bbox" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils_rec.refcoco_bbox_rec_process_result -metric_list: - - metric: refcoco_IoU - aggregation : !function utils_rec.refcoco_bbox_rec_iou - higher_is_better : true - - metric: refcoco_ACC@0.1 - aggregation : !function utils_rec.refcoco_bbox_rec_acc01 - higher_is_better : true - - metric: refcoco_ACC@0.3 - aggregation : !function utils_rec.refcoco_bbox_rec_acc03 - higher_is_better : true - - metric: refcoco_ACC@0.5 - aggregation : !function utils_rec.refcoco_bbox_rec_acc05 - higher_is_better : true - - metric: refcoco_ACC@0.7 - aggregation : !function utils_rec.refcoco_bbox_rec_acc07 - higher_is_better : true - - metric: refcoco_ACC@0.9 - aggregation : !function utils_rec.refcoco_bbox_rec_acc09 - higher_is_better : true - - metric: refcoco_Center_ACC - aggregation : !function utils_rec.refcoco_bbox_rec_center_acc - higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/refcocog/_default_template_bbox_yaml b/lmms_eval/tasks/refcocog/_default_template_bbox_yaml deleted file mode 100644 index 9abd94f..0000000 --- a/lmms_eval/tasks/refcocog/_default_template_bbox_yaml +++ /dev/null @@ -1,36 +0,0 @@ -dataset_path: lmms-lab/RefCOCOg -output_type: generate_until -doc_to_visual: !function utils.refcoco_bbox_doc_to_visual -doc_to_text: !function utils.refcoco_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils.refcoco_process_result -metric_list: - - metric: refcoco_Bleu_4 - aggregation : !function utils.refcoco_bleu4 - higher_is_better : true - - metric: refcoco_Bleu_3 - aggregation : !function utils.refcoco_bleu3 - higher_is_better : true - - metric: refcoco_Bleu_2 - aggregation : !function utils.refcoco_bleu2 - higher_is_better : true - - metric: refcoco_Bleu_1 - aggregation : !function utils.refcoco_bleu1 - higher_is_better : true - - metric: refcoco_METEOR - aggregation : !function utils.refcoco_meteor - higher_is_better : true - - metric: refcoco_ROUGE_L - aggregation : !function utils.refcoco_rougel - higher_is_better : true - - metric: refcoco_CIDEr - aggregation : !function utils.refcoco_cider - higher_is_better : true - #- metric: refcoco_SPICE - # aggregation : !function utils.refcoco_spice - # higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/refcocog/_default_template_seg_yaml b/lmms_eval/tasks/refcocog/_default_template_seg_yaml deleted file mode 100644 index a3a291b..0000000 --- a/lmms_eval/tasks/refcocog/_default_template_seg_yaml +++ /dev/null @@ -1,36 +0,0 @@ -dataset_path: lmms-lab/RefCOCOg -output_type: generate_until -doc_to_visual: !function utils.refcoco_seg_doc_to_visual -doc_to_text: !function utils.refcoco_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils.refcoco_process_result -metric_list: - - metric: refcoco_Bleu_4 - aggregation : !function utils.refcoco_bleu4 - higher_is_better : true - - metric: refcoco_Bleu_3 - aggregation : !function utils.refcoco_bleu3 - higher_is_better : true - - metric: refcoco_Bleu_2 - aggregation : !function utils.refcoco_bleu2 - higher_is_better : true - - metric: refcoco_Bleu_1 - aggregation : !function utils.refcoco_bleu1 - higher_is_better : true - - metric: refcoco_METEOR - aggregation : !function utils.refcoco_meteor - higher_is_better : true - - metric: refcoco_ROUGE_L - aggregation : !function utils.refcoco_rougel - higher_is_better : true - - metric: refcoco_CIDEr - aggregation : !function utils.refcoco_cider - higher_is_better : true - #- metric: refcoco_SPICE - # aggregation : !function utils.refcoco_spice - # higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/refcocog/_generate_config.py b/lmms_eval/tasks/refcocog/_generate_config.py deleted file mode 100644 index 935d074..0000000 --- a/lmms_eval/tasks/refcocog/_generate_config.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import yaml - -# splits = ["train", "test", "val"] -splits = ["test", "val"] -tasks = ["seg", "bbox"] - -if __name__ == "__main__": - dump_tasks = [] - for task in tasks: - for split in splits: - yaml_dict = {"group": f"refcocog_{task}", "task": f"refcocog_{task}_{split}", "include": f"_default_template_{task}_yaml", "test_split": split} - if split == "train": - yaml_dict.pop("group") - else: - dump_tasks.append(f"refcoco_{task}_{split}") - - save_path = f"./refcocog_{task}_{split}.yaml" - print(f"Saving to {save_path}") - with open(save_path, "w") as f: - yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False) - - group_dict = {"group": "refcocog", "task": dump_tasks} - - with open("./_refcoco.yaml", "w") as f: - yaml.dump(group_dict, f, default_flow_style=False, indent=4) diff --git a/lmms_eval/tasks/refcocog/_refcoco.yaml b/lmms_eval/tasks/refcocog/_refcoco.yaml deleted file mode 100644 index 1c8c398..0000000 --- a/lmms_eval/tasks/refcocog/_refcoco.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: refcocog -task: -- refcocog_seg_test -- refcocog_seg_val -- refcocog_bbox_test -- refcocog_bbox_val diff --git a/lmms_eval/tasks/refcocog/refcocog_bbox_rec_test.yaml b/lmms_eval/tasks/refcocog/refcocog_bbox_rec_test.yaml deleted file mode 100644 index 2f43597..0000000 --- a/lmms_eval/tasks/refcocog/refcocog_bbox_rec_test.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcocog_bbox_rec -task: refcocog_bbox_rec_test -include: _default_template_bbox_rec_yaml -test_split: test diff --git a/lmms_eval/tasks/refcocog/refcocog_bbox_rec_val.yaml b/lmms_eval/tasks/refcocog/refcocog_bbox_rec_val.yaml deleted file mode 100644 index 5e19397..0000000 --- a/lmms_eval/tasks/refcocog/refcocog_bbox_rec_val.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcocog_bbox_rec -task: refcocog_bbox_rec_val -include: _default_template_bbox_rec_yaml -test_split: val diff --git a/lmms_eval/tasks/refcocog/refcocog_bbox_test.yaml b/lmms_eval/tasks/refcocog/refcocog_bbox_test.yaml deleted file mode 100644 index f29fb57..0000000 --- a/lmms_eval/tasks/refcocog/refcocog_bbox_test.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcocog_bbox -task: refcocog_bbox_test -include: _default_template_bbox_yaml -test_split: test diff --git a/lmms_eval/tasks/refcocog/refcocog_bbox_val.yaml b/lmms_eval/tasks/refcocog/refcocog_bbox_val.yaml deleted file mode 100644 index 5a62900..0000000 --- a/lmms_eval/tasks/refcocog/refcocog_bbox_val.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcocog_bbox -task: refcocog_bbox_val -include: _default_template_bbox_yaml -test_split: val diff --git a/lmms_eval/tasks/refcocog/refcocog_seg_test.yaml b/lmms_eval/tasks/refcocog/refcocog_seg_test.yaml deleted file mode 100644 index a23efd8..0000000 --- a/lmms_eval/tasks/refcocog/refcocog_seg_test.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcocog_seg -task: refcocog_seg_test -include: _default_template_seg_yaml -test_split: test diff --git a/lmms_eval/tasks/refcocog/refcocog_seg_val.yaml b/lmms_eval/tasks/refcocog/refcocog_seg_val.yaml deleted file mode 100644 index 93edf7d..0000000 --- a/lmms_eval/tasks/refcocog/refcocog_seg_val.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: refcocog_seg -task: refcocog_seg_val -include: _default_template_seg_yaml -test_split: val diff --git a/lmms_eval/tasks/refcocog/utils.py b/lmms_eval/tasks/refcocog/utils.py deleted file mode 100644 index 4feb71c..0000000 --- a/lmms_eval/tasks/refcocog/utils.py +++ /dev/null @@ -1,135 +0,0 @@ -from PIL import ImageDraw -from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice -from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer -from pycocotools.coco import COCO - -COCO_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"] # , "SPICE"] - -import logging - -eval_logger = logging.getLogger("lmms-eval") - - -def refcoco_bbox_doc_to_visual(doc): - bbox = doc["bbox"] - image = doc["image"].convert("RGB") - draw = ImageDraw.Draw(image) - # Origin format (top x, top y, width, height) - bbox_xy = [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]] - draw.rectangle(bbox_xy, outline="red") - return [image.convert("RGB")] - - -def refcoco_seg_doc_to_visual(doc): - seg = doc["segmentation"] - image = doc["image"].convert("RGB") - draw = ImageDraw.Draw(image) - draw.polygon(seg) - return [image.convert("RGB")] - - -def refcoco_doc_to_text(doc): - # question = doc["question"] - # return f"{question}\nAnswer the question using a single word or phrase." - return "Provide a short description for this region." - - -def refcoco_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case coco_bleu), value: metric value - """ - pred = result[0] if len(result) > 0 else "" - ann_id = doc["question_id"] - data_dict = {"answer": doc["answer"], "pred": pred, "ann_id": ann_id} - return {f"refcoco_{metric}": data_dict for metric in COCO_METRICS} - - -def refcoco_aggregation_result(results, metric): - scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr")]#, (Spice(), "SPICE")] - scorers_dict = {s[1]: s for s in scorers} - - stored_results = [] - # In order to make the coco eval tools to successfully create index - # We need at least two dict in the dataset - # 'annotation' and 'images' - # 'annotation' exactly reproduce the original annotation - # 'images' however only need the image id which is contained in the file name - dataset = {"annotations": [], "images": []} - idx = 0 - ann_id = 0 - for result in results: - stored_results.append({"image_id": idx, "caption": result["pred"]}) - for s in result["answer"]: - dataset["annotations"].append({"image_id": idx, "caption": s, "id": ann_id}) - ann_id += 1 - - dataset["images"].append({"id": idx}) - idx += 1 - - coco = COCO() - # Manually create index here - coco.dataset = dataset - coco.createIndex() - - coco_result = coco.loadRes(stored_results) - coco_eval = COCOEvalCap(coco, coco_result) - - imgIds = coco_eval.params["image_id"] - gts = {} - res = {} - for imgId in imgIds: - gts[imgId] = coco_eval.coco.imgToAnns[imgId] - res[imgId] = coco_eval.cocoRes.imgToAnns[imgId] - - eval_logger.info("tokenization...") - tokenizer = PTBTokenizer() - gts = tokenizer.tokenize(gts) - res = tokenizer.tokenize(res) - - eval_logger.info(f"Computing {metric} scores...") - - score, scores = scorers_dict[metric][0].compute_score(gts, res) - # coco_eval.setEval(score, metric) - - # When metric is one of the Bleu, score will be a list - if type(score) == list: - n = int(metric.split("_")[-1]) - score = score[n - 1] - - return score - - -def refcoco_bleu4(results): - return refcoco_aggregation_result(results, "Bleu_4") - - -def refcoco_bleu3(results): - return refcoco_aggregation_result(results, "Bleu_3") - - -def refcoco_bleu2(results): - return refcoco_aggregation_result(results, "Bleu_2") - - -def refcoco_bleu1(results): - return refcoco_aggregation_result(results, "Bleu_1") - - -def refcoco_meteor(results): - return refcoco_aggregation_result(results, "METEOR") - - -def refcoco_rougel(results): - return refcoco_aggregation_result(results, "ROUGE_L") - - -def refcoco_cider(results): - return refcoco_aggregation_result(results, "CIDEr") - - -def refcoco_spice(results): - return refcoco_aggregation_result(results, "SPICE") diff --git a/lmms_eval/tasks/refcocog/utils_rec.py b/lmms_eval/tasks/refcocog/utils_rec.py deleted file mode 100644 index ec91bf9..0000000 --- a/lmms_eval/tasks/refcocog/utils_rec.py +++ /dev/null @@ -1,221 +0,0 @@ -import re -import logging -from datasets import Dataset - -eval_logger = logging.getLogger("lmms-eval") - -COCO_REC_METRICS = ["IoU", "ACC@0.1", "ACC@0.3", "ACC@0.5", "ACC@0.7", "ACC@0.9", "Center_ACC"] - - -def refcoco_bbox_rec_preprocess_dataset(dataset: Dataset): - # PIL image stored in dataset['image'] - # add `image_width` and `image_height` to the dataset - dataset = dataset.map(lambda x: {"image_width": x["image"].width, "image_height": x["image"].height}) - - # Original bbox format (top x, top y, width, height) - # Convert to (top-left x, top-left y, bottom-right x, bottom-right y) - # Normalize the bounding box coordinates to be between 0 and 1 - # using the image width and height - dataset = dataset.map( - lambda x: {"bbox": [x["bbox"][0] / x["image_width"], - x["bbox"][1] / x["image_height"], - (x["bbox"][0] + x["bbox"][2]) / x["image_width"], - (x["bbox"][1] + x["bbox"][3]) / x["image_height"]]} - ) - - # currently, the dataset has `answer` as a list of strings - # each answer should be its own row - # we will explode the dataset to have one row per answer - # duplicate the other columns - def explode_answers(example): - answers = example.pop('answer') - return [{'answer': answer, **example} for answer in answers] - - # Apply the function to each element, collecting the results - exploded_rows = [] - for example in dataset: - exploded_rows.extend(explode_answers(example)) - - # Create a new dataset from the exploded rows - new_dataset = Dataset.from_list(exploded_rows) - print(f"Exploded dataset from {len(dataset)} to {len(new_dataset)} rows") - - return new_dataset - - -def refcoco_bbox_rec_doc_to_visual(doc): - # Image is presented as is - image = doc["image"].convert("RGB") - return [image.convert("RGB")] - - -def refcoco_bbox_rec_doc_to_text(doc): - assert isinstance(doc['answer'], str), "Answer must be a string" - return "Bounding box coordinates are specified in the format (top-left x, top-left y, bottom-right x, bottom-right y). All values are floating point numbers bounded between 0 and 1. Please provide the bounding box coordinate of the region this sentence describes: " + doc['answer'] - - -def parse_float_sequence_within(input_str): - """ - Extract the first sequence of four floating-point numbers within square brackets from a string. - - Args: - input_str (str): A string that may contain a sequence of four floats within square brackets. - - Returns: - list: A list of four floats if the pattern is found, or a list of four zeros if the pattern is not found. - """ - # Define the regex pattern to find the first instance of four floats within square brackets - pattern = r'\[\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?)\s*\]' - - # Use re.search to find the first match of the pattern in the input string - match = re.search(pattern, input_str) - - # If a match is found, convert the captured groups into a list of floats - if match: - return [float(match.group(i)) for i in range(1, 5)] - - # If the input does not contain the pattern, return the null float sequence - return [0, 0, 0, 0] - - -def refcoco_bbox_rec_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name, value: metric value - """ - pred = result[0] if len(result) > 0 else "" - pred = parse_float_sequence_within(pred) - ann_id = doc["question_id"] - data_dict = {"answer": doc["answer"], "pred": pred, "ann_id": ann_id, 'bbox': doc['bbox']} - return {f"refcoco_{metric}": data_dict for metric in COCO_REC_METRICS} - - -def compute_iou(box1, box2): - """ - Compute the Intersection over Union (IoU) of two bounding boxes. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - Returns: - - float: IoU of box1 and box2. - """ - # Determine the coordinates of the intersection rectangle - x_left = max(box1[0], box2[0]) - y_top = max(box1[1], box2[1]) - x_right = min(box1[2], box2[2]) - y_bottom = min(box1[3], box2[3]) - - # Compute the area of intersection - intersection_area = max(0, x_right - x_left) * max(0, y_bottom - y_top) - - # Compute the area of both bounding boxes - box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1]) - box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1]) - - # Compute the area of the union - union_area = box1_area + box2_area - intersection_area - - # Compute the Intersection over Union - iou = intersection_area / union_area - - return iou - - -def compute_accuracy(box1, box2, threshold=0.5): - """ - Compute the accuracy of two bounding boxes based on a specified threshold. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - threshold (float): Threshold for the IoU to consider the prediction correct. - - Returns: - - float: Accuracy of the prediction based on the IoU threshold. - """ - iou = compute_iou(box1, box2) - return iou >= threshold - - -def compute_center_accuracy(box1, box2): - """ - Compute if the center point of box 2 is within box 1. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - Returns: - - bool: True if the center point of box 2 is within box 1, False otherwise. - """ - # Compute the center point of box 2 - center_x = (box2[0] + box2[2]) / 2 - center_y = (box2[1] + box2[3]) / 2 - - # Check if the center point is within box 1 - return box1[0] <= center_x <= box1[2] and box1[1] <= center_y <= box1[3] - - -def refcoco_bbox_rec_aggregation_result(results, metric): - """ - Aggregate the results of the RefCOCO evaluation task using the specified metric. - - Args: - - results (list of dict): List of result dictionaries. - - metric (str): Metric to use for aggregation. - - Returns: - - dict: Dictionary containing the aggregated results for the specified metric. - """ - scorers = { - 'IoU': compute_iou, - 'ACC@0.1': lambda x, y: compute_accuracy(x, y, 0.1), - 'ACC@0.3': lambda x, y: compute_accuracy(x, y, 0.3), - 'ACC@0.5': lambda x, y: compute_accuracy(x, y, 0.5), - 'ACC@0.7': lambda x, y: compute_accuracy(x, y, 0.7), - 'ACC@0.9': lambda x, y: compute_accuracy(x, y, 0.9), - 'Center_ACC': compute_center_accuracy - } - results_dict = {metric: []} - for result in results: - # Extract the ground truth and predicted bounding boxes - gt_bbox = result['bbox'] - pred_bbox = result['pred'] - # Compute the specified metric between the ground truth and predicted bounding boxes - score = scorers[metric](gt_bbox, pred_bbox) - results_dict[metric].append(score) - results_dict[metric] = sum(results_dict[metric]) / len(results_dict[metric]) - print(f"Aggregated {metric} score: {results_dict[metric]}") - return results_dict[metric] - - -def refcoco_bbox_rec_iou(results): - return refcoco_bbox_rec_aggregation_result(results, "IoU") - - -def refcoco_bbox_rec_acc01(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.1") - -def refcoco_bbox_rec_acc03(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.3") - - -def refcoco_bbox_rec_acc05(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.5") - - -def refcoco_bbox_rec_acc07(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.7") - - -def refcoco_bbox_rec_acc09(results): - return refcoco_bbox_rec_aggregation_result(results, "ACC@0.9") - - -def refcoco_bbox_rec_center_acc(results): - return refcoco_bbox_rec_aggregation_result(results, "Center_ACC") diff --git a/lmms_eval/tasks/scienceqa/scienceqa.yaml b/lmms_eval/tasks/scienceqa/scienceqa.yaml deleted file mode 100644 index 8190e3e..0000000 --- a/lmms_eval/tasks/scienceqa/scienceqa.yaml +++ /dev/null @@ -1,36 +0,0 @@ -dataset_path: lmms-lab/ScienceQA -dataset_name: ScienceQA-FULL -task: "scienceqa" -dataset_kwargs: - token: True -test_split: test -output_type: generate_until -doc_to_visual: !function utils.sqa_doc_to_visual -doc_to_text: !function utils.sqa_doc_to_text -doc_to_target: !function utils.sqa_doc_to_target -generation_kwargs: - max_new_tokens: 16 - temperature: 0 - do_sample: False -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true -process_results: !function utils.sqa_process_results -metadata: - - version: 0.0 - -model_specific_prompt_kwargs: - default: - format: default - pre_prompt: "" - post_prompt: "\nAnswer with the option's letter from the given choices directly." - qwen_vl: - format: qwen_vl - -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original - diff --git a/lmms_eval/tasks/scienceqa/scienceqa_full.yaml b/lmms_eval/tasks/scienceqa/scienceqa_full.yaml deleted file mode 100644 index 5f10bf4..0000000 --- a/lmms_eval/tasks/scienceqa/scienceqa_full.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: scienceqa_full -task: - - scienceqa - - scienceqa_img \ No newline at end of file diff --git a/lmms_eval/tasks/scienceqa/scienceqa_img.yaml b/lmms_eval/tasks/scienceqa/scienceqa_img.yaml deleted file mode 100644 index b6c7dcc..0000000 --- a/lmms_eval/tasks/scienceqa/scienceqa_img.yaml +++ /dev/null @@ -1,39 +0,0 @@ -dataset_path: lmms-lab/ScienceQA -dataset_name: ScienceQA-IMG -task: "scienceqa_img" -dataset_kwargs: - token: True -test_split: test -output_type: generate_until -doc_to_visual: !function utils.sqa_doc_to_visual -doc_to_text: !function utils.sqa_doc_to_text -doc_to_target: !function utils.sqa_doc_to_target -generation_kwargs: - max_new_tokens: 16 - temperature: 0 - do_sample: False -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true -process_results: !function utils.sqa_process_results -metadata: - - version: 0.0 - -model_specific_prompt_kwargs: - default: - format: default - pre_prompt: "" - post_prompt: "\nAnswer with the option's letter from the given choices directly." - qwen_vl: - format: qwen_vl - idefics2: - format: default - pre_prompt: "" - post_prompt: "\nAnswer:" -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original - diff --git a/lmms_eval/tasks/scienceqa/utils.py b/lmms_eval/tasks/scienceqa/utils.py deleted file mode 100644 index eed6b26..0000000 --- a/lmms_eval/tasks/scienceqa/utils.py +++ /dev/null @@ -1,44 +0,0 @@ -def sqa_doc_to_text(doc, model_specific_prompt_kwargs=None): - context, question, choices = doc["hint"], doc["question"], doc["choices"] - len_choices = len(choices) - options = [chr(ord("A") + i) for i in range(len_choices)] - choices_str = "\n".join([f"{option}. {choice}" for option, choice in zip(options, choices)]) - if model_specific_prompt_kwargs["format"] == "default": - if context: - context = f"Context: {context}\n" - - post_prompt = model_specific_prompt_kwargs["post_prompt"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - return f"{pre_prompt}{context}{question}\n{choices_str}{post_prompt}" - elif model_specific_prompt_kwargs["format"] == "qwen_vl": - prompt = "Context: {}\nQuestion: {}\nOptions: {}\nAnswer:" - context = context if context else "N/A" - prompt = prompt.format(context, question, choices_str) - return prompt - else: - raise ValueError(f"Unknown prompt format: {model_specific_prompt_kwargs}") - - -def sqa_doc_to_visual(doc): - if doc["image"] is None: - return [] - return [doc["image"].convert("RGB")] - - -def sqa_doc_to_target(doc): - len_choices = len(doc["choices"]) - options = [chr(ord("A") + i) for i in range(len_choices)] - return options[doc["answer"]] - - -def sqa_process_results(doc, results): - # I know this is weird, but it's how llava parse it. - target = sqa_doc_to_target(doc) - pred = results[0] - if pred == target: - return {"exact_match": 1.0} - # pattern: ^[A-Z]\. .* - if len(pred) >= 2 and pred[0].isupper() and pred[1] == ".": - result = 1.0 if pred[0] == target else 0.0 - return {"exact_match": result} - return {"exact_match": 0.0} diff --git a/lmms_eval/tasks/screenspot/README.md b/lmms_eval/tasks/screenspot/README.md deleted file mode 100644 index 840b36c..0000000 --- a/lmms_eval/tasks/screenspot/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# SceenSpot - -## GUI Grounding Benchmark: ScreenSpot - -ScreenSpot is an evaluation benchmark for GUI grounding, comprising over 1200 instructions from iOS, Android, macOS, Windows and Web environments, along with annotated element types (Text or Icon/Widget). - - -## Groups - -- `screenspot`: This group bundles both the original grounding task and the new instruction generation task. - -## Tasks -- `screenspot_rec_test`: the original evaluation of `{img} {instruction} --> {bounding box}` called grounding or Referring Expression Completion (REC); -- `screenspot_reg_test`: the new evaluation of `{img} {bounding box} --> {instruction}` called instruction generation or Referring Expression Generation (REG). - -### REC Metrics - -REC/Grounding requires that a model outputs a bounding box for the target element in the image. The evaluation metrics are: -- `IoU`: Intersection over Union (IoU) between the predicted bounding box and the ground truth bounding box. -- `ACC@IoIU`: We use `IoU` to create `ACC@IoU` metrics at different IoU thresholds where an output with an IoU above the threshold is considered correct. -- `CENTER ACC`: The predicted bounding box is considered correct if the center of the predicted bounding box is within the ground truth bounding box. This is what's reported in the paper. - -### REG Metrics - -REG/Generation requires that a model outputs the instruction that describes the target element in the image. Currently, this element will be highlighted in red in the image. The evaluation metrics are: -- `CIDEr`: The CIDEr metric is used to evaluate the quality of the generated instruction. As the paper doesn't consider this task, we have selected this metric as a standard for evaluating the quality of the generated instruction. This matches with what other works like ScreenAI have done for instruction generation for RICO datasets. - -## Baseline Scores - -As a Baseline, here is how LLaVA-v1.5-7b performs on the ScreenSpot dataset: -- `IoU`: 0.051 -- `ACC@0.1`: 0.195 -- `ACC@0.3`: 0.042 -- `ACC@0.5`: 0.006 -- `ACC@0.7`: 0.000 -- `ACC@0.9`: 0.000 -- `CENTER ACC`: 0.097 -- `CIDEr`: 0.097 - -## References - -- ArXiv: [SeeClick: Harnessing GUI Grounding for Advanced Visual GUI Agents](https://arxiv.org/abs/2401.10935) -- GitHub: [njucckevin/SeeClick](https://github.com/njucckevin/SeeClick) - -```bibtex -@misc{cheng2024seeclick, - title={SeeClick: Harnessing GUI Grounding for Advanced Visual GUI Agents}, - author={Kanzhi Cheng and Qiushi Sun and Yougang Chu and Fangzhi Xu and Yantao Li and Jianbing Zhang and Zhiyong Wu}, - year={2024}, - eprint={2401.10935}, - archivePrefix={arXiv}, - primaryClass={cs.HC} -} -``` diff --git a/lmms_eval/tasks/screenspot/_default_template_rec_yaml b/lmms_eval/tasks/screenspot/_default_template_rec_yaml deleted file mode 100644 index 7f9aad5..0000000 --- a/lmms_eval/tasks/screenspot/_default_template_rec_yaml +++ /dev/null @@ -1,33 +0,0 @@ -dataset_path: rootsautomation/ScreenSpot -output_type: generate_until -doc_to_visual: !function utils_rec.screenspot_rec_doc_to_visual -doc_to_text: !function utils_rec.screenspot_rec_doc_to_text -doc_to_target: "bbox" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils_rec.screenspot_rec_process_result -metric_list: - - metric: screenspot_IoU - aggregation : !function utils_rec.screenspot_rec_iou - higher_is_better : true - - metric: screenspot_ACC@0.1 - aggregation : !function utils_rec.screenspot_rec_acc01 - higher_is_better : true - - metric: screenspot_ACC@0.3 - aggregation : !function utils_rec.screenspot_rec_acc03 - higher_is_better : true - - metric: screenspot_ACC@0.5 - aggregation : !function utils_rec.screenspot_rec_acc05 - higher_is_better : true - - metric: screenspot_ACC@0.7 - aggregation : !function utils_rec.screenspot_rec_acc07 - higher_is_better : true - - metric: screenspot_ACC@0.9 - aggregation : !function utils_rec.screenspot_rec_acc09 - higher_is_better : true - - metric: screenspot_Center_ACC - aggregation : !function utils_rec.screenspot_rec_center_acc - higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/screenspot/_default_template_reg_yaml b/lmms_eval/tasks/screenspot/_default_template_reg_yaml deleted file mode 100644 index 0ef7205..0000000 --- a/lmms_eval/tasks/screenspot/_default_template_reg_yaml +++ /dev/null @@ -1,15 +0,0 @@ -dataset_path: rootsautomation/ScreenSpot -output_type: generate_until -doc_to_visual: !function utils.screenspot_bbox_doc_to_visual -doc_to_text: !function utils.screenspot_doc_to_text -doc_to_target: "instruction" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils.screenspot_process_result -metric_list: - - metric: screenspot_CIDEr - aggregation : !function utils.screenspot_cider - higher_is_better : true -metadata: - version: '0.0' \ No newline at end of file diff --git a/lmms_eval/tasks/screenspot/_screenspot.yaml b/lmms_eval/tasks/screenspot/_screenspot.yaml deleted file mode 100644 index c31f58d..0000000 --- a/lmms_eval/tasks/screenspot/_screenspot.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: screenspot -task: -- screenspot_reg_test -- screenspot_rec_test \ No newline at end of file diff --git a/lmms_eval/tasks/screenspot/screenspot_rec_test.yaml b/lmms_eval/tasks/screenspot/screenspot_rec_test.yaml deleted file mode 100644 index baccc82..0000000 --- a/lmms_eval/tasks/screenspot/screenspot_rec_test.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: screenspot_rec -task: screenspot_rec_test -include: _default_template_rec_yaml -test_split: test diff --git a/lmms_eval/tasks/screenspot/screenspot_reg_test.yaml b/lmms_eval/tasks/screenspot/screenspot_reg_test.yaml deleted file mode 100644 index ab06471..0000000 --- a/lmms_eval/tasks/screenspot/screenspot_reg_test.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: screenspot_reg -task: screenspot_reg_test -include: _default_template_reg_yaml -test_split: test diff --git a/lmms_eval/tasks/screenspot/utils.py b/lmms_eval/tasks/screenspot/utils.py deleted file mode 100644 index cb6ceda..0000000 --- a/lmms_eval/tasks/screenspot/utils.py +++ /dev/null @@ -1,126 +0,0 @@ -from PIL import ImageDraw -from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice -from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer -from pycocotools.coco import COCO - -# COCO_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"] # , "SPICE"] -COCO_METRICS = ["CIDEr"] - -import logging - -eval_logger = logging.getLogger("lmms-eval") - - -def screenspot_bbox_doc_to_visual(doc): - bbox = doc["bbox"] - image = doc["image"].convert("RGB") - draw = ImageDraw.Draw(image) - bbox_xy = [bbox[0], bbox[1], bbox[2], bbox[3]] - draw.rectangle(bbox_xy, outline="red", width=3) - return [image.convert("RGB")] - - -def screenspot_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case coco_bleu), value: metric value - """ - pred = result[0] if len(result) > 0 else "" - ann_id = doc["file_name"] - data_dict = {"instruction": doc["instruction"], "pred": pred, "ann_id": ann_id, 'data_type': doc['data_type'], 'data_source': doc['data_source']} - return {f"screenspot_{metric}": data_dict for metric in COCO_METRICS} - - -def screenspot_doc_to_text(doc): - return f"Direct a user to interact with the highlighted region [{doc['bbox'][0]:.2f}, {doc['bbox'][1]:.2f}, {doc['bbox'][2]:.2f}, {doc['bbox'][3]:.2f}]." - - -def screenspot_aggregation_result(results, metric): - # scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr"), (Spice(), "SPICE")] - scorers = [(Cider(), "CIDEr")] - scorers_dict = {s[1]: s for s in scorers} - - stored_results = [] - # In order to make the coco eval tools to successfully create index - # We need at least two dict in the dataset - # 'annotation' and 'images' - # 'annotation' exactly reproduce the original annotation - # 'images' however only need the image id which is contained in the file name - dataset = {"annotations": [], "images": []} - idx = 0 - ann_id = 0 - for result in results: - stored_results.append({"image_id": idx, "caption": result["pred"]}) - # for s in result["answer"]: - dataset["annotations"].append({"image_id": idx, "caption": result['instruction'], "id": ann_id}) - ann_id += 1 - - dataset["images"].append({"id": idx}) - idx += 1 - - coco = COCO() - # Manually create index here - coco.dataset = dataset - coco.createIndex() - - coco_result = coco.loadRes(stored_results) - coco_eval = COCOEvalCap(coco, coco_result) - - imgIds = coco_eval.params["image_id"] - gts = {} - res = {} - for imgId in imgIds: - gts[imgId] = coco_eval.coco.imgToAnns[imgId] - res[imgId] = coco_eval.cocoRes.imgToAnns[imgId] - - eval_logger.info("tokenization...") - tokenizer = PTBTokenizer() - gts = tokenizer.tokenize(gts) - res = tokenizer.tokenize(res) - - eval_logger.info(f"Computing {metric} scores...") - - score, scores = scorers_dict[metric][0].compute_score(gts, res) - # coco_eval.setEval(score, metric) - - # When metric is one of the Bleu, score will be a list - if type(score) == list: - n = int(metric.split("_")[-1]) - score = score[n - 1] - - return score - - -def screenspot_bleu4(results): - return screenspot_aggregation_result(results, "Bleu_4") - - -def screenspot_bleu3(results): - return screenspot_aggregation_result(results, "Bleu_3") - - -def screenspot_bleu2(results): - return screenspot_aggregation_result(results, "Bleu_2") - - -def screenspot_bleu1(results): - return screenspot_aggregation_result(results, "Bleu_1") - - -def screenspot_meteor(results): - return screenspot_aggregation_result(results, "METEOR") - - -def screenspot_rougel(results): - return screenspot_aggregation_result(results, "ROUGE_L") - - -def screenspot_cider(results): - return screenspot_aggregation_result(results, "CIDEr") - - -def screenspot_spice(results): - return screenspot_aggregation_result(results, "SPICE") diff --git a/lmms_eval/tasks/screenspot/utils_rec.py b/lmms_eval/tasks/screenspot/utils_rec.py deleted file mode 100644 index 5cdf5b6..0000000 --- a/lmms_eval/tasks/screenspot/utils_rec.py +++ /dev/null @@ -1,215 +0,0 @@ -import re -import logging -from datasets import Dataset - -eval_logger = logging.getLogger("lmms-eval") - -REC_METRICS = ["IoU", "ACC@0.1", "ACC@0.3", "ACC@0.5", "ACC@0.7", "ACC@0.9", "Center_ACC"] - - - -def screenspot_rec_doc_to_visual(doc): - # Image is presented as is - image = doc["image"].convert("RGB") - return [image.convert("RGB")] - - -def screenspot_rec_doc_to_text(doc): - return "Bounding box coordinates are specified in the format (top-left x, top-left y, bottom-right x, bottom-right y). All values are floating point numbers bounded between 0 and 1 with two decimal places of precision (e.g., 0.15). Please provide the bounding box coordinates of the region that corresponds to the command: " + doc["instruction"] - - -def parse_float_sequence_within(input_str): - """ - Extract the first sequence of four floating-point numbers within square brackets from a string. - - Args: - input_str (str): A string that may contain a sequence of four floats within square brackets. - - Returns: - list: A list of four floats if the pattern is found, or a list of four zeros if the pattern is not found. - """ - # Define the regex pattern to find the first instance of four floats within square brackets - pattern = r'\[\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?)\s*\]' - - # Use re.search to find the first match of the pattern in the input string - match = re.search(pattern, input_str) - - # If a match is found, convert the captured groups into a list of floats - if match: - return [float(match.group(i)) for i in range(1, 5)] - - # If the input does not contain the pattern, return the null float sequence - return [0, 0, 0, 0] - - -def screenspot_rec_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name, value: metric value - """ - pred = result[0] if len(result) > 0 else "" - pred = parse_float_sequence_within(pred) - ann_id = doc["file_name"] - data_dict = {"instruction": doc["instruction"], "pred": pred, "ann_id": ann_id, 'bbox': doc['bbox'], 'data_type': doc['data_type'], 'data_source': doc['data_source']} - return {f"screenspot_{metric}": data_dict for metric in REC_METRICS} - - -def compute_iou(box1, box2): - """ - Compute the Intersection over Union (IoU) of two bounding boxes. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - Returns: - - float: IoU of box1 and box2. - """ - # Determine the coordinates of the intersection rectangle - x_left = max(box1[0], box2[0]) - y_top = max(box1[1], box2[1]) - x_right = min(box1[2], box2[2]) - y_bottom = min(box1[3], box2[3]) - - # Compute the area of intersection - intersection_area = max(0, x_right - x_left) * max(0, y_bottom - y_top) - - # Compute the area of both bounding boxes - box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1]) - box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1]) - - # Compute the area of the union - union_area = box1_area + box2_area - intersection_area - - # Compute the Intersection over Union - iou = intersection_area / union_area - - return iou - - -def compute_accuracy(box1, box2, threshold=0.5): - """ - Compute the accuracy of two bounding boxes based on a specified threshold. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - threshold (float): Threshold for the IoU to consider the prediction correct. - - Returns: - - float: Accuracy of the prediction based on the IoU threshold. - """ - iou = compute_iou(box1, box2) - return iou >= threshold - - -def compute_center_accuracy(box1, box2): - """ - Compute if the center point of box 2 is within box 1. - - Parameters: - - box1 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - box2 (list of float): Bounding box [x_min, y_min, x_max, y_max]. - - Returns: - - bool: True if the center point of box 2 is within box 1, False otherwise. - """ - # Compute the center point of box 2 - center_x = (box2[0] + box2[2]) / 2 - center_y = (box2[1] + box2[3]) / 2 - - # Check if the center point is within box 1 - return box1[0] <= center_x <= box1[2] and box1[1] <= center_y <= box1[3] - - -def screenspot_rec_aggregation_result(results, metric): - """ - Aggregate the results of the screenspot evaluation task using the specified metric. - - Args: - - results (list of dict): List of result dictionaries. - - metric (str): Metric to use for aggregation. - - Returns: - - dict: Dictionary containing the aggregated results for the specified metric. - """ - scorers = { - 'IoU': compute_iou, - 'ACC@0.1': lambda x, y: compute_accuracy(x, y, 0.1), - 'ACC@0.3': lambda x, y: compute_accuracy(x, y, 0.3), - 'ACC@0.5': lambda x, y: compute_accuracy(x, y, 0.5), - 'ACC@0.7': lambda x, y: compute_accuracy(x, y, 0.7), - 'ACC@0.9': lambda x, y: compute_accuracy(x, y, 0.9), - 'Center_ACC': compute_center_accuracy - } - results_dict = { - metric: [], - metric + '-mobile_text': [], - metric + '-mobile_icon': [], - metric + '-web_text': [], - metric + '-web_icon': [], - metric + '-desktop_text': [], - metric + '-desktop_icon': [], - } - for result in results: - # Extract the ground truth and predicted bounding boxes - gt = result['bbox'] - pred = result['pred'] - - # Compute the specified metric between the ground truth and predicted bounding boxes - score = scorers[metric](gt, pred) - - results_dict[metric].append(score) - if result['data_type'] == 'text': - if 'ios' in result['data_source'] or 'android' in result['data_source']: - results_dict[metric + '-mobile_text'].append(score) - elif 'macos' in result['data_source'] or 'windows' in result['data_source']: - results_dict[metric + '-desktop_text'].append(score) - else: - results_dict[metric + '-web_text'].append(score) - else: - if 'ios' in result['data_source'] or 'android' in result['data_source']: - results_dict[metric + '-mobile_icon'].append(score) - elif 'macos' in result['data_source'] or 'windows' in result['data_source']: - results_dict[metric + '-desktop_icon'].append(score) - else: - results_dict[metric + '-web_icon'].append(score) - - for key in results_dict: - if len(results_dict[key]) == 0: - results_dict[key] = 0 - else: - results_dict[key] = sum(results_dict[key]) / len(results_dict[key]) - - print(f"{key}: {results_dict[key]:0.4f}") - return results_dict[metric] - - -def screenspot_rec_iou(results): - return screenspot_rec_aggregation_result(results, "IoU") - - -def screenspot_rec_acc01(results): - return screenspot_rec_aggregation_result(results, "ACC@0.1") - -def screenspot_rec_acc03(results): - return screenspot_rec_aggregation_result(results, "ACC@0.3") - - -def screenspot_rec_acc05(results): - return screenspot_rec_aggregation_result(results, "ACC@0.5") - - -def screenspot_rec_acc07(results): - return screenspot_rec_aggregation_result(results, "ACC@0.7") - - -def screenspot_rec_acc09(results): - return screenspot_rec_aggregation_result(results, "ACC@0.9") - - -def screenspot_rec_center_acc(results): - return screenspot_rec_aggregation_result(results, "Center_ACC") diff --git a/lmms_eval/tasks/seedbench/seedbench.yaml b/lmms_eval/tasks/seedbench/seedbench.yaml deleted file mode 100644 index 371d0ba..0000000 --- a/lmms_eval/tasks/seedbench/seedbench.yaml +++ /dev/null @@ -1,28 +0,0 @@ -dataset_path: lmms-lab/SEED-Bench -dataset_kwargs: - token: True -task: "seedbench" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.seed_doc_to_visual -doc_to_text: !function utils.seed_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - image_aspect_ratio: original -# The return value of process_results will be used by metrics -process_results: !function utils.seed_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: seed_image - aggregation: !function utils.seed_aggregation_result - higher_is_better: true - - metric: seed_video - aggregation: !function utils.seed_aggregation_result - higher_is_better: true - - metric: seed_all - aggregation: !function utils.seed_aggregation_result - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/seedbench/seedbench_ppl.yaml b/lmms_eval/tasks/seedbench/seedbench_ppl.yaml deleted file mode 100644 index e12729e..0000000 --- a/lmms_eval/tasks/seedbench/seedbench_ppl.yaml +++ /dev/null @@ -1,15 +0,0 @@ -dataset_path: lmms-lab/SEED-Bench -dataset_kwargs: - token: True -task: "seedbench_ppl" -test_split: test -output_type: multiple_choice -doc_to_visual: !function utils.seed_doc_to_visual -doc_to_text: !function utils.seed_doc_to_text_mc -doc_to_choice : !function utils.seed_doc_to_choice -doc_to_target: !function utils.seed_doc_to_mc_target -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: acc -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/seedbench/utils.py b/lmms_eval/tasks/seedbench/utils.py deleted file mode 100644 index c2938f1..0000000 --- a/lmms_eval/tasks/seedbench/utils.py +++ /dev/null @@ -1,60 +0,0 @@ -import json - - -def seed_doc_to_visual(doc): - return [image.convert("RGB") for image in doc["image"]] - - -def seed_doc_to_text(doc): - question = doc["question"] - question += "\n" + f"A. {doc['choice_a']}\n" - question += f"B. {doc['choice_b']}\n" - question += f"C. {doc['choice_c']}\n" - question += f"D. {doc['choice_d']}" - return f"{question}\nAnswer with the option's letter from the given choices directly." - - -def seed_process_result(doc, result): - pred = result[0].strip() - if len(pred) > 1: - pred = pred[0] - answer = doc["answer"] - data_type = doc["data_type"] - - return {f"seed_{data_type}": {"pred": pred, "answer": answer, "question_id": doc["question_id"]}, f"seed_all": {"pred": pred, "answer": answer, "question_id": doc["question_id"]}} - - -def seed_aggregation_result(results): - total_count = 0 - total_correct = 0 - for result in results: - if result["pred"] == result["answer"]: - total_correct += 1 - total_count += 1 - return total_correct / total_count - - -def seed_aggregation_result_all(results): - score = seed_aggregation_result(results) - stored_results = [] - for result in results: - stored_results.append({"question_id": result["question_id"], "prediction": result["pred"]}) - with open("./seed_submission.json", "w") as f: - json.dump(stored_results, f, indent=4) - print("Storing files for seed_submission ...") - - return score - - -def seed_doc_to_text_mc(doc): - question = doc["question"] - return f"{question} Answer :" - - -def seed_doc_to_choice(doc): - return [doc["choice_a"], doc["choice_b"], doc["choice_c"], doc["choice_d"]] - - -def seed_doc_to_mc_target(doc): - answer2choice = {"A": "choice_a", "B": "choice_b", "C": "choice_c", "D": "choice_d"} - return doc[answer2choice[doc["answer"]]] diff --git a/lmms_eval/tasks/seedbench_2/seedbench_2.yaml b/lmms_eval/tasks/seedbench_2/seedbench_2.yaml deleted file mode 100644 index c61e410..0000000 --- a/lmms_eval/tasks/seedbench_2/seedbench_2.yaml +++ /dev/null @@ -1,49 +0,0 @@ -dataset_path: lmms-lab/SEED-Bench-2 -dataset_kwargs: - token: True -task: "seedbench-2" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.seed_doc_to_visual -doc_to_text: !function utils.seed_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 16 - image_aspect_ratio: original -# The return value of process_results will be used by metrics -process_results: !function utils.seed_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: seed_Video - aggregation: !function utils.seed_aggregation_result - higher_is_better: true - - metric: seed_Multiple_Images - aggregation: !function utils.seed_aggregation_result - higher_is_better: true - - metric: seed_Image_&_Text_Generation - aggregation: !function utils.seed_aggregation_result - higher_is_better: true - - metric: seed_Single_Image - aggregation: !function utils.seed_aggregation_result - higher_is_better: true - - metric: seed_Image_Generation - aggregation: !function utils.seed_aggregation_result - higher_is_better: true - - metric: seed_Interleaved_Image - aggregation: !function utils.seed_aggregation_result - higher_is_better: true - - metric: seed_all - aggregation: !function utils.seed_aggregation_result - higher_is_better: true -metadata: - - version: 0.0 - -model_specific_prompt_kwargs: - llava : - img_token : - post_prompt : "Answer with the option's letter from the given choices directly." - gpt4V : - img_token : - post_prompt : "Answer with the option's letter from the given choices directly." \ No newline at end of file diff --git a/lmms_eval/tasks/seedbench_2/utils.py b/lmms_eval/tasks/seedbench_2/utils.py deleted file mode 100644 index f88ded9..0000000 --- a/lmms_eval/tasks/seedbench_2/utils.py +++ /dev/null @@ -1,58 +0,0 @@ -import json - - -def seed_doc_to_visual(doc): - return [image.convert("RGB") for image in doc["image"]] - - -def parse_choice_img(choice: str, img_token: str): - if "jpg" in choice or "png" in choice: - return img_token - return choice - - -def seed_doc_to_text(doc, model_specific_kwargs=None): - question = doc["question"] - question.replace("", model_specific_kwargs["img_token"]) - question += "\n" + f"A. {parse_choice_img(doc['choice_a'], model_specific_kwargs['img_token'])}\n" - question += f"B. {parse_choice_img(doc['choice_b'], model_specific_kwargs['img_token'])}\n" - question += f"C. {parse_choice_img(doc['choice_c'], model_specific_kwargs['img_token'])}\n" - question += f"D. {parse_choice_img(doc['choice_d'], model_specific_kwargs['img_token'])}" - if doc["data_type"] == "Image Generation": - num_img_in_question = len(doc["data_id"]) - 4 - prepend_tokens = [model_specific_kwargs["img_token"]] * num_img_in_question - question = " ".join(prepend_tokens) + "\n" + question - return f"{question}\n{model_specific_kwargs['post_prompt']}" - - -def seed_process_result(doc, result): - pred = result[0].strip() - if len(pred) > 1: - pred = pred[0] - answer = doc["answer"] - data_type = doc["data_type"].split(" ") - data_type = "_".join(data_type) - - return {f"seed_{data_type}": {"pred": pred, "answer": answer, "question_id": doc["question_id"]}, f"seed_all": {"pred": pred, "answer": answer, "question_id": doc["question_id"]}} - - -def seed_aggregation_result(results): - total_count = 0 - total_correct = 0 - for result in results: - if result["pred"] == result["answer"]: - total_correct += 1 - total_count += 1 - return total_correct / total_count if total_count != 0 else 0 - - -def seed_aggregation_result_all(results): - score = seed_aggregation_result(results) - stored_results = [] - for result in results: - stored_results.append({"question_id": result["question_id"], "prediction": result["pred"]}) - with open("./seed_submission.json", "w") as f: - json.dump(stored_results, f, indent=4) - print("Storing files for seed_submission ...") - - return score diff --git a/lmms_eval/tasks/stvqa/stvqa.yaml b/lmms_eval/tasks/stvqa/stvqa.yaml deleted file mode 100644 index 42ecaac..0000000 --- a/lmms_eval/tasks/stvqa/stvqa.yaml +++ /dev/null @@ -1,20 +0,0 @@ -dataset_path: lmms-lab/ST-VQA -task: "stvqa" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.stvqa_doc_to_visual -doc_to_text: !function utils.stvqa_doc_to_text -doc_to_target: "answers" -generation_kwargs: - max_new_tokens: 32 - temperature: 0 - do_sample: False -process_results: !function utils.stvqa_process_results -metric_list: - - metric: submission - aggregation: !function utils.stvqa_aggregate_submissions -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer the question using a single word or phrase." - \ No newline at end of file diff --git a/lmms_eval/tasks/stvqa/utils.py b/lmms_eval/tasks/stvqa/utils.py deleted file mode 100644 index 81fca5b..0000000 --- a/lmms_eval/tasks/stvqa/utils.py +++ /dev/null @@ -1,28 +0,0 @@ -import os -import json -import logging - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - - -def stvqa_doc_to_text(doc, model_specific_prompt_kwargs): - question = doc["question"] - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - post_prompt = model_specific_prompt_kwargs["post_prompt"] - return f"{pre_prompt}{question}{post_prompt}" - - -def stvqa_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def stvqa_process_results(doc, results): - answer = results[0] - return {"submission": {"question_id": int(doc["question_id"]), "answer": answer}} - - -def stvqa_aggregate_submissions(results, args): - file = generate_submission_file("stvqa_test_for_submission.json", args) - with open(file, "w") as f: - json.dump(results, f) - logging.getLogger("lmms-eval").info(f"Results saved to {file}") diff --git a/lmms_eval/tasks/textcaps/_default_template_textcaps_yaml b/lmms_eval/tasks/textcaps/_default_template_textcaps_yaml deleted file mode 100644 index bb71de4..0000000 --- a/lmms_eval/tasks/textcaps/_default_template_textcaps_yaml +++ /dev/null @@ -1,3 +0,0 @@ -model_specific_prompt_kwargs: - default: - prompt: Provide a one-sentence caption for the provided image. \ No newline at end of file diff --git a/lmms_eval/tasks/textcaps/textcaps.yaml b/lmms_eval/tasks/textcaps/textcaps.yaml deleted file mode 100644 index 39a040c..0000000 --- a/lmms_eval/tasks/textcaps/textcaps.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group : textcaps -task: - - textcaps_val - - textcaps_test \ No newline at end of file diff --git a/lmms_eval/tasks/textcaps/textcaps_test.yaml b/lmms_eval/tasks/textcaps/textcaps_test.yaml deleted file mode 100644 index 2636922..0000000 --- a/lmms_eval/tasks/textcaps/textcaps_test.yaml +++ /dev/null @@ -1,25 +0,0 @@ -dataset_path: lmms-lab/TextCaps -dataset_kwargs: - token: True -task : "textcaps_test" -group : "textcaps_caption" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.textcaps_doc_to_visual -doc_to_text: !function utils.textcaps_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 64 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.textcaps_test_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: textcaps_passthrough - aggregation : !function utils.textcaps_test_aggregation_result - higher_is_better : true -metadata: - - version: 0.0 -include: _default_template_textcaps_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/textcaps/textcaps_train.yaml b/lmms_eval/tasks/textcaps/textcaps_train.yaml deleted file mode 100644 index 931aeb1..0000000 --- a/lmms_eval/tasks/textcaps/textcaps_train.yaml +++ /dev/null @@ -1,48 +0,0 @@ -dataset_path: lmms-lab/TextCaps -dataset_kwargs: - token: True -task : "textcaps_train" -group : "textcaps_caption" -test_split: train -output_type: generate_until -doc_to_visual: !function utils.textcaps_doc_to_visual -doc_to_text: !function utils.textcaps_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.textcaps_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: textcaps_Bleu_4 - aggregation : !function utils.textcaps_bleu4 - higher_is_better : true - - metric: textcaps_Bleu_3 - aggregation : !function utils.textcaps_bleu3 - higher_is_better : true - - metric: textcaps_Bleu_2 - aggregation : !function utils.textcaps_bleu2 - higher_is_better : true - - metric: textcaps_Bleu_1 - aggregation : !function utils.textcaps_bleu1 - higher_is_better : true - - metric: textcaps_METEOR - aggregation : !function utils.textcaps_meteor - higher_is_better : true - - metric: textcaps_ROUGE_L - aggregation : !function utils.textcaps_rougel - higher_is_better : true - - metric: textcaps_CIDEr - aggregation : !function utils.textcaps_cider - higher_is_better : true - #- metric: textcaps_SPICE - # aggregation : !function utils.textcaps_spice - # higher_is_better : true -metadata: - - version: 0.0 -include: _default_template_textcaps_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/textcaps/textcaps_val.yaml b/lmms_eval/tasks/textcaps/textcaps_val.yaml deleted file mode 100644 index 41baec8..0000000 --- a/lmms_eval/tasks/textcaps/textcaps_val.yaml +++ /dev/null @@ -1,46 +0,0 @@ -dataset_path: lmms-lab/TextCaps -dataset_kwargs: - token: True -task: "textcaps_val" -group : "textcaps_caption" -test_split: val -output_type: generate_until -doc_to_visual: !function utils.textcaps_doc_to_visual -doc_to_text: !function utils.textcaps_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 64 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false -process_results: !function utils.textcaps_process_result -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -metric_list: - - metric: textcaps_Bleu_4 - aggregation : !function utils.textcaps_bleu4 - higher_is_better : true - - metric: textcaps_Bleu_3 - aggregation : !function utils.textcaps_bleu3 - higher_is_better : true - - metric: textcaps_Bleu_2 - aggregation : !function utils.textcaps_bleu2 - higher_is_better : true - - metric: textcaps_Bleu_1 - aggregation : !function utils.textcaps_bleu1 - higher_is_better : true - - metric: textcaps_METEOR - aggregation : !function utils.textcaps_meteor - higher_is_better : true - - metric: textcaps_ROUGE_L - aggregation : !function utils.textcaps_rougel - higher_is_better : true - - metric: textcaps_CIDEr - aggregation : !function utils.textcaps_cider - higher_is_better : true - #- metric: textcaps_SPICE - # aggregation : !function utils.textcaps_spice - # higher_is_better : true -metadata: - - version: 0.0 -include: _default_template_textcaps_yaml \ No newline at end of file diff --git a/lmms_eval/tasks/textcaps/utils.py b/lmms_eval/tasks/textcaps/utils.py deleted file mode 100644 index c63feae..0000000 --- a/lmms_eval/tasks/textcaps/utils.py +++ /dev/null @@ -1,150 +0,0 @@ -import os -import json -from pycocoevalcap.eval import COCOEvalCap, Bleu, Meteor, Rouge, Cider, Spice -from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer -from pycocotools.coco import COCO -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -import logging - -eval_logger = logging.getLogger("lmms-eval") - -dir_name = os.path.dirname(os.path.abspath(__file__)) - -TEXTCAPS_METRICS = ["Bleu_4", "Bleu_3", "Bleu_2", "Bleu_1", "METEOR", "ROUGE_L", "CIDEr"] # , "SPICE"] - - -def textcaps_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def textcaps_doc_to_text(doc, model_specific_prompt_kwargs=None): - return model_specific_prompt_kwargs["prompt"] - - -def textcaps_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name, value: metric value - """ - pred = result[0] if len(result) > 0 else "" - - data_dict = {"answer": doc["caption_str"], "pred": pred, "image_id": doc["image_id"]} - - return {f"textcaps_{metric}": data_dict for metric in TEXTCAPS_METRICS} - - -def textcaps_aggregation_result(results, metric, args=None): - scorers = [(Bleu(4), "Bleu_1"), (Bleu(4), "Bleu_2"), (Bleu(4), "Bleu_3"), (Bleu(4), "Bleu_4"), (Meteor(), "METEOR"), (Rouge(), "ROUGE_L"), (Cider(), "CIDEr")]#, (Spice(), "SPICE")] - scorers_dict = {s[1]: s for s in scorers} - - stored_results = [] - # In order to make the coco eval tools to successfully create index - # We need at least two dict in the dataset - # 'annotation' and 'images' - # 'annotation' exactly reproduce the original annotation - # 'images' however only need the image id which is contained in the file name - dataset = {"annotations": [], "images": []} - idx = 0 - for result in results: - stored_results.append({"image_id": result["image_id"], "caption": result["pred"]}) - for a in result["answer"]: - dataset["annotations"].append({"image_id": result["image_id"], "caption": a, "id": idx}) - idx += 1 - dataset["images"].append({"id": result["image_id"]}) - - coco = COCO() - # Manually create index here - coco.dataset = dataset - coco.createIndex() - - textcaps_result = coco.loadRes(stored_results) - textcaps_eval = COCOEvalCap(coco, textcaps_result) - - imgIds = textcaps_eval.params["image_id"] - gts = {} - res = {} - for imgId in imgIds: - gts[imgId] = textcaps_eval.coco.imgToAnns[imgId] - res[imgId] = textcaps_eval.cocoRes.imgToAnns[imgId] - - eval_logger.info("tokenization...") - tokenizer = PTBTokenizer() - gts = tokenizer.tokenize(gts) - res = tokenizer.tokenize(res) - - eval_logger.info(f"Computing {metric} scores...") - - score, scores = scorers_dict[metric][0].compute_score(gts, res) - # When metric is one of the Bleu, score will be a list - if type(score) == list: - n = int(metric.split("_")[-1]) - score = score[n - 1] - - path = generate_submission_file("textcaps_captions_val2014_alg_results.json", args) - - with open(path, "w") as f: - json.dump(stored_results, f, indent=4) - - eval_logger.info(f"Results saved to {path}") - - return score - - -def textcaps_bleu4(results, args=None): - return textcaps_aggregation_result(results, "Bleu_4", args) - - -def textcaps_bleu3(results, args=None): - return textcaps_aggregation_result(results, "Bleu_3", args) - - -def textcaps_bleu2(results, args=None): - return textcaps_aggregation_result(results, "Bleu_2", args) - - -def textcaps_bleu1(results, args=None): - return textcaps_aggregation_result(results, "Bleu_1", args) - - -def textcaps_meteor(results, args=None): - return textcaps_aggregation_result(results, "METEOR", args) - - -def textcaps_rougel(results, args=None): - return textcaps_aggregation_result(results, "ROUGE_L", args) - - -def textcaps_cider(results, args=None): - return textcaps_aggregation_result(results, "CIDEr", args) - - -def textcaps_spice(results, args=None): - return textcaps_aggregation_result(results, "SPICE", args) - - -def textcaps_test_process_result(doc, result): - """ - Args: - doc: a instance of the eval dataset - results: [pred] - Returns: - a dictionary with key: metric name (in this case textcaps_passthrough), value: metric value - """ - return {"textcaps_passthrough": {"pred": result, "image_id": doc["image_id"]}} - - -def textcaps_test_aggregation_result(results, args): - stored_results = [] - for result in results: - stored_results.append({"image_id": result["image_id"], "caption": result["pred"]}) - - path = generate_submission_file("textcaps_captions_test2014_alg_results.json", args) - eval_logger.info("Storing prediction that can be submitted to the server ...") - with open(path, "w") as f: - json.dump(stored_results, f, indent=4) - - eval_logger.info(f"Your test result has been stored into {path}. Make sure you also have the val result stored to submit to the server on https://codalab.lisn.upsaclay.fr/competitions/7404#participate.") diff --git a/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml b/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml deleted file mode 100644 index 282b401..0000000 --- a/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml +++ /dev/null @@ -1,17 +0,0 @@ -dataset_path: lmms-lab/textvqa -output_type: generate_until -doc_to_visual: !function utils.textvqa_doc_to_visual -doc_to_text: !function utils.textvqa_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" -process_results: !function utils.textvqa_process_results -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer the question using a single word or phrase." - ocr: false - qwen_vl: - pre_prompt: "" - post_prompt: " Answer:" diff --git a/lmms_eval/tasks/textvqa/_textvqa.yaml b/lmms_eval/tasks/textvqa/_textvqa.yaml deleted file mode 100644 index 506a2fb..0000000 --- a/lmms_eval/tasks/textvqa/_textvqa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: textvqa -task: -- textvqa_val -- textvqa_test \ No newline at end of file diff --git a/lmms_eval/tasks/textvqa/textvqa_test.yaml b/lmms_eval/tasks/textvqa/textvqa_test.yaml deleted file mode 100644 index 15b02c4..0000000 --- a/lmms_eval/tasks/textvqa/textvqa_test.yaml +++ /dev/null @@ -1,7 +0,0 @@ -task: textvqa_test -test_split: test -metric_list: - - metric: submission - aggregation: !function utils.textvqa_aggreate_submissions - higher_is_better: true -include: _default_template_textvqa_yaml diff --git a/lmms_eval/tasks/textvqa/textvqa_val.yaml b/lmms_eval/tasks/textvqa/textvqa_val.yaml deleted file mode 100644 index e23e9b2..0000000 --- a/lmms_eval/tasks/textvqa/textvqa_val.yaml +++ /dev/null @@ -1,12 +0,0 @@ -task: textvqa_val -test_split: validation -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - - metric: submission - aggregation: !function utils.textvqa_aggreate_submissions - higher_is_better: true -include: _default_template_textvqa_yaml diff --git a/lmms_eval/tasks/textvqa/utils.py b/lmms_eval/tasks/textvqa/utils.py deleted file mode 100644 index ea3b503..0000000 --- a/lmms_eval/tasks/textvqa/utils.py +++ /dev/null @@ -1,68 +0,0 @@ -import re -import os -import json -import yaml -import pathlib -import logging -import datetime -import statistics - -from lmms_eval.tasks._task_utils.vqa_eval_metric import EvalAIAnswerProcessor -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -eval_logger = logging.getLogger("lmms-eval") - - -def textvqa_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def textvqa_process_results(doc, result): - eval_ai_processor = EvalAIAnswerProcessor() - assert len(result) == 1, f"The result should be a list of length 1, but got {len(result)}." - resAns = eval_ai_processor(result[0]) - accuracy = 0 - - if "answers" in doc and doc["answers"] is not None: - gtAcc = [] - - for i in range(len(doc["answers"])): - doc["answers"][i] = eval_ai_processor(doc["answers"][i]) - - for i in range(len(doc["answers"])): - otherGTAns = [doc["answers"][j] for j in range(len(doc["answers"])) if i != j] - matchingAns = [item for item in otherGTAns if item == resAns] - acc = min(1, float(len(matchingAns)) / 3) - gtAcc.append(acc) - accuracy = statistics.mean(gtAcc) - - return { - "exact_match": accuracy, - "submission": { - "question_id": doc["question_id"], - "answer": resAns, - }, - } - - -def textvqa_doc_to_text(doc, model_specific_prompt_kwargs=None): - pre_prompt = "" - post_post = "" - ocr_ref = "" - if model_specific_prompt_kwargs: - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] - if "ocr" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["ocr"]: - ocr_ref = f"\nReference OCR token: {', '.join(doc['ocr_tokens'])}" - return f"{pre_prompt}{doc['question'].capitalize()}{ocr_ref}{post_prompt}" - - -def textvqa_aggreate_submissions(results, args): - now_date_time = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - path = generate_submission_file(f"textvqa_submission_{now_date_time}.json", args) - with open(path, "w") as f: - json.dump(results, f) - # print(f"Submission file saved to {path}") - eval_logger.info(f"Submission file saved to {path}") diff --git a/lmms_eval/tasks/vizwiz_vqa/_default_template_vqa_yaml b/lmms_eval/tasks/vizwiz_vqa/_default_template_vqa_yaml deleted file mode 100644 index b7dd413..0000000 --- a/lmms_eval/tasks/vizwiz_vqa/_default_template_vqa_yaml +++ /dev/null @@ -1,15 +0,0 @@ -dataset_path: lmms-lab/VizWiz-VQA -output_type: generate_until -doc_to_visual: !function utils.vizwiz_vqa_doc_to_visual -doc_to_text: !function utils.vizwiz_vqa_doc_to_text -doc_to_target: "answer" -generation_kwargs: - until: - - "ASSISTANT:" -metadata: - - version: 0.0 -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nWhen the provided information is insufficient, respond with 'Unanswerable'.\nAnswer the question using a single word or phrase." -process_results: !function utils.vizwiz_vqa_process_results diff --git a/lmms_eval/tasks/vizwiz_vqa/_generate_config.py b/lmms_eval/tasks/vizwiz_vqa/_generate_config.py deleted file mode 100644 index 725161d..0000000 --- a/lmms_eval/tasks/vizwiz_vqa/_generate_config.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import yaml - -splits = ["val", "test"] -tasks = ["vqa"] - -if __name__ == "__main__": - dump_tasks = [] - for task in tasks: - for split in splits: - yaml_dict = {"group": f"vizwiz_{task}", "task": f"vizwiz_{task}_{split}", "include": f"_default_template_{task}_yaml", "test_split": split} - if split == "train": - yaml_dict.pop("group") - else: - dump_tasks.append(f"vizwiz_{task}_{split}") - - save_path = f"./vizwiz_{task}_{split}.yaml" - print(f"Saving to {save_path}") - with open(save_path, "w") as f: - yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False) - - group_dict = {"group": "vizwiz_vqa", "task": dump_tasks} - - with open("./_vizwiz_vqa.yaml", "w") as f: - yaml.dump(group_dict, f, default_flow_style=False, indent=4) diff --git a/lmms_eval/tasks/vizwiz_vqa/_vizwiz_vqa.yaml b/lmms_eval/tasks/vizwiz_vqa/_vizwiz_vqa.yaml deleted file mode 100644 index 5502a10..0000000 --- a/lmms_eval/tasks/vizwiz_vqa/_vizwiz_vqa.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: vizwiz_vqa -task: -- vizwiz_vqa_val -- vizwiz_vqa_test \ No newline at end of file diff --git a/lmms_eval/tasks/vizwiz_vqa/utils.py b/lmms_eval/tasks/vizwiz_vqa/utils.py deleted file mode 100644 index 63afb4e..0000000 --- a/lmms_eval/tasks/vizwiz_vqa/utils.py +++ /dev/null @@ -1,70 +0,0 @@ -import re -import os -import json -import yaml -import pathlib -import logging -import datetime -import statistics - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file -from lmms_eval.tasks._task_utils.vqa_eval_metric import EvalAIAnswerProcessor - -eval_logger = logging.getLogger("lmms-eval") - - -def vizwiz_vqa_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def vizwiz_vqa_process_results(doc, result): - eval_ai_processor = EvalAIAnswerProcessor() - assert len(result) == 1, f"The result should be a list of length 1, but got {len(result)}." - resAns = eval_ai_processor(result[0]) - accuracy = 0 - - if "answers" in doc and doc["answers"] is not None: - gtAcc = [] - - for i in range(len(doc["answers"])): - doc["answers"][i] = eval_ai_processor(doc["answers"][i]) - - for i in range(len(doc["answers"])): - otherGTAns = [doc["answers"][j] for j in range(len(doc["answers"])) if i != j] - matchingAns = [item for item in otherGTAns if item == resAns] - acc = min(1, float(len(matchingAns)) / 3) - gtAcc.append(acc) - if gtAcc: - accuracy = statistics.mean(gtAcc) - else: - accuracy = 0 - - return { - "exact_match": accuracy, - "submission": { - "image": f"{doc['question_id']}.jpg", - "answer": resAns, - }, - } - - -def vizwiz_vqa_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = "" - post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] - text = f"{pre_prompt}{doc['question'].capitalize()}{post_prompt}" - return text - - -def vizwiz_vqa_aggreate_submissions(results, args): - now_date_time = datetime.datetime.now().strftime("%Y-%m%d-%H%M-%S") - submission_file_name = f"vizwiz_vqa-test-submission-{now_date_time}.json" - path = generate_submission_file(submission_file_name, args) - with open(path, "w") as f: - json.dump(results, f) - print(f"Submission file saved to {path}") diff --git a/lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_test.yaml b/lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_test.yaml deleted file mode 100644 index dec140f..0000000 --- a/lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_test.yaml +++ /dev/null @@ -1,14 +0,0 @@ -group: vizwiz_vqa -task: vizwiz_vqa_test -test_split: test -include: _default_template_vqa_yaml -process_results: !function utils.vizwiz_vqa_process_results -metric_list: - # - metric: exact_match - # aggregation: mean - # higher_is_better: true - # ignore_case: true - # ignore_punctuation: true - - metric: submission - aggregation: !function utils.vizwiz_vqa_aggreate_submissions - higher_is_better: true diff --git a/lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_val.yaml b/lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_val.yaml deleted file mode 100644 index ac8ecc9..0000000 --- a/lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_val.yaml +++ /dev/null @@ -1,13 +0,0 @@ -group: vizwiz_vqa -task: vizwiz_vqa_val -test_split: val -include: _default_template_vqa_yaml -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - # - metric: submission - # aggregation: !function utils.vizwiz_vqa_aggreate_submissions - # higher_is_better: true \ No newline at end of file diff --git a/lmms_eval/tasks/vqav2/_default_template_vqav2_yaml b/lmms_eval/tasks/vqav2/_default_template_vqav2_yaml deleted file mode 100644 index d3ce20f..0000000 --- a/lmms_eval/tasks/vqav2/_default_template_vqav2_yaml +++ /dev/null @@ -1,15 +0,0 @@ -dataset_path: lmms-lab/VQAv2 -dataset_kwargs: - token: True -output_type: generate_until -doc_to_visual: !function utils.vqav2_doc_to_visual -doc_to_text: !function utils.vqav2_doc_to_text -doc_to_target: "answer" -generation_kwargs: - max_new_tokens: 16 -metadata: - - version: 0.0 -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer the question using a single word or phrase." \ No newline at end of file diff --git a/lmms_eval/tasks/vqav2/_vqav2.yaml b/lmms_eval/tasks/vqav2/_vqav2.yaml deleted file mode 100644 index d6cadde..0000000 --- a/lmms_eval/tasks/vqav2/_vqav2.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: vqav2 -task: -- vqav2_val -- vqav2_test \ No newline at end of file diff --git a/lmms_eval/tasks/vqav2/utils.py b/lmms_eval/tasks/vqav2/utils.py deleted file mode 100644 index 1a3e908..0000000 --- a/lmms_eval/tasks/vqav2/utils.py +++ /dev/null @@ -1,89 +0,0 @@ -import re -import os -import json -import logging -import datetime -import statistics - -import lmms_eval.tasks._task_utils.file_utils as file_utils - -from lmms_eval.tasks._task_utils.vqa_eval_metric import EvalAIAnswerProcessor - - -eval_logger = logging.getLogger("lmms-eval") - - -def vqav2_doc_to_visual(doc): - return [doc["image"].convert("RGB")] - - -def vqav2_process_results(doc, result): - eval_ai_processor = EvalAIAnswerProcessor() - assert len(result) == 1, f"The result should be a list of length 1, but got {len(result)}." - resAns = eval_ai_processor(result[0]) - accuracy = 0 - - if "answers" in doc and doc["answers"] is not None: - for ansDic in doc["answers"]: - ansDic["answer"] = ansDic["answer"].replace("\n", " ") - ansDic["answer"] = ansDic["answer"].replace("\t", " ") - ansDic["answer"] = ansDic["answer"].strip() - gtAcc = [] - gtAnswers = [ans["answer"] for ans in doc["answers"]] - - if len(set(gtAnswers)) > 1: - for ansDic in doc["answers"]: - ansDic["answer"] = eval_ai_processor.process_punctuation(ansDic["answer"]) - ansDic["answer"] = eval_ai_processor.process_digit_article(ansDic["answer"]) - resAns = eval_ai_processor.process_punctuation(resAns) - resAns = eval_ai_processor.process_digit_article(resAns) - - for gtAnsDatum in doc["answers"]: - otherGTAns = [item for item in doc["answers"] if item != gtAnsDatum] - matchingAns = [item for item in otherGTAns if item["answer"] == resAns] - acc = min(1, float(len(matchingAns)) / 3) - gtAcc.append(acc) - accuracy = statistics.mean(gtAcc) - - return { - "exact_match": accuracy, - "submission": { - "question_id": doc["question_id"], - "answer": resAns, - }, - } - - -def vqav2_process_results_test(doc, result): - res = vqav2_process_results(doc, result) - return { - "submission": res["submission"], - } - - -def vqav2_process_results_val(doc, result): - res = vqav2_process_results(doc, result) - return { - "exact_match": res["exact_match"], - } - - -def vqav2_doc_to_text(doc, model_specific_prompt_kwargs=None): - if model_specific_prompt_kwargs is None: - model_specific_prompt_kwargs = {} - pre_prompt = "" - post_prompt = "" - if "pre_prompt" in model_specific_prompt_kwargs: - pre_prompt = model_specific_prompt_kwargs["pre_prompt"] - if "post_prompt" in model_specific_prompt_kwargs: - post_prompt = model_specific_prompt_kwargs["post_prompt"] - return f"{pre_prompt}{doc['question']}{post_prompt}" - - -def vqav2_aggreate_submissions(results, args): - now_date_time = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - submission_file_name = f"vqav2-test-submission-{now_date_time}.json" - path = file_utils.generate_submission_file(submission_file_name, args) - with open(path, "w") as f: - json.dump(results, f) - eval_logger.info(f"Submission file saved to {path}") diff --git a/lmms_eval/tasks/vqav2/vqav2_test.yaml b/lmms_eval/tasks/vqav2/vqav2_test.yaml deleted file mode 100644 index 94c6920..0000000 --- a/lmms_eval/tasks/vqav2/vqav2_test.yaml +++ /dev/null @@ -1,8 +0,0 @@ -task: "vqav2_test" -include: _default_template_vqav2_yaml -test_split: test -metric_list: - - metric: submission - aggregation: !function utils.vqav2_aggreate_submissions - higher_is_better: true -process_results: !function utils.vqav2_process_results_test diff --git a/lmms_eval/tasks/vqav2/vqav2_val.yaml b/lmms_eval/tasks/vqav2/vqav2_val.yaml deleted file mode 100644 index d24870b..0000000 --- a/lmms_eval/tasks/vqav2/vqav2_val.yaml +++ /dev/null @@ -1,10 +0,0 @@ -task: "vqav2_val" -include: _default_template_vqav2_yaml -test_split: validation -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true -process_results: !function utils.vqav2_process_results_val diff --git a/lmms_eval/tasks/websrc/README.md b/lmms_eval/tasks/websrc/README.md deleted file mode 100644 index a631e6d..0000000 --- a/lmms_eval/tasks/websrc/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# WebSRC - -## Paper - -Title: WebSRC: A Dataset for Web-Based Structural Reading Comprehension - -Abstract: https://arxiv.org/abs/2101.09465 - -Homepage: https://x-lance.github.io/WebSRC/# - -WebSRC is a dataset for web-based structural reading comprehension. -Its full train/dev/test split contains over 400k questions across 6.4k webpages. -This version of the dataset does not contain OCR or original HTML, it simply treats WebSRC as a image-and-text-based multimodal Q&A benchmark on webpage screenshots. - -## Citation - -```bibtex -@inproceedings{chen2021websrc, - title={WebSRC: A Dataset for Web-Based Structural Reading Comprehension}, - author={Chen, Xingyu and Zhao, Zihan and Chen, Lu and Ji, Jiabao and Zhang, Danyang and Luo, Ao and Xiong, Yuxuan and Yu, Kai}, - booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing}, - pages={4173--4185}, - year={2021} -} -``` - -## Groups & Tasks - -### Groups - -- `websrc`: Evaluates `websrc-val` and generates a submission file for `websrc-test`. - -### Tasks - -- `websrc-val`: Given a question and a web page, predict the answer. -- `websrc-test`: Given a question and a web page, predict the answer. Ground truth is not provided for this task. - -## Metrics - -This task uses SQUAD-style evaluation metrics, of which F1 score over tokens is used. -The orignal paper also uses Exact Match (EM) score, but this is not implemented here as that metric is more conducive for Encoder-only extraction models. - -### F1 Score - -F1 Score is the harmonic mean of precision and recall. -We calculate precision and recall at the token level, then compute the F1 score as normal using these values. - -### Test Submission - -When evaluaing on the test split, a prediction JSON will be compiled instead of metrics computed. -Instructions for submission are available on the [WebSRC homepage](https://x-lance.github.io/WebSRC/#) and in their [Original GitHub Repo](https://github.com/X-LANCE/WebSRC-Baseline#obtain-test-result). \ No newline at end of file diff --git a/lmms_eval/tasks/websrc/utils.py b/lmms_eval/tasks/websrc/utils.py deleted file mode 100644 index e76037c..0000000 --- a/lmms_eval/tasks/websrc/utils.py +++ /dev/null @@ -1,160 +0,0 @@ -from collections import defaultdict -import re -import ast -import base64 -import io -import random -import numpy as np -import os -import json -import logging -from PIL import Image - -from lmms_eval.tasks._task_utils.file_utils import generate_submission_file - -lmms_logger = logging.getLogger("lmms-eval") - -OPEN_ENDED_PROMPT = "Answer the question using a single word or phrase." - - -def construct_prompt(doc): - question = doc["question"] - question = f"{OPEN_ENDED_PROMPT}\n{question}" - return question - - -def websrc_doc_to_text(doc): - question = construct_prompt(doc) - return question - - -def websrc_doc_to_visual(doc): - img_bs64 = doc["image"] - img = Image.open(io.BytesIO(base64.b64decode(img_bs64))) - del doc['image'] - return [img] - - -def websrc_process_results(doc, results): - pred = results[0] - parsed_pred = pred - id = doc["page_id"] - websrc_ans = {"id": id, "domain": doc['domain'], "parsed_pred": parsed_pred} - if "answer" in doc: - websrc_ans["answer"] = doc["answer"] - - if 'id' in doc: - websrc_ans['question_id'] = doc['id'] - - return { - "websrc_squad_f1": websrc_ans, - "submission": { - websrc_ans['question_id']: pred, - } if 'question_id' in websrc_ans else None - } - - -def websrc_test_aggregate_results_for_submission(results, args): - path = generate_submission_file("websrc_test_for_submission.json", args) - with open(path, "w") as f: - out = {} - for result in results: - out.update(result) - json.dump(out, f, indent=4) - lmms_logger.info(f"Results saved to {path}.") - - -def websrc_aggregate_results(results): - evaluation_result = {} - - # Group results by domain - subset_to_eval_samples = defaultdict(list) - for result in results: - subset_to_eval_samples[result["domain"]].append(result) - - # Evaluate each domain - for subset, sub_eval_samples in subset_to_eval_samples.items(): - judge_dict, metric_dict = evaluate_websrc(sub_eval_samples) - metric_dict.update({"num_example": len(sub_eval_samples)}) - evaluation_result[subset] = metric_dict - - # Aggregate results for all domains - printable_results = {} - for domain in DOMAINS: - if domain not in evaluation_result: - continue - printable_results[domain] = { - "num": int(evaluation_result[domain]["num_example"]), - "f1": round(evaluation_result[domain]["f1"], 3), - } - all_ins_f1 = np.sum([cat_results["f1"] * cat_results["num_example"] for cat_results in evaluation_result.values()]) / sum( - [cat_results["num_example"] for cat_results in evaluation_result.values()] - ) - printable_results["Overall"] = { - "num": sum([cat_results["num_example"] for cat_results in evaluation_result.values()]), - "f1": round(all_ins_f1, 3), - } - print(printable_results) - return printable_results["Overall"]["f1"] - - -################## -# Helper functions written by official MMMU repo. -################## -DOMAINS = [ - 'auto', - 'book', - 'camera', - 'game', - 'jobs', - 'movie', - 'phone', - 'restaurant', - 'sports', - 'university', - 'hotel', -] - - -def evaluate_websrc(samples): - - def _normalize_str(string): - # lower it - string = string.lower() - - # strip leading and trailing whitespaces - string = string.strip() - - return string - - def _tokenize(text): - # Regex pattern to match words and isolate punctuation - pattern = r'\w+|[^\w\s]' - tokens = re.findall(pattern, text) - return tokens - - def _compute_f1(sa, sb): - sa = _normalize_str(sa) - sb = _normalize_str(sb) - - sa = _tokenize(sa) - sb = _tokenize(sb) - - sa = set(sa) - sb = set(sb) - - if len(sa) == 0 or len(sb) == 0: - return 0.0 - - comm = sa.intersection(sb) - prec = len(comm) / len(sb) - rec = len(comm) / len(sa) - f1 = 2 * prec * rec / (prec + rec) if prec + rec > 0 else 0 - return f1 - - judge_list = [] - for sample in samples: - judge_list.append(_compute_f1(sample["answer"], sample["parsed_pred"])) - - f1 = np.mean(judge_list) - return judge_list, {"f1": f1} diff --git a/lmms_eval/tasks/websrc/websrc.yaml b/lmms_eval/tasks/websrc/websrc.yaml deleted file mode 100644 index e32bed9..0000000 --- a/lmms_eval/tasks/websrc/websrc.yaml +++ /dev/null @@ -1,4 +0,0 @@ -group: websrc -task: -- websrc_val -- websrc_test diff --git a/lmms_eval/tasks/websrc/websrc_test.yaml b/lmms_eval/tasks/websrc/websrc_test.yaml deleted file mode 100644 index b19ec2c..0000000 --- a/lmms_eval/tasks/websrc/websrc_test.yaml +++ /dev/null @@ -1,19 +0,0 @@ -dataset_path: rootsautomation/websrc-test -task: "websrc_test" -test_split: test -output_type: generate_until -doc_to_visual: !function utils.websrc_doc_to_visual -doc_to_text: !function utils.websrc_doc_to_text -doc_to_target: "answer" -# The return value of process_results will be used by metrics -process_results: !function utils.websrc_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -generation_kwargs: - max_new_tokens: 16 - image_aspect_ratio: pad -metric_list: - - metric: submission - aggregation: !function utils.websrc_test_aggregate_results_for_submission - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file diff --git a/lmms_eval/tasks/websrc/websrc_val.yaml b/lmms_eval/tasks/websrc/websrc_val.yaml deleted file mode 100644 index aeac7d0..0000000 --- a/lmms_eval/tasks/websrc/websrc_val.yaml +++ /dev/null @@ -1,19 +0,0 @@ -dataset_path: rootsautomation/websrc -task: "websrc_val" -test_split: dev -output_type: generate_until -doc_to_visual: !function utils.websrc_doc_to_visual -doc_to_text: !function utils.websrc_doc_to_text -doc_to_target: "answer" -# The return value of process_results will be used by metrics -process_results: !function utils.websrc_process_results -# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results -generation_kwargs: - max_new_tokens: 16 - image_aspect_ratio: pad -metric_list: - - metric: websrc_squad_f1 - aggregation: !function utils.websrc_aggregate_results - higher_is_better: true -metadata: - - version: 0.0 \ No newline at end of file