delete inrelavant tasks

hanqiu-hq · Jun 5, 2024 · 6727093 · 6727093
1 parent 15f1ab4
commit 6727093
Show file tree

Hide file tree

Showing 260 changed files with 14 additions and 13,193 deletions.
diff --git a/README.md b/README.md
@@ -22,7 +22,7 @@ accelerate launch --num_processes=1 -m lmms_eval \
 accelerate launch --num_processes=1 -m lmms_eval \ 
 --model llava \ 
 --model_args pretrained="liuhaotian/llava-v1.5-7b" \
---tasks lhqa_complete_conversation,lhqa_complete_conversation_ppl,lhqa_complete_description,lhqa_complete_description_ppl \
+--tasks lhqa_complete_description,lhqa_complete_description_ppl,lhqa_complete_conversation,lhqa_complete_conversation_ppl \
 --batch_size 1 \
 --log_samples \ 
 --log_samples_suffix llava_v15_7b_lhqa_completion \
@@ -41,6 +41,19 @@ accelerate launch --num_processes=1 -m lmms_eval \
 | LLaVA-1.5-13b | llava        | "liuhaotian/llava-v1.5-13b"                                       |
 | LLaVA-1.6-7b  | llava        | "liuhaotian/llava-v1.6-mistral-7b,conv_template=mistral_instruct" |
 
+### The evaluated sub-items in LongHalQA are:
+
+
+| Hallucinaiton Discrimination     | Hallucination Completion       |
+|:---------------------------------|:-------------------------------|
+| lhqa_discrim_object_binary       | lhqa_complete_description      |
+| lhqa_discrim_description_binary  | lhqa_complete_description_ppl  |
+| lhqa_discrim_conversation_binary | lhqa_complete_conversation     |
+| lhqa_discrim_description_choice  | lhqa_complete_conversation_ppl |
+| lhqa_discrim_description_ppl     |                                |
+| lhqa_discrim_conversation_choice |                                |
+| lhqa_discrim_conversation_ppl    |                                |
+
 
 # Acknowledgement
 

diff --git a/lmms_eval/tasks/ai2d/ai2d.yaml b/lmms_eval/tasks/ai2d/ai2d.yaml
diff --git a/lmms_eval/tasks/ai2d/upload_ai2d.py b/lmms_eval/tasks/ai2d/upload_ai2d.py
diff --git a/lmms_eval/tasks/ai2d/utils.py b/lmms_eval/tasks/ai2d/utils.py
diff --git a/lmms_eval/tasks/chartqa/chartqa.yaml b/lmms_eval/tasks/chartqa/chartqa.yaml
diff --git a/lmms_eval/tasks/chartqa/upload_chartqa.py b/lmms_eval/tasks/chartqa/upload_chartqa.py