使用GLM4跑您这边提供的数据，效果很低，是因为大模型的问题吗？

[AryaFeng]

1.代码如下。

测试generate_then_read代码在glm9b下的性

from openai import OpenAI
import re
import json
from tqdm import tqdm

class ChatGLM4(object):

def __init__(self, api_key="EMPTY", base_url="http://x.x.x.x:8000/v1/",model_name="glm-4"):
    self.base_url = base_url
    self.model_name = model_name
    # 初始化 OpenAI 客户端，并增加错误处理
    try:
        self._client = OpenAI(api_key="glm-4", base_url=self.base_url)
    except Exception as e:
        raise ValueError(f"Failed to initialize OpenAI client: {e}")

def call(self, prompt, chat_history = []):

    # 将 chat_history 与当前 prompt 组合成请求
    messages = [{"role": "system", "content": "you are a helpful assistant!"}]
    # 添加之前的对话历史
    for user_content, assistant_content in chat_history:
        messages.append({"role": "user", "content": user_content})
        messages.append({"role": "system", "content": assistant_content})

    # 当前用户的输入
    messages.append({"role": "user", "content": prompt})
    # logger_file.info(f"messages: {messages}")
    # 发送请求
    try:
        response = self._client.chat.completions.create(
            model="glm-4",  # 或其他模型
            messages=messages,
            # max_tokens=,  # 可以调整返回的 token 数量
            n=1,  # 返回一个结果
            stop=None,  # 可设置 stop 标记
            temperature=0,  # 控制输出的创造性
            top_p = 0.1
        )
        return response

    except Exception as e:
        log.error(f"error:,Failed to get a response from OpenAI: {e}")
        raise ValueError(f"Failed to get a response from OpenAI: {e}")


def chat(self,prompt,chat_history=[]):
    response = self.call(prompt,chat_history)
    response = response.choices[0].message.content
    return response

llm = ChatGLM4()

with open("webq_test_gpt_output.json",'r') as f:
datas = json.load(f)

prompt = """
Refer to the passage below and answer the following question with just one entity. \n\n Passage: {background} \n\n Question: {query} \n\n The answer is
"""
results = []
for data in tqdm(datas):
question = data['question']
ctxs = data['ctxs']
answer = data['answers']
string = ""
for idx,ctx in enumerate(ctxs):
string += f"\n {idx}: {ctx['text']}"
final_prompt = prompt.format(background=string,query=question)
output= llm.chat(final_prompt)
results.append({"question":question,"answer":answer,"output":output})

with open('results2.json', 'w', encoding='utf-8') as f:
json.dump(results, f, ensure_ascii=False, indent=4) # 设置 ensure_ascii=False 以支持中文
得到结果如下

将输出转化为列表，

import json

results = []
with open("results2.json",'r') as f:
datas = json.load(f)

for data in datas:
question = data['question']
answer = data['answer']
output = [ ans.strip() for ans in data['output'][0].split(',')]
results.append({"question":question,"answer":answer,"output":output})

with open('results_output.json', 'w', encoding='utf-8') as f:
json.dump(results, f, ensure_ascii=False, indent=4) # 设置 ensure_ascii=False 以支持中文

得到结果如下：

最后调用eval进行评估

import regex
import json
import string
import unicodedata
from typing import List
import numpy as np
from collections import Counter

from rouge import Rouge

class SimpleTokenizer(object):
ALPHA_NUM = r'[\p{L}\p{N}\p{M}]+'
NON_WS = r'[^\p{Z}\p{C}]'

def __init__(self):
    """
    Args:
        annotators: None or empty set (only tokenizes).
    """
    self._regexp = regex.compile(
        '(%s)|(%s)' % (self.ALPHA_NUM, self.NON_WS),
        flags=regex.IGNORECASE + regex.UNICODE + regex.MULTILINE
    )

def tokenize(self, text, uncased=False):
    matches = [m for m in self._regexp.finditer(text)]
    if uncased:
        tokens = [m.group().lower() for m in matches]
    else:
        tokens = [m.group() for m in matches]
    return tokens

def check_answer(example, tokenizer) -> List[bool]:
"""Search through all the top docs to see if they have any of the answers."""
answers = example['answers']
ctxs = example['ctxs']

hits = []

for _, doc in enumerate(ctxs):
    text = doc['text']

    if text is None:  # cannot find the document for some reason
        hits.append(False)
        continue

    hits.append(has_answer(answers, text, tokenizer))

return hits

def has_answer(answers, text, tokenizer=SimpleTokenizer()) -> bool:
"""Check if a document contains an answer string."""
text = _normalize(text)
text = tokenizer.tokenize(text, uncased=True)

for answer in answers:
    answer = _normalize(answer)
    answer = tokenizer.tokenize(answer, uncased=True)
    for i in range(0, len(text) - len(answer) + 1):
        if answer == text[i: i + len(answer)]:
            return True
return False

def _normalize(text):
return unicodedata.normalize('NFD', text)

def normalize_answer(s):
def remove_articles(text):
return regex.sub(r'\b(a|an|the)\b', ' ', text)

def white_space_fix(text):
    return ' '.join(text.split())

def remove_punc(text):
    exclude = set(string.punctuation)
    return ''.join(ch for ch in text if ch not in exclude)

def lower(text):
    return text.lower()

return white_space_fix(remove_articles(remove_punc(lower(s))))

def exact_match_score(prediction, ground_truth):
return normalize_answer(prediction) == normalize_answer(ground_truth)

def ems(prediction, ground_truths):
return max([exact_match_score(prediction, gt) for gt in ground_truths])

def f1_score(prediction, ground_truth):
prediction_tokens = normalize_answer(prediction).split()
ground_truth_tokens = normalize_answer(ground_truth).split()
common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
num_same = sum(common.values())
if num_same == 0:
return 0
precision = 1.0 * num_same / len(prediction_tokens)
recall = 1.0 * num_same / len(ground_truth_tokens)
f1 = (2 * precision * recall) / (precision + recall)
return f1

def f1(prediction, ground_truths):
return max([f1_score(prediction, gt) for gt in ground_truths])

def rougel_score(prediction, ground_truth):

rouge = Rouge()

# no normalization

try:

scores = rouge.get_scores(prediction, ground_truth, avg=True)

except ValueError: # "Hypothesis is empty."

return 0.0

return scores["rouge-l"]["f"]

def rl(prediction, ground_truths):

return max([rougel_score(prediction, gt) for gt in ground_truths])

file-level evaluation ...

def eval_recall(infile):

tokenizer = SimpleTokenizer()
lines = open(infile, 'r').readlines()[1:]

has_answer_count = 0
answer_lengths = []
for line in lines:
    line = json.loads(line)
    answer = line['answer']
    output = ' || '.join(line['output'])

    if has_answer(answer, output, tokenizer):
        has_answer_count += 1

    answer_lengths.append(len(output.split()))

recall = round(has_answer_count/len(lines), 4)
lens = round(np.mean(answer_lengths), 4)

return recall, lens

def eval_question_answering(infile):

# lines = open(infile, 'r').readlines()[1:]
with open(infile,'r') as f:
    lines = json.load(f)

exact_match_count = 0
answer_lengths = []
for line in lines:
    # line = json.loads(line)
    answer = line['answer']
    output = line['output'][0]

    if ems(output, answer): # EM evaluation
        exact_match_count += 1
    
    answer_lengths.append(len(output.split()))

em = round(exact_match_count/len(lines), 4)
lens = round(np.mean(answer_lengths), 4)

return em, lens

def eval_fact_checking(infile):

tokenizer = SimpleTokenizer()
lines = open(infile, 'r').readlines()[1:]

exact_match_count = 0
answer_lengths = []
for line in lines:
    line = json.loads(line)
    answer = line['answer']
    output = line['output'][0]

    if answer == ["refutes"]:
        answer = ["refutes", "no", "false"]
    if answer == ["supports"]:
        answer = ["supports", "yes", "true"]

    if has_answer(answer, output, tokenizer):
        exact_match_count += 1
    
    answer_lengths.append(len(output.split()))

em = round(exact_match_count/len(lines), 4)
lens = round(np.mean(answer_lengths), 4)

return em, lens

def eval_dialogue_system(infile):

lines = open(infile, 'r').readlines()[1:]

f1_scores = []
rl_scores = []
answer_lengths = []
for line in lines:
    line = json.loads(line)
    answer = line['answer']
    output = line['output'][0]

    f1_scores.append(f1(output, answer))
    rl_scores.append(rl(output, answer))
    answer_lengths.append(len(output.split()))

F1 = round(np.mean(f1_scores), 4)
RL = round(np.mean(rl_scores), 4)
lens = round(np.mean(answer_lengths), 4)

return F1, RL, lens

emscore, length = eval_question_answering("results_output.json")
print(emscore)
print(length) 最后的结果只有
0.3179
2.6216
不清楚是因为大模型的问题吗。
webq_test_gpt_output.json来自于https://drive.google.com/drive/folders/1DNjTTOLKi24wohJKu1Z-v6b4izfymlLu 您这边提供的文件。希望可以得到您的回复