Skip to content

Commit

Permalink
Fix frequency/presence penalty
Browse files Browse the repository at this point in the history
This PR fixes the frequency/presence penalty for first token.
  • Loading branch information
cyx-6 committed Dec 21, 2023
1 parent e87f690 commit 3a026f5
Showing 1 changed file with 1 addition and 1 deletion.
2 changes: 1 addition & 1 deletion serve/mlc_serve/model/paged_cache_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ def _is_safe_to_sample(prob_like):
do_top_p |= top_ps[-1] < 1.0
do_top_k |= top_ks[-1] != vocab_size

if not param.presence_penalty == 0.0 or not param.frequency_penalty == 0 and bool(freq):
if (not param.presence_penalty == 0.0 or not param.frequency_penalty == 0) and bool(freq):
index = torch.from_numpy(np.array(list(freq.keys()))).to(device=logits.device)
src = torch.from_numpy(np.array(list(freq.values()))).type_as(logits).to(device=logits.device)
logits[i][index] -= src * param.frequency_penalty + param.presence_penalty
Expand Down

0 comments on commit 3a026f5

Please sign in to comment.