Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add sampling penalties and logit bias #125

Merged
merged 6 commits into from
Dec 20, 2023
Merged

Add sampling penalties and logit bias #125

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
293fb8b
Add sampling penalties and logit bias
cyx-6 Dec 19, 2023
78c1390
fix
cyx-6 Dec 20, 2023
b4c7d1f
fix lint
cyx-6 Dec 20, 2023
69eb2fd
vectorized sampling
cyx-6 Dec 20, 2023
941f2e4
apply code review suggestions
cyx-6 Dec 20, 2023
ae2ada5
fix
cyx-6 Dec 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions serve/benchmarks/benchmark_throughput.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ def run_mlc(
sampling_params = SamplingParams(
temperature=1.0,
top_p=1.0,
frequency_penalty=-1,
logit_bias={1: -1}
)

engine.add(
Expand Down
18 changes: 11 additions & 7 deletions serve/mlc_serve/model/paged_cache_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,18 @@ def _is_safe_to_sample(prob_like):
do_top_p |= top_ps[-1] < 1.0
do_top_k |= top_ks[-1] != vocab_size

if not param.presence_penalty == 0.0 or not param.frequency_penalty == 0:
for token_id, token_freq in freq.items():
logits[i][token_id] -= token_freq * param.frequency_penalty + param.presence_penalty
if not param.presence_penalty == 0.0 or not param.frequency_penalty == 0 and bool(freq):
freq_tensor = np.array(list(freq.items()))
index = torch.from_numpy(freq_tensor[..., 0]).to(device=logits.device)
src = torch.from_numpy(freq_tensor[..., 1]).type_as(logits).to(device=logits.device)
cyx-6 marked this conversation as resolved.
Show resolved Hide resolved
logits[i] = torch.scatter_add(logits[i], dim=0, index=index, src=-(src * param.frequency_penalty + param.presence_penalty))

if param.logit_bias:
for token_id, token_bias in param.logit_bias.items():
logits[i][token_id] += token_bias

bias_tensor = np.array(list(param.logit_bias.items()))
index = torch.from_numpy(bias_tensor[..., 0]).to(device=logits.device)
src = torch.from_numpy(bias_tensor[..., 1]).type_as(logits).to(device=logits.device)
cyx-6 marked this conversation as resolved.
Show resolved Hide resolved
logits[i] = torch.scatter_add(logits[i], dim=0, index=index, src=src)
cyx-6 marked this conversation as resolved.
Show resolved Hide resolved


logits_random = logits[mask_random]

Expand Down Expand Up @@ -478,7 +482,7 @@ def generate(
):
if not new_token in requests[i].sampling_params.appeared_tokens_freq:
requests[i].sampling_params.appeared_tokens_freq[new_token] = 0
request.sampling_params.appeared_tokens_freq[new_token] += 1
requests[i].sampling_params.appeared_tokens_freq[new_token] += 1
if sequence_id.sequence_index == PROMPT_SEQEUNCE_INDEX:
for seq_id in range(num_sequences[i]):
outputs.append(
Expand Down