Skip to content

Commit 586ee38

Browse files
committed
Address Percy's comments
1 parent d521257 commit 586ee38

File tree

2 files changed

+4
-4
lines changed

2 files changed

+4
-4
lines changed

src/benchmark/metrics/basic_metrics.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -549,9 +549,10 @@ def compute_efficiency_metrics(
549549

550550
# Total number of tokens in the completion.
551551
num_completion_tokens: int = sum([len(completion.tokens) for completion in request_state.result.completions])
552-
# Assume that tokens for different completions are generated sequentially (instead of batched).
553-
num_output_tokens: int = num_completion_tokens
554552
# Don't include prompt in number of generated tokens (e.g., for language modeling).
553+
# Assume that tokens for different completions are generated sequentially (instead of batched) when
554+
# computing num_output_tokens (for the purpose of runtime estimation).
555+
num_output_tokens: int = num_completion_tokens
555556
if request_state.request.echo_prompt:
556557
# num_prompt_tokens > num_output_tokens can happen if tokenizer doesn't round trip.
557558
if num_prompt_tokens <= num_output_tokens:
@@ -588,7 +589,6 @@ def compute_efficiency_metrics(
588589
else:
589590
training_energy_cost = None
590591

591-
# TODO: unify num_completion_tokens and num_output_tokens.
592592
stats = [
593593
Stat(MetricName("num_prompt_tokens")).add(num_prompt_tokens),
594594
Stat(MetricName("num_completion_tokens")).add(num_completion_tokens),

src/benchmark/static/schema.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -913,7 +913,7 @@ metric_groups:
913913
split: ${main_split}
914914
- name: num_prompt_tokens
915915
split: ${main_split}
916-
- name: num_completion_tokens
916+
- name: num_output_tokens
917917
split: ${main_split}
918918
- name: num_train_trials
919919
split: ${main_split}

0 commit comments

Comments
 (0)