Repository navigation
Expand file tree
/
Copy pathpooler.py
More file actions
193 lines (165 loc) · 6.92 KB
/
Copy pathpooler.py
File metadata and controls
193 lines (165 loc) · 6.92 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Literal, get_args
from pydantic import model_validator
from pydantic_core import ArgsKwargs
from vllm.config.utils import config
from vllm.logger import init_logger
from vllm.tasks import PoolingTask, check_removed_pooling_task
from vllm.utils.hashing import safe_hash
logger = init_logger(__name__)
SequencePoolingType = Literal["CLS", "LAST", "MEAN"]
SEQ_POOLING_TYPES: tuple[SequencePoolingType, ...] = get_args(SequencePoolingType)
TokenPoolingType = Literal["ALL", "STEP"]
TOK_POOLING_TYPES: tuple[TokenPoolingType, ...] = get_args(TokenPoolingType)
POOLER_CONFIG_LOG_FIELDS = (
"seq_pooling_type",
"tok_pooling_type",
"use_activation",
)
@config
class PoolerConfig:
"""Controls the behavior of output pooling in pooling models."""
task: PoolingTask | None = None
"""
The task used for pooling.
"""
pooling_type: SequencePoolingType | TokenPoolingType | None = None
"""
The pooling method used for pooling.
If set, `seq_pooling_type` or `tok_pooling_type` are automatically populated
with this field. Alternatively, users can set `seq_pooling_type` and
`tok_pooling_type` explicitly.
This field is mainly for user convenience. Internal code should always use
`seq_pooling_type` or `tok_pooling_type` instead of `pooling_type`.
"""
seq_pooling_type: SequencePoolingType | None = None
"""
The pooling method used for sequence pooling.
"""
tok_pooling_type: TokenPoolingType | None = None
"""
The pooling method used for tokenwise pooling.
"""
use_activation: bool | None = None
"""
Whether to apply activation function to the pooler outputs.
`None` uses the pooler's default, which is `True` in most cases.
"""
## for embedding models
dimensions: int | None = None
"""
Reduce the dimensions of embeddings if model
support matryoshka representation. Defaults to None.
"""
enable_chunked_processing: bool = False
"""
Whether to enable chunked processing for long inputs that exceed the model's
maximum position embeddings. When enabled, long inputs will be split into
chunks, processed separately, and then aggregated using weighted averaging.
This allows embedding models to handle arbitrarily long text without CUDA
errors. Defaults to False.
"""
max_embed_len: int | None = None
"""
Maximum input length allowed for embedding generation. When set, allows
inputs longer than max_embed_len to be accepted for embedding models.
When an input exceeds max_embed_len, it will be handled according to
the original max_model_len validation logic.
Defaults to None (i.e. set to max_model_len).
"""
## for classification models — affine score calibration
logit_mean: float | None = None
"""
If provided, subtract this value from classification logits before
activation. Used for affine score calibration (Platt scaling):
activation((logit - logit_mean) / logit_sigma). Defaults to None.
"""
logit_sigma: float | None = None
"""
If provided, divide the classification logits by this value after
mean subtraction. Used for affine score calibration (Platt scaling):
activation((logit - logit_mean) / logit_sigma). Defaults to None.
"""
## for reward models
step_tag_id: int | None = None
"""
If set, only the score corresponding to the `step_tag_id` in the
generated sentence should be returned. Otherwise, the scores for all tokens
are returned.
"""
returned_token_ids: list[int] | None = None
"""
A list of indices for the vocabulary dimensions to be extracted,
such as the token IDs of `good_token` and `bad_token` in the
`math-shepherd-mistral-7b-prm` model.
"""
@model_validator(mode="before")
@classmethod
def reject_removed_parameters(cls, data):
values = data.kwargs if isinstance(data, ArgsKwargs) else data
if not isinstance(values, dict):
return data
if "normalize" in values:
raise ValueError(
"Parameter `normalize` was removed; use `use_activation` instead."
)
check_removed_pooling_task(values.get("task"))
return data
def __post_init__(self) -> None:
if self.logit_sigma is not None and self.logit_sigma == 0:
raise ValueError("logit_sigma cannot be 0 (division by zero)")
if pooling_type := self.pooling_type:
if self.seq_pooling_type is not None:
raise ValueError(
"Cannot set both `pooling_type` and `seq_pooling_type`"
)
if self.tok_pooling_type is not None:
raise ValueError(
"Cannot set both `pooling_type` and `tok_pooling_type`"
)
if pooling_type in SEQ_POOLING_TYPES:
logger.debug(
"Resolved `pooling_type=%r` to `seq_pooling_type=%r`.",
pooling_type,
pooling_type,
)
self.seq_pooling_type = pooling_type # type: ignore[assignment]
elif pooling_type in TOK_POOLING_TYPES:
logger.debug(
"Resolved `pooling_type=%r` to `tok_pooling_type=%r`.",
pooling_type,
pooling_type,
)
self.tok_pooling_type = pooling_type # type: ignore[assignment]
else:
raise NotImplementedError(pooling_type)
def get_seq_pooling_type(self) -> SequencePoolingType:
if self.seq_pooling_type is None:
raise ValueError(
"seq_pooling_type is not set; it should be resolved by"
" ModelConfig before calling get_seq_pooling_type()"
)
return self.seq_pooling_type
def get_tok_pooling_type(self) -> TokenPoolingType:
if self.tok_pooling_type is None:
raise ValueError(
"tok_pooling_type is not set; it should be resolved by"
" ModelConfig before calling get_tok_pooling_type()"
)
return self.tok_pooling_type
def compute_hash(self) -> str:
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
Provide a hash that uniquely identifies all the configs
that affect the structure of the computation
graph from input ids/embeddings to the final hidden states,
excluding anything before input ids/embeddings and after
the final hidden states.
"""
# no factors to consider.
# this config will not affect the computation graph.
factors: list[Any] = []
hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
return hash_str