Repository navigation
Expand file tree
/
Copy pathlib.rs
More file actions
301 lines (285 loc) · 9.18 KB
/
Copy pathlib.rs
File metadata and controls
301 lines (285 loc) · 9.18 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
use std::{iter::zip, str::FromStr};
use hf_hub::{
api::sync::{ApiBuilder, ApiError},
Repo, RepoType,
};
#[cfg(feature = "num")]
use num_derive::{FromPrimitive, ToPrimitive};
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tokenizers::{Encoding, Tokenizer};
use tract_onnx::prelude::*;
/// ONNX export of MobileBERT fine-tuned for part-of-speech tagging,
/// downloaded from Hugging Face on first use.
pub const MODEL_REPO: &str = "onnx-community/mobilebert-finetuned-pos-ONNX";
/// Pinned so that [`LABELS`] and the snapshots stay valid.
pub const MODEL_REVISION: &str = "5b7653c4e2078ca6e6a7fd099c1f558eb066315b";
const MODEL_FILE: &str = "onnx/model.onnx";
/// `id2label` of the model, from
/// <https://huggingface.co/onnx-community/mobilebert-finetuned-pos-ONNX/blob/main/config.json>.
const LABELS: [PartOfSpeech; N_PART_OF_SPEECH as usize] = {
use PartOfSpeech::*;
[
O, CC, CD, DT, EX, FW, IN, JJ, JJR, JJS, MD, NN, NNP, NNPS, NNS, PDT, POS, PRP, RB, RBR,
RBS, RP, SYM, TO, UH, VB, VBD, VBG, VBN, VBP, VBZ, WDT, WP, WRB,
]
};
/// `max_position_embeddings` of the model.
const MAX_N_TOKEN: usize = 512;
const CLS_ID: i64 = 101;
const SEP_ID: i64 = 102;
/// Part-of-speech tagging model.
// 🧑 “fix natural_syntax_ls. It’s currently hard to install.”
pub struct POSModel {
tokenizer: Tokenizer,
model: Arc<TypedRunnableModel>,
}
impl POSModel {
pub fn try_default() -> Result<Self, POSModelError> {
let repo = ApiBuilder::from_env()
.with_progress(false)
.build()?
.repo(Repo::with_revision(
MODEL_REPO.into(),
RepoType::Model,
MODEL_REVISION.into(),
));
let tokenizer = Tokenizer::from_file(repo.get("tokenizer.json")?)?;
let model = tract_onnx::onnx()
.model_for_path(repo.get(MODEL_FILE)?)?
.into_optimized()?
.into_runnable()?;
Ok(Self { tokenizer, model })
}
/// Predict [`POSToken`]s for `input`, one per word.
pub fn predict(&self, input: &str) -> Result<Vec<POSToken>, POSModelError> {
let encoding = self.tokenizer.encode(input, false)?;
let mut predictions = Vec::with_capacity(encoding.len());
for ids in encoding.get_ids().chunks(MAX_N_TOKEN - 2) {
predictions.extend(self.classify(ids)?);
}
Ok(words(input, &encoding, &predictions))
}
/// Most likely part of speech and its confidence score for
/// each of the token `ids`.
fn classify(&self, ids: &[u32]) -> TractResult<Vec<(PartOfSpeech, f64)>> {
let n_token = ids.len() + 2;
let ids = [CLS_ID]
.into_iter()
.chain(ids.iter().map(|&id| id as i64))
.chain([SEP_ID])
.collect::<Vec<_>>();
let inputs = tvec![
tensor1(&ids),
tensor1(&vec![1i64; n_token]),
tensor1(&vec![0i64; n_token]),
]
.into_iter()
.map(|tensor| Ok(tensor.into_shape(&[1, n_token])?.into()))
.collect::<TractResult<TVec<TValue>>>()?;
let outputs = self.model.run(inputs)?;
let logits = outputs[0].to_plain_array_view::<f32>()?;
Ok(logits
.rows()
.into_iter()
.skip(1)
.take(n_token - 2)
.map(|row| {
let (i_max, max) = zip(0.., row)
.max_by(|(_, a), (_, b)| a.total_cmp(b))
.expect("The model outputs one logit per label.");
let sum = row.iter().map(|logit| (logit - max).exp()).sum::<f32>();
(LABELS[i_max], 1. / sum as f64)
})
.collect())
}
}
/// Merge the sub-word tokens in `encoding` of `input` into words,
/// each tagged with the prediction of its first token.
fn words(input: &str, encoding: &Encoding, predictions: &[(PartOfSpeech, f64)]) -> Vec<POSToken> {
let mut byte_words: Vec<(usize, usize, PartOfSpeech, f64)> = Vec::new();
let mut prev_word_id = None;
for ((word_id, &(begin, end)), &(tag, score)) in zip(
zip(encoding.get_word_ids(), encoding.get_offsets()),
predictions,
) {
match byte_words.last_mut() {
Some(word) if *word_id == prev_word_id => word.1 = end,
_ => byte_words.push((begin, end, tag, score)),
}
prev_word_id = *word_id;
}
let (mut i_byte, mut i_char) = (0, 0);
byte_words
.into_iter()
.map(|(begin, end, tag, score)| {
let word = &input[begin..end];
i_char += input[i_byte..begin].chars().count() as u32;
i_byte = begin;
POSToken {
word: word.into(),
score,
tag,
offset_begin: i_char,
offset_end: i_char + word.chars().count() as u32,
}
})
.collect()
}
#[derive(Debug, Error)]
pub enum POSModelError {
#[error("Downloading the model: {0}")]
Download(#[from] ApiError),
#[error("Tokenizing: {0}")]
Tokenizer(#[from] tokenizers::Error),
#[error("Running the model: {0}")]
Model(#[from] TractError),
}
/// A word tagged with its part of speech.
#[derive(Clone, Debug, Default, PartialEq, PartialOrd)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
pub struct POSToken {
/// String representation of the Token
pub word: String,
/// Confidence score
pub score: f64,
/// Part-of-speech tag
pub tag: PartOfSpeech,
/// Token offset beginning (in unicode points) relative to the input string
pub offset_begin: u32,
/// Token offset end (in unicode points) relative to the input string
pub offset_end: u32,
}
impl POSToken {
pub fn tag_with_confidence(&self, confidence: f64) -> Option<PartOfSpeech> {
(self.score > confidence).then_some(self.tag)
}
}
/// Enum representing part-of-speech labels of MobileBERT, from
/// <https://huggingface.co/mrm8488/mobilebert-finetuned-pos/resolve/main/config.json>.
// NOTE: ChatGPT generated the docstrings, so they may be inaccurate.
#[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
#[cfg_attr(feature = "num", derive(FromPrimitive, ToPrimitive))]
pub enum PartOfSpeech {
/// Coordinating conjunction
CC = 0,
/// Cardinal number
CD = 1,
/// Determiner
DT = 2,
/// Existential there
EX = 3,
/// Foreign word
FW = 4,
/// Preposition or subordinating conjunction
IN = 5,
/// Adjective
JJ = 6,
/// Adjective, comparative
JJR = 7,
/// Adjective, superlative
JJS = 8,
/// Modal
MD = 9,
/// Noun, singular or mass
NN = 10,
/// Proper noun, singular
NNP = 11,
/// Proper noun, plural
NNPS = 12,
/// Noun, plural
NNS = 13,
/// Other (not a part of speech)
#[default]
O = 14,
/// Predeterminer
PDT = 15,
/// Possessive ending
POS = 16,
/// Personal pronoun
PRP = 17,
/// Adverb
RB = 18,
/// Adverb, comparative
RBR = 19,
/// Adverb, superlative
RBS = 20,
/// Particle
RP = 21,
/// Symbol
SYM = 22,
/// to
TO = 23,
/// Interjection
UH = 24,
/// Verb, base form
VB = 25,
/// Verb, past tense
VBD = 26,
/// Verb, gerund or present participle
VBG = 27,
/// Verb, past participle
VBN = 28,
/// Verb, non-3rd person singular present
VBP = 29,
/// Verb, 3rd person singular present
VBZ = 30,
/// Wh-determiner
WDT = 31,
/// Wh-pronoun
WP = 32,
/// Wh-adverb
WRB = 33,
}
pub const N_PART_OF_SPEECH: u8 = 34;
impl FromStr for PartOfSpeech {
type Err = PartOfSpeechError;
fn from_str(input: &str) -> Result<Self, Self::Err> {
match input {
"CC" => Ok(Self::CC),
"CD" => Ok(Self::CD),
"DT" => Ok(Self::DT),
"EX" => Ok(Self::EX),
"FW" => Ok(Self::FW),
"IN" => Ok(Self::IN),
"JJ" => Ok(Self::JJ),
"JJR" => Ok(Self::JJR),
"JJS" => Ok(Self::JJS),
"MD" => Ok(Self::MD),
"NN" => Ok(Self::NN),
"NNP" => Ok(Self::NNP),
"NNPS" => Ok(Self::NNPS),
"NNS" => Ok(Self::NNS),
"O" => Ok(Self::O),
"PDT" => Ok(Self::PDT),
"POS" => Ok(Self::POS),
"PRP" => Ok(Self::PRP),
"RB" => Ok(Self::RB),
"RBR" => Ok(Self::RBR),
"RBS" => Ok(Self::RBS),
"RP" => Ok(Self::RP),
"SYM" => Ok(Self::SYM),
"TO" => Ok(Self::TO),
"UH" => Ok(Self::UH),
"VB" => Ok(Self::VB),
"VBD" => Ok(Self::VBD),
"VBG" => Ok(Self::VBG),
"VBN" => Ok(Self::VBN),
"VBP" => Ok(Self::VBP),
"VBZ" => Ok(Self::VBZ),
"WDT" => Ok(Self::WDT),
"WP" => Ok(Self::WP),
"WRB" => Ok(Self::WRB),
_ => Err(PartOfSpeechError::UnknownLabel(input.into())),
}
}
}
#[derive(Clone, Debug, Error)]
pub enum PartOfSpeechError {
#[error("Unknown part of speech label `{0}`")]
UnknownLabel(String),
}
#[cfg(test)]
mod tests;