Skip to content

Commit 5e0a7f7

Browse files
speedstorm1copybara-github
authored andcommitted
build: split the local-tokenizer extra so SentencePiece models skip PyTorch
fixes #2728 PiperOrigin-RevId: 1014934925
1 parent 6eb8523 commit 5e0a7f7

5 files changed

Lines changed: 96 additions & 4 deletions

File tree

‎README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1298,6 +1298,18 @@ print(response)
12981298

12991299
#### Local Count Tokens
13001300

1301+
The local tokenizer needs an extra. SentencePiece models such as
1302+
`gemini-2.5-flash` need only the light one. Models that use the Gemma 4
1303+
tokenizer, such as `gemini-3.5-flash`, also need Transformers and PyTorch.
1304+
Install one of the two extras:
1305+
1306+
```sh
1307+
# SentencePiece models only.
1308+
pip install "google-genai[local-tokenizer]"
1309+
# Also models that use the Gemma 4 tokenizer (adds Transformers and PyTorch).
1310+
pip install "google-genai[local-tokenizer-transformers]"
1311+
```
1312+
13011313
```python
13021314
from google.genai import local_tokenizer
13031315

‎google/genai/_local_tokenizer_loader.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -219,11 +219,11 @@ def get_huggingface_tokenizer(tokenizer_name: str) -> Any:
219219
# Load the processor which includes the tokenizer
220220
try:
221221
from transformers import AutoProcessor
222-
except ImportError:
222+
except ImportError as e:
223223
raise ImportError(
224-
"Please install transformers to use huggingface tokenizer: pip install"
225-
" transformers"
226-
) from ImportError
224+
"The transformers library is required to tokenize this model. Install"
225+
' it with: pip install "google-genai[local-tokenizer-transformers]"'
226+
) from e
227227
processor = AutoProcessor.from_pretrained( # type: ignore[no-untyped-call]
228228
GEMMA_TOKENIZER_TO_MODEL_NAMES[tokenizer_name]
229229
)
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
"""Tests for the local tokenizer install extras declared in pyproject.toml."""
16+
17+
import os
18+
import pathlib
19+
import re
20+
import tomllib
21+
22+
_TRANSFORMERS_STACK = frozenset(
23+
{'pillow', 'torch', 'torchvision', 'transformers'}
24+
)
25+
26+
27+
def _pyproject_path() -> pathlib.Path:
28+
"""Finds pyproject.toml, which the public export moves to the repository root."""
29+
candidates = []
30+
srcdir = os.environ.get('TEST_SRCDIR')
31+
if srcdir:
32+
candidates.append(
33+
pathlib.Path(srcdir)
34+
/ 'google3/third_party/py/google/genai/packaging/pyproject.toml'
35+
)
36+
for parent in pathlib.Path(__file__).resolve().parents:
37+
candidates.append(parent / 'packaging' / 'pyproject.toml')
38+
candidates.append(parent / 'pyproject.toml')
39+
for candidate in candidates:
40+
if candidate.is_file():
41+
return candidate
42+
raise FileNotFoundError('cannot find pyproject.toml')
43+
44+
45+
def _extra_packages(extra_name: str) -> set[str]:
46+
"""Returns the distribution names listed in one pyproject.toml extra."""
47+
with _pyproject_path().open('rb') as pyproject_file:
48+
pyproject = tomllib.load(pyproject_file)
49+
extras = pyproject['project']['optional-dependencies']
50+
return {re.split(r'[<>=!~;\[ ]', spec)[0] for spec in extras[extra_name]}
51+
52+
53+
def test_local_tokenizer_extra_excludes_transformers_stack():
54+
"""A SentencePiece model installs without Transformers and PyTorch."""
55+
packages = _extra_packages('local-tokenizer')
56+
57+
assert 'sentencepiece' in packages
58+
assert 'protobuf' in packages
59+
assert (packages & _TRANSFORMERS_STACK) == set()
60+
61+
62+
def test_local_tokenizer_transformers_extra_has_full_stack():
63+
"""The Gemma 4 extra stays a superset of the SentencePiece extra."""
64+
packages = _extra_packages('local-tokenizer-transformers')
65+
66+
assert packages >= {'sentencepiece', 'protobuf'} | _TRANSFORMERS_STACK

‎google/genai/tests/local_tokenizer/test_local_tokenizer_loader.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# limitations under the License.
1414
#
1515

16+
import sys
1617
import unittest
1718
from unittest.mock import MagicMock, mock_open, patch
1819

@@ -247,3 +248,12 @@ def test_get_sentencepiece_caching(
247248

248249
# Should only be loaded once due to lru_cache
249250
mock_get.assert_called_once()
251+
252+
253+
class TestGetHuggingfaceTokenizer(unittest.TestCase):
254+
255+
def test_get_huggingface_tokenizer_error_names_extra(self):
256+
"""A missing transformers install names the extra that provides it."""
257+
with patch.dict(sys.modules, {"transformers": None}):
258+
with self.assertRaisesRegex(ImportError, "local-tokenizer-transformers"):
259+
loader.get_huggingface_tokenizer("gemma4")

‎pyproject.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ httpx2 = []
4444
local-tokenizer = [
4545
"sentencepiece>=0.2.0",
4646
"protobuf",
47+
]
48+
local-tokenizer-transformers = [
49+
"sentencepiece>=0.2.0",
50+
"protobuf",
4751
"pillow",
4852
"torch",
4953
"torchvision",

0 commit comments

Comments
 (0)