Fix doctest

yfw · yfw · commit 2a449657b5fa · 2025-08-18T14:48:26.000-07:00
Signed-off-by: Yi-Fu Wu &lt;yifu.wu@gmail.com&gt;
diff --git a/nemo_rl/algorithms/utils.py b/nemo_rl/algorithms/utils.py
@@ -173,7 +173,7 @@ def get_tokenizer(tokenizer_config: TokenizerConfig) -> PreTrainedTokenizerBase:
         >>> from transformers import AutoTokenizer
         >>> from nemo_rl.algorithms.utils import get_tokenizer
         >>> # not specifying a chat template uses the tokenizer's default
-        >>> config = {"name": "meta-llama/Llama-3.2-1B-Instruct"}
+        >>> config = {"name": "meta-llama/Llama-3.2-1B-Instruct", "is_tokenizer_processor": False}
         >>> tokenizer = get_tokenizer(config)
         No chat template provided, using tokenizer's default
         >>> messages = [
@@ -186,7 +186,8 @@ def get_tokenizer(tokenizer_config: TokenizerConfig) -> PreTrainedTokenizerBase:
         >>> # Using a passthrough template
         >>> config = {
         ...     "name": "meta-llama/Llama-3.2-1B-Instruct",
-        ...     "chat_template": None
+        ...     "chat_template": None,
+        ...     "is_tokenizer_processor": False,
         ... }
         >>> tokenizer = get_tokenizer(config)
         Using passthrough chat template
@@ -196,7 +197,8 @@ def get_tokenizer(tokenizer_config: TokenizerConfig) -> PreTrainedTokenizerBase:
         >>> # Using a custom template
         >>> config = {
         ...     "name": "meta-llama/Llama-3.2-1B-Instruct",
-        ...     "chat_template": "{% for message in messages %}{{ ' START: ' + message['content'] + ' END.' }}{% endfor %}"
+        ...     "chat_template": "{% for message in messages %}{{ ' START: ' + message['content'] + ' END.' }}{% endfor %}",
+        ...     "is_tokenizer_processor": False,
         ... }
         >>> tokenizer = get_tokenizer(config)
         Using custom chat template
