Merge branch 'hn-fix-tokenizer' into 'main'

jaredcasper · jaredcasper · commit 236c7911d561 · 2024-07-29T12:55:47.000-07:00
Fix TikTokenizer decoding case

See merge request ADLR/megatron-lm!1827
diff --git a/megatron/inference/text_generation/tokenization.py b/megatron/inference/text_generation/tokenization.py
@@ -35,6 +35,8 @@ def detokenize_generations(tokens_gpu_tensor,
                                            'HuggingFaceTokenizer',
                                            'Llama2Tokenizer']:
                     word = tokenizer.decoder[token]
+                elif args.tokenizer_type == 'TikTokenizer':
+                    word = tokenizer.detokenize([token])
                 elif args.tokenizer_type in ['Llama3Tokenizer', 'MistralTokenizer']:
                     word = tokenizer.decode([token])
                 elif args.tokenizer_type == 'NullTokenizer':
