feat(core): make nmt normalization false by default

formermagic · Sep 17, 2021 · 583ac46 · 583ac46
1 parent f2857d9
commit 583ac46
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/git_t5/core/tokenizer_model.py b/git_t5/core/tokenizer_model.py
@@ -62,6 +62,7 @@ class SentencePieceTokenizerConfig(TokenizerConfig):
     trim_offsets: bool = False
     min_frequency: int = 2
     lowercase: bool = False
+    nmt_normalizer: bool = False
     remove_extra_spaces: bool = True
     unicode_normalizer: Optional[str] = "nfkc"
     unk_token: str = "<unk>"
@@ -127,7 +128,9 @@ def get_tokenizer(self) -> Tokenizer:
 
     def get_normalizer(self) -> normalizers.Normalizer:
         normalizer_list: List[normalizers.Normalizer] = []
-        normalizer_list.append(normalizers.Nmt())
+
+        if self.config.nmt_normalizer:
+            normalizer_list.append(normalizers.Nmt())
 
         if self.config.unicode_normalizer is not None:
             normalizer_list.append(unicode_normalizer(self.config.unicode_normalizer))