fix: correct WhitespaceSplit Pretokenizer handling of invisible space…

… chars
CodeWithKyrian · Sep 13, 2024 · 6ec3e3e · 6ec3e3e
1 parent e8a8a9a
commit 6ec3e3e
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/PreTokenizers/WhitespaceSplit.php b/src/PreTokenizers/WhitespaceSplit.php
@@ -14,6 +14,8 @@ public function __construct(protected array $config)
 
     public function preTokenizeText(string|array $text, array $options): array
     {
-        return explode(' ', $text);
+        preg_match_all('/\S+/', $text, $matches);
+
+        return $matches[0] ?? [];
     }
 }