Skip to content

Commit 901a049

Browse files
fix: preTokenizer null error when there's no text pair
1 parent 91db063 commit 901a049

File tree

1 file changed

+1
-1
lines changed

1 file changed

+1
-1
lines changed

src/PretrainedTokenizers/PretrainedTokenizer.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ public function encodePlus(
379379

380380
$combinedTokens = $this->postProcessor
381381
? $this->postProcessor->postProcess($tokens, $tokens2, addSpecialTokens: $addSpecialTokens)
382-
: new PostProcessedOutput(tokens: array_merge($tokens, $tokens2));
382+
: new PostProcessedOutput(tokens: array_merge($tokens ?? [], $tokens2 ?? []));
383383

384384

385385
$inputIds = $this->tokenizer->convertTokensToIds($combinedTokens->tokens);

0 commit comments

Comments
 (0)