diff --git a/decode_test.go b/decode_test.go index b81fbdda..d41ed480 100644 --- a/decode_test.go +++ b/decode_test.go @@ -842,8 +842,8 @@ func TestDecoder(t *testing.T) { // Multi bytes { - "v: あいうえお", - map[string]string{"v": "あいうえお"}, + "v: あいうえお\nv2: かきくけこ", + map[string]string{"v": "あいうえお", "v2": "かきくけこ"}, }, } for _, test := range tests { diff --git a/encode_test.go b/encode_test.go index 12bac5c3..8c9b853d 100644 --- a/encode_test.go +++ b/encode_test.go @@ -288,8 +288,8 @@ func TestEncoder(t *testing.T) { // Multi bytes { - "v: あいうえお\n", - map[string]string{"v": "あいうえお"}, + "v: あいうえお\nv2: かきくけこ\n", + map[string]string{"v": "あいうえお", "v2": "かきくけこ"}, }, } for _, test := range tests { diff --git a/scanner/context.go b/scanner/context.go index 497f2489..15a94e9e 100644 --- a/scanner/context.go +++ b/scanner/context.go @@ -20,8 +20,7 @@ type Context struct { literalOpt string } -func newContext(s string) *Context { - src := []rune(s) +func newContext(src []rune) *Context { return &Context{ idx: 0, size: len(src), diff --git a/scanner/scanner.go b/scanner/scanner.go index 9cb29635..5b3957a3 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -25,7 +25,7 @@ const ( // Scanner holds the scanner's internal state while processing a given text. // It can be allocated as part of another data structure but must be initialized via Init before use. type Scanner struct { - source string + source []rune sourcePos int sourceSize int line int @@ -486,7 +486,8 @@ func (s *Scanner) scan(ctx *Context) (pos int) { } // Init prepares the scanner s to tokenize the text src by setting the scanner at the beginning of src. -func (s *Scanner) Init(src string) { +func (s *Scanner) Init(text string) { + src := []rune(text) s.source = src s.sourcePos = 0 s.sourceSize = len(src)