Merge pull request #27 from goccy/feature/fix-multi-bytes-handling

Fix multi bytes handling

decode_test.go

@@ -842,8 +842,8 @@ func TestDecoder(t *testing.T) {
 
 		// Multi bytes
 		{
-			"v: あいうえお",
-			map[string]string{"v": "あいうえお"},
+			"v: あいうえお\nv2: かきくけこ",
+			map[string]string{"v": "あいうえお", "v2": "かきくけこ"},
 		},
 	}
 	for _, test := range tests {

encode_test.go

@@ -288,8 +288,8 @@ func TestEncoder(t *testing.T) {
 
 		// Multi bytes
 		{
-			"v: あいうえお\n",
-			map[string]string{"v": "あいうえお"},
+			"v: あいうえお\nv2: かきくけこ\n",
+			map[string]string{"v": "あいうえお", "v2": "かきくけこ"},
 		},
 	}
 	for _, test := range tests {

scanner/context.go

@@ -20,8 +20,7 @@ type Context struct {
 	literalOpt  string
 }
 
-func newContext(s string) *Context {
-	src := []rune(s)
+func newContext(src []rune) *Context {
 	return &Context{
 		idx:    0,
 		size:   len(src),

scanner/scanner.go

@@ -25,7 +25,7 @@ const (
 // Scanner holds the scanner's internal state while processing a given text.
 // It can be allocated as part of another data structure but must be initialized via Init before use.
 type Scanner struct {
-	source                 string
+	source                 []rune
 	sourcePos              int
 	sourceSize             int
 	line                   int
@@ -486,7 +486,8 @@ func (s *Scanner) scan(ctx *Context) (pos int) {
 }
 
 // Init prepares the scanner s to tokenize the text src by setting the scanner at the beginning of src.
-func (s *Scanner) Init(src string) {
+func (s *Scanner) Init(text string) {
+	src := []rune(text)
 	s.source = src
 	s.sourcePos = 0
 	s.sourceSize = len(src)