Merge pull request #26 from goccy/feature/fix-multi-bytes

Support multi bytes encoding/decoding

decode_test.go

@@ -839,6 +839,12 @@ func TestDecoder(t *testing.T) {
 				},
 			},
 		},
+
+		// Multi bytes
+		{
+			"v: あいうえお",
+			map[string]string{"v": "あいうえお"},
+		},
 	}
 	for _, test := range tests {
 		buf := bytes.NewBufferString(test.source)

encode_test.go

@@ -285,6 +285,12 @@ func TestEncoder(t *testing.T) {
 				} "a,flow"
 			}{struct{ B, D string }{"c", "e"}},
 		},
+
+		// Multi bytes
+		{
+			"v: あいうえお\n",
+			map[string]string{"v": "あいうえお"},
+		},
 	}
 	for _, test := range tests {
 		var buf bytes.Buffer

scanner/context.go

@@ -10,7 +10,7 @@ import (
 type Context struct {
 	idx         int
 	size        int
-	src         string
+	src         []rune
 	buf         []rune
 	obuf        []rune
 	tokens      token.Tokens
@@ -20,7 +20,8 @@ type Context struct {
 	literalOpt  string
 }
 
-func newContext(src string) *Context {
+func newContext(s string) *Context {
+	src := []rune(s)
 	return &Context{
 		idx:    0,
 		size:   len(src),
@@ -75,31 +76,31 @@ func (c *Context) next() bool {
 }
 
 func (c *Context) source(s, e int) string {
-	return c.src[s:e]
+	return string(c.src[s:e])
 }
 
 func (c *Context) previousChar() rune {
 	if c.idx > 0 {
-		return rune(c.src[c.idx-1])
+		return c.src[c.idx-1]
 	}
 	return rune(0)
 }
 
 func (c *Context) currentChar() rune {
-	return rune(c.src[c.idx])
+	return c.src[c.idx]
 }
 
 func (c *Context) nextChar() rune {
 	if c.size > c.idx+1 {
-		return rune(c.src[c.idx+1])
+		return c.src[c.idx+1]
 	}
 	return rune(0)
 }
 
 func (c *Context) repeatNum(r rune) int {
 	cnt := 0
 	for i := c.idx; i < c.size; i++ {
-		if rune(c.src[i]) == r {
+		if c.src[i] == r {
 			cnt++
 		} else {
 			break