Merge pull request #105 from goccy/feature/fix-escaping-quote

Fix handling of escaped character in quoted string
goccy · May 29, 2020 · ca90bf9 · ca90bf9
2 parents 545153e + c6ffce4
commit ca90bf9
Show file tree

Hide file tree

Showing 2 changed files with 138 additions and 19 deletions.
diff --git a/decode_test.go b/decode_test.go
@@ -315,11 +315,81 @@ func TestDecoder(t *testing.T) {
 			map[string]time.Time{"v": time.Date(2015, 2, 24, 18, 19, 39, 0, time.UTC)},
 		},
 
-		// Quoted values.
+		// Single Quoted values.
 		{
-			"'1': '\"2\"'",
-			map[interface{}]interface{}{"1": "\"2\""},
+			`'1': '2'`,
+			map[interface{}]interface{}{"1": `2`},
 		},
+		{
+			`'1': '"2"'`,
+			map[interface{}]interface{}{"1": `"2"`},
+		},
+		{
+			`'1': ''''`,
+			map[interface{}]interface{}{"1": `'`},
+		},
+		{
+			`'1': '''2'''`,
+			map[interface{}]interface{}{"1": `'2'`},
+		},
+		{
+			`'1': 'B''z'`,
+			map[interface{}]interface{}{"1": `B'z`},
+		},
+		{
+			`'1': '\'`,
+			map[interface{}]interface{}{"1": `\`},
+		},
+		{
+			`'1': '\\'`,
+			map[interface{}]interface{}{"1": `\\`},
+		},
+		{
+			`'1': '\"2\"'`,
+			map[interface{}]interface{}{"1": `\"2\"`},
+		},
+		{
+			`'1': '\\"2\\"'`,
+			map[interface{}]interface{}{"1": `\\"2\\"`},
+		},
+
+		// Double Quoted values.
+		{
+			`"1": "2"`,
+			map[interface{}]interface{}{"1": `2`},
+		},
+		{
+			`"1": "\"2\""`,
+			map[interface{}]interface{}{"1": `"2"`},
+		},
+		{
+			`"1": "\""`,
+			map[interface{}]interface{}{"1": `"`},
+		},
+		{
+			`"1": "X\"z"`,
+			map[interface{}]interface{}{"1": `X"z`},
+		},
+		{
+			`"1": "\\"`,
+			map[interface{}]interface{}{"1": `\`},
+		},
+		{
+			`"1": "\\\\"`,
+			map[interface{}]interface{}{"1": `\\`},
+		},
+		{
+			`"1": "\\\"2\\\""`,
+			map[interface{}]interface{}{"1": `\"2\"`},
+		},
+
+		/*
+			// TODO: Escape string
+			{
+				`"1": "a\x2Fb\u002Fc\U0000002Fd"`,
+				map[interface{}]interface{}{"1": `a/b/c/d`},
+			},
+		*/
 
 		{
 			"a: -b_c",

diff --git a/scanner/scanner.go b/scanner/scanner.go
@@ -193,32 +193,81 @@ func (s *Scanner) breakLiteral(ctx *Context) {
 	ctx.breakLiteral()
 }
 
-func (s *Scanner) scanQuote(ctx *Context, ch rune) (tk *token.Token, pos int) {
-	ctx.addOriginBuf(ch)
+func (s *Scanner) scanSingleQuote(ctx *Context) (tk *token.Token, pos int) {
+	ctx.addOriginBuf('\'')
 	startIndex := ctx.idx + 1
 	ctx.progress(1)
-	for idx, c := range ctx.src[startIndex:] {
+	src := ctx.src
+	size := len(src)
+	value := []rune{}
+	for idx := startIndex; idx < size; idx++ {
+		c := src[idx]
 		pos = idx + 1
 		ctx.addOriginBuf(c)
-		switch c {
-		case ch:
-			if ctx.previousChar() == '\\' {
-				continue
-			}
-			value := ctx.source(startIndex, startIndex+idx)
-			switch ch {
-			case '\'':
-				tk = token.SingleQuote(value, string(ctx.obuf), s.pos())
-			case '"':
-				tk = token.DoubleQuote(value, string(ctx.obuf), s.pos())
+		if c != '\'' {
+			value = append(value, c)
+			continue
+		}
+		if idx+1 < len(ctx.src) && ctx.src[idx+1] == '\'' {
+			// '' handle as ' character
+			value = append(value, c)
+			ctx.addOriginBuf(c)
+			idx++
+			continue
+		}
+		tk = token.SingleQuote(string(value), string(ctx.obuf), s.pos())
+		pos = len([]rune(value)) + 1
+		return
+	}
+	return
+}
+
+func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) {
+	ctx.addOriginBuf('"')
+	startIndex := ctx.idx + 1
+	ctx.progress(1)
+	src := ctx.src
+	size := len(src)
+	value := []rune{}
+	for idx := startIndex; idx < size; idx++ {
+		c := src[idx]
+		pos = idx + 1
+		ctx.addOriginBuf(c)
+		if c == '\\' {
+			if idx+1 < size {
+				nextChar := src[idx+1]
+				switch nextChar {
+				case '"':
+					ctx.addOriginBuf(nextChar)
+					value = append(value, nextChar)
+					idx++
+					continue
+				case '\\':
+					ctx.addOriginBuf(nextChar)
+					idx++
+				}
 			}
-			pos = len([]rune(value)) + 1
-			return
+			value = append(value, c)
+			continue
+		} else if c != '"' {
+			value = append(value, c)
+			continue
 		}
+
+		tk = token.DoubleQuote(string(value), string(ctx.obuf), s.pos())
+		pos = len([]rune(value)) + 1
+		return
 	}
 	return
 }
 
+func (s *Scanner) scanQuote(ctx *Context, ch rune) (tk *token.Token, pos int) {
+	if ch == '\'' {
+		return s.scanSingleQuote(ctx)
+	}
+	return s.scanDoubleQuote(ctx)
+}
+
 func (s *Scanner) scanTag(ctx *Context) (tk *token.Token, pos int) {
 	ctx.addOriginBuf('!')
 	ctx.progress(1) // skip '!' character