fix escape character in double-quote (#562)

goccy · Dec 1, 2024 · bf03d4d · bf03d4d
1 parent f4ccce9
commit bf03d4d
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 32 deletions.
diff --git a/decode_test.go b/decode_test.go
@@ -1107,6 +1107,16 @@ c:
 			"v: あいうえお\nv2: かきくけこ",
 			map[string]string{"v": "あいうえお", "v2": "かきくけこ"},
 		},
+		{
+			`
+- "Fun with \\"
+- "\" \a \b \e \f"
+- "\n \r \t \v \0"
+- "\  \_ \N \L \P \
+  \x41 \u0041 \U00000041"
+`,
+			[]string{"Fun with \\", "\" \u0007 \b \u001b \f", "\n \r \t \u000b \u0000", "\u0020 \u00a0 \u0085 \u2028 \u2029 A A A"},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.source, func(t *testing.T) {

diff --git a/scanner/scanner.go b/scanner/scanner.go
@@ -360,54 +360,74 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (*token.Token, error) {
 			nextChar := src[idx+1]
 			progress := 0
 			switch nextChar {
-			case 'b':
+			case '0':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, '\b')
-			case 'e':
+				value = append(value, 0x00)
+			case 'a':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, '\x1B')
-			case 'f':
+				value = append(value, 0x07)
+			case 'b':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, '\f')
+				value = append(value, 0x08)
+			case 't':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, 0x09)
 			case 'n':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, '\n')
+				value = append(value, 0x0A)
+			case 'v':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, 0x0B)
+			case 'f':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, 0x0C)
 			case 'r':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, '\r')
-			case 't':
+				value = append(value, 0x0D)
+			case 'e':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, '\t')
-			case 'v':
+				value = append(value, 0x1B)
+			case ' ':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, '\v')
-			case 'L': // LS (#x2028)
+				value = append(value, 0x20)
+			case '"':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, []rune{'\xE2', '\x80', '\xA8'}...)
-			case 'N': // NEL (#x85)
+				value = append(value, 0x22)
+			case '/':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, []rune{'\xC2', '\x85'}...)
-			case 'P': // PS (#x2029)
+				value = append(value, 0x2F)
+			case '\\':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, []rune{'\xE2', '\x80', '\xA9'}...)
-			case '_': // #xA0
+				value = append(value, 0x5C)
+			case 'N':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, []rune{'\xC2', '\xA0'}...)
-			case '"':
+				value = append(value, 0x85)
+			case '_':
 				progress = 1
 				ctx.addOriginBuf(nextChar)
-				value = append(value, nextChar)
+				value = append(value, 0xA0)
+			case 'L':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, 0x2028)
+			case 'P':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, 0x2029)
 			case 'x':
 				if idx+3 >= size {
 					progress = 1
@@ -438,10 +458,6 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (*token.Token, error) {
 					codeNum := hexRunesToInt(src[idx+2 : idx+progress+1])
 					value = append(value, rune(codeNum))
 				}
-			case '\\':
-				progress = 1
-				ctx.addOriginBuf(nextChar)
-				value = append(value, c)
 			case '\n':
 				isFirstLineChar = true
 				isNewLine = true
@@ -454,12 +470,6 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (*token.Token, error) {
 				progress = 1
 				ctx.addOriginBuf(nextChar)
 				value = append(value, nextChar)
-			case '0':
-				progress = 1
-				ctx.addOriginBuf(nextChar)
-				value = append(value, '\x00')
-			case ' ':
-				// skip escape character.
 			default:
 				s.progressColumn(ctx, 1)
 				return nil, ErrInvalidToken(