diff --git a/decode_test.go b/decode_test.go index 27dacc5d..7dea2bc9 100644 --- a/decode_test.go +++ b/decode_test.go @@ -315,11 +315,81 @@ func TestDecoder(t *testing.T) { map[string]time.Time{"v": time.Date(2015, 2, 24, 18, 19, 39, 0, time.UTC)}, }, - // Quoted values. + // Single Quoted values. { - "'1': '\"2\"'", - map[interface{}]interface{}{"1": "\"2\""}, + `'1': '2'`, + map[interface{}]interface{}{"1": `2`}, }, + { + `'1': '"2"'`, + map[interface{}]interface{}{"1": `"2"`}, + }, + { + `'1': ''''`, + map[interface{}]interface{}{"1": `'`}, + }, + { + `'1': '''2'''`, + map[interface{}]interface{}{"1": `'2'`}, + }, + { + `'1': 'B''z'`, + map[interface{}]interface{}{"1": `B'z`}, + }, + { + `'1': '\'`, + map[interface{}]interface{}{"1": `\`}, + }, + { + `'1': '\\'`, + map[interface{}]interface{}{"1": `\\`}, + }, + { + `'1': '\"2\"'`, + map[interface{}]interface{}{"1": `\"2\"`}, + }, + { + `'1': '\\"2\\"'`, + map[interface{}]interface{}{"1": `\\"2\\"`}, + }, + + // Double Quoted values. + { + `"1": "2"`, + map[interface{}]interface{}{"1": `2`}, + }, + { + `"1": "\"2\""`, + map[interface{}]interface{}{"1": `"2"`}, + }, + { + `"1": "\""`, + map[interface{}]interface{}{"1": `"`}, + }, + { + `"1": "X\"z"`, + map[interface{}]interface{}{"1": `X"z`}, + }, + { + `"1": "\\"`, + map[interface{}]interface{}{"1": `\`}, + }, + { + `"1": "\\\\"`, + map[interface{}]interface{}{"1": `\\`}, + }, + { + `"1": "\\\"2\\\""`, + map[interface{}]interface{}{"1": `\"2\"`}, + }, + + /* + // TODO: Escape string + { + `"1": "a\x2Fb\u002Fc\U0000002Fd"`, + map[interface{}]interface{}{"1": `a/b/c/d`}, + }, + */ { "a: -b_c", diff --git a/scanner/scanner.go b/scanner/scanner.go index 4bff2418..1089587a 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -193,32 +193,81 @@ func (s *Scanner) breakLiteral(ctx *Context) { ctx.breakLiteral() } -func (s *Scanner) scanQuote(ctx *Context, ch rune) (tk *token.Token, pos int) { - ctx.addOriginBuf(ch) +func (s *Scanner) scanSingleQuote(ctx *Context) (tk *token.Token, pos int) { + ctx.addOriginBuf('\'') startIndex := ctx.idx + 1 ctx.progress(1) - for idx, c := range ctx.src[startIndex:] { + src := ctx.src + size := len(src) + value := []rune{} + for idx := startIndex; idx < size; idx++ { + c := src[idx] pos = idx + 1 ctx.addOriginBuf(c) - switch c { - case ch: - if ctx.previousChar() == '\\' { - continue - } - value := ctx.source(startIndex, startIndex+idx) - switch ch { - case '\'': - tk = token.SingleQuote(value, string(ctx.obuf), s.pos()) - case '"': - tk = token.DoubleQuote(value, string(ctx.obuf), s.pos()) + if c != '\'' { + value = append(value, c) + continue + } + if idx+1 < len(ctx.src) && ctx.src[idx+1] == '\'' { + // '' handle as ' character + value = append(value, c) + ctx.addOriginBuf(c) + idx++ + continue + } + tk = token.SingleQuote(string(value), string(ctx.obuf), s.pos()) + pos = len([]rune(value)) + 1 + return + } + return +} + +func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) { + ctx.addOriginBuf('"') + startIndex := ctx.idx + 1 + ctx.progress(1) + src := ctx.src + size := len(src) + value := []rune{} + for idx := startIndex; idx < size; idx++ { + c := src[idx] + pos = idx + 1 + ctx.addOriginBuf(c) + if c == '\\' { + if idx+1 < size { + nextChar := src[idx+1] + switch nextChar { + case '"': + ctx.addOriginBuf(nextChar) + value = append(value, nextChar) + idx++ + continue + case '\\': + ctx.addOriginBuf(nextChar) + idx++ + } } - pos = len([]rune(value)) + 1 - return + value = append(value, c) + continue + } else if c != '"' { + value = append(value, c) + continue } + + tk = token.DoubleQuote(string(value), string(ctx.obuf), s.pos()) + pos = len([]rune(value)) + 1 + return } return } +func (s *Scanner) scanQuote(ctx *Context, ch rune) (tk *token.Token, pos int) { + if ch == '\'' { + return s.scanSingleQuote(ctx) + } + return s.scanDoubleQuote(ctx) +} + func (s *Scanner) scanTag(ctx *Context) (tk *token.Token, pos int) { ctx.addOriginBuf('!') ctx.progress(1) // skip '!' character