diff --git a/parser/parser.go b/parser/parser.go index 83c08877..75734cc7 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -74,12 +74,26 @@ func (p *parser) parseTag(ctx *context) (ast.Node, error) { return node, nil } +func (p *parser) removeLeftSideNewLineCharacter(src string) string { + return strings.TrimLeft(strings.TrimLeft(src, "\r"), "\n") +} + +func (p *parser) existsNewLineCharacter(src string) bool { + if strings.Index(src, "\n") > 0 { + return true + } + if strings.Index(src, "\r") > 0 { + return true + } + return false +} + func (p *parser) validateMapKey(tk *token.Token) error { if tk.Type != token.StringType { return nil } - origin := strings.TrimLeft(tk.Origin, "\n") - if strings.Index(origin, "\n") > 0 { + origin := p.removeLeftSideNewLineCharacter(tk.Origin) + if p.existsNewLineCharacter(origin) { return errors.ErrSyntax("unexpected key name", tk) } return nil diff --git a/parser/parser_test.go b/parser/parser_test.go index 272c5601..73a89ffa 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -2,6 +2,7 @@ package parser_test import ( "fmt" + "path/filepath" "strings" "testing" @@ -490,6 +491,26 @@ c: d } } +func TestNewLineChar(t *testing.T) { + for _, f := range []string{ + "lf.yml", + "cr.yml", + "crlf.yml", + } { + ast, err := parser.ParseFile(filepath.Join("testdata", f), 0) + if err != nil { + t.Fatalf("%+v", err) + } + actual := fmt.Sprintf("%v\n", ast) + expect := `a: "a" +b: 1 +` + if expect != actual { + t.Fatal("unexpected result") + } + } +} + func TestSyntaxError(t *testing.T) { sources := []string{ "a:\n- b\n c: d\n e: f\n g: h", diff --git a/parser/testdata/cr.yml b/parser/testdata/cr.yml new file mode 100644 index 00000000..8e558db1 --- /dev/null +++ b/parser/testdata/cr.yml @@ -0,0 +1 @@ +a: "a" b: 1 \ No newline at end of file diff --git a/parser/testdata/crlf.yml b/parser/testdata/crlf.yml new file mode 100644 index 00000000..0f25504e --- /dev/null +++ b/parser/testdata/crlf.yml @@ -0,0 +1,2 @@ +a: "a" +b: 1 diff --git a/parser/testdata/lf.yml b/parser/testdata/lf.yml new file mode 100644 index 00000000..416d8ba0 --- /dev/null +++ b/parser/testdata/lf.yml @@ -0,0 +1,2 @@ +a: "a" +b: 1 diff --git a/scanner/scanner.go b/scanner/scanner.go index d8bb770a..33d47e62 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -101,8 +101,18 @@ func (s *Scanner) isNeededKeepPreviousIndentNum(ctx *Context, c rune) bool { return false } +func (s *Scanner) isNewLineChar(c rune) bool { + if c == '\n' { + return true + } + if c == '\r' { + return true + } + return false +} + func (s *Scanner) updateIndent(ctx *Context, c rune) { - if s.isFirstCharAtLine && c == '\n' && ctx.isDocument() { + if s.isFirstCharAtLine && s.isNewLineChar(c) && ctx.isDocument() { return } if s.isFirstCharAtLine && c == ' ' { @@ -199,7 +209,7 @@ func (s *Scanner) scanTag(ctx *Context) (tk *token.Token, pos int) { pos = idx + 1 ctx.addOriginBuf(c) switch c { - case ' ', '\n': + case ' ', '\n', '\r': value := ctx.source(ctx.idx-1, ctx.idx+idx) tk = token.Tag(value, string(ctx.obuf), s.pos()) pos = len([]rune(value)) @@ -216,7 +226,7 @@ func (s *Scanner) scanComment(ctx *Context) (tk *token.Token, pos int) { pos = idx + 1 ctx.addOriginBuf(c) switch c { - case '\n': + case '\n', '\r': if ctx.previousChar() == '\\' { continue } @@ -237,7 +247,7 @@ func (s *Scanner) scanLiteral(ctx *Context, c rune) { ctx.addToken(token.New(value, string(ctx.obuf), s.pos())) ctx.resetBuffer() s.progressColumn(ctx, 1) - } else if c == '\n' { + } else if s.isNewLineChar(c) { if ctx.isLiteral { ctx.addBuf(c) } else { @@ -266,7 +276,7 @@ func (s *Scanner) scanLiteralHeader(ctx *Context) (pos int, err error) { pos = idx ctx.addOriginBuf(c) switch c { - case '\n': + case '\n', '\r': value := ctx.source(ctx.idx, ctx.idx+idx) opt := strings.TrimRight(value, " ") switch opt { @@ -337,8 +347,8 @@ func (s *Scanner) scan(ctx *Context) (pos int) { } else if s.isChangedToIndentStateDown() { s.addBufferedTokenIfExists(ctx) } else if s.isChangedToIndentStateEqual() { - // if first character is \n, buffer expect to raw folded literal - if len(ctx.obuf) > 0 && ctx.obuf[0] != '\n' { + // if first character is new line character, buffer expect to raw folded literal + if len(ctx.obuf) > 0 && !s.isNewLineChar(ctx.obuf[0]) { // doesn't raw folded literal s.addBufferedTokenIfExists(ctx) } @@ -436,7 +446,7 @@ func (s *Scanner) scan(ctx *Context) (pos int) { } case ':': nc := ctx.nextChar() - if nc == ' ' || nc == '\n' || ctx.isNextEOS() { + if nc == ' ' || s.isNewLineChar(nc) || ctx.isNextEOS() { // mapping value tk := s.bufferedToken(ctx) if tk != nil { @@ -463,7 +473,7 @@ func (s *Scanner) scan(ctx *Context) (pos int) { token, progress := s.scanTag(ctx) ctx.addToken(token) s.progressColumn(ctx, progress) - if c := ctx.previousChar(); c == '\n' { + if c := ctx.previousChar(); s.isNewLineChar(c) { s.progressLine(ctx) } pos += progress