diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 0d8f6481..a1775199 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -586,6 +586,60 @@ b: 1`, }, }, }, + { + name: "literal string without trailing whitespace", + src: `a: | + Text`, + expect: []testToken{ + { + line: 1, + column: 1, + value: "a", + }, + { + line: 1, + column: 2, + value: ":", + }, + { + line: 1, + column: 4, + value: "|", + }, + { + line: 2, + column: 3, + value: "Text", + }, + }, + }, + { + name: "folded string without trailing whitespace", + src: `a: > + Text`, + expect: []testToken{ + { + line: 1, + column: 1, + value: "a", + }, + { + line: 1, + column: 2, + value: ":", + }, + { + line: 1, + column: 4, + value: ">", + }, + { + line: 2, + column: 3, + value: "Text", + }, + }, + }, } for _, tc := range tests { diff --git a/scanner/scanner.go b/scanner/scanner.go index b0eac48d..2b5e8b25 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -377,7 +377,7 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) { case 'x': if idx+3 >= size { // TODO: need to return error - //err = xerrors.New("invalid escape character \\x") + // err = xerrors.New("invalid escape character \\x") return } codeNum := hexRunesToInt(src[idx+2 : idx+4]) @@ -387,7 +387,7 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) { case 'u': if idx+5 >= size { // TODO: need to return error - //err = xerrors.New("invalid escape character \\u") + // err = xerrors.New("invalid escape character \\u") return } codeNum := hexRunesToInt(src[idx+2 : idx+6]) @@ -397,7 +397,7 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) { case 'U': if idx+9 >= size { // TODO: need to return error - //err = xerrors.New("invalid escape character \\U") + // err = xerrors.New("invalid escape character \\U") return } codeNum := hexRunesToInt(src[idx+2 : idx+10]) @@ -512,9 +512,13 @@ func (s *Scanner) scanLiteral(ctx *Context, c rune) { if ctx.isEOS() { if ctx.isLiteral { ctx.addBuf(c) + } else if ctx.isFolded && !s.isNewLineChar(c) { + ctx.addBuf(c) } value := ctx.bufferedSrc() - ctx.addToken(token.String(string(value), string(ctx.obuf), s.pos())) + pos := s.pos() + pos.Column = s.docStartColumn + ctx.addToken(token.String(string(value), string(ctx.obuf), pos)) ctx.resetBuffer() s.progressColumn(ctx, 1) } else if s.isNewLineChar(c) {