fix: Avoid truncating last char from folded strings

As described in #433, if the last character of a folded string is the very last character in the file (no trailing whitespace), it's not included the token's `Value`. While fixing this, I also noticed that the column information for the folded and literal strings were the end rather than the beginning. I've (hopefully) fixed that as well. Fixes #433 Signed-off-by: Charith Ellawala <[email protected]>
goccy · Mar 13, 2024 · 1a67fbf · 1a67fbf
1 parent 4653a1b
commit 1a67fbf
Show file tree

Hide file tree

Showing 2 changed files with 62 additions and 4 deletions.
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
@@ -586,6 +586,60 @@ b: 1`,
 				},
 			},
 		},
+		{
+			name: "literal string without trailing whitespace",
+			src: `a: |
+  Text`,
+			expect: []testToken{
+				{
+					line:   1,
+					column: 1,
+					value:  "a",
+				},
+				{
+					line:   1,
+					column: 2,
+					value:  ":",
+				},
+				{
+					line:   1,
+					column: 4,
+					value:  "|",
+				},
+				{
+					line:   2,
+					column: 3,
+					value:  "Text",
+				},
+			},
+		},
+		{
+			name: "folded string without trailing whitespace",
+			src: `a: >
+  Text`,
+			expect: []testToken{
+				{
+					line:   1,
+					column: 1,
+					value:  "a",
+				},
+				{
+					line:   1,
+					column: 2,
+					value:  ":",
+				},
+				{
+					line:   1,
+					column: 4,
+					value:  ">",
+				},
+				{
+					line:   2,
+					column: 3,
+					value:  "Text",
+				},
+			},
+		},
 	}
 
 	for _, tc := range tests {

diff --git a/scanner/scanner.go b/scanner/scanner.go
@@ -377,7 +377,7 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) {
 				case 'x':
 					if idx+3 >= size {
 						// TODO: need to return error
-						//err = xerrors.New("invalid escape character \\x")
+						// err = xerrors.New("invalid escape character \\x")
 						return
 					}
 					codeNum := hexRunesToInt(src[idx+2 : idx+4])
@@ -387,7 +387,7 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) {
 				case 'u':
 					if idx+5 >= size {
 						// TODO: need to return error
-						//err = xerrors.New("invalid escape character \\u")
+						// err = xerrors.New("invalid escape character \\u")
 						return
 					}
 					codeNum := hexRunesToInt(src[idx+2 : idx+6])
@@ -397,7 +397,7 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) {
 				case 'U':
 					if idx+9 >= size {
 						// TODO: need to return error
-						//err = xerrors.New("invalid escape character \\U")
+						// err = xerrors.New("invalid escape character \\U")
 						return
 					}
 					codeNum := hexRunesToInt(src[idx+2 : idx+10])
@@ -512,9 +512,13 @@ func (s *Scanner) scanLiteral(ctx *Context, c rune) {
 	if ctx.isEOS() {
 		if ctx.isLiteral {
 			ctx.addBuf(c)
+		} else if ctx.isFolded && !s.isNewLineChar(c) {
+			ctx.addBuf(c)
 		}
 		value := ctx.bufferedSrc()
-		ctx.addToken(token.String(string(value), string(ctx.obuf), s.pos()))
+		pos := s.pos()
+		pos.Column = s.docStartColumn
+		ctx.addToken(token.String(string(value), string(ctx.obuf), pos))
 		ctx.resetBuffer()
 		s.progressColumn(ctx, 1)
 	} else if s.isNewLineChar(c) {