fix token offset (#524)

goccy · Nov 12, 2024 · 8808389 · 8808389
1 parent 3fa49bf
commit 8808389
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 3 deletions.
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
@@ -3186,3 +3186,32 @@ a: |invalid`,
 		})
 	}
 }
+
+func TestTokenOffset(t *testing.T) {
+	t.Run("crlf", func(t *testing.T) {
+		content := "project:\r\n  version: 1.2.3\r\n"
+		tokens := lexer.Tokenize(content)
+		if len(tokens) != 5 {
+			t.Fatalf("invalid token num. got %d", len(tokens))
+		}
+		if tokens[4].Value != "1.2.3" {
+			t.Fatalf("unexpected value. got %q", tokens[4].Value)
+		}
+		if tokens[4].Position.Offset != 22 {
+			t.Fatalf("unexpected offset. got %d", tokens[4].Position.Offset)
+		}
+	})
+	t.Run("lf", func(t *testing.T) {
+		content := "project:\n  version: 1.2.3\n"
+		tokens := lexer.Tokenize(content)
+		if len(tokens) != 5 {
+			t.Fatalf("invalid token num. got %d", len(tokens))
+		}
+		if tokens[4].Value != "1.2.3" {
+			t.Fatalf("unexpected value. got %q", tokens[4].Value)
+		}
+		if tokens[4].Position.Offset != 21 {
+			t.Fatalf("unexpected offset. got %d", tokens[4].Position.Offset)
+		}
+	})
+}
diff --git a/scanner/scanner.go b/scanner/scanner.go
@@ -606,8 +606,10 @@ func (s *Scanner) scanDocument(ctx *Context, c rune) error {
 
 func (s *Scanner) scanNewLine(ctx *Context, c rune) {
 	if len(ctx.buf) > 0 && s.savedPos == nil {
+		bufLen := len(ctx.bufferedSrc())
 		s.savedPos = s.pos()
-		s.savedPos.Column -= len(ctx.bufferedSrc())
+		s.savedPos.Column -= bufLen
+		s.savedPos.Offset -= bufLen
 	}
 
 	// if the following case, origin buffer has unnecessary two spaces.
@@ -631,6 +633,7 @@ func (s *Scanner) scanNewLine(ctx *Context, c rune) {
 	if c == '\r' && ctx.nextChar() == '\n' {
 		ctx.addOriginBuf('\r')
 		s.progress(ctx, 1)
+		s.offset++
 		c = '\n'
 	}
 

diff --git a/token/token.go b/token/token.go
@@ -765,8 +765,8 @@ func (t *Token) Clone() *Token {
 // Dump outputs token information to stdout for debugging.
 func (t *Token) Dump() {
 	fmt.Printf(
-		"[TYPE]:%q [CHARTYPE]:%q [INDICATOR]:%q [VALUE]:%q [ORG]:%q [POS(line:column:level)]: %d:%d:%d\n",
-		t.Type, t.CharacterType, t.Indicator, t.Value, t.Origin, t.Position.Line, t.Position.Column, t.Position.IndentLevel,
+		"[TYPE]:%q [CHARTYPE]:%q [INDICATOR]:%q [VALUE]:%q [ORG]:%q [POS(line:column:level:offset)]: %d:%d:%d:%d\n",
+		t.Type, t.CharacterType, t.Indicator, t.Value, t.Origin, t.Position.Line, t.Position.Column, t.Position.IndentLevel, t.Position.Offset,
 	)
 }