fix: Parse indented newlines correctly

Fixes the case where an indented newline causes invalid parse results. Consider the following input where the second line contains an indented new line. ```yaml a:·b␊ ···␊ c:·d␊ ``` Before this fix, the parser produces the following: ```yaml a: null b c: d ``` With this fix, the output is as expected. ```yaml a: b c: d ``` Signed-off-by: Charith Ellawala <[email protected]>
goccy · Apr 29, 2024 · 3541536 · 3541536
1 parent 4653a1b
commit 3541536
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 14 deletions.
diff --git a/parser/parser_test.go b/parser/parser_test.go
@@ -180,7 +180,8 @@ a: 0 - 1
 a: 0 - 1
 `,
 		},
-		{`
+		{
+			`
 - a:
    b: c
    d: e
@@ -456,7 +457,7 @@ d: eeeeeeeeeeeeeeeee
 		},
 		{
 			`
-a: b    
+a: b
   c
 `,
 			`
@@ -465,7 +466,7 @@ a: b c
 		},
 		{
 			`
-a:    
+a:
   b: c
 `,
 			`
@@ -475,7 +476,7 @@ a:
 		},
 		{
 			`
-a: b    
+a: b
 c: d
 `,
 			`
@@ -625,6 +626,20 @@ b: 1
 	}
 }
 
+func TestIndentedNewLine(t *testing.T) {
+	ast, err := parser.ParseFile(filepath.Join("testdata", "indented_new_line.yml"), 0)
+	if err != nil {
+		t.Fatalf("%+v", err)
+	}
+	actual := fmt.Sprintf("%v", ast)
+	expect := `a: b
+c: d
+`
+	if expect != actual {
+		t.Fatalf("Expected:\n%s\n\nActual:\n%s\n", expect, actual)
+	}
+}
+
 func TestSyntaxError(t *testing.T) {
 	tests := []struct {
 		source string
@@ -999,8 +1014,7 @@ func (c *pathCapturer) Visit(node ast.Node) ast.Visitor {
 	return c
 }
 
-type Visitor struct {
-}
+type Visitor struct{}
 
 func (v *Visitor) Visit(node ast.Node) ast.Visitor {
 	tk := node.GetToken()

diff --git a/parser/testdata/indented_new_line.yml b/parser/testdata/indented_new_line.yml
@@ -0,0 +1,4 @@
+# The line following a:b has 3 spaces
+a: b
+
+c: d
diff --git a/scanner/context.go b/scanner/context.go
@@ -25,13 +25,11 @@ type Context struct {
 	literalOpt         string
 }
 
-var (
-	ctxPool = sync.Pool{
-		New: func() interface{} {
-			return createContext()
-		},
-	}
-)
+var ctxPool = sync.Pool{
+	New: func() interface{} {
+		return createContext()
+	},
+}
 
 func createContext() *Context {
 	return &Context{
@@ -101,7 +99,7 @@ func (c *Context) addBuf(r rune) {
 
 func (c *Context) addOriginBuf(r rune) {
 	c.obuf = append(c.obuf, r)
-	if r != ' ' && r != '\t' {
+	if r != ' ' && r != '\t' && r != '\n' {
 		c.notSpaceOrgCharPos = len(c.obuf)
 	}
 }
@@ -110,6 +108,15 @@ func (c *Context) removeRightSpaceFromBuf() int {
 	trimmedBuf := c.obuf[:c.notSpaceOrgCharPos]
 	buflen := len(trimmedBuf)
 	diff := len(c.obuf) - buflen
+
+	// only calculate the space chopped up to the first newline
+	for i := c.notSpaceOrgCharPos; i < len(c.obuf); i++ {
+		if c.obuf[i] == '\n' || c.obuf[i] == '\r' {
+			diff = i - c.notSpaceOrgCharPos
+			break
+		}
+	}
+
 	if diff > 0 {
 		c.obuf = c.obuf[:buflen]
 		c.buf = c.bufferedSrc()