From 62a86c4e5de3f4a5c0a93f2e073cf873f4267c39 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Thu, 7 Nov 2019 23:45:39 +0900 Subject: [PATCH 1/4] Fix parsing of literal token --- ast/ast.go | 3 ++- parser/parser_test.go | 18 ++++++++++++++++++ scanner/context.go | 4 ++++ scanner/scanner.go | 28 ++++++++++++++++++++-------- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 8831b3b6..cf0e5db3 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -424,7 +424,8 @@ func (n *LiteralNode) GetValue() interface{} { // String literal to text func (n *LiteralNode) String() string { - return n.Value.String() + origin := n.Value.GetToken().Origin + return fmt.Sprintf("|\n%s", strings.TrimRight(strings.TrimRight(origin, " "), "\n")) } // MergeKeyNode type of merge key node diff --git a/parser/parser_test.go b/parser/parser_test.go index f7026f4f..49c09ac3 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -375,6 +375,24 @@ b: 2 c: 3 d: 4 ... +`, + }, + { + ` +a: + b: | + { + [ 1, 2 ] + } + c: d +`, + ` +a: + b: | + { + [ 1, 2 ] + } + c: d `, }, } diff --git a/scanner/context.go b/scanner/context.go index 15a94e9e..2292f9b5 100644 --- a/scanner/context.go +++ b/scanner/context.go @@ -62,6 +62,10 @@ func (c *Context) addOriginBuf(r rune) { c.obuf = append(c.obuf, r) } +func (c *Context) isDocument() bool { + return c.isLiteral || c.isFolded || c.isRawFolded +} + func (c *Context) isEOS() bool { return len(c.src)-1 <= c.idx } diff --git a/scanner/scanner.go b/scanner/scanner.go index cfab6f60..a3d83895 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -87,7 +87,7 @@ func (s *Scanner) progressLine(ctx *Context) { ctx.progress(1) } -func (s *Scanner) updateIndent(c rune) { +func (s *Scanner) updateIndent(ctx *Context, c rune) { if s.isFirstCharAtLine && c == ' ' { s.indentNum++ return @@ -119,10 +119,15 @@ func (s *Scanner) updateIndent(c rune) { s.indentState = IndentStateDown } } + s.isFirstCharAtLine = false + if ctx.isDocument() && s.isChangedToIndentStateUp() { + return + } else if c == '-' && ctx.bufferedSrc() != "" && s.isChangedToIndentStateUp() { + return + } s.prevIndentNum = s.indentNum s.prevIndentColumn = 0 s.prevIndentLevel = s.indentLevel - s.isFirstCharAtLine = false } func (s *Scanner) isChangedToIndentStateDown() bool { @@ -212,6 +217,7 @@ func (s *Scanner) scanLiteral(ctx *Context, c rune) { if ctx.isEOS() { value := ctx.bufferedSrc() ctx.addToken(token.New(value, string(ctx.obuf), s.pos())) + ctx.resetBuffer() } if c == '\n' { if ctx.isLiteral { @@ -280,13 +286,18 @@ func (s *Scanner) scan(ctx *Context) (pos int) { for ctx.next() { pos = ctx.nextPos() c := ctx.currentChar() - s.updateIndent(c) - if s.isChangedToIndentStateDown() { + s.updateIndent(ctx, c) + if ctx.isDocument() { + if s.isChangedToIndentStateEqual() || + s.isChangedToIndentStateDown() { + s.addBufferedTokenIfExists(ctx) + s.breakLiteral(ctx) + } else { + s.scanLiteral(ctx, c) + continue + } + } else if s.isChangedToIndentStateDown() { s.addBufferedTokenIfExists(ctx) - s.breakLiteral(ctx) - } else if ctx.isLiteral || ctx.isFolded || ctx.isRawFolded { - s.scanLiteral(ctx, c) - continue } else if s.isChangedToIndentStateEqual() { // if first character is \n, buffer expect to raw folded literal if len(ctx.obuf) > 0 && ctx.obuf[0] != '\n' { @@ -350,6 +361,7 @@ func (s *Scanner) scan(ctx *Context) (pos int) { tk := token.SequenceEntry(string(ctx.obuf), s.pos()) s.prevIndentColumn = tk.Position.Column ctx.addToken(tk) + ctx.resetBuffer() s.progressColumn(ctx, 1) return } From 0d1afe30542d481181bf7655a944eff729efbc03 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Thu, 7 Nov 2019 23:49:28 +0900 Subject: [PATCH 2/4] Remove not affected code --- scanner/scanner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index a3d83895..2bd6b27f 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -361,7 +361,6 @@ func (s *Scanner) scan(ctx *Context) (pos int) { tk := token.SequenceEntry(string(ctx.obuf), s.pos()) s.prevIndentColumn = tk.Position.Column ctx.addToken(tk) - ctx.resetBuffer() s.progressColumn(ctx, 1) return } From c61d2c3981c7ae940835965960e9615a8a2fcba2 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Thu, 7 Nov 2019 23:54:32 +0900 Subject: [PATCH 3/4] Refactor condition --- scanner/scanner.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 2bd6b27f..e11418e3 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -87,6 +87,19 @@ func (s *Scanner) progressLine(ctx *Context) { ctx.progress(1) } +func (s *Scanner) isNeededKeepPreviousIndentNum(ctx *Context, c rune) bool { + if !s.isChangedToIndentStateUp() { + return false + } + if ctx.isDocument() { + return true + } + if c == '-' && ctx.bufferedSrc() != "" { + return true + } + return false +} + func (s *Scanner) updateIndent(ctx *Context, c rune) { if s.isFirstCharAtLine && c == ' ' { s.indentNum++ @@ -120,9 +133,7 @@ func (s *Scanner) updateIndent(ctx *Context, c rune) { } } s.isFirstCharAtLine = false - if ctx.isDocument() && s.isChangedToIndentStateUp() { - return - } else if c == '-' && ctx.bufferedSrc() != "" && s.isChangedToIndentStateUp() { + if s.isNeededKeepPreviousIndentNum(ctx, c) { return } s.prevIndentNum = s.indentNum From 208e32895124290d06d0d47b877ab904b4fa79ac Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Fri, 8 Nov 2019 01:04:29 +0900 Subject: [PATCH 4/4] Include indent to literal token --- scanner/scanner.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scanner/scanner.go b/scanner/scanner.go index e11418e3..da73fced 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -34,6 +34,7 @@ type Scanner struct { prevIndentLevel int prevIndentNum int prevIndentColumn int + docStartColumn int indentLevel int indentNum int isFirstCharAtLine bool @@ -158,6 +159,7 @@ func (s *Scanner) addBufferedTokenIfExists(ctx *Context) { } func (s *Scanner) breakLiteral(ctx *Context) { + s.docStartColumn = 0 ctx.breakLiteral() } @@ -238,8 +240,14 @@ func (s *Scanner) scanLiteral(ctx *Context, c rune) { } s.progressLine(ctx) } else if s.isFirstCharAtLine && c == ' ' { + if 0 < s.docStartColumn && s.docStartColumn <= s.column { + ctx.addBuf(c) + } s.progressColumn(ctx, 1) } else { + if s.docStartColumn == 0 { + s.docStartColumn = s.column + } ctx.addBuf(c) s.progressColumn(ctx, 1) }