From 5e8343ae64cd163d239abf8104f15f70baf3367d Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Mon, 28 Oct 2024 16:51:35 +0900 Subject: [PATCH 1/3] refactor indent state --- lexer/lexer_test.go | 4 ++-- parser/parser_test.go | 14 ++++++++++++++ scanner/scanner.go | 26 ++++---------------------- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 7660ba1c..3a6d9126 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -22,7 +22,7 @@ func TestTokenize(t *testing.T) { CharacterType: token.CharacterTypeMiscellaneous, Indicator: token.NotIndicator, Value: "null", - Origin: "null\n", + Origin: "null\n\t\t", }, }, }, @@ -828,7 +828,7 @@ func TestTokenize(t *testing.T) { CharacterType: token.CharacterTypeMiscellaneous, Indicator: token.NotIndicator, Value: "123", - Origin: "123\n", + Origin: "123\n\t\t", }, }, }, diff --git a/parser/parser_test.go b/parser/parser_test.go index 3dc94401..2c64ce52 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -439,6 +439,20 @@ a: piyo `, }, + { + ` +v: | + a + b + c`, + ` +v: | + a + b + c +`, + }, + { ` a: | diff --git a/scanner/scanner.go b/scanner/scanner.go index f638a998..281a3560 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -162,21 +162,10 @@ func (s *Scanner) updateIndentState(ctx *Context) { if s.lastDelimColumn > 0 { if s.lastDelimColumn < s.column { s.indentState = IndentStateUp - } else if s.lastDelimColumn != s.column || s.prevLineIndentNum != s.indentNum { - // The following case ( current position is 'd' ), some variables becomes like here - // - lastDelimColumn: 1 of 'a' - // - indentNumBasedIndentState: IndentStateDown because d's indentNum(1) is less than c's indentNum(3). - // Therefore, s.lastDelimColumn(1) == s.column(1) is true, but we want to treat this as IndentStateDown. - // So, we look also current indentState value by the above prevLineIndentNum based logic, and determines finally indentState. - // --- - // a: - // b - // c - // d: e - // ^ - s.indentState = IndentStateDown } else { - s.indentState = IndentStateEqual + // If lastDelimColumn and s.column are the same, + // treat as Down state since it is the same column as delimiter. + s.indentState = IndentStateDown } } else { s.indentState = s.indentStateFromIndentNumDifference() @@ -635,8 +624,7 @@ func (s *Scanner) scan(ctx *Context) (pos int) { c := ctx.currentChar() s.updateIndent(ctx, c) if ctx.isDocument() { - if (s.indentNum == 0 && s.isChangedToIndentStateEqual()) || - s.isChangedToIndentStateDown() { + if s.isChangedToIndentStateDown() { s.addBufferedTokenIfExists(ctx) s.breakLiteral(ctx) } else { @@ -645,12 +633,6 @@ func (s *Scanner) scan(ctx *Context) (pos int) { } } else if s.isChangedToIndentStateDown() { s.addBufferedTokenIfExists(ctx) - } else if s.isChangedToIndentStateEqual() { - // if first character is new line character, buffer expect to raw folded literal - if len(ctx.obuf) > 0 && s.newLineCount(ctx.obuf) <= 1 { - // doesn't raw folded literal - s.addBufferedTokenIfExists(ctx) - } } switch c { case '{': From 670f8ffc694ef6dbf491cfa13f7a5bafe9acacf9 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Mon, 28 Oct 2024 17:00:02 +0900 Subject: [PATCH 2/3] refactor function position --- scanner/scanner.go | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 281a3560..d0e5e946 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -145,20 +145,7 @@ func (s *Scanner) updateIndentLevel() { } } -func (s *Scanner) indentStateFromIndentNumDifference() IndentState { - switch { - case s.prevLineIndentNum < s.indentNum: - return IndentStateUp - case s.prevLineIndentNum == s.indentNum: - return IndentStateEqual - default: - return IndentStateDown - } -} - func (s *Scanner) updateIndentState(ctx *Context) { - s.updateIndentLevel() - if s.lastDelimColumn > 0 { if s.lastDelimColumn < s.column { s.indentState = IndentStateUp @@ -172,6 +159,17 @@ func (s *Scanner) updateIndentState(ctx *Context) { } } +func (s *Scanner) indentStateFromIndentNumDifference() IndentState { + switch { + case s.prevLineIndentNum < s.indentNum: + return IndentStateUp + case s.prevLineIndentNum == s.indentNum: + return IndentStateEqual + default: + return IndentStateDown + } +} + func (s *Scanner) updateIndent(ctx *Context, c rune) { if s.isFirstCharAtLine && s.isNewLineChar(c) && ctx.isDocument() { return @@ -184,6 +182,7 @@ func (s *Scanner) updateIndent(ctx *Context, c rune) { s.indentState = IndentStateKeep return } + s.updateIndentLevel() s.updateIndentState(ctx) s.isFirstCharAtLine = false } @@ -623,16 +622,16 @@ func (s *Scanner) scan(ctx *Context) (pos int) { pos = ctx.nextPos() c := ctx.currentChar() s.updateIndent(ctx, c) + if s.isChangedToIndentStateDown() { + s.addBufferedTokenIfExists(ctx) + } if ctx.isDocument() { if s.isChangedToIndentStateDown() { - s.addBufferedTokenIfExists(ctx) s.breakLiteral(ctx) } else { s.scanLiteral(ctx, c) continue } - } else if s.isChangedToIndentStateDown() { - s.addBufferedTokenIfExists(ctx) } switch c { case '{': From fb45a19f9824baaacb45b0471a7fffd8830d31b4 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Mon, 28 Oct 2024 17:01:34 +0900 Subject: [PATCH 3/3] remove unnecessary function --- scanner/scanner.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index d0e5e946..d55e3bf4 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -195,10 +195,6 @@ func (s *Scanner) isChangedToIndentStateUp() bool { return s.indentState == IndentStateUp } -func (s *Scanner) isChangedToIndentStateEqual() bool { - return s.indentState == IndentStateEqual -} - func (s *Scanner) addBufferedTokenIfExists(ctx *Context) { ctx.addToken(s.bufferedToken(ctx)) }