diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 7660ba1c..3a6d9126 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -22,7 +22,7 @@ func TestTokenize(t *testing.T) { CharacterType: token.CharacterTypeMiscellaneous, Indicator: token.NotIndicator, Value: "null", - Origin: "null\n", + Origin: "null\n\t\t", }, }, }, @@ -828,7 +828,7 @@ func TestTokenize(t *testing.T) { CharacterType: token.CharacterTypeMiscellaneous, Indicator: token.NotIndicator, Value: "123", - Origin: "123\n", + Origin: "123\n\t\t", }, }, }, diff --git a/parser/parser_test.go b/parser/parser_test.go index 3dc94401..2c64ce52 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -439,6 +439,20 @@ a: piyo `, }, + { + ` +v: | + a + b + c`, + ` +v: | + a + b + c +`, + }, + { ` a: | diff --git a/scanner/scanner.go b/scanner/scanner.go index f638a998..d55e3bf4 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -145,44 +145,31 @@ func (s *Scanner) updateIndentLevel() { } } -func (s *Scanner) indentStateFromIndentNumDifference() IndentState { - switch { - case s.prevLineIndentNum < s.indentNum: - return IndentStateUp - case s.prevLineIndentNum == s.indentNum: - return IndentStateEqual - default: - return IndentStateDown - } -} - func (s *Scanner) updateIndentState(ctx *Context) { - s.updateIndentLevel() - if s.lastDelimColumn > 0 { if s.lastDelimColumn < s.column { s.indentState = IndentStateUp - } else if s.lastDelimColumn != s.column || s.prevLineIndentNum != s.indentNum { - // The following case ( current position is 'd' ), some variables becomes like here - // - lastDelimColumn: 1 of 'a' - // - indentNumBasedIndentState: IndentStateDown because d's indentNum(1) is less than c's indentNum(3). - // Therefore, s.lastDelimColumn(1) == s.column(1) is true, but we want to treat this as IndentStateDown. - // So, we look also current indentState value by the above prevLineIndentNum based logic, and determines finally indentState. - // --- - // a: - // b - // c - // d: e - // ^ - s.indentState = IndentStateDown } else { - s.indentState = IndentStateEqual + // If lastDelimColumn and s.column are the same, + // treat as Down state since it is the same column as delimiter. + s.indentState = IndentStateDown } } else { s.indentState = s.indentStateFromIndentNumDifference() } } +func (s *Scanner) indentStateFromIndentNumDifference() IndentState { + switch { + case s.prevLineIndentNum < s.indentNum: + return IndentStateUp + case s.prevLineIndentNum == s.indentNum: + return IndentStateEqual + default: + return IndentStateDown + } +} + func (s *Scanner) updateIndent(ctx *Context, c rune) { if s.isFirstCharAtLine && s.isNewLineChar(c) && ctx.isDocument() { return @@ -195,6 +182,7 @@ func (s *Scanner) updateIndent(ctx *Context, c rune) { s.indentState = IndentStateKeep return } + s.updateIndentLevel() s.updateIndentState(ctx) s.isFirstCharAtLine = false } @@ -207,10 +195,6 @@ func (s *Scanner) isChangedToIndentStateUp() bool { return s.indentState == IndentStateUp } -func (s *Scanner) isChangedToIndentStateEqual() bool { - return s.indentState == IndentStateEqual -} - func (s *Scanner) addBufferedTokenIfExists(ctx *Context) { ctx.addToken(s.bufferedToken(ctx)) } @@ -634,23 +618,16 @@ func (s *Scanner) scan(ctx *Context) (pos int) { pos = ctx.nextPos() c := ctx.currentChar() s.updateIndent(ctx, c) + if s.isChangedToIndentStateDown() { + s.addBufferedTokenIfExists(ctx) + } if ctx.isDocument() { - if (s.indentNum == 0 && s.isChangedToIndentStateEqual()) || - s.isChangedToIndentStateDown() { - s.addBufferedTokenIfExists(ctx) + if s.isChangedToIndentStateDown() { s.breakLiteral(ctx) } else { s.scanLiteral(ctx, c) continue } - } else if s.isChangedToIndentStateDown() { - s.addBufferedTokenIfExists(ctx) - } else if s.isChangedToIndentStateEqual() { - // if first character is new line character, buffer expect to raw folded literal - if len(ctx.obuf) > 0 && s.newLineCount(ctx.obuf) <= 1 { - // doesn't raw folded literal - s.addBufferedTokenIfExists(ctx) - } } switch c { case '{':