Merge pull request #208 from goccy/feature/fix-origin-buffer-for-ycat

Fix origin buffer for DocumentHeader and DocumentEnd and Directive
goccy · Mar 1, 2021 · 0b9febf · 0b9febf
2 parents 5bc735b + 560f570
commit 0b9febf
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 28 deletions.
diff --git a/printer/printer.go b/printer/printer.go
@@ -287,21 +287,6 @@ func (p *Printer) printBeforeTokens(tk *token.Token, minLine, extLine int) token
 	return tokens
 }
 
-func (p *Printer) addNewLineCharIfDocumentHeader(tk *token.Token) {
-	if tk.Prev == nil {
-		return
-	}
-	if tk.Type != token.DocumentHeaderType {
-		return
-	}
-	prev := tk.Prev
-	lineDiff := tk.Position.Line - prev.Position.Line
-	if p.isNewLineLastChar(prev.Origin) {
-		lineDiff--
-	}
-	tk.Origin = strings.Repeat("\n", lineDiff) + tk.Origin
-}
-
 func (p *Printer) printAfterTokens(tk *token.Token, maxLine int) token.Tokens {
 	tokens := token.Tokens{}
 	if tk == nil {
@@ -316,7 +301,6 @@ func (p *Printer) printAfterTokens(tk *token.Token, maxLine int) token.Tokens {
 	tk = minTk.Next
 	for tk != nil && tk.Position.Line <= maxLine {
 		clonedTk := tk.Clone()
-		p.addNewLineCharIfDocumentHeader(clonedTk)
 		tokens.Add(clonedTk)
 		tk = clonedTk.Next
 	}

diff --git a/scanner/scanner.go b/scanner/scanner.go
@@ -578,7 +578,7 @@ func (s *Scanner) scan(ctx *Context) (pos int) {
 			}
 		case '.':
 			if s.indentNum == 0 && s.column == 1 && ctx.repeatNum('.') == 3 {
-				ctx.addToken(token.DocumentEnd(s.pos()))
+				ctx.addToken(token.DocumentEnd(string(ctx.obuf)+"...", s.pos()))
 				s.progressColumn(ctx, 3)
 				pos += 2
 				return
@@ -594,7 +594,7 @@ func (s *Scanner) scan(ctx *Context) (pos int) {
 		case '-':
 			if s.indentNum == 0 && s.column == 1 && ctx.repeatNum('-') == 3 {
 				s.addBufferedTokenIfExists(ctx)
-				ctx.addToken(token.DocumentHeader(s.pos()))
+				ctx.addToken(token.DocumentHeader(string(ctx.obuf)+"---", s.pos()))
 				s.progressColumn(ctx, 3)
 				pos += 2
 				return
@@ -686,7 +686,7 @@ func (s *Scanner) scan(ctx *Context) (pos int) {
 			}
 		case '%':
 			if !ctx.existsBuffer() && s.indentNum == 0 {
-				ctx.addToken(token.Directive(s.pos()))
+				ctx.addToken(token.Directive(string(ctx.obuf)+"%", s.pos()))
 				s.progressColumn(ctx, 1)
 				return
 			}

diff --git a/token/token.go b/token/token.go
@@ -961,13 +961,13 @@ func DoubleQuote(value string, org string, pos *Position) *Token {
 }
 
 // Directive create token for Directive
-func Directive(pos *Position) *Token {
+func Directive(org string, pos *Position) *Token {
 	return &Token{
 		Type:          DirectiveType,
 		CharacterType: CharacterTypeIndicator,
 		Indicator:     DirectiveIndicator,
 		Value:         string(DirectiveCharacter),
-		Origin:        string(DirectiveCharacter),
+		Origin:        org,
 		Position:      pos,
 	}
 }
@@ -997,25 +997,25 @@ func MergeKey(org string, pos *Position) *Token {
 }
 
 // DocumentHeader create token for DocumentHeader
-func DocumentHeader(pos *Position) *Token {
+func DocumentHeader(org string, pos *Position) *Token {
 	return &Token{
 		Type:          DocumentHeaderType,
 		CharacterType: CharacterTypeMiscellaneous,
 		Indicator:     NotIndicator,
 		Value:         "---",
-		Origin:        "---",
+		Origin:        org,
 		Position:      pos,
 	}
 }
 
 // DocumentEnd create token for DocumentEnd
-func DocumentEnd(pos *Position) *Token {
+func DocumentEnd(org string, pos *Position) *Token {
 	return &Token{
 		Type:          DocumentEndType,
 		CharacterType: CharacterTypeMiscellaneous,
 		Indicator:     NotIndicator,
 		Value:         "...",
-		Origin:        "...",
+		Origin:        org,
 		Position:      pos,
 	}
 }

diff --git a/token/token_test.go b/token/token_test.go
@@ -24,11 +24,11 @@ func TestToken(t *testing.T) {
 		token.Folded(">", ">", pos),
 		token.SingleQuote("'", "'", pos),
 		token.DoubleQuote(`"`, `"`, pos),
-		token.Directive(pos),
+		token.Directive("%", pos),
 		token.Space(pos),
 		token.MergeKey("<<", pos),
-		token.DocumentHeader(pos),
-		token.DocumentEnd(pos),
+		token.DocumentHeader("---", pos),
+		token.DocumentEnd("...", pos),
 		token.New("1", "1", pos),
 		token.New("3.14", "3.14", pos),
 		token.New("-0b101010", "-0b101010", pos),