Fix scanning token of number in literal

goccy · Jun 1, 2020 · 0c228d5 · 0c228d5
1 parent 4368a8b
commit 0c228d5
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 10 deletions.
diff --git a/parser/parser_test.go b/parser/parser_test.go
@@ -66,6 +66,8 @@ func TestParser(t *testing.T) {
 		"- !tag\n  a: b\n  c: d\n",
 		"v:\n- A\n- |-\n  B\n  C\n",
 		"v:\n- A\n- >-\n  B\n  C\n",
+		"v: |-\n  0\n",
+		"v: |-\n  0\nx: 0",
 	}
 	for _, src := range sources {
 		fmt.Printf(src)

diff --git a/scanner/context.go b/scanner/context.go
@@ -190,7 +190,12 @@ func (c *Context) bufferedToken(pos *token.Position) *token.Token {
 	if len(source) == 0 {
 		return nil
 	}
-	tk := token.New(string(source), string(c.obuf), pos)
+	var tk *token.Token
+	if c.isDocument() {
+		tk = token.String(string(source), string(c.obuf), pos)
+	} else {
+		tk = token.New(string(source), string(c.obuf), pos)
+	}
 	c.resetBuffer()
 	return tk
 }
diff --git a/scanner/scanner.go b/scanner/scanner.go
@@ -312,7 +312,7 @@ func (s *Scanner) scanLiteral(ctx *Context, c rune) {
 			ctx.addBuf(c)
 		}
 		value := ctx.bufferedSrc()
-		ctx.addToken(token.New(string(value), string(ctx.obuf), s.pos()))
+		ctx.addToken(token.String(string(value), string(ctx.obuf), s.pos()))
 		ctx.resetBuffer()
 		s.progressColumn(ctx, 1)
 	} else if s.isNewLineChar(c) {

diff --git a/token/token.go b/token/token.go
@@ -628,14 +628,7 @@ func New(value string, org string, pos *Position) *Token {
 		}
 		return tk
 	}
-	return &Token{
-		Type:          StringType,
-		CharacterType: CharacterTypeMiscellaneous,
-		Indicator:     NotIndicator,
-		Value:         value,
-		Origin:        org,
-		Position:      pos,
-	}
+	return String(value, org, pos)
 }
 
 // Position type for position in YAML document
@@ -718,6 +711,18 @@ func (t Tokens) Dump() {
 	}
 }
 
+// String create token for String
+func String(value string, org string, pos *Position) *Token {
+	return &Token{
+		Type:          StringType,
+		CharacterType: CharacterTypeMiscellaneous,
+		Indicator:     NotIndicator,
+		Value:         value,
+		Origin:        org,
+		Position:      pos,
+	}
+}
+
 // SequenceEntry create token for SequenceEntry
 func SequenceEntry(org string, pos *Position) *Token {
 	return &Token{