fix number parsing

ast/ast.go

@@ -341,118 +341,30 @@ func Bool(tk *token.Token) *BoolNode {
 
 // Integer create node for integer value
 func Integer(tk *token.Token) *IntegerNode {
-	switch tk.Type {
-	case token.BinaryIntegerType:
-		// skip two characters because binary token starts with '0b'
-		parsedNum := parseNumber("0b", tk.Value)
-		if parsedNum.isNegative {
-			i, _ := strconv.ParseInt(parsedNum.String(), 2, 64)
-			return &IntegerNode{
-				BaseNode: &BaseNode{},
-				Token:    tk,
-				Value:    i,
-			}
-		}
-		i, _ := strconv.ParseUint(parsedNum.String(), 2, 64)
-		return &IntegerNode{
-			BaseNode: &BaseNode{},
-			Token:    tk,
-			Value:    i,
-		}
-	case token.OctetIntegerType:
-		// octet token starts with '0o' or '-0o' or '0' or '-0'
-		parsedNum := parseNumber("0o", tk.Value)
-		if parsedNum.isNegative {
-			i, _ := strconv.ParseInt(parsedNum.String(), 8, 64)
-			return &IntegerNode{
-				BaseNode: &BaseNode{},
-				Token:    tk,
-				Value:    i,
-			}
-		}
-		i, _ := strconv.ParseUint(parsedNum.String(), 8, 64)
-		return &IntegerNode{
-			BaseNode: &BaseNode{},
-			Token:    tk,
-			Value:    i,
-		}
-	case token.HexIntegerType:
-		// hex token starts with '0x' or '-0x'
-		parsedNum := parseNumber("0x", tk.Value)
-		if parsedNum.isNegative {
-			i, _ := strconv.ParseInt(parsedNum.String(), 16, 64)
-			return &IntegerNode{
-				BaseNode: &BaseNode{},
-				Token:    tk,
-				Value:    i,
-			}
-		}
-		i, _ := strconv.ParseUint(parsedNum.String(), 16, 64)
-		return &IntegerNode{
-			BaseNode: &BaseNode{},
-			Token:    tk,
-			Value:    i,
-		}
-	}
-	parsedNum := parseNumber("", tk.Value)
-	if parsedNum.isNegative {
-		i, _ := strconv.ParseInt(parsedNum.String(), 10, 64)
-		return &IntegerNode{
-			BaseNode: &BaseNode{},
-			Token:    tk,
-			Value:    i,
-		}
+	var v any
+	if num := token.ToNumber(tk.Value); num != nil {
+		v = num.Value
 	}
-	i, _ := strconv.ParseUint(parsedNum.String(), 10, 64)
 	return &IntegerNode{
 		BaseNode: &BaseNode{},
 		Token:    tk,
-		Value:    i,
-	}
-}
-
-type parsedNumber struct {
-	isNegative bool
-	num        string
-}
-
-func (n *parsedNumber) String() string {
-	if n.isNegative {
-		return "-" + n.num
-	}
-	return n.num
-}
-
-func parseNumber(prefix, value string) *parsedNumber {
-	isNegative := value[0] == '-'
-	trimmed := strings.TrimPrefix(value, "+")
-	trimmed = strings.TrimPrefix(trimmed, "-")
-	trimmed = strings.TrimPrefix(trimmed, prefix)
-
-	num := make([]rune, 0, len(trimmed))
-	for _, v := range trimmed {
-		if v == '_' {
-			continue
-		}
-		num = append(num, v)
-	}
-	if len(num) == 0 {
-		num = append(num, '0')
-	}
-	return &parsedNumber{
-		isNegative: isNegative,
-		num:        string(num),
+		Value:    v,
 	}
 }
 
 // Float create node for float value
 func Float(tk *token.Token) *FloatNode {
-	parsedNum := parseNumber("", tk.Value)
-	f, _ := strconv.ParseFloat(parsedNum.String(), 64)
+	var v float64
+	if num := token.ToNumber(tk.Value); num != nil && num.Type == token.NumberTypeFloat {
+		value, ok := num.Value.(float64)
+		if ok {
+			v = value
+		}
+	}
 	return &FloatNode{
 		BaseNode: &BaseNode{},
 		Token:    tk,
-		Value:    f,
+		Value:    v,
 	}
 }
 

lexer/lexer_test.go

@@ -86,6 +86,18 @@ func TestTokenize(t *testing.T) {
 				},
 			},
 		},
+		{
+			YAML: `0.123e+123`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.FloatType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "0.123e+123",
+					Origin:        "0.123e+123",
+				},
+			},
+		},
 		{
 			YAML: `{}
 		`,
@@ -2257,6 +2269,42 @@ s: >-3
 				},
 			},
 		},
+		{
+			YAML: `0b98765`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "0b98765",
+					Origin:        "0b98765",
+				},
+			},
+		},
+		{
+			YAML: `098765`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "098765",
+					Origin:        "098765",
+				},
+			},
+		},
+		{
+			YAML: `0o98765`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "0o98765",
+					Origin:        "0o98765",
+				},
+			},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.YAML, func(t *testing.T) {