Skip to content

Commit

Permalink
Fix parser by using yaml-test-suite (#542)
Browse files Browse the repository at this point in the history 
* fix alias in map key
* fix parsing of wrong-indented-sequence-item
* fix parsing of tab character before comment
* fix parsing of white space in quote
* fix parsing of tab character in quote
  • Loading branch information
@goccy
goccy authored Nov 17, 2024
1 parent c312882 commit ff5d41f
Show file tree
Hide file tree
Showing 9 changed files with 415 additions and 108 deletions.
21 changes: 21 additions & 0 deletions ast/ast.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1287,6 +1287,7 @@ type MappingValueNode struct {
Start *token.Token
Key MapKeyNode
Value Node
Anchor *AnchorNode
FootComment *CommentGroupNode
}

Expand Down Expand Up @@ -1597,6 +1598,10 @@ type AnchorNode struct {
Value Node
}

func (n *AnchorNode) stringWithoutComment() string {
return n.Value.String()
}

func (n *AnchorNode) SetName(name string) error {
if n.Name == nil {
return ErrInvalidAnchorName
Expand All @@ -1622,6 +1627,10 @@ func (n *AnchorNode) GetToken() *token.Token {
return n.Start
}

func (n *AnchorNode) GetValue() any {
return n.Value.GetToken().Value
}

// AddColumn add column number to child nodes recursively
func (n *AnchorNode) AddColumn(col int) {
n.Start.AddColumn(col)
Expand Down Expand Up @@ -1658,6 +1667,10 @@ type AliasNode struct {
Value Node
}

func (n *AliasNode) stringWithoutComment() string {
return n.Value.String()
}

func (n *AliasNode) SetName(name string) error {
if n.Value == nil {
return ErrInvalidAliasName
Expand All @@ -1683,6 +1696,10 @@ func (n *AliasNode) GetToken() *token.Token {
return n.Start
}

func (n *AliasNode) GetValue() any {
return n.Value.GetToken().Value
}

// AddColumn add column number to child nodes recursively
func (n *AliasNode) AddColumn(col int) {
n.Start.AddColumn(col)
Expand Down Expand Up @@ -1745,6 +1762,10 @@ type TagNode struct {
Value Node
}

func (n *TagNode) stringWithoutComment() string {
return n.Value.String()
}

// Read implements (io.Reader).Read
func (n *TagNode) Read(p []byte) (int, error) {
return readNode(p, n)
Expand Down
11 changes: 9 additions & 2 deletions parser/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ import (

// context context at parsing
type context struct {
path string
isFlow bool
path string
isFlow bool
isMapKey bool
}

var pathSpecialChars = []string{
Expand Down Expand Up @@ -49,6 +50,12 @@ func (c *context) withFlow(isFlow bool) *context {
return &ctx
}

func (c *context) withMapKey() *context {
ctx := *c
ctx.isMapKey = true
return &ctx
}

func newContext() *context {
return &context{
path: "$",
Expand Down
150 changes: 104 additions & 46 deletions parser/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ func (p *parser) parseTag(ctx *context) (*ast.TagNode, error) {
if typ == token.LiteralType || typ == token.FoldedType {
value, err = p.parseLiteral(ctx)
} else {
value = p.parseScalarValue(p.currentToken())
value, err = p.parseScalarValueWithComment(ctx, p.currentToken())
}
case token.SequenceTag,
token.SetTag:
Expand Down Expand Up @@ -429,7 +429,7 @@ func (p *parser) mapKeyText(n ast.Node) string {
}

func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
key, err := p.parseMapKey(ctx)
key, err := p.parseMapKey(ctx.withMapKey())
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -497,30 +497,55 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
if antk == nil {
return nil, errors.ErrSyntax("required ':' and map value", ntk)
}
if antk.Type != token.MappingValueType {
return nil, errors.ErrSyntax("required ':' and map value", antk)
}
p.progressIgnoreComment(1)
value, err := p.parseToken(ctx, p.currentToken())
var comment *ast.CommentGroupNode
if tk := p.currentToken(); tk.Type == token.CommentType {
comment = p.parseCommentOnly(ctx)
}
value, err := p.parseMappingValue(ctx)
if err != nil {
return nil, err
}
switch value.Type() {
case ast.MappingType:
c, _ := value.(*ast.MappingNode)
comment := c.GetComment()
for idx, v := range c.Values {
if comment != nil {
comment.SetPath(value.GetPath())
if err := value.SetComment(comment); err != nil {
return nil, err
}
}
switch v := value.(type) {
case *ast.MappingNode:
comment := v.GetComment()
for idx, val := range v.Values {
if idx == 0 && comment != nil {
if err := v.SetComment(comment); err != nil {
if err := val.SetComment(comment); err != nil {
return nil, err
}
}
node.Values = append(node.Values, v)
node.Values = append(node.Values, val)
}
case *ast.MappingValueNode:
node.Values = append(node.Values, v)
case *ast.AnchorNode:
switch anchorV := v.Value.(type) {
case *ast.MappingNode:
comment := anchorV.GetComment()
for idx, val := range anchorV.Values {
if idx == 0 && comment != nil {
if err := val.SetComment(comment); err != nil {
return nil, err
}
}
val.Anchor = v
node.Values = append(node.Values, val)
}
case *ast.MappingValueNode:
anchorV.Anchor = v
node.Values = append(node.Values, anchorV)
default:
return nil, fmt.Errorf("failed to parse mapping value node. anchor node is %s", anchorV.Type())
}
case ast.MappingValueType:
node.Values = append(node.Values, value.(*ast.MappingValueNode))
default:
return nil, fmt.Errorf("failed to parse mapping value node node is %s", value.Type())
return nil, fmt.Errorf("failed to parse mapping value node. node is %s", value.Type())
}
ntk = p.nextNotCommentToken()
antk = p.afterNextNotCommentToken()
Expand Down Expand Up @@ -657,11 +682,15 @@ func (p *parser) parseAnchor(ctx *context) (*ast.AnchorNode, error) {
return nil, errors.ErrSyntax("unexpected anchor. anchor name is undefined", tk)
}
p.progress(1) // skip anchor token
name, err := p.parseToken(ctx, p.currentToken())
anchorNameTk := p.currentToken()
anchorNameNode, err := p.parseScalarValueWithComment(ctx, anchorNameTk)
if err != nil {
return nil, err
}
anchor.Name = name
if anchorNameNode == nil {
return nil, errors.ErrSyntax("unexpected anchor. anchor name is not scalar value", anchorNameTk)
}
anchor.Name = anchorNameNode
ntk = p.nextToken()
if ntk == nil {
return nil, errors.ErrSyntax("unexpected anchor. anchor value is undefined", p.currentToken())
Expand All @@ -684,40 +713,39 @@ func (p *parser) parseAlias(ctx *context) (*ast.AliasNode, error) {
return nil, errors.ErrSyntax("unexpected alias. alias name is undefined", tk)
}
p.progress(1) // skip alias token
name, err := p.parseToken(ctx, p.currentToken())
aliasNameTk := p.currentToken()
aliasNameNode, err := p.parseScalarValueWithComment(ctx, aliasNameTk)
if err != nil {
return nil, err
}
alias.Value = name
if aliasNameNode == nil {
return nil, errors.ErrSyntax("unexpected alias. alias name is not scalar value", aliasNameTk)
}
alias.Value = aliasNameNode
return alias, nil
}

func (p *parser) parseMapKey(ctx *context) (ast.MapKeyNode, error) {
tk := p.currentToken()
if value := p.parseScalarValue(tk); value != nil {
if value, _ := p.parseScalarValueWithComment(ctx, tk); value != nil {
return value, nil
}
switch tk.Type {
case token.MergeKeyType:
return ast.MergeKey(tk), nil
case token.MappingKeyType:
return p.parseMappingKey(ctx)
case token.TagType:
return p.parseTag(ctx)
}
return nil, errors.ErrSyntax("unexpected mapping key", tk)
}

func (p *parser) parseStringValue(tk *token.Token) *ast.StringNode {
switch tk.Type {
case token.StringType,
token.SingleQuoteType,
token.DoubleQuoteType:
return ast.String(tk)
}
return nil
}

func (p *parser) parseScalarValueWithComment(ctx *context, tk *token.Token) (ast.ScalarNode, error) {
node := p.parseScalarValue(tk)
node, err := p.parseScalarValue(ctx, tk)
if err != nil {
return nil, err
}
if node == nil {
return nil, nil
}
Expand All @@ -731,28 +759,32 @@ func (p *parser) parseScalarValueWithComment(ctx *context, tk *token.Token) (ast
return node, nil
}

func (p *parser) parseScalarValue(tk *token.Token) ast.ScalarNode {
if node := p.parseStringValue(tk); node != nil {
return node
}
func (p *parser) parseScalarValue(ctx *context, tk *token.Token) (ast.ScalarNode, error) {
switch tk.Type {
case token.NullType:
return ast.Null(tk)
return ast.Null(tk), nil
case token.BoolType:
return ast.Bool(tk)
return ast.Bool(tk), nil
case token.IntegerType,
token.BinaryIntegerType,
token.OctetIntegerType,
token.HexIntegerType:
return ast.Integer(tk)
return ast.Integer(tk), nil
case token.FloatType:
return ast.Float(tk)
return ast.Float(tk), nil
case token.InfinityType:
return ast.Infinity(tk)
return ast.Infinity(tk), nil
case token.NanType:
return ast.Nan(tk)
return ast.Nan(tk), nil
case token.StringType, token.SingleQuoteType,
token.DoubleQuoteType:
return ast.String(tk), nil
case token.AnchorType:
return p.parseAnchor(ctx)
case token.AliasType:
return p.parseAlias(ctx)
}
return nil
return nil, nil
}

func (p *parser) parseDirective(ctx *context) (*ast.DirectiveNode, error) {
Expand Down Expand Up @@ -925,9 +957,14 @@ func (p *parser) createNodeFromToken(ctx *context, tk *token.Token) (ast.Node, e
if tk == nil {
return nil, nil
}
if tk.NextType() == token.MappingValueType {
node, err := p.parseMappingValue(ctx)
return node, err
if !ctx.isMapKey && tk.NextType() == token.MappingValueType {
return p.parseMappingValue(ctx)
}
if tk.Type == token.AliasType {
aliasValueTk := p.nextToken()
if aliasValueTk != nil && aliasValueTk.NextType() == token.MappingValueType {
return p.parseMappingValue(ctx)
}
}
node, err := p.parseScalarValueWithComment(ctx, tk)
if err != nil {
Expand Down Expand Up @@ -982,13 +1019,34 @@ func (p *parser) parse(ctx *context) (*ast.File, error) {
}
if doc, ok := node.(*ast.DocumentNode); ok {
file.Docs = append(file.Docs, doc)
} else if len(file.Docs) == 0 {
file.Docs = append(file.Docs, ast.Document(nil, node))
} else {
lastNode := p.comparableColumnNode(file.Docs[len(file.Docs)-1])
curNode := p.comparableColumnNode(node)
if lastNode.GetToken().Position.Column != curNode.GetToken().Position.Column {
return nil, errors.ErrSyntax("value is not allowed in this context", curNode.GetToken())
}
file.Docs = append(file.Docs, ast.Document(nil, node))
}
}
return file, nil
}

func (p *parser) comparableColumnNode(n ast.Node) ast.Node {
switch nn := n.(type) {
case *ast.MappingNode:
if len(nn.Values) != 0 {
return nn.Values[0].Key
}
case *ast.MappingValueNode:
return nn.Key
case *ast.DocumentNode:
return p.comparableColumnNode(nn.Body)
}
return n
}

type Mode uint

const (
Expand Down
7 changes: 3 additions & 4 deletions parser/parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1110,12 +1110,11 @@ b
- c
`,
`
[4:1] required ':' and map value
[3:1] unexpected key name
2 | a: 1
3 | b
> 4 | - c
> 3 | b
^
`,
4 | - c`,
},
{
`a: [`,
Expand Down
Loading

0 comments on commit ff5d41f

Please sign in to comment.