Skip to content

Commit

Permalink
fix parsing of invalid document
Browse files Browse the repository at this point in the history
  • Loading branch information
goccy committed Nov 2, 2024
1 parent 61505f3 commit 873d14e
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 0 deletions.
8 changes: 8 additions & 0 deletions lexer/lexer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2421,6 +2421,14 @@ a: |invalid`,
name: "invalid double-quoted",
src: `a: "\"key\": \"value:\"`,
},
{
name: "invalid document folded",
src: ">\n>",
},
{
name: "invalid document number",
src: ">\n1",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
Expand Down
27 changes: 27 additions & 0 deletions parser/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1099,6 +1099,33 @@ b: - 2
[1:8] ',' or ']' must be specified
> 1 | foo: [$[should not be allowed]]
^
`,
},
{
">\n>",
`
[2:1] found invalid token
1 | >
> 2 | >
^
`,
},
{
">\n1",
`
[2:1] found invalid token
1 | >
> 2 | 1
^
`,
},
{
"|\n1",
`
[2:1] found invalid token
1 | |
> 2 | 1
^
`,
},
}
Expand Down
10 changes: 10 additions & 0 deletions scanner/scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -837,6 +837,9 @@ func (s *Scanner) scanLiteralHeaderOption(ctx *Context) error {
case "", "+", "-",
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9":
hasComment := len(opt) < orgOptLen
if s.column == 1 {
s.lastDelimColumn = 1
}
if header == '|' {
if hasComment {
commentLen := orgOptLen - len(opt)
Expand Down Expand Up @@ -961,6 +964,13 @@ func (s *Scanner) scan(ctx *Context) error {
if tk := ctx.lastToken(); tk != nil {
// If literal/folded content is empty, no string token is added.
// Therefore, add an empty string token.
// But if literal/folded token column is 1, it is invalid at down state.
if tk.Position.Column == 1 {
return ErrInvalidToken(
"could not find document",
token.Invalid(string(ctx.obuf), s.pos()),
)
}
if tk.Type != token.StringType {
ctx.addToken(token.String("", "", s.pos()))
}
Expand Down

0 comments on commit 873d14e

Please sign in to comment.