fix parsing of invalid yaml

parser/parser.go

@@ -654,6 +654,10 @@ func (p *parser) parseLiteral(ctx *context) (*ast.LiteralNode, error) {
 	p.progress(1) // skip literal/folded token
 
 	tk := p.currentToken()
+	if tk == nil {
+		node.Value = ast.String(token.New("", "", node.Start.Position))
+		return node, nil
+	}
 	var comment *ast.CommentGroupNode
 	if tk.Type == token.CommentType {
 		comment = p.parseCommentOnly(ctx)

parser/parser_test.go

@@ -88,6 +88,10 @@ func TestParser(t *testing.T) {
 		"a_mk: \n  bd: 3\n",
 		"a: :a",
 		"{a: , b: c}",
+		"value: >\n",
+		"value: >\n\n",
+		"value: >\nother:",
+		"value: >\n\nother:",
 	}
 	for _, src := range sources {
 		if _, err := parser.Parse(lexer.Tokenize(src), 0); err != nil {
@@ -811,6 +815,22 @@ b: - 2
 [1:4] found invalid token
 >  1 | a: "\"key\": \"value:\"
           ^
+`,
+		},
+		{
+			`foo: [${should not be allowed}]`,
+			`
+[1:8] ',' or ']' must be specified
+>  1 | foo: [${should not be allowed}]
+              ^
+`,
+		},
+		{
+			`foo: [$[should not be allowed]]`,
+			`
+[1:8] ',' or ']' must be specified
+>  1 | foo: [$[should not be allowed]]
+              ^
 `,
 		},
 	}

scanner/scanner.go

@@ -620,11 +620,22 @@ func (s *Scanner) scanNewLine(ctx *Context, c rune) {
 	s.progressLine(ctx)
 }
 
+func (s *Scanner) isFlowMode() bool {
+	if s.startedFlowSequenceNum > 0 {
+		return true
+	}
+	if s.startedFlowMapNum > 0 {
+		return true
+	}
+	return false
+}
+
 func (s *Scanner) scanFlowMapStart(ctx *Context) bool {
-	if ctx.existsBuffer() {
+	if ctx.existsBuffer() && !s.isFlowMode() {
 		return false
 	}
 
+	s.addBufferedTokenIfExists(ctx)
 	ctx.addOriginBuf('{')
 	ctx.addToken(token.MappingStart(string(ctx.obuf), s.pos()))
 	s.startedFlowMapNum++
@@ -648,10 +659,11 @@ func (s *Scanner) scanFlowMapEnd(ctx *Context) bool {
 }
 
 func (s *Scanner) scanFlowArrayStart(ctx *Context) bool {
-	if ctx.existsBuffer() {
+	if ctx.existsBuffer() && !s.isFlowMode() {
 		return false
 	}
 
+	s.addBufferedTokenIfExists(ctx)
 	ctx.addOriginBuf('[')
 	ctx.addToken(token.SequenceStart(string(ctx.obuf), s.pos()))
 	s.startedFlowSequenceNum++
@@ -946,6 +958,13 @@ func (s *Scanner) scan(ctx *Context) error {
 		}
 		if ctx.isDocument() {
 			if s.isChangedToIndentStateDown() {
+				if tk := ctx.lastToken(); tk != nil {
+					// If literal/folded content is empty, no string token is added.
+					// Therefore, add an empty string token.
+					if tk.Type != token.StringType {
+						ctx.addToken(token.String("", "", s.pos()))
+					}
+				}
 				s.breakLiteral(ctx)
 			} else {
 				s.scanLiteral(ctx, c)