From b23273d19cfcba85a7f08630e5d6f002d38cbcc9 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 26 Nov 2024 20:12:49 +0900 Subject: [PATCH 1/3] fix parser --- parser/parser.go | 1 + scanner/context.go | 17 +++++++++++ scanner/scanner.go | 63 ++++++++++++++++++++++++++++++----------- yaml_test_suite_test.go | 10 +------ 4 files changed, 65 insertions(+), 26 deletions(-) diff --git a/parser/parser.go b/parser/parser.go index 47f6dae..0237a9b 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -323,6 +323,7 @@ func (p *parser) parseFlowMap(ctx *context) (*ast.MappingNode, error) { if err != nil { return nil, err } + ctx := ctx.withChild(p.mapKeyText(key)) colonTk := mapKeyTk.Group.Last() if p.isFlowMapDelim(ctx.nextToken()) { value, err := newNullNode(ctx, ctx.insertNullToken(colonTk)) diff --git a/scanner/context.go b/scanner/context.go index 6898d08..9220778 100644 --- a/scanner/context.go +++ b/scanner/context.go @@ -138,6 +138,13 @@ func (c *Context) updateDocumentNewLineState() { c.docLineIndentColumn = 0 } +func (c *Context) isIndentColumn(column int) bool { + if c.docFirstLineIndentColumn == 0 { + return column == 1 + } + return c.docFirstLineIndentColumn > column +} + func (c *Context) addDocumentIndent(column int) { if c.docFirstLineIndentColumn == 0 { return @@ -192,6 +199,16 @@ func (c *Context) addBuf(r rune) { } } +func (c *Context) addBufWithTab(r rune) { + if len(c.buf) == 0 && r == ' ' { + return + } + c.buf = append(c.buf, r) + if r != ' ' { + c.notSpaceCharPos = len(c.buf) + } +} + func (c *Context) addOriginBuf(r rune) { c.obuf = append(c.obuf, r) if r != ' ' && r != '\t' { diff --git a/scanner/scanner.go b/scanner/scanner.go index ed43245..9dda5c9 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -46,6 +46,7 @@ type Scanner struct { indentLevel int isFirstCharAtLine bool isAnchor bool + isAlias bool isDirective bool startedFlowSequenceNum int startedFlowMapNum int @@ -105,6 +106,7 @@ func (s *Scanner) progressLine(ctx *Context) { s.indentNum = 0 s.isFirstCharAtLine = true s.isAnchor = false + s.isAlias = false s.isDirective = false s.progress(ctx, 1) } @@ -516,7 +518,7 @@ func (s *Scanner) scanWhiteSpace(ctx *Context) bool { if ctx.isDocument() { return false } - if !s.isAnchor && !s.isFirstCharAtLine { + if !s.isAnchor && !s.isAlias && !s.isFirstCharAtLine { return false } @@ -528,6 +530,7 @@ func (s *Scanner) scanWhiteSpace(ctx *Context) bool { s.addBufferedTokenIfExists(ctx) s.isAnchor = false + s.isAlias = false return true } @@ -663,7 +666,7 @@ func (s *Scanner) scanDocument(ctx *Context, c rune) error { } else if s.isFirstCharAtLine && c == ' ' { ctx.addDocumentIndent(s.column) s.progressColumn(ctx, 1) - } else if s.isFirstCharAtLine && c == '\t' { + } else if s.isFirstCharAtLine && c == '\t' && ctx.isIndentColumn(s.column) { err := ErrInvalidToken( token.Invalid( "found a tab character where an indentation space is expected", @@ -683,7 +686,7 @@ func (s *Scanner) scanDocument(ctx *Context, c rune) error { return ErrInvalidToken(invalidTk) } ctx.updateDocumentNewLineInFolded(s.column) - ctx.addBuf(c) + ctx.addBufWithTab(c) s.progressColumn(ctx, 1) } return nil @@ -717,7 +720,7 @@ func (s *Scanner) scanNewLine(ctx *Context, c rune) { if ctx.isEOS() { s.addBufferedTokenIfExists(ctx) - } else if s.isAnchor { + } else if s.isAnchor || s.isAlias { s.addBufferedTokenIfExists(ctx) } if ctx.existsBuffer() && s.isFirstCharAtLine { @@ -812,13 +815,19 @@ func (s *Scanner) scanFlowEntry(ctx *Context, c rune) bool { return true } -func (s *Scanner) scanMapDelim(ctx *Context) bool { +func (s *Scanner) scanMapDelim(ctx *Context) (bool, error) { nc := ctx.nextChar() - if s.isDirective { - return false + if s.isDirective || s.isAnchor || s.isAlias { + return false, nil } if s.startedFlowMapNum <= 0 && nc != ' ' && nc != '\t' && !s.isNewLineChar(nc) && !ctx.isNextEOS() { - return false + return false, nil + } + + if strings.HasPrefix(strings.TrimPrefix(string(ctx.obuf), " "), "\t") && !strings.HasPrefix(string(ctx.buf), "\t") { + invalidTk := token.Invalid("tab character cannot use as a map key directly", string(ctx.obuf), s.pos()) + s.progressColumn(ctx, 1) + return false, ErrInvalidToken(invalidTk) } // mapping value @@ -836,7 +845,7 @@ func (s *Scanner) scanMapDelim(ctx *Context) bool { ctx.addToken(token.MappingValue(s.pos())) s.progressColumn(ctx, 1) ctx.clear() - return true + return true, nil } func (s *Scanner) scanDocumentStart(ctx *Context) bool { @@ -908,14 +917,20 @@ func (s *Scanner) scanRawFoldedChar(ctx *Context) bool { return true } -func (s *Scanner) scanSequence(ctx *Context) bool { +func (s *Scanner) scanSequence(ctx *Context) (bool, error) { if ctx.existsBuffer() { - return false + return false, nil } nc := ctx.nextChar() - if nc != 0 && nc != ' ' && !s.isNewLineChar(nc) { - return false + if nc != 0 && nc != ' ' && nc != '\t' && !s.isNewLineChar(nc) { + return false, nil + } + + if strings.HasPrefix(strings.TrimPrefix(string(ctx.obuf), " "), "\t") { + invalidTk := token.Invalid("tab character cannot use as a sequence delimiter", string(ctx.obuf), s.pos()) + s.progressColumn(ctx, 1) + return false, ErrInvalidToken(invalidTk) } s.addBufferedTokenIfExists(ctx) @@ -925,7 +940,7 @@ func (s *Scanner) scanSequence(ctx *Context) bool { ctx.addToken(tk) s.progressColumn(ctx, 1) ctx.clear() - return true + return true, nil } func (s *Scanner) scanDocumentHeader(ctx *Context) (bool, error) { @@ -1036,7 +1051,7 @@ func (s *Scanner) scanMapKey(ctx *Context) bool { } nc := ctx.nextChar() - if nc != ' ' { + if nc != ' ' && nc != '\t' { return false } @@ -1084,6 +1099,7 @@ func (s *Scanner) scanAlias(ctx *Context) bool { ctx.addOriginBuf('*') ctx.addToken(token.Alias(string(ctx.obuf), s.pos())) s.progressColumn(ctx, 1) + s.isAlias = true ctx.clear() return true } @@ -1107,6 +1123,11 @@ func (s *Scanner) scanReservedChar(ctx *Context, c rune) error { } func (s *Scanner) scanTab(ctx *Context, c rune) error { + if s.startedFlowSequenceNum > 0 || s.startedFlowMapNum > 0 { + // tabs character is allowed in flow mode. + return nil + } + if !s.isFirstCharAtLine { return nil } @@ -1185,7 +1206,11 @@ func (s *Scanner) scan(ctx *Context) error { if s.scanRawFoldedChar(ctx) { continue } - if s.scanSequence(ctx) { + scanned, err := s.scanSequence(ctx) + if err != nil { + return err + } + if scanned { continue } case '[': @@ -1201,7 +1226,11 @@ func (s *Scanner) scan(ctx *Context) error { continue } case ':': - if s.scanMapDelim(ctx) { + scanned, err := s.scanMapDelim(ctx) + if err != nil { + return err + } + if scanned { continue } case '|', '>': diff --git a/yaml_test_suite_test.go b/yaml_test_suite_test.go index 286becb..74ab1af 100644 --- a/yaml_test_suite_test.go +++ b/yaml_test_suite_test.go @@ -18,7 +18,6 @@ var failureTestNames = []string{ "aliases-in-flow-objects", "aliases-in-explicit-block-mapping", "aliases-in-implicit-block-mapping", - "allowed-characters-in-alias", "anchor-before-sequence-entry-on-same-line", "anchor-for-empty-node", "anchor-plus-alias", @@ -156,13 +155,7 @@ var failureTestNames = []string{ "syntax-character-edge-cases/00", "tab-at-beginning-of-line-followed-by-a-flow-mapping", "tab-indented-top-flow", - "tabs-in-various-contexts/001", - "tabs-in-various-contexts/002", - "tabs-in-various-contexts/004", - "tabs-in-various-contexts/005", - "tabs-in-various-contexts/006", - "tabs-in-various-contexts/008", - "tabs-in-various-contexts/010", + "tabs-in-various-contexts/003", "tabs-that-look-like-indentation/00", "tabs-that-look-like-indentation/01", "tabs-that-look-like-indentation/02", @@ -178,7 +171,6 @@ var failureTestNames = []string{ "tags-in-explicit-mapping", "tags-in-implicit-mapping", "tags-on-empty-scalars", - "three-dashes-and-content-without-space", "trailing-line-of-spaces/01", // last '\n' character is needed ? "various-combinations-of-explicit-block-mappings", // no json "various-trailing-comments", // no json From fe569b54db636cae3a7bd459717ccf52d5bff228 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 26 Nov 2024 21:58:21 +0900 Subject: [PATCH 2/3] fix anchor-for-empty-node --- parser/parser.go | 42 +++++++++++++++++++++++++++++++++++++++-- parser/parser_test.go | 14 ++++++++++++++ parser/token.go | 3 +++ yaml_test_suite_test.go | 2 -- 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/parser/parser.go b/parser/parser.go index 0237a9b..803d584 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -620,8 +620,10 @@ func (p *parser) parseMapValue(ctx *context, key ast.MapKeyNode, colonTk *Token) if ctx.isComment() { tk = ctx.nextNotCommentToken() } + keyCol := key.GetToken().Position.Column + keyLine := key.GetToken().Position.Line - if tk.Column() == key.GetToken().Position.Column && p.isMapToken(tk) { + if tk.Column() == keyCol && p.isMapToken(tk) { // in this case, // ---- // key: @@ -629,7 +631,25 @@ func (p *parser) parseMapValue(ctx *context, key ast.MapKeyNode, colonTk *Token) return newNullNode(ctx, ctx.insertNullToken(colonTk)) } - if tk.Column() < key.GetToken().Position.Column { + if tk.Line() == keyLine && tk.GroupType() == TokenGroupAnchorName && + ctx.nextToken().Column() == keyCol && p.isMapToken(ctx.nextToken()) { + // in this case, + // ---- + // key: &anchor + // next + group := &TokenGroup{ + Type: TokenGroupAnchor, + Tokens: []*Token{tk, ctx.createNullToken(tk)}, + } + anchor, err := p.parseAnchor(ctx.withGroup(group), group) + if err != nil { + return nil, err + } + ctx.goNext() + return anchor, nil + } + + if tk.Column() < keyCol { // in this case, // ---- // key: @@ -637,6 +657,24 @@ func (p *parser) parseMapValue(ctx *context, key ast.MapKeyNode, colonTk *Token) return newNullNode(ctx, ctx.insertNullToken(colonTk)) } + if tk.Line() == keyLine && tk.GroupType() == TokenGroupAnchorName && + ctx.nextToken().Column() < keyCol { + // in this case, + // ---- + // key: &anchor + // next + group := &TokenGroup{ + Type: TokenGroupAnchor, + Tokens: []*Token{tk, ctx.createNullToken(tk)}, + } + anchor, err := p.parseAnchor(ctx.withGroup(group), group) + if err != nil { + return nil, err + } + ctx.goNext() + return anchor, nil + } + value, err := p.parseToken(ctx, ctx.currentToken()) if err != nil { return nil, err diff --git a/parser/parser_test.go b/parser/parser_test.go index 9493e2d..174c897 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -667,6 +667,20 @@ a: - |2 b c: d +`, + }, + { + ` +a: + b: &anchor + c: &anchor2 +d: e +`, + ` +a: + b: &anchor null + c: &anchor2 null +d: e `, }, } diff --git a/parser/token.go b/parser/token.go index be00c92..9897492 100644 --- a/parser/token.go +++ b/parser/token.go @@ -311,6 +311,9 @@ func createAnchorAndAliasTokenGroups(tokens []*Token) ([]*Token, error) { }, } valueTk := tokens[i+2] + if tk.Line() == valueTk.Line() && valueTk.Type() == token.SequenceEntryType { + return nil, errors.ErrSyntax("sequence entries are not allowed after anchor on the same line", valueTk.RawToken()) + } if tk.Line() == valueTk.Line() && isScalarType(valueTk) { ret = append(ret, &Token{ Group: &TokenGroup{ diff --git a/yaml_test_suite_test.go b/yaml_test_suite_test.go index 74ab1af..c3ae330 100644 --- a/yaml_test_suite_test.go +++ b/yaml_test_suite_test.go @@ -18,8 +18,6 @@ var failureTestNames = []string{ "aliases-in-flow-objects", "aliases-in-explicit-block-mapping", "aliases-in-implicit-block-mapping", - "anchor-before-sequence-entry-on-same-line", - "anchor-for-empty-node", "anchor-plus-alias", "anchors-in-mapping", "anchors-with-colon-in-name", From 5ab916a79b7f87b08cf0d1ce699966757560d394 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 26 Nov 2024 22:03:51 +0900 Subject: [PATCH 3/3] add test case --- parser/parser_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/parser/parser_test.go b/parser/parser_test.go index 174c897..14980b5 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -953,6 +953,38 @@ foo: bar: null # comment baz: 1 +`, + }, + { + ` +{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "foo", + "labels": { + "app": "bar" + } + }, + "spec": { + "replicas": 3, + "selector": { + "matchLabels": { + "app": "bar" + } + }, + "template": { + "metadata": { + "labels": { + "app": "bar" + } + } + } + } +} +`, + ` +{"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": "foo", "labels": {"app": "bar"}}, "spec": {"replicas": 3, "selector": {"matchLabels": {"app": "bar"}}, "template": {"metadata": {"labels": {"app": "bar"}}}}} `, }, }