From 9c4ec6ef640b0386f40474190e42e04352f161cf Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Sun, 15 Dec 2024 00:25:43 +0900 Subject: [PATCH 1/3] fix parsing of tab characters --- scanner/scanner.go | 6 ++++++ yaml_test_suite_test.go | 9 --------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 4ef05e3..42e2763 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -1413,6 +1413,12 @@ func (s *Scanner) scan(ctx *Context) error { s.progressColumn(ctx, 1) continue } + if s.lastDelimColumn < s.column { + s.indentNum++ + ctx.addOriginBuf(c) + s.progressColumn(ctx, 1) + continue + } if err := s.scanTab(ctx, c); err != nil { return err } diff --git a/yaml_test_suite_test.go b/yaml_test_suite_test.go index ca52dbc..54dc8d3 100644 --- a/yaml_test_suite_test.go +++ b/yaml_test_suite_test.go @@ -42,10 +42,6 @@ var failureTestNames = []string{ "various-trailing-comments-1-3", // no json. "zero-indented-sequences-in-explicit-mapping-keys", // no json. - "legal-tab-after-indentation", // pass yamlv3. - "multiline-plain-value-with-tabs-on-empty-lines", // pass yamlv3. - "tabs-that-look-like-indentation/05", // pass yamlv3. - "colon-at-the-beginning-of-adjacent-flow-scalar", "comment-without-whitespace-after-doublequoted-scalar", "construct-binary", @@ -61,14 +57,9 @@ var failureTestNames = []string{ "spec-example-9-3-bare-documents", "spec-example-9-6-stream", "spec-example-9-6-stream-1-3", - "tab-at-beginning-of-line-followed-by-a-flow-mapping", - "tab-indented-top-flow", "tabs-in-various-contexts/003", - "tabs-that-look-like-indentation/00", "tabs-that-look-like-indentation/01", - "tabs-that-look-like-indentation/03", "tabs-that-look-like-indentation/04", - "tabs-that-look-like-indentation/07", "tag-shorthand-used-in-documents-but-only-defined-in-the-first", "trailing-line-of-spaces/01", // last '\n' character is needed ? "wrong-indented-flow-sequence", // error ? From 0b9a668a8f6ba22f0d91960d44169eac04e0c7d0 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Sun, 15 Dec 2024 00:45:18 +0900 Subject: [PATCH 2/3] fix parsing of tab indent in double-quote --- scanner/scanner.go | 12 +++++++++++- yaml_test_suite_test.go | 1 - 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 42e2763..13d2fff 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -338,7 +338,17 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (*token.Token, error) { } } continue - } else if isFirstLineChar && (c == ' ' || c == '\t') { + } else if isFirstLineChar && c == ' ' { + continue + } else if isFirstLineChar && c == '\t' { + if s.lastDelimColumn >= s.column { + return nil, ErrInvalidToken( + token.Invalid( + "tab character cannot be used for indentation in double-quoted text", + string(ctx.obuf), s.pos(), + ), + ) + } continue } else if c == '\\' { isFirstLineChar = false diff --git a/yaml_test_suite_test.go b/yaml_test_suite_test.go index 54dc8d3..becce12 100644 --- a/yaml_test_suite_test.go +++ b/yaml_test_suite_test.go @@ -58,7 +58,6 @@ var failureTestNames = []string{ "spec-example-9-6-stream", "spec-example-9-6-stream-1-3", "tabs-in-various-contexts/003", - "tabs-that-look-like-indentation/01", "tabs-that-look-like-indentation/04", "tag-shorthand-used-in-documents-but-only-defined-in-the-first", "trailing-line-of-spaces/01", // last '\n' character is needed ? From d5db17f715af065ea682fd5c1bd5dbfb3bbcb895 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Sun, 15 Dec 2024 00:56:19 +0900 Subject: [PATCH 3/3] fix parsing of tab indent in single quote --- scanner/scanner.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 13d2fff..fb068ed 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -250,7 +250,17 @@ func (s *Scanner) scanSingleQuote(ctx *Context) (*token.Token, error) { } } continue - } else if isFirstLineChar && (c == ' ' || c == '\t') { + } else if isFirstLineChar && c == ' ' { + continue + } else if isFirstLineChar && c == '\t' { + if s.lastDelimColumn >= s.column { + return nil, ErrInvalidToken( + token.Invalid( + "tab character cannot be used for indentation in single-quoted text", + string(ctx.obuf), s.pos(), + ), + ) + } continue } else if c != '\'' { value = append(value, c)