From b73cb947b756bb4879c3dc37eef08dcd1a965a42 Mon Sep 17 00:00:00 2001 From: Malcolm Holmes Date: Mon, 15 Jan 2024 13:51:20 +0000 Subject: [PATCH 1/2] First pass at test cases --- cmd/dev/main.go | 182 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 cmd/dev/main.go diff --git a/cmd/dev/main.go b/cmd/dev/main.go new file mode 100644 index 00000000..6b35aa02 --- /dev/null +++ b/cmd/dev/main.go @@ -0,0 +1,182 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "os" + "strings" + + "github.com/goccy/go-yaml" + "github.com/kr/pretty" +) + +func main() { + err := testBackslash() + if err != nil { + log.Printf("Test Backslash in key: %v", err) + } + err = testNewline() + if err != nil { + log.Printf("Test newline in key: %v", err) + } + err = testQuote() + if err != nil { + log.Printf("Test quote in key: %v", err) + } + file, err := testLoadYAMLFile() + if err != nil { + log.Printf("Test loading YAML file %s: %v", file, err) + } + file, err = testLoadJSONFile() + if err != nil { + log.Printf("Test loading JSON file %s: %v", file, err) + } +} + +func testNewline() error { + spec := map[string]string{ + "a": "a", + "c\nc": "cc", + "d": "d", + } + y, err := yaml.Marshal(spec) + if err != nil { + return err + } + + //y, err := os.ReadFile(filename) + decoder := yaml.NewDecoder(bytes.NewReader(y)) + var out map[string]interface{} + for i := 0; ; i++ { + err = decoder.Decode(&out) + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("Error on iteration %d: %v", i, err) + } + pretty.Print(i, out) + } + log.Printf("newline PASSED") + return nil +} + +func testBackslash() error { + spec := map[string]any{ + "outer": map[string]string{ + "b\\b": "b", + "d": "d", + }, + } + y, err := yaml.Marshal(spec) + if err != nil { + return err + } + + //y, err := os.ReadFile(filename) + decoder := yaml.NewDecoder(bytes.NewReader(y)) + var out map[string]interface{} + for i := 0; ; i++ { + err = decoder.Decode(&out) + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("Error on iteration %d: %v", i, err) + } + } + log.Printf("backslash PASSED") + return nil +} + +func testQuote() error { + spec := `--- + outer: + "a\"b\"c": a + "d\"e\"f": d +` + decoder := yaml.NewDecoder(strings.NewReader(spec)) + var out map[string]interface{} + for i := 0; ; i++ { + err := decoder.Decode(&out) + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("Error on iteration %d: %v", i, err) + } + } + log.Printf("quote PASSED") + return nil +} + +/** +bi3/dashboards/4u89zGTVk/dashboard-Vm10mGTVz.yaml <-- plain text/HTML?? +bi3/dashboards/7u49jPd7k/dashboard--49ZBkKnk.yaml <-- newline +bi3/dashboards/d40980f9-6366-458b-aac0-2c0a24e9fb6b/dashboard-b4e8bc14-9c16-4181-897c-e30631806d45.yaml <-- newline at beginning of expression + +**/ + +/* + "bi2/dashboards/WoVNxSA4k/dashboard-ae65738c-671b-4a51-b2e9-49d583b28793.yaml", //backslash + "bi2/dashboards/WoVNxSA4k/dashboard-cdc9172e-b37e-49fb-8e12-0e9ac806ea44.yaml", //backslash + "bi2/dashboards/WoVNxSA4k/dashboard-fcce96d1-a898-4726-97ff-59e70c80ad95.yaml", //backslash + "bi2/dashboards/QqKpDP0Vz/dashboard-e7be048f-625e-486b-a4d3-37a393fe834f.yaml", //backslash + "bi2/dashboards/FkdQ9Lsnz/dashboard-e2773f54-aebc-407a-8b48-2c7192db2921.yaml", //quote +*/ + +func testLoadYAMLFile() (string, error) { + file := "e85464d4-efcd-4351-ad68-3ccc935162fa.yaml" + log.Printf("Loading %s", file) + y, err := os.ReadFile(file) + if err != nil { + return file, err + } + + decoder := yaml.NewDecoder(bytes.NewReader(y)) + var out map[string]interface{} + for i := 0; ; i++ { + err = decoder.Decode(&out) + if err == io.EOF { + break + } + if err != nil { + return "", fmt.Errorf("Error on iteration %d: %v", i, err) + } + } + log.Printf("load %s file PASSED", file) + return file, nil +} + +func testLoadJSONFile() (string, error) { + spec := map[string]any{} + + file := "e2773f54-aebc-407a-8b48-2c7192db2921.json" + b, err := os.ReadFile(file) + if err != nil { + return file, err + } + err = json.Unmarshal(b, &spec) + + y, err := yaml.Marshal(spec) + if err != nil { + return file, err + } + + decoder := yaml.NewDecoder(bytes.NewReader(y)) + var out map[string]interface{} + for i := 0; ; i++ { + err = decoder.Decode(&out) + if err == io.EOF { + break + } + if err != nil { + return "", fmt.Errorf("Error on iteration %d: %v", i, err) + } + } + log.Printf("load %s file PASSED", file) + return file, nil +} From 1f79f66a4a32ce1e890a65be715ad57e2e2099e4 Mon Sep 17 00:00:00 2001 From: Malcolm Holmes Date: Mon, 15 Jan 2024 16:53:04 +0000 Subject: [PATCH 2/2] Add tests for failing decode/encode --- cmd/dev/main.go | 182 ------------------------------------------------ decode_test.go | 114 ++++++++++++++++++++++++++++++ encode_test.go | 33 +++++++++ 3 files changed, 147 insertions(+), 182 deletions(-) delete mode 100644 cmd/dev/main.go diff --git a/cmd/dev/main.go b/cmd/dev/main.go deleted file mode 100644 index 6b35aa02..00000000 --- a/cmd/dev/main.go +++ /dev/null @@ -1,182 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "log" - "os" - "strings" - - "github.com/goccy/go-yaml" - "github.com/kr/pretty" -) - -func main() { - err := testBackslash() - if err != nil { - log.Printf("Test Backslash in key: %v", err) - } - err = testNewline() - if err != nil { - log.Printf("Test newline in key: %v", err) - } - err = testQuote() - if err != nil { - log.Printf("Test quote in key: %v", err) - } - file, err := testLoadYAMLFile() - if err != nil { - log.Printf("Test loading YAML file %s: %v", file, err) - } - file, err = testLoadJSONFile() - if err != nil { - log.Printf("Test loading JSON file %s: %v", file, err) - } -} - -func testNewline() error { - spec := map[string]string{ - "a": "a", - "c\nc": "cc", - "d": "d", - } - y, err := yaml.Marshal(spec) - if err != nil { - return err - } - - //y, err := os.ReadFile(filename) - decoder := yaml.NewDecoder(bytes.NewReader(y)) - var out map[string]interface{} - for i := 0; ; i++ { - err = decoder.Decode(&out) - if err == io.EOF { - break - } - if err != nil { - return fmt.Errorf("Error on iteration %d: %v", i, err) - } - pretty.Print(i, out) - } - log.Printf("newline PASSED") - return nil -} - -func testBackslash() error { - spec := map[string]any{ - "outer": map[string]string{ - "b\\b": "b", - "d": "d", - }, - } - y, err := yaml.Marshal(spec) - if err != nil { - return err - } - - //y, err := os.ReadFile(filename) - decoder := yaml.NewDecoder(bytes.NewReader(y)) - var out map[string]interface{} - for i := 0; ; i++ { - err = decoder.Decode(&out) - if err == io.EOF { - break - } - if err != nil { - return fmt.Errorf("Error on iteration %d: %v", i, err) - } - } - log.Printf("backslash PASSED") - return nil -} - -func testQuote() error { - spec := `--- - outer: - "a\"b\"c": a - "d\"e\"f": d -` - decoder := yaml.NewDecoder(strings.NewReader(spec)) - var out map[string]interface{} - for i := 0; ; i++ { - err := decoder.Decode(&out) - if err == io.EOF { - break - } - if err != nil { - return fmt.Errorf("Error on iteration %d: %v", i, err) - } - } - log.Printf("quote PASSED") - return nil -} - -/** -bi3/dashboards/4u89zGTVk/dashboard-Vm10mGTVz.yaml <-- plain text/HTML?? -bi3/dashboards/7u49jPd7k/dashboard--49ZBkKnk.yaml <-- newline -bi3/dashboards/d40980f9-6366-458b-aac0-2c0a24e9fb6b/dashboard-b4e8bc14-9c16-4181-897c-e30631806d45.yaml <-- newline at beginning of expression - -**/ - -/* - "bi2/dashboards/WoVNxSA4k/dashboard-ae65738c-671b-4a51-b2e9-49d583b28793.yaml", //backslash - "bi2/dashboards/WoVNxSA4k/dashboard-cdc9172e-b37e-49fb-8e12-0e9ac806ea44.yaml", //backslash - "bi2/dashboards/WoVNxSA4k/dashboard-fcce96d1-a898-4726-97ff-59e70c80ad95.yaml", //backslash - "bi2/dashboards/QqKpDP0Vz/dashboard-e7be048f-625e-486b-a4d3-37a393fe834f.yaml", //backslash - "bi2/dashboards/FkdQ9Lsnz/dashboard-e2773f54-aebc-407a-8b48-2c7192db2921.yaml", //quote -*/ - -func testLoadYAMLFile() (string, error) { - file := "e85464d4-efcd-4351-ad68-3ccc935162fa.yaml" - log.Printf("Loading %s", file) - y, err := os.ReadFile(file) - if err != nil { - return file, err - } - - decoder := yaml.NewDecoder(bytes.NewReader(y)) - var out map[string]interface{} - for i := 0; ; i++ { - err = decoder.Decode(&out) - if err == io.EOF { - break - } - if err != nil { - return "", fmt.Errorf("Error on iteration %d: %v", i, err) - } - } - log.Printf("load %s file PASSED", file) - return file, nil -} - -func testLoadJSONFile() (string, error) { - spec := map[string]any{} - - file := "e2773f54-aebc-407a-8b48-2c7192db2921.json" - b, err := os.ReadFile(file) - if err != nil { - return file, err - } - err = json.Unmarshal(b, &spec) - - y, err := yaml.Marshal(spec) - if err != nil { - return file, err - } - - decoder := yaml.NewDecoder(bytes.NewReader(y)) - var out map[string]interface{} - for i := 0; ; i++ { - err = decoder.Decode(&out) - if err == io.EOF { - break - } - if err != nil { - return "", fmt.Errorf("Error on iteration %d: %v", i, err) - } - } - log.Printf("load %s file PASSED", file) - return file, nil -} diff --git a/decode_test.go b/decode_test.go index 2373804d..c8843269 100644 --- a/decode_test.go +++ b/decode_test.go @@ -2890,3 +2890,117 @@ func TestSameNameInineStruct(t *testing.T) { t.Fatalf("failed to decode") } } + +// Test newline in key as encoded by alternative YAML tools: +func TestUnmarshalNewlineInKey(t *testing.T) { + tests := []struct { + name string + spec string + }{ + { + name: "multiline-key-with-newline-01", + /* As rendered by this library from this JSON: + { + "a": "a", + "c\nc": "cc", + "d": "d" + } + */ + spec: `a: a +|- + c + c: cc +d: d +`, + }, + { + name: "quoted-key-with-newline", + spec: `a: a +"c\nc": "cc" +d: d +`, + }, + { + name: "multiline-key-with-newline-02", + // As encoded by alternative online YAML tools (e.g. https://www.json2yaml.com/) + spec: `a: a +? |- + c + c +: cc +d: d +`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + decoder := yaml.NewDecoder(strings.NewReader(test.spec)) + var out map[string]interface{} + for i := 0; ; i++ { + err := decoder.Decode(&out) + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Error on iteration %d: %v", i, err) + } + } + }) + } +} + +func TestUnmarshalBackslashInKey(t *testing.T) { + spec := `outer: + "b\\b": b + d : d +` + + decoder := yaml.NewDecoder(strings.NewReader(spec)) + var out map[string]interface{} + for i := 0; ; i++ { + err := decoder.Decode(&out) + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Error on iteration %d: %v", i, err) + } + } +} + +func TestUnmarshalQuotesInKey(t *testing.T) { + spec := `outer: + "a\"b\"c": a + "d\"e\"f": d +` + decoder := yaml.NewDecoder(strings.NewReader(spec)) + var out map[string]interface{} + for i := 0; ; i++ { + err := decoder.Decode(&out) + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Error on iteration %d: %v", i, err) + } + } +} + +func TestUnmarshalCollectedEscapesInKey(t *testing.T) { + spec := `outer: + "a\"b\"c\\d\ne": a + "d\"e\"f": d + ` + decoder := yaml.NewDecoder(strings.NewReader(spec)) + var out map[string]interface{} + for i := 0; ; i++ { + err := decoder.Decode(&out) + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Error on iteration %d: %v", i, err) + } + } +} diff --git a/encode_test.go b/encode_test.go index 3ff6f1c1..93b890c3 100644 --- a/encode_test.go +++ b/encode_test.go @@ -7,6 +7,7 @@ import ( "math" "reflect" "strconv" + "strings" "testing" "time" @@ -1630,3 +1631,35 @@ b: t.Fatalf("failed to encode. expected %s but got %s", expected, got) } } + +func TestMarshalNewlineInKey(t *testing.T) { + spec := map[string]string{ + "a": "a", + "c\nc": "cc", + "d": "d", + } + expected := []string{ + `a: a +"c\nc": "cc" +d: d +`, + `--- +a: a +? |- + c + c +: cc +d: d +`} + y, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("Error marshalling YAML: %v", err) + } + for _, exp := range expected { + if string(y) == exp { + return + } + } + + t.Fatalf("testNewlineEncode FAILED\nExpected: '%s'\nGot: '%s'", strings.Join(expected, "\nor\n"), string(y)) +}