From 2245c0089427aa848fc964aec7e7686c774bc10c Mon Sep 17 00:00:00 2001 From: John Gresty Date: Mon, 22 Jul 2024 15:34:04 +0100 Subject: [PATCH 1/8] feat: update go version 1.21 -> 1.22 I want to use refs in a loop, which behaves weirdly in 1.21 but they work properly in 1.22. The `exportloopref` linter detected the old behaviour which is not needed now it is fixed in 1.22. --- .circleci/config.yml | 4 ++-- .golangci.yml | 1 - Dockerfile | 2 +- go.mod | 2 +- shell.nix | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a5c32f0..0a0ff872 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ defaults: &defaults resource_class: small working_directory: ~/vervet docker: - - image: cimg/go:1.21-node + - image: cimg/go:1.22-node test_defaults: &test_defaults resource_class: medium @@ -54,7 +54,7 @@ jobs: command: npm install -g @stoplight/spectral@6.5.0 - checkout - go/install: - version: 1.21.3 + version: 1.22.3 - go/mod-download-cached - run: name: Verify testdata/output up to date diff --git a/.golangci.yml b/.golangci.yml index cf91fc4b..6f862f8f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -33,7 +33,6 @@ linters: - errname # - errorlint - Good to have linter but potential to introduce breaking changes. # - exhaustive - Causes too much noise at the moment. - - exportloopref - gci - gocritic - goconst diff --git a/Dockerfile b/Dockerfile index ae07f977..0a3dcc4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG GO_VERSION=1.21.3 +ARG GO_VERSION=1.22.3 ############### # Build stage # diff --git a/go.mod b/go.mod index 5b2c98f8..88406758 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/snyk/vervet/v7 -go 1.21 +go 1.22 require ( cloud.google.com/go/storage v1.34.1 diff --git a/shell.nix b/shell.nix index 07b7cd53..3fb57221 100644 --- a/shell.nix +++ b/shell.nix @@ -6,7 +6,7 @@ let }; in pkgs.mkShell { nativeBuildInputs = with pkgs.buildPackages; [ - go_1_21 + go_1_22 gopls gotools golangci-lint From 749416fb7bc055284866f460a014b76ec62df793 Mon Sep 17 00:00:00 2001 From: John Gresty Date: Mon, 22 Jul 2024 16:08:32 +0100 Subject: [PATCH 2/8] feat: add new codepath for building specs This codepath is an attempt to simplify the existing build process. It is a clean room implementation to avoid adopting any unnecessary steps from the old flow. Implemented as a new command to make it work alongside the old command. So far it only merges paths and not any other part of the spec. This will need to be worked on before it is suitable, however we can begin testing specs with this new command by releasing in progress versions. Eventually this will supplant the old flow when we are confident that it is feature complete. --- internal/cmd/cmd.go | 1 + internal/cmd/compiler.go | 29 +++ internal/simplebuild/build.go | 176 +++++++++++++++ internal/simplebuild/build_test.go | 330 +++++++++++++++++++++++++++++ 4 files changed, 536 insertions(+) create mode 100644 internal/simplebuild/build.go create mode 100644 internal/simplebuild/build_test.go diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 334a63b2..550f1e85 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -75,6 +75,7 @@ var CLIApp = cli.App{ Commands: []*cli.Command{ &BackstageCommand, &BuildCommand, + &SimpleBuildCommand, &FilterCommand, &GenerateCommand, &LocalizeCommand, diff --git a/internal/cmd/compiler.go b/internal/cmd/compiler.go index d1247878..380677ed 100644 --- a/internal/cmd/compiler.go +++ b/internal/cmd/compiler.go @@ -8,6 +8,7 @@ import ( "github.com/snyk/vervet/v7/config" "github.com/snyk/vervet/v7/internal/compiler" + "github.com/snyk/vervet/v7/internal/simplebuild" ) // BuildCommand is the `vervet build` subcommand. @@ -30,6 +31,34 @@ var BuildCommand = cli.Command{ Action: Build, } +var SimpleBuildCommand = cli.Command{ + Name: "simplebuild", + Usage: "Build versioned resources into versioned OpenAPI specs", + ArgsUsage: "[input resources root]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Aliases: []string{"c", "conf"}, + Usage: "Project configuration file", + }, + &cli.StringFlag{ + Name: "include", + Aliases: []string{"I"}, + Usage: "OpenAPI specification to include in build output", + }, + }, + Action: SimpleBuild, +} + +func SimpleBuild(ctx *cli.Context) error { + project, err := projectFromContext(ctx) + if err != nil { + return err + } + err = simplebuild.Build(ctx.Context, project) + return err +} + // Build compiles versioned resources into versioned API specs. func Build(ctx *cli.Context) error { project, err := projectFromContext(ctx) diff --git a/internal/simplebuild/build.go b/internal/simplebuild/build.go new file mode 100644 index 00000000..1e6dbea0 --- /dev/null +++ b/internal/simplebuild/build.go @@ -0,0 +1,176 @@ +package simplebuild + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/getkin/kin-openapi/openapi3" + + "github.com/snyk/vervet/v7" + "github.com/snyk/vervet/v7/config" + "github.com/snyk/vervet/v7/internal/files" +) + +func Build(ctx context.Context, project *config.Project) error { + for _, apiConfig := range project.APIs { + operations, err := LoadPaths(ctx, apiConfig) + if err != nil { + return err + } + docs, err := operations.Build() + if err != nil { + return err + } + err = docs.Write() + if err != nil { + return err + } + } + return nil +} + +type OpKey struct { + Path string + Method string +} + +type VersionedOp struct { + Version vervet.Version + Operation *openapi3.Operation +} + +type VersionSet []VersionedOp + +type Operations map[OpKey]VersionSet + +type VersionedDoc struct { + Version vervet.Version + Doc *openapi3.T +} +type DocSet []VersionedDoc + +func (ops Operations) Build() (DocSet, error) { + versionIndex := ops.Versions() + versions := versionIndex.Versions() + output := make(DocSet, len(versions)) + for idx, version := range versions { + output[idx] = VersionedDoc{ + Doc: &openapi3.T{}, + Version: version, + } + for path, spec := range ops { + op := spec.GetLatest(version) + if op == nil { + continue + } + output[idx].Doc.AddOperation(path.Path, path.Method, op) + } + } + return output, nil +} + +func (ops Operations) Versions() vervet.VersionIndex { + versionSet := map[vervet.Version]struct{}{} + for _, opSet := range ops { + for _, op := range opSet { + versionSet[op.Version] = struct{}{} + } + } + uniqueVersions := make(vervet.VersionSlice, len(versionSet)) + idx := 0 + for version := range versionSet { + uniqueVersions[idx] = version + idx++ + } + return vervet.NewVersionIndex(uniqueVersions) +} + +func (docs DocSet) Write() error { + for _, doc := range docs { + fmt.Println(doc.Version) + out, err := doc.Doc.MarshalJSON() + if err != nil { + return err + } + fmt.Println(string(out)) + } + return nil +} + +func LoadPaths(ctx context.Context, api *config.API) (Operations, error) { + operations := map[OpKey]VersionSet{} + + for _, resource := range api.Resources { + paths, err := ResourceSpecFiles(resource) + if err != nil { + return nil, err + } + for _, path := range paths { + versionDir := filepath.Dir(path) + versionStr := filepath.Base(versionDir) + + doc, err := vervet.NewDocumentFile(path) + if err != nil { + return nil, fmt.Errorf("failed to load spec from %q: %w", path, err) + } + + stabilityStr, err := vervet.ExtensionString(doc.T.Extensions, vervet.ExtSnykApiStability) + if err != nil { + return nil, err + } + if stabilityStr != "ga" { + versionStr = fmt.Sprintf("%s~%s", versionStr, stabilityStr) + } + version, err := vervet.ParseVersion(versionStr) + if err != nil { + return nil, fmt.Errorf("invalid version %q", versionStr) + } + + doc.InternalizeRefs(ctx, nil) + err = doc.ResolveRefs() + if err != nil { + return nil, fmt.Errorf("failed to localize refs: %w", err) + } + + for pathName, pathDef := range doc.T.Paths { + for opName, opDef := range pathDef.Operations() { + k := OpKey{ + Path: pathName, + Method: opName, + } + if operations[k] == nil { + operations[k] = []VersionedOp{} + } + operations[k] = append(operations[k], VersionedOp{ + Version: version, + Operation: opDef, + }) + } + } + } + } + + return operations, nil +} + +func ResourceSpecFiles(resource *config.ResourceSet) ([]string, error) { + return files.LocalFSSource{}.Match(resource) +} + +func (vs VersionSet) GetLatest(before vervet.Version) *openapi3.Operation { + var latest *VersionedOp + for _, versionedOp := range vs { + isBefore := versionedOp.Version.Compare(before) <= 0 + isLowerStability := versionedOp.Version.Stability.Compare(before.Stability) < 0 + if isBefore && !isLowerStability { + if latest == nil || versionedOp.Version.Compare(latest.Version) > 0 { + latest = &versionedOp + } + } + } + if latest == nil { + return nil + } + return latest.Operation +} diff --git a/internal/simplebuild/build_test.go b/internal/simplebuild/build_test.go new file mode 100644 index 00000000..c4f5f077 --- /dev/null +++ b/internal/simplebuild/build_test.go @@ -0,0 +1,330 @@ +package simplebuild_test + +import ( + "testing" + "time" + + qt "github.com/frankban/quicktest" + "github.com/getkin/kin-openapi/openapi3" + + "github.com/snyk/vervet/v7" + "github.com/snyk/vervet/v7/internal/simplebuild" +) + +func TestGetLatest(t *testing.T) { + c := qt.New(t) + + c.Run("gets the latest version", func(c *qt.C) { + before := vervet.MustParseVersion("3000-01-01") + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01"), + Operation: openapi3.NewOperation(), + }, + } + op := vs.GetLatest(before) + c.Assert(op, qt.Equals, vs[1].Operation) + }) + + c.Run("filters to before given date", func(c *qt.C) { + before := vervet.MustParseVersion("2024-02-15") + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01"), + Operation: openapi3.NewOperation(), + }, + } + op := vs.GetLatest(before) + c.Assert(op, qt.Equals, vs[2].Operation) + }) + + c.Run("ignores lower stabilities", func(c *qt.C) { + before := vervet.MustParseVersion("2024-06-01~beta") + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01~beta"), + Operation: openapi3.NewOperation(), + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-04-01~experimental"), + Operation: openapi3.NewOperation(), + }, + } + op := vs.GetLatest(before) + c.Assert(op, qt.Equals, vs[2].Operation) + }) + + c.Run("ignores lower stability", func(c *qt.C) { + before := vervet.MustParseVersion("2024-06-01") + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-05-01~beta"), + Operation: openapi3.NewOperation(), + }, + } + op := vs.GetLatest(before) + c.Assert(op, qt.IsNil) + }) +} + +func TestBuild(t *testing.T) { + c := qt.New(t) + + c.Run("copies paths to output", func(c *qt.C) { + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + c.Assert(output[0].Version.Date, qt.Equals, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + c.Assert(output[0].Doc.Paths["/foo"].Get, qt.IsNotNil) + }) + + c.Run("merges operations from the same version", func(c *qt.C) { + version := vervet.MustParseVersion("2024-01-01") + + getFoo := openapi3.NewOperation() + postFoo := openapi3.NewOperation() + getBar := openapi3.NewOperation() + + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: version, + Operation: getFoo, + }}, + simplebuild.OpKey{ + Path: "/foo", + Method: "POST", + }: simplebuild.VersionSet{{ + Version: version, + Operation: postFoo, + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: version, + Operation: getBar, + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + c.Assert(output[0].Version, qt.Equals, version) + c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[0].Doc.Paths["/foo"].Post, qt.Equals, postFoo) + c.Assert(output[0].Doc.Paths["/bar"].Get, qt.Equals, getBar) + }) + + c.Run("generates an output per unique version", func(c *qt.C) { + versions := []vervet.Version{ + vervet.MustParseVersion("2024-01-01"), + vervet.MustParseVersion("2024-01-02"), + vervet.MustParseVersion("2024-01-03"), + } + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: versions[0], + Operation: openapi3.NewOperation(), + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: versions[1], + Operation: openapi3.NewOperation(), + }, { + Version: versions[2], + Operation: openapi3.NewOperation(), + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + + outputVersions := make([]vervet.Version, len(output)) + for idx, out := range output { + outputVersions[idx] = out.Version + } + c.Assert(outputVersions, qt.DeepEquals, versions) + }) + + c.Run("merges distinct operations from previous versions", func(c *qt.C) { + versionA := vervet.MustParseVersion("2024-01-01") + versionB := vervet.MustParseVersion("2024-01-02") + versionC := vervet.MustParseVersion("2024-01-03") + + getFoo := openapi3.NewOperation() + postFoo := openapi3.NewOperation() + getBar := openapi3.NewOperation() + + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: versionA, + Operation: getFoo, + }}, + simplebuild.OpKey{ + Path: "/foo", + Method: "POST", + }: simplebuild.VersionSet{{ + Version: versionB, + Operation: postFoo, + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: versionC, + Operation: getBar, + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + + c.Assert(output[0].Version, qt.Equals, versionA) + c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[0].Doc.Paths["/foo"].Post, qt.IsNil) + c.Assert(output[0].Doc.Paths["/bar"], qt.IsNil) + + c.Assert(output[1].Version, qt.Equals, versionB) + c.Assert(output[1].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[1].Doc.Paths["/foo"].Post, qt.Equals, postFoo) + c.Assert(output[1].Doc.Paths["/bar"], qt.IsNil) + + c.Assert(output[2].Version, qt.Equals, versionC) + c.Assert(output[2].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[2].Doc.Paths["/foo"].Post, qt.Equals, postFoo) + c.Assert(output[2].Doc.Paths["/bar"].Get, qt.Equals, getBar) + }) + + c.Run("resolves operations to latest version with respect to output", func(c *qt.C) { + versionA := vervet.MustParseVersion("2024-01-01") + versionB := vervet.MustParseVersion("2024-01-02") + versionC := vervet.MustParseVersion("2024-01-03") + + getFooOld := openapi3.NewOperation() + getFooNew := openapi3.NewOperation() + getBar := openapi3.NewOperation() + + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: versionA, + Operation: getFooOld, + }, { + Version: versionC, + Operation: getFooNew, + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: versionB, + Operation: getBar, + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + + c.Assert(output[0].Version, qt.Equals, versionA) + c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFooOld) + c.Assert(output[0].Doc.Paths["/bar"], qt.IsNil) + + c.Assert(output[1].Version, qt.Equals, versionB) + c.Assert(output[1].Doc.Paths["/foo"].Get, qt.Equals, getFooOld) + c.Assert(output[1].Doc.Paths["/bar"].Get, qt.Equals, getBar) + + c.Assert(output[2].Version, qt.Equals, versionC) + c.Assert(output[2].Doc.Paths["/foo"].Get, qt.Equals, getFooNew) + c.Assert(output[2].Doc.Paths["/bar"].Get, qt.Equals, getBar) + }) + + c.Run("lower stabilities are not merged into higher", func(c *qt.C) { + versionBetaA := vervet.MustParseVersion("2024-01-01~beta") + versionGA := vervet.MustParseVersion("2024-01-02") + versionBetaB := vervet.MustParseVersion("2024-01-03~beta") + + getFoo := openapi3.NewOperation() + postFoo := openapi3.NewOperation() + getBar := openapi3.NewOperation() + + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: versionGA, + Operation: getFoo, + }}, + simplebuild.OpKey{ + Path: "/foo", + Method: "POST", + }: simplebuild.VersionSet{{ + Version: versionBetaB, + Operation: postFoo, + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{{ + Version: versionBetaA, + Operation: getBar, + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + + c.Assert(output[0].Version, qt.Equals, versionBetaA) + c.Assert(output[0].Doc.Paths["/foo"], qt.IsNil) + c.Assert(output[0].Doc.Paths["/bar"].Get, qt.Equals, getBar) + + c.Assert(output[1].Version, qt.Equals, versionGA) + c.Assert(output[1].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[1].Doc.Paths["/foo"].Post, qt.IsNil) + c.Assert(output[1].Doc.Paths["/bar"], qt.IsNil) + + c.Assert(output[2].Version, qt.Equals, versionBetaB) + c.Assert(output[2].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[2].Doc.Paths["/foo"].Post, qt.Equals, postFoo) + c.Assert(output[2].Doc.Paths["/bar"].Get, qt.Equals, getBar) + }) +} From dff66eb271d35968d0e2d8c638e4b9afbc0a9ae1 Mon Sep 17 00:00:00 2001 From: John Gresty Date: Thu, 25 Jul 2024 10:37:59 +0100 Subject: [PATCH 3/8] feat: roll up lower stabilities in simplebuild This is a change to how we are handling stabilities. Instead of generating an api spec for each stability at each version, we are only going to generate a single spec for each version date. Any endpoints which do not have a GA version will be carried through to the latest spec but marked as their original stability. Any endpoint, once a higher stability is released, cannot then go backwards in stability. For example a new beta version cannot be released after a ga version for any specific endpoint. Once an endpoint has hit ga status, all future versions must also be ga. --- internal/simplebuild/build.go | 51 +++++++++++--------- internal/simplebuild/build_test.go | 76 +++++++++++++++++------------- 2 files changed, 72 insertions(+), 55 deletions(-) diff --git a/internal/simplebuild/build.go b/internal/simplebuild/build.go index 1e6dbea0..d6185d4e 100644 --- a/internal/simplebuild/build.go +++ b/internal/simplebuild/build.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "time" "github.com/getkin/kin-openapi/openapi3" @@ -45,22 +46,21 @@ type VersionSet []VersionedOp type Operations map[OpKey]VersionSet type VersionedDoc struct { - Version vervet.Version - Doc *openapi3.T + VersionDate time.Time + Doc *openapi3.T } type DocSet []VersionedDoc func (ops Operations) Build() (DocSet, error) { - versionIndex := ops.Versions() - versions := versionIndex.Versions() - output := make(DocSet, len(versions)) - for idx, version := range versions { + versionDates := ops.VersionDates() + output := make(DocSet, len(versionDates)) + for idx, versionDate := range versionDates { output[idx] = VersionedDoc{ - Doc: &openapi3.T{}, - Version: version, + Doc: &openapi3.T{}, + VersionDate: versionDate, } for path, spec := range ops { - op := spec.GetLatest(version) + op := spec.GetLatest(versionDate) if op == nil { continue } @@ -70,25 +70,25 @@ func (ops Operations) Build() (DocSet, error) { return output, nil } -func (ops Operations) Versions() vervet.VersionIndex { - versionSet := map[vervet.Version]struct{}{} +func (ops Operations) VersionDates() []time.Time { + versionSet := map[time.Time]struct{}{} for _, opSet := range ops { for _, op := range opSet { - versionSet[op.Version] = struct{}{} + versionSet[op.Version.Date] = struct{}{} } } - uniqueVersions := make(vervet.VersionSlice, len(versionSet)) + uniqueVersions := make([]time.Time, len(versionSet)) idx := 0 for version := range versionSet { uniqueVersions[idx] = version idx++ } - return vervet.NewVersionIndex(uniqueVersions) + return uniqueVersions } func (docs DocSet) Write() error { for _, doc := range docs { - fmt.Println(doc.Version) + fmt.Println(doc.VersionDate.Format(time.DateOnly)) out, err := doc.Doc.MarshalJSON() if err != nil { return err @@ -158,15 +158,22 @@ func ResourceSpecFiles(resource *config.ResourceSet) ([]string, error) { return files.LocalFSSource{}.Match(resource) } -func (vs VersionSet) GetLatest(before vervet.Version) *openapi3.Operation { +func (vs VersionSet) GetLatest(before time.Time) *openapi3.Operation { var latest *VersionedOp for _, versionedOp := range vs { - isBefore := versionedOp.Version.Compare(before) <= 0 - isLowerStability := versionedOp.Version.Stability.Compare(before.Stability) < 0 - if isBefore && !isLowerStability { - if latest == nil || versionedOp.Version.Compare(latest.Version) > 0 { - latest = &versionedOp - } + if versionedOp.Version.Date.After(before) { + continue + } + if latest == nil { + latest = &versionedOp + continue + } + // Higher stabilities always take precedent + if versionedOp.Version.Stability.Compare(latest.Version.Stability) < 0 { + continue + } + if versionedOp.Version.Compare(latest.Version) > 0 { + latest = &versionedOp } } if latest == nil { diff --git a/internal/simplebuild/build_test.go b/internal/simplebuild/build_test.go index c4f5f077..61342ac7 100644 --- a/internal/simplebuild/build_test.go +++ b/internal/simplebuild/build_test.go @@ -1,6 +1,7 @@ package simplebuild_test import ( + "slices" "testing" "time" @@ -30,7 +31,7 @@ func TestGetLatest(t *testing.T) { Operation: openapi3.NewOperation(), }, } - op := vs.GetLatest(before) + op := vs.GetLatest(before.Date) c.Assert(op, qt.Equals, vs[1].Operation) }) @@ -50,12 +51,12 @@ func TestGetLatest(t *testing.T) { Operation: openapi3.NewOperation(), }, } - op := vs.GetLatest(before) + op := vs.GetLatest(before.Date) c.Assert(op, qt.Equals, vs[2].Operation) }) - c.Run("ignores lower stabilities", func(c *qt.C) { - before := vervet.MustParseVersion("2024-06-01~beta") + c.Run("selects highest stability", func(c *qt.C) { + before := vervet.MustParseVersion("2024-06-01") vs := simplebuild.VersionSet{ simplebuild.VersionedOp{ Version: vervet.MustParseVersion("2024-01-01"), @@ -74,21 +75,9 @@ func TestGetLatest(t *testing.T) { Operation: openapi3.NewOperation(), }, } - op := vs.GetLatest(before) + op := vs.GetLatest(before.Date) c.Assert(op, qt.Equals, vs[2].Operation) }) - - c.Run("ignores lower stability", func(c *qt.C) { - before := vervet.MustParseVersion("2024-06-01") - vs := simplebuild.VersionSet{ - simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-05-01~beta"), - Operation: openapi3.NewOperation(), - }, - } - op := vs.GetLatest(before) - c.Assert(op, qt.IsNil) - }) } func TestBuild(t *testing.T) { @@ -106,7 +95,7 @@ func TestBuild(t *testing.T) { } output, err := ops.Build() c.Assert(err, qt.IsNil) - c.Assert(output[0].Version.Date, qt.Equals, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + c.Assert(output[0].VersionDate, qt.Equals, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) c.Assert(output[0].Doc.Paths["/foo"].Get, qt.IsNotNil) }) @@ -142,7 +131,7 @@ func TestBuild(t *testing.T) { } output, err := ops.Build() c.Assert(err, qt.IsNil) - c.Assert(output[0].Version, qt.Equals, version) + c.Assert(output[0].VersionDate, qt.Equals, version.Date) c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFoo) c.Assert(output[0].Doc.Paths["/foo"].Post, qt.Equals, postFoo) c.Assert(output[0].Doc.Paths["/bar"].Get, qt.Equals, getBar) @@ -176,11 +165,19 @@ func TestBuild(t *testing.T) { output, err := ops.Build() c.Assert(err, qt.IsNil) - outputVersions := make([]vervet.Version, len(output)) + inputVersions := make([]time.Time, len(versions)) + for idx, in := range versions { + inputVersions[idx] = in.Date + } + slices.SortFunc(inputVersions, compareDates) + + outputVersions := make([]time.Time, len(output)) for idx, out := range output { - outputVersions[idx] = out.Version + outputVersions[idx] = out.VersionDate } - c.Assert(outputVersions, qt.DeepEquals, versions) + slices.SortFunc(outputVersions, compareDates) + + c.Assert(outputVersions, qt.DeepEquals, inputVersions) }) c.Run("merges distinct operations from previous versions", func(c *qt.C) { @@ -218,17 +215,19 @@ func TestBuild(t *testing.T) { output, err := ops.Build() c.Assert(err, qt.IsNil) - c.Assert(output[0].Version, qt.Equals, versionA) + slices.SortFunc(output, compareDocs) + + c.Assert(output[0].VersionDate, qt.Equals, versionA.Date) c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFoo) c.Assert(output[0].Doc.Paths["/foo"].Post, qt.IsNil) c.Assert(output[0].Doc.Paths["/bar"], qt.IsNil) - c.Assert(output[1].Version, qt.Equals, versionB) + c.Assert(output[1].VersionDate, qt.Equals, versionB.Date) c.Assert(output[1].Doc.Paths["/foo"].Get, qt.Equals, getFoo) c.Assert(output[1].Doc.Paths["/foo"].Post, qt.Equals, postFoo) c.Assert(output[1].Doc.Paths["/bar"], qt.IsNil) - c.Assert(output[2].Version, qt.Equals, versionC) + c.Assert(output[2].VersionDate, qt.Equals, versionC.Date) c.Assert(output[2].Doc.Paths["/foo"].Get, qt.Equals, getFoo) c.Assert(output[2].Doc.Paths["/foo"].Post, qt.Equals, postFoo) c.Assert(output[2].Doc.Paths["/bar"].Get, qt.Equals, getBar) @@ -265,20 +264,22 @@ func TestBuild(t *testing.T) { output, err := ops.Build() c.Assert(err, qt.IsNil) - c.Assert(output[0].Version, qt.Equals, versionA) + slices.SortFunc(output, compareDocs) + + c.Assert(output[0].VersionDate, qt.Equals, versionA.Date) c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFooOld) c.Assert(output[0].Doc.Paths["/bar"], qt.IsNil) - c.Assert(output[1].Version, qt.Equals, versionB) + c.Assert(output[1].VersionDate, qt.Equals, versionB.Date) c.Assert(output[1].Doc.Paths["/foo"].Get, qt.Equals, getFooOld) c.Assert(output[1].Doc.Paths["/bar"].Get, qt.Equals, getBar) - c.Assert(output[2].Version, qt.Equals, versionC) + c.Assert(output[2].VersionDate, qt.Equals, versionC.Date) c.Assert(output[2].Doc.Paths["/foo"].Get, qt.Equals, getFooNew) c.Assert(output[2].Doc.Paths["/bar"].Get, qt.Equals, getBar) }) - c.Run("lower stabilities are not merged into higher", func(c *qt.C) { + c.Run("lower stabilities are merged into higher", func(c *qt.C) { versionBetaA := vervet.MustParseVersion("2024-01-01~beta") versionGA := vervet.MustParseVersion("2024-01-02") versionBetaB := vervet.MustParseVersion("2024-01-03~beta") @@ -313,18 +314,27 @@ func TestBuild(t *testing.T) { output, err := ops.Build() c.Assert(err, qt.IsNil) - c.Assert(output[0].Version, qt.Equals, versionBetaA) + slices.SortFunc(output, compareDocs) + + c.Assert(output[0].VersionDate, qt.Equals, versionBetaA.Date) c.Assert(output[0].Doc.Paths["/foo"], qt.IsNil) c.Assert(output[0].Doc.Paths["/bar"].Get, qt.Equals, getBar) - c.Assert(output[1].Version, qt.Equals, versionGA) + c.Assert(output[1].VersionDate, qt.Equals, versionGA.Date) c.Assert(output[1].Doc.Paths["/foo"].Get, qt.Equals, getFoo) c.Assert(output[1].Doc.Paths["/foo"].Post, qt.IsNil) - c.Assert(output[1].Doc.Paths["/bar"], qt.IsNil) + c.Assert(output[0].Doc.Paths["/bar"].Get, qt.Equals, getBar) - c.Assert(output[2].Version, qt.Equals, versionBetaB) + c.Assert(output[2].VersionDate, qt.Equals, versionBetaB.Date) c.Assert(output[2].Doc.Paths["/foo"].Get, qt.Equals, getFoo) c.Assert(output[2].Doc.Paths["/foo"].Post, qt.Equals, postFoo) c.Assert(output[2].Doc.Paths["/bar"].Get, qt.Equals, getBar) }) } + +func compareDocs(a, b simplebuild.VersionedDoc) int { + return a.VersionDate.Compare(b.VersionDate) +} +func compareDates(a, b time.Time) int { + return a.Compare(b) +} From ceab3422ed60343c5a1d22458f192c59c9759743 Mon Sep 17 00:00:00 2001 From: John Gresty Date: Fri, 26 Jul 2024 11:27:20 +0100 Subject: [PATCH 4/8] feat: include component refs in simplebuild Refs are an OpenAPI concept where you can define part of a spec then use a JSON reference [https://datatracker.ietf.org/doc/html/draft-pbryan-zyp-json-ref-03] to include that block in another part of the document. These are used extensively at Snyk so we need to make sure that we include the referenced section in the compiled specs. Note that this does not check for conflicts, so components that reference the same path but in different source documents will overwrite each other silently. --- internal/simplebuild/build.go | 5 ++ internal/simplebuild/refs.go | 140 +++++++++++++++++++++++++++++ internal/simplebuild/refs_test.go | 144 ++++++++++++++++++++++++++++++ 3 files changed, 289 insertions(+) create mode 100644 internal/simplebuild/refs.go create mode 100644 internal/simplebuild/refs_test.go diff --git a/internal/simplebuild/build.go b/internal/simplebuild/build.go index d6185d4e..dfdaf31d 100644 --- a/internal/simplebuild/build.go +++ b/internal/simplebuild/build.go @@ -59,12 +59,17 @@ func (ops Operations) Build() (DocSet, error) { Doc: &openapi3.T{}, VersionDate: versionDate, } + refResolver := NewRefResolver(output[idx].Doc) for path, spec := range ops { op := spec.GetLatest(versionDate) if op == nil { continue } output[idx].Doc.AddOperation(path.Path, path.Method, op) + err := refResolver.Resolve(op) + if err != nil { + return nil, err + } } } return output, nil diff --git a/internal/simplebuild/refs.go b/internal/simplebuild/refs.go new file mode 100644 index 00000000..7fd7b331 --- /dev/null +++ b/internal/simplebuild/refs.go @@ -0,0 +1,140 @@ +package simplebuild + +import ( + "fmt" + "reflect" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/mitchellh/reflectwalk" +) + +// Refs are an OpenAPI concept where you can define part of a spec then use a +// JSON reference +// [https://datatracker.ietf.org/doc/html/draft-pbryan-zyp-json-ref-03] to +// include that block in another part of the document. +// +// For example a component might live at the top level then that is consumed +// elsewhere: +// +// components: +// +// parameters: +// foo: +// name: fooparam +// in: query +// +// paths: +// +// /foo: +// parameters: +// - $ref: "#/components/parameters/foo" +// +// openapi3 has several *Ref types which have Ref and Value fields, the Ref +// field is the string from the original document and Value is the block it +// points to if the ref is resolved, hen loading our documents we do this +// resolution to populate all Value fields. +// +// When serialising an openapi3.*Ref, if the Ref field is set then the Value +// field is ignored. Therefore we have two options, either add the components +// back into the document at the appropriate position or inline them. As some +// components are likely to be reused several times, we elect to do the former +// where possible. +// +// This class walks a given object and recursively copy any refs it finds back +// into the document at the path they are referenced from. +type refResolver struct { + doc *openapi3.T +} + +func NewRefResolver(doc *openapi3.T) refResolver { + return refResolver{doc: doc} +} + +func (rr *refResolver) Resolve(from any) error { + return reflectwalk.Walk(from, rr) +} + +// Implements reflectwalk.StructWalker. This function is called for every +// struct found when walking. +func (rr *refResolver) Struct(v reflect.Value) error { + ref := v.FieldByName("Ref") + value := v.FieldByName("Value") + if !ref.IsValid() || !value.IsValid() { + // This isn't a openapi3.*Ref so nothing to do + return nil + } + refLoc := ref.String() + if refLoc == "" { + // This ref has been inlined + return nil + } + // Create a new *Ref object to avoid mutating the original + derefed := reflect.New(v.Type()) + reflect.Indirect(derefed).FieldByName("Value").Set(value) + + return rr.deref(refLoc, derefed) +} + +// Implements reflectwalk.StructWalker. We work on whole structs so there is +// nothing to do here. +func (rr *refResolver) StructField(sf reflect.StructField, v reflect.Value) error { + return nil +} + +func (rr *refResolver) deref(ref string, value reflect.Value) error { + path := strings.Split(ref, "/") + if path[0] != "#" { + // All refs should have been resolved to the local document when + // loading so if we hit this case then we have not loaded the document + // correctly. + return fmt.Errorf("external ref %s", ref) + } + field := reflect.ValueOf(rr.doc) + // Need to forward declare err so field is not shadowed in the loop + var err error + for _, segment := range path[1:] { + // Maps are a special case since the key also needs to be created. + if field.Kind() == reflect.Map { + newValue := reflect.New(field.Type().Elem().Elem()) + field.SetMapIndex(reflect.ValueOf(segment), newValue) + field = newValue.Elem() + continue + } + // else we assume we are working on a struct + field, err = getField(segment, field) + if err != nil { + return fmt.Errorf("invalid ref %s: %w", ref, err) + } + + // A lot of the openapi3.T fields are pointers so if this is the first + // time we have encountered an object of this type we need to create + // the container. + if field.Kind() == reflect.Map && field.IsZero() { + newValue := reflect.MakeMap(field.Type()) + field.Set(newValue) + } else if field.IsNil() { + newValue := reflect.New(field.Type().Elem()) + field.Set(newValue) + } + } + field.Set(value.Elem()) + return nil +} + +func getField(tag string, object reflect.Value) (reflect.Value, error) { + reflectedObject := object.Type().Elem() + if reflectedObject.Kind() != reflect.Struct { + return reflect.Value{}, fmt.Errorf("object is not a struct") + } + for idx := 0; idx < reflectedObject.NumField(); idx++ { + structField := reflectedObject.Field(idx) + yamlTag := structField.Tag.Get("yaml") + // Remove tag options (eg "omitempty") + yamlField := strings.SplitN(yamlTag, ",", 2)[0] + if yamlField == tag { + return object.Elem().FieldByName(structField.Name), nil + } + } + return reflect.Value{}, fmt.Errorf("field %s not found on object", tag) +} diff --git a/internal/simplebuild/refs_test.go b/internal/simplebuild/refs_test.go new file mode 100644 index 00000000..2ebade08 --- /dev/null +++ b/internal/simplebuild/refs_test.go @@ -0,0 +1,144 @@ +package simplebuild_test + +import ( + "testing" + + qt "github.com/frankban/quicktest" + "github.com/getkin/kin-openapi/openapi3" + + "github.com/snyk/vervet/v7/internal/simplebuild" +) + +func TestResolveRefs(t *testing.T) { + c := qt.New(t) + + c.Run("copies ref value into referenced location", func(c *qt.C) { + param := &openapi3.Parameter{} + path := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/foo", + Value: param, + }}, + } + doc := openapi3.T{ + Paths: openapi3.Paths{ + "/foo": &path, + }, + } + + rr := simplebuild.NewRefResolver(&doc) + err := rr.Resolve(path) + c.Assert(err, qt.IsNil) + + c.Assert(doc.Components.Parameters["foo"].Value, qt.Equals, param) + }) + + c.Run("ignores refs on other parts of the doc", func(c *qt.C) { + param := &openapi3.Parameter{} + pathA := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/foo", + Value: param, + }}, + } + pathB := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/bar", + Value: param, + }}, + } + doc := openapi3.T{ + Paths: openapi3.Paths{ + "/foo": &pathA, + "/bar": &pathB, + }, + } + + rr := simplebuild.NewRefResolver(&doc) + err := rr.Resolve(pathA) + c.Assert(err, qt.IsNil) + + c.Assert(doc.Components.Parameters["bar"], qt.IsNil) + }) + + c.Run("merges refs from successive calls", func(c *qt.C) { + paramA := &openapi3.Parameter{} + pathA := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/foo", + Value: paramA, + }}, + } + paramB := &openapi3.Parameter{} + pathB := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/bar", + Value: paramB, + }}, + } + doc := openapi3.T{ + Paths: openapi3.Paths{ + "/foo": &pathA, + "/bar": &pathB, + }, + } + + rr := simplebuild.NewRefResolver(&doc) + err := rr.Resolve(pathA) + c.Assert(err, qt.IsNil) + err = rr.Resolve(pathB) + c.Assert(err, qt.IsNil) + + c.Assert(doc.Components.Parameters["foo"].Value, qt.Equals, paramA) + c.Assert(doc.Components.Parameters["bar"].Value, qt.Equals, paramB) + }) + + c.Run("recursively resolves components", func(c *qt.C) { + schema := &openapi3.Schema{} + param := &openapi3.Parameter{ + Schema: &openapi3.SchemaRef{ + Ref: "#/components/schemas/foo", + Value: schema, + }, + } + path := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/foo", + Value: param, + }}, + } + doc := openapi3.T{ + Paths: openapi3.Paths{ + "/foo": &path, + }, + } + + rr := simplebuild.NewRefResolver(&doc) + err := rr.Resolve(path) + c.Assert(err, qt.IsNil) + + c.Assert(doc.Components.Parameters["foo"].Value, qt.Equals, param) + c.Assert(doc.Components.Schemas["foo"].Value, qt.Equals, schema) + }) + + c.Run("ignores ref objects with no ref value", func(c *qt.C) { + param := &openapi3.Parameter{} + path := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Value: param, + }}, + } + doc := openapi3.T{ + Components: &openapi3.Components{}, + Paths: openapi3.Paths{ + "/foo": &path, + }, + } + + rr := simplebuild.NewRefResolver(&doc) + err := rr.Resolve(path) + c.Assert(err, qt.IsNil) + + c.Assert(doc.Components.Parameters["foo"], qt.IsNil) + }) +} From 4ae329cd9d317efb3338c2ac63d50c3606a3878d Mon Sep 17 00:00:00 2001 From: John Gresty Date: Wed, 31 Jul 2024 07:01:25 +0100 Subject: [PATCH 5/8] feat: write outputs in simplebuild Mimics the behaviour of build; writes the compiled specs to the directories defined in the api configs outputs in json and yaml formats along with an embed.go. --- internal/compiler/compiler.go | 4 +- internal/simplebuild/build.go | 23 ++---- internal/simplebuild/output.go | 134 +++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 17 deletions(-) create mode 100644 internal/simplebuild/output.go diff --git a/internal/compiler/compiler.go b/internal/compiler/compiler.go index 9dc36f19..f08d5fcf 100644 --- a/internal/compiler/compiler.go +++ b/internal/compiler/compiler.go @@ -262,7 +262,7 @@ func (c *Compiler) writeEmbedGo(pkgName string, a *api, versionSpecFiles []strin return err } defer f.Close() - err = embedGoTmpl.Execute(f, struct { + err = EmbedGoTmpl.Execute(f, struct { Package string API *api VersionSpecFiles []string @@ -282,7 +282,7 @@ func (c *Compiler) writeEmbedGo(pkgName string, a *api, versionSpecFiles []strin return nil } -var embedGoTmpl = template.Must(template.New("embed.go").Parse(` +var EmbedGoTmpl = template.Must(template.New("embed.go").Parse(` // Code generated by Vervet. DO NOT EDIT. package {{ .Package }} diff --git a/internal/simplebuild/build.go b/internal/simplebuild/build.go index dfdaf31d..2346a9b7 100644 --- a/internal/simplebuild/build.go +++ b/internal/simplebuild/build.go @@ -23,9 +23,14 @@ func Build(ctx context.Context, project *config.Project) error { if err != nil { return err } - err = docs.Write() - if err != nil { - return err + + // TODO: apply overlays + + if apiConfig.Output != nil { + err = docs.WriteOutputs(*apiConfig.Output) + if err != nil { + return err + } } } return nil @@ -91,18 +96,6 @@ func (ops Operations) VersionDates() []time.Time { return uniqueVersions } -func (docs DocSet) Write() error { - for _, doc := range docs { - fmt.Println(doc.VersionDate.Format(time.DateOnly)) - out, err := doc.Doc.MarshalJSON() - if err != nil { - return err - } - fmt.Println(string(out)) - } - return nil -} - func LoadPaths(ctx context.Context, api *config.API) (Operations, error) { operations := map[OpKey]VersionSet{} diff --git a/internal/simplebuild/output.go b/internal/simplebuild/output.go new file mode 100644 index 00000000..4a5981be --- /dev/null +++ b/internal/simplebuild/output.go @@ -0,0 +1,134 @@ +package simplebuild + +import ( + "fmt" + "os" + "path" + "path/filepath" + "time" + + "github.com/ghodss/yaml" + + "github.com/snyk/vervet/v7" + "github.com/snyk/vervet/v7/config" + "github.com/snyk/vervet/v7/internal/compiler" + "github.com/snyk/vervet/v7/internal/files" +) + +// Some services have a need to write specs to multiple destinations. This +// tends to happen in Typescript services in which we want to write specs to +// two places: +// - src/** for committing into git and ingesting into Backstage +// - dist/** for runtime module access to compiled specs. +// +// To maintain backwards compatibility we still allow a single path in the +// config file then normalise that here to an array. +func getOutputPaths(cfg config.Output) []string { + paths := cfg.Paths + if len(paths) == 0 && cfg.Path != "" { + paths = []string{cfg.Path} + } + return paths +} + +// WriteOutputs writes compiled specs to all directories specified by the given +// api config. Removes any existing builds if they are present. +func (docs DocSet) WriteOutputs(cfg config.Output) error { + paths := getOutputPaths(cfg) + + for _, dir := range paths { + err := os.RemoveAll(dir) + if err != nil { + return fmt.Errorf("clear output directory: %w", err) + } + } + + err := docs.Write(paths[0]) + if err != nil { + return fmt.Errorf("write output files: %w", err) + } + + for _, dir := range paths[1:] { + err := files.CopyDir(dir, paths[0], true) + if err != nil { + return fmt.Errorf("copy outputs: %w", err) + } + } + + return nil +} + +// Write writes compiled specs to a single directory in YAML and JSON formats. +// Unlike WriteOutputs this function assumes the destination directory does not +// already exist. +func (docs DocSet) Write(dir string) error { + err := os.MkdirAll(dir, 0777) + if err != nil { + return err + } + + versionSpecFiles := make([]string, len(docs)*2) + for idx, doc := range docs { + versionDir := path.Join(dir, doc.VersionDate.Format(time.DateOnly)) + err = os.MkdirAll(versionDir, 0755) + if err != nil { + return fmt.Errorf("make output directory: %w", err) + } + + jsonBuf, err := vervet.ToSpecJSON(doc.Doc) + if err != nil { + return fmt.Errorf("serialise spec to json: %w", err) + } + jsonSpecPath := path.Join(versionDir, "spec.json") + jsonEmbedPath, err := filepath.Rel(dir, jsonSpecPath) + if err != nil { + return fmt.Errorf("get relative output path: %w", err) + } + versionSpecFiles[idx*2] = jsonEmbedPath + err = os.WriteFile(jsonSpecPath, jsonBuf, 0644) + if err != nil { + return fmt.Errorf("write json file: %w", err) + } + fmt.Println(jsonSpecPath) + + yamlBuf, err := yaml.JSONToYAML(jsonBuf) + if err != nil { + return fmt.Errorf("convert spec to yaml: %w", err) + } + yamlBuf, err = vervet.WithGeneratedComment(yamlBuf) + if err != nil { + return fmt.Errorf("prepend yaml comment: %w", err) + } + yamlSpecPath := path.Join(versionDir, "spec.yaml") + yamlEmbedPath, err := filepath.Rel(dir, yamlSpecPath) + if err != nil { + return fmt.Errorf("get relative output path: %w", err) + } + versionSpecFiles[idx*2+1] = yamlEmbedPath + err = os.WriteFile(yamlSpecPath, yamlBuf, 0644) + if err != nil { + return fmt.Errorf("write yaml file: %w", err) + } + fmt.Println(yamlSpecPath) + } + return writeEmbedGo(dir, versionSpecFiles) +} + +// Go services embed the compiled specs in the binary to avoid loading them +// from the file system at runtime, this is done with the embed package. +func writeEmbedGo(dir string, versionSpecFiles []string) error { + embedPath := filepath.Join(dir, "embed.go") + f, err := os.Create(embedPath) + if err != nil { + return fmt.Errorf("create embed.go: %w", err) + } + defer f.Close() + + return compiler.EmbedGoTmpl.Execute(f, struct { + Package string + VersionSpecFiles []string + }{ + Package: filepath.Base(dir), + VersionSpecFiles: versionSpecFiles, + }) +} From 56509d2a3030ff77845ffcd184ddcd36ceafce03 Mon Sep 17 00:00:00 2001 From: John Gresty Date: Wed, 31 Jul 2024 08:26:40 +0100 Subject: [PATCH 6/8] feat: apply overlays in simplebuild Overlays are openapi 3 document fragments that are merged on top of ever compiled spec. These are useful for adding information that will be the same on every spec. Matches the functionality of overlays in build for feature parity. --- internal/simplebuild/build.go | 2 +- internal/simplebuild/overlays.go | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 internal/simplebuild/overlays.go diff --git a/internal/simplebuild/build.go b/internal/simplebuild/build.go index 2346a9b7..75ab7347 100644 --- a/internal/simplebuild/build.go +++ b/internal/simplebuild/build.go @@ -24,7 +24,7 @@ func Build(ctx context.Context, project *config.Project) error { return err } - // TODO: apply overlays + docs.ApplyOverlays(ctx, apiConfig.Overlays) if apiConfig.Output != nil { err = docs.WriteOutputs(*apiConfig.Output) diff --git a/internal/simplebuild/overlays.go b/internal/simplebuild/overlays.go new file mode 100644 index 00000000..23d6d51d --- /dev/null +++ b/internal/simplebuild/overlays.go @@ -0,0 +1,56 @@ +package simplebuild + +import ( + "context" + "fmt" + "os" + + "github.com/getkin/kin-openapi/openapi3" + + "github.com/snyk/vervet/v7" + "github.com/snyk/vervet/v7/config" +) + +func (docs DocSet) ApplyOverlays(ctx context.Context, cfgs []*config.Overlay) error { + overlays, err := loadOverlays(ctx, cfgs) + if err != nil { + return fmt.Errorf("load overlays: %w", err) + } + for _, doc := range docs { + for _, overlay := range overlays { + // NB: Will overwrite any existing definitions without warning. + err := vervet.Merge(doc.Doc, overlay, true) + if err != nil { + return fmt.Errorf("apply overlay: %w", err) + } + } + } + + return nil +} + +func loadOverlays(ctx context.Context, cfgs []*config.Overlay) ([]*openapi3.T, error) { + overlays := make([]*openapi3.T, len(cfgs)) + for idx, overlayCfg := range cfgs { + if overlayCfg.Include != "" { + doc, err := vervet.NewDocumentFile(overlayCfg.Include) + if err != nil { + return nil, fmt.Errorf("load include overlay: %w", err) + } + err = vervet.Localize(ctx, doc) + if err != nil { + return nil, fmt.Errorf("localise overlay: %w", err) + } + overlays[idx] = doc.T + } else if overlayCfg.Inline != "" { + docString := os.ExpandEnv(overlayCfg.Inline) + loader := openapi3.NewLoader() + doc, err := loader.LoadFromData([]byte(docString)) + if err != nil { + return nil, fmt.Errorf("load inline overlay: %w", err) + } + overlays[idx] = doc + } + } + return overlays, nil +} From 86046a21a086c6aff0419a481c4e358ba2456c6b Mon Sep 17 00:00:00 2001 From: John Gresty Date: Wed, 31 Jul 2024 13:51:06 +0100 Subject: [PATCH 7/8] feat: add snyk annotations on operations in simplebuild These snyk specific annotations help us when managing versions and aggregating. These differ slightly to the upstream build as they are per operation instead of per resource. --- internal/simplebuild/build.go | 65 +++++++- internal/simplebuild/build_test.go | 253 +++++++++++++++++++++-------- 2 files changed, 247 insertions(+), 71 deletions(-) diff --git a/internal/simplebuild/build.go b/internal/simplebuild/build.go index 75ab7347..07120ef8 100644 --- a/internal/simplebuild/build.go +++ b/internal/simplebuild/build.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "slices" "time" "github.com/getkin/kin-openapi/openapi3" @@ -19,12 +20,18 @@ func Build(ctx context.Context, project *config.Project) error { if err != nil { return err } + for _, op := range operations { + op.Annotate() + } docs, err := operations.Build() if err != nil { return err } - docs.ApplyOverlays(ctx, apiConfig.Overlays) + err = docs.ApplyOverlays(ctx, apiConfig.Overlays) + if err != nil { + return err + } if apiConfig.Output != nil { err = docs.WriteOutputs(*apiConfig.Output) @@ -42,8 +49,9 @@ type OpKey struct { } type VersionedOp struct { - Version vervet.Version - Operation *openapi3.Operation + Version vervet.Version + Operation *openapi3.Operation + ResourceName string } type VersionSet []VersionedOp @@ -107,6 +115,7 @@ func LoadPaths(ctx context.Context, api *config.API) (Operations, error) { for _, path := range paths { versionDir := filepath.Dir(path) versionStr := filepath.Base(versionDir) + resourceName := filepath.Base(filepath.Dir(filepath.Dir(path))) doc, err := vervet.NewDocumentFile(path) if err != nil { @@ -141,8 +150,9 @@ func LoadPaths(ctx context.Context, api *config.API) (Operations, error) { operations[k] = []VersionedOp{} } operations[k] = append(operations[k], VersionedOp{ - Version: version, - Operation: opDef, + Version: version, + Operation: opDef, + ResourceName: resourceName, }) } } @@ -179,3 +189,48 @@ func (vs VersionSet) GetLatest(before time.Time) *openapi3.Operation { } return latest.Operation } + +// Annotate adds Snyk specific extensions to openapi operations. These +// extensions are: +// - x-snyk-api-version: version where the operation was defined +// - x-snyk-api-releases: all versions of this api +// - x-snyk-deprecated-by: if there is a later version of this operation, the +// version of that operation +// - x-snyk-sunset-eligible: the date after this operation can be sunset +// - x-snyk-api-resource: what resource this operation acts on +// - x-snyk-api-lifecycle: status of the operation, can be one of: +// [ unreleased, released, deprecated, sunset ] +func (vs VersionSet) Annotate() { + slices.SortFunc(vs, func(a, b VersionedOp) int { + return a.Version.Compare(b.Version) + }) + + count := len(vs) + + releases := make([]string, count) + for idx, op := range vs { + releases[idx] = op.Version.String() + } + + for idx, op := range vs { + if op.Operation.Extensions == nil { + op.Operation.Extensions = make(map[string]interface{}, 6) + } + op.Operation.Extensions[vervet.ExtSnykApiResource] = op.ResourceName + op.Operation.Extensions[vervet.ExtSnykApiVersion] = op.Version.String() + op.Operation.Extensions[vervet.ExtSnykApiReleases] = releases + op.Operation.Extensions[vervet.ExtSnykApiLifecycle] = op.Version.LifecycleAt(time.Time{}).String() + if idx < (count - 1) { + laterVersion := vs[idx+1].Version + // Sanity check the later version actually deprecates this one + if !op.Version.DeprecatedBy(laterVersion) { + continue + } + op.Operation.Extensions[vervet.ExtSnykDeprecatedBy] = laterVersion.String() + sunsetDate, ok := op.Version.Sunset(laterVersion) + if ok { + op.Operation.Extensions[vervet.ExtSnykSunsetEligible] = sunsetDate.Format("2006-01-02") + } + } + } +} diff --git a/internal/simplebuild/build_test.go b/internal/simplebuild/build_test.go index 61342ac7..8250bffd 100644 --- a/internal/simplebuild/build_test.go +++ b/internal/simplebuild/build_test.go @@ -19,16 +19,19 @@ func TestGetLatest(t *testing.T) { before := vervet.MustParseVersion("3000-01-01") vs := simplebuild.VersionSet{ simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-01-01"), - Operation: openapi3.NewOperation(), + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }, simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-03-01"), - Operation: openapi3.NewOperation(), + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }, simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-02-01"), - Operation: openapi3.NewOperation(), + Version: vervet.MustParseVersion("2024-02-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }, } op := vs.GetLatest(before.Date) @@ -39,16 +42,19 @@ func TestGetLatest(t *testing.T) { before := vervet.MustParseVersion("2024-02-15") vs := simplebuild.VersionSet{ simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-01-01"), - Operation: openapi3.NewOperation(), + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }, simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-03-01"), - Operation: openapi3.NewOperation(), + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }, simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-02-01"), - Operation: openapi3.NewOperation(), + Version: vervet.MustParseVersion("2024-02-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }, } op := vs.GetLatest(before.Date) @@ -59,20 +65,24 @@ func TestGetLatest(t *testing.T) { before := vervet.MustParseVersion("2024-06-01") vs := simplebuild.VersionSet{ simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-01-01"), - Operation: openapi3.NewOperation(), + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }, simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-02-01~beta"), - Operation: openapi3.NewOperation(), + Version: vervet.MustParseVersion("2024-02-01~beta"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }, simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-03-01"), - Operation: openapi3.NewOperation(), + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }, simplebuild.VersionedOp{ - Version: vervet.MustParseVersion("2024-04-01~experimental"), - Operation: openapi3.NewOperation(), + Version: vervet.MustParseVersion("2024-04-01~experimental"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }, } op := vs.GetLatest(before.Date) @@ -88,9 +98,10 @@ func TestBuild(t *testing.T) { simplebuild.OpKey{ Path: "/foo", Method: "GET", - }: simplebuild.VersionSet{{ - Version: vervet.MustParseVersion("2024-01-01"), - Operation: openapi3.NewOperation(), + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", }}, } output, err := ops.Build() @@ -110,23 +121,26 @@ func TestBuild(t *testing.T) { simplebuild.OpKey{ Path: "/foo", Method: "GET", - }: simplebuild.VersionSet{{ - Version: version, - Operation: getFoo, + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: version, + Operation: getFoo, + ResourceName: "foo", }}, simplebuild.OpKey{ Path: "/foo", Method: "POST", - }: simplebuild.VersionSet{{ - Version: version, - Operation: postFoo, + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: version, + Operation: postFoo, + ResourceName: "foo", }}, simplebuild.OpKey{ Path: "/bar", Method: "GET", - }: simplebuild.VersionSet{{ - Version: version, - Operation: getBar, + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: version, + Operation: getBar, + ResourceName: "bar", }}, } output, err := ops.Build() @@ -147,19 +161,22 @@ func TestBuild(t *testing.T) { simplebuild.OpKey{ Path: "/foo", Method: "GET", - }: simplebuild.VersionSet{{ - Version: versions[0], - Operation: openapi3.NewOperation(), + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versions[0], + Operation: openapi3.NewOperation(), + ResourceName: "foo", }}, simplebuild.OpKey{ Path: "/bar", Method: "GET", - }: simplebuild.VersionSet{{ - Version: versions[1], - Operation: openapi3.NewOperation(), - }, { - Version: versions[2], - Operation: openapi3.NewOperation(), + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versions[1], + Operation: openapi3.NewOperation(), + ResourceName: "bar", + }, simplebuild.VersionedOp{ + Version: versions[2], + Operation: openapi3.NewOperation(), + ResourceName: "bar", }}, } output, err := ops.Build() @@ -193,23 +210,26 @@ func TestBuild(t *testing.T) { simplebuild.OpKey{ Path: "/foo", Method: "GET", - }: simplebuild.VersionSet{{ - Version: versionA, - Operation: getFoo, + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionA, + Operation: getFoo, + ResourceName: "foo", }}, simplebuild.OpKey{ Path: "/foo", Method: "POST", - }: simplebuild.VersionSet{{ - Version: versionB, - Operation: postFoo, + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionB, + Operation: postFoo, + ResourceName: "foo", }}, simplebuild.OpKey{ Path: "/bar", Method: "GET", - }: simplebuild.VersionSet{{ - Version: versionC, - Operation: getBar, + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionC, + Operation: getBar, + ResourceName: "bar", }}, } output, err := ops.Build() @@ -246,19 +266,21 @@ func TestBuild(t *testing.T) { simplebuild.OpKey{ Path: "/foo", Method: "GET", - }: simplebuild.VersionSet{{ - Version: versionA, - Operation: getFooOld, - }, { + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionA, + Operation: getFooOld, + ResourceName: "foo", + }, simplebuild.VersionedOp{ Version: versionC, Operation: getFooNew, }}, simplebuild.OpKey{ Path: "/bar", Method: "GET", - }: simplebuild.VersionSet{{ - Version: versionB, - Operation: getBar, + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionB, + Operation: getBar, + ResourceName: "bar", }}, } output, err := ops.Build() @@ -292,23 +314,26 @@ func TestBuild(t *testing.T) { simplebuild.OpKey{ Path: "/foo", Method: "GET", - }: simplebuild.VersionSet{{ - Version: versionGA, - Operation: getFoo, + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionGA, + Operation: getFoo, + ResourceName: "foo", }}, simplebuild.OpKey{ Path: "/foo", Method: "POST", - }: simplebuild.VersionSet{{ - Version: versionBetaB, - Operation: postFoo, + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionBetaB, + Operation: postFoo, + ResourceName: "foo", }}, simplebuild.OpKey{ Path: "/bar", Method: "GET", - }: simplebuild.VersionSet{{ - Version: versionBetaA, - Operation: getBar, + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionBetaA, + Operation: getBar, + ResourceName: "bar", }}, } output, err := ops.Build() @@ -332,6 +357,102 @@ func TestBuild(t *testing.T) { }) } +func TestAnnotate(t *testing.T) { + c := qt.New(t) + + c.Run("adds version dates and resource name to operations", func(c *qt.C) { + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01~beta"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "bar", + }, + } + vs.Annotate() + c.Assert(vs[0].Operation.Extensions[vervet.ExtSnykApiVersion], qt.Equals, "2024-01-01") + c.Assert(vs[0].Operation.Extensions[vervet.ExtSnykApiResource], qt.Equals, "foo") + c.Assert(vs[1].Operation.Extensions[vervet.ExtSnykApiVersion], qt.Equals, "2024-02-01~beta") + c.Assert(vs[1].Operation.Extensions[vervet.ExtSnykApiResource], qt.Equals, "foo") + c.Assert(vs[2].Operation.Extensions[vervet.ExtSnykApiVersion], qt.Equals, "2024-03-01") + c.Assert(vs[2].Operation.Extensions[vervet.ExtSnykApiResource], qt.Equals, "bar") + }) + + c.Run("adds a list of all other versions", func(c *qt.C) { + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01~beta"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "bar", + }, + } + vs.Annotate() + c.Assert( + vs[0].Operation.Extensions[vervet.ExtSnykApiReleases], + qt.DeepEquals, + []string{"2024-01-01", "2024-02-01~beta", "2024-03-01"}, + ) + c.Assert( + vs[1].Operation.Extensions[vervet.ExtSnykApiReleases], + qt.DeepEquals, + []string{"2024-01-01", "2024-02-01~beta", "2024-03-01"}, + ) + c.Assert( + vs[2].Operation.Extensions[vervet.ExtSnykApiReleases], + qt.DeepEquals, + []string{"2024-01-01", "2024-02-01~beta", "2024-03-01"}, + ) + }) + + c.Run("adds deprecation annotations on older versions", func(c *qt.C) { + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01~beta"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "bar", + }, + } + vs.Annotate() + c.Assert(vs[0].Operation.Extensions[vervet.ExtSnykDeprecatedBy], qt.Equals, "2024-02-01") + // beta sunsets after 91 days + c.Assert(vs[0].Operation.Extensions[vervet.ExtSnykSunsetEligible], qt.Equals, "2024-05-02") + c.Assert(vs[1].Operation.Extensions[vervet.ExtSnykDeprecatedBy], qt.Equals, "2024-03-01") + // ga sunsets after 181 days + c.Assert(vs[1].Operation.Extensions[vervet.ExtSnykSunsetEligible], qt.Equals, "2024-08-29") + c.Assert(vs[2].Operation.Extensions[vervet.ExtSnykDeprecatedBy], qt.IsNil) + c.Assert(vs[2].Operation.Extensions[vervet.ExtSnykSunsetEligible], qt.IsNil) + }) +} + func compareDocs(a, b simplebuild.VersionedDoc) int { return a.VersionDate.Compare(b.VersionDate) } From 0a9e1473b5a52e26072ce1c1d69930b6cca72a11 Mon Sep 17 00:00:00 2001 From: John Gresty Date: Thu, 1 Aug 2024 10:43:26 +0100 Subject: [PATCH 8/8] fix: update golangci-lint The version used in CI has issues with go 1.22 which prevents building this app. ifshort is fully inactivated in this version so will error if it is enabled. --- .circleci/config.yml | 2 +- .golangci.yml | 1 - Makefile | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a0ff872..1961e432 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ jobs: lint: docker: - - image: golangci/golangci-lint:v1.51.0 + - image: golangci/golangci-lint:v1.59.1 steps: - checkout - run: diff --git a/.golangci.yml b/.golangci.yml index 6f862f8f..f7f4c8b6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -42,7 +42,6 @@ linters: - gocyclo # - gosec - Good to have linter but potential to introduce breaking changes. # - forbidigo - Good to have linter but potential to introduce breaking changes. - - ifshort - lll - misspell - nakedret diff --git a/Makefile b/Makefile index 7e4e183a..cf7f519e 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO_BIN=$(shell pwd)/.bin/go SHELL:=env PATH=$(GO_BIN):$(PATH) $(SHELL) -GOCI_LINT_V?=v1.54.2 +GOCI_LINT_V?=v1.59.1 .PHONY: all all: lint test build