diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a5c32f0..1961e432 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ defaults: &defaults resource_class: small working_directory: ~/vervet docker: - - image: cimg/go:1.21-node + - image: cimg/go:1.22-node test_defaults: &test_defaults resource_class: medium @@ -54,7 +54,7 @@ jobs: command: npm install -g @stoplight/spectral@6.5.0 - checkout - go/install: - version: 1.21.3 + version: 1.22.3 - go/mod-download-cached - run: name: Verify testdata/output up to date @@ -65,7 +65,7 @@ jobs: lint: docker: - - image: golangci/golangci-lint:v1.51.0 + - image: golangci/golangci-lint:v1.59.1 steps: - checkout - run: diff --git a/.golangci.yml b/.golangci.yml index cf91fc4b..f7f4c8b6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -33,7 +33,6 @@ linters: - errname # - errorlint - Good to have linter but potential to introduce breaking changes. # - exhaustive - Causes too much noise at the moment. - - exportloopref - gci - gocritic - goconst @@ -43,7 +42,6 @@ linters: - gocyclo # - gosec - Good to have linter but potential to introduce breaking changes. # - forbidigo - Good to have linter but potential to introduce breaking changes. - - ifshort - lll - misspell - nakedret diff --git a/Dockerfile b/Dockerfile index ae07f977..0a3dcc4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG GO_VERSION=1.21.3 +ARG GO_VERSION=1.22.3 ############### # Build stage # diff --git a/Makefile b/Makefile index 7e4e183a..cf7f519e 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO_BIN=$(shell pwd)/.bin/go SHELL:=env PATH=$(GO_BIN):$(PATH) $(SHELL) -GOCI_LINT_V?=v1.54.2 +GOCI_LINT_V?=v1.59.1 .PHONY: all all: lint test build diff --git a/go.mod b/go.mod index 5b2c98f8..88406758 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/snyk/vervet/v7 -go 1.21 +go 1.22 require ( cloud.google.com/go/storage v1.34.1 diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 334a63b2..550f1e85 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -75,6 +75,7 @@ var CLIApp = cli.App{ Commands: []*cli.Command{ &BackstageCommand, &BuildCommand, + &SimpleBuildCommand, &FilterCommand, &GenerateCommand, &LocalizeCommand, diff --git a/internal/cmd/compiler.go b/internal/cmd/compiler.go index d1247878..380677ed 100644 --- a/internal/cmd/compiler.go +++ b/internal/cmd/compiler.go @@ -8,6 +8,7 @@ import ( "github.com/snyk/vervet/v7/config" "github.com/snyk/vervet/v7/internal/compiler" + "github.com/snyk/vervet/v7/internal/simplebuild" ) // BuildCommand is the `vervet build` subcommand. @@ -30,6 +31,34 @@ var BuildCommand = cli.Command{ Action: Build, } +var SimpleBuildCommand = cli.Command{ + Name: "simplebuild", + Usage: "Build versioned resources into versioned OpenAPI specs", + ArgsUsage: "[input resources root]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Aliases: []string{"c", "conf"}, + Usage: "Project configuration file", + }, + &cli.StringFlag{ + Name: "include", + Aliases: []string{"I"}, + Usage: "OpenAPI specification to include in build output", + }, + }, + Action: SimpleBuild, +} + +func SimpleBuild(ctx *cli.Context) error { + project, err := projectFromContext(ctx) + if err != nil { + return err + } + err = simplebuild.Build(ctx.Context, project) + return err +} + // Build compiles versioned resources into versioned API specs. func Build(ctx *cli.Context) error { project, err := projectFromContext(ctx) diff --git a/internal/compiler/compiler.go b/internal/compiler/compiler.go index 9dc36f19..f08d5fcf 100644 --- a/internal/compiler/compiler.go +++ b/internal/compiler/compiler.go @@ -262,7 +262,7 @@ func (c *Compiler) writeEmbedGo(pkgName string, a *api, versionSpecFiles []strin return err } defer f.Close() - err = embedGoTmpl.Execute(f, struct { + err = EmbedGoTmpl.Execute(f, struct { Package string API *api VersionSpecFiles []string @@ -282,7 +282,7 @@ func (c *Compiler) writeEmbedGo(pkgName string, a *api, versionSpecFiles []strin return nil } -var embedGoTmpl = template.Must(template.New("embed.go").Parse(` +var EmbedGoTmpl = template.Must(template.New("embed.go").Parse(` // Code generated by Vervet. DO NOT EDIT. package {{ .Package }} diff --git a/internal/simplebuild/build.go b/internal/simplebuild/build.go new file mode 100644 index 00000000..07120ef8 --- /dev/null +++ b/internal/simplebuild/build.go @@ -0,0 +1,236 @@ +package simplebuild + +import ( + "context" + "fmt" + "path/filepath" + "slices" + "time" + + "github.com/getkin/kin-openapi/openapi3" + + "github.com/snyk/vervet/v7" + "github.com/snyk/vervet/v7/config" + "github.com/snyk/vervet/v7/internal/files" +) + +func Build(ctx context.Context, project *config.Project) error { + for _, apiConfig := range project.APIs { + operations, err := LoadPaths(ctx, apiConfig) + if err != nil { + return err + } + for _, op := range operations { + op.Annotate() + } + docs, err := operations.Build() + if err != nil { + return err + } + + err = docs.ApplyOverlays(ctx, apiConfig.Overlays) + if err != nil { + return err + } + + if apiConfig.Output != nil { + err = docs.WriteOutputs(*apiConfig.Output) + if err != nil { + return err + } + } + } + return nil +} + +type OpKey struct { + Path string + Method string +} + +type VersionedOp struct { + Version vervet.Version + Operation *openapi3.Operation + ResourceName string +} + +type VersionSet []VersionedOp + +type Operations map[OpKey]VersionSet + +type VersionedDoc struct { + VersionDate time.Time + Doc *openapi3.T +} +type DocSet []VersionedDoc + +func (ops Operations) Build() (DocSet, error) { + versionDates := ops.VersionDates() + output := make(DocSet, len(versionDates)) + for idx, versionDate := range versionDates { + output[idx] = VersionedDoc{ + Doc: &openapi3.T{}, + VersionDate: versionDate, + } + refResolver := NewRefResolver(output[idx].Doc) + for path, spec := range ops { + op := spec.GetLatest(versionDate) + if op == nil { + continue + } + output[idx].Doc.AddOperation(path.Path, path.Method, op) + err := refResolver.Resolve(op) + if err != nil { + return nil, err + } + } + } + return output, nil +} + +func (ops Operations) VersionDates() []time.Time { + versionSet := map[time.Time]struct{}{} + for _, opSet := range ops { + for _, op := range opSet { + versionSet[op.Version.Date] = struct{}{} + } + } + uniqueVersions := make([]time.Time, len(versionSet)) + idx := 0 + for version := range versionSet { + uniqueVersions[idx] = version + idx++ + } + return uniqueVersions +} + +func LoadPaths(ctx context.Context, api *config.API) (Operations, error) { + operations := map[OpKey]VersionSet{} + + for _, resource := range api.Resources { + paths, err := ResourceSpecFiles(resource) + if err != nil { + return nil, err + } + for _, path := range paths { + versionDir := filepath.Dir(path) + versionStr := filepath.Base(versionDir) + resourceName := filepath.Base(filepath.Dir(filepath.Dir(path))) + + doc, err := vervet.NewDocumentFile(path) + if err != nil { + return nil, fmt.Errorf("failed to load spec from %q: %w", path, err) + } + + stabilityStr, err := vervet.ExtensionString(doc.T.Extensions, vervet.ExtSnykApiStability) + if err != nil { + return nil, err + } + if stabilityStr != "ga" { + versionStr = fmt.Sprintf("%s~%s", versionStr, stabilityStr) + } + version, err := vervet.ParseVersion(versionStr) + if err != nil { + return nil, fmt.Errorf("invalid version %q", versionStr) + } + + doc.InternalizeRefs(ctx, nil) + err = doc.ResolveRefs() + if err != nil { + return nil, fmt.Errorf("failed to localize refs: %w", err) + } + + for pathName, pathDef := range doc.T.Paths { + for opName, opDef := range pathDef.Operations() { + k := OpKey{ + Path: pathName, + Method: opName, + } + if operations[k] == nil { + operations[k] = []VersionedOp{} + } + operations[k] = append(operations[k], VersionedOp{ + Version: version, + Operation: opDef, + ResourceName: resourceName, + }) + } + } + } + } + + return operations, nil +} + +func ResourceSpecFiles(resource *config.ResourceSet) ([]string, error) { + return files.LocalFSSource{}.Match(resource) +} + +func (vs VersionSet) GetLatest(before time.Time) *openapi3.Operation { + var latest *VersionedOp + for _, versionedOp := range vs { + if versionedOp.Version.Date.After(before) { + continue + } + if latest == nil { + latest = &versionedOp + continue + } + // Higher stabilities always take precedent + if versionedOp.Version.Stability.Compare(latest.Version.Stability) < 0 { + continue + } + if versionedOp.Version.Compare(latest.Version) > 0 { + latest = &versionedOp + } + } + if latest == nil { + return nil + } + return latest.Operation +} + +// Annotate adds Snyk specific extensions to openapi operations. These +// extensions are: +// - x-snyk-api-version: version where the operation was defined +// - x-snyk-api-releases: all versions of this api +// - x-snyk-deprecated-by: if there is a later version of this operation, the +// version of that operation +// - x-snyk-sunset-eligible: the date after this operation can be sunset +// - x-snyk-api-resource: what resource this operation acts on +// - x-snyk-api-lifecycle: status of the operation, can be one of: +// [ unreleased, released, deprecated, sunset ] +func (vs VersionSet) Annotate() { + slices.SortFunc(vs, func(a, b VersionedOp) int { + return a.Version.Compare(b.Version) + }) + + count := len(vs) + + releases := make([]string, count) + for idx, op := range vs { + releases[idx] = op.Version.String() + } + + for idx, op := range vs { + if op.Operation.Extensions == nil { + op.Operation.Extensions = make(map[string]interface{}, 6) + } + op.Operation.Extensions[vervet.ExtSnykApiResource] = op.ResourceName + op.Operation.Extensions[vervet.ExtSnykApiVersion] = op.Version.String() + op.Operation.Extensions[vervet.ExtSnykApiReleases] = releases + op.Operation.Extensions[vervet.ExtSnykApiLifecycle] = op.Version.LifecycleAt(time.Time{}).String() + if idx < (count - 1) { + laterVersion := vs[idx+1].Version + // Sanity check the later version actually deprecates this one + if !op.Version.DeprecatedBy(laterVersion) { + continue + } + op.Operation.Extensions[vervet.ExtSnykDeprecatedBy] = laterVersion.String() + sunsetDate, ok := op.Version.Sunset(laterVersion) + if ok { + op.Operation.Extensions[vervet.ExtSnykSunsetEligible] = sunsetDate.Format("2006-01-02") + } + } + } +} diff --git a/internal/simplebuild/build_test.go b/internal/simplebuild/build_test.go new file mode 100644 index 00000000..8250bffd --- /dev/null +++ b/internal/simplebuild/build_test.go @@ -0,0 +1,461 @@ +package simplebuild_test + +import ( + "slices" + "testing" + "time" + + qt "github.com/frankban/quicktest" + "github.com/getkin/kin-openapi/openapi3" + + "github.com/snyk/vervet/v7" + "github.com/snyk/vervet/v7/internal/simplebuild" +) + +func TestGetLatest(t *testing.T) { + c := qt.New(t) + + c.Run("gets the latest version", func(c *qt.C) { + before := vervet.MustParseVersion("3000-01-01") + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + } + op := vs.GetLatest(before.Date) + c.Assert(op, qt.Equals, vs[1].Operation) + }) + + c.Run("filters to before given date", func(c *qt.C) { + before := vervet.MustParseVersion("2024-02-15") + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + } + op := vs.GetLatest(before.Date) + c.Assert(op, qt.Equals, vs[2].Operation) + }) + + c.Run("selects highest stability", func(c *qt.C) { + before := vervet.MustParseVersion("2024-06-01") + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01~beta"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-04-01~experimental"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + } + op := vs.GetLatest(before.Date) + c.Assert(op, qt.Equals, vs[2].Operation) + }) +} + +func TestBuild(t *testing.T) { + c := qt.New(t) + + c.Run("copies paths to output", func(c *qt.C) { + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + c.Assert(output[0].VersionDate, qt.Equals, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + c.Assert(output[0].Doc.Paths["/foo"].Get, qt.IsNotNil) + }) + + c.Run("merges operations from the same version", func(c *qt.C) { + version := vervet.MustParseVersion("2024-01-01") + + getFoo := openapi3.NewOperation() + postFoo := openapi3.NewOperation() + getBar := openapi3.NewOperation() + + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: version, + Operation: getFoo, + ResourceName: "foo", + }}, + simplebuild.OpKey{ + Path: "/foo", + Method: "POST", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: version, + Operation: postFoo, + ResourceName: "foo", + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: version, + Operation: getBar, + ResourceName: "bar", + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + c.Assert(output[0].VersionDate, qt.Equals, version.Date) + c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[0].Doc.Paths["/foo"].Post, qt.Equals, postFoo) + c.Assert(output[0].Doc.Paths["/bar"].Get, qt.Equals, getBar) + }) + + c.Run("generates an output per unique version", func(c *qt.C) { + versions := []vervet.Version{ + vervet.MustParseVersion("2024-01-01"), + vervet.MustParseVersion("2024-01-02"), + vervet.MustParseVersion("2024-01-03"), + } + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versions[0], + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versions[1], + Operation: openapi3.NewOperation(), + ResourceName: "bar", + }, simplebuild.VersionedOp{ + Version: versions[2], + Operation: openapi3.NewOperation(), + ResourceName: "bar", + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + + inputVersions := make([]time.Time, len(versions)) + for idx, in := range versions { + inputVersions[idx] = in.Date + } + slices.SortFunc(inputVersions, compareDates) + + outputVersions := make([]time.Time, len(output)) + for idx, out := range output { + outputVersions[idx] = out.VersionDate + } + slices.SortFunc(outputVersions, compareDates) + + c.Assert(outputVersions, qt.DeepEquals, inputVersions) + }) + + c.Run("merges distinct operations from previous versions", func(c *qt.C) { + versionA := vervet.MustParseVersion("2024-01-01") + versionB := vervet.MustParseVersion("2024-01-02") + versionC := vervet.MustParseVersion("2024-01-03") + + getFoo := openapi3.NewOperation() + postFoo := openapi3.NewOperation() + getBar := openapi3.NewOperation() + + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionA, + Operation: getFoo, + ResourceName: "foo", + }}, + simplebuild.OpKey{ + Path: "/foo", + Method: "POST", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionB, + Operation: postFoo, + ResourceName: "foo", + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionC, + Operation: getBar, + ResourceName: "bar", + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + + slices.SortFunc(output, compareDocs) + + c.Assert(output[0].VersionDate, qt.Equals, versionA.Date) + c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[0].Doc.Paths["/foo"].Post, qt.IsNil) + c.Assert(output[0].Doc.Paths["/bar"], qt.IsNil) + + c.Assert(output[1].VersionDate, qt.Equals, versionB.Date) + c.Assert(output[1].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[1].Doc.Paths["/foo"].Post, qt.Equals, postFoo) + c.Assert(output[1].Doc.Paths["/bar"], qt.IsNil) + + c.Assert(output[2].VersionDate, qt.Equals, versionC.Date) + c.Assert(output[2].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[2].Doc.Paths["/foo"].Post, qt.Equals, postFoo) + c.Assert(output[2].Doc.Paths["/bar"].Get, qt.Equals, getBar) + }) + + c.Run("resolves operations to latest version with respect to output", func(c *qt.C) { + versionA := vervet.MustParseVersion("2024-01-01") + versionB := vervet.MustParseVersion("2024-01-02") + versionC := vervet.MustParseVersion("2024-01-03") + + getFooOld := openapi3.NewOperation() + getFooNew := openapi3.NewOperation() + getBar := openapi3.NewOperation() + + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionA, + Operation: getFooOld, + ResourceName: "foo", + }, simplebuild.VersionedOp{ + Version: versionC, + Operation: getFooNew, + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionB, + Operation: getBar, + ResourceName: "bar", + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + + slices.SortFunc(output, compareDocs) + + c.Assert(output[0].VersionDate, qt.Equals, versionA.Date) + c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFooOld) + c.Assert(output[0].Doc.Paths["/bar"], qt.IsNil) + + c.Assert(output[1].VersionDate, qt.Equals, versionB.Date) + c.Assert(output[1].Doc.Paths["/foo"].Get, qt.Equals, getFooOld) + c.Assert(output[1].Doc.Paths["/bar"].Get, qt.Equals, getBar) + + c.Assert(output[2].VersionDate, qt.Equals, versionC.Date) + c.Assert(output[2].Doc.Paths["/foo"].Get, qt.Equals, getFooNew) + c.Assert(output[2].Doc.Paths["/bar"].Get, qt.Equals, getBar) + }) + + c.Run("lower stabilities are merged into higher", func(c *qt.C) { + versionBetaA := vervet.MustParseVersion("2024-01-01~beta") + versionGA := vervet.MustParseVersion("2024-01-02") + versionBetaB := vervet.MustParseVersion("2024-01-03~beta") + + getFoo := openapi3.NewOperation() + postFoo := openapi3.NewOperation() + getBar := openapi3.NewOperation() + + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionGA, + Operation: getFoo, + ResourceName: "foo", + }}, + simplebuild.OpKey{ + Path: "/foo", + Method: "POST", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionBetaB, + Operation: postFoo, + ResourceName: "foo", + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionBetaA, + Operation: getBar, + ResourceName: "bar", + }}, + } + output, err := ops.Build() + c.Assert(err, qt.IsNil) + + slices.SortFunc(output, compareDocs) + + c.Assert(output[0].VersionDate, qt.Equals, versionBetaA.Date) + c.Assert(output[0].Doc.Paths["/foo"], qt.IsNil) + c.Assert(output[0].Doc.Paths["/bar"].Get, qt.Equals, getBar) + + c.Assert(output[1].VersionDate, qt.Equals, versionGA.Date) + c.Assert(output[1].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[1].Doc.Paths["/foo"].Post, qt.IsNil) + c.Assert(output[0].Doc.Paths["/bar"].Get, qt.Equals, getBar) + + c.Assert(output[2].VersionDate, qt.Equals, versionBetaB.Date) + c.Assert(output[2].Doc.Paths["/foo"].Get, qt.Equals, getFoo) + c.Assert(output[2].Doc.Paths["/foo"].Post, qt.Equals, postFoo) + c.Assert(output[2].Doc.Paths["/bar"].Get, qt.Equals, getBar) + }) +} + +func TestAnnotate(t *testing.T) { + c := qt.New(t) + + c.Run("adds version dates and resource name to operations", func(c *qt.C) { + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01~beta"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "bar", + }, + } + vs.Annotate() + c.Assert(vs[0].Operation.Extensions[vervet.ExtSnykApiVersion], qt.Equals, "2024-01-01") + c.Assert(vs[0].Operation.Extensions[vervet.ExtSnykApiResource], qt.Equals, "foo") + c.Assert(vs[1].Operation.Extensions[vervet.ExtSnykApiVersion], qt.Equals, "2024-02-01~beta") + c.Assert(vs[1].Operation.Extensions[vervet.ExtSnykApiResource], qt.Equals, "foo") + c.Assert(vs[2].Operation.Extensions[vervet.ExtSnykApiVersion], qt.Equals, "2024-03-01") + c.Assert(vs[2].Operation.Extensions[vervet.ExtSnykApiResource], qt.Equals, "bar") + }) + + c.Run("adds a list of all other versions", func(c *qt.C) { + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01~beta"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "bar", + }, + } + vs.Annotate() + c.Assert( + vs[0].Operation.Extensions[vervet.ExtSnykApiReleases], + qt.DeepEquals, + []string{"2024-01-01", "2024-02-01~beta", "2024-03-01"}, + ) + c.Assert( + vs[1].Operation.Extensions[vervet.ExtSnykApiReleases], + qt.DeepEquals, + []string{"2024-01-01", "2024-02-01~beta", "2024-03-01"}, + ) + c.Assert( + vs[2].Operation.Extensions[vervet.ExtSnykApiReleases], + qt.DeepEquals, + []string{"2024-01-01", "2024-02-01~beta", "2024-03-01"}, + ) + }) + + c.Run("adds deprecation annotations on older versions", func(c *qt.C) { + vs := simplebuild.VersionSet{ + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-01-01~beta"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-02-01"), + Operation: openapi3.NewOperation(), + ResourceName: "foo", + }, + simplebuild.VersionedOp{ + Version: vervet.MustParseVersion("2024-03-01"), + Operation: openapi3.NewOperation(), + ResourceName: "bar", + }, + } + vs.Annotate() + c.Assert(vs[0].Operation.Extensions[vervet.ExtSnykDeprecatedBy], qt.Equals, "2024-02-01") + // beta sunsets after 91 days + c.Assert(vs[0].Operation.Extensions[vervet.ExtSnykSunsetEligible], qt.Equals, "2024-05-02") + c.Assert(vs[1].Operation.Extensions[vervet.ExtSnykDeprecatedBy], qt.Equals, "2024-03-01") + // ga sunsets after 181 days + c.Assert(vs[1].Operation.Extensions[vervet.ExtSnykSunsetEligible], qt.Equals, "2024-08-29") + c.Assert(vs[2].Operation.Extensions[vervet.ExtSnykDeprecatedBy], qt.IsNil) + c.Assert(vs[2].Operation.Extensions[vervet.ExtSnykSunsetEligible], qt.IsNil) + }) +} + +func compareDocs(a, b simplebuild.VersionedDoc) int { + return a.VersionDate.Compare(b.VersionDate) +} +func compareDates(a, b time.Time) int { + return a.Compare(b) +} diff --git a/internal/simplebuild/output.go b/internal/simplebuild/output.go new file mode 100644 index 00000000..4a5981be --- /dev/null +++ b/internal/simplebuild/output.go @@ -0,0 +1,134 @@ +package simplebuild + +import ( + "fmt" + "os" + "path" + "path/filepath" + "time" + + "github.com/ghodss/yaml" + + "github.com/snyk/vervet/v7" + "github.com/snyk/vervet/v7/config" + "github.com/snyk/vervet/v7/internal/compiler" + "github.com/snyk/vervet/v7/internal/files" +) + +// Some services have a need to write specs to multiple destinations. This +// tends to happen in Typescript services in which we want to write specs to +// two places: +// - src/** for committing into git and ingesting into Backstage +// - dist/** for runtime module access to compiled specs. +// +// To maintain backwards compatibility we still allow a single path in the +// config file then normalise that here to an array. +func getOutputPaths(cfg config.Output) []string { + paths := cfg.Paths + if len(paths) == 0 && cfg.Path != "" { + paths = []string{cfg.Path} + } + return paths +} + +// WriteOutputs writes compiled specs to all directories specified by the given +// api config. Removes any existing builds if they are present. +func (docs DocSet) WriteOutputs(cfg config.Output) error { + paths := getOutputPaths(cfg) + + for _, dir := range paths { + err := os.RemoveAll(dir) + if err != nil { + return fmt.Errorf("clear output directory: %w", err) + } + } + + err := docs.Write(paths[0]) + if err != nil { + return fmt.Errorf("write output files: %w", err) + } + + for _, dir := range paths[1:] { + err := files.CopyDir(dir, paths[0], true) + if err != nil { + return fmt.Errorf("copy outputs: %w", err) + } + } + + return nil +} + +// Write writes compiled specs to a single directory in YAML and JSON formats. +// Unlike WriteOutputs this function assumes the destination directory does not +// already exist. +func (docs DocSet) Write(dir string) error { + err := os.MkdirAll(dir, 0777) + if err != nil { + return err + } + + versionSpecFiles := make([]string, len(docs)*2) + for idx, doc := range docs { + versionDir := path.Join(dir, doc.VersionDate.Format(time.DateOnly)) + err = os.MkdirAll(versionDir, 0755) + if err != nil { + return fmt.Errorf("make output directory: %w", err) + } + + jsonBuf, err := vervet.ToSpecJSON(doc.Doc) + if err != nil { + return fmt.Errorf("serialise spec to json: %w", err) + } + jsonSpecPath := path.Join(versionDir, "spec.json") + jsonEmbedPath, err := filepath.Rel(dir, jsonSpecPath) + if err != nil { + return fmt.Errorf("get relative output path: %w", err) + } + versionSpecFiles[idx*2] = jsonEmbedPath + err = os.WriteFile(jsonSpecPath, jsonBuf, 0644) + if err != nil { + return fmt.Errorf("write json file: %w", err) + } + fmt.Println(jsonSpecPath) + + yamlBuf, err := yaml.JSONToYAML(jsonBuf) + if err != nil { + return fmt.Errorf("convert spec to yaml: %w", err) + } + yamlBuf, err = vervet.WithGeneratedComment(yamlBuf) + if err != nil { + return fmt.Errorf("prepend yaml comment: %w", err) + } + yamlSpecPath := path.Join(versionDir, "spec.yaml") + yamlEmbedPath, err := filepath.Rel(dir, yamlSpecPath) + if err != nil { + return fmt.Errorf("get relative output path: %w", err) + } + versionSpecFiles[idx*2+1] = yamlEmbedPath + err = os.WriteFile(yamlSpecPath, yamlBuf, 0644) + if err != nil { + return fmt.Errorf("write yaml file: %w", err) + } + fmt.Println(yamlSpecPath) + } + return writeEmbedGo(dir, versionSpecFiles) +} + +// Go services embed the compiled specs in the binary to avoid loading them +// from the file system at runtime, this is done with the embed package. +func writeEmbedGo(dir string, versionSpecFiles []string) error { + embedPath := filepath.Join(dir, "embed.go") + f, err := os.Create(embedPath) + if err != nil { + return fmt.Errorf("create embed.go: %w", err) + } + defer f.Close() + + return compiler.EmbedGoTmpl.Execute(f, struct { + Package string + VersionSpecFiles []string + }{ + Package: filepath.Base(dir), + VersionSpecFiles: versionSpecFiles, + }) +} diff --git a/internal/simplebuild/overlays.go b/internal/simplebuild/overlays.go new file mode 100644 index 00000000..23d6d51d --- /dev/null +++ b/internal/simplebuild/overlays.go @@ -0,0 +1,56 @@ +package simplebuild + +import ( + "context" + "fmt" + "os" + + "github.com/getkin/kin-openapi/openapi3" + + "github.com/snyk/vervet/v7" + "github.com/snyk/vervet/v7/config" +) + +func (docs DocSet) ApplyOverlays(ctx context.Context, cfgs []*config.Overlay) error { + overlays, err := loadOverlays(ctx, cfgs) + if err != nil { + return fmt.Errorf("load overlays: %w", err) + } + for _, doc := range docs { + for _, overlay := range overlays { + // NB: Will overwrite any existing definitions without warning. + err := vervet.Merge(doc.Doc, overlay, true) + if err != nil { + return fmt.Errorf("apply overlay: %w", err) + } + } + } + + return nil +} + +func loadOverlays(ctx context.Context, cfgs []*config.Overlay) ([]*openapi3.T, error) { + overlays := make([]*openapi3.T, len(cfgs)) + for idx, overlayCfg := range cfgs { + if overlayCfg.Include != "" { + doc, err := vervet.NewDocumentFile(overlayCfg.Include) + if err != nil { + return nil, fmt.Errorf("load include overlay: %w", err) + } + err = vervet.Localize(ctx, doc) + if err != nil { + return nil, fmt.Errorf("localise overlay: %w", err) + } + overlays[idx] = doc.T + } else if overlayCfg.Inline != "" { + docString := os.ExpandEnv(overlayCfg.Inline) + loader := openapi3.NewLoader() + doc, err := loader.LoadFromData([]byte(docString)) + if err != nil { + return nil, fmt.Errorf("load inline overlay: %w", err) + } + overlays[idx] = doc + } + } + return overlays, nil +} diff --git a/internal/simplebuild/refs.go b/internal/simplebuild/refs.go new file mode 100644 index 00000000..7fd7b331 --- /dev/null +++ b/internal/simplebuild/refs.go @@ -0,0 +1,140 @@ +package simplebuild + +import ( + "fmt" + "reflect" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/mitchellh/reflectwalk" +) + +// Refs are an OpenAPI concept where you can define part of a spec then use a +// JSON reference +// [https://datatracker.ietf.org/doc/html/draft-pbryan-zyp-json-ref-03] to +// include that block in another part of the document. +// +// For example a component might live at the top level then that is consumed +// elsewhere: +// +// components: +// +// parameters: +// foo: +// name: fooparam +// in: query +// +// paths: +// +// /foo: +// parameters: +// - $ref: "#/components/parameters/foo" +// +// openapi3 has several *Ref types which have Ref and Value fields, the Ref +// field is the string from the original document and Value is the block it +// points to if the ref is resolved, hen loading our documents we do this +// resolution to populate all Value fields. +// +// When serialising an openapi3.*Ref, if the Ref field is set then the Value +// field is ignored. Therefore we have two options, either add the components +// back into the document at the appropriate position or inline them. As some +// components are likely to be reused several times, we elect to do the former +// where possible. +// +// This class walks a given object and recursively copy any refs it finds back +// into the document at the path they are referenced from. +type refResolver struct { + doc *openapi3.T +} + +func NewRefResolver(doc *openapi3.T) refResolver { + return refResolver{doc: doc} +} + +func (rr *refResolver) Resolve(from any) error { + return reflectwalk.Walk(from, rr) +} + +// Implements reflectwalk.StructWalker. This function is called for every +// struct found when walking. +func (rr *refResolver) Struct(v reflect.Value) error { + ref := v.FieldByName("Ref") + value := v.FieldByName("Value") + if !ref.IsValid() || !value.IsValid() { + // This isn't a openapi3.*Ref so nothing to do + return nil + } + refLoc := ref.String() + if refLoc == "" { + // This ref has been inlined + return nil + } + // Create a new *Ref object to avoid mutating the original + derefed := reflect.New(v.Type()) + reflect.Indirect(derefed).FieldByName("Value").Set(value) + + return rr.deref(refLoc, derefed) +} + +// Implements reflectwalk.StructWalker. We work on whole structs so there is +// nothing to do here. +func (rr *refResolver) StructField(sf reflect.StructField, v reflect.Value) error { + return nil +} + +func (rr *refResolver) deref(ref string, value reflect.Value) error { + path := strings.Split(ref, "/") + if path[0] != "#" { + // All refs should have been resolved to the local document when + // loading so if we hit this case then we have not loaded the document + // correctly. + return fmt.Errorf("external ref %s", ref) + } + field := reflect.ValueOf(rr.doc) + // Need to forward declare err so field is not shadowed in the loop + var err error + for _, segment := range path[1:] { + // Maps are a special case since the key also needs to be created. + if field.Kind() == reflect.Map { + newValue := reflect.New(field.Type().Elem().Elem()) + field.SetMapIndex(reflect.ValueOf(segment), newValue) + field = newValue.Elem() + continue + } + // else we assume we are working on a struct + field, err = getField(segment, field) + if err != nil { + return fmt.Errorf("invalid ref %s: %w", ref, err) + } + + // A lot of the openapi3.T fields are pointers so if this is the first + // time we have encountered an object of this type we need to create + // the container. + if field.Kind() == reflect.Map && field.IsZero() { + newValue := reflect.MakeMap(field.Type()) + field.Set(newValue) + } else if field.IsNil() { + newValue := reflect.New(field.Type().Elem()) + field.Set(newValue) + } + } + field.Set(value.Elem()) + return nil +} + +func getField(tag string, object reflect.Value) (reflect.Value, error) { + reflectedObject := object.Type().Elem() + if reflectedObject.Kind() != reflect.Struct { + return reflect.Value{}, fmt.Errorf("object is not a struct") + } + for idx := 0; idx < reflectedObject.NumField(); idx++ { + structField := reflectedObject.Field(idx) + yamlTag := structField.Tag.Get("yaml") + // Remove tag options (eg "omitempty") + yamlField := strings.SplitN(yamlTag, ",", 2)[0] + if yamlField == tag { + return object.Elem().FieldByName(structField.Name), nil + } + } + return reflect.Value{}, fmt.Errorf("field %s not found on object", tag) +} diff --git a/internal/simplebuild/refs_test.go b/internal/simplebuild/refs_test.go new file mode 100644 index 00000000..2ebade08 --- /dev/null +++ b/internal/simplebuild/refs_test.go @@ -0,0 +1,144 @@ +package simplebuild_test + +import ( + "testing" + + qt "github.com/frankban/quicktest" + "github.com/getkin/kin-openapi/openapi3" + + "github.com/snyk/vervet/v7/internal/simplebuild" +) + +func TestResolveRefs(t *testing.T) { + c := qt.New(t) + + c.Run("copies ref value into referenced location", func(c *qt.C) { + param := &openapi3.Parameter{} + path := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/foo", + Value: param, + }}, + } + doc := openapi3.T{ + Paths: openapi3.Paths{ + "/foo": &path, + }, + } + + rr := simplebuild.NewRefResolver(&doc) + err := rr.Resolve(path) + c.Assert(err, qt.IsNil) + + c.Assert(doc.Components.Parameters["foo"].Value, qt.Equals, param) + }) + + c.Run("ignores refs on other parts of the doc", func(c *qt.C) { + param := &openapi3.Parameter{} + pathA := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/foo", + Value: param, + }}, + } + pathB := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/bar", + Value: param, + }}, + } + doc := openapi3.T{ + Paths: openapi3.Paths{ + "/foo": &pathA, + "/bar": &pathB, + }, + } + + rr := simplebuild.NewRefResolver(&doc) + err := rr.Resolve(pathA) + c.Assert(err, qt.IsNil) + + c.Assert(doc.Components.Parameters["bar"], qt.IsNil) + }) + + c.Run("merges refs from successive calls", func(c *qt.C) { + paramA := &openapi3.Parameter{} + pathA := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/foo", + Value: paramA, + }}, + } + paramB := &openapi3.Parameter{} + pathB := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/bar", + Value: paramB, + }}, + } + doc := openapi3.T{ + Paths: openapi3.Paths{ + "/foo": &pathA, + "/bar": &pathB, + }, + } + + rr := simplebuild.NewRefResolver(&doc) + err := rr.Resolve(pathA) + c.Assert(err, qt.IsNil) + err = rr.Resolve(pathB) + c.Assert(err, qt.IsNil) + + c.Assert(doc.Components.Parameters["foo"].Value, qt.Equals, paramA) + c.Assert(doc.Components.Parameters["bar"].Value, qt.Equals, paramB) + }) + + c.Run("recursively resolves components", func(c *qt.C) { + schema := &openapi3.Schema{} + param := &openapi3.Parameter{ + Schema: &openapi3.SchemaRef{ + Ref: "#/components/schemas/foo", + Value: schema, + }, + } + path := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Ref: "#/components/parameters/foo", + Value: param, + }}, + } + doc := openapi3.T{ + Paths: openapi3.Paths{ + "/foo": &path, + }, + } + + rr := simplebuild.NewRefResolver(&doc) + err := rr.Resolve(path) + c.Assert(err, qt.IsNil) + + c.Assert(doc.Components.Parameters["foo"].Value, qt.Equals, param) + c.Assert(doc.Components.Schemas["foo"].Value, qt.Equals, schema) + }) + + c.Run("ignores ref objects with no ref value", func(c *qt.C) { + param := &openapi3.Parameter{} + path := openapi3.PathItem{ + Parameters: []*openapi3.ParameterRef{{ + Value: param, + }}, + } + doc := openapi3.T{ + Components: &openapi3.Components{}, + Paths: openapi3.Paths{ + "/foo": &path, + }, + } + + rr := simplebuild.NewRefResolver(&doc) + err := rr.Resolve(path) + c.Assert(err, qt.IsNil) + + c.Assert(doc.Components.Parameters["foo"], qt.IsNil) + }) +} diff --git a/shell.nix b/shell.nix index 07b7cd53..3fb57221 100644 --- a/shell.nix +++ b/shell.nix @@ -6,7 +6,7 @@ let }; in pkgs.mkShell { nativeBuildInputs = with pkgs.buildPackages; [ - go_1_21 + go_1_22 gopls gotools golangci-lint