From 30bc0b2b1934555a0edf60cd8fba117917b798bd Mon Sep 17 00:00:00 2001 From: Jatin Naik Date: Tue, 6 Aug 2024 12:56:53 +0100 Subject: [PATCH] feat: change build to be a meta-build command, invoking old and simple build --- internal/cmd/cmd.go | 1 + internal/cmd/compiler.go | 109 +++++++++++++++------ internal/compiler/compiler.go | 31 +++--- internal/compiler/compiler_test.go | 5 +- internal/simplebuild/build.go | 25 ++++- internal/simplebuild/build_test.go | 58 +++++++++-- internal/simplebuild/output.go | 52 +++++++--- internal/simplebuild/output_test.go | 144 ++++++++++++++++++++++++++++ 8 files changed, 355 insertions(+), 70 deletions(-) create mode 100644 internal/simplebuild/output_test.go diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 550f1e85..4f9a30b7 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -75,6 +75,7 @@ var CLIApp = cli.App{ Commands: []*cli.Command{ &BackstageCommand, &BuildCommand, + &RetroBuildCommand, &SimpleBuildCommand, &FilterCommand, &GenerateCommand, diff --git a/internal/cmd/compiler.go b/internal/cmd/compiler.go index 380677ed..c228dc7f 100644 --- a/internal/cmd/compiler.go +++ b/internal/cmd/compiler.go @@ -6,74 +6,121 @@ import ( "github.com/urfave/cli/v2" + "github.com/snyk/vervet/v7" "github.com/snyk/vervet/v7/config" "github.com/snyk/vervet/v7/internal/compiler" "github.com/snyk/vervet/v7/internal/simplebuild" ) +var defaultPivotDate = vervet.MustParseVersion("2024-09-01") +var pivotDateCLIFlagName = "pivot-version" + +var buildFlags = []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Aliases: []string{"c", "conf"}, + Usage: "Project configuration file", + }, + &cli.StringFlag{ + Name: "include", + Aliases: []string{"I"}, + Usage: "OpenAPI specification to include in build output", + }, + &cli.StringFlag{ + Name: pivotDateCLIFlagName, + Aliases: []string{"P"}, + Usage: fmt.Sprintf( + "Pivot version after which new strategy versioning is used."+ + " Flag for testing only, recommend to use the default date(%s)", defaultPivotDate.String()), + Value: defaultPivotDate.String(), + }, +} + // BuildCommand is the `vervet build` subcommand. var BuildCommand = cli.Command{ Name: "build", Usage: "Build versioned resources into versioned OpenAPI specs", ArgsUsage: "[input resources root] [output api root]", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Aliases: []string{"c", "conf"}, - Usage: "Project configuration file", - }, - &cli.StringFlag{ - Name: "include", - Aliases: []string{"I"}, - Usage: "OpenAPI specification to include in build output", - }, - }, - Action: Build, + Flags: buildFlags, + Action: CombinedBuild, +} + +// RetroBuild is the `vervet build` subcommand. +var RetroBuildCommand = cli.Command{ + Name: "retrobuild", + Usage: "Build versioned resources into versioned OpenAPI specs", + ArgsUsage: "[input resources root] [output api root]", + Flags: buildFlags, + Action: RetroBuild, } var SimpleBuildCommand = cli.Command{ Name: "simplebuild", Usage: "Build versioned resources into versioned OpenAPI specs", ArgsUsage: "[input resources root]", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Aliases: []string{"c", "conf"}, - Usage: "Project configuration file", - }, - &cli.StringFlag{ - Name: "include", - Aliases: []string{"I"}, - Usage: "OpenAPI specification to include in build output", - }, - }, - Action: SimpleBuild, + Flags: buildFlags, + Action: SimpleBuild, } +// SimpleBuild compiles versioned resources into versioned API specs using the rolled up versioning strategy. func SimpleBuild(ctx *cli.Context) error { project, err := projectFromContext(ctx) if err != nil { return err } - err = simplebuild.Build(ctx.Context, project) + pivotDate, err := parsePivotDate(ctx) + if err != nil { + return fmt.Errorf("failed to parse pivot date %q: %w", pivotDate, err) + } + + err = simplebuild.Build(ctx.Context, project, pivotDate, false) return err } -// Build compiles versioned resources into versioned API specs. -func Build(ctx *cli.Context) error { +// CombinedBuild compiles versioned resources into versioned API specs +// invokes retorbuild and simplebuild based on the context. +func CombinedBuild(ctx *cli.Context) error { project, err := projectFromContext(ctx) if err != nil { return err } + pivotDate, err := parsePivotDate(ctx) + if err != nil { + return fmt.Errorf("failed to parse pivot date %q: %w", pivotDate, err) + } + comp, err := compiler.New(ctx.Context, project) if err != nil { return err } - err = comp.BuildAll(ctx.Context) + err = comp.BuildAll(ctx.Context, pivotDate) + if err != nil { + return err + } + + return simplebuild.Build(ctx.Context, project, pivotDate, true) +} + +func parsePivotDate(ctx *cli.Context) (vervet.Version, error) { + return vervet.ParseVersion(ctx.String(pivotDateCLIFlagName)) +} + +// RetroBuild compiles versioned resources into versioned API specs using the older versioning strategy. +// This is used for regenerating old versioned API specs only. +func RetroBuild(ctx *cli.Context) error { + project, err := projectFromContext(ctx) + if err != nil { + return err + } + pivotDate, err := parsePivotDate(ctx) + if err != nil { + return fmt.Errorf("failed to parse pivot date %q: %w", pivotDate, err) + } + comp, err := compiler.New(ctx.Context, project) if err != nil { return err } - return nil + return comp.BuildAll(ctx.Context, pivotDate) } func projectFromContext(ctx *cli.Context) (*config.Project, error) { diff --git a/internal/compiler/compiler.go b/internal/compiler/compiler.go index f08d5fcf..8e4faba0 100644 --- a/internal/compiler/compiler.go +++ b/internal/compiler/compiler.go @@ -122,20 +122,9 @@ func ResourceSpecFiles(rcConfig *config.ResourceSet) ([]string, error) { return files.LocalFSSource{}.Match(rcConfig) } -func (c *Compiler) apisEach(ctx context.Context, f func(ctx context.Context, apiName string) error) error { - var errs error - for apiName := range c.apis { - err := f(ctx, apiName) - if err != nil { - errs = multierr.Append(errs, err) - } - } - return errs -} - // Build builds an aggregate versioned OpenAPI spec for a specific API by name // in the project. -func (c *Compiler) Build(ctx context.Context, apiName string) error { +func (c *Compiler) Build(apiName string, stopVersion vervet.Version) error { api, ok := c.apis[apiName] if !ok { return fmt.Errorf("api not found (apis.%s)", apiName) @@ -156,7 +145,7 @@ func (c *Compiler) Build(ctx context.Context, apiName string) error { log.Printf("compiling API %s to output versions", apiName) var versionSpecFiles []string for rcIndex, rc := range api.resources { - specVersions, err := vervet.LoadSpecVersionsFileset(rc.sourceFiles) //nolint:contextcheck //acked + specVersions, err := vervet.LoadSpecVersionsFileset(rc.sourceFiles) if err != nil { return fmt.Errorf("failed to load spec versions: %+v (apis.%s.resources[%d])", err, apiName, rcIndex) @@ -172,6 +161,9 @@ func (c *Compiler) Build(ctx context.Context, apiName string) error { version, )) } + if version.Compare(stopVersion) >= 0 { + continue + } spec, err := specVersions.At(version) if err == vervet.ErrNoMatchingVersion { @@ -298,7 +290,14 @@ import "embed" var Versions embed.FS `[1:])) -// BuildAll builds all APIs in the project. -func (c *Compiler) BuildAll(ctx context.Context) error { - return c.apisEach(ctx, c.Build) +// BuildAll builds all APIs in the project, before the stop version. +func (c *Compiler) BuildAll(ctx context.Context, stopVersion vervet.Version) error { + var errs error + for apiName := range c.apis { + err := c.Build(apiName, stopVersion) //nolint:contextcheck // TODO: fix contextcheck in separate PR + if err != nil { + errs = multierr.Append(errs, err) + } + } + return errs } diff --git a/internal/compiler/compiler_test.go b/internal/compiler/compiler_test.go index 313c0814..e82eff2b 100644 --- a/internal/compiler/compiler_test.go +++ b/internal/compiler/compiler_test.go @@ -11,6 +11,7 @@ import ( qt "github.com/frankban/quicktest" + "github.com/snyk/vervet/v7" "github.com/snyk/vervet/v7/config" "github.com/snyk/vervet/v7/testdata" ) @@ -105,7 +106,7 @@ func TestCompilerSmoke(t *testing.T) { c.Assert(restApi.output, qt.Not(qt.IsNil)) // Build stage - err = compiler.BuildAll(ctx) + err = compiler.BuildAll(ctx, vervet.MustParseVersion("2024-06-01")) c.Assert(err, qt.IsNil) // Verify created files/folders are as expected @@ -141,7 +142,7 @@ func TestCompilerSmokePaths(t *testing.T) { c.Assert(err, qt.IsNil) // Build stage - err = compiler.BuildAll(ctx) + err = compiler.BuildAll(ctx, vervet.MustParseVersion("2024-06-01")) c.Assert(err, qt.IsNil) refOutputPath := testdata.Path("output") diff --git a/internal/simplebuild/build.go b/internal/simplebuild/build.go index 07120ef8..f058b497 100644 --- a/internal/simplebuild/build.go +++ b/internal/simplebuild/build.go @@ -14,7 +14,12 @@ import ( "github.com/snyk/vervet/v7/internal/files" ) -func Build(ctx context.Context, project *config.Project) error { +// Build compiles the versioned resources in a project configuration based on +// simplified versioning rules, after the start date. +func Build(ctx context.Context, project *config.Project, startDate vervet.Version, appendOutputFiles bool) error { + if time.Now().Before(startDate.Date) { + return nil + } for _, apiConfig := range project.APIs { operations, err := LoadPaths(ctx, apiConfig) if err != nil { @@ -23,7 +28,8 @@ func Build(ctx context.Context, project *config.Project) error { for _, op := range operations { op.Annotate() } - docs, err := operations.Build() + + docs, err := operations.Build(startDate) if err != nil { return err } @@ -34,7 +40,7 @@ func Build(ctx context.Context, project *config.Project) error { } if apiConfig.Output != nil { - err = docs.WriteOutputs(*apiConfig.Output) + err = docs.WriteOutputs(*apiConfig.Output, appendOutputFiles) if err != nil { return err } @@ -64,8 +70,9 @@ type VersionedDoc struct { } type DocSet []VersionedDoc -func (ops Operations) Build() (DocSet, error) { +func (ops Operations) Build(startVersion vervet.Version) (DocSet, error) { versionDates := ops.VersionDates() + versionDates = filterVersionByStartDate(versionDates, startVersion.Date) output := make(DocSet, len(versionDates)) for idx, versionDate := range versionDates { output[idx] = VersionedDoc{ @@ -88,6 +95,16 @@ func (ops Operations) Build() (DocSet, error) { return output, nil } +func filterVersionByStartDate(dates []time.Time, startDate time.Time) []time.Time { + resultDates := []time.Time{startDate} + for _, d := range dates { + if d.After(startDate) { + resultDates = append(resultDates, d) + } + } + return resultDates +} + func (ops Operations) VersionDates() []time.Time { versionSet := map[time.Time]struct{}{} for _, opSet := range ops { diff --git a/internal/simplebuild/build_test.go b/internal/simplebuild/build_test.go index 8250bffd..6f68031f 100644 --- a/internal/simplebuild/build_test.go +++ b/internal/simplebuild/build_test.go @@ -104,7 +104,7 @@ func TestBuild(t *testing.T) { ResourceName: "foo", }}, } - output, err := ops.Build() + output, err := ops.Build(vervet.MustParseVersion("2024-01-01")) c.Assert(err, qt.IsNil) c.Assert(output[0].VersionDate, qt.Equals, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) c.Assert(output[0].Doc.Paths["/foo"].Get, qt.IsNotNil) @@ -143,7 +143,7 @@ func TestBuild(t *testing.T) { ResourceName: "bar", }}, } - output, err := ops.Build() + output, err := ops.Build(vervet.MustParseVersion("2024-01-01")) c.Assert(err, qt.IsNil) c.Assert(output[0].VersionDate, qt.Equals, version.Date) c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFoo) @@ -179,7 +179,7 @@ func TestBuild(t *testing.T) { ResourceName: "bar", }}, } - output, err := ops.Build() + output, err := ops.Build(vervet.MustParseVersion("2024-01-01")) c.Assert(err, qt.IsNil) inputVersions := make([]time.Time, len(versions)) @@ -232,7 +232,7 @@ func TestBuild(t *testing.T) { ResourceName: "bar", }}, } - output, err := ops.Build() + output, err := ops.Build(vervet.MustParseVersion("2024-01-01")) c.Assert(err, qt.IsNil) slices.SortFunc(output, compareDocs) @@ -283,7 +283,7 @@ func TestBuild(t *testing.T) { ResourceName: "bar", }}, } - output, err := ops.Build() + output, err := ops.Build(vervet.MustParseVersion("2024-01-01")) c.Assert(err, qt.IsNil) slices.SortFunc(output, compareDocs) @@ -301,6 +301,52 @@ func TestBuild(t *testing.T) { c.Assert(output[2].Doc.Paths["/bar"].Get, qt.Equals, getBar) }) + c.Run("does not generate versions before pivot date", func(c *qt.C) { + versionA := vervet.MustParseVersion("2024-01-01") + versionB := vervet.MustParseVersion("2024-01-02") + versionC := vervet.MustParseVersion("2024-01-03") + + getFooOld := openapi3.NewOperation() + getFooNew := openapi3.NewOperation() + getBar := openapi3.NewOperation() + + ops := simplebuild.Operations{ + simplebuild.OpKey{ + Path: "/foo", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionA, + Operation: getFooOld, + ResourceName: "foo", + }, simplebuild.VersionedOp{ + Version: versionC, + Operation: getFooNew, + }}, + simplebuild.OpKey{ + Path: "/bar", + Method: "GET", + }: simplebuild.VersionSet{simplebuild.VersionedOp{ + Version: versionB, + Operation: getBar, + ResourceName: "bar", + }}, + } + output, err := ops.Build(vervet.MustParseVersion("2024-01-02")) + c.Assert(err, qt.IsNil) + + slices.SortFunc(output, compareDocs) + + c.Assert(len(output), qt.Equals, 2) + + c.Assert(output[0].VersionDate, qt.Equals, versionB.Date) + c.Assert(output[0].Doc.Paths["/foo"].Get, qt.Equals, getFooOld) + c.Assert(output[0].Doc.Paths["/bar"].Get, qt.Equals, getBar) + + c.Assert(output[1].VersionDate, qt.Equals, versionC.Date) + c.Assert(output[1].Doc.Paths["/foo"].Get, qt.Equals, getFooNew) + c.Assert(output[1].Doc.Paths["/bar"].Get, qt.Equals, getBar) + }) + c.Run("lower stabilities are merged into higher", func(c *qt.C) { versionBetaA := vervet.MustParseVersion("2024-01-01~beta") versionGA := vervet.MustParseVersion("2024-01-02") @@ -336,7 +382,7 @@ func TestBuild(t *testing.T) { ResourceName: "bar", }}, } - output, err := ops.Build() + output, err := ops.Build(vervet.MustParseVersion("2024-01-01")) c.Assert(err, qt.IsNil) slices.SortFunc(output, compareDocs) diff --git a/internal/simplebuild/output.go b/internal/simplebuild/output.go index 4a5981be..00bb2e8e 100644 --- a/internal/simplebuild/output.go +++ b/internal/simplebuild/output.go @@ -2,9 +2,11 @@ package simplebuild import ( "fmt" + "io/fs" "os" "path" "path/filepath" + "sort" "time" "github.com/ghodss/yaml" @@ -33,17 +35,19 @@ func getOutputPaths(cfg config.Output) []string { // WriteOutputs writes compiled specs to all directories specified by the given // api config. Removes any existing builds if they are present. -func (docs DocSet) WriteOutputs(cfg config.Output) error { +func (docs DocSet) WriteOutputs(cfg config.Output, appendOutputFiles bool) error { paths := getOutputPaths(cfg) - for _, dir := range paths { - err := os.RemoveAll(dir) - if err != nil { - return fmt.Errorf("clear output directory: %w", err) + if !appendOutputFiles { + for _, dir := range paths { + err := os.RemoveAll(dir) + if err != nil { + return fmt.Errorf("clear output directory: %w", err) + } } } - err := docs.Write(paths[0]) + err := docs.Write(paths[0], appendOutputFiles) if err != nil { return fmt.Errorf("write output files: %w", err) } @@ -61,14 +65,19 @@ func (docs DocSet) WriteOutputs(cfg config.Output) error { // Write writes compiled specs to a single directory in YAML and JSON formats. // Unlike WriteOutputs this function assumes the destination directory does not // already exist. -func (docs DocSet) Write(dir string) error { +func (docs DocSet) Write(dir string, appendOutputFiles bool) error { err := os.MkdirAll(dir, 0777) if err != nil { return err } + existingFiles, err := getExisingSpecFiles(dir) + if err != nil { + return fmt.Errorf("list existing files: %w", err) + } - versionSpecFiles := make([]string, len(docs)*2) - for idx, doc := range docs { + versionSpecFiles := make([]string, 0, len(existingFiles)+len(docs)*2) + versionSpecFiles = append(versionSpecFiles, existingFiles...) + for _, doc := range docs { versionDir := path.Join(dir, doc.VersionDate.Format(time.DateOnly)) err = os.MkdirAll(versionDir, 0755) if err != nil { @@ -84,7 +93,7 @@ func (docs DocSet) Write(dir string) error { if err != nil { return fmt.Errorf("get relative output path: %w", err) } - versionSpecFiles[idx*2] = jsonEmbedPath + versionSpecFiles = append(versionSpecFiles, jsonEmbedPath) err = os.WriteFile(jsonSpecPath, jsonBuf, 0644) if err != nil { return fmt.Errorf("write json file: %w", err) @@ -104,7 +113,7 @@ func (docs DocSet) Write(dir string) error { if err != nil { return fmt.Errorf("get relative output path: %w", err) } - versionSpecFiles[idx*2+1] = yamlEmbedPath + versionSpecFiles = append(versionSpecFiles, yamlEmbedPath) err = os.WriteFile(yamlSpecPath, yamlBuf, 0644) if err != nil { return fmt.Errorf("write yaml file: %w", err) @@ -114,6 +123,27 @@ func (docs DocSet) Write(dir string) error { return writeEmbedGo(dir, versionSpecFiles) } +func getExisingSpecFiles(dir string) ([]string, error) { + var outputFiles []string + err := filepath.WalkDir(dir, func(filePath string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || d.Name() == "embed.go" { + return nil + } + relativePath, err := filepath.Rel(dir, filePath) + if err != nil { + return err + } + outputFiles = append(outputFiles, relativePath) + return nil + }) + // Sort files for consistency + sort.Strings(outputFiles) + return outputFiles, err +} + // Go services embed the compiled specs in the binary to avoid loading them // from the file system at runtime, this is done with the embed package. func writeEmbedGo(dir string, versionSpecFiles []string) error { diff --git a/internal/simplebuild/output_test.go b/internal/simplebuild/output_test.go new file mode 100644 index 00000000..9afef9b8 --- /dev/null +++ b/internal/simplebuild/output_test.go @@ -0,0 +1,144 @@ +package simplebuild + +import ( + "os" + "path" + "path/filepath" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/getkin/kin-openapi/openapi3" + + "github.com/snyk/vervet/v7" + "github.com/snyk/vervet/v7/config" +) + +func TestDocSet_WriteOutputs(t *testing.T) { + c := qt.New(t) + + loader := openapi3.NewLoader() + testDoc, err := loader.LoadFromData([]byte(minimalSpec)) + c.Assert(err, qt.IsNil) + + type args struct { + cfg config.Output + appendOutputFiles bool + } + tests := []struct { + name string + docs DocSet + args args + wantErr bool + assert func(*testing.T, args) + setup func(*testing.T, args) + }{ + { + name: "write the doc sets to outputs", + args: args{ + cfg: config.Output{ + Path: t.TempDir(), + }, + }, + docs: DocSet{ + { + VersionDate: vervet.MustParseVersion("2024-01-01").Date, + Doc: testDoc, + }, + }, + wantErr: false, + assert: func(t *testing.T, args args) { + t.Helper() + files, err := filepath.Glob(filepath.Join(args.cfg.Path, "*")) + c.Assert(err, qt.IsNil) + c.Assert(files, qt.HasLen, 2) + goEmbedContents, err := os.ReadFile(path.Join(args.cfg.Path, "embed.go")) + c.Assert(err, qt.IsNil) + c.Assert(string(goEmbedContents), qt.Contains, "2024-01-01") + }, + }, + { + name: "clears dir if appendOutputFiles is false", + args: args{ + cfg: config.Output{ + Path: t.TempDir(), + }, + appendOutputFiles: false, + }, + docs: DocSet{ + { + VersionDate: vervet.MustParseVersion("2024-01-01").Date, + Doc: testDoc, + }, + }, + wantErr: false, + setup: func(t *testing.T, args args) { + t.Helper() + err = os.WriteFile(path.Join(args.cfg.Path, "existing-file"), []byte("existing"), 0644) + c.Assert(err, qt.IsNil) + }, + assert: func(t *testing.T, args args) { + t.Helper() + files, err := filepath.Glob(filepath.Join(args.cfg.Path, "*")) + c.Assert(err, qt.IsNil) + c.Assert(files, qt.HasLen, 2) + goEmbedContents, err := os.ReadFile(path.Join(args.cfg.Path, "embed.go")) + c.Assert(err, qt.IsNil) + c.Assert(string(goEmbedContents), qt.Contains, "2024-01-01") + }, + }, + + { + name: "merges files if appendOutputFiles is true, embeds existing files", + args: args{ + cfg: config.Output{ + Path: t.TempDir(), + }, + appendOutputFiles: true, + }, + docs: DocSet{ + { + VersionDate: vervet.MustParseVersion("2024-01-01").Date, + Doc: testDoc, + }, + }, + wantErr: false, + setup: func(t *testing.T, args args) { + t.Helper() + err = os.WriteFile(path.Join(args.cfg.Path, "2024-02-01"), []byte("existing"), 0644) + c.Assert(err, qt.IsNil) + err = os.WriteFile(path.Join(args.cfg.Path, "2024-02-02"), []byte("existing"), 0644) + c.Assert(err, qt.IsNil) + err = os.WriteFile(path.Join(args.cfg.Path, "2024-02-03"), []byte("existing"), 0644) + c.Assert(err, qt.IsNil) + }, + assert: func(t *testing.T, args args) { + t.Helper() + files, err := filepath.Glob(filepath.Join(args.cfg.Path, "*")) + c.Assert(err, qt.IsNil) + c.Assert(files, qt.HasLen, 2+3) + goEmbedContents, err := os.ReadFile(path.Join(args.cfg.Path, "embed.go")) + c.Assert(err, qt.IsNil) + c.Assert(string(goEmbedContents), qt.Contains, "2024-01-01") + c.Assert(string(goEmbedContents), qt.Contains, "2024-02-01") + c.Assert(string(goEmbedContents), qt.Contains, "2024-02-02") + c.Assert(string(goEmbedContents), qt.Contains, "2024-02-03") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup(t, tt.args) + } + if err := tt.docs.WriteOutputs(tt.args.cfg, tt.args.appendOutputFiles); (err != nil) != tt.wantErr { + t.Errorf("WriteOutputs() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.assert != nil { + tt.assert(t, tt.args) + } + }) + } +} + +var minimalSpec = `--- +paths: {}`