Skip to content

Commit

Permalink
Merge pull request #353 from snyk/feat/new-build
Browse files Browse the repository at this point in the history
feat: add new codepath for building specs
  • Loading branch information
jgresty authored Aug 1, 2024
2 parents 1073ffe + 0a9e147 commit dcdba9a
Show file tree
Hide file tree
Showing 15 changed files with 1,210 additions and 11 deletions.
6 changes: 3 additions & 3 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ defaults: &defaults
resource_class: small
working_directory: ~/vervet
docker:
- image: cimg/go:1.21-node
- image: cimg/go:1.22-node

test_defaults: &test_defaults
resource_class: medium
Expand Down Expand Up @@ -54,7 +54,7 @@ jobs:
command: npm install -g @stoplight/[email protected]
- checkout
- go/install:
version: 1.21.3
version: 1.22.3
- go/mod-download-cached
- run:
name: Verify testdata/output up to date
Expand All @@ -65,7 +65,7 @@ jobs:

lint:
docker:
- image: golangci/golangci-lint:v1.51.0
- image: golangci/golangci-lint:v1.59.1
steps:
- checkout
- run:
Expand Down
2 changes: 0 additions & 2 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ linters:
- errname
# - errorlint - Good to have linter but potential to introduce breaking changes.
# - exhaustive - Causes too much noise at the moment.
- exportloopref
- gci
- gocritic
- goconst
Expand All @@ -43,7 +42,6 @@ linters:
- gocyclo
# - gosec - Good to have linter but potential to introduce breaking changes.
# - forbidigo - Good to have linter but potential to introduce breaking changes.
- ifshort
- lll
- misspell
- nakedret
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
ARG GO_VERSION=1.21.3
ARG GO_VERSION=1.22.3

###############
# Build stage #
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ GO_BIN=$(shell pwd)/.bin/go

SHELL:=env PATH=$(GO_BIN):$(PATH) $(SHELL)

GOCI_LINT_V?=v1.54.2
GOCI_LINT_V?=v1.59.1

.PHONY: all
all: lint test build
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/snyk/vervet/v7

go 1.21
go 1.22

require (
cloud.google.com/go/storage v1.34.1
Expand Down
1 change: 1 addition & 0 deletions internal/cmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ var CLIApp = cli.App{
Commands: []*cli.Command{
&BackstageCommand,
&BuildCommand,
&SimpleBuildCommand,
&FilterCommand,
&GenerateCommand,
&LocalizeCommand,
Expand Down
29 changes: 29 additions & 0 deletions internal/cmd/compiler.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/snyk/vervet/v7/config"
"github.com/snyk/vervet/v7/internal/compiler"
"github.com/snyk/vervet/v7/internal/simplebuild"
)

// BuildCommand is the `vervet build` subcommand.
Expand All @@ -30,6 +31,34 @@ var BuildCommand = cli.Command{
Action: Build,
}

var SimpleBuildCommand = cli.Command{
Name: "simplebuild",
Usage: "Build versioned resources into versioned OpenAPI specs",
ArgsUsage: "[input resources root]",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "config",
Aliases: []string{"c", "conf"},
Usage: "Project configuration file",
},
&cli.StringFlag{
Name: "include",
Aliases: []string{"I"},
Usage: "OpenAPI specification to include in build output",
},
},
Action: SimpleBuild,
}

func SimpleBuild(ctx *cli.Context) error {
project, err := projectFromContext(ctx)
if err != nil {
return err
}
err = simplebuild.Build(ctx.Context, project)
return err
}

// Build compiles versioned resources into versioned API specs.
func Build(ctx *cli.Context) error {
project, err := projectFromContext(ctx)
Expand Down
4 changes: 2 additions & 2 deletions internal/compiler/compiler.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ func (c *Compiler) writeEmbedGo(pkgName string, a *api, versionSpecFiles []strin
return err
}
defer f.Close()
err = embedGoTmpl.Execute(f, struct {
err = EmbedGoTmpl.Execute(f, struct {
Package string
API *api
VersionSpecFiles []string
Expand All @@ -282,7 +282,7 @@ func (c *Compiler) writeEmbedGo(pkgName string, a *api, versionSpecFiles []strin
return nil
}

var embedGoTmpl = template.Must(template.New("embed.go").Parse(`
var EmbedGoTmpl = template.Must(template.New("embed.go").Parse(`
// Code generated by Vervet. DO NOT EDIT.
package {{ .Package }}
Expand Down
236 changes: 236 additions & 0 deletions internal/simplebuild/build.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,236 @@
package simplebuild

import (
"context"
"fmt"
"path/filepath"
"slices"
"time"

"github.com/getkin/kin-openapi/openapi3"

"github.com/snyk/vervet/v7"
"github.com/snyk/vervet/v7/config"
"github.com/snyk/vervet/v7/internal/files"
)

func Build(ctx context.Context, project *config.Project) error {
for _, apiConfig := range project.APIs {
operations, err := LoadPaths(ctx, apiConfig)
if err != nil {
return err
}
for _, op := range operations {
op.Annotate()
}
docs, err := operations.Build()
if err != nil {
return err
}

err = docs.ApplyOverlays(ctx, apiConfig.Overlays)
if err != nil {
return err
}

if apiConfig.Output != nil {
err = docs.WriteOutputs(*apiConfig.Output)
if err != nil {
return err
}
}
}
return nil
}

type OpKey struct {
Path string
Method string
}

type VersionedOp struct {
Version vervet.Version
Operation *openapi3.Operation
ResourceName string
}

type VersionSet []VersionedOp

type Operations map[OpKey]VersionSet

type VersionedDoc struct {
VersionDate time.Time
Doc *openapi3.T
}
type DocSet []VersionedDoc

func (ops Operations) Build() (DocSet, error) {
versionDates := ops.VersionDates()
output := make(DocSet, len(versionDates))
for idx, versionDate := range versionDates {
output[idx] = VersionedDoc{
Doc: &openapi3.T{},
VersionDate: versionDate,
}
refResolver := NewRefResolver(output[idx].Doc)
for path, spec := range ops {
op := spec.GetLatest(versionDate)
if op == nil {
continue
}
output[idx].Doc.AddOperation(path.Path, path.Method, op)
err := refResolver.Resolve(op)
if err != nil {
return nil, err
}
}
}
return output, nil
}

func (ops Operations) VersionDates() []time.Time {
versionSet := map[time.Time]struct{}{}
for _, opSet := range ops {
for _, op := range opSet {
versionSet[op.Version.Date] = struct{}{}
}
}
uniqueVersions := make([]time.Time, len(versionSet))
idx := 0
for version := range versionSet {
uniqueVersions[idx] = version
idx++
}
return uniqueVersions
}

func LoadPaths(ctx context.Context, api *config.API) (Operations, error) {
operations := map[OpKey]VersionSet{}

for _, resource := range api.Resources {
paths, err := ResourceSpecFiles(resource)
if err != nil {
return nil, err
}
for _, path := range paths {
versionDir := filepath.Dir(path)
versionStr := filepath.Base(versionDir)
resourceName := filepath.Base(filepath.Dir(filepath.Dir(path)))

doc, err := vervet.NewDocumentFile(path)
if err != nil {
return nil, fmt.Errorf("failed to load spec from %q: %w", path, err)
}

stabilityStr, err := vervet.ExtensionString(doc.T.Extensions, vervet.ExtSnykApiStability)
if err != nil {
return nil, err
}
if stabilityStr != "ga" {
versionStr = fmt.Sprintf("%s~%s", versionStr, stabilityStr)
}
version, err := vervet.ParseVersion(versionStr)
if err != nil {
return nil, fmt.Errorf("invalid version %q", versionStr)
}

doc.InternalizeRefs(ctx, nil)
err = doc.ResolveRefs()
if err != nil {
return nil, fmt.Errorf("failed to localize refs: %w", err)
}

for pathName, pathDef := range doc.T.Paths {
for opName, opDef := range pathDef.Operations() {
k := OpKey{
Path: pathName,
Method: opName,
}
if operations[k] == nil {
operations[k] = []VersionedOp{}
}
operations[k] = append(operations[k], VersionedOp{
Version: version,
Operation: opDef,
ResourceName: resourceName,
})
}
}
}
}

return operations, nil
}

func ResourceSpecFiles(resource *config.ResourceSet) ([]string, error) {
return files.LocalFSSource{}.Match(resource)
}

func (vs VersionSet) GetLatest(before time.Time) *openapi3.Operation {
var latest *VersionedOp
for _, versionedOp := range vs {
if versionedOp.Version.Date.After(before) {
continue
}
if latest == nil {
latest = &versionedOp
continue
}
// Higher stabilities always take precedent
if versionedOp.Version.Stability.Compare(latest.Version.Stability) < 0 {
continue
}
if versionedOp.Version.Compare(latest.Version) > 0 {
latest = &versionedOp
}
}
if latest == nil {
return nil
}
return latest.Operation
}

// Annotate adds Snyk specific extensions to openapi operations. These
// extensions are:
// - x-snyk-api-version: version where the operation was defined
// - x-snyk-api-releases: all versions of this api
// - x-snyk-deprecated-by: if there is a later version of this operation, the
// version of that operation
// - x-snyk-sunset-eligible: the date after this operation can be sunset
// - x-snyk-api-resource: what resource this operation acts on
// - x-snyk-api-lifecycle: status of the operation, can be one of:
// [ unreleased, released, deprecated, sunset ]
func (vs VersionSet) Annotate() {
slices.SortFunc(vs, func(a, b VersionedOp) int {
return a.Version.Compare(b.Version)
})

count := len(vs)

releases := make([]string, count)
for idx, op := range vs {
releases[idx] = op.Version.String()
}

for idx, op := range vs {
if op.Operation.Extensions == nil {
op.Operation.Extensions = make(map[string]interface{}, 6)
}
op.Operation.Extensions[vervet.ExtSnykApiResource] = op.ResourceName
op.Operation.Extensions[vervet.ExtSnykApiVersion] = op.Version.String()
op.Operation.Extensions[vervet.ExtSnykApiReleases] = releases
op.Operation.Extensions[vervet.ExtSnykApiLifecycle] = op.Version.LifecycleAt(time.Time{}).String()
if idx < (count - 1) {
laterVersion := vs[idx+1].Version
// Sanity check the later version actually deprecates this one
if !op.Version.DeprecatedBy(laterVersion) {
continue
}
op.Operation.Extensions[vervet.ExtSnykDeprecatedBy] = laterVersion.String()
sunsetDate, ok := op.Version.Sunset(laterVersion)
if ok {
op.Operation.Extensions[vervet.ExtSnykSunsetEligible] = sunsetDate.Format("2006-01-02")
}
}
}
}
Loading

0 comments on commit dcdba9a

Please sign in to comment.