Merge pull request #341 from snyk/fix/prevent-spec-over-generation

fix: prevent specs from being generated when nothing has changed

go.mod

@@ -43,8 +43,8 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -167,8 +167,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -191,8 +191,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/backstage/backstage_test.go

@@ -127,13 +127,9 @@ func TestLoadVersionsNoApis(t *testing.T) {
     - Registry_2021-06-07_experimental
     - Registry_2021-06-13_beta
     - Registry_2021-06-13_experimental
-    - Registry_2021-08-20_beta
     - Registry_2021-08-20_experimental
-    - Registry_2023-06-01_beta
     - Registry_2023-06-01_experimental
-    - Registry_2023-06-02_beta
     - Registry_2023-06-02_experimental
-    - Registry_2023-06-03_beta
     - Registry_2023-06-03_experimental
 ---
 `[1:]+string(vervetAPIs))
@@ -163,13 +159,9 @@ func TestLoadVersionsSomeApis(t *testing.T) {
     - Registry_2021-06-07_experimental
     - Registry_2021-06-13_beta
     - Registry_2021-06-13_experimental
-    - Registry_2021-08-20_beta
     - Registry_2021-08-20_experimental
-    - Registry_2023-06-01_beta
     - Registry_2023-06-01_experimental
-    - Registry_2023-06-02_beta
     - Registry_2023-06-02_experimental
-    - Registry_2023-06-03_beta
     - Registry_2023-06-03_experimental
     - someOtherApi
 ---

spec.go

@@ -217,8 +217,6 @@ func setOperationByName(path *openapi3.PathItem, opName string, op *openapi3.Ope
 	}
 }
 
-var stabilities = []Stability{StabilityExperimental, StabilityBeta, StabilityGA}
-
 func newSpecVersions(specs resourceVersionsSlice) (*SpecVersions, error) {
 	versions := specs.versions()
 	var versionDates []time.Time
@@ -229,20 +227,23 @@ func newSpecVersions(specs resourceVersionsSlice) (*SpecVersions, error) {
 	}
 
 	documentVersions := map[Version]*openapi3.T{}
-	for _, date := range versionDates {
-		for _, stability := range stabilities {
-			v := Version{Date: date, Stability: stability}
-			doc, err := specs.at(v)
-			if err == ErrNoMatchingVersion {
-				continue
-			} else if err != nil {
-				return nil, err
-			}
-			if doc.Extensions == nil {
-				doc.Extensions = map[string]interface{}{}
+	for _, spec := range specs {
+		for _, doc := range spec.versions {
+			for _, stability := range doc.Version.Stability.Resolvable() {
+				v := Version{Date: doc.Version.Date, Stability: stability}
+				doc, err := specs.at(v)
+				if err == ErrNoMatchingVersion {
+					continue
+				} else if err != nil {
+					return nil, err
+				}
+				documentVersions[v] = doc
+				if doc.Extensions == nil {
+					doc.Extensions = map[string]interface{}{}
+				}
+				doc.Extensions[ExtSnykApiVersion] = v.String()
+				documentVersions[v] = doc
 			}
-			doc.Extensions[ExtSnykApiVersion] = v.String()
-			documentVersions[v] = doc
 		}
 	}
 	sv := &SpecVersions{

spec_test.go

@@ -22,13 +22,9 @@ func TestSpecs(t *testing.T) {
 		MustParseVersion("2021-06-13~experimental"),
 		MustParseVersion("2021-06-13~beta"),
 		MustParseVersion("2021-08-20~experimental"),
-		MustParseVersion("2021-08-20~beta"),
 		MustParseVersion("2023-06-01~experimental"),
-		MustParseVersion("2023-06-01~beta"),
 		MustParseVersion("2023-06-02~experimental"),
-		MustParseVersion("2023-06-02~beta"),
 		MustParseVersion("2023-06-03~experimental"),
-		MustParseVersion("2023-06-03~beta"),
 	})
 
 	type expectResourceVersion struct {
@@ -154,3 +150,16 @@ func TestSpecs(t *testing.T) {
 		}
 	}
 }
+
+func TestDoesNotGenerateSpecsForStabilitiesWithNoChanges(t *testing.T) {
+	c := qt.New(t)
+	specs, err := LoadSpecVersions(testdata.Path("overgen"))
+	c.Assert(err, qt.IsNil)
+	versions := specs.Versions()
+	c.Assert(versions, qt.ContentEquals, VersionSlice{
+		MustParseVersion("2021-08-11"),
+		MustParseVersion("2021-08-11~beta"),
+		MustParseVersion("2021-08-11~experimental"),
+		MustParseVersion("2023-11-15~experimental"),
+	})
+}

testdata/catalog-vervet-apis.yaml

@@ -116,29 +116,6 @@ spec:
 # Generated by vervet, DO NOT EDIT
 apiVersion: backstage.io/v1alpha1
 kind: API
-metadata:
-  name: Registry_2021-08-20_beta
-  title: Registry 2021-08-20 beta
-  annotations:
-    api.snyk.io/generated-by: vervet
-  labels:
-    api.snyk.io/version-date: "2021-08-20"
-    api.snyk.io/version-lifecycle: released
-    api.snyk.io/version-stability: beta
-  tags:
-    - 2021-08
-    - beta
-    - released
-spec:
-  type: openapi
-  lifecycle: beta
-  owner: someone-else
-  definition:
-    $text: output/2021-08-20~beta/spec.json
----
-# Generated by vervet, DO NOT EDIT
-apiVersion: backstage.io/v1alpha1
-kind: API
 metadata:
   name: Registry_2021-08-20_experimental
   title: Registry 2021-08-20 experimental
@@ -162,29 +139,6 @@ spec:
 # Generated by vervet, DO NOT EDIT
 apiVersion: backstage.io/v1alpha1
 kind: API
-metadata:
-  name: Registry_2023-06-01_beta
-  title: Registry 2023-06-01 beta
-  annotations:
-    api.snyk.io/generated-by: vervet
-  labels:
-    api.snyk.io/version-date: "2023-06-01"
-    api.snyk.io/version-lifecycle: released
-    api.snyk.io/version-stability: beta
-  tags:
-    - 2023-06
-    - beta
-    - released
-spec:
-  type: openapi
-  lifecycle: beta
-  owner: someone-else
-  definition:
-    $text: output/2023-06-01~beta/spec.json
----
-# Generated by vervet, DO NOT EDIT
-apiVersion: backstage.io/v1alpha1
-kind: API
 metadata:
   name: Registry_2023-06-01_experimental
   title: Registry 2023-06-01 experimental
@@ -208,29 +162,6 @@ spec:
 # Generated by vervet, DO NOT EDIT
 apiVersion: backstage.io/v1alpha1
 kind: API
-metadata:
-  name: Registry_2023-06-02_beta
-  title: Registry 2023-06-02 beta
-  annotations:
-    api.snyk.io/generated-by: vervet
-  labels:
-    api.snyk.io/version-date: "2023-06-02"
-    api.snyk.io/version-lifecycle: released
-    api.snyk.io/version-stability: beta
-  tags:
-    - 2023-06
-    - beta
-    - released
-spec:
-  type: openapi
-  lifecycle: beta
-  owner: someone-else
-  definition:
-    $text: output/2023-06-02~beta/spec.json
----
-# Generated by vervet, DO NOT EDIT
-apiVersion: backstage.io/v1alpha1
-kind: API
 metadata:
   name: Registry_2023-06-02_experimental
   title: Registry 2023-06-02 experimental
@@ -254,29 +185,6 @@ spec:
 # Generated by vervet, DO NOT EDIT
 apiVersion: backstage.io/v1alpha1
 kind: API
-metadata:
-  name: Registry_2023-06-03_beta
-  title: Registry 2023-06-03 beta
-  annotations:
-    api.snyk.io/generated-by: vervet
-  labels:
-    api.snyk.io/version-date: "2023-06-03"
-    api.snyk.io/version-lifecycle: released
-    api.snyk.io/version-stability: beta
-  tags:
-    - 2023-06
-    - beta
-    - released
-spec:
-  type: openapi
-  lifecycle: beta
-  owner: someone-else
-  definition:
-    $text: output/2023-06-03~beta/spec.json
----
-# Generated by vervet, DO NOT EDIT
-apiVersion: backstage.io/v1alpha1
-kind: API
 metadata:
   name: Registry_2023-06-03_experimental
   title: Registry 2023-06-03 experimental