Merge pull request #347 from snyk/feat/merge-modules

Feat: merge vervet unground into main modules

.circleci/config.yml

@@ -2,6 +2,7 @@ version: 2.1
 
 orbs:
   go: circleci/[email protected]
+  node: circleci/[email protected]
   publish: snyk/publish@1
   prodsec: snyk/[email protected]
 
@@ -11,20 +12,12 @@ defaults: &defaults
   docker:
     - image: cimg/go:1.21-node
 
-test_vu_defaults: &test_vu_defaults
+test_defaults: &test_defaults
   resource_class: medium
-  working_directory: ~/vervet/vervet-underground
+  working_directory: ~/vervet
   machine:
     image: ubuntu-2004:2023.10.1
 
-vu_defaults: &vu_defaults
-  resource_class: small
-  working_directory: ~/vervet/vervet-underground
-  docker:
-    - image: cimg/go:1.21-node
-  environment:
-    DOCKER_BUILDKIT: 1
-
 ignore_main_branch_filter: &ignore_main_branch_filter
   filters:
     branches:
@@ -53,12 +46,15 @@ commands:
 
 jobs:
   test:
-    <<: *defaults
+    <<: *test_defaults
     steps:
+      - node/install
       - run:
           name: Install spectral
-          command: sudo npm install -g @stoplight/[email protected]
+          command: npm install -g @stoplight/[email protected]
       - checkout
+      - go/install:
+          version: 1.21.3
       - go/mod-download-cached
       - run:
           name: Verify testdata/output up to date
@@ -67,17 +63,6 @@ jobs:
           name: Run tests
           command: go test ./... -count=1
 
-  test-vu:
-    <<: *test_vu_defaults
-    steps:
-      - checkout:
-          path: ~/vervet
-      - go/install:
-          version: 1.21.3
-      - go/mod-download-cached
-      - run:
-          command: make test
-
   lint:
     docker:
       - image: golangci/golangci-lint:v1.51.0
@@ -86,16 +71,8 @@ jobs:
       - run:
           command: golangci-lint run -v ./...
 
-  lint-vu:
-    docker:
-      - image: golangci/golangci-lint:v1.51.0
-    steps:
-      - checkout
-      - run:
-          command: cd vervet-underground && golangci-lint run -v ./...
-
   security-scans:
-    <<: *vu_defaults
+    <<: *defaults
     steps:
       - checkout
       - prodsec/security_scans:
@@ -104,14 +81,14 @@ jobs:
           org: platformeng_api
 
   build-vu:
-    <<: *vu_defaults
+    <<: *defaults
     steps:
       - checkout:
           path: ~/vervet
       - gcr_auth
       - run:
           name: Build Docker Image
-          command: make build-docker
+          command: make build-docker APP=vervet-underground
       - publish/save-image:
           image_name: vervet-underground
 
@@ -135,18 +112,10 @@ workflows:
           name: Test
           <<: *ignore_main_branch_filter
 
-      - test-vu:
-          name: Test VU
-          <<: *ignore_main_branch_filter
-
       - lint:
           name: Lint
           <<: *ignore_main_branch_filter
 
-      - lint-vu:
-          name: Lint VU
-          <<: *ignore_main_branch_filter
-
       - security-scans:
           name: Security Scans
           context:
@@ -157,8 +126,8 @@ workflows:
           name: Build Docker Image
           context: snyk-docker-build
           requires:
-            - Test VU
-            - Lint VU
+            - Test
+            - Lint
 
       - prodsec/container-scan:
           name: Scan VU Container

vervet-underground/Dockerfile → Dockerfile

@@ -18,10 +18,10 @@ RUN go mod download
 COPY . .
 RUN --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=0 \
-    go build -v -o /go/bin/app ./cmd/api/main.go
+    go build -v -o /go/bin/app ./cmd/vu-api/main.go
 RUN --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=0 \
-    go build -v -o /go/bin/scraper ./cmd/scraper/main.go
+    go build -v -o /go/bin/scraper ./cmd/vu-scraper/main.go
 
 #################
 # Runtime stage #

Makefile

@@ -1,3 +1,4 @@
+APP:=vervet
 GO_BIN=$(shell pwd)/.bin/go
 
 SHELL:=env PATH=$(GO_BIN):$(PATH) $(SHELL)
@@ -8,9 +9,17 @@ GOCI_LINT_V?=v1.54.2
 all: lint test build
 
 .PHONY: build
-build:
+build: vervet vu-api vu-scraper
+
+vervet:
 	go build -a -o vervet ./cmd/vervet
 
+vu-api:
+	go build -a -o vu-api ./cmd/vu-api
+
+vu-scraper:
+	go build -a -o vu-scraper ./cmd/vu-scraper
+
 # Run go mod tidy yourself
 
 #----------------------------------------------------------------------------------
@@ -31,7 +40,13 @@ lint:
 
 .PHONY: lint-docker
 lint-docker:
-	docker run --rm -v $(shell pwd):/vervet -w /vervet golangci/golangci-lint:${GOCI_LINT_V} golangci-lint run -v ./...
+	docker run --rm -v $(shell pwd):/${APP} -w /${APP} golangci/golangci-lint:${GOCI_LINT_V} golangci-lint run -v ./...
+
+.PHONY: build-docker
+build-docker:
+	docker build \
+		-t ${APP}:${CIRCLE_WORKFLOW_ID} \
+		-t gcr.io/snyk-main/${APP}:${CIRCLE_SHA1} .
 
 #----------------------------------------------------------------------------------
 #  Ignores the test cache and forces a full test suite execution
@@ -52,7 +67,7 @@ clean:
 	$(RM) vervet
 
 .PHONY: install-tools
-install-tools: 
+install-tools:
 ifndef CI
 	mkdir -p ${GO_BIN}
 	curl -sSfL 'https://raw.githubusercontent.com/golangci/golangci-lint/${GOCI_LINT_V}/install.sh' | sh -s -- -b ${GO_BIN} ${GOCI_LINT_V}
@@ -66,3 +81,8 @@ format: ## Format source code with gofmt and golangci-lint
 .PHONY: tidy
 tidy:
 	go mod tidy -v
+
+.PHONY: start-vu
+start-vu:
+	go run cmd/vu-scraper/main.go
+	go run cmd/vu-api/main.go

README.md

@@ -250,3 +250,150 @@ To solve this:
 3. Remove the old OpenAPI specs.
 
 [Example PR](https://github.com/snyk/registry/pull/33489/files)
+
+
+# Vervet Underground
+
+# What Vervet Underground does and why
+
+In order to understand _why_ Vervet Underground exists and the problem it solves, you should first become familiar with the [API versioning scheme](https://github.com/snyk/sweater-comb/blob/main/docs/principles/version.md) that Vervet supports. The main idea is, an API may be authored in parts, each of those parts may be versioned, and all the distinct versions are assembled to produce a cohesive timeline of versions for the entire service.
+
+Just as Vervet compiles a timeline of OpenAPI versions for a single service from independently versioned parts, Vervet Underground (VU) compiles a timeline of OpenAPI spec versions for a SaaS from independently versioned microservices, each of which contributes parts of the SaaS API.
+
+# Service API aggregation by example
+
+## The pet store gets microservices
+
+To illustrate how this works in practice, let's deconstruct a pet store into two services:
+
+* `petfood.default.svc.cluster.local`, which knows about pet food.
+* `animals.default.svc.cluster.local`, which knows about animals.
+
+For sake of example, let's assume the following versions are published by each service:
+
+petfood has:
+- `2021-07-04~experimental`
+- `2021-08-09~beta`
+- `2021-08-09~experimental` (beta released, with some parts still experimental, so both are published)
+- `2021-09-14` .            (first GA)
+- `2021-09-14~beta`
+- `2021-09-14~experimental`
+
+animals has:
+- `2021-09-10~experimental`
+- `2021-10-04~experimental`
+- `2021-10-12~beta`
+- `2021-10-12~experimental`
+- `2021-11-05`
+- `2021-11-05~beta`
+- `2021-11-05~experimental`
+
+And, the OpenAPI spec for each version is available at `/openapi`. `/openapi` provides a JSON array of OpenAPI versions supported by the service, and `/openapi/{version}` fetches the OpenAPI spec for that particular `{version}`. For example, `GET http://petfood.default.svc.cluster.local/openapi/2021-09-14~experimental`.
+
+There is some nuance to this to be aware of. You'll notice that some dates have multiple versions with different stabilities. This can happen because on that date, there is more than one API version available at different stability levels.
+
+There are also some assumptions. These services are cooperatively contributing to the public pet store SaaS API. They cannot conflict with each other -- no overwriting each other's OpenAPI components, or publishing conflicting paths.
+
+## Pet store's public API
+
+From these service versions, what versions of the pet store API are published to the public SaaS consumer? Well, the union of all of them! So we should be able to enumerate these versions from the public SaaS API:
+
+```
+GET https://api.petstore.example.com/openapi
+
+200 OK
+Content-Type: application/json
+[
+  `2021-07-04~experimental`,   // Contains only petfood so far...
+  `2021-08-09~experimental`,
+  `2021-08-09~beta`,
+  `2021-09-10~beta`            // Petfood only (no beta version of animals yet...)
+  `2021-09-10~experimental`,   // First animals (experimental version) + petfood
+  `2021-09-14`,                // Petfood GA only, animals isn't GA yet
+  `2021-09-14~beta`,
+  `2021-09-14~experimental`,
+  `2021-10-04`,
+  `2021-10-04~beta`,
+  `2021-10-04~experimental`,
+  `2021-10-12`,
+  `2021-10-12~beta`,
+  `2021-10-12~experimental`,
+  `2021-11-05`,                // First GA release of animals, also has petfood GA (from most recent 2021-09-14)
+  `2021-11-05~beta`,
+  `2021-11-05~experimental`,
+]
+```
+
+## Past API releases can change
+
+The examples so far have been kept simple by assuming released API versions do not change. In practice, non-breaking changes are allowed to be made to existing versions of the API at any time. [Non-breaking changes](https://github.com/snyk/sweater-comb/blob/main/docs/principles/version.md#breaking-changes) must be additive and optional. It is fine to add new HTTP methods, endpoints, request parameters or response fields, so long that the added parameters or fields are not _required_ -- which existing generated client code would have no way of knowing about.
+
+With VU, we should be able to request a version of the public SaaS API _as it was on a given date_, regardless of the version release date.
+
+### Tracking a non-breaking change by example
+
+Let's assume that on 2021-11-08, the petfood service team adds a PATCH method to all their resources, to allow existing orders to be modified before they ship. It's a reasonable thing to do -- a new HTTP method doesn't break existing behavior! The team adds this method to every active (non-sunset) version retroactively -- why not? it was essentially the same backend code to implement it!
+
+Vervet Underground not only compiles the initially published APIs from component services, it tracks changes in these APIs and updates its SaaS-level view of the API accordingly. So, VU scrapes the /openapi endpoints of its services periodically and detects the API changes on 2021-11-08, even though no new API was explicitly released that day, and now represents it as a new "discovered" version:
+
+```
+GET https://api.petstore.example.com/openapi
+
+200 OK
+Content-Type: application/json
+[
+  `2021-07-04~experimental`,
+  ...
+  `2021-11-05`,
+  `2021-11-05~beta`,
+  `2021-11-05~experimental`,
+  `2021-11-08`,                // A wild new version appears!
+  `2021-11-08~beta`,
+  `2021-11-08~experimental`,
+]
+```
+
+## How VU tracks non-breaking changes (even our versions have versions!)
+
+VU scrapes the `/openapi` endpoints of each service and tracks the changes in each version found. This may be stored in a directory structure, such as:
+
+```
+services/petfood
+├── 2021-09-14~experimental
+│   ├── 2021-09-14_11_23_24.spec.yaml
+│   └── 2021-11-08_13_14_15.spec.yaml
+...
+```
+
+where the scraped OpenAPI is compared against the most recent last snapshot: if they differ, a new snapshot is taken.
+
+When the new `2021-11-08` snapshot is detected, this triggers a rebuild of the top-level SaaS OpenAPI specs with that new version added. The arrow of time eventually flows only one way and storage is cheap, so it's assumed that the compiled OpenAPI specs will be statically compiled up-front as service API changes are detected.
+
+This snapshot version should not be taken to represent a breaking-change release, which has different deprecation and sunsetting implications. It is only used to represent what the API looked like at a given point in time.
+
+### What this means from a public API perspective
+
+If a version date prior to `2021-11-08` resolves to `2021-09-14~experimental`, you should see the `2021-09-14~experimental` contributions to the API _as it would have appeared at that time_, in other words, you should see a view based on the `2021-07-04_11_23_24.spec.yaml` snapshot of `2021-09-14~experimental`.
+
+If a version date after `2021-11-08` matches `2021-09-14~experimental`, let's say a request for `2021-12-10~experimental`, then you should see it as it would appear after the non-breaking change, `2021-11-08_13_14_15.spec.yaml`.
+
+# Roadmap
+
+## Minimum Viable
+
+VU aggregates OpenAPI specs from multiple services and serves them up in a single place for:
+
+- Docs
+    - Docs will likely either render public `/openapi` directly client-side or periodically scrape & update
+- Routing public v3 `/openapi` to VU's aggregated OpenAPI versions
+- Add Akamai configuration to serve the blockstored specs initially for `/openapi`
+    - Formal frontend presentation will be later
+
+This unblocks docs for multi-service decomp.
+
+### Details
+
+- Could use block storage with a history of changes per service per version
+- Static config that tells VU where to scrape upstream OpenAPI from services (registry and friends)
+- Cron job to periodically scrape and update
+    - Or we can try to make this push and set up a webhook...

cmd/vervet/main.go

@@ -4,7 +4,7 @@ import (
 	"log"
 	"os"
 
-	"github.com/snyk/vervet/v6/internal/cmd"
+	"github.com/snyk/vervet/v7/internal/cmd"
 )
 
 func main() {

vervet-underground/cmd/api/main.go → cmd/vu-api/main.go

@@ -15,12 +15,12 @@ import (
 	"github.com/rs/zerolog/log"
 	"golang.org/x/sync/errgroup"
 
-	"vervet-underground/config"
-	"vervet-underground/internal/handler"
-	"vervet-underground/internal/storage"
-	"vervet-underground/internal/storage/disk"
-	"vervet-underground/internal/storage/gcs"
-	"vervet-underground/internal/storage/s3"
+	"github.com/snyk/vervet/v7/config"
+	"github.com/snyk/vervet/v7/internal/handler"
+	"github.com/snyk/vervet/v7/internal/storage"
+	"github.com/snyk/vervet/v7/internal/storage/disk"
+	"github.com/snyk/vervet/v7/internal/storage/gcs"
+	"github.com/snyk/vervet/v7/internal/storage/s3"
 )
 
 func main() {
@@ -37,7 +37,7 @@ func main() {
 
 	var cfg *config.ServerConfig
 	var err error
-	if cfg, err = config.Load(configJson); err != nil {
+	if cfg, err = config.LoadServerConfig(configJson); err != nil {
 		log.Fatal().Err(err).Msg("unable to load config")
 	}