Skip to content

Commit 0ad9173

Browse files
authored
Merge pull request #96 from snyk/feat/present-compiled-vu
Feat/present compiled vu
2 parents 4fca1ab + 81f72ae commit 0ad9173

File tree

6 files changed

+211
-50
lines changed

6 files changed

+211
-50
lines changed

.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,5 @@
55
.dccache
66
.vscode
77

8-
**/server
8+
**/server
9+
config.json

vervet-underground/internal/scraper/scraper.go

+64-21
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"io/ioutil"
99
"net/http"
1010
"net/url"
11+
"strings"
1112
"time"
1213

1314
"github.com/pkg/errors"
@@ -23,17 +24,28 @@ type Storage interface {
2324
// NotifyVersions tells the storage which versions are currently available.
2425
// This is the primary mechanism by which the storage layer discovers and
2526
// processes versions which are removed post-sunset.
26-
NotifyVersions(ctx context.Context, name string, versions []string, scrapeTime time.Time) error
27+
NotifyVersions(name string, versions []string, scrapeTime time.Time) error
28+
29+
// CollateVersions tells the storage to execute the compilation and
30+
// update all VU-formatted specs from all services and their
31+
// respective versions gathered.
32+
CollateVersions() error
2733

2834
// HasVersion returns whether the storage has already stored the service
2935
// API spec version at the given content digest.
30-
HasVersion(ctx context.Context, name string, version string, digest string) (bool, error)
36+
HasVersion(name string, version string, digest string) (bool, error)
3137

3238
// NotifyVersion tells the storage to store the given version contents at
3339
// the scrapeTime. The storage implementation must detect and ignore
3440
// duplicate version contents, as some services may not provide content
3541
// digest headers in their responses.
36-
NotifyVersion(ctx context.Context, name string, version string, contents []byte, scrapeTime time.Time) error
42+
NotifyVersion(name string, version string, contents []byte, scrapeTime time.Time) error
43+
44+
// Versions fetches the Storage Versions compiled by VU
45+
Versions() []string
46+
47+
// Version fetches the Storage Version spec compiled by VU
48+
Version(version string) ([]byte, error)
3749
}
3850

3951
// Scraper gets OpenAPI specs from a collection of services and updates storage
@@ -57,24 +69,32 @@ type Option func(*Scraper) error
5769
func New(cfg *config.ServerConfig, storage Storage, options ...Option) (*Scraper, error) {
5870
s := &Scraper{
5971
storage: storage,
60-
http: &http.Client{},
72+
http: &http.Client{Timeout: time.Second * 15},
6173
timeNow: time.Now,
6274
}
75+
err := setupScraper(s, cfg, options)
76+
if err != nil {
77+
return nil, err
78+
}
79+
return s, nil
80+
}
81+
82+
func setupScraper(s *Scraper, cfg *config.ServerConfig, options []Option) error {
6383
s.services = make([]service, len(cfg.Services))
6484
for i := range cfg.Services {
6585
u, err := url.Parse(cfg.Services[i] + "/openapi")
6686
if err != nil {
67-
return nil, errors.Wrapf(err, "invalid service %q", cfg.Services[i])
87+
return errors.Wrapf(err, "invalid service %q", cfg.Services[i])
6888
}
6989
s.services[i] = service{base: cfg.Services[i], url: u}
7090
}
7191
for i := range options {
7292
err := options[i](s)
7393
if err != nil {
74-
return nil, err
94+
return err
7595
}
7696
}
77-
return s, nil
97+
return nil
7898
}
7999

80100
// HTTPClient is a Scraper constructor Option that allows providing an
@@ -113,41 +133,52 @@ func (s *Scraper) Run(ctx context.Context) error {
113133
errs = multierr.Append(errs, err)
114134
}
115135
close(errCh)
136+
if errs != nil {
137+
return errs
138+
} else {
139+
err := s.collateVersions()
140+
errs = multierr.Append(errs, err)
141+
}
116142
return errs
117143
}
118144

119145
func (s *Scraper) scrape(ctx context.Context, scrapeTime time.Time, svc service) error {
120-
versions, err := s.getVersions(ctx, svc)
146+
versions, err := s.getVersions(svc)
121147
if err != nil {
122148
return errors.WithStack(err)
123149
}
124-
err = s.storage.NotifyVersions(ctx, svc.base, versions, scrapeTime)
150+
err = s.storage.NotifyVersions(svc.base, versions, scrapeTime)
125151
if err != nil {
126152
return errors.WithStack(err)
127153
}
128154
for i := range versions {
129155
// TODO: we might run this concurrently per live service pod if/when
130156
// we're more k8s aware, but we won't do that yet.
131-
contents, isNew, err := s.getNewVersion(ctx, svc, versions[i])
157+
contents, isNew, err := s.getNewVersion(svc, versions[i])
132158
if err != nil {
133159
return errors.WithStack(err)
134160
}
135161
if !isNew {
136162
continue
137163
}
138-
err = s.storage.NotifyVersion(ctx, svc.base, versions[i], contents, scrapeTime)
164+
err = s.storage.NotifyVersion(svc.base, versions[i], contents, scrapeTime)
139165
if err != nil {
140166
return errors.WithStack(err)
141167
}
142168
}
143169
return nil
144170
}
145171

146-
func (s *Scraper) getVersions(ctx context.Context, svc service) ([]string, error) {
147-
req, err := http.NewRequestWithContext(ctx, "GET", svc.url.String(), nil)
172+
func (s *Scraper) collateVersions() error {
173+
return s.storage.CollateVersions()
174+
}
175+
176+
func (s *Scraper) getVersions(svc service) ([]string, error) {
177+
req, err := http.NewRequest("GET", svc.url.String(), nil)
148178
if err != nil {
149179
return nil, errors.Wrap(err, "failed to create request")
150180
}
181+
151182
resp, err := s.http.Do(req)
152183
if err != nil {
153184
return nil, errors.Wrap(err, "request failed")
@@ -171,16 +202,18 @@ func httpError(r *http.Response) error {
171202
return errors.Errorf("request failed: HTTP %d", r.StatusCode)
172203
}
173204

174-
func (s *Scraper) getNewVersion(ctx context.Context, svc service, version string) ([]byte, bool, error) {
175-
isNew, err := s.hasNewVersion(ctx, svc, version)
205+
func (s *Scraper) getNewVersion(svc service, version string) ([]byte, bool, error) {
206+
// TODO: Services don't emit HEAD currently with compiled vervet
207+
// will need to enforce down the line
208+
isNew, err := s.hasNewVersion(svc, version)
176209
if err != nil {
177210
return nil, false, errors.WithStack(err)
178211
}
179212
if !isNew {
180213
return nil, false, nil
181214
}
182215

183-
req, err := http.NewRequestWithContext(ctx, "GET", svc.url.String()+"/"+version, nil)
216+
req, err := http.NewRequest("GET", svc.url.String()+"/"+version, nil)
184217
if err != nil {
185218
return nil, false, errors.Wrap(err, "failed to create request")
186219
}
@@ -192,7 +225,7 @@ func (s *Scraper) getNewVersion(ctx context.Context, svc service, version string
192225
if resp.StatusCode != http.StatusOK {
193226
return nil, false, httpError(resp)
194227
}
195-
if ct := resp.Header.Get("Content-Type"); ct != "application/json" {
228+
if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "application/json") {
196229
return nil, false, errors.Errorf("unexpected content type: %s", ct)
197230
}
198231
respContents, err := ioutil.ReadAll(resp.Body)
@@ -209,9 +242,9 @@ func (s *Scraper) getNewVersion(ctx context.Context, svc service, version string
209242
return respContents, true, nil
210243
}
211244

212-
func (s *Scraper) hasNewVersion(ctx context.Context, svc service, version string) (bool, error) {
245+
func (s *Scraper) hasNewVersion(svc service, version string) (bool, error) {
213246
// Check Digest to see if there's a new version
214-
req, err := http.NewRequestWithContext(ctx, "HEAD", svc.url.String()+"/"+version, nil)
247+
req, err := http.NewRequest("HEAD", svc.url.String()+"/"+version, nil)
215248
if err != nil {
216249
return false, errors.Wrap(err, "failed to create request")
217250
}
@@ -227,13 +260,23 @@ func (s *Scraper) hasNewVersion(ctx context.Context, svc service, version string
227260
if resp.StatusCode != http.StatusOK {
228261
return false, httpError(resp)
229262
}
230-
if ct := resp.Header.Get("Content-Type"); ct != "application/json" {
263+
264+
// Can be formed similarly: "application/json; charset: utf-8"
265+
if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "application/json") {
231266
return false, errors.Errorf("unexpected content type: %s", ct)
232267
}
233268
digest := storage.DigestHeader(resp.Header.Get("Digest"))
234269
if digest == "" {
235270
// Not providing a digest is fine, we'll just come back with a GET
236271
return true, nil
237272
}
238-
return s.storage.HasVersion(ctx, svc.base, version, digest)
273+
return s.storage.HasVersion(svc.base, version, digest)
274+
}
275+
276+
func (s *Scraper) Versions() []string {
277+
return s.storage.Versions()
278+
}
279+
280+
func (s *Scraper) Version(version string) ([]byte, error) {
281+
return s.storage.Version(version)
239282
}

vervet-underground/internal/scraper/scraper_test.go

+3-5
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ func TestScraper(t *testing.T) {
8888

8989
// No version digests should be known
9090
for _, test := range tests {
91-
ok, err := st.HasVersion(ctx, test.service, test.version, test.digest)
91+
ok, err := st.HasVersion(test.service, test.version, test.digest)
9292
c.Assert(err, qt.IsNil)
9393
c.Assert(ok, qt.IsFalse)
9494
}
@@ -99,7 +99,7 @@ func TestScraper(t *testing.T) {
9999

100100
// Version digests now known to storage
101101
for _, test := range tests {
102-
ok, err := st.HasVersion(ctx, test.service, test.version, test.digest)
102+
ok, err := st.HasVersion(test.service, test.version, test.digest)
103103
c.Assert(err, qt.IsNil)
104104
c.Assert(ok, qt.IsTrue)
105105
}
@@ -201,13 +201,11 @@ func TestScraperCollation(t *testing.T) {
201201

202202
// Version digests now known to storage
203203
for _, test := range tests {
204-
ok, err := st.HasVersion(ctx, test.service, test.version, test.digest)
204+
ok, err := st.HasVersion(test.service, test.version, test.digest)
205205
c.Assert(err, qt.IsNil)
206206
c.Assert(ok, qt.IsTrue)
207207
}
208208

209-
err = st.CollateVersions()
210-
c.Assert(err, qt.IsNil)
211209
collated, err := st.GetCollatedVersionSpecs()
212210
c.Assert(err, qt.IsNil)
213211
c.Assert(len(collated), qt.Equals, 4)

vervet-underground/internal/storage/mem/mem.go

+34-11
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,12 @@
55
package mem
66

77
import (
8-
"context"
9-
"github.com/pkg/errors"
108
"sort"
119
"sync"
1210
"time"
1311

1412
"github.com/getkin/kin-openapi/openapi3"
13+
"github.com/pkg/errors"
1514
"github.com/rs/zerolog/log"
1615
"github.com/snyk/vervet"
1716
"go.uber.org/multierr"
@@ -67,11 +66,11 @@ func New() *Storage {
6766
}
6867

6968
// NotifyVersions implements scraper.Storage.
70-
func (s *Storage) NotifyVersions(ctx context.Context, name string, versions []string, scrapeTime time.Time) error {
69+
func (s *Storage) NotifyVersions(name string, versions []string, scrapeTime time.Time) error {
7170
for _, version := range versions {
7271
// TODO: Add method to fetch contents here
7372
// TODO: implement notify versions; update sunset when versions are removed
74-
err := s.NotifyVersion(ctx, name, version, []byte{}, scrapeTime)
73+
err := s.NotifyVersion(name, version, []byte{}, scrapeTime)
7574
if err != nil {
7675
return err
7776
}
@@ -80,7 +79,7 @@ func (s *Storage) NotifyVersions(ctx context.Context, name string, versions []st
8079
}
8180

8281
// HasVersion implements scraper.Storage.
83-
func (s *Storage) HasVersion(ctx context.Context, name string, version string, digest string) (bool, error) {
82+
func (s *Storage) HasVersion(name string, version string, digest string) (bool, error) {
8483
s.mu.RLock()
8584
defer s.mu.RUnlock()
8685
revisions, ok := s.serviceVersionMappedRevisionSpecs[name][version]
@@ -93,7 +92,7 @@ func (s *Storage) HasVersion(ctx context.Context, name string, version string, d
9392
}
9493

9594
// NotifyVersion implements scraper.Storage.
96-
func (s *Storage) NotifyVersion(ctx context.Context, name string, version string, contents []byte, scrapeTime time.Time) error {
95+
func (s *Storage) NotifyVersion(name string, version string, contents []byte, scrapeTime time.Time) error {
9796
s.mu.Lock()
9897
defer s.mu.Unlock()
9998

@@ -145,6 +144,32 @@ func (s *Storage) NotifyVersion(ctx context.Context, name string, version string
145144
return nil
146145
}
147146

147+
// Versions implements scraper.Storage
148+
func (s *Storage) Versions() []string {
149+
s.mu.RLock()
150+
defer s.mu.RUnlock()
151+
stringVersions := make([]string, len(s.collatedVersions))
152+
for i, version := range s.collatedVersions {
153+
stringVersions[i] = version.String()
154+
}
155+
156+
return stringVersions
157+
}
158+
159+
// Version implements scraper.Storage
160+
func (s *Storage) Version(version string) ([]byte, error) {
161+
s.mu.RLock()
162+
defer s.mu.RUnlock()
163+
164+
parsedVersion, err := vervet.ParseVersion(version)
165+
if err != nil {
166+
return nil, err
167+
}
168+
169+
spec := s.collatedVersionedSpecs[*parsedVersion]
170+
return spec.MarshalJSON()
171+
}
172+
148173
// CollateVersions does the following:
149174
// - calls updateCollatedVersions for a slice of unique vervet.Version entries
150175
// - for each unique vervet.Version, run collateVersion to create a compiled VU openapi doc
@@ -206,13 +231,12 @@ func (s *Storage) collateVersion(version vervet.Version) error {
206231
// number of services maximum needed
207232
contentRevisions := make([]ContentRevision, 0)
208233

234+
s.mu.RLock()
209235
// preprocessing all relevant docs in byte format before combining
210236
for service, versionSlice := range s.serviceVersions {
211237
// If there is an exact match on versions 1-to-1
212-
s.mu.RLock()
213-
revisions, ok := s.serviceVersionMappedRevisionSpecs[service][version.String()]
214-
s.mu.RUnlock()
215238
var currentRevision ContentRevision
239+
revisions, ok := s.serviceVersionMappedRevisionSpecs[service][version.String()]
216240
if ok {
217241
// TODO: iterate through and take last contentRevision.
218242
// Could change to []ContentRevision in struct later
@@ -229,9 +253,7 @@ func (s *Storage) collateVersion(version vervet.Version) error {
229253
continue
230254
}
231255

232-
s.mu.RLock()
233256
revisions, ok = s.serviceVersionMappedRevisionSpecs[service][resolvedVersion.String()]
234-
s.mu.RUnlock()
235257
if ok {
236258
// TODO: iterate through and take last contentRevision.
237259
// Could change to []ContentRevision in struct later
@@ -242,6 +264,7 @@ func (s *Storage) collateVersion(version vervet.Version) error {
242264
}
243265
}
244266
}
267+
s.mu.RUnlock()
245268

246269
err := s.mergeContentRevisions(version, contentRevisions)
247270
if err != nil {

0 commit comments

Comments
 (0)