8
8
"io/ioutil"
9
9
"net/http"
10
10
"net/url"
11
+ "strings"
11
12
"time"
12
13
13
14
"github.com/pkg/errors"
@@ -23,17 +24,28 @@ type Storage interface {
23
24
// NotifyVersions tells the storage which versions are currently available.
24
25
// This is the primary mechanism by which the storage layer discovers and
25
26
// processes versions which are removed post-sunset.
26
- NotifyVersions (ctx context.Context , name string , versions []string , scrapeTime time.Time ) error
27
+ NotifyVersions (name string , versions []string , scrapeTime time.Time ) error
28
+
29
+ // CollateVersions tells the storage to execute the compilation and
30
+ // update all VU-formatted specs from all services and their
31
+ // respective versions gathered.
32
+ CollateVersions () error
27
33
28
34
// HasVersion returns whether the storage has already stored the service
29
35
// API spec version at the given content digest.
30
- HasVersion (ctx context. Context , name string , version string , digest string ) (bool , error )
36
+ HasVersion (name string , version string , digest string ) (bool , error )
31
37
32
38
// NotifyVersion tells the storage to store the given version contents at
33
39
// the scrapeTime. The storage implementation must detect and ignore
34
40
// duplicate version contents, as some services may not provide content
35
41
// digest headers in their responses.
36
- NotifyVersion (ctx context.Context , name string , version string , contents []byte , scrapeTime time.Time ) error
42
+ NotifyVersion (name string , version string , contents []byte , scrapeTime time.Time ) error
43
+
44
+ // Versions fetches the Storage Versions compiled by VU
45
+ Versions () []string
46
+
47
+ // Version fetches the Storage Version spec compiled by VU
48
+ Version (version string ) ([]byte , error )
37
49
}
38
50
39
51
// Scraper gets OpenAPI specs from a collection of services and updates storage
@@ -57,24 +69,32 @@ type Option func(*Scraper) error
57
69
func New (cfg * config.ServerConfig , storage Storage , options ... Option ) (* Scraper , error ) {
58
70
s := & Scraper {
59
71
storage : storage ,
60
- http : & http.Client {},
72
+ http : & http.Client {Timeout : time . Second * 15 },
61
73
timeNow : time .Now ,
62
74
}
75
+ err := setupScraper (s , cfg , options )
76
+ if err != nil {
77
+ return nil , err
78
+ }
79
+ return s , nil
80
+ }
81
+
82
+ func setupScraper (s * Scraper , cfg * config.ServerConfig , options []Option ) error {
63
83
s .services = make ([]service , len (cfg .Services ))
64
84
for i := range cfg .Services {
65
85
u , err := url .Parse (cfg .Services [i ] + "/openapi" )
66
86
if err != nil {
67
- return nil , errors .Wrapf (err , "invalid service %q" , cfg .Services [i ])
87
+ return errors .Wrapf (err , "invalid service %q" , cfg .Services [i ])
68
88
}
69
89
s .services [i ] = service {base : cfg .Services [i ], url : u }
70
90
}
71
91
for i := range options {
72
92
err := options [i ](s )
73
93
if err != nil {
74
- return nil , err
94
+ return err
75
95
}
76
96
}
77
- return s , nil
97
+ return nil
78
98
}
79
99
80
100
// HTTPClient is a Scraper constructor Option that allows providing an
@@ -113,41 +133,52 @@ func (s *Scraper) Run(ctx context.Context) error {
113
133
errs = multierr .Append (errs , err )
114
134
}
115
135
close (errCh )
136
+ if errs != nil {
137
+ return errs
138
+ } else {
139
+ err := s .collateVersions ()
140
+ errs = multierr .Append (errs , err )
141
+ }
116
142
return errs
117
143
}
118
144
119
145
func (s * Scraper ) scrape (ctx context.Context , scrapeTime time.Time , svc service ) error {
120
- versions , err := s .getVersions (ctx , svc )
146
+ versions , err := s .getVersions (svc )
121
147
if err != nil {
122
148
return errors .WithStack (err )
123
149
}
124
- err = s .storage .NotifyVersions (ctx , svc .base , versions , scrapeTime )
150
+ err = s .storage .NotifyVersions (svc .base , versions , scrapeTime )
125
151
if err != nil {
126
152
return errors .WithStack (err )
127
153
}
128
154
for i := range versions {
129
155
// TODO: we might run this concurrently per live service pod if/when
130
156
// we're more k8s aware, but we won't do that yet.
131
- contents , isNew , err := s .getNewVersion (ctx , svc , versions [i ])
157
+ contents , isNew , err := s .getNewVersion (svc , versions [i ])
132
158
if err != nil {
133
159
return errors .WithStack (err )
134
160
}
135
161
if ! isNew {
136
162
continue
137
163
}
138
- err = s .storage .NotifyVersion (ctx , svc .base , versions [i ], contents , scrapeTime )
164
+ err = s .storage .NotifyVersion (svc .base , versions [i ], contents , scrapeTime )
139
165
if err != nil {
140
166
return errors .WithStack (err )
141
167
}
142
168
}
143
169
return nil
144
170
}
145
171
146
- func (s * Scraper ) getVersions (ctx context.Context , svc service ) ([]string , error ) {
147
- req , err := http .NewRequestWithContext (ctx , "GET" , svc .url .String (), nil )
172
+ func (s * Scraper ) collateVersions () error {
173
+ return s .storage .CollateVersions ()
174
+ }
175
+
176
+ func (s * Scraper ) getVersions (svc service ) ([]string , error ) {
177
+ req , err := http .NewRequest ("GET" , svc .url .String (), nil )
148
178
if err != nil {
149
179
return nil , errors .Wrap (err , "failed to create request" )
150
180
}
181
+
151
182
resp , err := s .http .Do (req )
152
183
if err != nil {
153
184
return nil , errors .Wrap (err , "request failed" )
@@ -171,16 +202,18 @@ func httpError(r *http.Response) error {
171
202
return errors .Errorf ("request failed: HTTP %d" , r .StatusCode )
172
203
}
173
204
174
- func (s * Scraper ) getNewVersion (ctx context.Context , svc service , version string ) ([]byte , bool , error ) {
175
- isNew , err := s .hasNewVersion (ctx , svc , version )
205
+ func (s * Scraper ) getNewVersion (svc service , version string ) ([]byte , bool , error ) {
206
+ // TODO: Services don't emit HEAD currently with compiled vervet
207
+ // will need to enforce down the line
208
+ isNew , err := s .hasNewVersion (svc , version )
176
209
if err != nil {
177
210
return nil , false , errors .WithStack (err )
178
211
}
179
212
if ! isNew {
180
213
return nil , false , nil
181
214
}
182
215
183
- req , err := http .NewRequestWithContext ( ctx , "GET" , svc .url .String ()+ "/" + version , nil )
216
+ req , err := http .NewRequest ( "GET" , svc .url .String ()+ "/" + version , nil )
184
217
if err != nil {
185
218
return nil , false , errors .Wrap (err , "failed to create request" )
186
219
}
@@ -192,7 +225,7 @@ func (s *Scraper) getNewVersion(ctx context.Context, svc service, version string
192
225
if resp .StatusCode != http .StatusOK {
193
226
return nil , false , httpError (resp )
194
227
}
195
- if ct := resp .Header .Get ("Content-Type" ); ct != "application/json" {
228
+ if ct := resp .Header .Get ("Content-Type" ); ! strings . HasPrefix ( ct , "application/json" ) {
196
229
return nil , false , errors .Errorf ("unexpected content type: %s" , ct )
197
230
}
198
231
respContents , err := ioutil .ReadAll (resp .Body )
@@ -209,9 +242,9 @@ func (s *Scraper) getNewVersion(ctx context.Context, svc service, version string
209
242
return respContents , true , nil
210
243
}
211
244
212
- func (s * Scraper ) hasNewVersion (ctx context. Context , svc service , version string ) (bool , error ) {
245
+ func (s * Scraper ) hasNewVersion (svc service , version string ) (bool , error ) {
213
246
// Check Digest to see if there's a new version
214
- req , err := http .NewRequestWithContext ( ctx , "HEAD" , svc .url .String ()+ "/" + version , nil )
247
+ req , err := http .NewRequest ( "HEAD" , svc .url .String ()+ "/" + version , nil )
215
248
if err != nil {
216
249
return false , errors .Wrap (err , "failed to create request" )
217
250
}
@@ -227,13 +260,23 @@ func (s *Scraper) hasNewVersion(ctx context.Context, svc service, version string
227
260
if resp .StatusCode != http .StatusOK {
228
261
return false , httpError (resp )
229
262
}
230
- if ct := resp .Header .Get ("Content-Type" ); ct != "application/json" {
263
+
264
+ // Can be formed similarly: "application/json; charset: utf-8"
265
+ if ct := resp .Header .Get ("Content-Type" ); ! strings .HasPrefix (ct , "application/json" ) {
231
266
return false , errors .Errorf ("unexpected content type: %s" , ct )
232
267
}
233
268
digest := storage .DigestHeader (resp .Header .Get ("Digest" ))
234
269
if digest == "" {
235
270
// Not providing a digest is fine, we'll just come back with a GET
236
271
return true , nil
237
272
}
238
- return s .storage .HasVersion (ctx , svc .base , version , digest )
273
+ return s .storage .HasVersion (svc .base , version , digest )
274
+ }
275
+
276
+ func (s * Scraper ) Versions () []string {
277
+ return s .storage .Versions ()
278
+ }
279
+
280
+ func (s * Scraper ) Version (version string ) ([]byte , error ) {
281
+ return s .storage .Version (version )
239
282
}
0 commit comments