diff --git a/pkg/kudoctl/cmd/repo_index.go b/pkg/kudoctl/cmd/repo_index.go index 2d3d33ce7..4fc3c68fc 100644 --- a/pkg/kudoctl/cmd/repo_index.go +++ b/pkg/kudoctl/cmd/repo_index.go @@ -140,7 +140,10 @@ func (ri *repoIndexCmd) run() error { if err != nil { return err } - merge(index, mergeIndex) + err = client.Merge(index, mergeIndex) + if err != nil { + return err + } } if err := index.WriteFile(ri.fs, target); err != nil { @@ -150,20 +153,6 @@ func (ri *repoIndexCmd) run() error { return nil } -func merge(index *repo.IndexFile, mergeIndex *repo.IndexFile) { - // index is the master, any dups in the merged in index will have what is local replace those entries - for _, pvs := range mergeIndex.Entries { - for _, pv := range pvs { - err := index.AddPackageVersion(pv) - // this is most likely to be a duplicate pv, which we ignore (but will log at the right v) - if err != nil { - // todo: add verbose logging here - continue - } - } - } -} - func (ri *repoIndexCmd) mergeRepoConfig() (*repo.Configuration, error) { if ri.mergeRepoName != "" { return ri.repoConfig(ri.mergeRepoName) diff --git a/pkg/kudoctl/cmd/repo_index_test.go b/pkg/kudoctl/cmd/repo_index_test.go index b6a06b46b..bdc9c80fc 100644 --- a/pkg/kudoctl/cmd/repo_index_test.go +++ b/pkg/kudoctl/cmd/repo_index_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "fmt" "io/ioutil" "path/filepath" "testing" @@ -92,7 +93,17 @@ func TestRepoIndexCmd_MergeIndex(t *testing.T) { mergeFile, _ := repo.ParseIndexFile(mergeBytes) resultBuf := &bytes.Buffer{} - merge(indexFile, mergeFile) + + p, err := filepath.Abs("testdata/include-index") + assert.NoError(t, err) + config := &repo.Configuration{ + URL: fmt.Sprintf("file://%s", p), + } + client, err := repo.NewClient(config) + assert.NoError(t, err) + + err = client.Merge(indexFile, mergeFile) + assert.NoError(t, err) if err := indexFile.Write(resultBuf); err != nil { t.Fatal(err) } diff --git a/pkg/kudoctl/util/repo/index.go b/pkg/kudoctl/util/repo/index.go index 9b8ddbbc5..6b200cac6 100644 --- a/pkg/kudoctl/util/repo/index.go +++ b/pkg/kudoctl/util/repo/index.go @@ -22,6 +22,7 @@ const defaultURL = "http://localhost/" type IndexFile struct { APIVersion string `json:"apiVersion"` Entries map[string]PackageVersions `json:"entries"` + Includes []string `json:"includes,omitempty"` Generated *time.Time `json:"generated"` } @@ -104,13 +105,29 @@ func (i IndexFile) Write(w io.Writer) error { return err } +// DuplicateError is returned for a duplicate entry +type DuplicateError struct { + pv *PackageVersion +} + +// Implement the Error interface. +func (e *DuplicateError) Error() string { + return fmt.Sprintf("operator %q version: %v_%v already exists", e.pv.Name, e.pv.AppVersion, e.pv.OperatorVersion) +} + +// Defines what is is +func (e *DuplicateError) Is(target error) bool { + _, ok := target.(*DuplicateError) + return ok +} + // AddPackageVersion adds an entry to the IndexFile (does not allow dups) func (i *IndexFile) AddPackageVersion(pv *PackageVersion) error { name := pv.Name appVersion := pv.AppVersion operatorVersion := pv.OperatorVersion if operatorVersion == "" { - return fmt.Errorf("operator '%v' is missing operator version", name) + return fmt.Errorf("operator %q is missing operator version", name) } if i.Entries == nil { i.Entries = make(map[string]PackageVersions) @@ -126,7 +143,7 @@ func (i *IndexFile) AddPackageVersion(pv *PackageVersion) error { // loop thru all... don't allow dups for _, ver := range vs { if ver.AppVersion == appVersion && ver.OperatorVersion == operatorVersion { - return fmt.Errorf("operator '%v' version: %v_%v already exists", name, appVersion, operatorVersion) + return &DuplicateError{ver} } } diff --git a/pkg/kudoctl/util/repo/index_test.go b/pkg/kudoctl/util/repo/index_test.go index c5c86336f..8c7c9d713 100644 --- a/pkg/kudoctl/util/repo/index_test.go +++ b/pkg/kudoctl/util/repo/index_test.go @@ -174,8 +174,8 @@ func TestAddPackageVersionErrorConditions(t *testing.T) { pv *PackageVersion err string }{ - {"duplicate version", dup, "operator 'flink' version: 0.7.0_0.3.0 already exists"}, - {"no version", &missing, "operator 'flink' is missing operator version"}, + {"duplicate version", dup, "operator \"flink\" version: 0.7.0_0.3.0 already exists"}, + {"no version", &missing, "operator \"flink\" is missing operator version"}, {"good additional version", &good, ""}, {"good additional package", &g2, ""}, } diff --git a/pkg/kudoctl/util/repo/repo_operator.go b/pkg/kudoctl/util/repo/repo_operator.go index d0db1b000..6021f7cd6 100644 --- a/pkg/kudoctl/util/repo/repo_operator.go +++ b/pkg/kudoctl/util/repo/repo_operator.go @@ -2,6 +2,7 @@ package repo import ( "bytes" + "errors" "fmt" "io/ioutil" "net/url" @@ -9,6 +10,7 @@ import ( "github.com/spf13/afero" + "github.com/kudobuilder/kudo/pkg/kudoctl/clog" "github.com/kudobuilder/kudo/pkg/kudoctl/http" "github.com/kudobuilder/kudo/pkg/kudoctl/kudohome" ) @@ -50,20 +52,34 @@ func NewClient(conf *Configuration) (*Client, error) { // DownloadIndexFile fetches the index file from a repository. func (c *Client) DownloadIndexFile() (*IndexFile, error) { - var indexURL string parsedURL, err := url.Parse(c.Config.URL) if err != nil { return nil, fmt.Errorf("parsing config url: %w", err) } - parsedURL.Path = fmt.Sprintf("%s/index.yaml", strings.TrimSuffix(parsedURL.Path, "/")) + h := make(map[string]bool) + return c.downloadIndexFile(parsedURL, h) +} - indexURL = parsedURL.String() +func (c *Client) downloadIndexFile(url *url.URL, urlHistory map[string]bool) (*IndexFile, error) { var resp *bytes.Buffer - if strings.HasPrefix(indexURL, "file:") { - b, _ := ioutil.ReadFile(parsedURL.Path) + var err error + // we need the index.yaml at the url provided + url.Path = fmt.Sprintf("%s/index.yaml", strings.TrimSuffix(url.Path, "/")) + if val, ok := urlHistory[url.String()]; ok { + // if we have seen the url previous we don't process it + clog.V(1).Printf("duplicate url %v ignored", val) + return nil, nil + } + urlHistory[url.String()] = true + + if url.Scheme == "file" || strings.HasPrefix(url.String(), "file:") { + b, err := ioutil.ReadFile(url.Path) + if err != nil { + return nil, err + } resp = bytes.NewBuffer(b) } else { - resp, err = c.Client.Get(indexURL) + resp, err = c.Client.Get(url.String()) } if err != nil { return nil, fmt.Errorf("getting index url: %w", err) @@ -75,5 +91,38 @@ func (c *Client) DownloadIndexFile() (*IndexFile, error) { } indexFile, err := ParseIndexFile(indexBytes) + for _, include := range indexFile.Includes { + iURL, err := url.Parse(include) + if err != nil { + return nil, clog.Errorf("unable to parse include url for %s", include) + } + nextIndex, err := c.downloadIndexFile(iURL, urlHistory) + if err != nil { + return nil, err + } + err = c.Merge(indexFile, nextIndex) + if err != nil { + return nil, err + } + } + return indexFile, err } + +// Merge combines the Entries of 2 index files. The first index file is the master +// the second is merged into the first. Any duplicates are ignored. +func (c *Client) Merge(index *IndexFile, mergeIndex *IndexFile) error { + // index is the master, any dups in the merged in index will have what is local replace those entries + for _, pvs := range mergeIndex.Entries { + for _, pv := range pvs { + err := index.AddPackageVersion(pv) + if errors.Is(err, &DuplicateError{}) { + clog.V(1).Printf("ignoring duplicate for %q: appver: %v, opver: %v", pv.Name, pv.AppVersion, pv.OperatorVersion) + continue + } else if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/kudoctl/util/repo/repo_test.go b/pkg/kudoctl/util/repo/repo_test.go index 7cb77b92b..c874ada75 100644 --- a/pkg/kudoctl/util/repo/repo_test.go +++ b/pkg/kudoctl/util/repo/repo_test.go @@ -42,3 +42,35 @@ func TestLoadRepositories(t *testing.T) { assert.Equal(t, r.CurrentConfiguration().Name, Default.Name) assert.Equal(t, r.CurrentConfiguration().URL, Default.URL) } + +func TestDownloadMultiRepo(t *testing.T) { + + p, err := filepath.Abs("testdata/include-index") + assert.NoError(t, err) + config := &Configuration{ + URL: fmt.Sprintf("file://%s", p), + } + client, err := NewClient(config) + assert.NoError(t, err) + index, err := client.DownloadIndexFile() + assert.NoError(t, err) + // mysql package only there, if include worked + assert.NotNil(t, index.Entries["mysql"]) + + // the merge for flink will have 1 dup that doesn't merge + flink, err := index.FindFirstMatch("flink", "", "0.3.0") + assert.NoError(t, err) + assert.Equal(t, "correct flink", flink.Description) + + // and a new version that does merge + flink, err = index.FindFirstMatch("flink", "", "0.4.0") + assert.NoError(t, err) + assert.Equal(t, "0.4.0", flink.OperatorVersion) + + // and a version that is nested in a repository + flink, err = index.FindFirstMatch("flink", "", "0.4.1") + assert.NoError(t, err) + assert.Equal(t, "0.4.1", flink.OperatorVersion) + assert.Equal(t, "this merges and overwrites the version in nested-included-repo", flink.Description) + +} diff --git a/pkg/kudoctl/util/repo/testdata/include-index/index.yaml b/pkg/kudoctl/util/repo/testdata/include-index/index.yaml new file mode 100644 index 000000000..f28debea7 --- /dev/null +++ b/pkg/kudoctl/util/repo/testdata/include-index/index.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +entries: + flink: + - appVersion: 0.7.0 + description: correct flink + digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd + maintainers: + - email: + name: Tom Runyon + - email: + name: Ken Sipe + name: flink + operatorVersion: 0.3.0 + urls: + - http://kudo.dev/flink +includes: + - ../included-repo +generated: "2020-04-19T15:04:00Z" diff --git a/pkg/kudoctl/util/repo/testdata/included-repo/index.yaml b/pkg/kudoctl/util/repo/testdata/included-repo/index.yaml new file mode 100644 index 000000000..f53a8a71f --- /dev/null +++ b/pkg/kudoctl/util/repo/testdata/included-repo/index.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +entries: + mysql: + - appVersion: "5.7" + digest: ad2451eaf68896490a2864afbb3fcd81e6fe3368e4bf001f8a23dc9cbcddf49a + maintainers: + - email: nick@dischord.org + name: Nick Jones + name: mysql + operatorVersion: 0.2.0 + urls: + - https://kudo-repository.storage.googleapis.com/0.10.0/mysql-5.7_0.2.0.tgz + flink: + - appVersion: 0.7.0 + description: wrong flink (should NOT be included) + digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd + maintainers: + - email: + name: Tom Runyon + - email: + name: Ken Sipe + name: flink + operatorVersion: 0.3.0 + - appVersion: 0.8.0 + description: this merges because the version doesn't exist in parent + digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd + maintainers: + - email: + name: Tom Runyon + - email: + name: Ken Sipe + name: flink + operatorVersion: 0.4.0 + urls: + - http://kudo.dev/flink + - appVersion: 0.8.1 + description: this merges and overwrites the version in nested-included-repo + digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd + maintainers: + - email: + name: Tom Runyon + - email: + name: Ken Sipe + name: flink + operatorVersion: 0.4.1 + urls: + - http://kudo.dev/flink +includes: + - ../nested-included-repo +generated: "2020-04-19T15:04:00Z" + + diff --git a/pkg/kudoctl/util/repo/testdata/nested-included-repo/index.yaml b/pkg/kudoctl/util/repo/testdata/nested-included-repo/index.yaml new file mode 100644 index 000000000..c9f90e51b --- /dev/null +++ b/pkg/kudoctl/util/repo/testdata/nested-included-repo/index.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +entries: + mysql: + - appVersion: "5.7" + digest: ad2451eaf68896490a2864afbb3fcd81e6fe3368e4bf001f8a23dc9cbcddf49a + maintainers: + - email: nick@dischord.org + name: Nick Jones + name: mysql + operatorVersion: 0.2.0 + urls: + - https://kudo-repository.storage.googleapis.com/0.10.0/mysql-5.7_0.2.0.tgz + flink: + - appVersion: 0.7.0 + description: wrong flink (should NOT be included) + digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd + maintainers: + - email: + name: Tom Runyon + - email: + name: Ken Sipe + name: flink + operatorVersion: 0.3.0 + - appVersion: 0.8.1 + description: this will get overwritten by included-repo + digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd + maintainers: + - email: + name: Tom Runyon + - email: + name: Ken Sipe + name: flink + operatorVersion: 0.4.1 + urls: + - http://kudo.dev/flink + +generated: "2020-04-19T15:04:00Z" + +