Providing Repo Include Feature

pkg/kudoctl/cmd/repo_index.go

@@ -140,7 +140,10 @@ func (ri *repoIndexCmd) run() error {
 		if err != nil {
 			return err
 		}
-		merge(index, mergeIndex)
+		err = client.Merge(index, mergeIndex)
+		if err != nil {
+			return err
+		}
 	}
 
 	if err := index.WriteFile(ri.fs, target); err != nil {
@@ -150,20 +153,6 @@ func (ri *repoIndexCmd) run() error {
 	return nil
 }
 
-func merge(index *repo.IndexFile, mergeIndex *repo.IndexFile) {
-	// index is the master, any dups in the merged in index will have what is local replace those entries
-	for _, pvs := range mergeIndex.Entries {
-		for _, pv := range pvs {
-			err := index.AddPackageVersion(pv)
-			// this is most likely to be a duplicate pv, which we ignore (but will log at the right v)
-			if err != nil {
-				// todo: add verbose logging here
-				continue
-			}
-		}
-	}
-}
-
 func (ri *repoIndexCmd) mergeRepoConfig() (*repo.Configuration, error) {
 	if ri.mergeRepoName != "" {
 		return ri.repoConfig(ri.mergeRepoName)

pkg/kudoctl/cmd/repo_index_test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"bytes"
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
 	"testing"
@@ -92,7 +93,17 @@ func TestRepoIndexCmd_MergeIndex(t *testing.T) {
 	mergeFile, _ := repo.ParseIndexFile(mergeBytes)
 
 	resultBuf := &bytes.Buffer{}
-	merge(indexFile, mergeFile)
+
+	p, err := filepath.Abs("testdata/include-index")
+	assert.NoError(t, err)
+	config := &repo.Configuration{
+		URL: fmt.Sprintf("file://%s", p),
+	}
+	client, err := repo.NewClient(config)
+	assert.NoError(t, err)
+
+	err = client.Merge(indexFile, mergeFile)
+	assert.NoError(t, err)
 	if err := indexFile.Write(resultBuf); err != nil {
 		t.Fatal(err)
 	}

pkg/kudoctl/util/repo/index.go

@@ -22,6 +22,7 @@ const defaultURL = "http://localhost/"
 type IndexFile struct {
 	APIVersion string                     `json:"apiVersion"`
 	Entries    map[string]PackageVersions `json:"entries"`
+	Includes   []string                   `json:"includes,omitempty"`
 	Generated  *time.Time                 `json:"generated"`
 }
 
@@ -104,13 +105,29 @@ func (i IndexFile) Write(w io.Writer) error {
 	return err
 }
 
+// DuplicateError is returned for a duplicate entry
+type DuplicateError struct {
+	pv *PackageVersion
+}
+
+// Implement the Error interface.
+func (e *DuplicateError) Error() string {
+	return fmt.Sprintf("operator %q version: %v_%v already exists", e.pv.Name, e.pv.AppVersion, e.pv.OperatorVersion)
+}
+
+// Defines what is is
+func (e *DuplicateError) Is(target error) bool {
+	_, ok := target.(*DuplicateError)
+	return ok
+}
+
 // AddPackageVersion adds an entry to the IndexFile (does not allow dups)
 func (i *IndexFile) AddPackageVersion(pv *PackageVersion) error {
 	name := pv.Name
 	appVersion := pv.AppVersion
 	operatorVersion := pv.OperatorVersion
 	if operatorVersion == "" {
-		return fmt.Errorf("operator '%v' is missing operator version", name)
+		return fmt.Errorf("operator %q is missing operator version", name)
 	}
 	if i.Entries == nil {
 		i.Entries = make(map[string]PackageVersions)
@@ -126,7 +143,7 @@ func (i *IndexFile) AddPackageVersion(pv *PackageVersion) error {
 	// loop thru all... don't allow dups
 	for _, ver := range vs {
 		if ver.AppVersion == appVersion && ver.OperatorVersion == operatorVersion {
-			return fmt.Errorf("operator '%v' version: %v_%v already exists", name, appVersion, operatorVersion)
+			return &DuplicateError{ver}
 		}
 	}
 

pkg/kudoctl/util/repo/index_test.go

@@ -174,8 +174,8 @@ func TestAddPackageVersionErrorConditions(t *testing.T) {
 		pv   *PackageVersion
 		err  string
 	}{
-		{"duplicate version", dup, "operator 'flink' version: 0.7.0_0.3.0 already exists"},
-		{"no version", &missing, "operator 'flink' is missing operator version"},
+		{"duplicate version", dup, "operator \"flink\" version: 0.7.0_0.3.0 already exists"},
+		{"no version", &missing, "operator \"flink\" is missing operator version"},
 		{"good additional version", &good, ""},
 		{"good additional package", &g2, ""},
 	}

pkg/kudoctl/util/repo/repo_operator.go

@@ -2,13 +2,15 @@ package repo
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"strings"
 
 	"github.com/spf13/afero"
 
+	"github.com/kudobuilder/kudo/pkg/kudoctl/clog"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/http"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/kudohome"
 )
@@ -50,20 +52,34 @@ func NewClient(conf *Configuration) (*Client, error) {
 
 // DownloadIndexFile fetches the index file from a repository.
 func (c *Client) DownloadIndexFile() (*IndexFile, error) {
-	var indexURL string
 	parsedURL, err := url.Parse(c.Config.URL)
 	if err != nil {
 		return nil, fmt.Errorf("parsing config url: %w", err)
 	}
-	parsedURL.Path = fmt.Sprintf("%s/index.yaml", strings.TrimSuffix(parsedURL.Path, "/"))
+	h := make(map[string]bool)
+	return c.downloadIndexFile(parsedURL, h)
+}
 
-	indexURL = parsedURL.String()
+func (c *Client) downloadIndexFile(url *url.URL, urlHistory map[string]bool) (*IndexFile, error) {
 	var resp *bytes.Buffer
-	if strings.HasPrefix(indexURL, "file:") {
-		b, _ := ioutil.ReadFile(parsedURL.Path)
+	var err error
+	// we need the index.yaml at the url provided
+	url.Path = fmt.Sprintf("%s/index.yaml", strings.TrimSuffix(url.Path, "/"))
+	if val, ok := urlHistory[url.String()]; ok {
+		// if we have seen the url previous we don't process it
+		clog.V(1).Printf("duplicate url %v ignored", val)
+		return nil, nil
+	}
+	urlHistory[url.String()] = true
+
+	if url.Scheme == "file" || strings.HasPrefix(url.String(), "file:") {
+		b, err := ioutil.ReadFile(url.Path)
+		if err != nil {
+			return nil, err
+		}
 		resp = bytes.NewBuffer(b)
 	} else {
-		resp, err = c.Client.Get(indexURL)
+		resp, err = c.Client.Get(url.String())
 	}
 	if err != nil {
 		return nil, fmt.Errorf("getting index url: %w", err)
@@ -75,5 +91,38 @@ func (c *Client) DownloadIndexFile() (*IndexFile, error) {
 	}
 
 	indexFile, err := ParseIndexFile(indexBytes)
+	for _, include := range indexFile.Includes {
+		iURL, err := url.Parse(include)
+		if err != nil {
+			return nil, clog.Errorf("unable to parse include url for %s", include)
+		}
+		nextIndex, err := c.downloadIndexFile(iURL, urlHistory)
+		if err != nil {
+			return nil, err
+		}
+		err = c.Merge(indexFile, nextIndex)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return indexFile, err
 }
+
+// Merge combines the Entries of 2 index files.   The first index file is the master
+// the second is merged into the first.  Any duplicates are ignored.
+func (c *Client) Merge(index *IndexFile, mergeIndex *IndexFile) error {
+	// index is the master, any dups in the merged in index will have what is local replace those entries
+	for _, pvs := range mergeIndex.Entries {
+		for _, pv := range pvs {
+			err := index.AddPackageVersion(pv)
+			if errors.Is(err, &DuplicateError{}) {
+				clog.V(1).Printf("ignoring duplicate for %q: appver: %v, opver: %v", pv.Name, pv.AppVersion, pv.OperatorVersion)
+				continue
+			} else if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/kudoctl/util/repo/repo_test.go

@@ -42,3 +42,35 @@ func TestLoadRepositories(t *testing.T) {
 	assert.Equal(t, r.CurrentConfiguration().Name, Default.Name)
 	assert.Equal(t, r.CurrentConfiguration().URL, Default.URL)
 }
+
+func TestDownloadMultiRepo(t *testing.T) {
+
+	p, err := filepath.Abs("testdata/include-index")
+	assert.NoError(t, err)
+	config := &Configuration{
+		URL: fmt.Sprintf("file://%s", p),
+	}
+	client, err := NewClient(config)
+	assert.NoError(t, err)
+	index, err := client.DownloadIndexFile()
+	assert.NoError(t, err)
+	// mysql package only there, if include worked
+	assert.NotNil(t, index.Entries["mysql"])
+
+	// the merge for flink will have 1 dup that doesn't merge
+	flink, err := index.FindFirstMatch("flink", "", "0.3.0")
+	assert.NoError(t, err)
+	assert.Equal(t, "correct flink", flink.Description)
+
+	// and a new version that does merge
+	flink, err = index.FindFirstMatch("flink", "", "0.4.0")
+	assert.NoError(t, err)
+	assert.Equal(t, "0.4.0", flink.OperatorVersion)
+
+	// and a version that is nested in a repository
+	flink, err = index.FindFirstMatch("flink", "", "0.4.1")
+	assert.NoError(t, err)
+	assert.Equal(t, "0.4.1", flink.OperatorVersion)
+	assert.Equal(t, "this merges and overwrites the version in nested-included-repo", flink.Description)
+
+}

pkg/kudoctl/util/repo/testdata/include-index/index.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+entries:
+  flink:
+    - appVersion: 0.7.0
+      description: correct flink
+      digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+      maintainers:
+        - email: <[email protected]>
+          name: Tom Runyon
+        - email: <[email protected]>
+          name: Ken Sipe
+      name: flink
+      operatorVersion: 0.3.0
+      urls:
+        - http://kudo.dev/flink
+includes:
+  - ../included-repo
+generated: "2020-04-19T15:04:00Z"

pkg/kudoctl/util/repo/testdata/included-repo/index.yaml

@@ -0,0 +1,52 @@
+apiVersion: v1
+entries:
+  mysql:
+    - appVersion: "5.7"
+      digest: ad2451eaf68896490a2864afbb3fcd81e6fe3368e4bf001f8a23dc9cbcddf49a
+      maintainers:
+        - email: [email protected]
+          name: Nick Jones
+      name: mysql
+      operatorVersion: 0.2.0
+      urls:
+        - https://kudo-repository.storage.googleapis.com/0.10.0/mysql-5.7_0.2.0.tgz
+  flink:
+    - appVersion: 0.7.0
+      description: wrong flink (should NOT be included)
+      digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+      maintainers:
+        - email: <[email protected]>
+          name: Tom Runyon
+        - email: <[email protected]>
+          name: Ken Sipe
+      name: flink
+      operatorVersion: 0.3.0
+    - appVersion: 0.8.0
+      description: this merges because the version doesn't exist in parent
+      digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+      maintainers:
+        - email: <[email protected]>
+          name: Tom Runyon
+        - email: <[email protected]>
+          name: Ken Sipe
+      name: flink
+      operatorVersion: 0.4.0
+      urls:
+        - http://kudo.dev/flink
+    - appVersion: 0.8.1
+      description: this merges and overwrites the version in nested-included-repo
+      digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+      maintainers:
+        - email: <[email protected]>
+          name: Tom Runyon
+        - email: <[email protected]>
+          name: Ken Sipe
+      name: flink
+      operatorVersion: 0.4.1
+      urls:
+        - http://kudo.dev/flink
+includes:
+  - ../nested-included-repo
+generated: "2020-04-19T15:04:00Z"
+
+

pkg/kudoctl/util/repo/testdata/nested-included-repo/index.yaml

@@ -0,0 +1,39 @@
+apiVersion: v1
+entries:
+  mysql:
+    - appVersion: "5.7"
+      digest: ad2451eaf68896490a2864afbb3fcd81e6fe3368e4bf001f8a23dc9cbcddf49a
+      maintainers:
+        - email: [email protected]
+          name: Nick Jones
+      name: mysql
+      operatorVersion: 0.2.0
+      urls:
+        - https://kudo-repository.storage.googleapis.com/0.10.0/mysql-5.7_0.2.0.tgz
+  flink:
+    - appVersion: 0.7.0
+      description: wrong flink (should NOT be included)
+      digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+      maintainers:
+        - email: <[email protected]>
+          name: Tom Runyon
+        - email: <[email protected]>
+          name: Ken Sipe
+      name: flink
+      operatorVersion: 0.3.0
+    - appVersion: 0.8.1
+      description: this will get overwritten by included-repo
+      digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+      maintainers:
+        - email: <[email protected]>
+          name: Tom Runyon
+        - email: <[email protected]>
+          name: Ken Sipe
+      name: flink
+      operatorVersion: 0.4.1
+      urls:
+        - http://kudo.dev/flink
+
+generated: "2020-04-19T15:04:00Z"
+
+