update config.yaml.example

fix query
add limit
add close method
add new test cases
add mock

Co-authored-by: Justin Sherrill <[email protected]>

Author: rverdile

README.md

@@ -24,10 +24,11 @@ t, err := tangy.New(dbConfig, tangy.Logger{Enabled: false})
 if err != nil {
     return err
 }
+defer t.Close()
 
-// Use Tangy to search for RPMs, by name, that are associated to a specific repository version
+// Use Tangy to search for RPMs, by name, that are associated to a specific repository version, returning up to the first 100 results
 versionHref := "/pulp/e1c6bee3/api/v3/repositories/rpm/rpm/018c1c95-4281-76eb-b277-842cbad524f4/versions/1/"
-rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja")
+rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja", 100)
 if err != nil {
 return err
 }
@@ -56,4 +57,7 @@ The default values provided in config.yaml.example will work with this server.
 `make compose-down`
 
 #### Clean container volumes
-`make compose-clean`
+`make compose-clean`
+
+### Mocking
+Tangy also exports a mock interface you can regenerate using the [mockery](https://github.com/vektra/mockery) tool.

configs/config.yaml.example

@@ -9,7 +9,7 @@ database:
 # Configuration options for the pulp server
 server:
   url: "http://localhost:8087"
-  username: "pulp"
+  username: "admin"
   password: "password"
   storage_type: "local"
   download_policy: "on_demand"

example.go

@@ -27,6 +27,7 @@ func main() {
 		fmt.Println(err)
 		return
 	}
+	defer t.Close()
 
 	// Call helper function that creates and syncs a repository
 	versionHref, err := CreateRepositoryVersion()
@@ -35,8 +36,8 @@ func main() {
 		return
 	}
 
-	// Use Tangy to search for RPMs, by name, that are associated to a specific repository version
-	rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja")
+	// Use Tangy to search for RPMs, by name, that are associated to a specific repository version, returning up to the first 100 results
+	rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja", 100)
 	if err != nil {
 		fmt.Println(err)
 		return
@@ -58,7 +59,7 @@ func CreateRepositoryVersion() (versionHref string, err error) {
 		DownloadPolicy: "on_demand",
 	})
 
-	domainName := "example_domain"
+	domainName := "example-domain"
 
 	// Create domain and repository, then sync repository, to create a new repository version with rpm packages
 	_, err = rpmZest.LookupOrCreateDomain(domainName)

go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/content-services/zest/release/v2023 v2023.11.1701177874
+	github.com/google/uuid v1.4.0
 	github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb
 	github.com/jackc/pgx/v5 v5.5.1
 	github.com/rs/zerolog v1.31.0
@@ -32,6 +33,7 @@ require (
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect

go.sum

@@ -10,6 +10,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
@@ -61,6 +63,7 @@ github.com/spf13/viper v1.18.1 h1:rmuU42rScKWlhhJDyXZRKJQHXFX02chSVW1IvkPGiVM=
 github.com/spf13/viper v1.18.1/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

internal/test/integration/rpm_test.go

@@ -18,10 +18,13 @@ type RpmSuite struct {
 	client     *zestwrapper.RpmZest
 	tangy      tangy.Tangy
 	domainName string
+	remoteHref string
+	repoHref   string
 }
 
 const testRepoName = "rpm modular"
-const testRepoURL = "https://fixtures.pulpproject.org/rpm-modular/"
+const testRepoURL = "https://jlsherrill.fedorapeople.org/fake-repos/revision/one/"
+const testRepoURLTwo = "https://jlsherrill.fedorapeople.org/fake-repos/revision/two/"
 
 func (r *RpmSuite) CreateTestRepository(t *testing.T) {
 	domainName := RandStringBytes(10)
@@ -33,13 +36,27 @@ func (r *RpmSuite) CreateTestRepository(t *testing.T) {
 	repoHref, remoteHref, err := r.client.CreateRepository(domainName, testRepoName, testRepoURL)
 	require.NoError(t, err)
 
+	r.repoHref = repoHref
+	r.remoteHref = remoteHref
+
 	syncTask, err := r.client.SyncRpmRepository(repoHref, remoteHref)
 	require.NoError(t, err)
 
 	_, err = r.client.PollTask(syncTask)
 	require.NoError(t, err)
 }
 
+func (r *RpmSuite) UpdateTestRepository(t *testing.T, url string) {
+	err := r.client.UpdateRemote(r.remoteHref, url)
+	require.NoError(t, err)
+
+	syncTask, err := r.client.SyncRpmRepository(r.repoHref, r.remoteHref)
+	require.NoError(t, err)
+
+	_, err = r.client.PollTask(syncTask)
+	require.NoError(t, err)
+}
+
 func TestRpmSuite(t *testing.T) {
 	s := config.Get().Server
 	rpmZest := zestwrapper.NewRpmZest(context.Background(), s)
@@ -51,7 +68,7 @@ func TestRpmSuite(t *testing.T) {
 		Port:     dbConfig.Port,
 		User:     dbConfig.User,
 		Password: dbConfig.Password,
-	}, tangy.Logger{Enabled: false})
+	}, tangy.Logger{})
 	require.NoError(t, err)
 
 	r := RpmSuite{}
@@ -64,12 +81,51 @@ func TestRpmSuite(t *testing.T) {
 func (r *RpmSuite) TestRpmRepositoryVersionPackageSearch() {
 	resp, err := r.client.GetRpmRepositoryByName(r.domainName, testRepoName)
 	require.NoError(r.T(), err)
-	versionHref := resp.LatestVersionHref
-	require.NotNil(r.T(), versionHref)
+	firstVersionHref := resp.LatestVersionHref
+	require.NotNil(r.T(), firstVersionHref)
+
+	// Search first repository version
+	search, err := r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "bea", 100)
+	assert.NoError(r.T(), err)
+	assert.Equal(r.T(), search[0].Name, "bear")
+	search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "cam", 100)
+	assert.NoError(r.T(), err)
+	assert.Empty(r.T(), search)
+
+	// Create second repository version
+	r.UpdateTestRepository(r.T(), testRepoURLTwo)
+	resp, err = r.client.GetRpmRepositoryByName(r.domainName, testRepoName)
+	require.NoError(r.T(), err)
+	secondVersionHref := resp.LatestVersionHref
+	require.NotNil(r.T(), secondVersionHref)
+
+	// Search second repository version, should have new package
+	search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*secondVersionHref}, "bea", 100)
+	assert.NoError(r.T(), err)
+	assert.Equal(r.T(), search[0].Name, "bear")
+	search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*secondVersionHref}, "cam", 100)
+	assert.NoError(r.T(), err)
+	assert.Equal(r.T(), search[0].Name, "camel")
+
+	// Re-search the first version, should be the same
+	search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "bea", 100)
+	assert.NoError(r.T(), err)
+	assert.Equal(r.T(), search[0].Name, "bear")
+	search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "cam", 100)
+	assert.NoError(r.T(), err)
+	assert.Empty(r.T(), search)
+
+	// Search both versions
+	search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref, *secondVersionHref}, "a", 100)
+	assert.NoError(r.T(), err)
+	assert.Len(r.T(), search, 2)
+	assert.Equal(r.T(), search[0].Name, "bear")
+	assert.Equal(r.T(), search[1].Name, "camel")
 
-	search, err := r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*versionHref}, "ninja")
+	// Test search limit
+	search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*secondVersionHref}, "a", 1)
 	assert.NoError(r.T(), err)
-	assert.Equal(r.T(), search[0].Name, "ninja-build")
+	assert.Len(r.T(), search, 1)
 }
 
 func RandStringBytes(n int) string {

internal/zestwrapper/rpm.go

@@ -116,6 +116,17 @@ func (r *RpmZest) CreateRepository(domain, name, url string) (repoHref string, r
 	return *resp.PulpHref, *remoteResponse.PulpHref, nil
 }
 
+func (r *RpmZest) UpdateRemote(remoteHref string, url string) error {
+	_, httpResp, err := r.client.RemotesRpmAPI.RemotesRpmRpmPartialUpdate(r.ctx, remoteHref).PatchedrpmRpmRemote(zest.PatchedrpmRpmRemote{Url: &url}).Execute()
+	if httpResp != nil {
+		defer httpResp.Body.Close()
+	}
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (r *RpmZest) SyncRpmRepository(rpmRpmRepositoryHref string, remoteHref string) (string, error) {
 	rpmRepositoryHref := *zest.NewRpmRepositorySyncURL()
 	rpmRepositoryHref.SetRemote(remoteHref)

pkg/tangy/interface.go

@@ -50,6 +50,13 @@ type tangyImpl struct {
 	logger Logger
 }
 
+//go:generate mockery --name Tangy --filename tangy_mock.go --inpackage
 type Tangy interface {
-	RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string) ([]RpmPackageSearch, error)
+	RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string, limit int) ([]RpmPackageSearch, error)
+	Close()
+}
+
+// Close closes the DB connection pool
+func (t *tangyImpl) Close() {
+	t.pool.Close()
 }

pkg/tangy/rpm.go

@@ -2,36 +2,64 @@ package tangy
 
 import (
 	"context"
+	"fmt"
+	"strconv"
 	"strings"
 
+	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5"
 )
 
+const DefaultLimit = 500
+
 type RpmPackageSearch struct {
 	Name    string
 	Summary string
 }
 
-func (t *tangyImpl) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string) ([]RpmPackageSearch, error) {
+// RpmRepositoryVersionPackageSearch search for RPMs, by name, associated to repository hrefs, returning an amount up to limit
+func (t *tangyImpl) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string, limit int) ([]RpmPackageSearch, error) {
 	conn, err := t.pool.Acquire(ctx)
 	if err != nil {
 		return nil, err
 	}
 	defer conn.Release()
 
-	query := `
-		SELECT DISTINCT ON (rp.name) rp.name, rp.summary
-		FROM core_repositoryversion crv
-		INNER JOIN core_repository cr on crv.repository_id = cr.pulp_id
-		INNER JOIN core_repositorycontent crc on cr.pulp_id = crc.repository_id
-		INNER JOIN core_content cc on crc.content_id = cc.pulp_id
-		INNER JOIN rpm_package rp on cc.pulp_id = rp.content_ptr_id
-		WHERE CONCAT(crv.repository_id, '/', crv.number) = ANY ($1)
-		AND rp.name ILIKE CONCAT( '%', $2::text, '%')
-		ORDER BY rp.name ASC
-	`
-	concatenatedIdAndVersion := parseRepositoryVersionHrefs(hrefs)
-	rows, err := conn.Query(context.Background(), query, concatenatedIdAndVersion, search)
+	if limit == 0 {
+		limit = DefaultLimit
+	}
+
+	repositoryIDs, versions, err := parseRepositoryVersionHrefs(hrefs)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing repository version hrefs: %w", err)
+	}
+
+	var query string
+	for i := 0; i < len(repositoryIDs); i++ {
+		id := repositoryIDs[i]
+		ver := versions[i]
+
+		query += fmt.Sprintf(`
+			SELECT DISTINCT ON (rp.name) rp.name, rp.summary
+		    FROM rpm_package rp
+		    WHERE rp.content_ptr_id IN (
+    		    SELECT crc.content_id
+    		    FROM core_repositorycontent crc
+    		    INNER JOIN core_repositoryversion crv ON (crc.version_added_id = crv.pulp_id)
+    		    LEFT OUTER JOIN core_repositoryversion crv2 ON (crc.version_removed_id = crv2.pulp_id)
+    		    WHERE crv.repository_id = '%v' AND crv.number <= %v AND NOT (crv2.number <= %v AND crv2.number IS NOT NULL)
+				AND rp.name ILIKE CONCAT( '%%', '%v'::text, '%%') ORDER BY rp.name ASC LIMIT %v
+            )
+		`, id, ver, ver, search, limit)
+
+		if i == len(repositoryIDs)-1 {
+			query += ";"
+			break
+		}
+
+		query += "UNION"
+	}
+	rows, err := conn.Query(context.Background(), query)
 	if err != nil {
 		return nil, err
 	}
@@ -42,11 +70,25 @@ func (t *tangyImpl) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs
 	return rpms, nil
 }
 
-func parseRepositoryVersionHrefs(hrefs []string) (concatenatedIdAndVersion []string) {
+func parseRepositoryVersionHrefs(hrefs []string) (repositoryIDs []string, versions []int, err error) {
 	// /pulp/e1c6bee3/api/v3/repositories/rpm/rpm/018c1c95-4281-76eb-b277-842cbad524f4/versions/1/
 	for _, href := range hrefs {
 		splitHref := strings.Split(href, "/")
-		concatenatedIdAndVersion = append(concatenatedIdAndVersion, splitHref[8]+"/"+splitHref[10])
+		id := splitHref[8]
+		num := splitHref[10]
+
+		_, err = uuid.Parse(id)
+		if err != nil {
+			return nil, nil, fmt.Errorf("%v is not a valid uuid", id)
+		}
+
+		ver, err := strconv.Atoi(num)
+		if err != nil {
+			return nil, nil, fmt.Errorf("%v is not a valid integer", num)
+		}
+
+		repositoryIDs = append(repositoryIDs, id)
+		versions = append(versions, ver)
 	}
 	return
 }