Reaper optimization.

Signed-off-by: Jeff Ortel <[email protected]>

reaper/bucket.go

@@ -1,22 +1,21 @@
 package reaper
 
 import (
+	"os"
+	"time"
+
 	liberr "github.com/jortel/go-utils/error"
 	"github.com/konveyor/tackle2-hub/model"
 	"github.com/konveyor/tackle2-hub/nas"
 	"gorm.io/gorm"
-	"os"
-	"time"
 )
 
-//
 // BucketReaper bucket reaper.
 type BucketReaper struct {
 	// DB
 	DB *gorm.DB
 }
 
-//
 // Run Executes the reaper.
 // A bucket is deleted when it is no longer referenced and the TTL has expired.
 func (r *BucketReaper) Run() {
@@ -27,13 +26,24 @@ func (r *BucketReaper) Run() {
 		Log.Error(err, "")
 		return
 	}
-	for _, bucket := range list {
-		busy, err := r.busy(&bucket)
+	if len(list) == 0 {
+		return
+	}
+	ids := make(map[uint]byte)
+	finder := RefFinder{DB: r.DB}
+	for _, m := range []any{
+		&model.Application{},
+		&model.TaskGroup{},
+		&model.Task{},
+	} {
+		err := finder.Find(m, "bucket", ids)
 		if err != nil {
 			Log.Error(err, "")
 			continue
 		}
-		if busy {
+	}
+	for _, bucket := range list {
+		if _, found := ids[bucket.ID]; found {
 			if bucket.Expiration != nil {
 				bucket.Expiration = nil
 				err = r.DB.Save(&bucket).Error
@@ -60,29 +70,6 @@ func (r *BucketReaper) Run() {
 	}
 }
 
-//
-// busy determines if anything references the bucket.
-func (r *BucketReaper) busy(bucket *model.Bucket) (busy bool, err error) {
-	nRef := int64(0)
-	var n int64
-	ref := RefCounter{DB: r.DB}
-	for _, m := range []interface{}{
-		&model.Application{},
-		&model.TaskGroup{},
-		&model.Task{},
-	} {
-		n, err = ref.Count(m, "bucket", bucket.ID)
-		if err != nil {
-			Log.Error(err, "")
-			continue
-		}
-		nRef += n
-	}
-	busy = nRef > 0
-	return
-}
-
-//
 // Delete bucket.
 func (r *BucketReaper) delete(bucket *model.Bucket) (err error) {
 	err = nas.RmDir(bucket.Path)

reaper/file.go

@@ -1,21 +1,20 @@
 package reaper
 
 import (
+	"os"
+	"time"
+
 	liberr "github.com/jortel/go-utils/error"
 	"github.com/konveyor/tackle2-hub/model"
 	"gorm.io/gorm"
-	"os"
-	"time"
 )
 
-//
 // FileReaper file reaper.
 type FileReaper struct {
 	// DB
 	DB *gorm.DB
 }
 
-//
 // Run Executes the reaper.
 // A file is deleted when it is no longer referenced and the TTL has expired.
 func (r *FileReaper) Run() {
@@ -26,13 +25,25 @@ func (r *FileReaper) Run() {
 		Log.Error(err, "")
 		return
 	}
-	for _, file := range list {
-		busy, err := r.busy(&file)
+	if len(list) == 0 {
+		return
+	}
+	ids := make(map[uint]byte)
+	finder := RefFinder{DB: r.DB}
+	for _, m := range []any{
+		&model.Task{},
+		&model.TaskReport{},
+		&model.Rule{},
+		&model.Target{},
+	} {
+		err := finder.Find(m, "file", ids)
 		if err != nil {
 			Log.Error(err, "")
 			continue
 		}
-		if busy {
+	}
+	for _, file := range list {
+		if _, found := ids[file.ID]; found {
 			if file.Expiration != nil {
 				file.Expiration = nil
 				err = r.DB.Save(&file).Error
@@ -59,30 +70,6 @@ func (r *FileReaper) Run() {
 	}
 }
 
-//
-// busy determines if anything references the file.
-func (r *FileReaper) busy(file *model.File) (busy bool, err error) {
-	nRef := int64(0)
-	var n int64
-	ref := RefCounter{DB: r.DB}
-	for _, m := range []interface{}{
-		&model.TaskReport{},
-		&model.RuleSet{},
-		&model.Rule{},
-		&model.Target{},
-	} {
-		n, err = ref.Count(m, "file", file.ID)
-		if err != nil {
-			Log.Error(err, "")
-			continue
-		}
-		nRef += n
-	}
-	busy = nRef > 0
-	return
-}
-
-//
 // Delete file.
 func (r *FileReaper) delete(file *model.File) (err error) {
 	err = os.Remove(file.Path)

reaper/manager.go

@@ -2,12 +2,13 @@ package reaper
 
 import (
 	"context"
+	"time"
+
 	"github.com/jortel/go-utils/logr"
 	"github.com/konveyor/tackle2-hub/settings"
 	"github.com/konveyor/tackle2-hub/task"
 	"gorm.io/gorm"
 	k8s "sigs.k8s.io/controller-runtime/pkg/client"
-	"time"
 )
 
 const (
@@ -21,7 +22,6 @@ var (
 
 type Task = task.Task
 
-//
 // Manager provides task management.
 type Manager struct {
 	// DB
@@ -30,7 +30,6 @@ type Manager struct {
 	Client k8s.Client
 }
 
-//
 // Run the manager.
 func (m *Manager) Run(ctx context.Context) {
 	registered := []Reaper{
@@ -65,14 +64,12 @@ func (m *Manager) Run(ctx context.Context) {
 	}()
 }
 
-//
 // Pause.
 func (m *Manager) pause() {
 	d := Unit * time.Duration(Settings.Frequency.Reaper)
 	time.Sleep(d)
 }
 
-//
 // Reaper interface.
 type Reaper interface {
 	Run()

reaper/ref.go

@@ -1,52 +1,43 @@
 package reaper
 
 import (
+	"fmt"
+	"reflect"
+
 	liberr "github.com/jortel/go-utils/error"
 	"gorm.io/gorm"
-	"reflect"
-	"fmt"
 )
 
+// RefFinder provides model inspection for files
+// tagged with:
 //
-// RefCounter provides model inspection for files
-// tagged with: ref:<kind>.
-type RefCounter struct {
+//	ref:<kind>
+//	[]ref:<kind>
+type RefFinder struct {
 	// DB
 	DB *gorm.DB
 }
 
-//
-// Count find & count references.
-func (r *RefCounter) Count(m interface{}, kind string, pk uint) (nRef int64, err error) {
-	db := r.DB.Model(m)
-	fields := 0
-	j := 0
+// Find returns a map of all references for the model and kind.
+func (r *RefFinder) Find(m any, kind string, ids map[uint]byte) (err error) {
+	var nfields []string
+	var jfields []string
 	add := func(ft reflect.StructField) {
 		tag, found := ft.Tag.Lookup("ref")
 		if found && tag == kind {
-			db = db.Or(ft.Name, pk)
-			fields++
+			nfields = append(
+				nfields,
+				ft.Name)
 			return
 		}
 		if found && tag == "[]"+kind {
-			db = db.Joins(
-				fmt.Sprintf(
-					",json_each(%s) j%d",
-					ft.Name,
-					j))
-			db = db.Or(
-				fmt.Sprintf(
-					"json_extract(j%d.value,?)=?",
-					j),
-				"$.id",
-				pk)
-			fields++
-			j++
-			return
+			jfields = append(
+				jfields,
+				ft.Name)
 		}
 	}
-	var find func(interface{})
-	find = func(object interface{}) {
+	var find func(any)
+	find = func(object any) {
 		mt := reflect.TypeOf(object)
 		mv := reflect.ValueOf(object)
 		if mt.Kind() == reflect.Ptr {
@@ -77,21 +68,58 @@ func (r *RefCounter) Count(m interface{}, kind string, pk uint) (nRef int64, err
 				add(ft)
 			case reflect.Slice:
 				add(ft)
+			default:
 			}
 		}
 	}
 	find(m)
-	if fields == 0 {
+	if len(nfields)+len(jfields) == 0 {
 		return
 	}
-	err = db.Count(&nRef).Error
+	db := r.DB.Model(m)
+	if Log.V(1).Enabled() {
+		db = db.Debug()
+	}
+	var fields []string
+	var list []map[string]any
+	for i := range nfields {
+		fields = append(fields, nfields[i])
+	}
+	for i := range jfields {
+		fields = append(
+			fields,
+			fmt.Sprintf(
+				"json_extract(j%d.value,'$.id')",
+				i))
+		db = db.Joins(
+			fmt.Sprintf(
+				",json_each(%s) j%d",
+				jfields[i],
+				i))
+	}
+	db = db.Select(fields)
+	err = db.Find(&list).Error
 	if err != nil {
 		err = liberr.Wrap(
 			err,
 			"object",
 			reflect.TypeOf(m).Name(),
 		)
 	}
+	for _, ref := range list {
+		for _, v := range ref {
+			switch n := v.(type) {
+			case uint:
+				ids[n] = 0
+			case *uint:
+				if n != nil {
+					ids[*n] = 0
+				}
+			case int64:
+				ids[uint(n)] = 0
+			}
+		}
+	}
 
 	return
 }