add catalog pkg to load catalog sources and a simple yaml catalog imp…

…lementation scaffold

Signed-off-by: Dhiraj Bokde <[email protected]>

cmd/proxy.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"github.com/kubeflow/model-registry/internal/catalog"
 	"net/http"
 	"time"
 
@@ -40,8 +41,11 @@ func runProxyServer(cmd *cobra.Command, args []string) error {
 	}
 	if disableService != CatalogService {
 
-		// TODO read yaml catalog file and instantiate ModelCatalogAPI implementations
-		ModelCatalogServiceAPIService := openapi.NewModelCatalogServiceAPIService(map[string]openapi.ModelCatalogApi{})
+		sources, err := catalog.LoadCatalogSources(proxyCfg.CatalogsConfigPath)
+		if err != nil {
+			return fmt.Errorf("error loading catalog sources: %v", err)
+		}
+		ModelCatalogServiceAPIService := openapi.NewModelCatalogServiceAPIService(sources)
 		ModelCatalogServiceAPIController := openapi.NewModelCatalogServiceAPIController(ModelCatalogServiceAPIService)
 		routers = append(routers, ModelCatalogServiceAPIController)
 		glog.Infof("started catalog service")
@@ -95,6 +99,7 @@ func init() {
 	proxyCmd.Flags().StringVar(&proxyCfg.MLMDHostname, "mlmd-hostname", proxyCfg.MLMDHostname, "MLMD hostname")
 	proxyCmd.Flags().IntVar(&proxyCfg.MLMDPort, "mlmd-port", proxyCfg.MLMDPort, "MLMD port")
 
+	proxyCmd.Flags().StringVar(&proxyCfg.CatalogsConfigPath, "catalogs-path", proxyCfg.DisableService, "Path for Model Catalog source configuration yaml file")
 	proxyCmd.Flags().StringVar(&proxyCfg.DisableService, "disable-service", proxyCfg.DisableService, "Optional name of service/endpoint to disable, can be either \"catalog\" or \"registry\"")
 }
 
@@ -107,7 +112,9 @@ type ProxyConfig struct {
 	MLMDHostname string
 	MLMDPort     int
 
-	DisableService string
+	// Catalog sources config file path
+	CatalogsConfigPath string
+	DisableService     string
 }
 
 var proxyCfg = ProxyConfig{

internal/catalog/catalog.go

@@ -0,0 +1,67 @@
+package catalog
+
+import (
+	"context"
+	"fmt"
+	"github.com/kubeflow/model-registry/pkg/openapi"
+	yaml3 "gopkg.in/yaml.v3"
+	"log"
+	"os"
+)
+
+// ModelCatalogApi is implemented by catalog types, e.g. YamlCatalog
+type ModelCatalogApi interface {
+	GetCatalogModel(ctx context.Context, modelId string) (openapi.CatalogModel, error)
+	GetCatalogModelVersion(ctx context.Context, modelId string, versionId string) (openapi.CatalogModelVersion, error)
+	GetCatalogModelVersions(ctx context.Context, modelId string, nameParam string, externalIdParam string, pageSizeParam string, orderByParam openapi.OrderByField, sortOrderParam openapi.SortOrder, offsetParam string) (openapi.CatalogModelVersionList, error)
+	GetCatalogModels(ctx context.Context, nameParam string, externalIdParam string, pageSizeParam string, orderByParam openapi.OrderByField, sortOrderParam openapi.SortOrder, offsetParam string) (openapi.CatalogModelList, error)
+	GetCatalogSource() (openapi.CatalogSource, error)
+}
+
+type CatalogSource struct {
+	openapi.CatalogSource
+
+	// Catalog type to use, must match one of the registered types
+	Type string `json:"type"`
+
+	// private properties used for configuring the catalog connection based on catalog implementation
+	PrivateProperties *map[string]openapi.MetadataValue `json:"privateProperties,omitempty"`
+}
+
+type CatalogsConfig struct {
+	Catalogs []CatalogSource `json:"catalogs"`
+}
+
+type CatalogTypeRegisterFunc func (source *CatalogSource) ModelCatalogApi
+
+var catalogTypes = make(map[string]CatalogTypeRegisterFunc, 0)
+
+func RegisterCatalogType(catalogType string, callback CatalogTypeRegisterFunc) {
+	catalogTypes[catalogType] = callback
+}
+
+func LoadCatalogSources(catalogsPath string) (map[string]ModelCatalogApi, error) {
+	config := CatalogsConfig{}
+	f, err := os.Open(catalogsPath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	decoder := yaml3.NewDecoder(f)
+	if err := decoder.Decode(&config); err != nil {
+		log.Fatal(err)
+	}
+
+	catalogs := make(map[string]ModelCatalogApi)
+	for _, catalog := range config.Catalogs {
+		catalogType := catalog.Type
+		registerFunc, ok := catalogTypes[catalogType]
+		if !ok {
+			return nil, fmt.Errorf("catalog type %s not registered", catalogType)
+		}
+		id := catalog.GetId()
+		catalogs[id] = registerFunc(&catalog)
+	}
+	return catalogs, nil
+}

internal/catalog/yaml_catalog.go

@@ -0,0 +1,71 @@
+package catalog
+
+import (
+	"context"
+	"github.com/kubeflow/model-registry/pkg/openapi"
+)
+
+type YamlCatalog struct {
+	Models []struct {
+		Name                     string   `yaml:"name"`
+		Provider                 string   `yaml:"provider"`
+		Description              string   `yaml:"description"`
+		ReadmeLink               string   `yaml:"readmeLink"`
+		Language                 []string `yaml:"language"`
+		License                  string   `yaml:"license"`
+		LicenseLink              string   `yaml:"licenseLink"`
+		LibraryName              string   `yaml:"libraryName"`
+		Tags                     []string `yaml:"tags"`
+		Tasks                    []string `yaml:"tasks"`
+		CreateTimeSinceEpoch     int64    `yaml:"createTimeSinceEpoch"`
+		LastUpdateTimeSinceEpoch int64    `yaml:"lastUpdateTimeSinceEpoch"`
+		BaseModel                []struct {
+			Catalog    string `yaml:"catalog"`
+			Repository string `yaml:"repository"`
+			Model      string `yaml:"model"`
+		} `yaml:"baseModel,omitempty"`
+	} `yaml:"models"`
+}
+
+type yamlCatalogImpl struct {
+	contents *YamlCatalog
+	source   *CatalogSource
+}
+
+func (y yamlCatalogImpl) GetCatalogModel(ctx context.Context, modelId string) (openapi.CatalogModel, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (y yamlCatalogImpl) GetCatalogModelVersion(ctx context.Context, modelId string, versionId string) (openapi.CatalogModelVersion, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (y yamlCatalogImpl) GetCatalogModelVersions(ctx context.Context, modelId string, nameParam string, externalIdParam string, pageSizeParam string, orderByParam openapi.OrderByField, sortOrderParam openapi.SortOrder, offsetParam string) (openapi.CatalogModelVersionList, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (y yamlCatalogImpl) GetCatalogModels(ctx context.Context, nameParam string, externalIdParam string, pageSizeParam string, orderByParam openapi.OrderByField, sortOrderParam openapi.SortOrder, offsetParam string) (openapi.CatalogModelList, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (y yamlCatalogImpl) GetCatalogSource() (openapi.CatalogSource, error) {
+	return y.source.CatalogSource, nil
+}
+
+// TODO start background thread to watch file
+
+var _ ModelCatalogApi = &yamlCatalogImpl{}
+
+func NewYamlCatalog(source *CatalogSource) ModelCatalogApi {
+	// TODO read file contents from config
+	var contents YamlCatalog
+	return &yamlCatalogImpl{source: source, contents: &contents}
+}
+
+func init() {
+	RegisterCatalogType("yaml", NewYamlCatalog)
+}

internal/server/openapi/api_model_catalog_service_service.go

@@ -12,23 +12,16 @@ package openapi
 import (
 	"context"
 	"fmt"
+	"github.com/kubeflow/model-registry/internal/catalog"
 	"github.com/kubeflow/model-registry/pkg/openapi"
 	"net/http"
 )
 
-type ModelCatalogApi interface {
-	GetCatalogModel(ctx context.Context, modelId string) (openapi.CatalogModel, error)
-	GetCatalogModelVersion(ctx context.Context, modelId string, versionId string) (openapi.CatalogModelVersion, error)
-	GetCatalogModelVersions(ctx context.Context, modelId string, nameParam string, externalIdParam string, pageSizeParam string, orderByParam openapi.OrderByField, sortOrderParam openapi.SortOrder, offsetParam string) (openapi.CatalogModelVersionList, error)
-	GetCatalogModels(ctx context.Context, nameParam string, externalIdParam string, pageSizeParam string, orderByParam openapi.OrderByField, sortOrderParam openapi.SortOrder, offsetParam string) (openapi.CatalogModelList, error)
-	GetCatalogSource() (openapi.CatalogSource, error)
-}
-
 // ModelCatalogServiceAPIService is a service that implements the logic for the ModelCatalogServiceAPIServicer
 // This service should implement the business logic for every endpoint for the ModelCatalogServiceAPI s.coreApi.
 // Include any external packages or services that will be required by this service.
 type ModelCatalogServiceAPIService struct {
-	modelCatalogs map[string]ModelCatalogApi
+	modelCatalogs map[string]catalog.ModelCatalogApi
 }
 
 func (m ModelCatalogServiceAPIService) GetCatalogModel(ctx context.Context, id string, modelId string) (ImplResponse, error) {
@@ -112,7 +105,7 @@ func missingCatalogError(id string) (ImplResponse, error) {
 }
 
 // NewModelCatalogServiceAPIService creates a default api service
-func NewModelCatalogServiceAPIService(modelCatalogs map[string]ModelCatalogApi) ModelCatalogServiceAPIServicer {
+func NewModelCatalogServiceAPIService(modelCatalogs map[string]catalog.ModelCatalogApi) ModelCatalogServiceAPIServicer {
 	return &ModelCatalogServiceAPIService{
 		modelCatalogs: modelCatalogs,
 	}