Merge pull request #1175 from blublinsky/additional-providers

add additional vllm based providers

Author: openshift-merge-bot[bot]

internal/controller/lcore/assets_test.go

@@ -65,8 +65,9 @@ func TestBuildLlamaStackYAML_SupportedProvider(t *testing.T) {
 }
 
 func TestBuildLlamaStackYAML_UnsupportedProvider(t *testing.T) {
-	// Test unsupported providers
-	unsupportedProviders := []string{"watsonx", "bam", "rhoai_vllm", "rhelai_vllm"}
+	// Test unsupported providers (watsonx, bam are not supported)
+	// Note: rhoai_vllm and rhelai_vllm are now supported via OpenAI compatibility
+	unsupportedProviders := []string{"watsonx", "bam"}
 
 	for _, providerType := range unsupportedProviders {
 		t.Run(providerType, func(t *testing.T) {
@@ -116,6 +117,94 @@ func TestBuildLlamaStackYAML_UnsupportedProvider(t *testing.T) {
 	}
 }
 
+func TestBuildLlamaStackYAML_OpenAICompatibleProviders(t *testing.T) {
+	// Test that vLLM providers (rhoai_vllm, rhelai_vllm) use remote::vllm provider type
+	vllmProviders := []string{"rhoai_vllm", "rhelai_vllm"}
+
+	for _, providerType := range vllmProviders {
+		t.Run(providerType, func(t *testing.T) {
+			// Create a test CR with vLLM provider
+			cr := &olsv1alpha1.OLSConfig{
+				Spec: olsv1alpha1.OLSConfigSpec{
+					LLMConfig: olsv1alpha1.LLMSpec{
+						Providers: []olsv1alpha1.ProviderSpec{
+							{
+								Name: "test-provider",
+								Type: providerType,
+								URL:  "https://test-vllm-endpoint.com/v1",
+								Models: []olsv1alpha1.ModelSpec{
+									{Name: "test-model"},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			// Build the YAML - should succeed
+			ctx := context.Background()
+			yamlOutput, err := buildLlamaStackYAML(nil, ctx, cr)
+
+			// Verify no error is returned
+			if err != nil {
+				t.Fatalf("Unexpected error for supported provider '%s': %v", providerType, err)
+			}
+
+			// Verify it's valid YAML
+			var result map[string]interface{}
+			err = yaml.Unmarshal([]byte(yamlOutput), &result)
+			if err != nil {
+				t.Fatalf("buildLlamaStackYAML produced invalid YAML for '%s': %v", providerType, err)
+			}
+
+			// Verify provider is configured as remote::vllm
+			providers, ok := result["providers"].(map[string]interface{})
+			if !ok {
+				t.Fatalf("providers section not found or invalid type")
+			}
+
+			inference, ok := providers["inference"].([]interface{})
+			if !ok || len(inference) == 0 {
+				t.Fatalf("inference providers not found or empty")
+			}
+
+			// Find the test provider (not the sentence-transformers one)
+			var testProvider map[string]interface{}
+			for _, provider := range inference {
+				p, ok := provider.(map[string]interface{})
+				if !ok {
+					continue
+				}
+				if p["provider_id"] == "test-provider" {
+					testProvider = p
+					break
+				}
+			}
+
+			if testProvider == nil {
+				t.Fatalf("Test provider not found in inference providers")
+			}
+
+			// Verify it's configured as vLLM (not OpenAI)
+			if testProvider["provider_type"] != "remote::vllm" {
+				t.Errorf("Expected provider_type 'remote::vllm' for %s, got '%v'", providerType, testProvider["provider_type"])
+			}
+
+			// Verify URL is present in config
+			config, ok := testProvider["config"].(map[string]interface{})
+			if !ok {
+				t.Fatalf("provider config not found or invalid type")
+			}
+
+			if url, ok := config["url"].(string); !ok || url == "" {
+				t.Errorf("Expected URL to be configured for %s provider", providerType)
+			}
+
+			t.Logf("Successfully validated '%s' provider uses remote::vllm", providerType)
+		})
+	}
+}
+
 func TestBuildLlamaStackYAML_AzureProvider(t *testing.T) {
 	// Create a fake secret with API token for Azure provider
 	secret := &corev1.Secret{

internal/controller/lcore/config.go

@@ -74,15 +74,16 @@ func buildLlamaStackCoreConfig(_ reconciler.Reconciler, _ *olsv1alpha1.OLSConfig
 	return map[string]interface{}{
 		"version":    "2",
 		"image_name": "minimal-viable-llama-stack-configuration",
-		// Minimal APIs for RAG + MCP: agents (for MCP), files, inference, safety (required by agents), tool_runtime, vector_io
-		// Commented out: datasetio, eval, post_training, scoring, telemetry (not available in this Llama Stack version)
-		"apis":                   []string{"agents" /* "datasetio", "eval", */, "files", "inference" /* , "post_training", */, "safety" /* , "scoring", "telemetry" */, "tool_runtime", "vector_io"},
+		// Minimal APIs for RAG + MCP: agents (for MCP), files, inference, safety (required by agents), telemetry, tool_runtime, vector_io
+		// Commented out: datasetio, eval, post_training, scoring (not needed for basic RAG + MCP)
+		// Commented out: datasetio, eval, post_training, prompts, scoring, telemetry
+		"apis":                   []string{"agents" /* "datasetio", "eval", */, "files", "inference" /* , "post_training", */, "safety" /* , "scoring", "telemetry"*/, "tool_runtime", "vector_io"},
 		"benchmarks":             []interface{}{},
 		"container_image":        nil,
 		"datasets":               []interface{}{},
 		"external_providers_dir": nil,
 		"inference_store": map[string]interface{}{
-			"db_path": "/tmp/llama-stack/inference_store.db",
+			"db_path": ".llama/distributions/ollama/inference_store.db",
 			"type":    "sqlite",
 		},
 		"logging": nil,
@@ -205,15 +206,23 @@ func buildLlamaStackInferenceProviders(_ reconciler.Reconciler, _ context.Contex
 		envVarName := utils.ProviderNameToEnvVarName(provider.Name)
 
 		// Map OLSConfig provider types to Llama Stack provider types
-		// Note: Only providers supported by Llama Stack are included
 		switch provider.Type {
-		case "openai":
-			providerConfig["provider_type"] = "remote::openai"
+		case "openai", "rhoai_vllm", "rhelai_vllm":
 			config := map[string]interface{}{}
-
-			// Set environment variable name for API key
-			// Llama Stack will substitute ${env.VAR_NAME} with the actual env var value
-			config["api_key"] = fmt.Sprintf("${env.%s_API_KEY}", envVarName)
+			// Determine the appropriate Llama Stack provider type
+			// - OpenAI uses remote::openai (validates against OpenAI model whitelist)
+			// - vLLM uses remote::vllm (accepts any custom model names)
+			if provider.Type == "openai" {
+				providerConfig["provider_type"] = "remote::openai"
+				// Set API key from environment variable
+				// Llama Stack will substitute ${env.VAR_NAME} with the actual env var value
+				config["api_key"] = fmt.Sprintf("${env.%s_API_KEY}", envVarName)
+			} else {
+				providerConfig["provider_type"] = "remote::vllm"
+				// Set API key from environment variable
+				// Llama Stack will substitute ${env.VAR_NAME} with the actual env var value
+				config["api_token"] = fmt.Sprintf("${env.%s_API_KEY}", envVarName)
+			}
 
 			// Add custom URL if specified
 			if provider.URL != "" {
@@ -246,14 +255,14 @@ func buildLlamaStackInferenceProviders(_ reconciler.Reconciler, _ context.Contex
 			}
 			providerConfig["config"] = config
 
-		case "watsonx", "rhoai_vllm", "rhelai_vllm", "bam":
+		case "watsonx", "bam":
 			// These providers are not supported by Llama Stack
 			// They are handled directly by lightspeed-stack (LCS), not Llama Stack
-			return nil, fmt.Errorf("provider type '%s' (provider '%s') is not currently supported by Llama Stack. Supported types: openai, azure_openai", provider.Type, provider.Name)
+			return nil, fmt.Errorf("provider type '%s' (provider '%s') is not currently supported by Llama Stack. Supported types: openai, azure_openai, rhoai_vllm, rhelai_vllm", provider.Type, provider.Name)
 
 		default:
 			// Unknown provider type
-			return nil, fmt.Errorf("unknown provider type '%s' (provider '%s'). Supported types: openai, azure_openai", provider.Type, provider.Name)
+			return nil, fmt.Errorf("unknown provider type '%s' (provider '%s'). Supported types: openai, azure_openai, rhoai_vllm, rhelai_vllm", provider.Type, provider.Name)
 		}
 
 		providers = append(providers, providerConfig)