Add support for environment specific apphosting.yaml files

PiperOrigin-RevId: 651902568
Change-Id: I21e6fd85a278a3645a419716cf8d302aac5086b2

cmd/firebase/preparer/main.go

@@ -28,6 +28,7 @@ import (
 var (
 	apphostingYAMLFilePath        = flag.String("apphostingyaml_filepath", "", "File path to user defined apphosting.yaml")
 	projectID                     = flag.String("project_id", "", "User's GCP project ID")
+	environmentName               = flag.String("environment_name", "", "Environment name tied to the build, if applicable")
 	appHostingYAMLOutputFilePath  = flag.String("apphostingyaml_output_filepath", "", "File path to write the validated and formatted apphosting.yaml to")
 	dotEnvOutputFilePath          = flag.String("dot_env_output_filepath", "", "File path to write the output .env file to")
 	backendRootDirectory          = flag.String("backend_root_directory", "", "File path to the application directory specified by the user")
@@ -67,6 +68,7 @@ func main() {
 		SecretClient:                  secretClient,
 		AppHostingYAMLPath:            *apphostingYAMLFilePath,
 		ProjectID:                     *projectID,
+		EnvironmentName:               *environmentName,
 		AppHostingYAMLOutputFilePath:  *appHostingYAMLOutputFilePath,
 		EnvDereferencedOutputFilePath: *dotEnvOutputFilePath,
 		BackendRootDirectory:          *backendRootDirectory,

pkg/firebase/apphostingschema/apphostingschema.go

@@ -174,6 +174,58 @@ func Sanitize(schema *AppHostingSchema) {
 	schema.Env = santizeEnv(schema.Env)
 }
 
+// Merge app hosting schemas with priority given to any environment specific overrides
+func mergeAppHostingSchemas(appHostingSchema *AppHostingSchema, envSpecificSchema *AppHostingSchema) {
+	// Merge RunConfig
+	if envSpecificSchema.RunConfig.CPU != nil {
+		appHostingSchema.RunConfig.CPU = envSpecificSchema.RunConfig.CPU
+	}
+	if envSpecificSchema.RunConfig.MemoryMiB != nil {
+		appHostingSchema.RunConfig.MemoryMiB = envSpecificSchema.RunConfig.MemoryMiB
+	}
+	if envSpecificSchema.RunConfig.Concurrency != nil {
+		appHostingSchema.RunConfig.Concurrency = envSpecificSchema.RunConfig.Concurrency
+	}
+	if envSpecificSchema.RunConfig.MaxInstances != nil {
+		appHostingSchema.RunConfig.MaxInstances = envSpecificSchema.RunConfig.MaxInstances
+	}
+	if envSpecificSchema.RunConfig.MinInstances != nil {
+		appHostingSchema.RunConfig.MinInstances = envSpecificSchema.RunConfig.MinInstances
+	}
+
+	// Merge Environment Variables
+	envVarMap := make(map[string]*EnvironmentVariable)
+	for i := range appHostingSchema.Env {
+		envVarMap[appHostingSchema.Env[i].Variable] = &appHostingSchema.Env[i]
+	}
+
+	for _, envVar := range envSpecificSchema.Env {
+		if existingVar, exists := envVarMap[envVar.Variable]; exists {
+			// Overwrite existing variable
+			*existingVar = envVar
+		} else {
+			// Add new variable
+			appHostingSchema.Env = append(appHostingSchema.Env, envVar)
+		}
+	}
+}
+
+// MergeWithEnvironmentSpecificYAML merges the environment specific apphosting.<environmentName>.yaml with the base apphosting schema found in apphosting.yaml
+func MergeWithEnvironmentSpecificYAML(appHostingSchema *AppHostingSchema, appHostingYAMLPath string, environmentName string) error {
+	if environmentName == "" {
+		return nil
+	}
+
+	envSpecificYAMLPath := filepath.Join(filepath.Dir(appHostingYAMLPath), fmt.Sprintf("apphosting.%v.yaml", environmentName))
+	envSpecificSchema, err := ReadAndValidateAppHostingSchemaFromFile(envSpecificYAMLPath)
+	if err != nil {
+		return fmt.Errorf("reading in and validating apphosting.%v.yaml at path %v: %w", environmentName, envSpecificYAMLPath, err)
+	}
+
+	mergeAppHostingSchemas(appHostingSchema, &envSpecificSchema)
+	return nil
+}
+
 // WriteToFile writes the given app hosting schema to the specified path.
 func (schema *AppHostingSchema) WriteToFile(outputFilePath string) error {
 	fileData, err := yaml.Marshal(schema)

pkg/firebase/apphostingschema/apphostingschema_test.go

@@ -76,7 +76,7 @@ func TestReadAndValidateAppHostingSchemaFromFile(t *testing.T) {
 			}
 
 			if diff := cmp.Diff(test.wantAppHostingSchema, s); diff != "" {
-				t.Errorf("unexpected YAML for test %q, (+got, -want):\n%v", test.desc, diff)
+				t.Errorf("unexpected YAML for test %q, (-want, +got):\n%v", test.desc, diff)
 			}
 
 			// Error Path
@@ -158,11 +158,166 @@ func TestSanitize(t *testing.T) {
 	for _, test := range testCases {
 		Sanitize(&test.inputSchema)
 		if diff := cmp.Diff(test.wantSchema, test.inputSchema); diff != "" {
-			t.Errorf("unexpected sanitized envVars for test %q (+got, -want):\n%v", test.desc, diff)
+			t.Errorf("unexpected sanitized envVars for test %q (-want, +got):\n%v", test.desc, diff)
 		}
 	}
 }
 
+func TestMergeWithEnvironmentSpecificYAML(t *testing.T) {
+	testCases := []struct {
+		desc               string
+		appHostingSchema   AppHostingSchema
+		appHostingYAMLPath string
+		environmentName    string
+		wantSchema         AppHostingSchema
+	}{
+		{
+			desc: "Merge apphosting.yaml and apphosting.<environmentName>.yaml properly",
+			appHostingSchema: AppHostingSchema{
+				RunConfig: RunConfig{
+					CPU:          proto.Float32(3),
+					MemoryMiB:    proto.Int32(1024),
+					MaxInstances: proto.Int32(4),
+					MinInstances: proto.Int32(0),
+					Concurrency:  proto.Int32(100),
+				},
+				Env: []EnvironmentVariable{
+					EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}},
+					EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}},
+					EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"},
+				},
+			},
+			appHostingYAMLPath: testdata.MustGetPath("testdata/apphosting.yaml"),
+			environmentName:    "staging",
+			wantSchema: AppHostingSchema{
+				RunConfig: RunConfig{
+					CPU:          proto.Float32(1),
+					MemoryMiB:    proto.Int32(512),
+					MaxInstances: proto.Int32(4),
+					MinInstances: proto.Int32(0),
+					Concurrency:  proto.Int32(5),
+				},
+				Env: []EnvironmentVariable{
+					EnvironmentVariable{Variable: "API_URL", Value: "api.staging.service.com", Availability: []string{"BUILD"}},
+					EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}},
+					EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"},
+					EnvironmentVariable{Variable: "DATABASE_URL", Secret: "secretStagingDatabaseURL"},
+				},
+			},
+		},
+		{
+			desc: "Don't modify apphosting.yaml when apphosting.<environmentName>.yaml is empty",
+			appHostingSchema: AppHostingSchema{
+				RunConfig: RunConfig{
+					CPU:          proto.Float32(3),
+					MemoryMiB:    proto.Int32(1024),
+					MaxInstances: proto.Int32(4),
+					MinInstances: proto.Int32(0),
+					Concurrency:  proto.Int32(100),
+				},
+				Env: []EnvironmentVariable{
+					EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}},
+					EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}},
+					EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"},
+				},
+			},
+			appHostingYAMLPath: testdata.MustGetPath("testdata/apphosting.yaml"),
+			environmentName:    "empty",
+			wantSchema: AppHostingSchema{
+				RunConfig: RunConfig{
+					CPU:          proto.Float32(3),
+					MemoryMiB:    proto.Int32(1024),
+					MaxInstances: proto.Int32(4),
+					MinInstances: proto.Int32(0),
+					Concurrency:  proto.Int32(100),
+				},
+				Env: []EnvironmentVariable{
+					EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}},
+					EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}},
+					EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"},
+				},
+			},
+		},
+		{
+			desc: "Don't modify apphosting.yaml when environment name isn't passed in",
+			appHostingSchema: AppHostingSchema{
+				RunConfig: RunConfig{
+					CPU:          proto.Float32(3),
+					MemoryMiB:    proto.Int32(1024),
+					MaxInstances: proto.Int32(4),
+					MinInstances: proto.Int32(0),
+					Concurrency:  proto.Int32(100),
+				},
+				Env: []EnvironmentVariable{
+					EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}},
+					EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}},
+					EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"},
+				},
+			},
+			appHostingYAMLPath: testdata.MustGetPath("testdata/apphosting.yaml"),
+			environmentName:    "",
+			wantSchema: AppHostingSchema{
+				RunConfig: RunConfig{
+					CPU:          proto.Float32(3),
+					MemoryMiB:    proto.Int32(1024),
+					MaxInstances: proto.Int32(4),
+					MinInstances: proto.Int32(0),
+					Concurrency:  proto.Int32(100),
+				},
+				Env: []EnvironmentVariable{
+					EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}},
+					EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}},
+					EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"},
+				},
+			},
+		},
+		{
+			desc: "Use apphosting.yaml when apphosting.<environmentName>.yaml is not found",
+			appHostingSchema: AppHostingSchema{
+				RunConfig: RunConfig{
+					CPU:          proto.Float32(3),
+					MemoryMiB:    proto.Int32(1024),
+					MaxInstances: proto.Int32(4),
+					MinInstances: proto.Int32(0),
+					Concurrency:  proto.Int32(100),
+				},
+				Env: []EnvironmentVariable{
+					EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}},
+					EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}},
+					EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"},
+				},
+			},
+			appHostingYAMLPath: testdata.MustGetPath("testdata/apphosting.yaml"),
+			environmentName:    "missingfile",
+			wantSchema: AppHostingSchema{
+				RunConfig: RunConfig{
+					CPU:          proto.Float32(3),
+					MemoryMiB:    proto.Int32(1024),
+					MaxInstances: proto.Int32(4),
+					MinInstances: proto.Int32(0),
+					Concurrency:  proto.Int32(100),
+				},
+				Env: []EnvironmentVariable{
+					EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}},
+					EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}},
+					EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"},
+				},
+			},
+		},
+	}
+
+	for _, test := range testCases {
+		if err := MergeWithEnvironmentSpecificYAML(&test.appHostingSchema, test.appHostingYAMLPath, test.environmentName); err != nil {
+			t.Fatalf("unexpected error for TestMergeWithEnvironmentSpecificYAML(%q): %v", test.desc, err)
+		}
+
+		if diff := cmp.Diff(test.wantSchema, test.appHostingSchema); diff != "" {
+			t.Errorf("unexpected merged apphosting schema for test %q (-want, +got):\n%v", test.desc, diff)
+		}
+	}
+
+}
+
 func TestWriteToFile(t *testing.T) {
 	testDir := t.TempDir()
 
@@ -221,7 +376,7 @@ func TestWriteToFile(t *testing.T) {
 		}
 
 		if diff := cmp.Diff(test.inputSchema, actualSchema); diff != "" {
-			t.Errorf("unexpected schema for test %q, (+got, -want):\n%v", test.desc, diff)
+			t.Errorf("unexpected schema for test %q, (-want, +got):\n%v", test.desc, diff)
 		}
 	}
 }

pkg/firebase/apphostingschema/testdata/apphosting.staging.yaml

@@ -0,0 +1,15 @@
+schemaVersion: '3.0.0'
+
+runConfig:
+  cpu: 1
+  memoryMiB: 512
+  concurrency: 5
+
+env:
+  - variable: API_URL
+    value: api.staging.service.com
+    availability:
+      - BUILD
+
+  - variable: DATABASE_URL
+    secret: secretStagingDatabaseURL

pkg/firebase/preparer/preparer.go

@@ -32,6 +32,7 @@ type Options struct {
 	SecretClient                  secrets.SecretManager
 	AppHostingYAMLPath            string
 	ProjectID                     string
+	EnvironmentName               string
 	AppHostingYAMLOutputFilePath  string
 	EnvDereferencedOutputFilePath string
 	BackendRootDirectory          string
@@ -57,13 +58,17 @@ func Prepare(ctx context.Context, opts Options) error {
 			return fmt.Errorf("reading in and validating apphosting.yaml at path %v: %w", opts.AppHostingYAMLPath, err)
 		}
 
+		if err = apphostingschema.MergeWithEnvironmentSpecificYAML(&appHostingYAML, opts.AppHostingYAMLPath, opts.EnvironmentName); err != nil {
+			return fmt.Errorf("merging with environment specific apphosting.%v.yaml: %w", opts.EnvironmentName, err)
+		}
+
 		apphostingschema.Sanitize(&appHostingYAML)
 
-		if err := secrets.Normalize(appHostingYAML.Env, opts.ProjectID); err != nil {
+		if err = secrets.Normalize(appHostingYAML.Env, opts.ProjectID); err != nil {
 			return fmt.Errorf("normalizing apphosting.yaml fields: %w", err)
 		}
 
-		if err := secrets.PinVersions(ctx, opts.SecretClient, appHostingYAML.Env); err != nil {
+		if err = secrets.PinVersions(ctx, opts.SecretClient, appHostingYAML.Env); err != nil {
 			return fmt.Errorf("pinning secrets in apphosting.yaml: %w", err)
 		}