diff --git a/cmd/firebase/preparer/main.go b/cmd/firebase/preparer/main.go index b6c5832b1..6f7a5a6b1 100644 --- a/cmd/firebase/preparer/main.go +++ b/cmd/firebase/preparer/main.go @@ -28,6 +28,7 @@ import ( var ( apphostingYAMLFilePath = flag.String("apphostingyaml_filepath", "", "File path to user defined apphosting.yaml") projectID = flag.String("project_id", "", "User's GCP project ID") + environmentName = flag.String("environment_name", "", "Environment name tied to the build, if applicable") appHostingYAMLOutputFilePath = flag.String("apphostingyaml_output_filepath", "", "File path to write the validated and formatted apphosting.yaml to") dotEnvOutputFilePath = flag.String("dot_env_output_filepath", "", "File path to write the output .env file to") backendRootDirectory = flag.String("backend_root_directory", "", "File path to the application directory specified by the user") @@ -67,6 +68,7 @@ func main() { SecretClient: secretClient, AppHostingYAMLPath: *apphostingYAMLFilePath, ProjectID: *projectID, + EnvironmentName: *environmentName, AppHostingYAMLOutputFilePath: *appHostingYAMLOutputFilePath, EnvDereferencedOutputFilePath: *dotEnvOutputFilePath, BackendRootDirectory: *backendRootDirectory, diff --git a/pkg/firebase/apphostingschema/apphostingschema.go b/pkg/firebase/apphostingschema/apphostingschema.go index 8f3fb34e3..ef6227701 100644 --- a/pkg/firebase/apphostingschema/apphostingschema.go +++ b/pkg/firebase/apphostingschema/apphostingschema.go @@ -174,6 +174,58 @@ func Sanitize(schema *AppHostingSchema) { schema.Env = santizeEnv(schema.Env) } +// Merge app hosting schemas with priority given to any environment specific overrides +func mergeAppHostingSchemas(appHostingSchema *AppHostingSchema, envSpecificSchema *AppHostingSchema) { + // Merge RunConfig + if envSpecificSchema.RunConfig.CPU != nil { + appHostingSchema.RunConfig.CPU = envSpecificSchema.RunConfig.CPU + } + if envSpecificSchema.RunConfig.MemoryMiB != nil { + appHostingSchema.RunConfig.MemoryMiB = envSpecificSchema.RunConfig.MemoryMiB + } + if envSpecificSchema.RunConfig.Concurrency != nil { + appHostingSchema.RunConfig.Concurrency = envSpecificSchema.RunConfig.Concurrency + } + if envSpecificSchema.RunConfig.MaxInstances != nil { + appHostingSchema.RunConfig.MaxInstances = envSpecificSchema.RunConfig.MaxInstances + } + if envSpecificSchema.RunConfig.MinInstances != nil { + appHostingSchema.RunConfig.MinInstances = envSpecificSchema.RunConfig.MinInstances + } + + // Merge Environment Variables + envVarMap := make(map[string]*EnvironmentVariable) + for i := range appHostingSchema.Env { + envVarMap[appHostingSchema.Env[i].Variable] = &appHostingSchema.Env[i] + } + + for _, envVar := range envSpecificSchema.Env { + if existingVar, exists := envVarMap[envVar.Variable]; exists { + // Overwrite existing variable + *existingVar = envVar + } else { + // Add new variable + appHostingSchema.Env = append(appHostingSchema.Env, envVar) + } + } +} + +// MergeWithEnvironmentSpecificYAML merges the environment specific apphosting..yaml with the base apphosting schema found in apphosting.yaml +func MergeWithEnvironmentSpecificYAML(appHostingSchema *AppHostingSchema, appHostingYAMLPath string, environmentName string) error { + if environmentName == "" { + return nil + } + + envSpecificYAMLPath := filepath.Join(filepath.Dir(appHostingYAMLPath), fmt.Sprintf("apphosting.%v.yaml", environmentName)) + envSpecificSchema, err := ReadAndValidateAppHostingSchemaFromFile(envSpecificYAMLPath) + if err != nil { + return fmt.Errorf("reading in and validating apphosting.%v.yaml at path %v: %w", environmentName, envSpecificYAMLPath, err) + } + + mergeAppHostingSchemas(appHostingSchema, &envSpecificSchema) + return nil +} + // WriteToFile writes the given app hosting schema to the specified path. func (schema *AppHostingSchema) WriteToFile(outputFilePath string) error { fileData, err := yaml.Marshal(schema) diff --git a/pkg/firebase/apphostingschema/apphostingschema_test.go b/pkg/firebase/apphostingschema/apphostingschema_test.go index 1de88a491..0a4ac9af2 100644 --- a/pkg/firebase/apphostingschema/apphostingschema_test.go +++ b/pkg/firebase/apphostingschema/apphostingschema_test.go @@ -76,7 +76,7 @@ func TestReadAndValidateAppHostingSchemaFromFile(t *testing.T) { } if diff := cmp.Diff(test.wantAppHostingSchema, s); diff != "" { - t.Errorf("unexpected YAML for test %q, (+got, -want):\n%v", test.desc, diff) + t.Errorf("unexpected YAML for test %q, (-want, +got):\n%v", test.desc, diff) } // Error Path @@ -158,11 +158,166 @@ func TestSanitize(t *testing.T) { for _, test := range testCases { Sanitize(&test.inputSchema) if diff := cmp.Diff(test.wantSchema, test.inputSchema); diff != "" { - t.Errorf("unexpected sanitized envVars for test %q (+got, -want):\n%v", test.desc, diff) + t.Errorf("unexpected sanitized envVars for test %q (-want, +got):\n%v", test.desc, diff) } } } +func TestMergeWithEnvironmentSpecificYAML(t *testing.T) { + testCases := []struct { + desc string + appHostingSchema AppHostingSchema + appHostingYAMLPath string + environmentName string + wantSchema AppHostingSchema + }{ + { + desc: "Merge apphosting.yaml and apphosting..yaml properly", + appHostingSchema: AppHostingSchema{ + RunConfig: RunConfig{ + CPU: proto.Float32(3), + MemoryMiB: proto.Int32(1024), + MaxInstances: proto.Int32(4), + MinInstances: proto.Int32(0), + Concurrency: proto.Int32(100), + }, + Env: []EnvironmentVariable{ + EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}}, + EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}}, + EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"}, + }, + }, + appHostingYAMLPath: testdata.MustGetPath("testdata/apphosting.yaml"), + environmentName: "staging", + wantSchema: AppHostingSchema{ + RunConfig: RunConfig{ + CPU: proto.Float32(1), + MemoryMiB: proto.Int32(512), + MaxInstances: proto.Int32(4), + MinInstances: proto.Int32(0), + Concurrency: proto.Int32(5), + }, + Env: []EnvironmentVariable{ + EnvironmentVariable{Variable: "API_URL", Value: "api.staging.service.com", Availability: []string{"BUILD"}}, + EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}}, + EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"}, + EnvironmentVariable{Variable: "DATABASE_URL", Secret: "secretStagingDatabaseURL"}, + }, + }, + }, + { + desc: "Don't modify apphosting.yaml when apphosting..yaml is empty", + appHostingSchema: AppHostingSchema{ + RunConfig: RunConfig{ + CPU: proto.Float32(3), + MemoryMiB: proto.Int32(1024), + MaxInstances: proto.Int32(4), + MinInstances: proto.Int32(0), + Concurrency: proto.Int32(100), + }, + Env: []EnvironmentVariable{ + EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}}, + EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}}, + EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"}, + }, + }, + appHostingYAMLPath: testdata.MustGetPath("testdata/apphosting.yaml"), + environmentName: "empty", + wantSchema: AppHostingSchema{ + RunConfig: RunConfig{ + CPU: proto.Float32(3), + MemoryMiB: proto.Int32(1024), + MaxInstances: proto.Int32(4), + MinInstances: proto.Int32(0), + Concurrency: proto.Int32(100), + }, + Env: []EnvironmentVariable{ + EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}}, + EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}}, + EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"}, + }, + }, + }, + { + desc: "Don't modify apphosting.yaml when environment name isn't passed in", + appHostingSchema: AppHostingSchema{ + RunConfig: RunConfig{ + CPU: proto.Float32(3), + MemoryMiB: proto.Int32(1024), + MaxInstances: proto.Int32(4), + MinInstances: proto.Int32(0), + Concurrency: proto.Int32(100), + }, + Env: []EnvironmentVariable{ + EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}}, + EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}}, + EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"}, + }, + }, + appHostingYAMLPath: testdata.MustGetPath("testdata/apphosting.yaml"), + environmentName: "", + wantSchema: AppHostingSchema{ + RunConfig: RunConfig{ + CPU: proto.Float32(3), + MemoryMiB: proto.Int32(1024), + MaxInstances: proto.Int32(4), + MinInstances: proto.Int32(0), + Concurrency: proto.Int32(100), + }, + Env: []EnvironmentVariable{ + EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}}, + EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}}, + EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"}, + }, + }, + }, + { + desc: "Use apphosting.yaml when apphosting..yaml is not found", + appHostingSchema: AppHostingSchema{ + RunConfig: RunConfig{ + CPU: proto.Float32(3), + MemoryMiB: proto.Int32(1024), + MaxInstances: proto.Int32(4), + MinInstances: proto.Int32(0), + Concurrency: proto.Int32(100), + }, + Env: []EnvironmentVariable{ + EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}}, + EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}}, + EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"}, + }, + }, + appHostingYAMLPath: testdata.MustGetPath("testdata/apphosting.yaml"), + environmentName: "missingfile", + wantSchema: AppHostingSchema{ + RunConfig: RunConfig{ + CPU: proto.Float32(3), + MemoryMiB: proto.Int32(1024), + MaxInstances: proto.Int32(4), + MinInstances: proto.Int32(0), + Concurrency: proto.Int32(100), + }, + Env: []EnvironmentVariable{ + EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD", "RUNTIME"}}, + EnvironmentVariable{Variable: "STORAGE_BUCKET", Value: "mybucket.appspot.com", Availability: []string{"RUNTIME"}}, + EnvironmentVariable{Variable: "API_KEY", Secret: "secretIDforAPI"}, + }, + }, + }, + } + + for _, test := range testCases { + if err := MergeWithEnvironmentSpecificYAML(&test.appHostingSchema, test.appHostingYAMLPath, test.environmentName); err != nil { + t.Fatalf("unexpected error for TestMergeWithEnvironmentSpecificYAML(%q): %v", test.desc, err) + } + + if diff := cmp.Diff(test.wantSchema, test.appHostingSchema); diff != "" { + t.Errorf("unexpected merged apphosting schema for test %q (-want, +got):\n%v", test.desc, diff) + } + } + +} + func TestWriteToFile(t *testing.T) { testDir := t.TempDir() @@ -221,7 +376,7 @@ func TestWriteToFile(t *testing.T) { } if diff := cmp.Diff(test.inputSchema, actualSchema); diff != "" { - t.Errorf("unexpected schema for test %q, (+got, -want):\n%v", test.desc, diff) + t.Errorf("unexpected schema for test %q, (-want, +got):\n%v", test.desc, diff) } } } diff --git a/pkg/firebase/apphostingschema/testdata/apphosting.empty.yaml b/pkg/firebase/apphostingschema/testdata/apphosting.empty.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/firebase/apphostingschema/testdata/apphosting.staging.yaml b/pkg/firebase/apphostingschema/testdata/apphosting.staging.yaml new file mode 100644 index 000000000..c878f72e2 --- /dev/null +++ b/pkg/firebase/apphostingschema/testdata/apphosting.staging.yaml @@ -0,0 +1,15 @@ +schemaVersion: '3.0.0' + +runConfig: + cpu: 1 + memoryMiB: 512 + concurrency: 5 + +env: + - variable: API_URL + value: api.staging.service.com + availability: + - BUILD + + - variable: DATABASE_URL + secret: secretStagingDatabaseURL \ No newline at end of file diff --git a/pkg/firebase/preparer/preparer.go b/pkg/firebase/preparer/preparer.go index af8a1ec29..aba42cc23 100644 --- a/pkg/firebase/preparer/preparer.go +++ b/pkg/firebase/preparer/preparer.go @@ -32,6 +32,7 @@ type Options struct { SecretClient secrets.SecretManager AppHostingYAMLPath string ProjectID string + EnvironmentName string AppHostingYAMLOutputFilePath string EnvDereferencedOutputFilePath string BackendRootDirectory string @@ -57,13 +58,17 @@ func Prepare(ctx context.Context, opts Options) error { return fmt.Errorf("reading in and validating apphosting.yaml at path %v: %w", opts.AppHostingYAMLPath, err) } + if err = apphostingschema.MergeWithEnvironmentSpecificYAML(&appHostingYAML, opts.AppHostingYAMLPath, opts.EnvironmentName); err != nil { + return fmt.Errorf("merging with environment specific apphosting.%v.yaml: %w", opts.EnvironmentName, err) + } + apphostingschema.Sanitize(&appHostingYAML) - if err := secrets.Normalize(appHostingYAML.Env, opts.ProjectID); err != nil { + if err = secrets.Normalize(appHostingYAML.Env, opts.ProjectID); err != nil { return fmt.Errorf("normalizing apphosting.yaml fields: %w", err) } - if err := secrets.PinVersions(ctx, opts.SecretClient, appHostingYAML.Env); err != nil { + if err = secrets.PinVersions(ctx, opts.SecretClient, appHostingYAML.Env); err != nil { return fmt.Errorf("pinning secrets in apphosting.yaml: %w", err) } diff --git a/pkg/firebase/preparer/preparer_test.go b/pkg/firebase/preparer/preparer_test.go index 0014606d0..66bba4a17 100644 --- a/pkg/firebase/preparer/preparer_test.go +++ b/pkg/firebase/preparer/preparer_test.go @@ -32,6 +32,7 @@ func TestPrepare(t *testing.T) { desc string appHostingYAMLPath string projectID string + environmentName string wantEnvMap map[string]string wantSchema apphostingschema.AppHostingSchema }{ @@ -39,29 +40,31 @@ func TestPrepare(t *testing.T) { desc: "properly prepare apphosting.yaml", appHostingYAMLPath: appHostingYAMLPath, projectID: "test-project", + environmentName: "staging", wantEnvMap: map[string]string{ - "API_URL": "api.service.com", - "VAR_QUOTED_SPECIAL": "api2.service.com::", - "VAR_SPACED": "api3 - service - com", - "VAR_SINGLE_QUOTES": "I said, 'I'm learning YAML!'", - "VAR_DOUBLE_QUOTES": "\"api4.service.com\"", - "MULTILINE_VAR": "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n", - "VAR_NUMBER": "12345", - "API_KEY": secretString, - "PINNED_API_KEY": secretString, - "VERBOSE_API_KEY": secretString, - "PINNED_VERBOSE_API_KEY": secretString, + "API_URL": "api.staging.service.com", + "VAR_QUOTED_SPECIAL": "api2.service.com::", + "VAR_SPACED": "api3 - service - com", + "VAR_SINGLE_QUOTES": "I said, 'I'm learning YAML!'", + "VAR_DOUBLE_QUOTES": "\"api4.service.com\"", + "MULTILINE_VAR": "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n", + "VAR_NUMBER": "12345", + "API_KEY": secretString, + "PINNED_API_KEY": secretString, + "VERBOSE_API_KEY": secretString, + "PINNED_VERBOSE_API_KEY": secretString, + "STAGING_SECRET_VARIABLE": secretString, }, wantSchema: apphostingschema.AppHostingSchema{ RunConfig: apphostingschema.RunConfig{ - CPU: proto.Float32(3), - MemoryMiB: proto.Int32(1024), + CPU: proto.Float32(1), + MemoryMiB: proto.Int32(512), Concurrency: proto.Int32(100), - MaxInstances: proto.Int32(4), + MaxInstances: proto.Int32(2), MinInstances: proto.Int32(0), }, Env: []apphostingschema.EnvironmentVariable{ - apphostingschema.EnvironmentVariable{Variable: "API_URL", Value: "api.service.com", Availability: []string{"BUILD"}}, + apphostingschema.EnvironmentVariable{Variable: "API_URL", Value: "api.staging.service.com", Availability: []string{"BUILD", "RUNTIME"}}, apphostingschema.EnvironmentVariable{Variable: "VAR_QUOTED_SPECIAL", Value: "api2.service.com::", Availability: []string{"BUILD", "RUNTIME"}}, apphostingschema.EnvironmentVariable{Variable: "VAR_SPACED", Value: "api3 - service - com", Availability: []string{"BUILD", "RUNTIME"}}, apphostingschema.EnvironmentVariable{Variable: "VAR_SINGLE_QUOTES", Value: "I said, 'I'm learning YAML!'", Availability: []string{"BUILD", "RUNTIME"}}, @@ -73,6 +76,7 @@ func TestPrepare(t *testing.T) { apphostingschema.EnvironmentVariable{Variable: "PINNED_API_KEY", Secret: pinnedSecretName, Availability: []string{"BUILD", "RUNTIME"}}, apphostingschema.EnvironmentVariable{Variable: "VERBOSE_API_KEY", Secret: latestSecretName, Availability: []string{"BUILD", "RUNTIME"}}, apphostingschema.EnvironmentVariable{Variable: "PINNED_VERBOSE_API_KEY", Secret: pinnedSecretName, Availability: []string{"BUILD", "RUNTIME"}}, + apphostingschema.EnvironmentVariable{Variable: "STAGING_SECRET_VARIABLE", Secret: pinnedSecretName, Availability: []string{"BUILD", "RUNTIME"}}, }, }, }, @@ -120,6 +124,7 @@ func TestPrepare(t *testing.T) { SecretClient: fakeSecretClient, AppHostingYAMLPath: test.appHostingYAMLPath, ProjectID: test.projectID, + EnvironmentName: test.environmentName, AppHostingYAMLOutputFilePath: outputFilePathYAML, EnvDereferencedOutputFilePath: outputFilePathEnv, BackendRootDirectory: "", @@ -127,7 +132,7 @@ func TestPrepare(t *testing.T) { } if err := Prepare(context.Background(), opts); err != nil { - t.Errorf("Error in test '%v'. Error was %v", test.desc, err) + t.Fatalf("Error in test '%v'. Error was %v", test.desc, err) } // Check dereferenced secret material env file @@ -137,7 +142,7 @@ func TestPrepare(t *testing.T) { } if diff := cmp.Diff(test.wantEnvMap, actualEnvMapDereferenced); diff != "" { - t.Errorf("Unexpected env map for test %v (+got, -want):\n%v", test.desc, diff) + t.Errorf("Unexpected env map for test %v (-want, +got):\n%v", test.desc, diff) } // Check app hosting schema @@ -147,7 +152,7 @@ func TestPrepare(t *testing.T) { } if diff := cmp.Diff(test.wantSchema, actualAppHostingSchema); diff != "" { - t.Errorf("unexpected prepared YAML schema for test %q (+got, -want):\n%v", test.desc, diff) + t.Errorf("unexpected prepared YAML schema for test %q (-want, +got):\n%v", test.desc, diff) } } } diff --git a/pkg/firebase/preparer/testdata/apphosting.staging.yaml b/pkg/firebase/preparer/testdata/apphosting.staging.yaml new file mode 100644 index 000000000..01e3d71d5 --- /dev/null +++ b/pkg/firebase/preparer/testdata/apphosting.staging.yaml @@ -0,0 +1,18 @@ +schemaVersion: '3.0.0' + +runConfig: + cpu: 1 + memoryMiB: 512 + maxInstances: 2 + +env: + # Should override the value of API_URL in apphosting.yaml + - variable: API_URL + value: api.staging.service.com + availability: + - BUILD + - RUNTIME + + # Append this to apphosting.yaml since it isn't present there + - variable: STAGING_SECRET_VARIABLE + secret: secretID@11 \ No newline at end of file diff --git a/pkg/firebase/preparer/testdata/apphosting.yaml b/pkg/firebase/preparer/testdata/apphosting.yaml index 0d91630ae..2a6702c02 100644 --- a/pkg/firebase/preparer/testdata/apphosting.yaml +++ b/pkg/firebase/preparer/testdata/apphosting.yaml @@ -8,7 +8,7 @@ runConfig: concurrency: 100 env: - # Simple mapping from env var to value. By default will be applied with BUILD and RUNTIME availability + # Simple mapping from env var to value. By default this will be applied with BUILD and RUNTIME availability - variable: API_URL value: api.service.com availability: @@ -59,7 +59,7 @@ env: # Same as API_KEY above but with a pinned version. See go/firestack-secrets for more information on this format. - variable: PINNED_API_KEY - secret: secretID@11 # Secret will be pinned to version 5 + secret: secretID@11 # Secret will be pinned to version 11 # Same as API_KEY above but with the long form secret reference as defined by secret manager. - variable: VERBOSE_API_KEY diff --git a/pkg/firebase/secrets/secrets.go b/pkg/firebase/secrets/secrets.go index 9002658f2..00057a506 100644 --- a/pkg/firebase/secrets/secrets.go +++ b/pkg/firebase/secrets/secrets.go @@ -100,7 +100,14 @@ func PinVersions(ctx context.Context, client SecretManager, env []apphostingsche if ev.Secret != "" && strings.HasSuffix(ev.Secret, latestSuffix) { n, err := getSecretVersion(ctx, client, ev.Secret) if err != nil { - return fmt.Errorf("calling GetSecretVersion with name=%v: %w", ev.Secret, err) + return fmt.Errorf( + "calling GetSecretVersion with name=%v: %w. "+ + "If the secret already exists in your project, please grant your App Hosting backend "+ + "access to it with the CLI command 'firebase apphosting:secrets:grantaccess'. "+ + "See https://firebase.google.com/docs/app-hosting/configure#secret-parameters for more information", + ev.Secret, + err, + ) } env[i].Secret = n }