From 96e626e6dccd1c0113d7ff20af69e7ec7666fbf1 Mon Sep 17 00:00:00 2001 From: GCP Buildpacks Team Date: Thu, 1 Aug 2024 12:05:47 -0700 Subject: [PATCH] Add support for server-side environment variables in the build plane PiperOrigin-RevId: 658489839 Change-Id: I20a7049694d30cea7bc45c4238aba2a71a624759 --- cmd/firebase/preparer/main.go | 2 + pkg/firebase/envvars/BUILD.bazel | 6 +- pkg/firebase/envvars/envvars.go | 15 +++++ pkg/firebase/envvars/envvars_test.go | 78 ++++++++++++++++++++++++++ pkg/firebase/preparer/preparer.go | 34 +++++++---- pkg/firebase/preparer/preparer_test.go | 75 ++++++++++++++++++++++++- 6 files changed, 196 insertions(+), 14 deletions(-) diff --git a/cmd/firebase/preparer/main.go b/cmd/firebase/preparer/main.go index 6f7a5a6b1..7799f17bf 100644 --- a/cmd/firebase/preparer/main.go +++ b/cmd/firebase/preparer/main.go @@ -33,6 +33,7 @@ var ( dotEnvOutputFilePath = flag.String("dot_env_output_filepath", "", "File path to write the output .env file to") backendRootDirectory = flag.String("backend_root_directory", "", "File path to the application directory specified by the user") buildpackConfigOutputFilePath = flag.String("buildpack_config_output_filepath", "", "File path to write the buildpack config to") + serverSideEnvVars = flag.String("server_side_env_vars", "", "List of server side env vars to set. An empty string indicates server side environment variables are disabled. Any other value indicates enablement and to use these vars over yaml defined env vars.") ) func main() { @@ -73,6 +74,7 @@ func main() { EnvDereferencedOutputFilePath: *dotEnvOutputFilePath, BackendRootDirectory: *backendRootDirectory, BuildpackConfigOutputFilePath: *buildpackConfigOutputFilePath, + ServerSideEnvVars: *serverSideEnvVars, } err = preparer.Prepare(context.Background(), opts) diff --git a/pkg/firebase/envvars/BUILD.bazel b/pkg/firebase/envvars/BUILD.bazel index 5487c9b40..ecba6339f 100644 --- a/pkg/firebase/envvars/BUILD.bazel +++ b/pkg/firebase/envvars/BUILD.bazel @@ -8,6 +8,7 @@ go_library( name = "envvars", srcs = ["envvars.go"], importpath = "github.com/GoogleCloudPlatform/buildpacks/" + package_name(), + deps = ["//pkg/firebase/apphostingschema"], ) go_test( @@ -15,5 +16,8 @@ go_test( srcs = ["envvars_test.go"], embed = [":envvars"], rundir = ".", - deps = ["@com_github_google_go-cmp//cmp:go_default_library"], + deps = [ + "//pkg/firebase/apphostingschema", + "@com_github_google_go-cmp//cmp:go_default_library", + ], ) diff --git a/pkg/firebase/envvars/envvars.go b/pkg/firebase/envvars/envvars.go index 22e1d48aa..682cb044f 100644 --- a/pkg/firebase/envvars/envvars.go +++ b/pkg/firebase/envvars/envvars.go @@ -18,12 +18,15 @@ package envvars import ( "bufio" + "encoding/json" "fmt" "io/ioutil" "os" "sort" "strconv" "strings" + + "github.com/GoogleCloudPlatform/buildpacks/pkg/firebase/apphostingschema" ) // Write produces a file where each like has the format KEY=VALUE. We aren't using the @@ -84,3 +87,15 @@ func marshal(envMap map[string]string) (string, error) { sort.Strings(lines) return strings.Join(lines, "\n"), nil } + +// ParseEnvVarsFromString parses the server side environment variables from a string to a list of EnvironmentVariables. +func ParseEnvVarsFromString(serverSideEnvVars string) ([]apphostingschema.EnvironmentVariable, error) { + var parsedServerSideEnvVars []apphostingschema.EnvironmentVariable + + err := json.Unmarshal([]byte(serverSideEnvVars), &parsedServerSideEnvVars) + if err != nil { + return parsedServerSideEnvVars, fmt.Errorf("unmarshalling server side env var %v: %w", serverSideEnvVars, err) + } + + return parsedServerSideEnvVars, nil +} diff --git a/pkg/firebase/envvars/envvars_test.go b/pkg/firebase/envvars/envvars_test.go index 1807d0218..5640879e8 100644 --- a/pkg/firebase/envvars/envvars_test.go +++ b/pkg/firebase/envvars/envvars_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" + "github.com/GoogleCloudPlatform/buildpacks/pkg/firebase/apphostingschema" "github.com/google/go-cmp/cmp" ) @@ -117,3 +118,80 @@ VAR_SPACED=api3 - service - com } } } + +func TestParseEnvVarsFromString(t *testing.T) { + testCases := []struct { + desc string + serverSideEnvVars string + wantEnvVars []apphostingschema.EnvironmentVariable + wantErr bool + }{ + { + desc: "Parse server side env vars correctly", + serverSideEnvVars: ` + [ + { + "Variable": "SERVER_SIDE_ENV_VAR_NUMBER", + "Value": "3457934845", + "Availability": ["BUILD", "RUNTIME"] + }, + { + "Variable": "SERVER_SIDE_ENV_VAR_MULTILINE_FROM_SERVER_SIDE", + "Value": "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n", + "Availability": ["BUILD"] + }, + { + "Variable": "SERVER_SIDE_ENV_VAR_QUOTED_SPECIAL", + "Value": "api_from_server_side.service.com::", + "Availability": ["RUNTIME"] + }, + { + "Variable": "SERVER_SIDE_ENV_VAR_SPACED", + "Value": "api979 - service - com", + "Availability": ["BUILD"] + }, + { + "Variable": "SERVER_SIDE_ENV_VAR_SINGLE_QUOTES", + "Value": "I said, 'I'm learning GOLANG!'", + "Availability": ["BUILD"] + }, + { + "Variable": "SERVER_SIDE_ENV_VAR_DOUBLE_QUOTES", + "Value": "\"api41.service.com\"", + "Availability": ["BUILD", "RUNTIME"] + } + ] + `, + wantEnvVars: []apphostingschema.EnvironmentVariable{ + {Variable: "SERVER_SIDE_ENV_VAR_NUMBER", Value: "3457934845", Availability: []string{"BUILD", "RUNTIME"}}, + {Variable: "SERVER_SIDE_ENV_VAR_MULTILINE_FROM_SERVER_SIDE", Value: "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n", Availability: []string{"BUILD"}}, + {Variable: "SERVER_SIDE_ENV_VAR_QUOTED_SPECIAL", Value: "api_from_server_side.service.com::", Availability: []string{"RUNTIME"}}, + {Variable: "SERVER_SIDE_ENV_VAR_SPACED", Value: "api979 - service - com", Availability: []string{"BUILD"}}, + {Variable: "SERVER_SIDE_ENV_VAR_SINGLE_QUOTES", Value: "I said, 'I'm learning GOLANG!'", Availability: []string{"BUILD"}}, + {Variable: "SERVER_SIDE_ENV_VAR_DOUBLE_QUOTES", Value: "\"api41.service.com\"", Availability: []string{"BUILD", "RUNTIME"}}, + }, + }, + { + desc: "Empty list of server side env vars", + serverSideEnvVars: "[]", + wantEnvVars: []apphostingschema.EnvironmentVariable{}, + }, + { + desc: "Malformed server side env vars string", + serverSideEnvVars: "a malformed string", + wantEnvVars: nil, + wantErr: true, + }, + } + + for _, test := range testCases { + parsedServerSideEnvVars, err := ParseEnvVarsFromString(test.serverSideEnvVars) + gotErr := err != nil + if gotErr != test.wantErr { + t.Errorf("ParseEnvVarsFromString(%q) = %v, want error presence = %v", test.desc, err, test.wantErr) + } + if diff := cmp.Diff(test.wantEnvVars, parsedServerSideEnvVars); diff != "" { + t.Errorf("unexpected env vars for test %q, (+got, -want):\n%v", test.desc, diff) + } + } +} diff --git a/pkg/firebase/preparer/preparer.go b/pkg/firebase/preparer/preparer.go index aba42cc23..ef7fcf2a1 100644 --- a/pkg/firebase/preparer/preparer.go +++ b/pkg/firebase/preparer/preparer.go @@ -37,6 +37,7 @@ type Options struct { EnvDereferencedOutputFilePath string BackendRootDirectory string BuildpackConfigOutputFilePath string + ServerSideEnvVars string } // Prepare performs pre-build logic for App Hosting backends including: @@ -49,10 +50,9 @@ type Options struct { func Prepare(ctx context.Context, opts Options) error { dereferencedEnvMap := map[string]string{} // Env map with dereferenced secret material appHostingYAML := apphostingschema.AppHostingSchema{} + var err error if opts.AppHostingYAMLPath != "" { - var err error - appHostingYAML, err = apphostingschema.ReadAndValidateAppHostingSchemaFromFile(opts.AppHostingYAMLPath) if err != nil { return fmt.Errorf("reading in and validating apphosting.yaml at path %v: %w", opts.AppHostingYAMLPath, err) @@ -61,20 +61,30 @@ func Prepare(ctx context.Context, opts Options) error { if err = apphostingschema.MergeWithEnvironmentSpecificYAML(&appHostingYAML, opts.AppHostingYAMLPath, opts.EnvironmentName); err != nil { return fmt.Errorf("merging with environment specific apphosting.%v.yaml: %w", opts.EnvironmentName, err) } + } - apphostingschema.Sanitize(&appHostingYAML) - - if err = secrets.Normalize(appHostingYAML.Env, opts.ProjectID); err != nil { - return fmt.Errorf("normalizing apphosting.yaml fields: %w", err) + // Use server side env vars instead of apphosting.yaml. + if opts.ServerSideEnvVars != "" { + parsedServerSideEnvVars, err := envvars.ParseEnvVarsFromString(opts.ServerSideEnvVars) + if err != nil { + return fmt.Errorf("parsing server side env vars %v: %w", opts.ServerSideEnvVars, err) } - if err = secrets.PinVersions(ctx, opts.SecretClient, appHostingYAML.Env); err != nil { - return fmt.Errorf("pinning secrets in apphosting.yaml: %w", err) - } + appHostingYAML.Env = parsedServerSideEnvVars + } - if dereferencedEnvMap, err = secrets.GenerateBuildDereferencedEnvMap(ctx, opts.SecretClient, appHostingYAML.Env); err != nil { - return fmt.Errorf("dereferencing secrets in apphosting.yaml: %w", err) - } + apphostingschema.Sanitize(&appHostingYAML) + + if err := secrets.Normalize(appHostingYAML.Env, opts.ProjectID); err != nil { + return fmt.Errorf("normalizing apphosting.yaml fields: %w", err) + } + + if err := secrets.PinVersions(ctx, opts.SecretClient, appHostingYAML.Env); err != nil { + return fmt.Errorf("pinning secrets in apphosting.yaml: %w", err) + } + + if dereferencedEnvMap, err = secrets.GenerateBuildDereferencedEnvMap(ctx, opts.SecretClient, appHostingYAML.Env); err != nil { + return fmt.Errorf("dereferencing secrets in apphosting.yaml: %w", err) } if err := appHostingYAML.WriteToFile(opts.AppHostingYAMLOutputFilePath); err != nil { diff --git a/pkg/firebase/preparer/preparer_test.go b/pkg/firebase/preparer/preparer_test.go index 66bba4a17..91da89178 100644 --- a/pkg/firebase/preparer/preparer_test.go +++ b/pkg/firebase/preparer/preparer_test.go @@ -2,6 +2,7 @@ package preparer import ( "context" + "encoding/json" "hash/crc32" "testing" @@ -33,6 +34,7 @@ func TestPrepare(t *testing.T) { appHostingYAMLPath string projectID string environmentName string + serverSideEnvVars []apphostingschema.EnvironmentVariable wantEnvMap map[string]string wantSchema apphostingschema.AppHostingSchema }{ @@ -87,6 +89,67 @@ func TestPrepare(t *testing.T) { wantEnvMap: map[string]string{}, wantSchema: apphostingschema.AppHostingSchema{}, }, + { + desc: "server side env vars with apphosting.yaml", + appHostingYAMLPath: appHostingYAMLPath, + projectID: "test-project", + serverSideEnvVars: []apphostingschema.EnvironmentVariable{ + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_NUMBER", Value: "54321", Availability: []string{"BUILD", "RUNTIME"}}, + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_MULTILINE_FROM_SERVER_SIDE", Value: "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n", Availability: []string{"BUILD"}}, + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_QUOTED_SPECIAL", Value: "api_from_server_side.service.com::", Availability: []string{"RUNTIME"}}, + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_SPACED", Value: "api3 - service - com", Availability: []string{"BUILD"}}, + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_SINGLE_QUOTES", Value: "GOLANG is awesome!'", Availability: []string{"BUILD"}}, + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_DOUBLE_QUOTES", Value: "\"api4.service.com\"", Availability: []string{"BUILD", "RUNTIME"}}, + }, + wantEnvMap: map[string]string{ + "SERVER_SIDE_ENV_VAR_NUMBER": "54321", + "SERVER_SIDE_ENV_VAR_MULTILINE_FROM_SERVER_SIDE": "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n", + "SERVER_SIDE_ENV_VAR_SPACED": "api3 - service - com", + "SERVER_SIDE_ENV_VAR_SINGLE_QUOTES": "GOLANG is awesome!'", + "SERVER_SIDE_ENV_VAR_DOUBLE_QUOTES": "\"api4.service.com\"", + }, + wantSchema: apphostingschema.AppHostingSchema{ + RunConfig: apphostingschema.RunConfig{ + CPU: proto.Float32(3), + MemoryMiB: proto.Int32(1024), + Concurrency: proto.Int32(100), + MaxInstances: proto.Int32(4), + MinInstances: proto.Int32(0), + }, + Env: []apphostingschema.EnvironmentVariable{ + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_NUMBER", Value: "54321", Availability: []string{"BUILD", "RUNTIME"}}, + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_MULTILINE_FROM_SERVER_SIDE", Value: "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n", Availability: []string{"BUILD"}}, + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_QUOTED_SPECIAL", Value: "api_from_server_side.service.com::", Availability: []string{"RUNTIME"}}, + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_SPACED", Value: "api3 - service - com", Availability: []string{"BUILD"}}, + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_SINGLE_QUOTES", Value: "GOLANG is awesome!'", Availability: []string{"BUILD"}}, + apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_DOUBLE_QUOTES", Value: "\"api4.service.com\"", Availability: []string{"BUILD", "RUNTIME"}}, + }, + }, + }, + { + desc: "server side env vars enabled but empty without apphosting.yaml", + appHostingYAMLPath: "", + projectID: "test-project", + serverSideEnvVars: []apphostingschema.EnvironmentVariable{}, + wantEnvMap: map[string]string{}, + wantSchema: apphostingschema.AppHostingSchema{}, + }, + { + desc: "server side env vars enabled but empty with apphosting.yaml", + appHostingYAMLPath: appHostingYAMLPath, + projectID: "test-project", + serverSideEnvVars: []apphostingschema.EnvironmentVariable{}, + wantEnvMap: map[string]string{}, + wantSchema: apphostingschema.AppHostingSchema{ + RunConfig: apphostingschema.RunConfig{ + CPU: proto.Float32(3), + MemoryMiB: proto.Int32(1024), + Concurrency: proto.Int32(100), + MaxInstances: proto.Int32(4), + MinInstances: proto.Int32(0), + }, + }, + }, } fakeSecretClient := &fakesecretmanager.FakeSecretClient{ @@ -120,6 +183,16 @@ func TestPrepare(t *testing.T) { // Testing happy paths for _, test := range testCases { + // Convert server side env vars to string + serverSideEnvVars := "" + if test.serverSideEnvVars != nil { + parsedServerSideEnvVars, err := json.Marshal(test.serverSideEnvVars) + if err != nil { + t.Errorf("Error in json marshalling serverSideEnvVars '%v'. Error was %v", test.serverSideEnvVars, err) + return + } + serverSideEnvVars = string(parsedServerSideEnvVars) + } opts := Options{ SecretClient: fakeSecretClient, AppHostingYAMLPath: test.appHostingYAMLPath, @@ -129,12 +202,12 @@ func TestPrepare(t *testing.T) { EnvDereferencedOutputFilePath: outputFilePathEnv, BackendRootDirectory: "", BuildpackConfigOutputFilePath: outputFilePathBuildpackConfig, + ServerSideEnvVars: serverSideEnvVars, } if err := Prepare(context.Background(), opts); err != nil { t.Fatalf("Error in test '%v'. Error was %v", test.desc, err) } - // Check dereferenced secret material env file actualEnvMapDereferenced, err := envvars.Read(outputFilePathEnv) if err != nil {