Add support for server-side environment variables in the build plane

PiperOrigin-RevId: 658489839
Change-Id: I20a7049694d30cea7bc45c4238aba2a71a624759

cmd/firebase/preparer/main.go

@@ -33,6 +33,7 @@ var (
 	dotEnvOutputFilePath          = flag.String("dot_env_output_filepath", "", "File path to write the output .env file to")
 	backendRootDirectory          = flag.String("backend_root_directory", "", "File path to the application directory specified by the user")
 	buildpackConfigOutputFilePath = flag.String("buildpack_config_output_filepath", "", "File path to write the buildpack config to")
+	serverSideEnvVars             = flag.String("server_side_env_vars", "", "List of server side env vars to set. An empty string indicates server side environment variables are disabled. Any other value indicates enablement and to use these vars over yaml defined env vars.")
 )
 
 func main() {
@@ -73,6 +74,7 @@ func main() {
 		EnvDereferencedOutputFilePath: *dotEnvOutputFilePath,
 		BackendRootDirectory:          *backendRootDirectory,
 		BuildpackConfigOutputFilePath: *buildpackConfigOutputFilePath,
+		ServerSideEnvVars:             *serverSideEnvVars,
 	}
 
 	err = preparer.Prepare(context.Background(), opts)

pkg/firebase/envvars/BUILD.bazel

@@ -8,12 +8,16 @@ go_library(
     name = "envvars",
     srcs = ["envvars.go"],
     importpath = "github.com/GoogleCloudPlatform/buildpacks/" + package_name(),
+    deps = ["//pkg/firebase/apphostingschema"],
 )
 
 go_test(
     name = "envvars_test",
     srcs = ["envvars_test.go"],
     embed = [":envvars"],
     rundir = ".",
-    deps = ["@com_github_google_go-cmp//cmp:go_default_library"],
+    deps = [
+        "//pkg/firebase/apphostingschema",
+        "@com_github_google_go-cmp//cmp:go_default_library",
+    ],
 )

pkg/firebase/envvars/envvars.go

@@ -18,12 +18,15 @@ package envvars
 
 import (
 	"bufio"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"sort"
 	"strconv"
 	"strings"
+
+	"github.com/GoogleCloudPlatform/buildpacks/pkg/firebase/apphostingschema"
 )
 
 // Write produces a file where each like has the format KEY=VALUE. We aren't using the
@@ -84,3 +87,15 @@ func marshal(envMap map[string]string) (string, error) {
 	sort.Strings(lines)
 	return strings.Join(lines, "\n"), nil
 }
+
+// ParseEnvVarsFromString parses the server side environment variables from a string to a list of EnvironmentVariables.
+func ParseEnvVarsFromString(serverSideEnvVars string) ([]apphostingschema.EnvironmentVariable, error) {
+	var parsedServerSideEnvVars []apphostingschema.EnvironmentVariable
+
+	err := json.Unmarshal([]byte(serverSideEnvVars), &parsedServerSideEnvVars)
+	if err != nil {
+		return parsedServerSideEnvVars, fmt.Errorf("unmarshalling server side env var %v: %w", serverSideEnvVars, err)
+	}
+
+	return parsedServerSideEnvVars, nil
+}

pkg/firebase/envvars/envvars_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/GoogleCloudPlatform/buildpacks/pkg/firebase/apphostingschema"
 	"github.com/google/go-cmp/cmp"
 )
 
@@ -117,3 +118,80 @@ VAR_SPACED=api3 - service -  com
 		}
 	}
 }
+
+func TestParseEnvVarsFromString(t *testing.T) {
+	testCases := []struct {
+		desc              string
+		serverSideEnvVars string
+		wantEnvVars       []apphostingschema.EnvironmentVariable
+		wantErr           bool
+	}{
+		{
+			desc: "Parse server side env vars correctly",
+			serverSideEnvVars: `
+			[
+				{
+					"Variable": "SERVER_SIDE_ENV_VAR_NUMBER",
+					"Value": "3457934845",
+					"Availability": ["BUILD", "RUNTIME"]
+				},
+				{
+					"Variable": "SERVER_SIDE_ENV_VAR_MULTILINE_FROM_SERVER_SIDE",
+					"Value": "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n",
+					"Availability": ["BUILD"]
+				},
+				{
+					"Variable": "SERVER_SIDE_ENV_VAR_QUOTED_SPECIAL",
+					"Value": "api_from_server_side.service.com::",
+					"Availability": ["RUNTIME"]
+				},
+				{
+					"Variable": "SERVER_SIDE_ENV_VAR_SPACED",
+					"Value": "api979 - service -  com",
+					"Availability": ["BUILD"]
+				},
+				{
+					"Variable": "SERVER_SIDE_ENV_VAR_SINGLE_QUOTES",
+					"Value": "I said, 'I'm learning GOLANG!'",
+					"Availability": ["BUILD"]
+				},
+				{
+					"Variable": "SERVER_SIDE_ENV_VAR_DOUBLE_QUOTES",
+					"Value": "\"api41.service.com\"",
+					"Availability": ["BUILD", "RUNTIME"]
+				}
+			]
+		`,
+			wantEnvVars: []apphostingschema.EnvironmentVariable{
+				{Variable: "SERVER_SIDE_ENV_VAR_NUMBER", Value: "3457934845", Availability: []string{"BUILD", "RUNTIME"}},
+				{Variable: "SERVER_SIDE_ENV_VAR_MULTILINE_FROM_SERVER_SIDE", Value: "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n", Availability: []string{"BUILD"}},
+				{Variable: "SERVER_SIDE_ENV_VAR_QUOTED_SPECIAL", Value: "api_from_server_side.service.com::", Availability: []string{"RUNTIME"}},
+				{Variable: "SERVER_SIDE_ENV_VAR_SPACED", Value: "api979 - service -  com", Availability: []string{"BUILD"}},
+				{Variable: "SERVER_SIDE_ENV_VAR_SINGLE_QUOTES", Value: "I said, 'I'm learning GOLANG!'", Availability: []string{"BUILD"}},
+				{Variable: "SERVER_SIDE_ENV_VAR_DOUBLE_QUOTES", Value: "\"api41.service.com\"", Availability: []string{"BUILD", "RUNTIME"}},
+			},
+		},
+		{
+			desc:              "Empty list of server side env vars",
+			serverSideEnvVars: "[]",
+			wantEnvVars:       []apphostingschema.EnvironmentVariable{},
+		},
+		{
+			desc:              "Malformed server side env vars string",
+			serverSideEnvVars: "a malformed string",
+			wantEnvVars:       nil,
+			wantErr:           true,
+		},
+	}
+
+	for _, test := range testCases {
+		parsedServerSideEnvVars, err := ParseEnvVarsFromString(test.serverSideEnvVars)
+		gotErr := err != nil
+		if gotErr != test.wantErr {
+			t.Errorf("ParseEnvVarsFromString(%q) = %v, want error presence = %v", test.desc, err, test.wantErr)
+		}
+		if diff := cmp.Diff(test.wantEnvVars, parsedServerSideEnvVars); diff != "" {
+			t.Errorf("unexpected env vars for test %q, (+got, -want):\n%v", test.desc, diff)
+		}
+	}
+}

pkg/firebase/preparer/preparer.go

@@ -37,6 +37,7 @@ type Options struct {
 	EnvDereferencedOutputFilePath string
 	BackendRootDirectory          string
 	BuildpackConfigOutputFilePath string
+	ServerSideEnvVars             string
 }
 
 // Prepare performs pre-build logic for App Hosting backends including:
@@ -49,10 +50,9 @@ type Options struct {
 func Prepare(ctx context.Context, opts Options) error {
 	dereferencedEnvMap := map[string]string{} // Env map with dereferenced secret material
 	appHostingYAML := apphostingschema.AppHostingSchema{}
+	var err error
 
 	if opts.AppHostingYAMLPath != "" {
-		var err error
-
 		appHostingYAML, err = apphostingschema.ReadAndValidateAppHostingSchemaFromFile(opts.AppHostingYAMLPath)
 		if err != nil {
 			return fmt.Errorf("reading in and validating apphosting.yaml at path %v: %w", opts.AppHostingYAMLPath, err)
@@ -61,20 +61,30 @@ func Prepare(ctx context.Context, opts Options) error {
 		if err = apphostingschema.MergeWithEnvironmentSpecificYAML(&appHostingYAML, opts.AppHostingYAMLPath, opts.EnvironmentName); err != nil {
 			return fmt.Errorf("merging with environment specific apphosting.%v.yaml: %w", opts.EnvironmentName, err)
 		}
+	}
 
-		apphostingschema.Sanitize(&appHostingYAML)
-
-		if err = secrets.Normalize(appHostingYAML.Env, opts.ProjectID); err != nil {
-			return fmt.Errorf("normalizing apphosting.yaml fields: %w", err)
+	// Use server side env vars instead of apphosting.yaml.
+	if opts.ServerSideEnvVars != "" {
+		parsedServerSideEnvVars, err := envvars.ParseEnvVarsFromString(opts.ServerSideEnvVars)
+		if err != nil {
+			return fmt.Errorf("parsing server side env vars %v: %w", opts.ServerSideEnvVars, err)
 		}
 
-		if err = secrets.PinVersions(ctx, opts.SecretClient, appHostingYAML.Env); err != nil {
-			return fmt.Errorf("pinning secrets in apphosting.yaml: %w", err)
-		}
+		appHostingYAML.Env = parsedServerSideEnvVars
+	}
 
-		if dereferencedEnvMap, err = secrets.GenerateBuildDereferencedEnvMap(ctx, opts.SecretClient, appHostingYAML.Env); err != nil {
-			return fmt.Errorf("dereferencing secrets in apphosting.yaml: %w", err)
-		}
+	apphostingschema.Sanitize(&appHostingYAML)
+
+	if err := secrets.Normalize(appHostingYAML.Env, opts.ProjectID); err != nil {
+		return fmt.Errorf("normalizing apphosting.yaml fields: %w", err)
+	}
+
+	if err := secrets.PinVersions(ctx, opts.SecretClient, appHostingYAML.Env); err != nil {
+		return fmt.Errorf("pinning secrets in apphosting.yaml: %w", err)
+	}
+
+	if dereferencedEnvMap, err = secrets.GenerateBuildDereferencedEnvMap(ctx, opts.SecretClient, appHostingYAML.Env); err != nil {
+		return fmt.Errorf("dereferencing secrets in apphosting.yaml: %w", err)
 	}
 
 	if err := appHostingYAML.WriteToFile(opts.AppHostingYAMLOutputFilePath); err != nil {

pkg/firebase/preparer/preparer_test.go

@@ -2,6 +2,7 @@ package preparer
 
 import (
 	"context"
+	"encoding/json"
 	"hash/crc32"
 	"testing"
 
@@ -33,6 +34,7 @@ func TestPrepare(t *testing.T) {
 		appHostingYAMLPath string
 		projectID          string
 		environmentName    string
+		serverSideEnvVars  []apphostingschema.EnvironmentVariable
 		wantEnvMap         map[string]string
 		wantSchema         apphostingschema.AppHostingSchema
 	}{
@@ -87,6 +89,67 @@ func TestPrepare(t *testing.T) {
 			wantEnvMap:         map[string]string{},
 			wantSchema:         apphostingschema.AppHostingSchema{},
 		},
+		{
+			desc:               "server side env vars with apphosting.yaml",
+			appHostingYAMLPath: appHostingYAMLPath,
+			projectID:          "test-project",
+			serverSideEnvVars: []apphostingschema.EnvironmentVariable{
+				apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_NUMBER", Value: "54321", Availability: []string{"BUILD", "RUNTIME"}},
+				apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_MULTILINE_FROM_SERVER_SIDE", Value: "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n", Availability: []string{"BUILD"}},
+				apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_QUOTED_SPECIAL", Value: "api_from_server_side.service.com::", Availability: []string{"RUNTIME"}},
+				apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_SPACED", Value: "api3 - service -  com", Availability: []string{"BUILD"}},
+				apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_SINGLE_QUOTES", Value: "GOLANG is awesome!'", Availability: []string{"BUILD"}},
+				apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_DOUBLE_QUOTES", Value: "\"api4.service.com\"", Availability: []string{"BUILD", "RUNTIME"}},
+			},
+			wantEnvMap: map[string]string{
+				"SERVER_SIDE_ENV_VAR_NUMBER":                     "54321",
+				"SERVER_SIDE_ENV_VAR_MULTILINE_FROM_SERVER_SIDE": "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n",
+				"SERVER_SIDE_ENV_VAR_SPACED":                     "api3 - service -  com",
+				"SERVER_SIDE_ENV_VAR_SINGLE_QUOTES":              "GOLANG is awesome!'",
+				"SERVER_SIDE_ENV_VAR_DOUBLE_QUOTES":              "\"api4.service.com\"",
+			},
+			wantSchema: apphostingschema.AppHostingSchema{
+				RunConfig: apphostingschema.RunConfig{
+					CPU:          proto.Float32(3),
+					MemoryMiB:    proto.Int32(1024),
+					Concurrency:  proto.Int32(100),
+					MaxInstances: proto.Int32(4),
+					MinInstances: proto.Int32(0),
+				},
+				Env: []apphostingschema.EnvironmentVariable{
+					apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_NUMBER", Value: "54321", Availability: []string{"BUILD", "RUNTIME"}},
+					apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_MULTILINE_FROM_SERVER_SIDE", Value: "211 Broadway\\nApt. 17\\nNew York, NY 10019\\n", Availability: []string{"BUILD"}},
+					apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_QUOTED_SPECIAL", Value: "api_from_server_side.service.com::", Availability: []string{"RUNTIME"}},
+					apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_SPACED", Value: "api3 - service -  com", Availability: []string{"BUILD"}},
+					apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_SINGLE_QUOTES", Value: "GOLANG is awesome!'", Availability: []string{"BUILD"}},
+					apphostingschema.EnvironmentVariable{Variable: "SERVER_SIDE_ENV_VAR_DOUBLE_QUOTES", Value: "\"api4.service.com\"", Availability: []string{"BUILD", "RUNTIME"}},
+				},
+			},
+		},
+		{
+			desc:               "server side env vars enabled but empty without apphosting.yaml",
+			appHostingYAMLPath: "",
+			projectID:          "test-project",
+			serverSideEnvVars:  []apphostingschema.EnvironmentVariable{},
+			wantEnvMap:         map[string]string{},
+			wantSchema:         apphostingschema.AppHostingSchema{},
+		},
+		{
+			desc:               "server side env vars enabled but empty with apphosting.yaml",
+			appHostingYAMLPath: appHostingYAMLPath,
+			projectID:          "test-project",
+			serverSideEnvVars:  []apphostingschema.EnvironmentVariable{},
+			wantEnvMap:         map[string]string{},
+			wantSchema: apphostingschema.AppHostingSchema{
+				RunConfig: apphostingschema.RunConfig{
+					CPU:          proto.Float32(3),
+					MemoryMiB:    proto.Int32(1024),
+					Concurrency:  proto.Int32(100),
+					MaxInstances: proto.Int32(4),
+					MinInstances: proto.Int32(0),
+				},
+			},
+		},
 	}
 
 	fakeSecretClient := &fakesecretmanager.FakeSecretClient{
@@ -120,6 +183,16 @@ func TestPrepare(t *testing.T) {
 
 	// Testing happy paths
 	for _, test := range testCases {
+		// Convert server side env vars to string
+		serverSideEnvVars := ""
+		if test.serverSideEnvVars != nil {
+			parsedServerSideEnvVars, err := json.Marshal(test.serverSideEnvVars)
+			if err != nil {
+				t.Errorf("Error in json marshalling serverSideEnvVars '%v'. Error was %v", test.serverSideEnvVars, err)
+				return
+			}
+			serverSideEnvVars = string(parsedServerSideEnvVars)
+		}
 		opts := Options{
 			SecretClient:                  fakeSecretClient,
 			AppHostingYAMLPath:            test.appHostingYAMLPath,
@@ -129,12 +202,12 @@ func TestPrepare(t *testing.T) {
 			EnvDereferencedOutputFilePath: outputFilePathEnv,
 			BackendRootDirectory:          "",
 			BuildpackConfigOutputFilePath: outputFilePathBuildpackConfig,
+			ServerSideEnvVars:             serverSideEnvVars,
 		}
 
 		if err := Prepare(context.Background(), opts); err != nil {
 			t.Fatalf("Error in test '%v'. Error was %v", test.desc, err)
 		}
-
 		// Check dereferenced secret material env file
 		actualEnvMapDereferenced, err := envvars.Read(outputFilePathEnv)
 		if err != nil {