feat: add file resource provider for EG standalone mode (envoyproxy#3159

) * add validations for envoy-gateway file resource type Signed-off-by: shawnh2 <[email protected]> * improve eg validation and add resource provider interface for various provider Signed-off-by: shawnh2 <[email protected]> * extract common gatewayapi layer translate logic in egctl translate Signed-off-by: shawnh2 <[email protected]> * add notifier support Signed-off-by: shawnh2 <[email protected]> * fix lint and move read yaml bytes function back to translate Signed-off-by: shawnh2 <[email protected]> * add resources store support Signed-off-by: shawnh2 <[email protected]> * fix lint Signed-off-by: shawnh2 <[email protected]> * fix ci Signed-off-by: shawnh2 <[email protected]> * update infra provider api and address comments Signed-off-by: shawnh2 <[email protected]> * update custom provider comments and validate method test Signed-off-by: shawnh2 <[email protected]> * restore extension manager and add health probe server for file provider Signed-off-by: shawnh2 <[email protected]> * update envoy gateway helper functions Signed-off-by: shawnh2 <[email protected]> * add some unit tests Signed-off-by: shawnh2 <[email protected]> * properly handle the remove event for the file provider Signed-off-by: shawnh2 <[email protected]> * fix lint Signed-off-by: shawnh2 <[email protected]> * no default to k8s for infra provider Signed-off-by: shawnh2 <[email protected]> * fix runner Signed-off-by: shawnh2 <[email protected]> --------- Signed-off-by: shawnh2 <[email protected]> Co-authored-by: Xunzhuo <[email protected]>
deszhou · Sep 11, 2024 · f5693f6 · f5693f6
1 parent c109131
commit f5693f6
Show file tree

Hide file tree

Showing 26 changed files with 1,330 additions and 125 deletions.
diff --git a/api/v1alpha1/envoygateway_helpers.go b/api/v1alpha1/envoygateway_helpers.go
@@ -237,6 +237,7 @@ func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubern
  if r.Kubernetes.ShutdownManager == nil {
  r.Kubernetes.ShutdownManager = &ShutdownManager{Image: ptr.To(DefaultShutdownManagerImage)}
  }
+
  return r.Kubernetes
 }
 

diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go
@@ -174,7 +174,7 @@ type ExtensionAPISettings struct {
 // EnvoyGatewayProvider defines the desired configuration of a provider.
 // +union
 type EnvoyGatewayProvider struct {
- // Type is the type of provider to use. Supported types are "Kubernetes".
+ // Type is the type of provider to use. Supported types are "Kubernetes", "Custom".
  //
  // +unionDiscriminator
  Type ProviderType `json:"type"`
@@ -186,7 +186,7 @@ type EnvoyGatewayProvider struct {
  Kubernetes *EnvoyGatewayKubernetesProvider `json:"kubernetes,omitempty"`
 
  // Custom defines the configuration for the Custom provider. This provider
- // allows you to define a specific resource provider and a infrastructure
+ // allows you to define a specific resource provider and an infrastructure
  // provider.
  //
  // +optional
@@ -271,7 +271,11 @@ type EnvoyGatewayCustomProvider struct {
  // This provider is used to specify the provider to be used
  // to provide an environment to deploy the out resources like
  // the Envoy Proxy data plane.
- Infrastructure EnvoyGatewayInfrastructureProvider `json:"infrastructure"`
+ //
+ // Infrastructure is optional, if provider is not specified,
+ // No infrastructure provider is available.
+ // +optional
+ Infrastructure *EnvoyGatewayInfrastructureProvider `json:"infrastructure,omitempty"`
 }
 
 // ResourceProviderType defines the types of custom resource providers supported by Envoy Gateway.
@@ -300,7 +304,7 @@ type EnvoyGatewayResourceProvider struct {
 // EnvoyGatewayFileResourceProvider defines configuration for the File Resource provider.
 type EnvoyGatewayFileResourceProvider struct {
  // Paths are the paths to a directory or file containing the resource configuration.
- // Recursive sub directories are not currently supported.
+ // Recursive subdirectories are not currently supported.
  Paths []string `json:"paths"`
 }
 

diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go
@@ -47,16 +47,15 @@ type GroupVersionKind struct {
 
 // ProviderType defines the types of providers supported by Envoy Gateway.
 //
-// +kubebuilder:validation:Enum=Kubernetes
+// +kubebuilder:validation:Enum=Kubernetes;Custom
 type ProviderType string
 
 const (
  // ProviderTypeKubernetes defines the "Kubernetes" provider.
  ProviderTypeKubernetes ProviderType = "Kubernetes"
 
- // ProviderTypeFile defines the "File" provider. This type is not implemented
- // until https://github.com/envoyproxy/gateway/issues/1001 is fixed.
- ProviderTypeFile ProviderType = "File"
+ // ProviderTypeCustom defines the "Custom" provider.
+ ProviderTypeCustom ProviderType = "Custom"
 )
 
 // KubernetesDeploymentSpec defines the desired state of the Kubernetes deployment resource.

diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go
@@ -35,6 +35,10 @@ func ValidateEnvoyGateway(eg *egv1a1.EnvoyGateway) error {
  if err := validateEnvoyGatewayKubernetesProvider(eg.Provider.Kubernetes); err != nil {
  return err
  }
+ case egv1a1.ProviderTypeCustom:
+ if err := validateEnvoyGatewayCustomProvider(eg.Provider.Custom); err != nil {
+ return err
+ }
  default:
  return fmt.Errorf("unsupported provider type")
  }
@@ -79,6 +83,54 @@ func validateEnvoyGatewayKubernetesProvider(provider *egv1a1.EnvoyGatewayKuberne
  return nil
 }
 
+func validateEnvoyGatewayCustomProvider(provider *egv1a1.EnvoyGatewayCustomProvider) error {
+ if provider == nil {
+ return fmt.Errorf("empty custom provider settings")
+ }
+
+ if err := validateEnvoyGatewayCustomResourceProvider(provider.Resource); err != nil {
+ return err
+ }
+
+ if err := validateEnvoyGatewayCustomInfrastructureProvider(provider.Infrastructure); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func validateEnvoyGatewayCustomResourceProvider(resource egv1a1.EnvoyGatewayResourceProvider) error {
+ switch resource.Type {
+ case egv1a1.ResourceProviderTypeFile:
+ if resource.File == nil {
+ return fmt.Errorf("field 'file' should be specified when resource type is 'File'")
+ }
+
+ if len(resource.File.Paths) == 0 {
+ return fmt.Errorf("no paths were assigned for file resource provider to watch")
+ }
+ default:
+ return fmt.Errorf("unsupported resource provider: %s", resource.Type)
+ }
+ return nil
+}
+
+func validateEnvoyGatewayCustomInfrastructureProvider(infra *egv1a1.EnvoyGatewayInfrastructureProvider) error {
+ if infra == nil {
+ return nil
+ }
+
+ switch infra.Type {
+ case egv1a1.InfrastructureProviderTypeHost:
+ if infra.Host == nil {
+ return fmt.Errorf("field 'host' should be specified when infrastructure type is 'Host'")
+ }
+ default:
+ return fmt.Errorf("unsupported infrastructure provdier: %s", infra.Type)
+ }
+ return nil
+}
+
 func validateEnvoyGatewayLogging(logging *egv1a1.EnvoyGatewayLogging) error {
  if logging == nil || len(logging.Level) == 0 {
  return nil

diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go
@@ -68,11 +68,173 @@ func TestValidateEnvoyGateway(t *testing.T) {
  expect: false,
  },
  {
- name: "unsupported provider",
+ name: "nil custom provider",
  eg: &egv1a1.EnvoyGateway{
  EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
- Gateway: egv1a1.DefaultGateway(),
- Provider: &egv1a1.EnvoyGatewayProvider{Type: egv1a1.ProviderTypeFile},
+ Gateway: egv1a1.DefaultGateway(),
+ Provider: &egv1a1.EnvoyGatewayProvider{
+ Type: egv1a1.ProviderTypeCustom,
+ Custom: nil,
+ },
+ },
+ },
+ expect: false,
+ },
+ {
+ name: "empty custom provider",
+ eg: &egv1a1.EnvoyGateway{
+ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+ Gateway: egv1a1.DefaultGateway(),
+ Provider: &egv1a1.EnvoyGatewayProvider{
+ Type: egv1a1.ProviderTypeCustom,
+ Custom: &egv1a1.EnvoyGatewayCustomProvider{},
+ },
+ },
+ },
+ expect: false,
+ },
+ {
+ name: "custom provider with file resource provider and host infra provider",
+ eg: &egv1a1.EnvoyGateway{
+ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+ Gateway: egv1a1.DefaultGateway(),
+ Provider: &egv1a1.EnvoyGatewayProvider{
+ Type: egv1a1.ProviderTypeCustom,
+ Custom: &egv1a1.EnvoyGatewayCustomProvider{
+ Resource: egv1a1.EnvoyGatewayResourceProvider{
+ Type: egv1a1.ResourceProviderTypeFile,
+ File: &egv1a1.EnvoyGatewayFileResourceProvider{
+ Paths: []string{"foo", "bar"},
+ },
+ },
+ Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{
+ Type: egv1a1.InfrastructureProviderTypeHost,
+ Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{},
+ },
+ },
+ },
+ },
+ },
+ expect: true,
+ },
+ {
+ name: "custom provider with file provider and k8s infra provider",
+ eg: &egv1a1.EnvoyGateway{
+ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+ Gateway: egv1a1.DefaultGateway(),
+ Provider: &egv1a1.EnvoyGatewayProvider{
+ Type: egv1a1.ProviderTypeCustom,
+ Custom: &egv1a1.EnvoyGatewayCustomProvider{
+ Resource: egv1a1.EnvoyGatewayResourceProvider{
+ Type: egv1a1.ResourceProviderTypeFile,
+ File: &egv1a1.EnvoyGatewayFileResourceProvider{
+ Paths: []string{"foo", "bar"},
+ },
+ },
+ },
+ },
+ },
+ },
+ expect: true,
+ },
+ {
+ name: "custom provider with unsupported resource provider",
+ eg: &egv1a1.EnvoyGateway{
+ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+ Gateway: egv1a1.DefaultGateway(),
+ Provider: &egv1a1.EnvoyGatewayProvider{
+ Type: egv1a1.ProviderTypeCustom,
+ Custom: &egv1a1.EnvoyGatewayCustomProvider{
+ Resource: egv1a1.EnvoyGatewayResourceProvider{
+ Type: "foobar",
+ },
+ },
+ },
+ },
+ },
+ expect: false,
+ },
+ {
+ name: "custom provider with file provider but no file struct",
+ eg: &egv1a1.EnvoyGateway{
+ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+ Gateway: egv1a1.DefaultGateway(),
+ Provider: &egv1a1.EnvoyGatewayProvider{
+ Type: egv1a1.ProviderTypeCustom,
+ Custom: &egv1a1.EnvoyGatewayCustomProvider{
+ Resource: egv1a1.EnvoyGatewayResourceProvider{
+ Type: egv1a1.ResourceProviderTypeFile,
+ },
+ },
+ },
+ },
+ },
+ expect: false,
+ },
+ {
+ name: "custom provider with file provider and host infra provider but no host struct",
+ eg: &egv1a1.EnvoyGateway{
+ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+ Gateway: egv1a1.DefaultGateway(),
+ Provider: &egv1a1.EnvoyGatewayProvider{
+ Type: egv1a1.ProviderTypeCustom,
+ Custom: &egv1a1.EnvoyGatewayCustomProvider{
+ Resource: egv1a1.EnvoyGatewayResourceProvider{
+ Type: egv1a1.ResourceProviderTypeFile,
+ File: &egv1a1.EnvoyGatewayFileResourceProvider{
+ Paths: []string{"a", "b"},
+ },
+ },
+ Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{
+ Type: egv1a1.InfrastructureProviderTypeHost,
+ },
+ },
+ },
+ },
+ },
+ expect: false,
+ },
+ {
+ name: "custom provider with file provider and unsupported infra provider",
+ eg: &egv1a1.EnvoyGateway{
+ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+ Gateway: egv1a1.DefaultGateway(),
+ Provider: &egv1a1.EnvoyGatewayProvider{
+ Type: egv1a1.ProviderTypeCustom,
+ Custom: &egv1a1.EnvoyGatewayCustomProvider{
+ Resource: egv1a1.EnvoyGatewayResourceProvider{
+ Type: egv1a1.ResourceProviderTypeFile,
+ File: &egv1a1.EnvoyGatewayFileResourceProvider{
+ Paths: []string{"a", "b"},
+ },
+ },
+ Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{
+ Type: "foobar",
+ },
+ },
+ },
+ },
+ },
+ expect: false,
+ },
+ {
+ name: "custom provider with file provider and host infra provider but no paths assign in resource",
+ eg: &egv1a1.EnvoyGateway{
+ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+ Gateway: egv1a1.DefaultGateway(),
+ Provider: &egv1a1.EnvoyGatewayProvider{
+ Type: egv1a1.ProviderTypeCustom,
+ Custom: &egv1a1.EnvoyGatewayCustomProvider{
+ Resource: egv1a1.EnvoyGatewayResourceProvider{
+ Type: egv1a1.ResourceProviderTypeFile,
+ File: &egv1a1.EnvoyGatewayFileResourceProvider{},
+ },
+ Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{
+ Type: egv1a1.InfrastructureProviderTypeHost,
+ Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{},
+ },
+ },
+ },
  },
  },
  expect: false,

diff --git a/api/v1alpha1/validation/envoyproxy_validate_test.go b/api/v1alpha1/validation/envoyproxy_validate_test.go
@@ -52,7 +52,7 @@ func TestValidateEnvoyProxy(t *testing.T) {
  },
  Spec: egv1a1.EnvoyProxySpec{
  Provider: &egv1a1.EnvoyProxyProvider{
- Type: egv1a1.ProviderTypeFile,
+ Type: egv1a1.ProviderTypeCustom,
  },
  },
  },

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml
@@ -10227,6 +10227,7 @@ spec:
  optional auxiliary control planes. Supported types are "Kubernetes".
  enum:
  - Kubernetes
+ - Custom
  type: string
  required:
  - type

diff --git a/go.mod b/go.mod
@@ -214,7 +214,7 @@ require (
  github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
  github.com/evanphx/json-patch v5.9.0+incompatible
  github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
- github.com/fsnotify/fsnotify v1.7.0 // indirect
+ github.com/fsnotify/fsnotify v1.7.0
  github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
  github.com/go-errors/errors v1.5.1 // indirect
  github.com/go-logr/stdr v1.2.2 // indirect