diff --git a/README.md b/README.md index 8153a93..4aceceb 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ The `k8s-pvc-tagger` watches for new PersistentVolumeClaims and when new AWS EBS `--allow-all-tags` - Allow all tags to be set via the PVC; even those used by the EBS/EFS controllers. Use with caution! +`--copy-labels` - A csv encoded list of label keys from the PVC that will be used to set tags on Volumes. Use `*` to copy all labels from the PVC. + #### Annotations `k8s-pvc-tagger/ignore` - When this annotation is set (any value) it will ignore this PVC and not add any tags to it @@ -36,6 +38,10 @@ NOTE: Until version `v1.2.0` the legacy annotation prefix of `aws-ebs-tagger` wi 4. The cmdline arg `--default-tags={"me": "touge"}` and the annotation `k8s-pvc-tagger/tags: | {"cost-center": "abc", "environment": "prod"}` will create the tags `me=touge`, `cost-center=abc` and `environment=prod` on the EBS/EFS Volume +5. The cmdline arg `--copy-labels '*'` will create a tag from each label on the PVC with the exception of the those used by the controllers unless `--allow-all-tags` is specified. + +6. The cmdline arg `--copy-labels 'cost-center,environment'` will copy the `cost-center` and `environment` labels from the PVC onto the cloud volume. + #### ignored tags The following tags are ignored by default @@ -72,12 +78,45 @@ metadata: {"OwnerID": "{{ .Namespace }}/{{ .Name }}"} ``` +### Multi-cloud support + +Currently supported clouds: AWS, GCP. + +Only one mode is active at a given time. Specify the cloud `k8s-pvc-tagger` is running in with the `--cloud` flag. Either `aws` or `gcp`. + +If not specified `--cloud aws` is the default mode. + +> NOTE: GCP labels have constraints that do not match the contraints allowed by Kubernetes labels. When running in GCP mode labels will be modified to fit GCP's constraints, if necessary. The main difference is `.` and `/` are not allowed, so a label such as `dom.tld/key` will be converted to `dom-tld_key`. + ### Installation #### AWS IAM Role You need to create an AWS IAM Role that can be used by `k8s-pvc-tagger`. For EKS clusters, an [IAM Role for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html) should be used instead of using an AWS access key/secret. For non-EKS clusters, I recommend using a tool like [kube2iam](https://github.com/jtblin/kube2iam). An example policy is in [examples/iam-role.json](examples/iam-role.json). +#### GCP Service Account + +You need a GCP Service Account (GSA) that can be used by `k8s-pvc-tagger`. For GKE clusters, [Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) should be used instead of a static JSON key. + +It is recommended you create a custom IAM role for use by `k8s-pvc-tagger`. The permissions needed are: + +- compute.disks.get +- compute.disks.list +- compute.disks.setLabels + +An example terraform resources is in [examples/gcp-custom-role.tf](examples/gcp-custom-role.tf). + +Or, with `gcloud`: + +```sh +gcloud iam roles create CustomDiskRole \ + --project= \ + --title="k8s-pvc-tagger" \ + --description="Custom role to manage disk permissions" \ + --permissions="compute.disks.get,compute.disks.list,compute.disks.setLabels" \ + --stage="GA" +``` + #### Install via helm ``` diff --git a/aws.go b/aws.go index 103d1b7..7c0a70d 100644 --- a/aws.go +++ b/aws.go @@ -35,10 +35,8 @@ import ( log "github.com/sirupsen/logrus" ) -var ( - // awsSession the AWS Session - awsSession *session.Session -) +// awsSession the AWS Session +var awsSession *session.Session const ( // Matching strings for region @@ -215,7 +213,6 @@ func (client *FSxClient) addFSxVolumeTags(volumeID string, tags map[string]strin ResourceARN: describeFileSystemOutput.FileSystems[0].ResourceARN, Tags: convertTagsToFSxTags(tags), }) - if err != nil { log.Errorln("Could not FSx create tags for volumeID:", volumeID, err) promActionsTotal.With(prometheus.Labels{"status": "error", "storageclass": storageclass}).Inc() @@ -240,7 +237,6 @@ func (client *FSxClient) deleteFSxVolumeTags(volumeID string, tags []*string, st ResourceARN: describeVolumesOutput.Volumes[0].ResourceARN, TagKeys: tags, }) - if err != nil { log.Errorln("Could not FSx delete tags for volumeID:", volumeID, err) promActionsTotal.With(prometheus.Labels{"status": "error", "storageclass": storageclass}).Inc() diff --git a/examples/gcp-custom-role.tf b/examples/gcp-custom-role.tf new file mode 100644 index 0000000..513c7a5 --- /dev/null +++ b/examples/gcp-custom-role.tf @@ -0,0 +1,11 @@ +resource "google_project_iam_custom_role" "k8s-pvc-tagger" { + project = var.gcp_project + role_id = "k8s-pvc-tagger" + title = "k8s-pvc-tagger" + description = "A Custom role with minimum permission set for k8s-pvc-tagger" + permissions = [ + "compute.disks.get", + "compute.disks.list", + "compute.disks.setLabels", + ] +} diff --git a/gcp.go b/gcp.go new file mode 100644 index 0000000..9dda6ab --- /dev/null +++ b/gcp.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "fmt" + "maps" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + "google.golang.org/api/compute/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +type GCPClient interface { + GetDisk(project, zone, name string) (*compute.Disk, error) + SetDiskLabels(project, zone, name string, labelReq *compute.ZoneSetLabelsRequest) (*compute.Operation, error) + GetGCEOp(project, zone, name string) (*compute.Operation, error) +} + +type gcpClient struct { + gce *compute.Service +} + +func newGCPClient(ctx context.Context) (GCPClient, error) { + client, err := compute.NewService(ctx) + if err != nil { + return nil, err + } + return &gcpClient{gce: client}, nil +} + +func (c *gcpClient) GetDisk(project, zone, name string) (*compute.Disk, error) { + return c.gce.Disks.Get(project, zone, name).Do() +} + +func (c *gcpClient) SetDiskLabels(project, zone, name string, labelReq *compute.ZoneSetLabelsRequest) (*compute.Operation, error) { + return c.gce.Disks.SetLabels(project, zone, name, labelReq).Do() +} + +func (c *gcpClient) GetGCEOp(project, zone, name string) (*compute.Operation, error) { + return c.gce.ZoneOperations.Get(project, zone, name).Do() +} + +func addPDVolumeLabels(c GCPClient, volumeID string, labels map[string]string, storageclass string) { + sanitizedLabels := sanitizeLabelsForGCP(labels) + log.Debugf("labels to add to PD volume: %s: %s", volumeID, sanitizedLabels) + + project, location, name, err := parseVolumeID(volumeID) + if err != nil { + log.Error(err) + return + } + disk, err := c.GetDisk(project, location, name) + if err != nil { + log.Error(err) + return + } + + // merge existing disk labels with new labels: + updatedLabels := make(map[string]string) + if disk.Labels != nil { + updatedLabels = maps.Clone(disk.Labels) + } + maps.Copy(updatedLabels, sanitizedLabels) + if maps.Equal(disk.Labels, updatedLabels) { + log.Debug("labels already set on PD") + return + } + + req := &compute.ZoneSetLabelsRequest{ + Labels: updatedLabels, + LabelFingerprint: disk.LabelFingerprint, + } + op, err := c.SetDiskLabels(project, location, name, req) + if err != nil { + log.Errorf("failed to set labels on PD: %s", err) + promActionsTotal.With(prometheus.Labels{"status": "error", "storageclass": storageclass}).Inc() + return + } + + waitForCompletion := func(_ context.Context) (bool, error) { + resp, err := c.GetGCEOp(project, location, op.Name) + if err != nil { + return false, fmt.Errorf("failed to set labels on PD %s: %s", disk.Name, err) + } + return resp.Status == "DONE", nil + } + if err := wait.PollUntilContextTimeout(context.TODO(), + time.Second, + time.Minute, + false, + waitForCompletion); err != nil { + log.Errorf("set label operation failed: %s", err) + promActionsTotal.With(prometheus.Labels{"status": "error", "storageclass": storageclass}).Inc() + return + } + + log.Debug("successfully set labels on PD") + promActionsTotal.With(prometheus.Labels{"status": "success", "storageclass": storageclass}).Inc() +} + +func deletePDVolumeLabels(c GCPClient, volumeID string, keys []string, storageclass string) { + if len(keys) == 0 { + return + } + sanitizedKeys := sanitizeKeysForGCP(keys) + log.Debugf("labels to delete from PD volume: %s: %s", volumeID, sanitizedKeys) + + project, location, name, err := parseVolumeID(volumeID) + if err != nil { + log.Error(err) + return + } + disk, err := c.GetDisk(project, location, name) + if err != nil { + log.Error(err) + return + } + // if disk.Labels is nil, then there are no labels to delete + if disk.Labels == nil { + return + } + + updatedLabels := maps.Clone(disk.Labels) + for _, k := range sanitizedKeys { + delete(updatedLabels, k) + } + if maps.Equal(disk.Labels, updatedLabels) { + return + } + + req := &compute.ZoneSetLabelsRequest{ + Labels: updatedLabels, + LabelFingerprint: disk.LabelFingerprint, + } + op, err := c.SetDiskLabels(project, location, name, req) + if err != nil { + log.Errorf("failed to delete labels from PD: %s", err) + promActionsTotal.With(prometheus.Labels{"status": "error", "storageclass": storageclass}).Inc() + return + } + + waitForCompletion := func(_ context.Context) (bool, error) { + resp, err := c.GetGCEOp(project, location, op.Name) + if err != nil { + return false, fmt.Errorf("failed retrieve status of label update operation: %s", err) + } + return resp.Status == "DONE", nil + } + if err := wait.PollUntilContextTimeout(context.TODO(), + time.Second, + time.Minute, + false, + waitForCompletion); err != nil { + promActionsTotal.With(prometheus.Labels{"status": "error", "storageclass": storageclass}).Inc() + log.Errorf("delete label operation failed: %s", err) + return + } + + log.Debug("successfully deleted labels from PD") + promActionsTotal.With(prometheus.Labels{"status": "success", "storageclass": storageclass}).Inc() +} + +func parseVolumeID(id string) (string, string, string, error) { + parts := strings.Split(id, "/") + if len(parts) < 5 { + return "", "", "", fmt.Errorf("invalid volume handle format") + } + project := parts[1] + location := parts[3] + name := parts[5] + return project, location, name, nil +} + +func sanitizeLabelsForGCP(labels map[string]string) map[string]string { + newLabels := make(map[string]string, len(labels)) + for k, v := range labels { + newLabels[sanitizeKeyForGCP(k)] = sanitizeValueForGCP(v) + } + return newLabels +} + +func sanitizeKeysForGCP(keys []string) []string { + newKeys := make([]string, len(keys)) + for i, k := range keys { + newKeys[i] = sanitizeKeyForGCP(k) + } + return newKeys +} + +// sanitizeKeyForGCP sanitizes a Kubernetes label key to fit GCP's label key constraints +func sanitizeKeyForGCP(key string) string { + key = strings.ToLower(key) + key = strings.NewReplacer("/", "_", ".", "-").Replace(key) // Replace disallowed characters + key = strings.TrimRight(key, "-_") // Ensure it does not end with '-' or '_' + + if len(key) > 63 { + key = key[:63] + } + return key +} + +// sanitizeKeyForGCP sanitizes a Kubernetes label value to fit GCP's label value constraints +func sanitizeValueForGCP(value string) string { + if len(value) > 63 { + value = value[:63] + } + return value +} diff --git a/gcp_test.go b/gcp_test.go new file mode 100644 index 0000000..840ad33 --- /dev/null +++ b/gcp_test.go @@ -0,0 +1,258 @@ +package main + +import ( + "maps" + "reflect" + "strings" + "testing" + + "google.golang.org/api/compute/v1" +) + +type fakeGCPClient struct { + fakeGetDisk func(project, zone, name string) (*compute.Disk, error) + fakeSetDiskLabels func(project, zone, name string, labelReq *compute.ZoneSetLabelsRequest) (*compute.Operation, error) + fakeGetGCEOp func(project, zone, name string) (*compute.Operation, error) + + setLabelsCalled bool +} + +func (c *fakeGCPClient) GetDisk(project, zone, name string) (*compute.Disk, error) { + if c.fakeGetDisk == nil { + return nil, nil + } + return c.fakeGetDisk(project, zone, name) +} + +func (c *fakeGCPClient) SetDiskLabels(project, zone, name string, labelReq *compute.ZoneSetLabelsRequest) (*compute.Operation, error) { + c.setLabelsCalled = true + if c.fakeSetDiskLabels == nil { + return nil, nil + } + return c.fakeSetDiskLabels(project, zone, name, labelReq) +} + +func (c *fakeGCPClient) GetGCEOp(project, zone, name string) (*compute.Operation, error) { + if c.fakeSetDiskLabels == nil { + return nil, nil + } + return c.fakeGetGCEOp(project, zone, name) +} + +func setupFakeGCPClient(t *testing.T, currentLabels map[string]string, expectedSetLabels map[string]string) *fakeGCPClient { + return &fakeGCPClient{ + fakeGetDisk: func(project, zone, name string) (*compute.Disk, error) { + return &compute.Disk{Labels: currentLabels}, nil + }, + fakeSetDiskLabels: func(project, zone, name string, labelReq *compute.ZoneSetLabelsRequest) (*compute.Operation, error) { + if !maps.Equal(labelReq.Labels, expectedSetLabels) { + t.Errorf("SetDiskLabels(), got labels = %v, want = %v", labelReq.Labels, expectedSetLabels) + } + return &compute.Operation{Status: "PENDING"}, nil + }, + fakeGetGCEOp: func(project, zone, name string) (*compute.Operation, error) { + return &compute.Operation{Status: "DONE"}, nil + }, + } +} + +func TestAddPDVolumeLabels(t *testing.T) { + tests := []struct { + name string + volumeID string + currentLabels map[string]string + newPvcLabels map[string]string + expectSetLabelsCalled bool + expectedSetLabels map[string]string + }{ + { + name: "add new labels", + volumeID: "projects/myproject/zones/myzone/disks/mydisk", + currentLabels: map[string]string{"key1": "val1", "key2": "val2"}, + newPvcLabels: map[string]string{"foo": "bar", "dom.tld/key": "value"}, + expectSetLabelsCalled: true, + expectedSetLabels: map[string]string{"key1": "val1", "key2": "val2", "foo": "bar", "dom-tld_key": "value"}, + }, + { + name: "labels already set", + volumeID: "projects/myproject/zones/myzone/disks/mydisk", + currentLabels: map[string]string{"key1": "val1", "key2": "val2"}, + expectSetLabelsCalled: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := setupFakeGCPClient(t, tt.currentLabels, tt.expectedSetLabels) + + addPDVolumeLabels(client, tt.volumeID, tt.newPvcLabels, "storage-ssd") + + if client.setLabelsCalled != tt.expectSetLabelsCalled { + t.Error("SetDiskLabels() was not called") + } + }) + } +} + +func TestDeletePDVolumeLabels(t *testing.T) { + tests := []struct { + name string + volumeID string + currentLabels map[string]string + labelsToDelete []string + expectSetLabelsCalled bool + expectedSetLabels map[string]string + }{ + { + name: "delete existing labels", + volumeID: "projects/myproject/zones/myzone/disks/mydisk", + currentLabels: map[string]string{"key1": "val1", "key2": "val2", "dom-tld_key": "bar"}, + labelsToDelete: []string{"key1", "dom.tld/key"}, + expectSetLabelsCalled: true, + expectedSetLabels: map[string]string{"key2": "val2"}, + }, + { + name: "no labels to delete", + volumeID: "projects/myproject/zones/myzone/disks/mydisk", + currentLabels: map[string]string{"key1": "val1", "key2": "val2"}, + labelsToDelete: []string{}, + expectSetLabelsCalled: false, + }, + { + name: "no matching labels to delete", + volumeID: "projects/myproject/zones/myzone/disks/mydisk", + currentLabels: map[string]string{"key1": "val1", "key2": "val2"}, + labelsToDelete: []string{"foo"}, + expectSetLabelsCalled: false, + }, + { + name: "all labels deleted", + volumeID: "projects/myproject/zones/myzone/disks/mydisk", + currentLabels: map[string]string{"key1": "val1"}, + labelsToDelete: []string{"key1"}, + expectSetLabelsCalled: true, + expectedSetLabels: map[string]string{}, + }, + { + name: "no labels on disk", + volumeID: "projects/myproject/zones/myzone/disks/mydisk", + currentLabels: nil, + labelsToDelete: []string{"foo"}, + expectSetLabelsCalled: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := setupFakeGCPClient(t, tt.currentLabels, tt.expectedSetLabels) + + deletePDVolumeLabels(client, tt.volumeID, tt.labelsToDelete, "storage-ssd") + + if client.setLabelsCalled != tt.expectSetLabelsCalled { + t.Error("SetDiskLabels() was not called") + } + }) + } +} + +func TestSanitizeLabelsForGCP(t *testing.T) { + tests := []struct { + name string + labels map[string]string + want map[string]string + }{ + { + name: "simple labels", + labels: map[string]string{ + "Example/Key": "Example Value", + "Another.Key": "Another Value", + }, + want: map[string]string{ + "example_key": "Example Value", + "another-key": "Another Value", + }, + }, + { + name: "labels with special characters", + labels: map[string]string{ + "Domain.com/Key": "Value_1", + "Project.Version": "Version-1.2.3", + }, + want: map[string]string{ + "domain-com_key": "Value_1", + "project-version": "Version-1.2.3", + }, + }, + { + name: "labels exceeding maximum length", + labels: map[string]string{ + strings.Repeat("a", 70): strings.Repeat("b", 70), + }, + want: map[string]string{ + strings.Repeat("a", 63): strings.Repeat("b", 63), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := sanitizeLabelsForGCP(tt.labels); !reflect.DeepEqual(got, tt.want) { + t.Errorf("sanitizeLabelsForGCP(), got = %v, want = %v", got, tt.want) + } + }) + } +} + +func TestParseVolumeID(t *testing.T) { + tests := []struct { + name string + id string + wantProject string + wantLocation string + wantName string + wantErr bool + }{ + { + name: "valid volume ID", + id: "projects/my-project/zones/us-central1/disks/my-disk", + wantProject: "my-project", + wantLocation: "us-central1", + wantName: "my-disk", + wantErr: false, + }, + { + name: "missing parts", + id: "projects/my-project/zones/", + wantProject: "", + wantLocation: "", + wantName: "", + wantErr: true, + }, + { + name: "empty input", + id: "", + wantProject: "", + wantLocation: "", + wantName: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + project, location, name, err := parseVolumeID(tt.id) + if (err != nil) != tt.wantErr { + t.Errorf("parseVolumeID() error = %v, wantErr %v", err, tt.wantErr) + } + if project != tt.wantProject { + t.Errorf("Expected project %q, got %q", tt.wantProject, project) + } + if location != tt.wantLocation { + t.Errorf("Expected location %q, got %q", tt.wantLocation, location) + } + if name != tt.wantName { + t.Errorf("Expected name %q, got %q", tt.wantName, name) + } + }) + } +} diff --git a/go.mod b/go.mod index e9a2f29..7d013c0 100644 --- a/go.mod +++ b/go.mod @@ -4,29 +4,39 @@ go 1.21 require ( github.com/aws/aws-sdk-go v1.49.9 - github.com/google/uuid v1.5.0 + github.com/google/uuid v1.6.0 github.com/prometheus/client_golang v1.17.0 github.com/sirupsen/logrus v1.9.3 + google.golang.org/api v0.180.0 k8s.io/api v0.29.0 k8s.io/apimachinery v0.29.0 k8s.io/client-go v0.29.0 ) require ( + cloud.google.com/go/auth v0.4.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/swag v0.22.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.4 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -41,14 +51,21 @@ require ( github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.15.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index aed212f..d94fbcf 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,39 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg= +cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/aws/aws-sdk-go v1.49.9 h1:4xoyi707rsifB1yMsd5vGbAH21aBzwpL3gNRMSmjIyc= github.com/aws/aws-sdk-go v1.49.9/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= @@ -24,15 +44,30 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -41,8 +76,15 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= +github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -80,6 +122,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= @@ -93,73 +136,118 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/api v0.180.0 h1:M2D87Yo0rGBPWpo1orwfCLehUUL6E7/TYe5gvMQWDh4= +google.golang.org/api v0.180.0/go.mod h1:51AiyoEg1MJPSZ9zvklA8VnRILPXxn1iVen9v25XHAE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be h1:Zz7rLWqp0ApfsR/l7+zSHhY3PMiH2xqgxlfYfAfNpoU= +google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -171,6 +259,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= diff --git a/kubernetes.go b/kubernetes.go index 947f740..39a7df0 100644 --- a/kubernetes.go +++ b/kubernetes.go @@ -25,11 +25,11 @@ import ( "errors" "fmt" "html/template" - "io/ioutil" "net/url" "os" "path/filepath" "regexp" + "slices" "strings" "github.com/aws/aws-sdk-go/aws" @@ -56,6 +56,16 @@ var ( const ( // Matching strings for volume operations. regexpEFSVolumeID = `^fs-\w+::(fsap-\w+)$` + + // supported AWS storage provisioners: + AWS_EBS_CSI = "ebs.csi.aws.com" + AWS_EBS_LEGACY = "kubernetes.io/aws-ebs" + AWS_EFS_CSI = "efs.csi.aws.com" + AWS_FSX_CSI = "fsx.csi.aws.com" + + // supported GCP storage provisioners: + GCP_PD_CSI = "pd.csi.storage.gke.io" + GCP_PD_LEGACY = "kubernetes.io/gce-pd" ) type TagTemplate struct { @@ -93,7 +103,7 @@ func buildConfigFromFlags(kubeconfig string, context string) (*rest.Config, erro } func watchForPersistentVolumeClaims(ch chan struct{}, watchNamespace string) { - + var err error var factory informers.SharedInformerFactory log.WithFields(log.Fields{"namespace": watchNamespace}).Infoln("Starting informer") if watchNamespace == "" { @@ -104,43 +114,63 @@ func watchForPersistentVolumeClaims(ch chan struct{}, watchNamespace string) { informer := factory.Core().V1().PersistentVolumeClaims().Informer() - efsClient, _ := newEFSClient() - ec2Client, _ := newEC2Client() - fsxClient, _ := newFSxClient() + var efsClient *EFSClient + var ec2Client *EBSClient + var fsxClient *FSxClient + var gcpClient GCPClient + + switch cloud { + case AWS: + efsClient, _ = newEFSClient() + ec2Client, _ = newEC2Client() + fsxClient, _ = newFSxClient() + case GCP: + gcpClient, err = newGCPClient(context.Background()) + if err != nil { + log.Fatalln("failed to create GCP client", err) + } + } - _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err = informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { pvc := getPVC(obj) - if !provisionedByAwsEfs(pvc) && !provisionedByAwsEbs(pvc) && !provisionedByAwsFsx(pvc) { - return - } log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Infoln("New PVC Added to Store") volumeID, tags, err := processPersistentVolumeClaim(pvc) if err != nil || len(tags) == 0 { return } - if provisionedByAwsEfs(pvc) { - efsClient.addEFSVolumeTags(volumeID, tags, *pvc.Spec.StorageClassName) - } - if provisionedByAwsEbs(pvc) { - ec2Client.addEBSVolumeTags(volumeID, tags, *pvc.Spec.StorageClassName) - } - if provisionedByAwsFsx(pvc) { - fsxClient.addFSxVolumeTags(volumeID, tags, *pvc.Spec.StorageClassName) + + switch cloud { + case AWS: + if !provisionedByAwsEfs(pvc) && !provisionedByAwsEbs(pvc) && !provisionedByAwsFsx(pvc) { + return + } + + if provisionedByAwsEfs(pvc) { + efsClient.addEFSVolumeTags(volumeID, tags, *pvc.Spec.StorageClassName) + } + if provisionedByAwsEbs(pvc) { + ec2Client.addEBSVolumeTags(volumeID, tags, *pvc.Spec.StorageClassName) + } + if provisionedByAwsFsx(pvc) { + fsxClient.addFSxVolumeTags(volumeID, tags, *pvc.Spec.StorageClassName) + } + case GCP: + if !provisionedByGcpPD(pvc) { + return + } + addPDVolumeLabels(gcpClient, volumeID, tags, *pvc.Spec.StorageClassName) } }, - UpdateFunc: func(old, new interface{}) { + UpdateFunc: func(old, new interface{}) { newPVC := getPVC(new) oldPVC := getPVC(old) if newPVC.ResourceVersion == oldPVC.ResourceVersion { log.WithFields(log.Fields{"namespace": newPVC.GetNamespace(), "pvc": newPVC.GetName()}).Debugln("ResourceVersion are the same") return } - if !provisionedByAwsEfs(newPVC) && !provisionedByAwsEbs(newPVC) && !provisionedByAwsFsx(newPVC) { - return - } if newPVC.Spec.VolumeName == "" { log.WithFields(log.Fields{"namespace": newPVC.GetNamespace(), "pvc": newPVC.GetName()}).Debugln("PersistentVolume not created yet") return @@ -149,41 +179,67 @@ func watchForPersistentVolumeClaims(ch chan struct{}, watchNamespace string) { log.WithFields(log.Fields{"namespace": newPVC.GetNamespace(), "pvc": newPVC.GetName()}).Debugln("PersistentVolumeClaim is being deleted") return } - log.WithFields(log.Fields{"namespace": newPVC.GetNamespace(), "pvc": newPVC.GetName()}).Infoln("Need to reconcile tags") + volumeID, tags, err := processPersistentVolumeClaim(newPVC) if err != nil { return } - if len(tags) > 0 { - if provisionedByAwsEfs(newPVC) { - efsClient.addEFSVolumeTags(volumeID, tags, *newPVC.Spec.StorageClassName) + + switch cloud { + case AWS: + if !provisionedByAwsEfs(newPVC) && !provisionedByAwsEbs(newPVC) && !provisionedByAwsFsx(newPVC) { + return } - if provisionedByAwsEbs(newPVC) { - ec2Client.addEBSVolumeTags(volumeID, tags, *newPVC.Spec.StorageClassName) + + if len(tags) > 0 { + if provisionedByAwsEfs(newPVC) { + efsClient.addEFSVolumeTags(volumeID, tags, *newPVC.Spec.StorageClassName) + } + if provisionedByAwsEbs(newPVC) { + ec2Client.addEBSVolumeTags(volumeID, tags, *newPVC.Spec.StorageClassName) + } + if provisionedByAwsFsx(newPVC) { + fsxClient.addFSxVolumeTags(volumeID, tags, *newPVC.Spec.StorageClassName) + } } - if provisionedByAwsFsx(newPVC) { - fsxClient.addFSxVolumeTags(volumeID, tags, *newPVC.Spec.StorageClassName) + oldTags := buildTags(oldPVC) + var deletedTags []string + var deletedTagsPtr []*string + for k := range oldTags { + if _, ok := tags[k]; !ok { + deletedTags = append(deletedTags, k) + deletedTagsPtr = append(deletedTagsPtr, &k) + } } - } - oldTags := buildTags(oldPVC) - var deletedTags []string - var deletedTagsPtr []*string - for k := range oldTags { - if _, ok := tags[k]; !ok { - deletedTags = append(deletedTags, k) - deletedTagsPtr = append(deletedTagsPtr, &k) + if len(deletedTags) > 0 { + if provisionedByAwsEfs(newPVC) { + efsClient.deleteEFSVolumeTags(volumeID, deletedTags, *oldPVC.Spec.StorageClassName) + } + if provisionedByAwsEbs(newPVC) { + ec2Client.deleteEBSVolumeTags(volumeID, deletedTags, *oldPVC.Spec.StorageClassName) + } + if provisionedByAwsFsx(newPVC) { + fsxClient.deleteFSxVolumeTags(volumeID, deletedTagsPtr, *oldPVC.Spec.StorageClassName) + } } - } - if len(deletedTags) > 0 { - if provisionedByAwsEfs(newPVC) { - efsClient.deleteEFSVolumeTags(volumeID, deletedTags, *oldPVC.Spec.StorageClassName) + case GCP: + if !provisionedByGcpPD(newPVC) { + return + } + + if len(tags) > 0 { + addPDVolumeLabels(gcpClient, volumeID, tags, *newPVC.Spec.StorageClassName) } - if provisionedByAwsEbs(newPVC) { - ec2Client.deleteEBSVolumeTags(volumeID, deletedTags, *oldPVC.Spec.StorageClassName) + oldTags := buildTags(oldPVC) + var deletedTags []string + for k := range oldTags { + if _, ok := tags[k]; !ok { + deletedTags = append(deletedTags, k) + } } - if provisionedByAwsFsx(newPVC) { - fsxClient.deleteFSxVolumeTags(volumeID, deletedTagsPtr, *oldPVC.Spec.StorageClassName) + if len(deletedTags) > 0 { + deletePDVolumeLabels(gcpClient, volumeID, deletedTags, *newPVC.Spec.StorageClassName) } } }, @@ -244,7 +300,6 @@ func parseAWSEFSVolumeID(k8sVolumeID string) string { } func buildTags(pvc *corev1.PersistentVolumeClaim) map[string]string { - tags := map[string]string{} customTags := map[string]string{} var tagString string @@ -283,6 +338,24 @@ func buildTags(pvc *corev1.PersistentVolumeClaim) map[string]string { tags[k] = v } + if len(copyLabels) > 0 { + for k, v := range pvc.GetLabels() { + if copyLabels[0] == "*" || slices.Contains(copyLabels, k) { + if !isValidTagName(k) { + if !allowAllTags { + log.Warnln(k, "is a restricted tag. Skipping...") + promInvalidTagsTotal.With(prometheus.Labels{"storageclass": *pvc.Spec.StorageClassName}).Inc() + promInvalidTagsLegacyTotal.Inc() + continue + } else { + log.Warnln(k, "is a restricted tag but still allowing it to be set...") + } + } + tags[k] = v + } + } + } + var legacyOk bool tagString, ok := annotations[annotationPrefix+"/tags"] // if the annotationPrefix has been changed, then we don't compare to the legacyAnnotationPrefix anymore @@ -327,7 +400,6 @@ func buildTags(pvc *corev1.PersistentVolumeClaim) map[string]string { } func renderTagTemplates(pvc *corev1.PersistentVolumeClaim, tags map[string]string) map[string]string { - tplData := TagTemplate{ Name: pvc.GetName(), Namespace: pvc.GetNamespace(), @@ -375,8 +447,8 @@ func provisionedByAwsEfs(pvc *corev1.PersistentVolumeClaim) bool { return false } - if provisionedBy == "efs.csi.aws.com" { - log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln("efs.csi.aws.com volume") + if provisionedBy == AWS_EFS_CSI { + log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln(AWS_EFS_CSI + " volume") return true } return false @@ -394,11 +466,12 @@ func provisionedByAwsEbs(pvc *corev1.PersistentVolumeClaim) bool { return false } - if provisionedBy == "kubernetes.io/aws-ebs" { - log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln("kubernetes.io/aws-ebs volume") + switch provisionedBy { + case AWS_EBS_LEGACY: + log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln(AWS_EBS_LEGACY + " volume") return true - } else if provisionedBy == "ebs.csi.aws.com" { - log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln("ebs.csi.aws.com volume") + case AWS_EBS_CSI: + log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln(AWS_EBS_CSI + " volume") return true } return false @@ -416,8 +489,31 @@ func provisionedByAwsFsx(pvc *corev1.PersistentVolumeClaim) bool { return false } - if provisionedBy == "fsx.csi.aws.com" { - log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln("fsx.csi.aws.com volume") + if provisionedBy == AWS_FSX_CSI { + log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln(AWS_FSX_CSI + " volume") + return true + } + return false +} + +func provisionedByGcpPD(pvc *corev1.PersistentVolumeClaim) bool { + annotations := pvc.GetAnnotations() + if annotations == nil { + return false + } + + provisionedBy, ok := getProvisionedBy(annotations) + if !ok { + log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln("no volume.kubernetes.io/storage-provisioner annotation") + return false + } + + switch provisionedBy { + case GCP_PD_LEGACY: + log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln(GCP_PD_LEGACY + " volume") + return true + case GCP_PD_CSI: + log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName()}).Debugln(GCP_PD_CSI + " volume") return true } return false @@ -446,21 +542,28 @@ func processPersistentVolumeClaim(pvc *corev1.PersistentVolumeClaim) (string, ma log.Errorf("cannot get volume.kubernetes.io/storage-provisioner annotation") return "", nil, errors.New("cannot get volume.kubernetes.io/storage-provisioner annotation") } - if provisionedBy == "ebs.csi.aws.com" { + + switch provisionedBy { + case AWS_EBS_CSI: if pv.Spec.CSI != nil { volumeID = pv.Spec.CSI.VolumeHandle } else { volumeID = parseAWSEBSVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) } - } else if provisionedBy == "efs.csi.aws.com" { + case AWS_EFS_CSI: if pv.Spec.CSI != nil { volumeID = parseAWSEFSVolumeID(pv.Spec.CSI.VolumeHandle) } - } else if provisionedBy == "kubernetes.io/aws-ebs" { + case AWS_EBS_LEGACY: volumeID = parseAWSEBSVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) - } else if provisionedBy == "fsx.csi.aws.com" { + case AWS_FSX_CSI: + volumeID = pv.Spec.CSI.VolumeHandle + case GCP_PD_LEGACY: + volumeID = pv.Spec.GCEPersistentDisk.PDName + case GCP_PD_CSI: volumeID = pv.Spec.CSI.VolumeHandle } + log.WithFields(log.Fields{"namespace": pvc.GetNamespace(), "pvc": pvc.GetName(), "volumeID": volumeID}).Debugln("parsed volumeID:", volumeID) if len(volumeID) == 0 { log.Errorf("Cannot parse VolumeID") @@ -472,7 +575,7 @@ func processPersistentVolumeClaim(pvc *corev1.PersistentVolumeClaim) (string, ma func getCurrentNamespace() string { // Fall back to the namespace associated with the service account token, if available - if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { + if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { if ns := strings.TrimSpace(string(data)); len(ns) > 0 { return ns } diff --git a/kubernetes_test.go b/kubernetes_test.go index 6802ff3..59ddd5a 100644 --- a/kubernetes_test.go +++ b/kubernetes_test.go @@ -143,7 +143,6 @@ func Test_isValidTagName(t *testing.T) { } func Test_provisionedByAwsEbs(t *testing.T) { - pvc := &corev1.PersistentVolumeClaim{} pvc.SetName("my-pvc") pvc.Spec.StorageClassName = &dummyStorageClassName @@ -155,12 +154,12 @@ func Test_provisionedByAwsEbs(t *testing.T) { }{ { name: "valid provisioner in-tree aws-ebs legacy annotation", - annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": "kubernetes.io/aws-ebs"}, + annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": AWS_EBS_LEGACY}, want: true, }, { name: "valid provisioner ebs.csi.aws.com", - annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": "ebs.csi.aws.com"}, + annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": AWS_EBS_CSI}, want: true, }, { @@ -170,12 +169,12 @@ func Test_provisionedByAwsEbs(t *testing.T) { }, { name: "valid provisioner in-tree aws-ebs legacy annotation", - annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": "kubernetes.io/aws-ebs"}, + annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": AWS_EBS_LEGACY}, want: true, }, { name: "valid provisioner ebs.csi.aws.com legacy annotation", - annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": "ebs.csi.aws.com"}, + annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": AWS_EBS_CSI}, want: true, }, { @@ -200,7 +199,6 @@ func Test_provisionedByAwsEbs(t *testing.T) { } func Test_provisionedByAwsEfs(t *testing.T) { - pvc := &corev1.PersistentVolumeClaim{} pvc.SetName("my-pvc") pvc.Spec.StorageClassName = &dummyStorageClassName @@ -212,7 +210,7 @@ func Test_provisionedByAwsEfs(t *testing.T) { }{ { name: "valid provisioner efs.csi.aws.com", - annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": "efs.csi.aws.com"}, + annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": AWS_EFS_CSI}, want: true, }, { @@ -222,7 +220,7 @@ func Test_provisionedByAwsEfs(t *testing.T) { }, { name: "valid provisioner efs.csi.aws.com legacy annotation", - annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": "efs.csi.aws.com"}, + annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": AWS_EFS_CSI}, want: true, }, { @@ -247,7 +245,6 @@ func Test_provisionedByAwsEfs(t *testing.T) { } func Test_provisionedByAwsFsx(t *testing.T) { - pvc := &corev1.PersistentVolumeClaim{} pvc.SetName("my-pvc") pvc.Spec.StorageClassName = &dummyStorageClassName @@ -259,7 +256,7 @@ func Test_provisionedByAwsFsx(t *testing.T) { }{ { name: "valid provisioner fsx.csi.aws.com", - annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": "fsx.csi.aws.com"}, + annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": AWS_FSX_CSI}, want: true, }, { @@ -269,7 +266,7 @@ func Test_provisionedByAwsFsx(t *testing.T) { }, { name: "valid provisioner fsx.csi.aws.com legacy annotation", - annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": "fsx.csi.aws.com"}, + annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": AWS_FSX_CSI}, want: true, }, { @@ -293,8 +290,64 @@ func Test_provisionedByAwsFsx(t *testing.T) { } } -func Test_buildTags(t *testing.T) { +func Test_provisionedByGcpPD(t *testing.T) { + pvc := &corev1.PersistentVolumeClaim{} + pvc.SetName("my-pvc") + dummyStorageClassName := "dummy-storage-class" + pvc.Spec.StorageClassName = &dummyStorageClassName + tests := []struct { + name string + annotations map[string]string + want bool + }{ + { + name: "valid provisioner in-tree gce-pd legacy annotation", + annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": GCP_PD_LEGACY}, + want: true, + }, + { + name: "valid provisioner pd.csi.storage.gke.io", + annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": GCP_PD_CSI}, + want: true, + }, + { + name: "invalid provisioner", + annotations: map[string]string{"volume.kubernetes.io/storage-provisioner": "something else"}, + want: false, + }, + { + name: "valid provisioner in-tree gce-pd legacy annotation beta", + annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": GCP_PD_LEGACY}, + want: true, + }, + { + name: "valid provisioner pd.csi.storage.gke.io legacy annotation", + annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": GCP_PD_CSI}, + want: true, + }, + { + name: "invalid provisioner legacy annotation", + annotations: map[string]string{"volume.beta.kubernetes.io/storage-provisioner": "something else"}, + want: false, + }, + { + name: "provisioner not set", + annotations: map[string]string{"some annotation": "something else"}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pvc.SetAnnotations(tt.annotations) + if got := provisionedByGcpPD(pvc); got != tt.want { + t.Errorf("provisionedByGcpPD() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_buildTags(t *testing.T) { pvc := &corev1.PersistentVolumeClaim{} pvc.SetName("my-pvc") pvc.Spec.StorageClassName = &dummyStorageClassName @@ -303,6 +356,8 @@ func Test_buildTags(t *testing.T) { name string defaultTags map[string]string allowAllTags bool + copyLabels []string + pvcLabels map[string]string annotations map[string]string want map[string]string tagFormat string @@ -566,10 +621,43 @@ func Test_buildTags(t *testing.T) { annotations: map[string]string{"k8s-pvc-tagger/tags": "{\"foo\": \"selected\"}", "aws-ebs-tagger/ignore": ""}, want: map[string]string{}, }, + { + name: "copy-labels flag is set as wildcard", + defaultTags: map[string]string{}, + allowAllTags: false, + copyLabels: []string{"*"}, + pvcLabels: map[string]string{"foo": "bar", "dom.tld/key": "value"}, + want: map[string]string{"foo": "bar", "dom.tld/key": "value"}, + }, + { + name: "copy-labels flag with one matching label", + defaultTags: map[string]string{}, + allowAllTags: false, + copyLabels: []string{"foo"}, + pvcLabels: map[string]string{"foo": "bar", "dom.tld/key": "value"}, + want: map[string]string{"foo": "bar"}, + }, + { + name: "copy-labels flag with multiple matching labels", + defaultTags: map[string]string{}, + allowAllTags: false, + copyLabels: []string{"foo", "dom.tld/key"}, + pvcLabels: map[string]string{"foo": "bar", "dom.tld/key": "value"}, + want: map[string]string{"foo": "bar", "dom.tld/key": "value"}, + }, + { + name: "copy-labels with default tags", + defaultTags: map[string]string{"quux": "baz"}, + allowAllTags: false, + copyLabels: []string{"foo", "dom.tld/key"}, + pvcLabels: map[string]string{"foo": "bar", "dom.tld/key": "value"}, + want: map[string]string{"quux": "baz", "foo": "bar", "dom.tld/key": "value"}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { pvc.SetAnnotations(tt.annotations) + pvc.SetLabels(tt.pvcLabels) defaultTags = tt.defaultTags allowAllTags = tt.allowAllTags if tt.tagFormat != "" { @@ -577,6 +665,9 @@ func Test_buildTags(t *testing.T) { } else { tagFormat = "json" } + if tt.copyLabels != nil { + copyLabels = tt.copyLabels + } if got := buildTags(pvc); !reflect.DeepEqual(got, tt.want) { t.Errorf("buildTags() = %v, want %v", got, tt.want) } @@ -587,7 +678,6 @@ func Test_buildTags(t *testing.T) { } func Test_annotationPrefix(t *testing.T) { - pvc := &corev1.PersistentVolumeClaim{} pvc.SetName("my-pvc") defaultAnnotationPrefix := annotationPrefix @@ -675,7 +765,7 @@ func Test_processEBSPersistentVolumeClaim(t *testing.T) { }{ { name: "csi with valid tags and volume id", - provisionedBy: "ebs.csi.aws.com", + provisionedBy: AWS_EBS_CSI, tagsAnnotation: "{\"foo\": \"bar\"}", volumeName: volumeName, wantedTags: map[string]string{"foo": "bar"}, @@ -684,7 +774,7 @@ func Test_processEBSPersistentVolumeClaim(t *testing.T) { }, { name: "in-tree with valid tags and volume id", - provisionedBy: "kubernetes.io/aws-ebs", + provisionedBy: AWS_EBS_LEGACY, tagsAnnotation: "{\"foo\": \"bar\"}", volumeName: volumeName, volumeID: "aws://us-east-1a/vol-12345", @@ -694,7 +784,7 @@ func Test_processEBSPersistentVolumeClaim(t *testing.T) { }, { name: "in-tree with valid tags and invalid volume id", - provisionedBy: "kubernetes.io/aws-ebs", + provisionedBy: AWS_EBS_LEGACY, tagsAnnotation: "{\"foo\": \"bar\"}", volumeName: volumeName, volumeID: "aws://us-east-1a/abc123", @@ -713,7 +803,7 @@ func Test_processEBSPersistentVolumeClaim(t *testing.T) { }, { name: "in-tree with missing PV", - provisionedBy: "kubernetes.io/aws-ebs", + provisionedBy: AWS_EBS_LEGACY, tagsAnnotation: "{\"foo\": \"bar\"}", volumeName: "asdf", volumeID: "aws://us-east-1a/vol-12345", @@ -730,7 +820,7 @@ func Test_processEBSPersistentVolumeClaim(t *testing.T) { }) var pvSpec corev1.PersistentVolumeSpec - if tt.provisionedBy == "ebs.csi.aws.com" { + if tt.provisionedBy == AWS_EBS_CSI { pvSpec = corev1.PersistentVolumeSpec{ StorageClassName: dummyStorageClassName, PersistentVolumeSource: corev1.PersistentVolumeSource{ @@ -769,7 +859,6 @@ func Test_processEBSPersistentVolumeClaim(t *testing.T) { } }) } - } func Test_processEFSPersistentVolumeClaim(t *testing.T) { @@ -790,7 +879,7 @@ func Test_processEFSPersistentVolumeClaim(t *testing.T) { }{ { name: "csi with valid tags and volume id", - provisionedBy: "efs.csi.aws.com", + provisionedBy: AWS_EFS_CSI, tagsAnnotation: "{\"foo\": \"bar\"}", volumeName: volumeName, volumeID: "fs-05b82f74723423::fsap-06cc098e562d23425", @@ -800,7 +889,7 @@ func Test_processEFSPersistentVolumeClaim(t *testing.T) { }, { name: "csi with valid tags and invalid volume id", - provisionedBy: "efs.csi.aws.com", + provisionedBy: AWS_EFS_CSI, tagsAnnotation: "{\"foo\": \"bar\"}", volumeName: volumeName, volumeID: "asdf://us-east-1a/vol-12345", @@ -819,7 +908,7 @@ func Test_processEFSPersistentVolumeClaim(t *testing.T) { }, { name: "csi with missing PV", - provisionedBy: "efs.csi.aws.com", + provisionedBy: AWS_EFS_CSI, tagsAnnotation: "{\"foo\": \"bar\"}", volumeName: "asdf", volumeID: "fs-05b82f74723423::fsap-06cc098e562d23425", @@ -836,7 +925,7 @@ func Test_processEFSPersistentVolumeClaim(t *testing.T) { }) var pvSpec corev1.PersistentVolumeSpec - if tt.provisionedBy == "efs.csi.aws.com" { + if tt.provisionedBy == AWS_EFS_CSI { pvSpec = corev1.PersistentVolumeSpec{ PersistentVolumeSource: corev1.PersistentVolumeSource{ CSI: &corev1.CSIPersistentVolumeSource{ @@ -865,11 +954,124 @@ func Test_processEFSPersistentVolumeClaim(t *testing.T) { } }) } +} +func Test_processGCPPDPersistentVolumeClaim(t *testing.T) { + volumeName := "pvc-1234" + pvc := &corev1.PersistentVolumeClaim{} + pvc.SetName("my-pvc") + pvc.Spec.VolumeName = volumeName + + tests := []struct { + name string + provisionedBy string + tagsAnnotation string + volumeID string + volumeName string + wantedTags map[string]string + wantedVolumeID string + wantedErr bool + }{ + { + name: "csi with valid tags and volume id", + provisionedBy: GCP_PD_CSI, + tagsAnnotation: "{\"foo\": \"bar\"}", + volumeName: volumeName, + volumeID: "projects/my-project/zones/us-east1-a/disks/my-disk", + wantedTags: map[string]string{"foo": "bar"}, + wantedVolumeID: "projects/my-project/zones/us-east1-a/disks/my-disk", + wantedErr: false, + }, + { + name: "in-tree with valid tags and volume id", + provisionedBy: GCP_PD_LEGACY, + tagsAnnotation: "{\"foo\": \"bar\"}", + volumeName: volumeName, + volumeID: "projects/my-project/zones/us-east1-a/disks/my-disk", + wantedTags: map[string]string{"foo": "bar"}, + wantedVolumeID: "projects/my-project/zones/us-east1-a/disks/my-disk", + wantedErr: false, + }, + { + name: "in-tree with valid tags and invalid volume id", + provisionedBy: GCP_PD_LEGACY, + tagsAnnotation: "{\"foo\": \"bar\"}", + volumeName: volumeName, + volumeID: "", + wantedTags: nil, + wantedVolumeID: "", + wantedErr: true, + }, + { + name: "other with valid tags and volume id", + provisionedBy: "foo", + tagsAnnotation: "{\"foo\": \"bar\"}", + volumeName: volumeName, + wantedTags: nil, + wantedVolumeID: "", + wantedErr: true, + }, + { + name: "in-tree with missing PV", + provisionedBy: GCP_PD_LEGACY, + tagsAnnotation: "{\"foo\": \"bar\"}", + volumeName: "asdf", + volumeID: "projects/my-project/zones/us-east1-a/disks/my-disk", + wantedTags: nil, + wantedVolumeID: "", + wantedErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pvc.SetAnnotations(map[string]string{ + annotationPrefix + "/tags": tt.tagsAnnotation, + "volume.kubernetes.io/storage-provisioner": tt.provisionedBy, + }) + + var pvSpec corev1.PersistentVolumeSpec + if tt.provisionedBy == GCP_PD_CSI { + pvSpec = corev1.PersistentVolumeSpec{ + StorageClassName: dummyStorageClassName, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + VolumeHandle: tt.wantedVolumeID, + }, + }, + } + } else { + pvSpec = corev1.PersistentVolumeSpec{ + StorageClassName: dummyStorageClassName, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + GCEPersistentDisk: &corev1.GCEPersistentDiskVolumeSource{ + PDName: tt.volumeID, + }, + }, + } + } + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.volumeName, + Annotations: map[string]string{}, + }, + Spec: pvSpec, + } + k8sClient = fake.NewSimpleClientset(pv) + volumeID, tags, err := processPersistentVolumeClaim(pvc) + if (err == nil) == tt.wantedErr { + t.Errorf("processPersistentVolumeClaim() err = %v, wantedErr %v", err, tt.wantedErr) + } + if volumeID != tt.wantedVolumeID { + t.Errorf("processPersistentVolumeClaim() volumeID = %v, want %v", volumeID, tt.wantedVolumeID) + } + if !reflect.DeepEqual(tags, tt.wantedTags) { + t.Errorf("processPersistentVolumeClaim() tags = %v, want %v", tags, tt.wantedTags) + } + }) + } } func Test_templatedTags(t *testing.T) { - pvc := &corev1.PersistentVolumeClaim{} pvc.SetName("my-pvc") pvc.SetNamespace("my-namespace") diff --git a/main.go b/main.go index 3194eb6..5b913e3 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,8 @@ var ( watchNamespace string tagFormat string = "json" allowAllTags bool + cloud string + copyLabels []string promActionsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "k8s_pvc_tagger_actions_total", @@ -87,6 +89,11 @@ var ( }) ) +const ( + AWS = "aws" + GCP = "gcp" +) + func init() { if logFormatEnv == "" || strings.ToLower(logFormatEnv) == "json" { log.SetFormatter(&log.JSONFormatter{}) @@ -110,6 +117,7 @@ func init() { } func main() { + var err error var kubeconfig string var kubeContext string var region string @@ -119,6 +127,7 @@ func main() { var defaultTagsString string var statusPort string var metricsPort string + var copyLabelsString string flag.StringVar(&kubeconfig, "kubeconfig", "", "absolute path to the kubeconfig file") flag.StringVar(&kubeContext, "context", "", "the context to use") @@ -133,6 +142,8 @@ func main() { flag.StringVar(&statusPort, "status-port", "8000", "The healthz port") flag.StringVar(&metricsPort, "metrics-port", "8001", "The prometheus metrics port") flag.BoolVar(&allowAllTags, "allow-all-tags", false, "Whether or not to allow any tag, even Kubernetes assigned ones, to be set") + flag.StringVar(&cloud, "cloud", AWS, "The cloud provider (aws or gcp)") + flag.StringVar(©LabelsString, "copy-labels", "", "Comma-separated list of PVC labels to copy to volumes. Use '*' to copy all labels. (default \"\")") flag.Parse() if leaseLockName == "" { @@ -145,6 +156,35 @@ func main() { } } + switch cloud { + case AWS: + log.Infoln("Running in AWS mode") + // Parse AWS_REGION environment variable. + if len(region) == 0 { + region, _ = getMetadataRegion() + log.WithFields(log.Fields{"region": region}).Debugln("ec2Metadata region") + } + ok, err := regexp.Match(regexpAWSRegion, []byte(region)) + if err != nil { + log.Fatalln("Failed to parse AWS_REGION:", err.Error()) + } + if !ok { + log.Fatalln("Given AWS_REGION does not match AWS Region format.") + } + awsSession = createAWSSession(region) + if awsSession == nil { + err = fmt.Errorf("nil AWS session: %v", awsSession) + if err != nil { + log.Println(err.Error()) + } + os.Exit(1) + } + case GCP: + log.Infoln("Running in GCP mode") + default: + log.Fatalln("Cloud provider must be either aws or gcp") + } + defaultTags = make(map[string]string) if defaultTagsString != "" { log.Debugln("defaultTagsString:", defaultTagsString) @@ -159,25 +199,9 @@ func main() { } log.WithFields(log.Fields{"tags": defaultTags}).Infoln("Default Tags") - // Parse AWS_REGION environment variable. - if len(region) == 0 { - region, _ = getMetadataRegion() - log.WithFields(log.Fields{"region": region}).Debugln("ec2Metadata region") - } - ok, err := regexp.Match(regexpAWSRegion, []byte(region)) - if err != nil { - log.Fatalln("Failed to parse AWS_REGION:", err.Error()) - } - if !ok { - log.Fatalln("Given AWS_REGION does not match AWS Region format.") - } - awsSession = createAWSSession(region) - if awsSession == nil { - err = fmt.Errorf("nil AWS session: %v", awsSession) - if err != nil { - log.Println(err.Error()) - } - os.Exit(1) + if copyLabelsString != "" { + copyLabels = parseCopyLabels(copyLabelsString) + log.Infof("Copying PVC labels to tags: %v", copyLabels) } k8sClient, err = BuildClient(kubeconfig, kubeContext) @@ -286,7 +310,6 @@ func main() { }, }, }) - } func statusHandler(w http.ResponseWriter, r *http.Request) { @@ -305,7 +328,6 @@ func statusHandler(w http.ResponseWriter, r *http.Request) { } func runWatchNamespaceTask(ctx context.Context, namespace string) { - // Make the informer's channel here so we can close it when the // context is Done() ch := make(chan struct{}) @@ -316,7 +338,6 @@ func runWatchNamespaceTask(ctx context.Context, namespace string) { } func parseCsv(value string) map[string]string { - tags := make(map[string]string) for _, s := range strings.Split(value, ",") { if len(s) == 0 { @@ -338,3 +359,13 @@ func parseCsv(value string) map[string]string { return tags } + +func parseCopyLabels(copyLabelsString string) []string { + if copyLabelsString == "*" { + return []string{"*"} + } + // remove empty strings from final list, eg: "foo,,bar" -> ["foo" "bar"]: + return strings.FieldsFunc(copyLabelsString, func(c rune) bool { + return c == ',' + }) +} diff --git a/main_test.go b/main_test.go index d1105d1..136e528 100644 --- a/main_test.go +++ b/main_test.go @@ -73,3 +73,49 @@ func Test_parseCsv(t *testing.T) { }) } } + +func Test_parseCopyLabels(t *testing.T) { + tests := []struct { + name string + copyLabelsString string + want []string + }{ + { + name: "copy all labels", + copyLabelsString: "*", + want: []string{"*"}, + }, + { + name: "copy one label", + copyLabelsString: "foo", + want: []string{"foo"}, + }, + { + name: "copy multiple labels", + copyLabelsString: "foo,bar", + want: []string{"foo", "bar"}, + }, + { + name: "empty values in list", + copyLabelsString: "foo,bar", + want: []string{"foo", "bar"}, + }, + { + name: "copy no labels", + copyLabelsString: "", + want: []string{}, + }, + { + name: "empty values in list are removed", + copyLabelsString: "foo,,bar", + want: []string{"foo", "bar"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := parseCopyLabels(tt.copyLabelsString); !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseCopyLabels() = %v, want %v", got, tt.want) + } + }) + } +}