Skip to content

Commit 8710394

Browse files
devanshjain7devanshjain7
committed
add azure driver
1 parent c372b6d commit 8710394

File tree

3 files changed

+222
-0
lines changed

3 files changed

+222
-0
lines changed

‎metadata/pkg/drivers/azure/azure.go

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
// Copyright 2023 The SODA Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package azure
16+
17+
import (
18+
"context"
19+
"sync"
20+
21+
log "github.com/sirupsen/logrus"
22+
23+
"github.com/Azure/azure-storage-blob-go/azblob"
24+
backendpb "github.com/opensds/multi-cloud/backend/proto"
25+
"github.com/opensds/multi-cloud/metadata/pkg/db"
26+
"github.com/opensds/multi-cloud/metadata/pkg/model"
27+
pb "github.com/opensds/multi-cloud/metadata/proto"
28+
)
29+
30+
type AzureAdapter struct {
31+
backend *backendpb.BackendDetail
32+
serviceURL azblob.ServiceURL
33+
}
34+
35+
var ObjectList = func(containerURL azblob.ContainerURL, bucketName string) ([]*model.MetaObject, int64, error) {
36+
37+
var numObjects int = 0
38+
var totSize int64 = 0
39+
objectArray := []*model.MetaObject{}
40+
41+
blobResponse, err := containerURL.ListBlobsFlatSegment(context.Background(), azblob.Marker{}, azblob.ListBlobsSegmentOptions{})
42+
43+
if err != nil {
44+
log.Fatal(err)
45+
}
46+
47+
for _, blob := range blobResponse.Segment.BlobItems {
48+
numObjects += 1
49+
blobURL := containerURL.NewBlobURL(blob.Name)
50+
obj := &model.MetaObject{}
51+
objectArray = append(objectArray, obj)
52+
obj.ObjectName = blob.Name
53+
obj.LastModifiedDate = &blob.Properties.LastModified
54+
obj.Size = *blob.Properties.ContentLength
55+
totSize += obj.Size
56+
obj.StorageClass = string(blob.Properties.AccessTier)
57+
58+
if blob.Properties.DestinationSnapshot != nil {
59+
obj.RedirectLocation = *blob.Properties.DestinationSnapshot
60+
}
61+
62+
obj.ReplicationStatus = string(blob.Properties.CopyStatus)
63+
64+
if blob.Properties.ContentType != nil {
65+
obj.ObjectType = *blob.Properties.ContentType
66+
}
67+
props, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{})
68+
if err != nil {
69+
log.Error("get properties failed", err)
70+
}
71+
72+
obj.Metadata = props.NewMetadata()
73+
}
74+
75+
return objectArray, totSize, nil
76+
77+
}
78+
79+
func GetBucketMeta(idx int, container azblob.ContainerItem, serviceURL azblob.ServiceURL, bucketArray []*model.MetaBucket, wg *sync.WaitGroup) {
80+
81+
defer wg.Done()
82+
83+
buck := &model.MetaBucket{}
84+
bucketArray[idx] = buck
85+
buck.Name = container.Name
86+
buck.CreationDate = &container.Properties.LastModified
87+
88+
containerURL := serviceURL.NewContainerURL(buck.Name)
89+
objectArray, totalSize, err := ObjectList(containerURL, buck.Name)
90+
91+
if err != nil {
92+
return
93+
}
94+
95+
buck.Objects = objectArray
96+
buck.NumberOfObjects = len(objectArray)
97+
buck.TotalSize = totalSize
98+
99+
bucketArray[idx] = buck
100+
101+
props, err := containerURL.GetProperties(context.Background(), azblob.LeaseAccessConditions{})
102+
103+
if err != nil {
104+
log.Errorf("failed to get bucket tags. failed wth error: %v", err)
105+
} else {
106+
buck.BucketTags = props.NewMetadata()
107+
}
108+
109+
acl, err := containerURL.GetAccessPolicy(context.Background(), azblob.LeaseAccessConditions{})
110+
111+
if err != nil {
112+
log.Errorf("unable to get bucket Acl. failed with error: %v", err)
113+
} else {
114+
access := []*model.Access{}
115+
116+
for _, item := range acl.Items {
117+
acc := &model.Access{}
118+
acc.ID = item.ID
119+
acc.Permission = item.AccessPolicy.Permission
120+
access = append(access, acc)
121+
}
122+
buck.BucketAcl = access
123+
}
124+
}
125+
126+
func BucketList(serviceURL azblob.ServiceURL) ([]*model.MetaBucket, error) {
127+
response, err := serviceURL.ListContainersSegment(context.Background(), azblob.Marker{}, azblob.ListContainersSegmentOptions{})
128+
129+
if err != nil {
130+
log.Errorf("unable to list buckets. failed with err: %v", err)
131+
}
132+
133+
bucketArray := make([]*model.MetaBucket, len(response.ContainerItems))
134+
135+
wg := sync.WaitGroup{}
136+
for idx, container := range response.ContainerItems {
137+
wg.Add(1)
138+
go GetBucketMeta(idx, container, serviceURL, bucketArray, &wg)
139+
}
140+
wg.Wait()
141+
142+
return bucketArray, nil
143+
}
144+
145+
func (ad *AzureAdapter) SyncMetadata(ctx context.Context, in *pb.SyncMetadataRequest) error {
146+
147+
buckArr, err := BucketList(ad.serviceURL)
148+
if err != nil {
149+
log.Errorf("metadata collection for backend id: %v failed with error: %v", ad.backend.Id, err)
150+
return err
151+
}
152+
153+
metaBackend := model.MetaBackend{}
154+
metaBackend.Id = ad.backend.Id
155+
metaBackend.BackendName = ad.backend.Name
156+
metaBackend.Type = ad.backend.Type
157+
metaBackend.Region = ad.backend.Region
158+
metaBackend.Buckets = buckArr
159+
metaBackend.NumberOfBuckets = int32(len(buckArr))
160+
newContext := context.TODO()
161+
err = db.DbAdapter.CreateMetadata(newContext, metaBackend)
162+
163+
return err
164+
}
165+
166+
func (ad *AzureAdapter) DownloadObject() {
167+
168+
}

‎metadata/pkg/drivers/azure/factory.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Copyright 2023 The SODA Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package azure
16+
17+
import (
18+
"net/url"
19+
20+
"github.com/Azure/azure-storage-blob-go/azblob"
21+
"github.com/opensds/multi-cloud/backend/pkg/utils/constants"
22+
backendpb "github.com/opensds/multi-cloud/backend/proto"
23+
driver "github.com/opensds/multi-cloud/metadata/pkg/drivers/cloudfactory"
24+
)
25+
26+
type AzureBlobDriverFactory struct {
27+
}
28+
29+
func (factory *AzureBlobDriverFactory) CreateDriver(backend *backendpb.BackendDetail) (driver.CloudDriver, error) {
30+
endpoint := backend.Endpoint
31+
AccessKeyID := backend.Access
32+
AccessKeySecret := backend.Security
33+
34+
credential, err := azblob.NewSharedKeyCredential(AccessKeyID, AccessKeySecret)
35+
36+
if err != nil {
37+
return nil, err
38+
}
39+
40+
p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
41+
42+
u, _ := url.Parse(endpoint)
43+
44+
serviceURL := azblob.NewServiceURL(*u, p)
45+
46+
adap := &AzureAdapter{backend: backend, serviceURL: serviceURL}
47+
48+
return adap, nil
49+
}
50+
51+
func init() {
52+
driver.RegisterDriverFactory(constants.BackendTypeAzure, &AzureBlobDriverFactory{})
53+
}

‎metadata/pkg/drivers/init.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,6 @@ package drivers
22

33
import (
44
_ "github.com/opensds/multi-cloud/metadata/pkg/drivers/aws"
5+
_ "github.com/opensds/multi-cloud/metadata/pkg/drivers/azure"
56
_ "github.com/opensds/multi-cloud/metadata/pkg/drivers/gcp"
67
)

0 commit comments

Comments
 (0)