add support oracle oci object storage (langgenius#5616)

Author: hymvp

api/.env.example

@@ -72,6 +72,13 @@ TENCENT_COS_SECRET_ID=your-secret-id
 TENCENT_COS_REGION=your-region
 TENCENT_COS_SCHEME=your-scheme
 
+# OCI Storage configuration
+OCI_ENDPOINT=your-endpoint
+OCI_BUCKET_NAME=your-bucket-name
+OCI_ACCESS_KEY=your-access-key
+OCI_SECRET_KEY=your-secret-key
+OCI_REGION=your-region
+
 # CORS configuration
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*

api/configs/middleware/__init__.py

@@ -7,6 +7,7 @@
 from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
 from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
 from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
+from configs.middleware.storage.oci_storage_config import OCIStorageConfig
 from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
 from configs.middleware.vdb.chroma_config import ChromaConfig
 from configs.middleware.vdb.milvus_config import MilvusConfig
@@ -167,6 +168,7 @@ class MiddlewareConfig(
     GoogleCloudStorageConfig,
     TencentCloudCOSStorageConfig,
     S3StorageConfig,
+    OCIStorageConfig,
 
     # configs of vdb and vdb providers
     VectorStoreConfig,

api/configs/middleware/storage/oci_storage_config.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class OCIStorageConfig(BaseModel):
+    """
+    OCI storage configs
+    """
+
+    OCI_ENDPOINT: Optional[str] = Field(
+        description='OCI storage endpoint',
+        default=None,
+    )
+
+    OCI_REGION: Optional[str] = Field(
+        description='OCI storage region',
+        default=None,
+    )
+
+    OCI_BUCKET_NAME: Optional[str] = Field(
+        description='OCI storage bucket name',
+        default=None,
+    )
+
+    OCI_ACCESS_KEY: Optional[str] = Field(
+        description='OCI storage access key',
+        default=None,
+    )
+
+    OCI_SECRET_KEY: Optional[str] = Field(
+        description='OCI storage secret key',
+        default=None,
+    )
+

api/extensions/ext_storage.py

@@ -7,6 +7,7 @@
 from extensions.storage.azure_storage import AzureStorage
 from extensions.storage.google_storage import GoogleStorage
 from extensions.storage.local_storage import LocalStorage
+from extensions.storage.oci_storage import OCIStorage
 from extensions.storage.s3_storage import S3Storage
 from extensions.storage.tencent_storage import TencentStorage
 
@@ -37,6 +38,10 @@ def init_app(self, app: Flask):
             self.storage_runner = TencentStorage(
                 app=app
             )
+        elif storage_type == 'oci-storage':
+            self.storage_runner = OCIStorage(
+                app=app
+            )        
         else:
             self.storage_runner = LocalStorage(app=app)
 

api/extensions/storage/oci_storage.py

@@ -0,0 +1,64 @@
+from collections.abc import Generator
+from contextlib import closing
+
+import boto3
+from botocore.exceptions import ClientError
+from flask import Flask
+
+from extensions.storage.base_storage import BaseStorage
+
+
+class OCIStorage(BaseStorage):
+    def __init__(self, app: Flask):
+        super().__init__(app)
+        app_config = self.app.config
+        self.bucket_name = app_config.get('OCI_BUCKET_NAME')
+        self.client = boto3.client(
+                    's3',
+                    aws_secret_access_key=app_config.get('OCI_SECRET_KEY'),
+                    aws_access_key_id=app_config.get('OCI_ACCESS_KEY'),
+                    endpoint_url=app_config.get('OCI_ENDPOINT'),
+                    region_name=app_config.get('OCI_REGION')
+                )
+
+    def save(self, filename, data):
+        self.client.put_object(Bucket=self.bucket_name, Key=filename, Body=data)
+
+    def load_once(self, filename: str) -> bytes:
+        try:
+            with closing(self.client) as client:
+                data = client.get_object(Bucket=self.bucket_name, Key=filename)['Body'].read()
+        except ClientError as ex:
+            if ex.response['Error']['Code'] == 'NoSuchKey':
+                raise FileNotFoundError("File not found")
+            else:
+                raise
+        return data
+
+    def load_stream(self, filename: str) -> Generator:
+        def generate(filename: str = filename) -> Generator:
+            try:
+                with closing(self.client) as client:
+                    response = client.get_object(Bucket=self.bucket_name, Key=filename)
+                    yield from response['Body'].iter_chunks()
+            except ClientError as ex:
+                if ex.response['Error']['Code'] == 'NoSuchKey':
+                    raise FileNotFoundError("File not found")
+                else:
+                    raise
+        return generate()
+
+    def download(self, filename, target_filepath):
+        with closing(self.client) as client:
+            client.download_file(self.bucket_name, filename, target_filepath)
+
+    def exists(self, filename):
+        with closing(self.client) as client:
+            try:
+                client.head_object(Bucket=self.bucket_name, Key=filename)
+                return True
+            except:
+                return False
+
+    def delete(self, filename):
+        self.client.delete_object(Bucket=self.bucket_name, Key=filename)

docker/docker-compose.yaml

@@ -64,6 +64,11 @@ x-shared-env: &shared-api-worker-env
   TENCENT_COS_SECRET_ID: ${TENCENT_COS_SECRET_ID:-}
   TENCENT_COS_REGION: ${TENCENT_COS_REGION:-}
   TENCENT_COS_SCHEME: ${TENCENT_COS_SCHEME:-}
+  OCI_ENDPOINT: ${OCI_ENDPOINT:-}
+  OCI_BUCKET_NAME: ${OCI_BUCKET_NAME:-}
+  OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-}
+  OCI_SECRET_KEY: ${OCI_SECRET_KEY:-}
+  OCI_REGION: ${OCI_REGION:-}
   VECTOR_STORE: ${VECTOR_STORE:-weaviate}
   WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080}
   WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih}