diff --git a/Containerfiles/etcd-backup/Containerfile b/Containerfiles/etcd-backup/Containerfile index 899b0086..f33097ae 100644 --- a/Containerfiles/etcd-backup/Containerfile +++ b/Containerfiles/etcd-backup/Containerfile @@ -6,13 +6,13 @@ ENV ETCDCTL_ENDPOINTS "https://127.0.0.1:2379" ENV ETCDCTL_CACERT "/etc/ssl/etcd/ssl/ca.crt" ENV ETCDCTL_KEY "/etc/ssl/etcd/ssl/healthcheck-client.key" ENV ETCDCTL_CERT "/etc/ssl/etcd/ssl/healthcheck-client.crt" -ENV S3_ENDPOINT "10.74.8.135" -ENV S3_PORT "8081" +ENV S3_HOST "https://swift.api.sjc3.rackspacecloud.com" +ENV S3_REGION "SJC3" ENV S3_ACCESS_KEY "abcd" ENV S3_SECRET_KEY "abcd" RUN apk add --update --no-cache bash ca-certificates tzdata openssl -RUN pip install boto +RUN pip install boto3 botocore RUN wget https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \ && tar xzf etcd-${ETCD_VERSION}-linux-amd64.tar.gz \ diff --git a/Containerfiles/etcd-backup/backup.py b/Containerfiles/etcd-backup/backup.py index 8324c4f3..09d95eb6 100644 --- a/Containerfiles/etcd-backup/backup.py +++ b/Containerfiles/etcd-backup/backup.py @@ -1,7 +1,8 @@ import os import sys -import boto -import boto.s3.connection +import boto3 +from botocore.client import Config +from botocore.exceptions import NoCredentialsError, PartialCredentialsError, ClientError def get_env_variables(): @@ -9,65 +10,114 @@ def get_env_variables(): access_key = os.getenv("ACCESS_KEY") secret_key = os.getenv("SECRET_KEY") host = os.getenv("S3_HOST") + region = os.getenv("S3_REGION", "SJC3") - port_str = os.getenv("S3_PORT", "8081") - try: - port = int(port_str) - except ValueError: - raise ValueError( - f"Environment variable 'S3_PORT' has an invalid value: {port_str}" + if not all([access_key, secret_key, host]): + print( + "Error: Missing one or more environment variables: ACCESS_KEY, SECRET_KEY, S3_HOST" ) + sys.exit(1) - # Properly convert the 'S3_HOST_SSL' environment variable to a boolean - secure_str = os.getenv("S3_HOST_SSL", "false").lower() - secure = secure_str in ["true", "1", "t", "y", "yes"] - - return access_key, secret_key, host, port, secure + return access_key, secret_key, host, region -def create_s3_connection(access_key, secret_key, host, port, secure): - """Create S3 connection.""" - conn = boto.connect_s3( - aws_access_key_id=access_key, - aws_secret_access_key=secret_key, - host=host, - port=port, - is_secure=secure, - calling_format=boto.s3.connection.OrdinaryCallingFormat(), - ) - return conn +def create_s3_connection(access_key, secret_key, host, region): + """Create S3 connection using Boto3 with error handling.""" + try: + s3_client = boto3.client( + "s3", + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + endpoint_url=host, + region_name=region, + config=Config(signature_version="s3v4"), # Swift often requires SigV4 + ) + # Test connection + s3_client.list_buckets() + return s3_client + except NoCredentialsError: + print("Error: No credentials provided.") + except PartialCredentialsError: + print("Error: Incomplete credentials provided.") + except ClientError as e: + print(f"Client error while creating connection: {e}") + except Exception as e: + print(f"Unexpected error creating S3 connection: {e}") + + return None + + +def upload_file_to_bucket(conn, file_to_upload, bucket_name): + """Upload a file to a specific S3 bucket using Boto3.""" + try: + # Check if the bucket exists + conn.head_bucket(Bucket=bucket_name) + except ClientError as e: + print(f"Bucket '{bucket_name}' does not exist or is not accessible.") + return + key = os.path.basename(file_to_upload) -def upload_file_to_bucket(conn, file_to_upload): - """Upload a file to a specific S3 bucket.""" - bucket_name = "etcd-backup-bucket" - bucket = conn.get_bucket(bucket_name) - key = bucket.new_key(os.path.basename(file_to_upload)) - key.set_contents_from_filename(file_to_upload) + try: + conn.upload_file(file_to_upload, bucket_name, key) + print(f"File '{file_to_upload}' uploaded successfully to {bucket_name}/{key}.") + except FileNotFoundError: + print(f"Error: File '{file_to_upload}' not found.") + except ClientError as e: + print(f"Client error during upload: {e}") + except Exception as e: + print(f"Unexpected error uploading file: {e}") def list_all_buckets(conn): - """List all buckets.""" - for bucket in conn.get_all_buckets(): - print( - "{name}\t{created}".format( - name=bucket.name, - created=bucket.creation_date, - ) - ) + """List all buckets using Boto3.""" + try: + response = conn.list_buckets() + for bucket in response["Buckets"]: + print(f"{bucket['Name']}\t{bucket['CreationDate']}") + except ClientError as e: + print(f"Client error listing buckets: {e}") + except Exception as e: + print(f"Unexpected error listing buckets: {e}") + + +def create_bucket_if_not_exists(conn, bucket_name): + """Create the bucket if it does not exist.""" + try: + conn.head_bucket(Bucket=bucket_name) + except ClientError as e: + error_code = e.response["Error"]["Code"] + if error_code == "404": + try: + conn.create_bucket(Bucket=bucket_name) + print(f"Bucket '{bucket_name}' created successfully.") + except ClientError as create_error: + print(f"Client error creating bucket: {create_error}") + except Exception as create_error: + print(f"Unexpected error creating bucket: {create_error}") + else: + print(f"Error accessing bucket '{bucket_name}': {e}") + except Exception as e: + print(f"Unexpected error checking bucket: {e}") def main(): """Main function.""" if len(sys.argv) != 2: - print("Usage: python your_script.py ") + print("Usage: python backup.py ") sys.exit(1) file_to_upload = sys.argv[1] - access_key, secret_key, host, port, secure = get_env_variables() - conn = create_s3_connection(access_key, secret_key, host, port, secure) - upload_file_to_bucket(conn, file_to_upload) - list_all_buckets(conn) + access_key, secret_key, host, region = get_env_variables() + conn = create_s3_connection(access_key, secret_key, host, region) + + if not conn: + print("Failed to create S3 connection.") + sys.exit(1) + + create_bucket_if_not_exists(conn, "etcd-backups") + upload_file_to_bucket(conn, file_to_upload, "etcd-backups") + # list_all_buckets(conn) if __name__ == "__main__": diff --git a/base-kustomize/backups/etcd/etcd-backup.yaml b/base-kustomize/backups/etcd/etcd-backup.yaml index 5a1936d4..3fa7f0ee 100644 --- a/base-kustomize/backups/etcd/etcd-backup.yaml +++ b/base-kustomize/backups/etcd/etcd-backup.yaml @@ -3,7 +3,7 @@ kind: CronJob metadata: name: etcd-backup spec: - schedule: "0 */3 * * *" + schedule: "0 1 * * *" successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 concurrencyPolicy: Allow @@ -13,7 +13,7 @@ spec: spec: initContainers: - name: etcd-backup - image: csengteam/etcd-backup:v0.0.4 + image: csengteam/etcd-backup:v0.0.5 env: - name: ETCDCTL_API valueFrom: @@ -48,10 +48,10 @@ spec: readOnly: true - mountPath: /data/etcd-backup name: etcd-backup - restartPolicy: OnFailure + restartPolicy: Never containers: - name: backup-to-s3 - image: csengteam/etcd-backup:v0.0.4 + image: csengteam/etcd-backup:v0.0.5 env: - name: ACCESS_KEY valueFrom: @@ -68,11 +68,11 @@ spec: secretKeyRef: name: etcd-backup-secrets key: S3_HOST - - name: S3_PORT + - name: S3_REGION valueFrom: secretKeyRef: name: etcd-backup-secrets - key: S3_PORT + key: S3_REGION command: ["/bin/bash", "-c"] args: ["python /backup.py /data/etcd-backup/etcd-snapshot-$(date +%Y-%m-%d).db"] volumeMounts: diff --git a/docs/etcd-backup.md b/docs/etcd-backup.md index a96fddd1..9845c60c 100644 --- a/docs/etcd-backup.md +++ b/docs/etcd-backup.md @@ -27,7 +27,7 @@ kubectl --namespace openstack \ --from-literal=ACCESS_KEY="sadbq4bcva2392dasflkdsp" \ --from-literal=SECRET_KEY="aldskflkjpoq32ibdsfko23bnalkfdao2" \ --from-literal=S3_HOST="127.0.0.1" \ - --from-literal=S3_PORT="8081" \ + --from-literal=S3_REGION="SJC3" \ --from-literal=ETCDCTL_API="3" \ --from-literal=ETCDCTL_ENDPOINTS="https://127.0.0.1:2379" \ --from-literal=ETCDCTL_CACERT="/etc/ssl/etcd/ssl/ca.pem" \