Merge pull request #122 from ursais/max

[IMP] backup: Support different platform for filestore, backup and remote.

Author: max3903

backup/entrypoint.sh

@@ -7,7 +7,6 @@ set -e
 
 # Environment variables
 : ${RUNNING_ENV:='dev'}
-: ${PLATFORM:='do'}
 # PostgreSQL
 : ${PGHOST:='localhost'}
 : ${PGPORT:=5432}
@@ -16,18 +15,45 @@ set -e
 : ${PGPASSWORD:='postgres'}
 : ${PGDEFAULTDB:='postgres'}
 : ${PGSSLMODE:='prefer'}
-# AWS / DO
-: ${AWS_HOST:='false'}
-: ${AWS_REGION:='false'}
-: ${AWS_ACCESS_KEY_ID:='false'}
-: ${AWS_SECRET_ACCESS_KEY:='false'}
-: ${AWS_BUCKETNAME:='false'}
-# Azure
-: ${AZURE_STORAGE_ACCOUNT_URL:='false'}
+# Filestore
+: ${FILESTORE_PLATFORM:='aws'}
+## AWS / DO
+: ${FILESTORE_AWS_HOST:='false'}
+: ${FILESTORE_AWS_REGION:='false'}
+: ${FILESTORE_AWS_ACCESS_KEY_ID:='false'}
+: ${FILESTORE_AWS_SECRET_ACCESS_KEY:='false'}
+: ${FILESTORE_AWS_BUCKETNAME:='false'}
+## Azure
+: ${FILESTORE_AZURE_STORAGE_ACCOUNT_URL:='false'}
+# BACKUP
+: ${BACKUP_PLATFORM:='aws'}
+## AWS / DO
+: ${BACKUP_AWS_HOST:='false'}
+: ${BACKUP_AWS_REGION:='false'}
+: ${BACKUP_AWS_ACCESS_KEY_ID:='false'}
+: ${BACKUP_AWS_SECRET_ACCESS_KEY:='false'}
+: ${BACKUP_AWS_BUCKETNAME:='false'}
+## Azure
+: ${BACKUP_AZURE_STORAGE_ACCOUNT_URL:='false'}
+# REMOTE
+: ${REMOTE_ENABLED:='false'}
+: ${REMOTE_PLATFORM:='aws'}
+## AWS / DO
+: ${REMOTE_AWS_HOST:='false'}
+: ${REMOTE_AWS_REGION:='false'}
+: ${REMOTE_AWS_ACCESS_KEY_ID:='false'}
+: ${REMOTE_AWS_SECRET_ACCESS_KEY:='false'}
+: ${REMOTE_AWS_BUCKETNAME:='false'}
+## Azure
+: ${REMOTE_AZURE_STORAGE_ACCOUNT_URL:='false'}
+
+export FILESTORE_BUCKET=`echo $FILESTORE_AWS_BUCKETNAME | sed -e "s/{db}/$PGDATABASE/g"`
+export BACKUP_BUCKET=`echo $BACKUP_AWS_BUCKETNAME | sed -e "s/{db}/$PGDATABASE/g"`
+[ "$REMOTE_ENABLED" == "true" ] && export REMOTE_BUCKET=`echo $REMOTE_AWS_BUCKETNAME | sed -e "s/{db}/$PGDATABASE/g"`
 # Date in UTC
 export TODAY=$(date -u +%Y%m%d)
 export YESTERDAY=$(date -d "1 day ago" -u +%Y%m%d)
-export LASTWEEK=$(date -d "1 week ago" -u +%Y%m%d)
+export LASTMONTH=$(date -d "1 month ago" -u +%Y%m%d)
 # For dockerize
 export TEMPLATES=/templates
 
@@ -36,21 +62,51 @@ function config_rclone() {
   echo "Configure rclone"
   mkdir -p $HOME/.config/rclone
   dockerize -template $TEMPLATES/rclone.conf.tmpl:$HOME/.config/rclone/rclone.conf
-  case "$PLATFORM" in
+  [ "$DEBUG" == "1" ] && echo "Rclone configuration file:" && cat $HOME/.config/rclone/rclone.conf
+  # FILESTORE
+  case "$FILESTORE_PLATFORM" in
+    "aws")
+      ;;
+    "azure")
+      ;;
+    "do")
+      export FILESTORE_SPACE=`echo $FILESTORE_AWS_HOST | sed -e "s/.$FILESTORE_AWS_REGION.*$//"`
+      ;;
+    *)
+      echo "I don't know how to configure rclone for $FILESTORE_PLATFORM."
+      exit 1
+      ;;
+  esac
+  # BACKUP
+  case "$BACKUP_PLATFORM" in
     "aws")
-      export SPACE=""
       ;;
     "azure")
-      export SPACE=""
       ;;
     "do")
-      export SPACE=`echo $AWS_HOST | sed -e "s/.$AWS_REGION.*$//"`
+      export BACKUP_SPACE=`echo $BACKUP_AWS_HOST | sed -e "s/.$BACKUP_AWS_REGION.*$//"`
       ;;
     *)
-      echo "I don't know how to configure rclone for $PLATFORM."
+      echo "I don't know how to configure rclone for $BACKUP_PLATFORM."
       exit 1
       ;;
   esac
+  # REMOTE
+  if [ "$REMOTE_ENABLED" == "true" ]; then
+    case "$REMOTE_PLATFORM" in
+      "aws")
+        ;;
+      "azure")
+        ;;
+      "do")
+        export REMOTE_SPACE=`echo $REMOTE_AWS_HOST | sed -e "s/.$REMOTE_AWS_REGION.*$//"`
+        ;;
+      *)
+        echo "I don't know how to configure rclone for $REMOTE_PLATFORM."
+        exit 1
+        ;;
+    esac
+  fi
 }
 
 # Common functions
@@ -62,6 +118,7 @@ function restore_odoo_database() {
  UPDATE ir_cron SET active = 'f';
  UPDATE ir_mail_server SET active = 'f';
  UPDATE fetchmail_server SET active = 'f';
+ UPDATE ir_config_parameter SET value = 'null' WHERE key = 'database.uuid';
  " $PGDATABASE
 }
 
@@ -71,13 +128,20 @@ function backup() {
     "odoo")
       echo "Dump the database $PGDATABASE"
       pg_dump --clean $PGDATABASE | gzip > /tmp/$RUNNING_ENV-$PGDATABASE-$TODAY.sql.gz
-      echo "Push it to the container"
-      rclone copy /tmp/$RUNNING_ENV-$PGDATABASE-$TODAY.sql.gz remote:/$SPACE/backup/
-      echo "Duplicate the filestore"
-      rclone sync remote:/$SPACE/$RUNNING_ENV-$PGDATABASE/ remote:/$SPACE/backup/$RUNNING_ENV-$PGDATABASE-$TODAY/
-      echo "Cleanup last week backup"
-      rclone purge remote:/$SPACE/backup/$RUNNING_ENV-$PGDATABASE-$LASTWEEK/
-      rclone purge remote:/$SPACE/backup/$RUNNING_ENV-$PGDATABASE-$LASTWEEK.sql.gz
+      echo "Push it to backup"
+      rclone copy /tmp/$RUNNING_ENV-$PGDATABASE-$TODAY.sql.gz backup:/$BACKUP_SPACE/$BACKUP_BUCKET/
+      echo "Sync the filestore to backup"
+      rclone sync filestore:/$FILESTORE_SPACE/$FILESTORE_BUCKET/ backup:/$BACKUP_SPACE/$BACKUP_BUCKET/$RUNNING_ENV-$PGDATABASE-$TODAY/
+      echo "Cleanup last month copy on backup"
+      rclone purge backup:/$BACKUP_SPACE/$BACKUP_BUCKET/$RUNNING_ENV-$PGDATABASE-$LASTMONTH/
+      rclone purge backup:/$BACKUP_SPACE/$BACKUP_BUCKET/$RUNNING_ENV-$PGDATABASE-$LASTMONTH.sql.gz
+      if [ $REMOTE_ENABLED == 'true' ]; then
+        echo "Push, sync and cleanup to/on remote"
+        rclone copy /tmp/$RUNNING_ENV-$PGDATABASE-$TODAY.sql.gz remote:/$REMOTE_SPACE/$REMOTE_BUCKET/
+        rclone sync filestore:/$FILESTORE_SPACE/$FILESTORE_BUCKET/ remote:/$REMOTE_SPACE/$REMOTE_BUCKET/$RUNNING_ENV-$PGDATABASE-$TODAY/
+        rclone purge remote:/$REMOTE_SPACE/$REMOTE_BUCKET/$RUNNING_ENV-$PGDATABASE-$LASTMONTH/
+        rclone purge remote:/$REMOTE_SPACE/$REMOTE_BUCKET/$RUNNING_ENV-$PGDATABASE-$LASTMONTH.sql.gz
+      fi
       ;;
     *)
       echo "Backup profile does not exist. I don't know how to backup $1."
@@ -93,12 +157,10 @@ function restore() {
       dropdb --if-exists $PGDATABASE
       echo "Create $PGDATABASE database"
       createdb $PGDATABASE
-      echo "Delete current $PGDATABASE filestore"
-      rclone purge remote:/$SPACE/$RUNNING_ENV-$PGDATABASE/
       echo "Download yesterday's backup"
-      rclone copy remote:/$SPACE/backup/production-master-$YESTERDAY.sql.gz /tmp/
-      echo "Copy the filestore"
-      rclone sync remote:/$SPACE/backup/production-master-$YESTERDAY/ remote:/$SPACE/$RUNNING_ENV-$PGDATABASE/
+      rclone copy backup:/$BACKUP_SPACE/$BACKUP_BUCKET/production-master-$YESTERDAY.sql.gz /tmp/
+      echo "Sync the filestore"
+      rclone sync backup:/$BACKUP_SPACE/$BACKUP_BUCKET/production-master-$YESTERDAY/ filestore:/$FILESTORE_SPACE/$FILESTORE_BUCKET/
       echo "Restore database dump"
       restore_odoo_database
       ;;
@@ -109,6 +171,8 @@ function restore() {
   esac
 }
 
+[ "$DEBUG" == "1" ] && env | sort
+
 config_rclone
 
 $1 $2

backup/templates/rclone.conf.tmpl

@@ -1,24 +1,82 @@
-{{ if contains "aws" .Env.PLATFORM }}
-[remote]
+{{ if contains "aws" .Env.FILESTORE_PLATFORM }}
+[filestore]
+type = s3
+provider = AWS
+env_auth = false
+access_key_id = {{ default "" .Env.FILESTORE_AWS_ACCESS_KEY_ID }}
+secret_access_key = {{ default "" .Env.FILESTORE_AWS_SECRET_ACCESS_KEY }}
+region = {{ default "" .Env.FILESTORE_AWS_REGION }}
+location_constraint = {{ default "" .Env.FILESTORE_AWS_REGION }}
+acl = private
+{{ end }}
+{{ if contains "azure" .Env.FILESTORE_PLATFORM }}
+[filestore]
+type = azureblob
+sas_url = {{ default "" .Env.FILESTORE_AZURE_STORAGE_ACCOUNT_URL }}
+{{ end }}
+{{ if contains "do" .Env.FILESTORE_PLATFORM }}
+[filestore]
+type = s3
+provider = DigitalOcean
+env_auth = true
+access_key_id = {{ default "" .Env.FILESTORE_AWS_ACCESS_KEY_ID }}
+secret_access_key = {{ default "" .Env.FILESTORE_AWS_SECRET_ACCESS_KEY }}
+endpoint = {{ default "" .Env.FILESTORE_AWS_REGION }}.digitaloceanspaces.com
+acl = private
+{{ end }}
+
+{{ if contains "aws" .Env.BACKUP_PLATFORM }}
+[backup]
 type = s3
 provider = AWS
+env_auth = false
+access_key_id = {{ default "" .Env.BACKUP_AWS_ACCESS_KEY_ID }}
+secret_access_key = {{ default "" .Env.BACKUP_AWS_SECRET_ACCESS_KEY }}
+region = {{ default "" .Env.BACKUP_AWS_REGION }}
+location_constraint = {{ default "" .Env.BACKUP_AWS_REGION }}
+acl = private
+{{ end }}
+{{ if contains "azure" .Env.BACKUP_PLATFORM }}
+[backup]
+type = azureblob
+sas_url = {{ default "" .Env.BACKUP_AZURE_STORAGE_ACCOUNT_URL }}
+{{ end }}
+{{ if contains "do" .Env.BACKUP_PLATFORM }}
+[backup]
+type = s3
+provider = DigitalOcean
 env_auth = true
-access_key_id = {{ default "" .Env.AWS_ACCESS_KEY_ID }}
-secret_access_key = {{ default "" .Env.AWS_SECRET_ACCESS_KEY }}
+access_key_id = {{ default "" .Env.BACKUP_AWS_ACCESS_KEY_ID }}
+secret_access_key = {{ default "" .Env.BACKUP_AWS_SECRET_ACCESS_KEY }}
+endpoint = {{ default "" .Env.BACKUP_AWS_REGION }}.digitaloceanspaces.com
 acl = private
 {{ end }}
-{{ if contains "azure" .Env.PLATFORM }}
+
+{{ if .Env.REMOTE_ENABLED }}
+{{ if contains "aws" .Env.REMOTE_PLATFORM }}
+[remote]
+type = s3
+provider = AWS
+env_auth = false
+access_key_id = {{ default "" .Env.REMOTE_AWS_ACCESS_KEY_ID }}
+secret_access_key = {{ default "" .Env.REMOTE_AWS_SECRET_ACCESS_KEY }}
+region = {{ default "" .Env.REMOTE_AWS_REGION }}
+location_constraint = {{ default "" .Env.REMOTE_AWS_REGION }}
+acl = private
+{{ end }}
+{{ if contains "azure" .Env.REMOTE_PLATFORM }}
 [remote]
 type = azureblob
-sas_url = {{ default "" .Env.AZURE_STORAGE_ACCOUNT_URL }}
+sas_url = {{ default "" .Env.REMOTE_AZURE_STORAGE_ACCOUNT_URL }}
 {{ end }}
-{{ if contains "do" .Env.PLATFORM }}
+{{ if contains "do" .Env.REMOTE_PLATFORM }}
 [remote]
 type = s3
 provider = DigitalOcean
 env_auth = true
-access_key_id = {{ default "" .Env.AWS_ACCESS_KEY_ID }}
-secret_access_key = {{ default "" .Env.AWS_SECRET_ACCESS_KEY }}
-endpoint = {{ default "" .Env.AWS_REGION }}.digitaloceanspaces.com
+access_key_id = {{ default "" .Env.REMOTE_AWS_ACCESS_KEY_ID }}
+secret_access_key = {{ default "" .Env.REMOTE_AWS_SECRET_ACCESS_KEY }}
+endpoint = {{ default "" .Env.REMOTE_AWS_REGION }}.digitaloceanspaces.com
 acl = private
 {{ end }}
+{{ end }}