contrib: added the common mixins library

contrib/mixins/README.md

@@ -0,0 +1,65 @@
+# Common mixins library
+
+A library of reusable mixins to simplify common tasks needed when taking backups.
+
+Currently, this library contains mixins for:
+* Dumping databases to restic stdin (mysql, mariadb, pgsql)
+* Taking temporary snapshots (lvm, btrfs)
+* Temporarily freezing VM disk images (virsh/kvm)
+
+## Example Usage
+
+`profiles.toml`
+
+```toml
+version = "2"
+
+# downloaded by "https://raw.githubusercontent.com/creativeprojects/resticprofile/master/contrib/mixins/get.sh" 
+includes = ["mixins-*.yaml"]
+
+[profiles.__base]
+repo = "..."
+
+[proflies.default]
+inherit = "__base"
+[proflies.default.backup]
+use = {name = "snapshot-btrfs", FROM = "/opt/data", TO = "/opt/data_snapshot"}
+source = "/opt/data_snapshot"
+
+[proflies.mysql]
+inherit = "__base"
+[proflies.mysql.backup]
+use = {name = "database-mysql", DATABASE = "dbname", USER="dbuser", PASSWORD_FILE="/path/to/password.txt"}
+
+[proflies.vms]
+inherit = "__base"
+[proflies.vms.backup]
+use = [
+    {name = "snapshot-virsh", DOMAIN = "vmname1", DUMPXML = "/opt/vms/vmname1.xml"},
+    {name = "snapshot-virsh", DOMAIN = "vmname2", DUMPXML = "/opt/vms/vmname2.xml"},
+    {name = "snapshot-virsh", DOMAIN = "vmname2", DUMPXML = "/opt/vms/vmname2.xml"},
+]
+source = "/opt/vms/"
+includes = ["*.qcow2", "*.xml"]
+```
+
+## Setup
+
+### Posix environment:
+
+```shell
+cd /etc/resticprofile \
+&& curl -sL https://raw.githubusercontent.com/creativeprojects/resticprofile/master/contrib/mixins/get.sh | sh -
+```
+
+### Powershell environment:
+
+At the moment the mixins library doesn't support powershell. Contributions are welcome.
+
+## Disclaimer
+
+Please note that the actions that some of the mixins perform can lead to data loss. This lies in the nature of  
+creating and removing snapshots with system privileges. You should carefully read through the actions inside 
+the mixins and consider if the mixin is safe for your use case. Please report bugs when you find any.
+
+This library is under the [GPL3](../../LICENSE) license like all other parts of the project.

contrib/mixins/database.yaml

@@ -0,0 +1,107 @@
+---
+title: "Mixins for database backups in posix shell environments"
+github_repo: "github.com/creativeprojects/resticprofile"
+license: "GPL3"
+copyright: |-
+  This file is part of resticprofile (github.com/creativeprojects/resticprofile).
+  Copyright (c) 2024 resticprofile authors.
+
+  This program is free software: you can redistribute it and/or modify  
+  it under the terms of the GNU General Public License as published by  
+  the Free Software Foundation, version 3.
+
+  This program is distributed in the hope that it will be useful, but 
+  WITHOUT ANY WARRANTY; without even the implied warranty of 
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License 
+  along with this program. If not, see <http://www.gnu.org/licenses/>.
+---
+mixins:
+
+  ###
+  # MySQL / MariaDB dump from stdin
+  #
+  # Usage
+  #
+  #   backup:
+  #     use: {name: "database-mysql", DATABASE: "dbname", HOST: "dbhost", USER: "dbuser", PASSWORD_FILE: "dbpassword.txt"}
+  #
+  database-mysql:
+    default-vars:
+      HOST: "${MYSQLDUMP_HOST}"
+      PORT: "${MYSQLDUMP_PORT}"
+      USER: "${MYSQLDUMP_USER}"
+      PASSWORD: "${MYSQLDUMP_PASSWORD}"
+      PASSWORD_FILE: "${MYSQLDUMP_PASSWORD_FILE}"
+      DATABASE: "--all-databases"
+      FILENAME: "mysql-dump.sql"
+      DEFAULT_OPTS: "${MYSQLDUMP_DEFAULT_OPTS:-'--order-by-primary'}"
+      OPTS: ""
+
+    # creating a defaults-file for mysqldump if the PASSWORD is non-empty
+    run-before...: |
+      defaults=""
+      __pw="${PASSWORD}"
+      if [ -n "${__pw}" ] || [ -n "${PASSWORD_FILE}" ] ; then 
+        defaults="{{ tempFile("mysql-default.conf") }}"
+        chmod 0600 "$defaults"
+      fi
+      [ -f "$defaults" ] && cat > "$defaults" <<-EOF
+          [mysqldump]
+          password = "${__pw:-"$(cat "${PASSWORD_FILE}")"}"
+      EOF
+      echo "MYSQL_DEFAULTS_FILE=${defaults}" >> "{{ env }}"
+
+    # defining stdin command to that writes the mysql dump
+    stdin-filename: "${FILENAME}"
+    stdin-command: >
+      host="$( [ -z "${HOST}" ] || echo "--host='${HOST}'" )"
+      port="$( [ -z "${PORT}" ] || echo "--port='${PORT}'" )"
+      user="$( [ -z "${USER}" ] || echo "-u '${USER}'" )"
+      pass="$( [ -z "${MYSQL_DEFAULTS_FILE}" ] || echo "--defaults-file='${MYSQL_DEFAULTS_FILE}'" )"
+      mysqldump $user $pass $host $port ${DEFAULT_OPTS} ${OPTS} ${DATABASE}
+    source: []
+
+
+  ###
+  # Postgres dump from stdin
+  #
+  # Usage
+  #
+  #   backup:
+  #     use: {name: "database-pgsql", DATABASE: "dbname", HOST: "dbhost", USER: "dbuser", PASSWORD_FILE: "dbpassword.txt"}
+  #
+  database-pgsql:
+    default-vars:
+      HOST: "${PGDUMP_HOST}"
+      PORT: "${PGDUMP_PORT}"
+      USER: "${PGDUMP_USER}"
+      PASSWORD: "${PGDUMP_PASSWORD}"
+      PASSWORD_FILE: "${PGDUMP_PASSWORD_FILE}"
+      DATABASE: ""
+      FILENAME: "pgsql-dump.sql"
+      DEFAULT_OPTS: "${PGDUMP_DEFAULT_OPTS:-'--clean --if-exists --quote-all-identifiers --serializable-deferrable'}"
+      OPTS: ""
+
+    # creating a .pgpass file for pg_dump if the PASSWORD is non-empty
+    run-before...: |
+      pgpass=""
+      __pw="${PASSWORD}"
+      if [ -n "${__pw}" ] || [ -n "${PASSWORD_FILE}" ] ; then 
+        pgpass="{{ tempFile(".pgpass") }}"
+        echo "*:*:*:*:${__pw:-"$(cat "${PASSWORD_FILE}")"}" > "$pgpass"
+        chmod 0600 "$pgpass" 
+      fi
+      echo "PGPASS_FILE=${pgpass}" >> "{{ env }}" 
+
+    # defining stdin command to that writes the mysql dump
+    stdin-filename: "${FILENAME}"
+    stdin-command: >
+      [ -z "${PGPASS_FILE}" ] || export HOME=$(dirname "${PGPASS_FILE}") 
+      ; host="$( [ -z "${HOST}" ] || echo "--host='${HOST}'" )"
+        port="$( [ -z "${PORT}" ] || echo "--port='${PORT}'" )"
+        user="$( [ -z "${USER}" ] || echo "--username='${USER}'" )"
+        pg_dump $user --no-password $host $port ${DEFAULT_OPTS} ${OPTS} ${DATABASE}
+    source: []

contrib/mixins/get.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+set -e
+
+MIXINS="
+  database
+  snapshot
+"
+
+PREFIX="${RESTICPROFILE_MIXINS_PREFIX:-"mixins"}"
+TEMP_FILE="${TMPDIR:-/tmp}/.rp-mixin.tmp"
+GIT_BRANCH="${RESTICPROFILE_BRANCH:-"master"}"
+GIT_BASE_URL="${RESTICPROFILE_GIT_URL:-"https://raw.githubusercontent.com/creativeprojects/resticprofile/${GIT_BRANCH}/contrib/mixins"}"
+
+move_download_to() {
+  [ -s "$TEMP_FILE" ] && mv -f "$TEMP_FILE" "$1"
+  return $?
+}
+
+download() {
+  result=0
+  if which -s curl ; then
+    curl -fsL "$1" > "$TEMP_FILE" && move_download_to "$2"
+    result=$?
+  elif which -s wget ; then
+    wget -nv -O "$TEMP_FILE" "$1" && move_download_to "$2"
+    result=$?
+  else
+    echo "neither curl nor wget found, cannot load $1"
+    result=1
+  fi
+
+  [ -e "$TEMP_FILE" ] && rm "$TEMP_FILE"
+  return $result
+}
+
+download_all() {
+  dir=""
+  if [ -n "$1" ] && [ -d "$1" ] ; then
+    dir="$1/"
+    echo "downloading to $dir"
+  fi
+  for m in $MIXINS ; do
+      url="$GIT_BASE_URL/${m}.yaml"
+      dest="${dir}${PREFIX}-${m}.yaml"
+      echo "getting $url > $dest"
+      download "$url" "$dest" || echo "failed"
+  done
+}
+
+download_all "$1"

contrib/mixins/snapshot.yaml

@@ -0,0 +1,151 @@
+---
+title: "Mixins for snapshot creation in posix shell environments"
+github_repo: "github.com/creativeprojects/resticprofile"
+license: "GPL3"
+copyright: |-
+  This file is part of resticprofile (github.com/creativeprojects/resticprofile).
+  Copyright (c) 2024 resticprofile authors.
+
+  This program is free software: you can redistribute it and/or modify  
+  it under the terms of the GNU General Public License as published by  
+  the Free Software Foundation, version 3.
+
+  This program is distributed in the hope that it will be useful, but 
+  WITHOUT ANY WARRANTY; without even the implied warranty of 
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License 
+  along with this program. If not, see <http://www.gnu.org/licenses/>.
+---
+mixins:
+  ##
+  # Creates a temporary snapshot of a btrfs volume
+  # Usage
+  #
+  #   backup:
+  #     use:
+  #       - {name: "snapshot-btrfs", FROM: "/opt/data", TO: "/opt/data_snapshot"}
+  #     source: "/opt/data_snapshot"
+  #
+  snapshot-btrfs:
+    default-vars:
+      FROM: ""
+      TO: ""
+      IF_EXISTS: "fail"  # one of (delete, continue, fail)
+
+    ...run-before: |
+      if [ ! -d "${FROM}" ]       ; then echo "source volume (${FROM}) is not existing" ; exit 1 ; fi 
+      if [ -z "${TO}" ]           ; then echo "snapshot destination is not specified" ; exit 1 ; fi
+      if [ "${TO}" == "${FROM}" ] ; then echo "snapshot source & destination must differ" ; exit 1 ; fi
+      ;
+      if [ -d "${TO}" ] ; then
+        if [ "${IF_EXISTS}" == "delete" ] ; then
+          btrfs subvolume delete "${TO}"
+        elif [ "${IF_EXISTS}" == "continue" ] ; then
+          echo "${TO} already existing, continuing without creating a new snapshot" ; exit 0
+        else
+          echo "${TO} already existing, snapshot failed" ; exit 1
+        fi
+      fi
+      ;
+      btrfs subvolume snapshot -r "${FROM}" "${TO}" \
+      && echo "btrfs:${FROM}:${TO}--" >> "{{ tempFile "mixins-lib-snapshots.list" }}" 
+
+    run-finally...: |
+      if [ -d "${TO}" ] && grep -q "btrfs:${FROM}:${TO}--" "{{ tempFile "mixins-lib-snapshots.list" }}" ; then
+        btrfs subvolume delete "${TO}"
+      fi
+
+  ##
+  # Creates a temporary snapshot of a lvm volume
+  # Usage
+  #
+  #   backup:
+  #     use:
+  #       - {name: "snapshot-lvm", FROM: "/dev/vg00/data", TO: "/mnt/data_snapshot"}
+  #     source: "/mnt/data_snapshot"
+  #
+  snapshot-lvm:
+    default-vars:
+      FROM: ""
+      TO: ""
+      SNAPSHOT_NAME: ""
+      DEFAULT_OPTS: "-l100%FREE"
+      OPTS: ""
+
+    ...run-before: |
+      if [ ! -e "${FROM}" ]       ; then echo "source volume (${FROM}) is not existing" ; exit 1 ; fi 
+      if [ -z "${TO}" ]           ; then echo "snapshot destination is not specified" ; exit 1 ; fi 
+      if [ "${TO}" == "${FROM}" ] ; then echo "snapshot source & destination must differ" ; exit 1 ; fi
+      ( [ -d "${TO}" ] || mkdir -p "${TO}" ) || exit 1
+      ;
+      export snap_name="$( [ -z "${SNAPSHOT_NAME}" ] || echo "${SNAPSHOT_NAME}" )"
+      snap_name="restic_${snap_name:-"$(basename "${TO}")"}"
+      export snap_dev="$(dirname "${FROM}")/${snap_name}"
+      ;
+      lvcreate ${DEFAULT_OPTS} ${OPTS} --name "${snap_name}" --snapshot "${FROM}" \
+      && mount "${snap_dev}" "${TO}" \
+      && echo "lvm:${snap_dev}:${TO}--" >> "{{ tempFile "mixins-lib-snapshots.list" }}"
+
+    run-finally...: |
+      export snap_name="$( [ -z "${SNAPSHOT_NAME}" ] || echo "${SNAPSHOT_NAME}" )" 
+      snap_name="restic_${snap_name:-"$(basename "${TO}")"}" 
+      export snap_dev="$(dirname "${FROM}")/${snap_name}"
+      ;
+      if [ -d "${TO}" ] && grep -q "lvm:${snap_dev}:${TO}--" "{{ tempFile "mixins-lib-snapshots.list" }}" ; then 
+        umount "${TO}" \
+        && lvremove --force "${snap_dev}" 
+      fi
+
+
+  ##
+  # Temporarily freezes the image of a VM managed by virsh so that it can be backed-up while the keeps VM running
+  # Usage
+  #
+  #   backup:
+  #     use:
+  #       - {name: "snapshot-virsh", DOMAIN: "vmname", DUMPXML: "/opt/vms/vmname-definition.xml"}
+  #       - {name: "snapshot-virsh", DOMAIN: "vm2", DUMPXML: "/opt/vms/vm2-definition.xml"}
+  #       - {name: "snapshot-virsh", DOMAIN: "vm3"}
+  #       - {name: "snapshot-virsh", DOMAIN: "vm4"}
+  #       - {name: "snapshot-virsh", DOMAIN: "vm4-without-quest-additions", OPTS: ""}
+  #       # OPTIONAL: - "snapshot-virsh-aa-teardown" # place last if snapshot restores block with apparmor)
+  #     source:
+  #       - /opt/vms
+  #
+  snapshot-virsh:
+    default-vars:
+      DOMAIN: ""
+      DUMPXML: ""
+      LD_PATH: "/var/db"
+      LD_SUFFIX: "livedata.qcow2"
+      LD_DISCSPEC: "vda"
+      SNAPSHOT_SUFFIX: "restic-backup"
+      DEFAULT_OPTS: "--atomic --no-metadata"
+      OPTS: "--quiesce" # (fsync in guest OS, needs guest additions, highly recommended)
+
+    excludes...:
+      - "*-${LD_SUFFIX}"
+
+    ...run-before: >
+      virsh snapshot-create-as --domain "${DOMAIN}" --name "${DOMAIN}-${SNAPSHOT_SUFFIX}"
+        --diskspec "${LD_DISCSPEC},file=${LD_PATH}/${DOMAIN}-${LD_SUFFIX}"
+        --disk-only ${DEFAULT_OPTS} ${OPTS}
+      && echo "virsh:${LD_PATH}/${DOMAIN}-${LD_SUFFIX}--" >> "{{ tempFile "mixins-lib-snapshots.list" }}"
+      && ( [ -z "${DUMPXML}" ] 
+           || virsh dumpxml --domain "${DOMAIN}" --inactive --migratable > "${DUMPXML}" ) 
+
+    run-finally...: >
+      grep -q "virsh:${LD_PATH}/${DOMAIN}-${LD_SUFFIX}--" "{{ tempFile "mixins-lib-snapshots.list" }}"
+      && virsh blockcommit --domain "${DOMAIN}" ${LD_DISCSPEC} --wait --active
+      && virsh blockjob --domain "${DOMAIN}" "${LD_PATH}/${DOMAIN}-${LD_SUFFIX}" --pivot
+      && rm -f "${LD_PATH}/${DOMAIN}-${LD_SUFFIX}"
+
+  #
+  # Utility to teardown apparmor before other run finals (e.g. blockcommit / blockjob) and restart it afterward
+  # See "snapshot-virsh"
+  #
+  snapshot-virsh-aa-teardown:
+    ...run-finally: aa-teardown
+    run-finally...: service apparmor restart