diff --git a/contrib/mixins/README.md b/contrib/mixins/README.md new file mode 100644 index 00000000..82ff9273 --- /dev/null +++ b/contrib/mixins/README.md @@ -0,0 +1,65 @@ +# Common mixins library + +A library of reusable mixins to simplify common tasks needed when taking backups. + +Currently, this library contains mixins for: +* Dumping databases to restic stdin (mysql, mariadb, pgsql) +* Taking temporary snapshots (lvm, btrfs) +* Temporarily freezing VM disk images (virsh/kvm) + +## Example Usage + +`profiles.toml` + +```toml +version = "2" + +# downloaded by "https://raw.githubusercontent.com/creativeprojects/resticprofile/master/contrib/mixins/get.sh" +includes = ["mixins-*.yaml"] + +[profiles.__base] +repo = "..." + +[proflies.default] +inherit = "__base" +[proflies.default.backup] +use = {name = "snapshot-btrfs", FROM = "/opt/data", TO = "/opt/data_snapshot"} +source = "/opt/data_snapshot" + +[proflies.mysql] +inherit = "__base" +[proflies.mysql.backup] +use = {name = "database-mysql", DATABASE = "dbname", USER="dbuser", PASSWORD_FILE="/path/to/password.txt"} + +[proflies.vms] +inherit = "__base" +[proflies.vms.backup] +use = [ + {name = "snapshot-virsh", DOMAIN = "vmname1", DUMPXML = "/opt/vms/vmname1.xml"}, + {name = "snapshot-virsh", DOMAIN = "vmname2", DUMPXML = "/opt/vms/vmname2.xml"}, + {name = "snapshot-virsh", DOMAIN = "vmname2", DUMPXML = "/opt/vms/vmname2.xml"}, +] +source = "/opt/vms/" +includes = ["*.qcow2", "*.xml"] +``` + +## Setup + +### Posix environment: + +```shell +cd /etc/resticprofile \ +&& curl -sL https://raw.githubusercontent.com/creativeprojects/resticprofile/master/contrib/mixins/get.sh | sh - +``` + +### Powershell environment: + +At the moment the mixins library doesn't support powershell. Contributions are welcome. + +## Disclaimer + +Please note that the actions that some of the mixins perform can lead to data loss. This lies in the nature of +creating and removing snapshots with system privileges. You should carefully read through the actions inside +the mixins and consider if the mixin is safe for your use case. Please report bugs when you find any. + +This library is under the [GPL3](../../LICENSE) license like all other parts of the project. diff --git a/contrib/mixins/database.yaml b/contrib/mixins/database.yaml new file mode 100644 index 00000000..d6c8ba69 --- /dev/null +++ b/contrib/mixins/database.yaml @@ -0,0 +1,107 @@ +--- +title: "Mixins for database backups in posix shell environments" +github_repo: "github.com/creativeprojects/resticprofile" +license: "GPL3" +copyright: |- + This file is part of resticprofile (github.com/creativeprojects/resticprofile). + Copyright (c) 2024 resticprofile authors. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +--- +mixins: + + ### + # MySQL / MariaDB dump from stdin + # + # Usage + # + # backup: + # use: {name: "database-mysql", DATABASE: "dbname", HOST: "dbhost", USER: "dbuser", PASSWORD_FILE: "dbpassword.txt"} + # + database-mysql: + default-vars: + HOST: "${MYSQLDUMP_HOST}" + PORT: "${MYSQLDUMP_PORT}" + USER: "${MYSQLDUMP_USER}" + PASSWORD: "${MYSQLDUMP_PASSWORD}" + PASSWORD_FILE: "${MYSQLDUMP_PASSWORD_FILE}" + DATABASE: "--all-databases" + FILENAME: "mysql-dump.sql" + DEFAULT_OPTS: "${MYSQLDUMP_DEFAULT_OPTS:-'--order-by-primary'}" + OPTS: "" + + # creating a defaults-file for mysqldump if the PASSWORD is non-empty + run-before...: | + defaults="" + __pw="${PASSWORD}" + if [ -n "${__pw}" ] || [ -n "${PASSWORD_FILE}" ] ; then + defaults="{{ tempFile("mysql-default.conf") }}" + chmod 0600 "$defaults" + fi + [ -f "$defaults" ] && cat > "$defaults" <<-EOF + [mysqldump] + password = "${__pw:-"$(cat "${PASSWORD_FILE}")"}" + EOF + echo "MYSQL_DEFAULTS_FILE=${defaults}" >> "{{ env }}" + + # defining stdin command to that writes the mysql dump + stdin-filename: "${FILENAME}" + stdin-command: > + host="$( [ -z "${HOST}" ] || echo "--host='${HOST}'" )" + port="$( [ -z "${PORT}" ] || echo "--port='${PORT}'" )" + user="$( [ -z "${USER}" ] || echo "-u '${USER}'" )" + pass="$( [ -z "${MYSQL_DEFAULTS_FILE}" ] || echo "--defaults-file='${MYSQL_DEFAULTS_FILE}'" )" + mysqldump $user $pass $host $port ${DEFAULT_OPTS} ${OPTS} ${DATABASE} + source: [] + + + ### + # Postgres dump from stdin + # + # Usage + # + # backup: + # use: {name: "database-pgsql", DATABASE: "dbname", HOST: "dbhost", USER: "dbuser", PASSWORD_FILE: "dbpassword.txt"} + # + database-pgsql: + default-vars: + HOST: "${PGDUMP_HOST}" + PORT: "${PGDUMP_PORT}" + USER: "${PGDUMP_USER}" + PASSWORD: "${PGDUMP_PASSWORD}" + PASSWORD_FILE: "${PGDUMP_PASSWORD_FILE}" + DATABASE: "" + FILENAME: "pgsql-dump.sql" + DEFAULT_OPTS: "${PGDUMP_DEFAULT_OPTS:-'--clean --if-exists --quote-all-identifiers --serializable-deferrable'}" + OPTS: "" + + # creating a .pgpass file for pg_dump if the PASSWORD is non-empty + run-before...: | + pgpass="" + __pw="${PASSWORD}" + if [ -n "${__pw}" ] || [ -n "${PASSWORD_FILE}" ] ; then + pgpass="{{ tempFile(".pgpass") }}" + echo "*:*:*:*:${__pw:-"$(cat "${PASSWORD_FILE}")"}" > "$pgpass" + chmod 0600 "$pgpass" + fi + echo "PGPASS_FILE=${pgpass}" >> "{{ env }}" + + # defining stdin command to that writes the mysql dump + stdin-filename: "${FILENAME}" + stdin-command: > + [ -z "${PGPASS_FILE}" ] || export HOME=$(dirname "${PGPASS_FILE}") + ; host="$( [ -z "${HOST}" ] || echo "--host='${HOST}'" )" + port="$( [ -z "${PORT}" ] || echo "--port='${PORT}'" )" + user="$( [ -z "${USER}" ] || echo "--username='${USER}'" )" + pg_dump $user --no-password $host $port ${DEFAULT_OPTS} ${OPTS} ${DATABASE} + source: [] diff --git a/contrib/mixins/get.sh b/contrib/mixins/get.sh new file mode 100755 index 00000000..7efd16b6 --- /dev/null +++ b/contrib/mixins/get.sh @@ -0,0 +1,50 @@ +#!/bin/sh +set -e + +MIXINS=" + database + snapshot +" + +PREFIX="${RESTICPROFILE_MIXINS_PREFIX:-"mixins"}" +TEMP_FILE="${TMPDIR:-/tmp}/.rp-mixin.tmp" +GIT_BRANCH="${RESTICPROFILE_BRANCH:-"master"}" +GIT_BASE_URL="${RESTICPROFILE_GIT_URL:-"https://raw.githubusercontent.com/creativeprojects/resticprofile/${GIT_BRANCH}/contrib/mixins"}" + +move_download_to() { + [ -s "$TEMP_FILE" ] && mv -f "$TEMP_FILE" "$1" + return $? +} + +download() { + result=0 + if which -s curl ; then + curl -fsL "$1" > "$TEMP_FILE" && move_download_to "$2" + result=$? + elif which -s wget ; then + wget -nv -O "$TEMP_FILE" "$1" && move_download_to "$2" + result=$? + else + echo "neither curl nor wget found, cannot load $1" + result=1 + fi + + [ -e "$TEMP_FILE" ] && rm "$TEMP_FILE" + return $result +} + +download_all() { + dir="" + if [ -n "$1" ] && [ -d "$1" ] ; then + dir="$1/" + echo "downloading to $dir" + fi + for m in $MIXINS ; do + url="$GIT_BASE_URL/${m}.yaml" + dest="${dir}${PREFIX}-${m}.yaml" + echo "getting $url > $dest" + download "$url" "$dest" || echo "failed" + done +} + +download_all "$1" \ No newline at end of file diff --git a/contrib/mixins/snapshot.yaml b/contrib/mixins/snapshot.yaml new file mode 100644 index 00000000..eb270432 --- /dev/null +++ b/contrib/mixins/snapshot.yaml @@ -0,0 +1,151 @@ +--- +title: "Mixins for snapshot creation in posix shell environments" +github_repo: "github.com/creativeprojects/resticprofile" +license: "GPL3" +copyright: |- + This file is part of resticprofile (github.com/creativeprojects/resticprofile). + Copyright (c) 2024 resticprofile authors. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +--- +mixins: + ## + # Creates a temporary snapshot of a btrfs volume + # Usage + # + # backup: + # use: + # - {name: "snapshot-btrfs", FROM: "/opt/data", TO: "/opt/data_snapshot"} + # source: "/opt/data_snapshot" + # + snapshot-btrfs: + default-vars: + FROM: "" + TO: "" + IF_EXISTS: "fail" # one of (delete, continue, fail) + + ...run-before: | + if [ ! -d "${FROM}" ] ; then echo "source volume (${FROM}) is not existing" ; exit 1 ; fi + if [ -z "${TO}" ] ; then echo "snapshot destination is not specified" ; exit 1 ; fi + if [ "${TO}" == "${FROM}" ] ; then echo "snapshot source & destination must differ" ; exit 1 ; fi + ; + if [ -d "${TO}" ] ; then + if [ "${IF_EXISTS}" == "delete" ] ; then + btrfs subvolume delete "${TO}" + elif [ "${IF_EXISTS}" == "continue" ] ; then + echo "${TO} already existing, continuing without creating a new snapshot" ; exit 0 + else + echo "${TO} already existing, snapshot failed" ; exit 1 + fi + fi + ; + btrfs subvolume snapshot -r "${FROM}" "${TO}" \ + && echo "btrfs:${FROM}:${TO}--" >> "{{ tempFile "mixins-lib-snapshots.list" }}" + + run-finally...: | + if [ -d "${TO}" ] && grep -q "btrfs:${FROM}:${TO}--" "{{ tempFile "mixins-lib-snapshots.list" }}" ; then + btrfs subvolume delete "${TO}" + fi + + ## + # Creates a temporary snapshot of a lvm volume + # Usage + # + # backup: + # use: + # - {name: "snapshot-lvm", FROM: "/dev/vg00/data", TO: "/mnt/data_snapshot"} + # source: "/mnt/data_snapshot" + # + snapshot-lvm: + default-vars: + FROM: "" + TO: "" + SNAPSHOT_NAME: "" + DEFAULT_OPTS: "-l100%FREE" + OPTS: "" + + ...run-before: | + if [ ! -e "${FROM}" ] ; then echo "source volume (${FROM}) is not existing" ; exit 1 ; fi + if [ -z "${TO}" ] ; then echo "snapshot destination is not specified" ; exit 1 ; fi + if [ "${TO}" == "${FROM}" ] ; then echo "snapshot source & destination must differ" ; exit 1 ; fi + ( [ -d "${TO}" ] || mkdir -p "${TO}" ) || exit 1 + ; + export snap_name="$( [ -z "${SNAPSHOT_NAME}" ] || echo "${SNAPSHOT_NAME}" )" + snap_name="restic_${snap_name:-"$(basename "${TO}")"}" + export snap_dev="$(dirname "${FROM}")/${snap_name}" + ; + lvcreate ${DEFAULT_OPTS} ${OPTS} --name "${snap_name}" --snapshot "${FROM}" \ + && mount "${snap_dev}" "${TO}" \ + && echo "lvm:${snap_dev}:${TO}--" >> "{{ tempFile "mixins-lib-snapshots.list" }}" + + run-finally...: | + export snap_name="$( [ -z "${SNAPSHOT_NAME}" ] || echo "${SNAPSHOT_NAME}" )" + snap_name="restic_${snap_name:-"$(basename "${TO}")"}" + export snap_dev="$(dirname "${FROM}")/${snap_name}" + ; + if [ -d "${TO}" ] && grep -q "lvm:${snap_dev}:${TO}--" "{{ tempFile "mixins-lib-snapshots.list" }}" ; then + umount "${TO}" \ + && lvremove --force "${snap_dev}" + fi + + + ## + # Temporarily freezes the image of a VM managed by virsh so that it can be backed-up while the keeps VM running + # Usage + # + # backup: + # use: + # - {name: "snapshot-virsh", DOMAIN: "vmname", DUMPXML: "/opt/vms/vmname-definition.xml"} + # - {name: "snapshot-virsh", DOMAIN: "vm2", DUMPXML: "/opt/vms/vm2-definition.xml"} + # - {name: "snapshot-virsh", DOMAIN: "vm3"} + # - {name: "snapshot-virsh", DOMAIN: "vm4"} + # - {name: "snapshot-virsh", DOMAIN: "vm4-without-quest-additions", OPTS: ""} + # # OPTIONAL: - "snapshot-virsh-aa-teardown" # place last if snapshot restores block with apparmor) + # source: + # - /opt/vms + # + snapshot-virsh: + default-vars: + DOMAIN: "" + DUMPXML: "" + LD_PATH: "/var/db" + LD_SUFFIX: "livedata.qcow2" + LD_DISCSPEC: "vda" + SNAPSHOT_SUFFIX: "restic-backup" + DEFAULT_OPTS: "--atomic --no-metadata" + OPTS: "--quiesce" # (fsync in guest OS, needs guest additions, highly recommended) + + excludes...: + - "*-${LD_SUFFIX}" + + ...run-before: > + virsh snapshot-create-as --domain "${DOMAIN}" --name "${DOMAIN}-${SNAPSHOT_SUFFIX}" + --diskspec "${LD_DISCSPEC},file=${LD_PATH}/${DOMAIN}-${LD_SUFFIX}" + --disk-only ${DEFAULT_OPTS} ${OPTS} + && echo "virsh:${LD_PATH}/${DOMAIN}-${LD_SUFFIX}--" >> "{{ tempFile "mixins-lib-snapshots.list" }}" + && ( [ -z "${DUMPXML}" ] + || virsh dumpxml --domain "${DOMAIN}" --inactive --migratable > "${DUMPXML}" ) + + run-finally...: > + grep -q "virsh:${LD_PATH}/${DOMAIN}-${LD_SUFFIX}--" "{{ tempFile "mixins-lib-snapshots.list" }}" + && virsh blockcommit --domain "${DOMAIN}" ${LD_DISCSPEC} --wait --active + && virsh blockjob --domain "${DOMAIN}" "${LD_PATH}/${DOMAIN}-${LD_SUFFIX}" --pivot + && rm -f "${LD_PATH}/${DOMAIN}-${LD_SUFFIX}" + + # + # Utility to teardown apparmor before other run finals (e.g. blockcommit / blockjob) and restart it afterward + # See "snapshot-virsh" + # + snapshot-virsh-aa-teardown: + ...run-finally: aa-teardown + run-finally...: service apparmor restart diff --git a/docs/content/configuration/examples.md b/docs/content/configuration/examples.md index 62e952fb..7292af17 100644 --- a/docs/content/configuration/examples.md +++ b/docs/content/configuration/examples.md @@ -619,3 +619,49 @@ mysql { {{% /tab %}} {{< /tabs >}} + +## Using the common mixins library + +The following example shows how the [common mixins library](https://github.com/creativeprojects/resticprofile/tree/master/contrib/mixins) can be used to apply common tasks related to backups. + +{{< tabs groupid="config" >}} +{{% tab title="toml" %}} + +```toml +version = "2" + +# downloaded by "https://raw.githubusercontent.com/creativeprojects/resticprofile/master/contrib/mixins/get.sh" +includes = ["mixins-*.yaml"] + +[profiles.__base] +repository = "/repo" +password-file = "key" + +[proflies.default] +inherit = "__base" +[proflies.default.backup] +use = {name = "snapshot-btrfs", FROM = "/opt/data", TO = "/opt/data_snapshot"} +source = "/opt/data_snapshot" +``` +{{% /tab %}} +{{% tab title="yaml" %}} + +```yaml +version: "2" + +# downloaded by "https://raw.githubusercontent.com/creativeprojects/resticprofile/master/contrib/mixins/get.sh" +includes: ["mixins-*.yaml"] + +profiles: + __base: + repository: "/repo" + password-file: "key" + + default: + inherit: "__base" + backup: + use: {name: "snapshot-btrfs", FROM: "/opt/data", TO: "/opt/data_snapshot"} + source: "/opt/data_snapshot" +``` +{{% /tab %}} +{{< /tabs >}} diff --git a/docs/content/configuration/templates.md b/docs/content/configuration/templates.md index 9d7a64bb..f313248a 100644 --- a/docs/content/configuration/templates.md +++ b/docs/content/configuration/templates.md @@ -600,6 +600,7 @@ resticprofile supports the following set of own functions in all templates: * `{{ with list "A" "B" "C" "D" | map "key" }} {{ .key | join "-" }} {{ end }}` => ` A-B-C-D ` * `{{ tempDir }}` => `/tmp/resticprofile.../t` - unique OS specific existing temporary directory * `{{ tempFile "filename" }}` => `/tmp/resticprofile.../t/filename` - unique OS specific existing temporary file +* `{{ privateTempFile "filename" }}` => `/tmp/resticprofile.../t/filename` - similar to `tempFile` but ensures that file is accessible by the user that started resticprofile only. Not supported in all OS, fails with a parse error when unsupported. * `{{ env }}` => `/tmp/resticprofile.../t/profile.env` - unique OS specific existing temporary file that is added to the current profile env-files list All `{{ temp* }}` functions guarantee that returned temporary directories and files are existing & writable. diff --git a/util/templates/functions.go b/util/templates/functions.go index b5ac9768..5f4c9a3d 100644 --- a/util/templates/functions.go +++ b/util/templates/functions.go @@ -45,25 +45,26 @@ import ( // - {{ tempFile "filename" }} => "/path/to/unique-tempdir/filename" func TemplateFuncs(funcs ...map[string]any) (templateFuncs map[string]any) { templateFuncs = map[string]any{ - "contains": func(search any, src any) bool { return strings.Contains(toString(src), toString(search)) }, - "matches": func(ptn string, src any) bool { return mustCompile(ptn).MatchString(toString(src)) }, - "replace": func(old, new, src string) string { return strings.ReplaceAll(src, old, new) }, - "replaceR": func(ptn, repl, src string) string { return mustCompile(ptn).ReplaceAllString(src, repl) }, - "lower": strings.ToLower, - "upper": strings.ToUpper, - "trim": strings.TrimSpace, - "trimPrefix": func(prefix, src string) string { return strings.TrimPrefix(src, prefix) }, - "trimSuffix": func(suffix, src string) string { return strings.TrimSuffix(src, suffix) }, - "split": func(sep, src string) []any { return collect.From(strings.Split(src, sep), toAny[string]) }, - "splitR": func(ptn, src string) []any { return collect.From(mustCompile(ptn).Split(src, -1), toAny[string]) }, - "join": func(sep string, src []any) string { return strings.Join(collect.From(src, toString), sep) }, - "list": func(args ...any) []any { return args }, - "map": toMap, - "base64": func(src any) string { return base64.StdEncoding.EncodeToString([]byte(toString(src))) }, - "hex": func(src any) string { return hex.EncodeToString([]byte(toString(src))) }, - "tempDir": TempDir, - "tempFile": TempFile, - "env": func() string { return TempFile(".env.none") }, // satisfies the {{env}} interface w.o. functionality + "contains": func(search any, src any) bool { return strings.Contains(toString(src), toString(search)) }, + "matches": func(ptn string, src any) bool { return mustCompile(ptn).MatchString(toString(src)) }, + "replace": func(old, new, src string) string { return strings.ReplaceAll(src, old, new) }, + "replaceR": func(ptn, repl, src string) string { return mustCompile(ptn).ReplaceAllString(src, repl) }, + "lower": strings.ToLower, + "upper": strings.ToUpper, + "trim": strings.TrimSpace, + "trimPrefix": func(prefix, src string) string { return strings.TrimPrefix(src, prefix) }, + "trimSuffix": func(suffix, src string) string { return strings.TrimSuffix(src, suffix) }, + "split": func(sep, src string) []any { return collect.From(strings.Split(src, sep), toAny[string]) }, + "splitR": func(ptn, src string) []any { return collect.From(mustCompile(ptn).Split(src, -1), toAny[string]) }, + "join": func(sep string, src []any) string { return strings.Join(collect.From(src, toString), sep) }, + "list": func(args ...any) []any { return args }, + "map": toMap, + "base64": func(src any) string { return base64.StdEncoding.EncodeToString([]byte(toString(src))) }, + "hex": func(src any) string { return hex.EncodeToString([]byte(toString(src))) }, + "tempDir": TempDir, + "tempFile": TempFile, + "privateTempFile": MustPrivateTempFile, + "env": func() string { return TempFile(".env.none") }, // satisfies the {{env}} interface w.o. functionality } // aliases @@ -140,6 +141,15 @@ func PrivateTempFile(name string) (filename string, err error) { return } +// MustPrivateTempFile returns a strictly private temp file or panics if this is not supported (e.g. on Windows) +func MustPrivateTempFile(name string) string { + if filename, err := PrivateTempFile(name); err == nil { + return filename + } else { + panic(fmt.Errorf("failed creating private file %q (may be unsupported by this OS): %w", filename, err)) + } +} + // EnvFileReceiverFunc declares the backend interface for the "{{env}}" template function type EnvFileReceiverFunc func() (profileKey string, receiveFile func(string)) diff --git a/util/templates/functions_test.go b/util/templates/functions_test.go index 5170fad0..aa3fcd2c 100644 --- a/util/templates/functions_test.go +++ b/util/templates/functions_test.go @@ -22,6 +22,7 @@ func TestTemplateFuncs(t *testing.T) { tests := []struct { template, expected string + panics bool }{ {template: `{{ "some string" | contains "some" }}`, expected: `true`}, {template: `{{ "some string" | contains "else" }}`, expected: `false`}, @@ -52,6 +53,7 @@ func TestTemplateFuncs(t *testing.T) { {template: `{{ tempDir }}`, expected: dir}, // constant results when repeated {template: `{{ tempFile "test.txt" }}`, expected: file}, {template: `{{ tempFile "test.txt" }}`, expected: file}, // constant results when repeated + {template: `{{ privateTempFile "test.txt" }}`, expected: file, panics: platform.IsWindows()}, {template: `{{ env }}`, expected: TempFile(".env.none")}, {template: `{{ "a & b\n" | html }}`, expected: "a & b\n"}, {template: `{{ "a & b\n" | urlquery }}`, expected: "a+%26+b%0A"}, @@ -74,9 +76,13 @@ func TestTemplateFuncs(t *testing.T) { require.NotNil(t, tpl) buffer.Reset() - err = tpl.Execute(buffer, nil) - assert.NoError(t, err) - assert.Equal(t, test.expected, buffer.String()) + if test.panics { + assert.Panics(t, func() { _ = tpl.Execute(buffer, nil) }) + } else { + err = tpl.Execute(buffer, nil) + assert.NoError(t, err) + assert.Equal(t, test.expected, buffer.String()) + } }) }