From 293ff86b9107ce67e800ce1e54d3782e4bb7107b Mon Sep 17 00:00:00 2001 From: Juergen Kellerer Date: Fri, 13 Aug 2021 20:50:36 +0200 Subject: [PATCH] Added default config for unix packages --- contrib/posix/README.md | 40 ++++ contrib/posix/conf.d/check.conf | 58 ++++++ contrib/posix/conf.d/hooks.conf | 19 ++ contrib/posix/conf.d/metrics.conf | 15 ++ contrib/posix/conf.d/prune.conf | 32 +++ contrib/posix/conf.d/repository.conf | 67 +++++++ contrib/posix/profiles.conf | 183 ++++++++++++++++++ .../posix/profiles.d/fs-snapshot.yaml.sample | 54 ++++++ contrib/posix/profiles.d/system.toml | 38 ++++ 9 files changed, 506 insertions(+) create mode 100644 contrib/posix/README.md create mode 100644 contrib/posix/conf.d/check.conf create mode 100644 contrib/posix/conf.d/hooks.conf create mode 100644 contrib/posix/conf.d/metrics.conf create mode 100644 contrib/posix/conf.d/prune.conf create mode 100644 contrib/posix/conf.d/repository.conf create mode 100644 contrib/posix/profiles.conf create mode 100644 contrib/posix/profiles.d/fs-snapshot.yaml.sample create mode 100644 contrib/posix/profiles.d/system.toml diff --git a/contrib/posix/README.md b/contrib/posix/README.md new file mode 100644 index 00000000..a5b4e102 --- /dev/null +++ b/contrib/posix/README.md @@ -0,0 +1,40 @@ +# Default configuration for POSIX systems + +**Layout for `/etc/resticprofile`**: + +* `profiles.conf` - host centric default configuration +* `profiles.d/*` - host centric backup profiles (`*.toml` & `*.yaml`) +* `conf.d/*` - overrides & extra configuration + +The layout is used in `deb`, `rpm` and `apk` packages of `resticprofile` + +**Generated files**: +* `conf.d/default-repository.secret` - during installation, only if missing + +**Referenced files and paths**: +* `conf.d/default-repository-self-signed-pub.pem` - TLS public cert (self-signed only) +* `conf.d/default-repository-client.pem` - TLS client cert +* `/var/lib/prometheus/node-exporter/resticprofile-*.prom` - Prometheus files +* `$TMPDIR/resticprofile-*` - Status and lock files + +# Quick Start + +## Installation + +* RPM: `rpm -i "resticprofile-VERSION-ARCH.rpm"` +* DEB: `dpkg -i "resticprofile-VERSION-ARCH.deb"` + +## Configuration +Setup repository and validate system backup profile: +```shell +cd /etc/resticprofile/ +vim conf.d/repository.conf +vim profiles.d/system.toml +``` + +## Test config and backup +```shell +resticprofile -n root show +resticprofile -n root --dry-run backup +resticprofile -n root backup +``` diff --git a/contrib/posix/conf.d/check.conf b/contrib/posix/conf.d/check.conf new file mode 100644 index 00000000..e9ca1370 --- /dev/null +++ b/contrib/posix/conf.d/check.conf @@ -0,0 +1,58 @@ + +## +# Groups for profiles that check & verify repositories +# Usage +# - `resticprofile -n check-all` +# - `resticprofile -n check-all schedule` +# - `resticprofile -n check-all unschedule` +# - `resticprofile -n verify-all` +# - `resticprofile -n verify-all schedule` +# - `resticprofile -n verify-all unschedule` +[groups] +check-all = [ "check" ] +verify-all = [ "verify" ] + + +## +# Profile "check" may be used to schedule repository checks +# +# Usage +# - `resticprofile -n check` +# - `resticprofile -n check schedule` +# - `resticprofile -n check unschedule` +# +[check] +# Operate on the pepository defined in the "base" profile +inherit = "base" +initialize = false +default-command = "check" + +# Configuring the "check" command in profile "check" +[check.check] +schedule = "daily" +schedule-lock-wait = "4h" + + +## +# Profile "verify" may be used to schedule deep repository checks +# +# Usage +# - `resticprofile -n verify` +# - `resticprofile -n verify schedule` +# - `resticprofile -n verify unschedule` +# +[verify] +# Operate on the pepository defined in the "base" profile +inherit = "base" +initialize = false +default-command = "check" + +# Configuring the "check" command in profile "verify" +[verify.check] +schedule = "monthly" +schedule-lock-wait = "48h" +# Read the entire repository for verification +read-data = true +# Read a subset of the repository for verification +#read-data-subset = "15%" + diff --git a/contrib/posix/conf.d/hooks.conf b/contrib/posix/conf.d/hooks.conf new file mode 100644 index 00000000..0c5725e3 --- /dev/null +++ b/contrib/posix/conf.d/hooks.conf @@ -0,0 +1,19 @@ + +## +# Action hooks for profiles that derive from "base" +[base] +# Actions to run before any profile task +#run-before = [ +# 'echo ">>> ${PROFILE_NAME} - BEGIN ${PROFILE_COMMAND}"', +#] + +# Actions to run after a profile task +#run-after = [ +# 'echo "<<< ${PROFILE_NAME} - END ${PROFILE_COMMAND}"', +#] + +# Actions to run when a profile task has failed +run-after-fail = [ + 'echo "!!! ${PROFILE_NAME} - FAILED ${PROFILE_COMMAND}" - ERROR: ${ERROR}"', + # 'resticprofile-send-error admin@localhost', +] diff --git a/contrib/posix/conf.d/metrics.conf b/contrib/posix/conf.d/metrics.conf new file mode 100644 index 00000000..8d99b48d --- /dev/null +++ b/contrib/posix/conf.d/metrics.conf @@ -0,0 +1,15 @@ + + +## +# Metric collection for profiles that derive from "base" +[base.backup] +# Toggles full "restic" output capture to allow collecting backup metrics +# for "status-file" and "prometheus-(save-to-file|push)" +#extended-status = true + +# Write backup metrics as JSON (requires extended-status = true) +#status-file = "{{.TempDir}}/resticprofile-{{.Profile.Name}}-status.json" + +# Export backup metrics to Prometheus (requires extended-status = true) +#prometheus-save-to-file = "/var/lib/prometheus/node-exporter/{{.Profile.Name}}.prom" +#prometheus-push = "http://host:9091/" diff --git a/contrib/posix/conf.d/prune.conf b/contrib/posix/conf.d/prune.conf new file mode 100644 index 00000000..0397a855 --- /dev/null +++ b/contrib/posix/conf.d/prune.conf @@ -0,0 +1,32 @@ + +## +# Group for profiles that prune repositories +# Usage +# - `resticprofile -n prune-all` +# - `resticprofile -n prune-all schedule` +# - `resticprofile -n prune-all unschedule` +[groups] +prune-all = [ "prune" ] + + +## +# Profile "prune" may be used to schedule pruning of the default repository +# which reclaims space that is no longer occupied by removed snapshots from +# profiles that use this repository. +# +# Usage +# - `resticprofile -n prune` +# - `resticprofile -n prune schedule` +# - `resticprofile -n prune unschedule` +# +[prune] +# Operate on the pepository defined in the "base" profile +inherit = "base" +initialize = false +default-command = "prune" + +# Configuring the "prune" command in profile "prune" +[prune.prune] +schedule = "daily" +schedule-lock-wait = "4h" + diff --git a/contrib/posix/conf.d/repository.conf b/contrib/posix/conf.d/repository.conf new file mode 100644 index 00000000..a43dfa4f --- /dev/null +++ b/contrib/posix/conf.d/repository.conf @@ -0,0 +1,67 @@ + +## +# Repository configuration +# See https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html +# + +## +# Default repository (used in all derived profiles unless redefined) +[default] +# Local: Repository mounted to local folder +repository = "local:/backup" +#run-before = [ 'mountpoint -q /backup' ] + +# SFTP: (requires password-less public-key auth for the user running restic) +#repository = "sftp:user@host:/restic-repo" +#repository = "sftp://user@[::1]:2222//restic-repo" + +# REST server: (https://github.com/restic/rest-server) +#repository = "rest:https://user:pass@host:8000/my_backup_repo/" +#cacert = "conf.d/default-repository-self-signed-pub.pem" +#tls-client-cert = "conf.d/default-repository-client.pem" + +# S3 storage (see [default.env]) +#repository = "s3:s3.amazonaws.com/bucket_name" +#repository = "s3:http://host:9000/bucket_name" +#repository = "s3:https://host/bucket_name" +#cacert = "conf.d/default-repository-self-signed-pub.pem" + +# Azure storage (see [default.env]) +#repository = "azure:container_name:/" + +# Repository password file +password-file = "conf.d/default-repository.secret" + +## +# Environment variables to pass to "restic" +[default.env] +# S3 Storage (AWS, Minio, etc.) +#AWS_ACCESS_KEY_ID = "id" +#AWS_SECRET_ACCESS_KEY = "key" + +# Azure Blob Storage +#AZURE_ACCOUNT_NAME = "storage_account" +#AZURE_ACCOUNT_KEY = "key" + + +## +# Initialize the repository (if empty) for profiles deriving from "base" +[base] +# Initialize a repository if none exists at the specified location +initialize = true + + +## +# Example: Secondary repository +# Other repository for profiles inheriting from "other-repository-base": +# +# [other-repository-base] +# inherit = "base" +# repository = "local:/backup-other" +# +# Usage: +# [my-profile-other] +# inherit = "other-repository-base" +# [my-profile-other.backup] +# source = "/path" +# diff --git a/contrib/posix/profiles.conf b/contrib/posix/profiles.conf new file mode 100644 index 00000000..5fe96997 --- /dev/null +++ b/contrib/posix/profiles.conf @@ -0,0 +1,183 @@ + +# ----------------------------------------------------------------------------- +# Resticprofile backup profiles for "restic" backups +# See https://github.com/creativeprojects/resticprofile +# +# Note: This configuration file should not be changed directly. +# Add overrides to "conf.d" and profiles to "profiles.d". +# ----------------------------------------------------------------------------- + +## +# Loading config overrides from "conf.d" and profiles from "profiles.d": +includes = [ + 'conf.d/*.conf', + 'profiles.d/*.toml', + 'profiles.d/*.yaml', +] + + +## +# The global section +[global] +# initialize a repository if none exists at the specified location +# (can be overriden in individual profiles) +initialize = false + +# restic IO priority +ionice = false +#ionice-class = 2 +#ionice-level = 6 + +# restic CPU-priority ( idle | background | low | normal | high ) +priority = "low" + +# scheduler to use: "systemd" (default) or "crond" +#scheduler = "systemd" + +# what to run when no command is specified +default-command = "snapshots" + +# Optional: Specify path to restic +#restic-binary = "/usr/local/bin/restic" + + +## +# The "default" profile +# +# This profile is choosen when no profile is explicitly specified and +# is configured to support commands on the default repository like +# "snapshots", "check", "prune" & "mount". +# +# It is also the parent to the "base" profile which means that profiles +# inheriting from "base" also inherit all settings from "default" +# +# The default profile should not be used directly for running backups +# nor should it contain schedules. +[default] +## Locks +# Prevent concurrent invocation of a profile +lock = "{{.TempDir}}/resticprofile-{{.Profile.Name}}.lock" +# Detect stale locks and unlock automatically +force-inactive-lock = true + + +## +# Backup defaults +[default.backup] + +# Hostname and tags to identify backup snapshots in the repository +#### {{block "conf:default-backup-tags" .}} +# Multiple tags can be used, defaulting to profile name. +# Important: Set the same tags for "backup" and "retention" +tag = [ "{{.Profile.Name}}" ] +#### {{end}} +#### {{block "conf:default-backup-host" .}} +# Specify a hostname or leave it at 'true' for the current hostname +host = true +#### {{end}} + +# Exclude known cache files & folders from backups +exclude-caches = true + +# Exclude nested filesystems +one-file-system = true + +# Toggle whether a failure in reading a backup source is considered an error +#no-error-on-warning = false + +# Wait on acquiring locks when running the profile on a schedule +schedule-lock-wait = "30m" + +# Specify the user that runs profile tasks on a schedule +# "system" - root runs the profile tasks +# "user" - user that created the schedule runs the profile tasks +schedule-permission = "system" + +# Toggle verbose output for troubleshooting +#verbose = false + + +## +# Snapshot retention defaults +[default.retention] +# Remove obsolete snapshots prior to starting a backup +before-backup = false +# Remove obsolete snapshots after a successful backup +after-backup = true + +# Copying "host" and "tags" blocks to identify snapshots to retain or remove +# Note: "host" and "tag" must be in sync between "backup" and "retention" +{{template "conf:default-backup-host" .}} +# In retention, tags can be copied from backup with `true`: +tags = true + +# Copying backup source paths to identify snapshots to retain or remove +# Set to "false" or a list of paths to disable or customize the path filter +path = true + +# Specify the snapshots to keep when checking for obsolete snapshots +# Snapshots that do not match any condition are removed +keep-tag = [ "forever" ] +keep-last = 3 +#keep-hourly = 1 +#keep-daily = 1 +#keep-weekly = 1 +#keep-monthly = 1 +#keep-yearly = 1 +#keep-within = "30d" + +# Use compact format for listing snapshots +#compact = false + +# Enable to prune the repository immediatelly as snapshots are removed +# Prune can be expensive. Consider scheduling prune (see "conf.d/prune.conf") +#prune = true + + +## +# Defaults for showing snapshots of this host +# Usage: +# - `resticprofile snapshots` to view snapshots of this host +[default.snapshots] +# Copying "host" block to identify snapshots to list +{{template "conf:default-backup-host" .}} + + +## +# Defaults for mounting snapshot of this host +# Usage: +# - `resticprofile mount /mnt/restore` to mount snapshots of this host +[default.mount] +# Copying "host" block to identify snapshots to mount +{{template "conf:default-backup-host" .}} + + + +## +# The "base" profile is the base for all other profiles to inherit from +# +# Note: Profiles that do not inherit from "base" will run on built-in +# defaults instead and have to take care of a full profile setup. +[base] +# Inherit all settings from the "default" profile +inherit = "default" + + +## +# Configures the "snapshots" command for profiles inheriting from "base" +# Usage: +# - `resticprofile -n profileName snapshots` to view snapshots of one profile +[base.snapshots] +# Copying "host" and "tags" blocks to identify snapshots to list +{{template "conf:default-backup-host" .}} +{{template "conf:default-backup-tags" .}} + + +## +# Configures the "mount" command for profiles inheriting from "base" +# Usage: +# - `resticprofile -n profileName mount /mnt/restore` to mount snapshots +[base.mount] +# Copying "host" and "tags" block to identify snapshots to mount +{{template "conf:default-backup-host" .}} +{{template "conf:default-backup-tags" .}} \ No newline at end of file diff --git a/contrib/posix/profiles.d/fs-snapshot.yaml.sample b/contrib/posix/profiles.d/fs-snapshot.yaml.sample new file mode 100644 index 00000000..d37c9d57 --- /dev/null +++ b/contrib/posix/profiles.d/fs-snapshot.yaml.sample @@ -0,0 +1,54 @@ +## +# Example profile for applications whose files must be backed-up from a readonly snapshot. +# +applications: + inherit: base + + backup: + source: + - /opt/apps/_backup + - /mnt/data_backup + - /opt/vms/my-vm1.xml + - /opt/vms/my-vm1.qcow2 + + # Create snapshots of supported sources prior to running a backup + run-before: + # Snapshot on Btrfs (mounted on /opt/apps/_backup) + - btrfs subvolume snapshot -r /opt/apps/ /opt/apps/_backup + + # Snapshot on LVM (mounted on /opt/apps/_backup) + - lvcreate -l100%FREE -s -n data_backup /dev/vg00/data && mount /dev/vg00/data_backup /mnt/data_backup + + # Snapshot & config dump for libvirt VMs (VM disk images are readonly during backup) + - virsh dumpxml "my-vm1" > /opt/vms/my-vm1.xml + - {{template "libvirt-create-snapshot" "my-vm1"}} + + # Release snapshots after backup + run-finally: + # Cleanup on Btrfs + - btrfs subvolume delete /opt/apps/_backup + + # Cleanup on LVM + - umount /mnt/data_backup && lvremove -f /dev/vg00/data_backup + + # Cleanup for libvirt VMs (VM disk image receives changes from livedata file) + - {{template "libvirt-delete-snapshot" "my-vm1"}} + + +##### +# Config Templates +# Use +# - `resticprofile -n applications show` to see rendered result +# +{{define "libvirt-create-snapshot" -}} + virsh snapshot-create-as --domain "{{.}}" --name "{{.}}-backup" {{- " " -}} + --diskspec "vda,file=/var/db/{{.}}-livedata.qcow2" {{- " " -}} + --disk-only --atomic --no-metadata --quiesce +{{- end}} +# +{{define "libvirt-delete-snapshot" -}} + virsh blockcommit --domain "{{.}}" vda --wait --active {{- " " -}} + && virsh blockjob --domain "{{.}}" "/var/db/{{.}}-livedata.qcow2" --pivot {{- " " -}} + && rm -f /var/db/{{.}}-livedata.qcow2 +{{- end}} +##### diff --git a/contrib/posix/profiles.d/system.toml b/contrib/posix/profiles.d/system.toml new file mode 100644 index 00000000..70eb089c --- /dev/null +++ b/contrib/posix/profiles.d/system.toml @@ -0,0 +1,38 @@ +## +# The "root" profile +# +# Backup $HOME of root and 'etc' folders once a day +# +# Usage: +# - `resticprofile -n root show` +# - `resticprofile -n root --dry-run backup` +# - `resticprofile -n root backup` +# - `resticprofile -n root snapshots` +# - `resticprofile -n root mount /restore` +# - `resticprofile -n root status` +# - `resticprofile -n root schedule` +# - `resticprofile -n root unschedule` +# +[root] +inherit = "base" + +## +# Backup sources and schedule +[root.backup] +source = [ + "/etc", + "/usr/local/etc", + "/opt/local/etc", + "~root/", +] +schedule = "daily" + +## +# Keep last 14 days and the latest from each of last 8 weeks and 6 months +# Total snapshots to keep: 24 +# (14 days, weeks not covered by days and months not covered by weeks) +[root.retention] +keep-last = false +keep-daily = 14 +keep-weekly = 8 +keep-monthly = 6