From 58cb637f72ca7a43250d411489bfe7c988ebb939 Mon Sep 17 00:00:00 2001 From: BA1001 Date: Sun, 27 Nov 2022 21:30:10 +0100 Subject: [PATCH] Add borgmatic-wrapper script --- usr/local/jilits/borgmatic-wrapper.sh | 148 ++++++++++++++++++++++++++ usr/local/jilits/libcommon.sh | 5 + usr/local/jilits/libdatatype.sh | 9 ++ usr/local/jilits/libdatetime.sh | 13 +++ usr/local/jilits/libstate.sh | 102 ++++++++++++++++++ usr/local/jilits/libverbosity.sh | 62 +++++++++++ 6 files changed, 339 insertions(+) create mode 100755 usr/local/jilits/borgmatic-wrapper.sh create mode 100644 usr/local/jilits/libcommon.sh create mode 100644 usr/local/jilits/libdatatype.sh create mode 100644 usr/local/jilits/libdatetime.sh create mode 100644 usr/local/jilits/libstate.sh create mode 100644 usr/local/jilits/libverbosity.sh diff --git a/usr/local/jilits/borgmatic-wrapper.sh b/usr/local/jilits/borgmatic-wrapper.sh new file mode 100755 index 0000000..13f16a6 --- /dev/null +++ b/usr/local/jilits/borgmatic-wrapper.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +# Performs borgmatic backups and checks, given that +# enough time has elapsed since the last one performed. +# cerberus, 2022-02-03-1 + +set -o pipefail + +[ -f /opt/borg/bin/borg ] && borg=/opt/borg/bin/borg || borg="$(command -v borg)" +[ -f /opt/borg/bin/borgmatic ] && borgmatic=/opt/borg/bin/borgmatic || borgmatic="$(command -v borgmatic)" + +export LOG_FILE="/var/log/borgmatic-wrapper.log" +export STATE_DIR="/var/tmp/borgmatic-wrapper" +LIB_DIR="${LIB_DIR:-/usr/local/jilits}" + +script_name="$(basename -- "$0")" +max_minutes_since_last_backup="" +max_minutes_since_last_check=720 + +. "$LIB_DIR/libverbosity.sh" || exit 127 +. "$LIB_DIR/libcommon.sh" || exit 127 +. "$LIB_DIR/libdatatype.sh" || exit 127 +. "$LIB_DIR/libdatetime.sh" || exit 127 +. "$LIB_DIR/libstate.sh" || exit 127 + +eggsit() { + rc=$? + [ $rc -eq 0 ] && echlog "Script exiting ($rc)..." + [ $rc -ne 0 ] && errlog "Script exiting ($rc)..." + exit $rc +} +trap "eggsit" EXIT + +get_last_backup_ts() { + local ts state_ttl="$((6 * 60 * 60))" + ts="$(get_state CachedLastBackupTimestamp)" + + if [ -n "$ts" ]; then + echo "$ts" + return 0 + fi + + ts="$(sudo "$borgmatic" list --last 1 --successful --json | jq -r '.[0].archives[0].start')" + set_state CachedLastBackupTimestamp "$ts" "$state_ttl" + echo "$ts" +} + +minutes_since_last_backup() { + local ue + ue="$(dt_to_ue "$(get_last_backup_ts)")" + echo "$((($(current_ue) - ue) / 60))" +} + +remote_is_online() { + local host port + host="$(cut -d: -f1 <<<"$1")" + port="$(cut -d: -f2 <<<"$1")" + nc -zvw1 "${host:?Missing remote host}" "${port:?Missing remote port}" | tee -a "$LOG_FILE" >/dev/null +} + +perform_backup() { + echlog "Running backup." + + rm_state CachedLastBackupTimestamp + rm_state LastBackupPrettyTimestamp + + "$borgmatic" create --files --stats --verbosity 2 --syslog-verbosity -1 2>&1 | tee -a "$LOG_FILE" || return 1 + set_state LastBackupPrettyTimestamp "$(dt_to_pretty "$(get_last_backup_ts)")" "$((4 * 60 * 60))" + + "$borgmatic" prune --files --stats --verbosity 2 --syslog-verbosity -1 2>&1 | tee -a "$LOG_FILE" || return 2 +} + +perform_check() { + echlog "Running check." + + rm_state LastCheckUnixEpoch + + "$borgmatic" check --progress --verbosity 2 --syslog-verbosity -1 2>&1 | tee -a "$LOG_FILE" || return 1 + set_state LastCheckUnixEpoch "$(current_ue)" "$((24 * 60 * 60))" +} + +function usage() { + echo "Usage: $script_name [OPTIONS]" + echo + echo "Options:" + echo " -h Display help." + echo " -b [MINUTES] Max minutes since last backup. " + echo " -c [MINUTES] Max minutes since last check." + echo " -r [HOST:PORT] Ensure remote is online before running actions." +} + +while getopts ":b:c:r:h" arg; do + case $arg in + b) max_minutes_since_last_backup=$OPTARG ;; + c) max_minutes_since_last_check=$OPTARG ;; + r) remote_host_port=$OPTARG ;; + h) + usage + exit 0 + ;; + :) + echo "$script_name: Must supply a value with -$OPTARG." >&2 + usage + exit 1 + ;; + ?) + echo "Invalid option: -${OPTARG}." + echo + usage + exit 2 + ;; + esac +done + +rotlog 10000000 + +if ! is_int "$max_minutes_since_last_backup"; then + usage + echlog + echxit "ERROR: Invalid or unset value for argument -b: '$max_minutes_since_last_backup'" 2 +fi + +if ! im_root; then + echxit "ERROR: This script must be run as root" 1 +elif ! command -v jq >/dev/null; then + echxit "ERROR: jq not installed" 3 +elif pgrep -f "$borg" >/dev/null; then + echxit "ERROR: Borg seems to be active already" 11 +elif [ -n "$remote_host_port" ]; then + if ! remote_is_online "$remote_host_port"; then + echxit "ERROR: Unable to reach $remote_host_port" 12 + fi +fi + +echlog "The last backup was carried out $(minutes_since_last_backup) minutes ago (>=$max_minutes_since_last_backup)." +if [[ "$(minutes_since_last_backup)" -ge "$max_minutes_since_last_backup" ]]; then + perform_backup || echxit "ERROR: Backup returned errors" 21 +fi + +last_check_unixepoch="$(get_state LastCheckUnixEpoch)" || last_check_unixepoch=0 +minutes_since_last_check="$((($(current_ue) - last_check_unixepoch) / 60))" +echo "The last check was carried out $minutes_since_last_check minutes ago (>=$max_minutes_since_last_check)." | tee -a "$LOG_FILE" + +if [[ "$minutes_since_last_check" -ge "$max_minutes_since_last_check" ]]; then + perform_check || echxit "ERROR: Check returned errors" 21 +fi + +echo "Script finished." diff --git a/usr/local/jilits/libcommon.sh b/usr/local/jilits/libcommon.sh new file mode 100644 index 0000000..068617b --- /dev/null +++ b/usr/local/jilits/libcommon.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +im_root() { + [ "$EUID" -eq 0 ] || return 1 +} diff --git a/usr/local/jilits/libdatatype.sh b/usr/local/jilits/libdatatype.sh new file mode 100644 index 0000000..2af0cc6 --- /dev/null +++ b/usr/local/jilits/libdatatype.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +is_int() { + [[ $1 =~ ^[0-9]+$ ]] || return 1 +} + +is_float() { + [[ $1 =~ ^[0-9]+(\.[0-9]+)*$ ]] || return 1 +} diff --git a/usr/local/jilits/libdatetime.sh b/usr/local/jilits/libdatetime.sh new file mode 100644 index 0000000..9a0c93c --- /dev/null +++ b/usr/local/jilits/libdatetime.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +current_ue() { + date +%s +} + +dt_to_ue() { + date --date="${1:?Datetime not set}" +%s +} + +dt_to_pretty() { + date --date="${1:?Datetime not set}" +"%Y-%m-%d %H:%M:%S" +} diff --git a/usr/local/jilits/libstate.sh b/usr/local/jilits/libstate.sh new file mode 100644 index 0000000..c923c12 --- /dev/null +++ b/usr/local/jilits/libstate.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# Make sure the state directory exists and create it if not +ensure_state_dir() { + local state_dir="${STATE_DIR:?}" + + if ! [ -d "$(dirname "$state_dir")" ]; then + errlog "ERROR: Parent directory of STATE_DIR '$state_dir' doesn't exist." + return 1 + elif ! [ -d "$state_dir" ]; then + if ! mkdir "$state_dir"; then + errlog "ERROR: Failed to create STATE_DIR '$state_dir'." + return 2 + fi + fi +} + +# Get the configured TTL for a state key +get_state_ttl() { + local state_file="${1:?Missing state file}" + local ttl_file="$state_file.ttl" + local state_ttl + + state_ttl="$(cat "$ttl_file" 2>/dev/null)" + + if ! is_int "$state_ttl"; then + errlog "WARN: Invalid value '$state_ttl' in '$ttl_file'. Assuming value 0." + state_ttl=0 + fi + + echo "$state_ttl" +} + +# Get the age of a state key +get_state_age() { + local state_file="${1:?Missing state file}" + local state_mod + + state_mod="$(stat -c%Y "$state_file")" + echo "$(($(current_ue) - state_mod))" +} + +# Check if a state has expired +state_is_expired() { + local state_file="${1:?Missing state file}" + local state_ttl state_mod + + state_ttl="$(get_state_ttl "$state_file")" + [ "$state_ttl" -eq 0 ] && return 1 + [ "$(get_state_age "$state_file")" -gt "$state_ttl" ] && return 0 +} + +# Remove a state +rm_state() { + local key="${1:?Missing key}" + local state_file="$STATE_DIR/$key" + local ttl_file="$state_file.ttl" + rm "$state_file" "$ttl_file" +} + +# Check if a state key name is valid +state_key_is_valid() { + local key="${1:?Missing key}" regex="^([0-9]|[a-z]|[A-Z]|-)*$" + + if ! [[ "$key" =~ $regex ]]; then + errlog "ERROR: Key '$key' doesn't match regex '$regex'." + return 1 + fi +} + +# Write a value to a state key and optionally set a TTL +set_state() { + local key="${1:?Missing key}" value="$2" ttl="${3:-0}" + local state_file="$STATE_DIR/$key" + local ttl_file="$state_file.ttl" + + state_key_is_valid "$key" || return 1 + ensure_state_dir || return 2 + + echo "$value" >"$state_file" + echo "$ttl" >"$ttl_file" +} + +# Read a value from a state key +get_state() { + local key="${1:?Missing key}" + local state_file="$STATE_DIR/$key" + + state_key_is_valid "$key" || return 1 + ensure_state_dir || return 2 + + if ! [ -f "$state_file" ]; then + errlog "WARN: State file doesn't exist: '$state_file'" + return 3 + elif state_is_expired "$state_file"; then + errlog "WARN: State file has expired: '$state_file'" + rm_state "$key" + return 4 + fi + + cat "$state_file" +} diff --git a/usr/local/jilits/libverbosity.sh b/usr/local/jilits/libverbosity.sh new file mode 100644 index 0000000..f37b23c --- /dev/null +++ b/usr/local/jilits/libverbosity.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Write to log +wrlog() { + echo "$1" >>"${LOG_FILE:?}" +} + +# Write to stdout/stderr and log +echlog() { + local dt + dt="$(date +"%Y-%m-%dT%H:%M:%S")" + + if [ -z "$1" ] && [ -z "$2" ]; then + echo "" + wrlog "" + elif [ -n "$1" ]; then + echo "$dt $1" + wrlog "$dt $1" + elif [ -n "$2" ]; then + echo "$dt $2" 1>&2 + wrlog "$dt $2" + fi +} + +# Write to stderr +errlog() { + echlog "" "$1" +} + +# Rotate the log file +# TODO: Implement support for multiple rollovers and bz2 compression +rotlog() { + local filesize maxsize="${1:-10000000}" + + if ! [ -f "$LOG_FILE" ]; then + echlog "File doesn't exist: '$LOG_FILE'" + return 0 + fi + + filesize="$(stat -c%s "$LOG_FILE")" + + if ((filesize > maxsize)); then + echlog "Rotating log ($filesize > $maxsize)" + [ -f "$LOG_FILE.0" ] && rm "$LOG_FILE.0" + mv "$LOG_FILE" "$LOG_FILE.0" + fi +} + +# Echo $1 and exit with code $2 +echxit() { + if [ "$2" -eq 0 ]; then + echlog "$1" + exit "$2" + elif [ "$2" -gt 0 ]; then + errlog "$1" + exit "$2" + fi + + errlog "$1" + errlog "Invalid exit code specified to echxit" + exit 255 +}