b9e8694c61
- Engine: same targeted-run fix as apiscp-kopia (env BACKUP_SITES/SYSTEM/PATHS/ DATABASES now override the sourced config, so `run --site` scopes correctly instead of backing up every site). - docs/REFERENCE.md: exhaustive reference (every command, verb, config key, installer, automation, both GUIs, security model, recovery), plus a README Documentation section.
318 lines
12 KiB
Bash
318 lines
12 KiB
Bash
#!/bin/sh
|
|
#
|
|
# borg-apiscp-backup: an ApisCP-aware backup engine for BorgBackup.
|
|
#
|
|
# On every run, in order:
|
|
# 1. Refresh ApisCP's per-site database dumps (backup_dbs.php) FIRST, so the
|
|
# logical .sql exports under each site are current before archiving them.
|
|
# 2. Create one Borg archive per site subtree (shadow, info), named with a
|
|
# structured prefix (siteN-shadow-{now}) so per-site restore and retention
|
|
# are glob-addressable. Borg preserves POSIX ACLs and xattrs natively, so
|
|
# no metadata sidecar is required (unlike the kopia engine).
|
|
# 3. Dump and archive each site's databases via ApisCP's site-context export.
|
|
# 4. Archive the shared system paths (_system-{now}) and any custom paths.
|
|
# 5. Apply retention with `borg prune` per archive prefix.
|
|
# 6. Emit Prometheus textfile metrics and, if configured, email the outcome.
|
|
#
|
|
# Config: /etc/apiscp-borg/config (see apiscp-borg.config.example).
|
|
# Exit status: 0 if every archive was created, 1 if any failed.
|
|
|
|
set -u
|
|
|
|
# --- locate the shared library -------------------------------------------------
|
|
_self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
|
for _cand in \
|
|
"${APISCP_BORG_LIB:-}" \
|
|
"$_self_dir/../lib/apiscp-borg-common.sh" \
|
|
/usr/local/lib/apiscp-borg/apiscp-borg-common.sh \
|
|
/usr/lib/apiscp-borg/apiscp-borg-common.sh; do
|
|
[ -n "$_cand" ] && [ -r "$_cand" ] && { . "$_cand"; _lib_loaded=1; break; }
|
|
done
|
|
[ "${_lib_loaded:-0}" = 1 ] || { echo "FATAL: cannot find apiscp-borg-common.sh" >&2; exit 1; }
|
|
|
|
# --- configuration -------------------------------------------------------------
|
|
CONFIG_FILE="${APISCP_BORG_CONFIG:-/etc/apiscp-borg/config}"
|
|
# Capture run-scoped overrides passed via the environment (e.g. by
|
|
# `borg-apiscp-repo run --site X`, which sets these via systemd-run) BEFORE
|
|
# sourcing the config, then re-apply them AFTER, so a targeted run is not
|
|
# clobbered by the persistent config defaults. `${VAR-x}` (single dash)
|
|
# distinguishes unset (use config) from set-but-empty (an explicit override).
|
|
_env_sites="${BACKUP_SITES-__unset__}"
|
|
_env_system="${BACKUP_SYSTEM-__unset__}"
|
|
_env_paths="${BACKUP_PATHS-__unset__}"
|
|
_env_db="${BACKUP_DATABASES-__unset__}"
|
|
# shellcheck source=/dev/null
|
|
[ -r "$CONFIG_FILE" ] && . "$CONFIG_FILE"
|
|
[ "$_env_sites" != "__unset__" ] && BACKUP_SITES="$_env_sites"
|
|
[ "$_env_system" != "__unset__" ] && BACKUP_SYSTEM="$_env_system"
|
|
[ "$_env_paths" != "__unset__" ] && BACKUP_PATHS="$_env_paths"
|
|
[ "$_env_db" != "__unset__" ] && BACKUP_DATABASES="$_env_db"
|
|
|
|
: "${APNSCP_ROOT:=/usr/local/apnscp}"
|
|
: "${APNSCP_CMD:=/usr/local/apnscp/bin/cmd}"
|
|
: "${VIRTBASE:=/home/virtual}"
|
|
: "${STAGE:=/var/lib/apiscp-borg}"
|
|
: "${RUN_DB_EXPORT:=1}"
|
|
: "${DB_EXPORT_CMD:=$APNSCP_ROOT/bin/scripts/backup_dbs.php}"
|
|
: "${SYSTEM_PATHS:=/etc /opt}"
|
|
: "${SYSTEM_GLOBS:=/var/log/mailer_table* /var/lib/mysql/mysql-grants* /var/lib/pgsql/*/backups /root/apnscp* /root/license*}"
|
|
: "${METRICS_DIR:=}"
|
|
: "${LOCK_FILE:=/run/apiscp-borg.lock}"
|
|
: "${BACKUP_SITES:=all}"
|
|
: "${BACKUP_SYSTEM:=1}"
|
|
: "${BACKUP_DATABASES:=1}"
|
|
: "${BACKUP_PATHS:=}"
|
|
: "${BACKUP_EXCLUDES:=}"
|
|
: "${CAPTURE_META_SIDECAR:=0}"
|
|
: "${NOTIFY_EMAIL:=}"
|
|
: "${NOTIFY_ON:=failure}"
|
|
: "${NOTIFY_FROM:=}"
|
|
: "${RETENTION_KEEP_WITHIN:=}"
|
|
: "${RETENTION_KEEP_DAILY:=}"
|
|
: "${RETENTION_KEEP_WEEKLY:=}"
|
|
: "${RETENTION_KEEP_MONTHLY:=}"
|
|
: "${RETENTION_KEEP_ANNUAL:=}"
|
|
: "${BORG_REPO:=}"
|
|
: "${BORG_RSH:=ssh -o BatchMode=yes}"
|
|
|
|
[ -n "$BORG_REPO" ] || akp_die "BORG_REPO is not set (edit $CONFIG_FILE)"
|
|
export BORG_REPO BORG_RSH
|
|
[ -n "${BORG_PASSPHRASE:-}" ] && export BORG_PASSPHRASE
|
|
# Do not block on interactive prompts (unknown repo, etc.) in an unattended run.
|
|
export BORG_RELOCATED_REPO_ACCESS_IS_OK="${BORG_RELOCATED_REPO_ACCESS_IS_OK:-no}"
|
|
|
|
# akp_site_selected SITE: succeed if SITE should be backed up per BACKUP_SITES.
|
|
akp_site_selected() {
|
|
[ "$BACKUP_SITES" = "all" ] && return 0
|
|
[ "$BACKUP_SITES" = "none" ] && return 1
|
|
_want="$1"; _ifs=$IFS; IFS=,
|
|
for _s in $BACKUP_SITES; do
|
|
_s=$(printf '%s' "$_s" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
|
|
[ "$_s" = "$_want" ] && { IFS=$_ifs; return 0; }
|
|
done
|
|
IFS=$_ifs; return 1
|
|
}
|
|
|
|
# --- single-instance lock ------------------------------------------------------
|
|
if command -v flock >/dev/null 2>&1; then
|
|
exec 9>"$LOCK_FILE" || akp_die "cannot open lock $LOCK_FILE"
|
|
flock -n 9 || akp_die "another run holds $LOCK_FILE; aborting"
|
|
fi
|
|
|
|
# --- metrics + notify accounting ----------------------------------------------
|
|
_start_epoch=$(date +%s 2>/dev/null || echo 0)
|
|
OK=0
|
|
FAILED=0
|
|
DB_EXPORT_RC=-1
|
|
_reached_end=0
|
|
|
|
_detect_metrics_dir() {
|
|
[ -n "$METRICS_DIR" ] && { printf '%s' "$METRICS_DIR"; return; }
|
|
for d in /var/lib/node_exporter/textfile_collector /var/lib/prometheus/node-exporter /var/lib/prometheus/node_exporter; do
|
|
[ -d "$d" ] && { printf '%s' "$d"; return; }
|
|
done
|
|
printf ''
|
|
}
|
|
|
|
write_metrics() {
|
|
_mdir=$(_detect_metrics_dir)
|
|
[ -n "$_mdir" ] || { akp_warn "no textfile collector dir; metrics not written"; return; }
|
|
_now=$(date +%s 2>/dev/null || echo 0)
|
|
_dur=$(( _now - _start_epoch ))
|
|
_run_ok=0; [ "$FAILED" -eq 0 ] && _run_ok=1
|
|
_tmp="$_mdir/.apiscp-borg.prom.$$"
|
|
{
|
|
echo "# HELP apiscp_borg_run_success Whether the last run archived every source (1) or not (0)."
|
|
echo "# TYPE apiscp_borg_run_success gauge"
|
|
echo "apiscp_borg_run_success $_run_ok"
|
|
echo "# HELP apiscp_borg_archives_ok Archives created successfully in the last run."
|
|
echo "# TYPE apiscp_borg_archives_ok gauge"
|
|
echo "apiscp_borg_archives_ok $OK"
|
|
echo "# HELP apiscp_borg_archives_failed Archives that failed in the last run."
|
|
echo "# TYPE apiscp_borg_archives_failed gauge"
|
|
echo "apiscp_borg_archives_failed $FAILED"
|
|
echo "# HELP apiscp_borg_duration_seconds Wall-clock duration of the last run."
|
|
echo "# TYPE apiscp_borg_duration_seconds gauge"
|
|
echo "apiscp_borg_duration_seconds $_dur"
|
|
echo "# HELP apiscp_borg_db_export_success backup_dbs.php exit (1 ok, 0 failed, -1 skipped)."
|
|
echo "# TYPE apiscp_borg_db_export_success gauge"
|
|
echo "apiscp_borg_db_export_success $DB_EXPORT_RC"
|
|
echo "# HELP apiscp_borg_last_run_timestamp_seconds Unix time the last run finished."
|
|
echo "# TYPE apiscp_borg_last_run_timestamp_seconds gauge"
|
|
echo "apiscp_borg_last_run_timestamp_seconds $_now"
|
|
} > "$_tmp" 2>/dev/null && mv -f "$_tmp" "$_mdir/apiscp-borg.prom" 2>/dev/null \
|
|
|| akp_warn "failed to write metrics to $_mdir"
|
|
}
|
|
|
|
notify() {
|
|
[ -n "${NOTIFY_EMAIL:-}" ] || return 0
|
|
_n_ok=0; { [ "$FAILED" -eq 0 ] && [ "$_reached_end" -eq 1 ]; } && _n_ok=1
|
|
case "${NOTIFY_ON:-failure}" in
|
|
never) return 0 ;;
|
|
always) ;;
|
|
*) [ "$_n_ok" -eq 1 ] && return 0 ;;
|
|
esac
|
|
_n_host=$(hostname 2>/dev/null || echo apiscp)
|
|
_n_dur=$(( $(date +%s 2>/dev/null || echo 0) - _start_epoch ))
|
|
_n_res=$( [ "$_n_ok" -eq 1 ] && echo success || echo FAILURE )
|
|
_n_db=$( case "$DB_EXPORT_RC" in 1) echo ok ;; 0) echo failed ;; *) echo skipped ;; esac )
|
|
_n_body=$(printf 'apiscp-borg backup run on %s\n\nResult: %s\nFinished: %s\nArchives: %s ok, %s failed\nDatabases: %s\nDuration: %ss\nRepository: %s\n' \
|
|
"$_n_host" "$_n_res" "$(_akp_now)" "$OK" "$FAILED" "$_n_db" "$_n_dur" "$BORG_REPO")
|
|
[ "$_n_ok" -eq 1 ] || _n_body="$_n_body
|
|
See 'journalctl -t ${APISCP_BORG_LOG_TAG:-apiscp-borg}' on $_n_host for details."
|
|
akp_send_mail "$NOTIFY_EMAIL" "[apiscp-borg] backup $_n_res on $_n_host" "$_n_body" "${NOTIFY_FROM:-}" \
|
|
&& akp_log "notification emailed to $NOTIFY_EMAIL" || akp_warn "notification email failed"
|
|
}
|
|
|
|
on_exit() { write_metrics; notify; }
|
|
trap on_exit EXIT INT TERM
|
|
|
|
# Build the shared --exclude argument list (always exclude the restore staging).
|
|
_add_excludes() {
|
|
set -- --exclude '*/.borg-restore' --exclude '/.borg-restore'
|
|
_oifs=$IFS; IFS=',
|
|
'
|
|
for _p in ${BACKUP_EXCLUDES:-}; do
|
|
[ -n "$_p" ] && set -- "$@" --exclude "$_p"
|
|
done
|
|
IFS=$_oifs
|
|
EXCLUDE_ARGS="$*" # only used for logging; real calls use "$@" below
|
|
BORG_EXCLUDES_SET=1
|
|
# Export via positional re-use is awkward in sh; callers rebuild instead.
|
|
}
|
|
|
|
# create_archive NAME PATH...: borg create with excludes; update counters.
|
|
create_archive() {
|
|
_name="$1"; shift
|
|
[ "$#" -ge 1 ] || return 0
|
|
# Assemble excludes fresh each call.
|
|
set -- "$@" # keep source paths in "$@"
|
|
_srcs="$*"
|
|
# shellcheck disable=SC2086
|
|
if _create_with_excludes "$_name" "$@"; then
|
|
OK=$(( OK + 1 )); akp_log "archived $_name"
|
|
else
|
|
FAILED=$(( FAILED + 1 )); akp_warn "archive FAILED: $_name ($_srcs)"
|
|
fi
|
|
}
|
|
|
|
# _create_with_excludes NAME SRC...: run borg create '::NAME-{now}' with excludes.
|
|
_create_with_excludes() {
|
|
_cn="$1"; shift
|
|
# Build: create [excludes] ::NAME-{now} SRC...
|
|
set -- ::"$_cn"-{now} "$@"
|
|
# Prepend excludes.
|
|
_ex_oifs=$IFS; IFS=',
|
|
'
|
|
for _p in ${BACKUP_EXCLUDES:-}; do
|
|
[ -n "$_p" ] && set -- --exclude "$_p" "$@"
|
|
done
|
|
IFS=$_ex_oifs
|
|
set -- --exclude '*/.borg-restore' --exclude '/.borg-restore' "$@"
|
|
akp_borg create --compression "${BORG_COMPRESSION:-zstd}" "$@" >/dev/null 2>&1
|
|
}
|
|
|
|
# prune_prefix PREFIX: apply retention to archives named PREFIX-* if any keep rule set.
|
|
prune_prefix() {
|
|
_pref="$1"
|
|
set --
|
|
[ -n "$RETENTION_KEEP_WITHIN" ] && set -- "$@" --keep-within "$RETENTION_KEEP_WITHIN"
|
|
[ -n "$RETENTION_KEEP_DAILY" ] && set -- "$@" --keep-daily "$RETENTION_KEEP_DAILY"
|
|
[ -n "$RETENTION_KEEP_WEEKLY" ] && set -- "$@" --keep-weekly "$RETENTION_KEEP_WEEKLY"
|
|
[ -n "$RETENTION_KEEP_MONTHLY" ] && set -- "$@" --keep-monthly "$RETENTION_KEEP_MONTHLY"
|
|
[ -n "$RETENTION_KEEP_ANNUAL" ] && set -- "$@" --keep-yearly "$RETENTION_KEEP_ANNUAL"
|
|
[ "$#" -ge 1 ] || return 0
|
|
if akp_borg prune --glob-archives "$_pref-*" "$@" >/dev/null 2>&1; then
|
|
akp_log "pruned $_pref-* per retention"
|
|
else
|
|
akp_warn "prune failed for $_pref-*"
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================
|
|
akp_log "run start (repo: $BORG_REPO)"
|
|
|
|
# 1. Refresh per-site database exports before archiving them.
|
|
if [ "$RUN_DB_EXPORT" = "1" ]; then
|
|
if [ -x "$DB_EXPORT_CMD" ] || [ -r "$DB_EXPORT_CMD" ]; then
|
|
akp_log "running ApisCP database export: $DB_EXPORT_CMD"
|
|
if "$DB_EXPORT_CMD" >/dev/null 2>&1; then DB_EXPORT_RC=1; else DB_EXPORT_RC=0; akp_warn "database export returned non-zero"; fi
|
|
else
|
|
akp_warn "DB_EXPORT_CMD not found ($DB_EXPORT_CMD); skipping"
|
|
fi
|
|
fi
|
|
|
|
# 2 + 3. Per-site archives and databases.
|
|
if [ -d "$VIRTBASE" ]; then
|
|
for sitedir in "$VIRTBASE"/site[0-9]*; do
|
|
[ -d "$sitedir" ] || continue
|
|
site=$(basename "$sitedir")
|
|
akp_site_selected "$site" || { akp_log "skip $site (not in BACKUP_SITES)"; continue; }
|
|
|
|
[ -d "$sitedir/shadow" ] && create_archive "$site-shadow" "$sitedir/shadow"
|
|
[ -d "$sitedir/info" ] && create_archive "$site-info" "$sitedir/info"
|
|
|
|
if [ "$BACKUP_DATABASES" = "1" ]; then
|
|
dbdom=$(akp_site_domain "$site")
|
|
if [ -z "$dbdom" ]; then
|
|
akp_warn "no domain for $site; skipping database dump"
|
|
elif [ ! -x "$APNSCP_CMD" ] && [ ! -r "$APNSCP_CMD" ]; then
|
|
akp_warn "APNSCP_CMD not found ($APNSCP_CMD); skipping database dump for $site"
|
|
else
|
|
db_siterel="/tmp/.apiscp-borg-db"
|
|
dbroot="$sitedir/fst$db_siterel"
|
|
rm -rf "$dbroot"
|
|
_dbany=0
|
|
for eng in mysql pgsql; do
|
|
dblist=$("$APNSCP_CMD" -o json -d "$dbdom" "$eng:list_databases" 2>/dev/null) || continue
|
|
[ -n "$dblist" ] || continue
|
|
for db in $(printf '%s\n' "$dblist" | grep -oE '"[A-Za-z0-9_-]+"' | sed 's/"//g'); do
|
|
[ -n "$db" ] || continue
|
|
if "$APNSCP_CMD" -d "$dbdom" "$eng:export" "$db" "$db_siterel/$eng/$db.sql" >/dev/null 2>&1; then
|
|
_dbany=1; akp_log "exported $eng db $db for $site"
|
|
else
|
|
akp_warn "failed to export $eng db $db for $site"
|
|
fi
|
|
done
|
|
done
|
|
[ "$_dbany" = 1 ] && [ -d "$dbroot" ] && create_archive "$site-db" "$dbroot"
|
|
rm -rf "$dbroot"
|
|
fi
|
|
fi
|
|
|
|
# Per-site retention.
|
|
prune_prefix "$site-shadow"
|
|
prune_prefix "$site-info"
|
|
prune_prefix "$site-db"
|
|
done
|
|
else
|
|
akp_warn "VIRTBASE $VIRTBASE not present; no per-site backups"
|
|
fi
|
|
|
|
# 4. Shared system paths (one archive), then custom paths.
|
|
if [ "$BACKUP_SYSTEM" = "1" ]; then
|
|
# Expand globs; keep only existing paths.
|
|
set --
|
|
# shellcheck disable=SC2086
|
|
for p in $SYSTEM_PATHS $SYSTEM_GLOBS; do
|
|
[ -e "$p" ] && set -- "$@" "$p"
|
|
done
|
|
[ "$#" -ge 1 ] && create_archive "_system" "$@"
|
|
prune_prefix "_system"
|
|
else
|
|
akp_log "skipping shared system paths (BACKUP_SYSTEM=0)"
|
|
fi
|
|
|
|
if [ -n "$BACKUP_PATHS" ]; then
|
|
_paths=$(printf '%s' "$BACKUP_PATHS" | tr ',' ' ')
|
|
for cp in $_paths; do
|
|
[ -e "$cp" ] || { akp_warn "skip custom path (missing): $cp"; continue; }
|
|
slug=$(akp_slug "$cp")
|
|
create_archive "_custom-$slug" "$cp"
|
|
prune_prefix "_custom-$slug"
|
|
done
|
|
fi
|
|
|
|
akp_log "run done: $OK ok, $FAILED failed"
|
|
_reached_end=1
|
|
[ "$FAILED" -eq 0 ]
|