#!/bin/sh
#
# borg-apiscp-backup: an ApisCP-aware backup engine for BorgBackup.
#
# On every run, in order:
#   1. Refresh ApisCP's per-site database dumps (backup_dbs.php) FIRST, so the
#      logical .sql exports under each site are current before archiving them.
#   2. Create one Borg archive per site subtree (shadow, info), named with a
#      structured prefix (siteN-shadow-{now}) so per-site restore and retention
#      are glob-addressable. Borg preserves POSIX ACLs and xattrs natively, so
#      no metadata sidecar is required (unlike the kopia engine).
#   3. Dump and archive each site's databases via ApisCP's site-context export.
#   4. Archive the shared system paths (_system-{now}) and any custom paths.
#   5. Apply retention with `borg prune` per archive prefix.
#   6. Emit Prometheus textfile metrics and, if configured, email the outcome.
#
# Config: /etc/apiscp-borg/config (see apiscp-borg.config.example).
# Exit status: 0 if every archive was created, 1 if any failed.

set -u

# --- locate the shared library -------------------------------------------------
_self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
for _cand in \
	"${APISCP_BORG_LIB:-}" \
	"$_self_dir/../lib/apiscp-borg-common.sh" \
	/usr/local/lib/apiscp-borg/apiscp-borg-common.sh \
	/usr/lib/apiscp-borg/apiscp-borg-common.sh; do
	[ -n "$_cand" ] && [ -r "$_cand" ] && { . "$_cand"; _lib_loaded=1; break; }
done
[ "${_lib_loaded:-0}" = 1 ] || { echo "FATAL: cannot find apiscp-borg-common.sh" >&2; exit 1; }

# --- configuration -------------------------------------------------------------
CONFIG_FILE="${APISCP_BORG_CONFIG:-/etc/apiscp-borg/config}"
# Capture run-scoped overrides passed via the environment (e.g. by
# `borg-apiscp-repo run --site X`, which sets these via systemd-run) BEFORE
# sourcing the config, then re-apply them AFTER, so a targeted run is not
# clobbered by the persistent config defaults. `${VAR-x}` (single dash)
# distinguishes unset (use config) from set-but-empty (an explicit override).
_env_sites="${BACKUP_SITES-__unset__}"
_env_system="${BACKUP_SYSTEM-__unset__}"
_env_paths="${BACKUP_PATHS-__unset__}"
_env_db="${BACKUP_DATABASES-__unset__}"
# shellcheck source=/dev/null
[ -r "$CONFIG_FILE" ] && . "$CONFIG_FILE"
[ "$_env_sites" != "__unset__" ]  && BACKUP_SITES="$_env_sites"
[ "$_env_system" != "__unset__" ] && BACKUP_SYSTEM="$_env_system"
[ "$_env_paths" != "__unset__" ]  && BACKUP_PATHS="$_env_paths"
[ "$_env_db" != "__unset__" ]     && BACKUP_DATABASES="$_env_db"

: "${APNSCP_ROOT:=/usr/local/apnscp}"
: "${APNSCP_CMD:=/usr/local/apnscp/bin/cmd}"
: "${VIRTBASE:=/home/virtual}"
: "${STAGE:=/var/lib/apiscp-borg}"
: "${RUN_DB_EXPORT:=1}"
: "${DB_EXPORT_CMD:=$APNSCP_ROOT/bin/scripts/backup_dbs.php}"
: "${SYSTEM_PATHS:=/etc /opt}"
: "${SYSTEM_GLOBS:=/var/log/mailer_table* /var/lib/mysql/mysql-grants* /var/lib/pgsql/*/backups /root/apnscp* /root/license*}"
: "${METRICS_DIR:=}"
: "${LOCK_FILE:=/run/apiscp-borg.lock}"
: "${BACKUP_SITES:=all}"
: "${BACKUP_SYSTEM:=1}"
: "${BACKUP_DATABASES:=1}"
: "${BACKUP_PATHS:=}"
: "${BACKUP_EXCLUDES:=}"
: "${CAPTURE_META_SIDECAR:=0}"
: "${NOTIFY_EMAIL:=}"
: "${NOTIFY_ON:=failure}"
: "${NOTIFY_FROM:=}"
: "${RETENTION_KEEP_WITHIN:=}"
: "${RETENTION_KEEP_DAILY:=}"
: "${RETENTION_KEEP_WEEKLY:=}"
: "${RETENTION_KEEP_MONTHLY:=}"
: "${RETENTION_KEEP_ANNUAL:=}"
: "${BORG_REPO:=}"
: "${BORG_RSH:=ssh -o BatchMode=yes}"

[ -n "$BORG_REPO" ] || akp_die "BORG_REPO is not set (edit $CONFIG_FILE)"
export BORG_REPO BORG_RSH
[ -n "${BORG_PASSPHRASE:-}" ] && export BORG_PASSPHRASE
# Do not block on interactive prompts (unknown repo, etc.) in an unattended run.
export BORG_RELOCATED_REPO_ACCESS_IS_OK="${BORG_RELOCATED_REPO_ACCESS_IS_OK:-no}"

# akp_site_selected SITE: succeed if SITE should be backed up per BACKUP_SITES.
akp_site_selected() {
	[ "$BACKUP_SITES" = "all" ] && return 0
	[ "$BACKUP_SITES" = "none" ] && return 1
	_want="$1"; _ifs=$IFS; IFS=,
	for _s in $BACKUP_SITES; do
		_s=$(printf '%s' "$_s" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
		[ "$_s" = "$_want" ] && { IFS=$_ifs; return 0; }
	done
	IFS=$_ifs; return 1
}

# --- single-instance lock ------------------------------------------------------
if command -v flock >/dev/null 2>&1; then
	exec 9>"$LOCK_FILE" || akp_die "cannot open lock $LOCK_FILE"
	flock -n 9 || akp_die "another run holds $LOCK_FILE; aborting"
fi

# --- metrics + notify accounting ----------------------------------------------
_start_epoch=$(date +%s 2>/dev/null || echo 0)
OK=0
FAILED=0
DB_EXPORT_RC=-1
_reached_end=0

_detect_metrics_dir() {
	[ -n "$METRICS_DIR" ] && { printf '%s' "$METRICS_DIR"; return; }
	for d in /var/lib/node_exporter/textfile_collector /var/lib/prometheus/node-exporter /var/lib/prometheus/node_exporter; do
		[ -d "$d" ] && { printf '%s' "$d"; return; }
	done
	printf ''
}

write_metrics() {
	_mdir=$(_detect_metrics_dir)
	[ -n "$_mdir" ] || { akp_warn "no textfile collector dir; metrics not written"; return; }
	_now=$(date +%s 2>/dev/null || echo 0)
	_dur=$(( _now - _start_epoch ))
	_run_ok=0; [ "$FAILED" -eq 0 ] && _run_ok=1
	_tmp="$_mdir/.apiscp-borg.prom.$$"
	{
		echo "# HELP apiscp_borg_run_success Whether the last run archived every source (1) or not (0)."
		echo "# TYPE apiscp_borg_run_success gauge"
		echo "apiscp_borg_run_success $_run_ok"
		echo "# HELP apiscp_borg_archives_ok Archives created successfully in the last run."
		echo "# TYPE apiscp_borg_archives_ok gauge"
		echo "apiscp_borg_archives_ok $OK"
		echo "# HELP apiscp_borg_archives_failed Archives that failed in the last run."
		echo "# TYPE apiscp_borg_archives_failed gauge"
		echo "apiscp_borg_archives_failed $FAILED"
		echo "# HELP apiscp_borg_duration_seconds Wall-clock duration of the last run."
		echo "# TYPE apiscp_borg_duration_seconds gauge"
		echo "apiscp_borg_duration_seconds $_dur"
		echo "# HELP apiscp_borg_db_export_success backup_dbs.php exit (1 ok, 0 failed, -1 skipped)."
		echo "# TYPE apiscp_borg_db_export_success gauge"
		echo "apiscp_borg_db_export_success $DB_EXPORT_RC"
		echo "# HELP apiscp_borg_last_run_timestamp_seconds Unix time the last run finished."
		echo "# TYPE apiscp_borg_last_run_timestamp_seconds gauge"
		echo "apiscp_borg_last_run_timestamp_seconds $_now"
	} > "$_tmp" 2>/dev/null && mv -f "$_tmp" "$_mdir/apiscp-borg.prom" 2>/dev/null \
		|| akp_warn "failed to write metrics to $_mdir"
}

notify() {
	[ -n "${NOTIFY_EMAIL:-}" ] || return 0
	_n_ok=0; { [ "$FAILED" -eq 0 ] && [ "$_reached_end" -eq 1 ]; } && _n_ok=1
	case "${NOTIFY_ON:-failure}" in
		never)  return 0 ;;
		always) ;;
		*)      [ "$_n_ok" -eq 1 ] && return 0 ;;
	esac
	_n_host=$(hostname 2>/dev/null || echo apiscp)
	_n_dur=$(( $(date +%s 2>/dev/null || echo 0) - _start_epoch ))
	_n_res=$( [ "$_n_ok" -eq 1 ] && echo success || echo FAILURE )
	_n_db=$( case "$DB_EXPORT_RC" in 1) echo ok ;; 0) echo failed ;; *) echo skipped ;; esac )
	_n_body=$(printf 'apiscp-borg backup run on %s\n\nResult:    %s\nFinished:  %s\nArchives:  %s ok, %s failed\nDatabases: %s\nDuration:  %ss\nRepository: %s\n' \
		"$_n_host" "$_n_res" "$(_akp_now)" "$OK" "$FAILED" "$_n_db" "$_n_dur" "$BORG_REPO")
	[ "$_n_ok" -eq 1 ] || _n_body="$_n_body
See 'journalctl -t ${APISCP_BORG_LOG_TAG:-apiscp-borg}' on $_n_host for details."
	akp_send_mail "$NOTIFY_EMAIL" "[apiscp-borg] backup $_n_res on $_n_host" "$_n_body" "${NOTIFY_FROM:-}" \
		&& akp_log "notification emailed to $NOTIFY_EMAIL" || akp_warn "notification email failed"
}

on_exit() { write_metrics; notify; }
trap on_exit EXIT INT TERM

# Build the shared --exclude argument list (always exclude the restore staging).
_add_excludes() {
	set -- --exclude '*/.borg-restore' --exclude '/.borg-restore'
	_oifs=$IFS; IFS=',
'
	for _p in ${BACKUP_EXCLUDES:-}; do
		[ -n "$_p" ] && set -- "$@" --exclude "$_p"
	done
	IFS=$_oifs
	EXCLUDE_ARGS="$*"   # only used for logging; real calls use "$@" below
	BORG_EXCLUDES_SET=1
	# Export via positional re-use is awkward in sh; callers rebuild instead.
}

# create_archive NAME PATH...: borg create with excludes; update counters.
create_archive() {
	_name="$1"; shift
	[ "$#" -ge 1 ] || return 0
	# Assemble excludes fresh each call.
	set -- "$@"   # keep source paths in "$@"
	_srcs="$*"
	# shellcheck disable=SC2086
	if _create_with_excludes "$_name" "$@"; then
		OK=$(( OK + 1 )); akp_log "archived $_name"
	else
		FAILED=$(( FAILED + 1 )); akp_warn "archive FAILED: $_name ($_srcs)"
	fi
}

# _create_with_excludes NAME SRC...: run borg create '::NAME-{now}' with excludes.
_create_with_excludes() {
	_cn="$1"; shift
	# Build: create [excludes] ::NAME-{now} SRC...
	set -- ::"$_cn"-{now} "$@"
	# Prepend excludes.
	_ex_oifs=$IFS; IFS=',
'
	for _p in ${BACKUP_EXCLUDES:-}; do
		[ -n "$_p" ] && set -- --exclude "$_p" "$@"
	done
	IFS=$_ex_oifs
	set -- --exclude '*/.borg-restore' --exclude '/.borg-restore' "$@"
	akp_borg create --compression "${BORG_COMPRESSION:-zstd}" "$@" >/dev/null 2>&1
}

# prune_prefix PREFIX: apply retention to archives named PREFIX-* if any keep rule set.
prune_prefix() {
	_pref="$1"
	set --
	[ -n "$RETENTION_KEEP_WITHIN" ]  && set -- "$@" --keep-within "$RETENTION_KEEP_WITHIN"
	[ -n "$RETENTION_KEEP_DAILY" ]   && set -- "$@" --keep-daily "$RETENTION_KEEP_DAILY"
	[ -n "$RETENTION_KEEP_WEEKLY" ]  && set -- "$@" --keep-weekly "$RETENTION_KEEP_WEEKLY"
	[ -n "$RETENTION_KEEP_MONTHLY" ] && set -- "$@" --keep-monthly "$RETENTION_KEEP_MONTHLY"
	[ -n "$RETENTION_KEEP_ANNUAL" ]  && set -- "$@" --keep-yearly "$RETENTION_KEEP_ANNUAL"
	[ "$#" -ge 1 ] || return 0
	if akp_borg prune --glob-archives "$_pref-*" "$@" >/dev/null 2>&1; then
		akp_log "pruned $_pref-* per retention"
	else
		akp_warn "prune failed for $_pref-*"
	fi
}

# ==============================================================================
akp_log "run start (repo: $BORG_REPO)"

# 1. Refresh per-site database exports before archiving them.
if [ "$RUN_DB_EXPORT" = "1" ]; then
	if [ -x "$DB_EXPORT_CMD" ] || [ -r "$DB_EXPORT_CMD" ]; then
		akp_log "running ApisCP database export: $DB_EXPORT_CMD"
		if "$DB_EXPORT_CMD" >/dev/null 2>&1; then DB_EXPORT_RC=1; else DB_EXPORT_RC=0; akp_warn "database export returned non-zero"; fi
	else
		akp_warn "DB_EXPORT_CMD not found ($DB_EXPORT_CMD); skipping"
	fi
fi

# 2 + 3. Per-site archives and databases.
if [ -d "$VIRTBASE" ]; then
	for sitedir in "$VIRTBASE"/site[0-9]*; do
		[ -d "$sitedir" ] || continue
		site=$(basename "$sitedir")
		akp_site_selected "$site" || { akp_log "skip $site (not in BACKUP_SITES)"; continue; }

		[ -d "$sitedir/shadow" ] && create_archive "$site-shadow" "$sitedir/shadow"
		[ -d "$sitedir/info" ]   && create_archive "$site-info"   "$sitedir/info"

		if [ "$BACKUP_DATABASES" = "1" ]; then
			dbdom=$(akp_site_domain "$site")
			if [ -z "$dbdom" ]; then
				akp_warn "no domain for $site; skipping database dump"
			elif [ ! -x "$APNSCP_CMD" ] && [ ! -r "$APNSCP_CMD" ]; then
				akp_warn "APNSCP_CMD not found ($APNSCP_CMD); skipping database dump for $site"
			else
				db_siterel="/tmp/.apiscp-borg-db"
				dbroot="$sitedir/fst$db_siterel"
				rm -rf "$dbroot"
				_dbany=0
				for eng in mysql pgsql; do
					dblist=$("$APNSCP_CMD" -o json -d "$dbdom" "$eng:list_databases" 2>/dev/null) || continue
					[ -n "$dblist" ] || continue
					for db in $(printf '%s\n' "$dblist" | grep -oE '"[A-Za-z0-9_-]+"' | sed 's/"//g'); do
						[ -n "$db" ] || continue
						if "$APNSCP_CMD" -d "$dbdom" "$eng:export" "$db" "$db_siterel/$eng/$db.sql" >/dev/null 2>&1; then
							_dbany=1; akp_log "exported $eng db $db for $site"
						else
							akp_warn "failed to export $eng db $db for $site"
						fi
					done
				done
				[ "$_dbany" = 1 ] && [ -d "$dbroot" ] && create_archive "$site-db" "$dbroot"
				rm -rf "$dbroot"
			fi
		fi

		# Per-site retention.
		prune_prefix "$site-shadow"
		prune_prefix "$site-info"
		prune_prefix "$site-db"
	done
else
	akp_warn "VIRTBASE $VIRTBASE not present; no per-site backups"
fi

# 4. Shared system paths (one archive), then custom paths.
if [ "$BACKUP_SYSTEM" = "1" ]; then
	# Expand globs; keep only existing paths.
	set --
	# shellcheck disable=SC2086
	for p in $SYSTEM_PATHS $SYSTEM_GLOBS; do
		[ -e "$p" ] && set -- "$@" "$p"
	done
	[ "$#" -ge 1 ] && create_archive "_system" "$@"
	prune_prefix "_system"
else
	akp_log "skipping shared system paths (BACKUP_SYSTEM=0)"
fi

if [ -n "$BACKUP_PATHS" ]; then
	_paths=$(printf '%s' "$BACKUP_PATHS" | tr ',' ' ')
	for cp in $_paths; do
		[ -e "$cp" ] || { akp_warn "skip custom path (missing): $cp"; continue; }
		slug=$(akp_slug "$cp")
		create_archive "_custom-$slug" "$cp"
		prune_prefix "_custom-$slug"
	done
fi

akp_log "run done: $OK ok, $FAILED failed"
_reached_end=1
[ "$FAILED" -eq 0 ]