#!/bin/bash
#
# https://git.math.uzh.ch/typo3/qfq/-/blob/master/extension/Resources/Public/script/emailSync.sh
#
# Watchdog for the QFQ email-sync.php daemon(s): starts one daemon per configured TYPO3 instance,
# monitors them, restarts on failure and sends an alert mail after repeated failures.
#
# Deployment:
#   1. Copy this script and emailSync.service to the host (e.g. /etc/scripts/emailSync.sh,
#      /etc/systemd/system/emailSync.service), make the script executable.
#   2. Set SCRIPTS below to the absolute path(s) of email-sync.php (one per instance).
#   3. Set ADMIN_EMAIL (and SERVICE_USER if your webserver does not run as www-data).
#   4. systemctl daemon-reload && systemctl enable --now emailSync
#
# OAuth2 note (Outlook/Office365 accounts): the daemons READ AND WRITE the OAuth2 token files
# (plain JSON, mode 0600). Those files are created/owned by the webserver user (browser
# authorization in the QFQ emailSync report), so the daemons must run as the SAME user - see
# SERVICE_USER. If this script is started as root, it drops privileges via runuser automatically.
# An expired access token can end a daemon (IMAP reconnect fails); the automatic restart then
# resolves a fresh token via the refresh token. Occasional single restarts of instances with
# OAuth2 accounts are therefore normal - the alert mail only fires after MAX_ATTEMPTS consecutive
# failed restarts.

# ---------------------------------------------------------------------------------------------
# Configuration - adjust to your host
# ---------------------------------------------------------------------------------------------

# Alert mail recipient (requires a working 'mail' command / MTA on the host).
ADMIN_EMAIL="${EMAILSYNC_ADMIN_EMAIL:-root@localhost}"
EMAIL_SUBJECT="Email Sync Service Failure Alert"

# The daemons must run as the owner of the OAuth2 token files (= webserver user).
SERVICE_USER="${EMAILSYNC_SERVICE_USER:-www-data}"

# PHP executable.
PHP_BIN="${EMAILSYNC_PHP_BIN:-/usr/bin/php}"

# Absolute path(s) to email-sync.php - one entry per TYPO3 instance.
SCRIPTS=(
    "/var/www/html/typo3conf/ext/qfq/Classes/External/email-sync.php"
)

MAX_ATTEMPTS=10
ATTEMPT_DELAY=40
# Healthy cycles are logged as a single heartbeat line per ~1h (120 cycles x 30s) instead of
# per-cycle status spam in the journal.
HEARTBEAT_CYCLES=120
HEARTBEAT_COUNTER=0

# ---------------------------------------------------------------------------------------------

SERVER_DOMAIN=$(hostname -f)
SERVICE_PATH="${SERVICE_FILE_PATH:-Unknown}"

# Build the PHP launcher: run directly when already SERVICE_USER, drop privileges when root,
# otherwise warn - OAuth2 token files created by the web UI would not be read/writable.
if [ "$(id -un)" = "$SERVICE_USER" ]; then
    PHP_CMD=("$PHP_BIN")
elif [ "$(id -u)" -eq 0 ]; then
    PHP_CMD=(runuser -u "$SERVICE_USER" -- "$PHP_BIN")
    echo "$(date): Running as root - daemons are started as '$SERVICE_USER' via runuser"
else
    PHP_CMD=("$PHP_BIN")
    echo "$(date): WARNING - running as '$(id -un)' instead of '$SERVICE_USER': OAuth2 token files (mode 0600, owned by $SERVICE_USER) may not be accessible"
fi

# Set up trap to kill child processes when script receives SIGTERM
trap 'echo "Received SIGTERM, shutting down..."; kill $(jobs -p); exit 0' TERM INT

# Function to send email
send_alert_email() {
    local script_path=$1
    local attempts=$2
    local error_output=$3

    if command -v realpath >/dev/null 2>&1; then
        if [[ "$script_path" != "ALL SCRIPTS"* ]]; then
            script_path=$(realpath "$script_path" 2>/dev/null || echo "$script_path")
        fi
    fi

    local email_body="
ALERT: Script failure detected

Server: $SERVER_DOMAIN
Service file: $SERVICE_PATH
Script: $script_path
Failed attempts to restart: $attempts
Time: $(date)
"

    if [ -n "$error_output" ]; then
        email_body+="

PHP Error Output:
----------------
$error_output
"
    fi

    email_body+="

This message was automatically generated by the email sync monitoring service.
"

    if command -v mail >/dev/null 2>&1; then
        echo -e "$email_body" | mail -s "$EMAIL_SUBJECT on $SERVER_DOMAIN" "$ADMIN_EMAIL"
        echo "$(date): Email alert sent for $script_path on $SERVER_DOMAIN"
    else
        echo "$(date): WARNING - 'mail' command not found, cannot send alert for $script_path"
    fi
}

# Capture diagnostic output of a failing script with a SINGLE sync run ('-s').
# Never probe with the default daemon mode: it polls forever and would block the watchdog
# (and run a duplicate sync). OAuth2 token errors show up here immediately.
probe_script_error() {
    local script_path=$1
    local output rc

    output=$(timeout 60 "${PHP_CMD[@]}" "$script_path" -s 2>&1)
    rc=$?
    if [ $rc -eq 124 ]; then
        output+=$'\n(single-sync probe timed out after 60s)'
    fi

    echo "$output"
}

echo "$(date): Initial startup of all scripts"

PIDS=()
FAILURES=()
EMAIL_SENT=()
ERROR_OUTPUTS=()

for ((i=0; i<${#SCRIPTS[@]}; i++)); do
    FULL_PATH="${SCRIPTS[i]}"

    "${PHP_CMD[@]}" "$FULL_PATH" > /dev/null 2>&1 &
    PIDS[i]=$!
    FAILURES[i]=0
    EMAIL_SENT[i]=false
    ERROR_OUTPUTS[i]=""

    sleep 1
    if ps -p ${PIDS[i]} > /dev/null; then
        echo "$(date): Started $(basename "${SCRIPTS[i]}") with PID: ${PIDS[i]}"
    else
        echo "$(date): WARNING - Failed to start $(basename "${SCRIPTS[i]}")"
        PIDS[i]=0
    fi
done

echo "$(date): Monitoring PIDs: ${PIDS[*]}"

# Monitor and restart if needed
while true; do
    atleastOneRunning=false

    for ((i=0; i<${#SCRIPTS[@]}; i++)); do
        script_name="${SCRIPTS[i]}"

        if [ ${PIDS[i]} -ne 0 ] && ps -p ${PIDS[i]} > /dev/null; then
            atleastOneRunning=true
            # Log state changes only - healthy scripts stay silent (see heartbeat below)
            if [ ${FAILURES[i]} -gt 0 ]; then
                echo "$(date): $script_name (PID ${PIDS[i]}) recovered after ${FAILURES[i]} failure(s)"
            fi
            FAILURES[i]=0
            EMAIL_SENT[i]=false
            ERROR_OUTPUTS[i]=""
        else
            echo "$(date): $script_name (PID ${PIDS[i]}) is NOT running, attempting restart..."
            FAILURES[i]=$((FAILURES[i] + 1))

            # Send email alert when MAX_ATTEMPTS is reached and no email has been sent yet
            if [ ${FAILURES[i]} -ge $MAX_ATTEMPTS ] && [ "${EMAIL_SENT[i]}" = false ]; then
                # Probe only now, right before the alert: the probe is a full single-sync run
                # (IMAP fetch + DB writes, up to 60s) - too expensive for every 30s crash-loop cycle.
                echo "$(date): $script_name failed ${FAILURES[i]} times, probing error output and sending alert"
                ERROR_OUTPUTS[i]=$(probe_script_error "$script_name")
                send_alert_email "$script_name" ${FAILURES[i]} "${ERROR_OUTPUTS[i]}"
                EMAIL_SENT[i]=true
            fi

            "${PHP_CMD[@]}" "$script_name" > /dev/null 2>&1 &
            PIDS[i]=$!

            sleep 1
            if ps -p ${PIDS[i]} > /dev/null; then
                echo "$(date): Restarted $script_name with new PID: ${PIDS[i]}"
                atleastOneRunning=true
            else
                echo "$(date): FAILED to restart $script_name"
                PIDS[i]=0
            fi
        fi
    done

    all_exhausted=true
    if [ "$atleastOneRunning" = false ]; then

        for ((i=0; i<${#SCRIPTS[@]}; i++)); do
            if [ ${FAILURES[i]} -lt $MAX_ATTEMPTS ]; then
                all_exhausted=false
                break
            fi
        done

        if [ "$all_exhausted" = true ]; then
            echo "$(date): CRITICAL: All processes have failed to start, exiting wrapper script"

            ALL_ERRORS=""
            for ((i=0; i<${#SCRIPTS[@]}; i++)); do
                if [ -n "${ERROR_OUTPUTS[i]}" ]; then
                    ALL_ERRORS="${ALL_ERRORS}--- ${SCRIPTS[i]} ---
    ${ERROR_OUTPUTS[i]}

    "
                fi
            done

            SCRIPT_LIST="ALL SCRIPTS:"
            for script in "${SCRIPTS[@]}"; do
                SCRIPT_LIST="${SCRIPT_LIST}
    - $script"
            done

            send_alert_email "$SCRIPT_LIST" "$MAX_ATTEMPTS+" "$ALL_ERRORS"
            exit 1
        fi
    fi

    if [ "$atleastOneRunning" = false ]; then
        echo "$(date): Status check complete - no script running, retry in $ATTEMPT_DELAY"
        sleep $ATTEMPT_DELAY
    else
        # Healthy cycles stay silent - the hourly heartbeat shows the watchdog itself is alive.
        HEARTBEAT_COUNTER=$((HEARTBEAT_COUNTER + 1))
        if [ $HEARTBEAT_COUNTER -ge $HEARTBEAT_CYCLES ]; then
            echo "$(date): Heartbeat - monitoring PIDs: ${PIDS[*]}"
            HEARTBEAT_COUNTER=0
        fi
        sleep 30
    fi
done
