diff --git a/CHANGELOG.md b/CHANGELOG.md index fff6e88..51ba3d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ This project does not follow semantic versioning (it is a script collection, not ## [Unreleased] +### Changed + +- `monitor-cpu-load.sh`: the `[RESOLVED]` email now reports when the incident was alerted and resolved, the peak load and its time, and the cause, CPU per subscription, offenders and top processes from the peak. The peak snapshot is kept in `STATE_DIR/incident_peak` while an incident is open. + ## v2026.10.10 ### Fixed diff --git a/monitor-cpu-load/README.md b/monitor-cpu-load/README.md index 3d36aea..f0edc50 100644 --- a/monitor-cpu-load/README.md +++ b/monitor-cpu-load/README.md @@ -41,7 +41,8 @@ Linux only (`monitor-cpu-load.sh`), AlmaLinux Plesk servers, root required. No W - `[ALERT]`: sustained load that fail2ban is not handling; the email lists CPU per subscription, the busiest offenders (with the jails holding them) and the likely cause - `[NOTICE]`: fail2ban is banning the attack but load stayed high for another full window -- `[REMINDER]` / `[RESOLVED]`: the incident is still ongoing (every `REMINDER_HOURS`), or load is back to normal +- `[REMINDER]`: the incident is still ongoing (every `REMINDER_HOURS`); it repeats the full analysis +- `[RESOLVED]`: load is back to normal; the email gives when it alerted and resolved, the peak load and its time, and the cause, CPU per subscription, offenders and processes captured at the peak ## Configuration diff --git a/monitor-cpu-load/monitor-cpu-load.sh b/monitor-cpu-load/monitor-cpu-load.sh index 3cb294c..1d46a52 100755 --- a/monitor-cpu-load/monitor-cpu-load.sh +++ b/monitor-cpu-load/monitor-cpu-load.sh @@ -295,6 +295,7 @@ SMTP_FROM="${SMTP_FROM:-}" SAMPLES_FILE="${STATE_DIR}/samples" STATE_FILE="${STATE_DIR}/state" +PEAK_DIR="${STATE_DIR}/incident_peak" PIDFILE="${STATE_DIR}/monitor.pid" HOSTNAME_SHORT="$(hostname -s 2>/dev/null || hostname)" WORK="" @@ -409,6 +410,8 @@ load_state() { INCIDENT_START=0 LAST_ALERT_TS=0 LAST_REMINDER_TS=0 + INCIDENT_PEAK_LOAD=0 + INCIDENT_PEAK_TS=0 ALERT_SENT="false" [ -f "${STATE_FILE}" ] || return 0 @@ -425,11 +428,16 @@ load_state() { ALERT_SENT="true" fi ;; - MITIGATED_SINCE|INCIDENT_START|LAST_ALERT_TS|LAST_REMINDER_TS) + MITIGATED_SINCE|INCIDENT_START|LAST_ALERT_TS|LAST_REMINDER_TS|INCIDENT_PEAK_TS) if [[ "${value}" =~ ^[0-9]+$ ]]; then printf -v "${key}" '%s' "${value}" fi ;; + INCIDENT_PEAK_LOAD) + if [[ "${value}" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then + INCIDENT_PEAK_LOAD="${value}" + fi + ;; esac done < "${STATE_FILE}" return 0 @@ -442,6 +450,8 @@ save_state() { echo "INCIDENT_START=${INCIDENT_START}" echo "LAST_ALERT_TS=${LAST_ALERT_TS}" echo "LAST_REMINDER_TS=${LAST_REMINDER_TS}" + echo "INCIDENT_PEAK_LOAD=${INCIDENT_PEAK_LOAD}" + echo "INCIDENT_PEAK_TS=${INCIDENT_PEAK_TS}" echo "ALERT_SENT=${ALERT_SENT}" } > "${STATE_FILE}.tmp" mv -f "${STATE_FILE}.tmp" "${STATE_FILE}" @@ -635,23 +645,71 @@ build_html_report() { printf '

fail2ban: banned offenders account for %s%% of attack traffic (%s of %s requests); handling threshold %s%%.

\n' \ "${HANDLED_PCT}" "${BANNED_TOTAL}" "${ATTACK_TOTAL}" "${HANDLED_PERCENT}" + print_analysis_tables "${WORK}" + printf '\n' +} + +# Args: directory holding sub_cpu, sub_requests, offenders and proc_comm_cpu +print_analysis_tables() { + local dir="$1" printf '

CPU by subscription (%% of one core)

\n' - head -n 10 "${WORK}/sub_cpu" | awk '{ printf "%s\t%s\t%s\n", $1, $2, ($3 == 1) ? "subscription" : "system user" }' \ + head -n 10 "${dir}/sub_cpu" | awk '{ printf "%s\t%s\t%s\n", $1, $2, ($3 == 1) ? "subscription" : "system user" }' \ | html_table "Subscription / user" "CPU %" "Type" - if [ -s "${WORK}/sub_requests" ]; then + if [ -s "${dir}/sub_requests" ]; then printf '

Web requests in the last %s minutes (busiest subscriptions)

\n' "${ANALYSIS_MINUTES}" - html_table "Subscription" "Requests" < "${WORK}/sub_requests" + html_table "Subscription" "Requests" < "${dir}/sub_requests" fi - if [ -s "${WORK}/offenders" ]; then + if [ -s "${dir}/offenders" ]; then printf '

Top offenders

\n' - head -n 10 "${WORK}/offenders" | awk -F'\t' '{ printf "%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, $4, ($5 == "-") ? "NOT banned" : "banned: " $5 }' \ + head -n 10 "${dir}/offenders" | awk -F'\t' '{ printf "%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, $4, ($5 == "-") ? "NOT banned" : "banned: " $5 }' \ | html_table "IP" "Requests" "401/403/404" "Probe hits" "fail2ban" fi printf '

Top processes by command (%% of one core)

\n' - head -n 5 "${WORK}/proc_comm_cpu" | awk '{ printf "%s\t%s\n", $1, $2 }' | html_table "Command" "CPU %" + head -n 5 "${dir}/proc_comm_cpu" | awk '{ printf "%s\t%s\n", $1, $2 }' | html_table "Command" "CPU %" +} + +# Keeps the analysis from the highest-load run of the current incident for the [RESOLVED] email. +update_incident_peak() { + if ! awk -v now="${LOAD_NOW}" -v peak="${INCIDENT_PEAK_LOAD}" 'BEGIN { exit !(now + 0 > peak + 0) }'; then + return 0 + fi + local file + INCIDENT_PEAK_LOAD="${LOAD_NOW}" + INCIDENT_PEAK_TS=$(date +%s) + mkdir -p "${PEAK_DIR}" + for file in sub_cpu sub_requests offenders proc_comm_cpu; do + cp -f "${WORK}/${file}" "${PEAK_DIR}/${file}" 2>/dev/null || : > "${PEAK_DIR}/${file}" + done + printf '%s\n' "${CAUSE}" > "${PEAK_DIR}/cause" + printf '%s\t%s\t%s\n' "${HANDLED_PCT}" "${BANNED_TOTAL}" "${ATTACK_TOTAL}" > "${PEAK_DIR}/f2b" +} + +# Args: incident duration in minutes +build_resolved_report() { + local minutes="$1" peak_cause="" pct banned total + printf '\n

CPU load back to normal

\n' + printf '

Server: %s
Load now: %s (threshold %s = %s x %s cores)
' \ + "$(html_escape "${HOSTNAME_SHORT}")" "${LOAD_NOW}" "${THRESHOLD}" "${LOAD_THRESHOLD_FACTOR}" "${CORES}" + printf 'Alerted at: %s
Resolved at: %s (about %s minutes later)
' \ + "$(date -d "@${INCIDENT_START}" '+%Y-%m-%d %H:%M %Z')" "$(date '+%Y-%m-%d %H:%M %Z')" "${minutes}" + if [ "${INCIDENT_PEAK_TS}" -gt 0 ]; then + printf 'Peak load: %s at %s
' "${INCIDENT_PEAK_LOAD}" "$(date -d "@${INCIDENT_PEAK_TS}" '+%H:%M')" + fi + if [ -r "${PEAK_DIR}/cause" ]; then + IFS= read -r peak_cause < "${PEAK_DIR}/cause" || true + printf 'Cause at peak: %s' "$(html_escape "${peak_cause}")" + fi + printf '

\n' + if [ -r "${PEAK_DIR}/f2b" ]; then + IFS=$'\t' read -r pct banned total < "${PEAK_DIR}/f2b" || true + printf '

fail2ban at peak: banned offenders accounted for %s%% of attack traffic (%s of %s requests).

\n' \ + "${pct:-0}" "${banned:-0}" "${total:-0}" + printf '

At the peak

\n' + print_analysis_tables "${PEAK_DIR}" + fi printf '\n' } @@ -733,6 +791,9 @@ raise_incident() { STATE="incident" INCIDENT_START="${now}" ALERT_SENT="false" + INCIDENT_PEAK_LOAD=0 + INCIDENT_PEAK_TS=0 + rm -rf "${PEAK_DIR}" log_message "INCIDENT ${kind}: load=${LOAD_NOW} threshold=${THRESHOLD} cause=\"${CAUSE}\" banned=${HANDLED_PCT}% of ${ATTACK_TOTAL} attack requests top=$(head -n 3 "${WORK}/sub_cpu" | awk '{ printf "%s(%s%%) ", $1, $2 }')" @@ -788,6 +849,10 @@ handle_sustained() { log_message "Incident ongoing: load=${LOAD_NOW} cause=\"${CAUSE}\"" ;; esac + + if [ "${STATE}" = "incident" ]; then + update_incident_peak + fi } handle_recovered() { @@ -799,8 +864,9 @@ handle_recovered() { log_message "RESOLVED after ${minutes} minutes: load=${LOAD_NOW}" if [ "${ALERT_SENT}" = "true" ]; then send_email "[RESOLVED] ${HOSTNAME_SHORT}: CPU load back to normal" \ - "

CPU load back to normal

Load is ${LOAD_NOW} (threshold ${THRESHOLD}). The incident lasted about ${minutes} minutes.

" || true + "$(build_resolved_report "${minutes}")" || true fi + rm -rf "${PEAK_DIR}" ;; mitigated) log_message "Load recovered after attack was handled by fail2ban" @@ -809,6 +875,8 @@ handle_recovered() { STATE="normal" MITIGATED_SINCE=0 INCIDENT_START=0 + INCIDENT_PEAK_LOAD=0 + INCIDENT_PEAK_TS=0 ALERT_SENT="false" }