#!/usr/bin/env bash
# =============================================================================
# V-Connect PBX — Trunk health monitor (READ-ONLY)
# =============================================================================
# Samples inbound AND outbound trunk health and appends one line per run to
# /var/log/pbx-trunk-monitor.log. Designed to run from cron every 30s.
#
# WHY THIS EXISTS
#   meyers-motors had intermittent inbound + outbound call failures. To pin the
#   cause we need a continuous record of: is the provider registration up, is
#   the trunk endpoint reachable (qualify), how many calls are active, and have
#   any outbound calls failed (CONGESTION / CHANUNAVAIL / SIP 4xx-5xx) since the
#   last sample.
#
#   The first version of this monitor logged blank reg=/ecn= columns because it
#   called bare `asterisk`, which is NOT on cron's minimal PATH (asterisk lives
#   in /usr/sbin). This script resolves the binary by absolute path so it works
#   under cron.
#
# SAFE: changes nothing. Only reads Asterisk CLI + the CDR table.
#
# Install (run once on the box):
#   sudo bash /var/www/html/scripts/trunk-monitor.sh --install
# Manual one-shot sample (prints the line and appends it):
#   sudo bash /var/www/html/scripts/trunk-monitor.sh
# =============================================================================

set -uo pipefail

TRUNK="${PBX_MONITOR_TRUNK:-ECN}"
LOG="/var/log/pbx-trunk-monitor.log"
OFFSET_FILE="/var/lib/pbx-trunk-monitor.offset"   # byte offset into asterisk full log
AST_FULL_LOG="/var/log/asterisk/full"

# --- Resolve the asterisk binary by absolute path (cron-safe) ----------------
AST=""
for c in /usr/sbin/asterisk /usr/bin/asterisk "$(command -v asterisk 2>/dev/null)"; do
    if [[ -n "$c" && -x "$c" ]]; then AST="$c"; break; fi
done
ASTDB() { [[ -n "$AST" ]] && "$AST" -rx "$1" 2>/dev/null; }

# --- Resolve mysql for CDR-based outbound failure counting -------------------
MYSQL=""
for c in /usr/bin/mysql /usr/local/bin/mysql "$(command -v mysql 2>/dev/null)"; do
    if [[ -n "$c" && -x "$c" ]]; then MYSQL="$c"; break; fi
done
SQL() { [[ -n "$MYSQL" ]] && "$MYSQL" -N -u root asterisk -e "$1" 2>/dev/null; }

# =============================================================================
# --install: drop a cron entry that runs this every 30s
# =============================================================================
if [[ "${1:-}" == "--install" ]]; then
    SELF="$(readlink -f "$0")"
    CRON_LINE="* * * * * /usr/bin/env bash $SELF >/dev/null 2>&1"
    CRON_LINE2="* * * * * ( sleep 30 ; /usr/bin/env bash $SELF ) >/dev/null 2>&1"
    tmp="$(mktemp)"
    # Drop any prior monitor cron lines (this script's path AND the older
    # /usr/local/bin/trunk-monitor.sh install) so we never double-log.
    crontab -l 2>/dev/null | grep -v "$SELF" | grep -vi 'trunk-monitor' > "$tmp" || true
    echo "$CRON_LINE"  >> "$tmp"
    echo "$CRON_LINE2" >> "$tmp"
    crontab "$tmp"
    rm -f "$tmp"
    touch "$LOG"; chmod 644 "$LOG"
    echo "Installed: $SELF runs every 30s -> $LOG (trunk=$TRUNK)"
    exit 0
fi

# =============================================================================
# 1) Registration state for the trunk
#    "pjsip show registrations" trunk line looks like:
#      ECN/sip:...    ECN    Registered (exp. 2973s)
#    Grab the status WORD (Registered/Rejected/...) and the expiry seconds
#    separately so the column reads e.g. reg=Registered/2971s.
# =============================================================================
REGLINE="$(ASTDB 'pjsip show registrations' \
        | awk -v t="$TRUNK" 'tolower($0) ~ tolower(t) && $0 !~ /ServerURI|====/ {print; exit}')"
REG_STATUS="$(printf '%s' "$REGLINE" | grep -oiE 'Registered|Rejected|Unregistered|Registering|Stopped|None' | head -n1)"
REG_EXP="$(printf '%s' "$REGLINE" | grep -oiE '[0-9]+s' | head -n1)"
if [[ -n "$REG_STATUS" ]]; then
    REG="$REG_STATUS${REG_EXP:+/$REG_EXP}"
else
    REG="none"
fi

# =============================================================================
# 2) Endpoint qualify state (is the trunk reachable right now?)
#    "pjsip show endpoint <T>" prints a status line; capture Avail/Unavail.
# =============================================================================
ECN_STATE="$(ASTDB "pjsip show endpoint $TRUNK" \
        | awk '/Contact:|Aor:/ {next} /Avail|Unavail|Unreachable|Reachable|NonQual|Created|Removed/ {
                 if (match($0, /(Avail|Unavail|Unreachable|Reachable|NonQual|Created|Removed)[a-z]*/)) {
                     print substr($0, RSTART, RLENGTH); exit
                 }
             }')"
# Fall back to contact RTT/status if endpoint summary gave nothing.
if [[ -z "$ECN_STATE" ]]; then
    ECN_STATE="$(ASTDB "pjsip show contacts" \
        | awk -v t="$TRUNK" 'tolower($0) ~ tolower(t) {print; exit}' \
        | grep -oiE 'Avail|Unavail|Unreachable|Reachable|NonQual' | head -n1)"
fi
[[ -z "$ECN_STATE" ]] && ECN_STATE="unknown"

# =============================================================================
# 3) Active channels / calls (outbound capacity signal)
# =============================================================================
CHANS="$(ASTDB 'core show channels count' | awk '/active channel/ {print $1; exit}')"
CALLS="$(ASTDB 'core show channels count' | awk '/active call/ {print $1; exit}')"
[[ -z "$CHANS" ]] && CHANS="?"
[[ -z "$CALLS" ]] && CALLS="?"

# =============================================================================
# 3b) Local device reachability (THE phone-side signal)
#     The trunk can be perfectly healthy while every desk phone has dropped its
#     registration/qualify — which makes inbound ring groups AND internal calls
#     return NO ANSWER even though ECN is up. Track how many PJSIP contacts are
#     reachable vs total so a mass phone drop is captured. The trunk's own
#     contact is excluded so this reflects phones only.
# =============================================================================
CONTACTS="$(ASTDB 'pjsip show contacts')"
DEV_TOTAL="$(printf '%s\n' "$CONTACTS" | grep -c 'Contact:')"
DEV_AVAIL="$(printf '%s\n' "$CONTACTS" | grep -aE 'Contact:' | grep -aic 'Avail')"
# Subtract the trunk contact if it is currently listed/avail so devs= reflects phones.
if printf '%s\n' "$CONTACTS" | grep -aE 'Contact:' | grep -ai "$TRUNK" | grep -aiq 'Avail'; then
    DEV_TOTAL=$((DEV_TOTAL>0 ? DEV_TOTAL-1 : 0))
    DEV_AVAIL=$((DEV_AVAIL>0 ? DEV_AVAIL-1 : 0))
fi
DEVS="${DEV_AVAIL}/${DEV_TOTAL}"

# =============================================================================
# 4) Outbound failures since last sample (CDR-based, the strongest signal)
#    Counts CDRs in the last 2 minutes whose disposition indicates the call
#    could not complete out the trunk: FAILED / CONGESTION / BUSY is normal but
#    a *burst* of FAILED/CONGESTION is the outage signature.
# =============================================================================
OUTFAIL="$(SQL "SELECT COUNT(*) FROM cdr WHERE calldate >= (NOW() - INTERVAL 2 MINUTE) AND disposition IN ('FAILED','CONGESTION');")"
[[ -z "$OUTFAIL" ]] && OUTFAIL="?"

# =============================================================================
# 5) New SIP error responses on the trunk in the asterisk log since last run
#    Uses a saved byte offset so we only scan NEW log lines (cheap even on a
#    large log). Looks for 403/408/503/603 and auth failures referencing the
#    trunk — these accompany outbound registration/call drops.
# =============================================================================
SIPERR=0
if [[ -r "$AST_FULL_LOG" ]]; then
    SIZE="$(stat -c %s "$AST_FULL_LOG" 2>/dev/null || echo 0)"
    PREV="$(cat "$OFFSET_FILE" 2>/dev/null || echo 0)"
    # Log rotated (smaller than last offset) → start from 0.
    [[ "$SIZE" -lt "$PREV" ]] && PREV=0
    if [[ "$SIZE" -gt "$PREV" ]]; then
        SIPERR="$(tail -c +"$((PREV+1))" "$AST_FULL_LOG" 2>/dev/null \
            | grep -aiE "$TRUNK" \
            | grep -aicE '403 Forbidden|408 Request Timeout|503 Service Unavailable|603 Decline|Failed to authenticate|registration.*(Rejected|stopped|Unregistered)')"
    fi
    echo "$SIZE" > "$OFFSET_FILE" 2>/dev/null || true
fi
[[ -z "$SIPERR" ]] && SIPERR=0

# =============================================================================
# 6) Load average (always available from /proc)
# =============================================================================
LOAD="$(awk '{print $1}' /proc/loadavg 2>/dev/null)"
[[ -z "$LOAD" ]] && LOAD="?"

TS="$(date '+%F %T')"
LINE="$TS | reg=$REG | ecn=$ECN_STATE | devs=$DEVS | chans=$CHANS | calls=$CALLS | outfail2m=$OUTFAIL | siperr=$SIPERR | load=$LOAD"

echo "$LINE" >> "$LOG"
echo "$LINE"