gandalf/modem.py

"""Read-only client for AT&T BGW-series fiber gateways (tested on BGW320-505).

The gateway has no API, but its status pages are served without a login, so
this scrapes them over HTTP. Only GET requests to the fixed status pages below
are ever made: nothing here can change gateway settings.

Units come from the gateway's own help text, checked against live readings:
  Rx/Tx power  tenths of a dBm (-169 -> -16.9 dBm)
  Temperature  degrees C
  Uptime       seconds (despite the help text claiming D:H:M:S)

Deliberately not used, because the page disagrees with itself:
  Vcc / Tx Bias readings  values do not match the documented scales, so only
                          their alarm/warning flags are kept
  Rx LOS State            reads 1 on a healthy link; "Optical WAN Operational
                          Status" is the reliable signal
  Last Change             does not count up, despite the help text
"""
import html
import logging
import re
from datetime import datetime, timezone
from typing import Dict, List, Optional

import requests

logger = logging.getLogger('gandalf.modem')

_FIBER_PAGE     = '/cgi-bin/fiberstat.ha'
_BROADBAND_PAGE = '/cgi-bin/broadbandstatistics.ha'
_SYSINFO_PAGE   = '/cgi-bin/sysinfo.ha'

_ROW_RE = re.compile(r'<th\b[^>]*>(.*?)</th>\s*<td\b[^>]*>(.*?)</td>', re.I | re.S)
_TAG_RE = re.compile(r'<[^>]+>')
# e.g. "0                 (Threshold -292)": alarm flag, then the limit
_DOM_CELL_RE = re.compile(r'<td\b[^>]*>\s*(\d+)\s*\(Threshold\s+(-?\d+)\)\s*</td>', re.I)

# Page heading -> (result key, scale to real units; None = reading is unusable)
_DOM_METRICS = {
    'Temperature': ('temperature', 1.0),
    'Vcc':         ('vcc',         None),
    'Tx Bias':     ('tx_bias',     None),
    'Tx Power':    ('tx_power',    0.1),
    'Rx Power':    ('rx_power',    0.1),
}

_COUNTERS = {
    'rx_bytes':  'Receive Bytes',
    'tx_bytes':  'Transmit Bytes',
    'rx_errors': 'Receive Errors',
    'tx_errors': 'Transmit Errors',
    'rx_drops':  'Receive Drops',
    'tx_drops':  'Transmit Drops',
}


class ModemUnreachable(Exception):
    """The gateway did not answer (power, cabling, or routing problem)."""


class ModemDataInvalid(Exception):
    """The gateway answered but the page was not what we can parse.

    AT&T pushes firmware automatically, so this is how a layout change or a
    newly login-protected page shows up.
    """


def parse_rows(page: str) -> Dict[str, str]:
    """Return {label: value} for the page's <th>/<td> rows; the first label wins.

    First wins because labels repeat: "Transmit Packets" appears under both
    IPv4 and IPv6, and the IPv4 block comes first.
    """
    rows: Dict[str, str] = {}
    for label, value in _ROW_RE.findall(page):
        key = html.unescape(_TAG_RE.sub('', label)).strip()
        if key:
            rows.setdefault(key, ' '.join(html.unescape(_TAG_RE.sub('', value)).split()))
    return rows


def parse_dom_metrics(page: str) -> Dict[str, dict]:
    """Parse the fiber page's Temperature / Vcc / Tx Bias / Tx Power / Rx Power tables."""
    metrics: Dict[str, dict] = {}
    for heading, (key, scale) in _DOM_METRICS.items():
        m = re.search(
            rf'<h1>\s*{re.escape(heading)}(?:&nbsp;|\s)+Currently\s+(-?\d+)\s*</h1>(.*?)</table>',
            page, re.I | re.S,
        )
        if not m:
            continue
        cells = _DOM_CELL_RE.findall(m.group(2))
        if len(cells) != 4:  # alarm low, alarm high, warning low, warning high
            continue

        def limit(raw: str, scale: Optional[float] = scale) -> Optional[float]:
            return round(int(raw) * scale, 1) if scale is not None else None

        metrics[key] = {
            'value':       limit(m.group(1)),
            'alarm_low':   limit(cells[0][1]),
            'alarm_high':  limit(cells[1][1]),
            'warn_low':    limit(cells[2][1]),
            'warn_high':   limit(cells[3][1]),
            'alarm_flag':  cells[0][0] != '0' or cells[1][0] != '0',
            'warn_flag':   cells[2][0] != '0' or cells[3][0] != '0',
        }
    return metrics


def metric_status(metric: dict) -> str:
    """Classify a DOM metric as 'alarm', 'warning', or 'ok'.

    Trusts the module's own flags, and also compares the reading to its limits
    in case the flags are latched off.
    """
    value = metric.get('value')

    def outside(low: Optional[float], high: Optional[float]) -> bool:
        if value is None:
            return False
        return (low is not None and value <= low) or (high is not None and value >= high)

    if metric.get('alarm_flag') or outside(metric.get('alarm_low'), metric.get('alarm_high')):
        return 'alarm'
    if metric.get('warn_flag') or outside(metric.get('warn_low'), metric.get('warn_high')):
        return 'warning'
    return 'ok'


def _to_int(raw: Optional[str]) -> Optional[int]:
    try:
        return int(raw)  # type: ignore[arg-type]
    except (TypeError, ValueError):
        return None


def wan_problems(data: dict) -> List[str]:
    """Return human-readable reasons the WAN is down; empty when it looks healthy."""
    problems = []
    if (data.get('broadband_state') or '').lower() != 'up':
        problems.append(f'broadband connection is {data.get("broadband_state") or "unknown"}')
    if (data.get('optical_state') or '').lower() != 'up':
        problems.append(f'optical WAN is {data.get("optical_state") or "unknown"}')
    pon = data.get('pon_state')
    if pon and 'O5' not in pon:  # O5 is the PON "operation" state; older firmware may omit the field
        problems.append(f'PON link state is {pon}')
    return problems


def build_status(fiber_page: str, broadband_page: str, sysinfo_page: str = '') -> dict:
    """Combine the status pages into one dict. Raises ModemDataInvalid if unusable."""
    fiber = parse_rows(fiber_page)
    broadband = parse_rows(broadband_page)
    sysinfo = parse_rows(sysinfo_page)

    if 'Optical WAN Operational Status' not in fiber:
        raise ModemDataInvalid('fiber page is missing "Optical WAN Operational Status"')
    if 'Broadband Connection' not in broadband:
        raise ModemDataInvalid('broadband page is missing "Broadband Connection"')

    metrics = parse_dom_metrics(fiber_page)
    data = {
        'model':            sysinfo.get('Model Number', ''),
        'firmware':         sysinfo.get('Software Version', ''),
        'uptime_s':         _to_int(sysinfo.get('Time Since Last Reboot')),
        'broadband_state':  broadband.get('Broadband Connection', ''),
        'source':           broadband.get('Broadband Connection Source', ''),
        'wan_ip':           broadband.get('Broadband IPv4 Address', ''),
        'pon_state':        broadband.get('PON Link Status', ''),
        'uni_state':        broadband.get('UNI Status', ''),
        'line_speed_mbps':  _to_int(broadband.get('Current Speed (Mbps)')),
        'optical_state':    fiber.get('Optical WAN Operational Status', ''),
        'wavelength':       fiber.get('Wave Length', ''),
        'optics_part_no':   fiber.get('Vendor PN', ''),
        'optics':           metrics,
        'counters':         {k: _to_int(broadband.get(label)) for k, label in _COUNTERS.items()},
    }

    # With a healthy link the optics must be readable; if not, the layout changed.
    # With a cut fiber they may legitimately be blank, so don't report that as invalid.
    if not wan_problems(data) and 'rx_power' not in metrics:
        raise ModemDataInvalid('link is up but no Rx power reading was found')
    return data


def counter_rates(current: dict, previous: Optional[dict], elapsed_s: float) -> Dict[str, Optional[float]]:
    """Return per-second rates between counter snapshots.

    A rate is None rather than a bogus value on the first poll, or when a counter
    goes backwards because the gateway rebooted.
    """
    rates: Dict[str, Optional[float]] = {}
    for key, now_val in current.items():
        then = (previous or {}).get(key)
        if now_val is None or then is None or elapsed_s <= 0 or now_val < then:
            rates[key] = None
        else:
            rates[key] = (now_val - then) / elapsed_s
    return rates


class ModemClient:
    def __init__(self, cfg: dict):
        self.base_url = cfg['url'].rstrip('/')
        self.timeout = cfg.get('timeout', 10)
        self.session = requests.Session()

    def _get(self, path: str) -> str:
        try:
            # Protected pages redirect or render a login form, so never follow redirects.
            resp = self.session.get(f'{self.base_url}{path}', timeout=self.timeout, allow_redirects=False)
        except requests.RequestException as e:
            raise ModemUnreachable(f'{path}: {e}') from e
        if resp.status_code != 200:
            raise ModemDataInvalid(f'{path} returned HTTP {resp.status_code}')
        if re.search(r'<title>\s*Login', resp.text, re.I):
            raise ModemDataInvalid(f'{path} now requires a login')
        return resp.text

    def poll(self) -> dict:
        """Return gateway status. Raises ModemUnreachable or ModemDataInvalid."""
        fiber = self._get(_FIBER_PAGE)
        broadband = self._get(_BROADBAND_PAGE)
        try:
            sysinfo = self._get(_SYSINFO_PAGE)
        except (ModemUnreachable, ModemDataInvalid) as e:
            # Model and uptime are nice to have, not worth failing the poll over.
            logger.warning(f'Modem system info unavailable: {e}')
            sysinfo = ''
        data = build_status(fiber, broadband, sysinfo)
        data['updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
        return data


# ---------------------------------------------------------------------------
# WAN history: outages and gateway restarts, which are too brief to catch by polling
# ---------------------------------------------------------------------------
LOG_WAN_FAILED = 'NETWORK_WAN_FAILED_2'
LOG_WAN_RESTORED = 'NETWORK_WAN_RESTORED_2'
LOG_WAN_UNSTABLE = 'NETWORK_WAN_FAILED_MULTIPLE_TIMES_2'
LOG_MODEM_RESTARTED = 'SMART_POWER_MODEM_RESTARTED'
WAN_LOG_KEYS = (LOG_WAN_FAILED, LOG_WAN_RESTORED, LOG_WAN_UNSTABLE, LOG_MODEM_RESTARTED)

_HISTORY_LIMIT = 50
# The RPDU's log entry and the gateway's calculated boot time can differ by this much.
_RESTART_MATCH_WINDOW_S = 600


def apply_wan_events(history: dict, events: List[dict]) -> List[dict]:
    """Fold UniFi log events into *history* in place and return the ones worth a ticket.

    Each event is {'key', 'ts' (epoch seconds), 'device', 'wan'}.
    """
    outages = history.setdefault('outages', [])
    restarts = history.setdefault('restarts', [])
    actionable = []
    for e in sorted(events, key=lambda ev: ev['ts']):
        key = e['key']
        if key == LOG_WAN_FAILED:
            if not any(o.get('restored_at') is None for o in outages):
                outages.append({'failed_at': e['ts'], 'restored_at': None, 'wan': e.get('wan', '')})
        elif key == LOG_WAN_RESTORED:
            for o in reversed(outages):
                if o.get('restored_at') is None:
                    o['restored_at'] = e['ts']
                    break
        elif key == LOG_MODEM_RESTARTED:
            restarts.append({'at': e['ts'], 'by': e.get('device') or 'smart power', 'confirmed': False})
            actionable.append(e)
        elif key == LOG_WAN_UNSTABLE:
            actionable.append(e)
    del outages[:-_HISTORY_LIMIT]
    del restarts[:-_HISTORY_LIMIT]
    return actionable


def record_uptime_reset(history: dict, boot_ts: float) -> None:
    """Note that the gateway's uptime went backwards, attributing it to a logged restart if possible."""
    restarts = history.setdefault('restarts', [])
    for r in reversed(restarts):
        if abs(r['at'] - boot_ts) <= _RESTART_MATCH_WINDOW_S:
            r['confirmed'] = True
            return
    restarts.append({'at': boot_ts, 'by': None, 'confirmed': True})
    del restarts[:-_HISTORY_LIMIT]