diff --git a/README.md b/README.md index d629dd7..e708492 100644 --- a/README.md +++ b/README.md @@ -532,6 +532,16 @@ for read-only checks, stored in `monitor_state.modem_checks`: - **The gateway's own speed tests** (AT&T runs them from the modem), read with the NAT check; the latest down and up results are shown on the panel as a reference for what the line itself delivers. +**Restart modem** (admins, WAN / Fiber panel, needs `modem.access_code`): after a confirmation it +1. switches the `uptime_kuma` maintenance window on (if that fails it stops and asks whether to go on + without it), +2. starts Gandalf maintenance (a global suppression for up to 20 minutes, shown in the banner), +3. posts the gateway's Restart Device form. If that fails, steps 1 and 2 are undone. + +The monitor then waits for the gateway to answer with its WAN up and an uptime showing it restarted +(at least 3 minutes later), switches the Kuma window off, ends the suppression, and records how long it +took (`monitor_state.modem_restart`). After 30 minutes it ends both regardless. One restart at a time. + The login posts md5(access code + nonce), as the gateway's own page does, so the code itself is never sent. The session is kept between checks and renewed when it expires. A refused code is reported in Needs attention and not tried again for 30 minutes, and any other failure pauses the checks for 10 minutes, so @@ -680,6 +690,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/ | `modem.failure_threshold` | Consecutive failures before ticketing a modem alert (default: same as `failure_threshold`). Raise it if gateway reboots or automatic firmware updates cut tickets. | | `modem.rx_drift_db` | Rx drop below the 7-day median that raises `fiber_rx_drift` (default: 3.0) | | `modem.rx_drift_min_samples` | Readings required before drift alerts start (default: 288) | +| `uptime_kuma.url`, `.username`, `.password`, `.maintenance_id` | Optional. Lets the Restart modem button switch an Uptime Kuma maintenance window on and off (Kuma 2.x, over its Socket.IO login). Create the window once in Kuma with the *manual* strategy and the monitors a modem restart takes down, pause it, and put its id here. The account must not use two-factor login. | | `modem.access_code` | Optional. The gateway's Device Access Code (printed on its label). With it, Gandalf logs in to read NAT session counts and to ping from the gateway when the internet is slow (see WAN modem notes). Keep it only in `config.json`, which is git-ignored and mode 600. | --- @@ -865,7 +876,8 @@ curl http://10.10.10.48:9090/api/v1/query?query=up | jq '.data.result[] | select gandalf/ ├── app.py # Flask web app (routes, auth, API endpoints) ├── monitor.py # Background daemon (Prometheus, UniFi, Pulse, alert logic) -├── modem.py # AT&T gateway client: status pages, plus optional logged-in NAT and ping checks +├── kuma.py # Switches one Uptime Kuma maintenance window on/off (Restart modem) +├── modem.py # AT&T gateway client: status pages, plus optional logged-in checks and restart ├── devices.py # Device inventory for search and device pages ├── topology.py # The topology graph: devices, cables, depth, last-seen links ├── cutsheet.py # Cutsheet validation, path tracing, plan-vs-actual checks, CSV diff --git a/app.py b/app.py index d1f61eb..214340e 100644 --- a/app.py +++ b/app.py @@ -27,6 +27,8 @@ import cutsheet import db import devices import diagnose +import kuma +import modem as modem_api import topology from monitor import PulseClient, UnifiClient @@ -557,10 +559,71 @@ def api_modem(): data['wan_history'] = _state_json('wan_history') data['wan_quality'] = _state_json('wan_quality') or None data['modem_checks'] = _state_json('modem_checks') or None + data['modem_restart'] = _state_json('modem_restart') or None + data['can_restart'] = bool((_config().get('modem') or {}).get('access_code')) data['configured'] = True return jsonify(data) +# How long the silence lasts if the monitor never sees the modem come back to end it sooner +MODEM_RESTART_SILENCE_MIN = 20 + + +@app.route('/api/modem/restart', methods=['POST']) +@require_auth +@require_admin +def api_modem_restart(): + """Restart the AT&T gateway, with alerts silenced first. + + Body: {"confirm": true} and, after a Kuma failure the admin chose to accept, + "without_kuma": true. Order: Uptime Kuma maintenance on, Gandalf maintenance on, then + the restart; the first two are undone if the restart can't be sent. The monitor ends + both once the modem is back (monitor._finish_modem_restart). + """ + body = _json_body() + if body.get('confirm') is not True: + return jsonify({'error': 'confirm must be true'}), 400 + cfg = _config() + modem_cfg = cfg.get('modem') or {} + if not modem_cfg.get('url') or not modem_cfg.get('access_code'): + return jsonify({'error': 'Set modem.access_code in config.json first'}), 400 + pending = _state_json('modem_restart') + if pending and not pending.get('finished_at') and time.time() - pending.get('at', 0) < 1800: + return jsonify({'error': 'A restart is already in progress'}), 409 + + user = _get_user()['username'] + kuma_cfg = cfg.get('uptime_kuma') or {} + kuma_on = False + if kuma.configured(kuma_cfg): + try: + kuma.set_maintenance(kuma_cfg, True) + kuma_on = True + except kuma.KumaError as e: + logger.warning(f'Uptime Kuma maintenance could not be started: {e}') + if not body.get('without_kuma'): + return jsonify({'error': f'Uptime Kuma: {e}', 'kuma_failed': True}), 502 + + sup_id = db.create_suppression('all', '', '', 'Modem restart from Gandalf', user, + MODEM_RESTART_SILENCE_MIN) + try: + result = modem_api.ModemAdmin(modem_cfg).restart() + except (modem_api.ModemUnreachable, modem_api.ModemDataInvalid, modem_api.ModemLoginFailed) as e: + logger.error(f'Modem restart failed: {e}') + db.deactivate_suppression(sup_id) + if kuma_on: + try: + kuma.set_maintenance(kuma_cfg, False) + except kuma.KumaError as ke: + logger.error(f'Uptime Kuma maintenance could not be ended: {ke}') + return jsonify({'error': f'The modem did not restart: {e}'}), 502 + + db.set_state('modem_restart', {'at': time.time(), 'by': user, 'result': result, + 'suppression_id': sup_id, 'kuma': kuma_on, 'finished_at': None}) + logger.warning(f'Modem restart requested by {user} ({result}); Kuma maintenance ' + f'{"on" if kuma_on else "not set"}, suppression #{sup_id}') + return jsonify({'ok': True, 'result': result, 'kuma': kuma_on, 'suppression_id': sup_id}) + + @app.route('/api/devices') @require_auth def api_devices(): diff --git a/kuma.py b/kuma.py new file mode 100644 index 0000000..eb3ddcd --- /dev/null +++ b/kuma.py @@ -0,0 +1,56 @@ +"""Switch one Uptime Kuma maintenance window on and off (Kuma 2.x, over its Socket.IO API). + +Kuma has no REST API for maintenance, so this logs in over the same socket its web UI +uses. It only ever resumes or pauses the one window named in config, which is created +once in Kuma ("manual" strategy, covering the monitors that a modem restart takes down): + + "uptime_kuma": {"url": "http://10.10.10.25:3001", "username": "...", "password": "...", + "maintenance_id": 7} + +Nothing else in Kuma is read or changed. +""" +import logging +from typing import Callable, Optional + +import socketio + +logger = logging.getLogger('gandalf.kuma') + + +class KumaError(Exception): + """Kuma couldn't be reached, refused the login, or refused the change.""" + + +def configured(cfg: Optional[dict]) -> bool: + cfg = cfg or {} + return all(cfg.get(k) for k in ('url', 'username', 'password', 'maintenance_id')) + + +def set_maintenance(cfg: dict, active: bool, client_factory: Optional[Callable] = None) -> None: + """Resume (active=True) or pause the configured maintenance window. Raises KumaError.""" + timeout = cfg.get('timeout', 10) + factory = client_factory or (lambda: socketio.Client( + reconnection=False, ssl_verify=cfg.get('verify_ssl', True))) + sio = factory() + try: + sio.connect(cfg['url'], transports=['websocket'], wait_timeout=timeout) + res = sio.call('login', {'username': cfg['username'], 'password': cfg['password'], 'token': ''}, + timeout=timeout) or {} + if not res.get('ok'): + if res.get('tokenRequired'): + raise KumaError('the Kuma account has two-factor login, which Gandalf cannot answer') + raise KumaError(f'login refused: {res.get("msg") or "no reason given"}') + event = 'resumeMaintenance' if active else 'pauseMaintenance' + res = sio.call(event, int(cfg['maintenance_id']), timeout=timeout) or {} + if not res.get('ok'): + raise KumaError(f'{event} refused: {res.get("msg") or "no reason given"}') + logger.info(f'Uptime Kuma maintenance {cfg["maintenance_id"]} {"on" if active else "off"}') + except KumaError: + raise + except Exception as e: # socketio raises its own errors and plain timeouts + raise KumaError(f'{type(e).__name__}: {e}') from e + finally: + try: + sio.disconnect() + except Exception: # nosec B110 - nothing left to clean up + pass diff --git a/modem.py b/modem.py index de8a15a..5073373 100644 --- a/modem.py +++ b/modem.py @@ -1,13 +1,14 @@ -"""Read-only client for AT&T BGW-series fiber gateways (tested on BGW320-505). +"""Client for AT&T BGW-series fiber gateways (tested on BGW320-505). The gateway has no API, but its status pages are served without a login, so -this scrapes them over HTTP. Only GET requests to the fixed status pages below -are ever made: nothing here can change gateway settings. +ModemClient scrapes them over HTTP with GET requests only: it cannot change +anything on the gateway. With an access code configured, ModemAdmin (at the end) also logs in to read the NAT session counts, the firewall log and speed test history, and to run the -Troubleshoot page's ping from the gateway. -Those are a login, page reads and a ping test: still no setting is changed. +Troubleshoot page's ping from the gateway. None of that changes a setting. Its +restart() is the one call that does something to the gateway, and it runs only +when an admin presses Restart modem. Units come from the gateway's own help text, checked against live readings: Rx/Tx power tenths of a dBm (-169 -> -16.9 dBm) @@ -392,6 +393,8 @@ _NAT_PAGE = '/cgi-bin/nattable.ha' _DIAG_PAGE = '/cgi-bin/diag.ha' _LOGS_PAGE = '/cgi-bin/logs.ha' _SPEED_PAGE = '/cgi-bin/speed.ha' +_RESTART_PAGE = '/cgi-bin/restart.ha' +_RESTARTING_PAGE = '/cgi-bin/restarting.ha' # Firewall log row: number, ISO time (gateway's local clock), source, destination, protocol, reason _FW_ROW_RE = re.compile(r']*>\s*]*>\s*\d+\s*\s*\s*([^<]+?)\s*\s*\s*([^<]*?)\s*' r'\s*\s*([^<]*?)\s*\s*\s*([^<]*?)\s*\s*\s*([^<]*?)\s*\s*', re.I) @@ -536,6 +539,31 @@ class ModemAdmin: def speed_tests(self) -> List[dict]: return parse_speed_tests(self._page(_SPEED_PAGE)) + def restart(self) -> str: + """Restart the gateway from its Restart Device page: the one change this client makes, + and only when an admin asks. + + Two steps, as in a browser: posting the form only answers with a redirect to + restarting.ha, and loading that page is what starts the restart (firmware 6.40; a + post alone leaves the gateway running). Returns 'accepted' once that page says it is + restarting, or 'unconfirmed' when the gateway dropped the connection first. + """ + nonce = _NONCE_RE.search(self._page(_RESTART_PAGE)) + if not nonce: + raise ModemDataInvalid('Restart page has no nonce') + timeout = max(self.timeout, 30) + try: + resp = self.session.post(f'{self.base_url}{_RESTART_PAGE}', allow_redirects=False, + timeout=timeout, data={'nonce': nonce.group(1), 'Restart': 'Restart'}) + if resp.status_code == 302 and _RESTARTING_PAGE in (resp.headers.get('Location') or ''): + resp = self.session.get(f'{self.base_url}{_RESTARTING_PAGE}', allow_redirects=False, + timeout=timeout) + except (requests.ConnectionError, requests.Timeout): + return 'unconfirmed' + if resp.status_code != 200 or is_login_page(resp.text) or 'restarting' not in resp.text.lower(): + raise ModemDataInvalid(f'Restart was not started (HTTP {resp.status_code})') + return 'accepted' + def ping(self, target: str, wait_s: float = 30) -> dict: """Ping *target* from the gateway (its Troubleshoot page; 5 packets) and return the result.""" nonce = _NONCE_RE.search(self._page(_DIAG_PAGE)) diff --git a/monitor.py b/monitor.py index 6343ab2..43279f4 100644 --- a/monitor.py +++ b/monitor.py @@ -24,6 +24,7 @@ from urllib3.exceptions import InsecureRequestWarning import db import cutsheet import devices +import kuma import modem as modem_api import ticket_text @@ -1973,6 +1974,50 @@ class NetworkMonitor: if quality is not None: db.set_state('wan_quality', quality) + _RESTART_SETTLE_S = 180 # the modem answering this soon may not have restarted yet + _RESTART_GIVE_UP_S = 1800 + + def _finish_modem_restart(self) -> None: + """After the Restart modem button: once the modem answers again with its WAN up (or + after 30 minutes regardless), end the Uptime Kuma window and the Gandalf silence the + button started, and note how long it took.""" + raw = db.get_state('modem_restart') + try: + state = json.loads(raw) if raw else None + except ValueError: + state = None + if not state or state.get('finished_at'): + return + now = time.time() + age = now - state.get('at', now) + raw = db.get_state('modem_stats') + try: + stats = json.loads(raw) if raw else {} + except ValueError: + stats = {} + uptime = stats.get('uptime_s') + rebooted = uptime is not None and uptime < age + back = (age >= self._RESTART_SETTLE_S and stats.get('reachable') + and not stats.get('wan_problems') and (rebooted or uptime is None)) + if not back and age < self._RESTART_GIVE_UP_S: + return + if state.get('kuma') and not state.get('kuma_ended'): + try: + kuma.set_maintenance(self.cfg.get('uptime_kuma') or {}, False) + state['kuma_ended'] = True + except kuma.KumaError as e: + logger.warning(f'Ending the Uptime Kuma window after the modem restart: {e}') + if age < self._RESTART_GIVE_UP_S: + return # try again next cycle + state['kuma_error'] = str(e) + if state.get('suppression_id'): + db.deactivate_suppression(state['suppression_id']) + state.update(finished_at=now, back=bool(back), rebooted=rebooted, + back_after_s=round(age) if back else None) + db.set_state('modem_restart', state) + logger.info(f'Modem restart finished: {"back" if back else "gave up waiting"} after {round(age)} s' + f'{", rebooted" if rebooted else ""}') + _NAT_EVERY_S = 1800 # NAT sessions and the gateway's speed test history _FIREWALL_EVERY_S = 300 # the firewall log only holds about ten minutes _FIREWALL_HOURS = 24 @@ -2247,6 +2292,12 @@ class NetworkMonitor: except Exception as e: logger.error(f'WAN quality collection failed: {e}', exc_info=True) + # An admin's modem restart: end the silence once the modem is back + try: + self._finish_modem_restart() + except Exception as e: + logger.error(f'Finishing the modem restart failed: {e}', exc_info=True) + # Logged-in modem checks (NAT sessions; a ping from the modem when the internet is slow) try: self._collect_modem_checks() diff --git a/requirements.txt b/requirements.txt index 9ef2819..098cf5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ ldap3>=2.9 pymysql>=1.1.0 requests>=2.31.0 urllib3>=2.0.0 +python-socketio[client]>=5.8 diff --git a/static/app.js b/static/app.js index 65e2824..938ff6b 100644 --- a/static/app.js +++ b/static/app.js @@ -470,6 +470,47 @@ function _modemQuality(q) { `; } +// A restart from the Restart modem button: in progress, or finished in the last hour +function _modemRestartNote(r) { + if (!r || !r.at) return ''; + const who = r.by ? ` by ${r.by}` : ''; + if (!r.finished_at) { + return `
Restart requested${lt.escHtml(who)} ${lt.escHtml(_modemWhen(r.at))}. ` + + `Alerts are silenced${r.kuma ? ' here and in Uptime Kuma' : ''} until the modem is back.
`; + } + if (Date.now() / 1000 - r.finished_at > 3600) return ''; + const how = r.back ? `back after ${_modemDuration(r.back_after_s)}` : 'did not come back within 30 minutes'; + return `
Restarted${lt.escHtml(who)} ${lt.escHtml(_modemWhen(r.at))}: ` + + `${lt.escHtml(how)}. Alerts are on again.
`; +} + +async function restartModem(btn) { + const msg = 'Restart the AT&T modem?\n\nThe internet will be down for about 3 to 5 minutes. ' + + 'Gandalf alerts and the Uptime Kuma maintenance window are switched on first, and switched off ' + + 'again when the modem is back.'; + if (!window.confirm(msg)) return; + btn.disabled = true; + try { + let res; + try { + res = await lt.api.post('/api/modem/restart', { confirm: true }); + } catch (err) { + if (!String(err.message).startsWith('Uptime Kuma:')) throw err; + if (!window.confirm(`${err.message}\n\nRestart anyway? Uptime Kuma will report the outage.`)) { + btn.disabled = false; + return; + } + res = await lt.api.post('/api/modem/restart', { confirm: true, without_kuma: true }); + } + showToast(res.result === 'accepted' ? 'Modem restarting' : 'Restart sent; the modem dropped the connection', + 'success'); + window.location.reload(); + } catch (err) { + btn.disabled = false; + showToast(err.message || 'Could not restart the modem', 'error'); + } +} + function updateModem(data) { const section = document.getElementById('modem-section'); const body = document.getElementById('modem-body'); @@ -477,6 +518,18 @@ function updateModem(data) { section.hidden = !data.configured; if (!data.configured) return; + const restartBtn = document.getElementById('modem-restart-btn'); + if (restartBtn) { + const pending = data.modem_restart && !data.modem_restart.finished_at; + restartBtn.hidden = !data.can_restart; + restartBtn.disabled = Boolean(pending); + if (!restartBtn.dataset.bound) { + restartBtn.dataset.bound = '1'; + restartBtn.addEventListener('click', () => restartModem(restartBtn)); + } + } + const restartNote = _modemRestartNote(data.modem_restart); + // On the dashboard, link to the modem's page. On that page, it would link to itself. const detailsLink = document.getElementById('modem-device-link'); if (detailsLink && typeof DEVICE_NAME === 'undefined' && data.name) { @@ -487,7 +540,7 @@ function updateModem(data) { const title = document.getElementById('modem-title'); if (!data.reachable) { - body.innerHTML = ` + body.innerHTML = `${restartNote}
Modem status unavailable: ${lt.escHtml(data.error || 'unknown error')} Last attempt ${lt.escHtml(data.updated || '-')}
${_modemQuality(data.wan_quality)} @@ -508,7 +561,7 @@ function updateModem(data) { const errs = (counters.rx_errors || 0) + (counters.tx_errors || 0); const drops = (counters.rx_drops || 0) + (counters.tx_drops || 0); - body.innerHTML = ` + body.innerHTML = `${restartNote} ${problems.length ? `
WAN down: ${lt.escHtml(problems.join('; '))}
` : ''}
${_modemStat('WAN', problems.length ? 'DOWN' : 'UP', problems.length ? 'lt-text-red' : 'lt-text-green', diff --git a/static/style.css b/static/style.css index c3ea836..62cd111 100644 --- a/static/style.css +++ b/static/style.css @@ -1849,3 +1849,8 @@ body > .lt-footer { flex-shrink: 0; } #events-table td { max-width: none; } #events-table td.desc-cell { white-space: normal; text-align: right; } } + +/* ── WAN / Fiber: restart button and progress note ── */ +.modem-actions { display: inline-flex; gap: 6px; } +.modem-restart-note { margin: 8px 12px 0; padding: 6px 10px; font-size: .8em; + border: 1px solid var(--accent-amber); color: var(--text-primary); } diff --git a/templates/_modem_panel.html b/templates/_modem_panel.html index 0455650..c0a8365 100644 --- a/templates/_modem_panel.html +++ b/templates/_modem_panel.html @@ -1,7 +1,13 @@