diff --git a/README.md b/README.md
index 21a351a..905dcff 100644
--- a/README.md
+++ b/README.md
@@ -111,6 +111,9 @@ Ping-only (no node_exporter): **pbs** (10.10.10.3)
- WAN / Fiber panel: WAN and PON state, Rx/Tx power, optics temperature, throughput, errors, gateway uptime and last restart, 7-day Rx trend, recent WAN outages
- UniFi device table (switches, APs, gateway)
- Active alerts table with severity, target, consecutive failures, ticket link
+- Follow-up flags (see Follow-up below): a ⏳ badge on alerts active for days, a "ticket closed" badge when the
+ ticket was closed but the alert is still active, and a Needs Follow-up panel of tickets left open after their
+ alert cleared. A "TO FOLLOW UP" chip in the status bar counts them all.
- Quick-suppress modal: apply timed or manual suppression from any alert row
- Auto-refreshes every 30 seconds via `/api/status` + `/api/network`
@@ -381,6 +384,20 @@ A note that fails to post is retried every cycle for 24 hours. Posting needs a T
`read_write` scope; if Tinker answers 401 or 403, the monitor logs one error and stops trying until it is
restarted.
+### Follow-up
+
+Gandalf only comments on tickets, so a ticket can stay open long after its problem is fixed, and an alert can
+sit active for days once everyone has stopped looking. The dashboard flags both:
+
+- **Stale alert**: active for `monitor.stale_alert_days` (default 3) or longer. Shown as a ⏳ badge with its age.
+- **Ticket closed**: the alert is still active but its ticket was closed in Tinker, so nobody is tracking it.
+- **Ticket left open**: every alert on a ticket cleared at least `monitor.stale_ticket_hours` (default 24) ago,
+ but the ticket is still Open, Pending or In Progress. Listed in the Needs Follow-up panel.
+
+The monitor reads the list of open tickets from Tinker's `api/tickets_api.php` every 15 minutes (one request,
+read scope only) and saves it. If that list is over 2 hours old, because Tinker or the key stopped working,
+the ticket flags are left off rather than guessed. Stale alerts are still flagged.
+
### WAN modem notes
The alarm and warning limits are read from the gateway on every poll, so they are never hard-coded here.
@@ -508,6 +525,7 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
| `ticket_api.url` | Tinker Tickets `create_ticket_api.php` URL |
| `ticket_api.api_key` | Tinker Tickets bearer token. Needs `read_write` scope to add recovery notes |
| `ticket_api.comment_url` | Tinker comment endpoint (default: `api/ticket_comment_api.php` beside `ticket_api.url`) |
+| `ticket_api.read_url` | Tinker ticket read endpoint, used to find tickets left open (default: `api/tickets_api.php` beside `ticket_api.url`) |
| `gandalf.public_url` | Gandalf's own address, for links in tickets (default: `https://gandalf.lotusguild.org`) |
| `rack.units`, `rack.name` | The single rack's height and name on `/rack` (defaults: 42, "Rack 1") |
| `racks` | Several racks: `[{"name": "...", "units": 42}, ...]`, in tab order; overrides `rack` |
@@ -525,6 +543,8 @@ Shared by both processes. Located in the working directory (`/var/www/html/prod/
| `monitor.unifi_failure_threshold` | Consecutive failures before ticketing a UniFi device as offline (default: same as `failure_threshold`). Set higher than `failure_threshold`: UniFi switches/APs can take several minutes to rejoin after a firmware-update reboot, so a low threshold cuts tickets for momentary maintenance reboots instead of real failures. |
| `monitor.cluster_threshold` | Hosts with failures to trigger cluster-wide P1 (default: 3) |
| `monitor.ping_hosts` | Hosts checked only by ping (no node_exporter) |
+| `monitor.stale_alert_days` | Days an alert can stay active before the dashboard flags it as stale (default: 3) |
+| `monitor.stale_ticket_hours` | Hours a ticket can stay open after all its alerts cleared before it is listed under Needs Follow-up (default: 24) |
| `cutsheet.mismatch_ticket_hours` | Hours a cutsheet mismatch must last before it opens a P4 ticket (default: 24; `0` never tickets). The mismatch shows on the dashboard and the switch's page straight away, so recabling never pages anyone. Suppressing the switch or its port also stops the ticket. |
| `modem.url` | AT&T gateway base URL. Omit the `modem` block to disable modem monitoring. Must be reachable from the monitor host. |
| `modem.name` | Name used for the modem's alerts, tickets, and suppressions (default: `att-modem`) |
diff --git a/app.py b/app.py
index 92a2754..b5245f9 100644
--- a/app.py
+++ b/app.py
@@ -274,6 +274,72 @@ def _annotate_suppressions(events: list, suppressions: list) -> None:
ev['is_suppressed'] = db.check_suppressed(suppressions, 'host', name, detail)
+# The monitor refreshes the open ticket list every 15 minutes; older than this and
+# it has stopped working, so nothing is flagged rather than flagging on stale data.
+_OPEN_TICKETS_MAX_AGE_S = 2 * 3600
+
+
+def _stale_settings() -> tuple:
+ """(days an alert can stay active before it counts as stale,
+ hours a ticket can stay open after its alert cleared)."""
+ mon = _config().get('monitor', {})
+ try:
+ days = float(mon.get('stale_alert_days', 3))
+ hours = int(mon.get('stale_ticket_hours', 24))
+ except (TypeError, ValueError):
+ days, hours = 3.0, 24
+ return days, hours
+
+
+def _open_tickets() -> Optional[dict]:
+ """{ticket_id: status} of open Tinker tickets, or None when the list is missing or out of date."""
+ try:
+ raw = db.get_state('open_tickets')
+ data = json.loads(raw) if raw else None
+ if not data or time.time() - float(data.get('checked_at', 0)) > _OPEN_TICKETS_MAX_AGE_S:
+ return None
+ return {str(k): v for k, v in (data.get('tickets') or {}).items()}
+ except Exception as e:
+ logger.error(f'Reading open ticket list failed: {e}')
+ return None
+
+
+def _annotate_staleness(events: list, open_tickets: Optional[dict]) -> None:
+ """Add stale_days (set when an alert has been active too long) and ticket_closed
+ (its ticket was closed in Tinker while the alert is still active) to each event."""
+ days, _ = _stale_settings()
+ now = datetime.now(timezone.utc)
+ for ev in events:
+ ev['stale_days'] = None
+ ev['ticket_closed'] = False
+ try:
+ first = datetime.fromisoformat(str(ev.get('first_seen', '')).replace('Z', '+00:00'))
+ if first.tzinfo is None:
+ first = first.replace(tzinfo=timezone.utc)
+ age = (now - first).total_seconds() / 86400
+ if age >= days:
+ ev['stale_days'] = int(age)
+ except ValueError:
+ pass
+ tid = ev.get('ticket_id')
+ if tid and open_tickets is not None and str(tid) not in open_tickets:
+ ev['ticket_closed'] = True
+
+
+def _stale_view(events: list) -> list:
+ """Flag stale active alerts in place; return tickets left open after their alerts cleared."""
+ open_tickets = _open_tickets()
+ _annotate_staleness(events, open_tickets)
+ if open_tickets is None:
+ return []
+ _, hours = _stale_settings()
+ try:
+ return db.get_cleared_tickets(list(open_tickets), older_than_hours=hours)
+ except Exception as e:
+ logger.error(f'Finding tickets left open failed: {e}')
+ return []
+
+
def _state_json(key: str) -> dict:
"""Return a monitor_state JSON blob as a dict, or {} if missing or unreadable."""
raw = db.get_state(key)
@@ -307,6 +373,7 @@ def index():
snapshot = {}
suppressions = db.get_active_suppressions()
_annotate_suppressions(events, suppressions)
+ stale_tickets = _stale_view(events)
recent_resolved = db.get_recent_resolved(hours=24, limit=10)
return render_template(
'index.html',
@@ -318,6 +385,8 @@ def index():
last_check=last_check,
suppressions=suppressions,
recent_resolved=recent_resolved,
+ stale_tickets=stale_tickets,
+ stale_alerts=sum(1 for e in events if e.get('stale_days') is not None or e.get('ticket_closed')),
daemon_ok=_daemon_ok(last_check),
)
@@ -419,11 +488,13 @@ def api_status():
active = db.get_active_events(limit=_PAGE_LIMIT)
suppressions = db.get_active_suppressions()
_annotate_suppressions(active, suppressions)
+ stale_tickets = _stale_view(active)
last_check = db.get_state('last_check', 'Never')
return jsonify({
'summary': db.get_status_summary(),
'last_check': last_check,
'events': active,
+ 'stale_tickets': stale_tickets,
'total_active': db.count_active_events(),
'daemon_ok': _daemon_ok(last_check),
})
diff --git a/db.py b/db.py
index a9f207f..88839f3 100644
--- a/db.py
+++ b/db.py
@@ -247,6 +247,49 @@ def get_active_events(limit: int = 200, offset: int = 0) -> list:
return rows
+def get_cleared_tickets(open_ids: List[str], older_than_hours: int = 24) -> list:
+ """Of the given open tickets, those whose every alert cleared at least `older_than_hours` ago.
+
+ One row per ticket, describing its most recent alert, newest clearance first:
+ {ticket_id, event_type, target_name, target_detail, severity, first_seen, cleared_at, events}
+ """
+ ids = sorted({str(i) for i in open_ids if i})
+ if not ids:
+ return []
+ placeholders = ','.join(['%s'] * len(ids))
+ with get_conn() as conn:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""SELECT ticket_id, event_type, target_name, target_detail, severity,
+ first_seen, resolved_at
+ FROM network_events WHERE ticket_id IN ({placeholders})
+ ORDER BY first_seen""", # nosec B608: placeholders are only %s markers
+ ids,
+ )
+ rows = cur.fetchall()
+ cur.execute('SELECT NOW() AS now')
+ now = cur.fetchone()['now']
+ by_ticket: dict = {}
+ for r in rows:
+ by_ticket.setdefault(str(r['ticket_id']), []).append(r)
+ out = []
+ for tid, events in by_ticket.items():
+ if any(e['resolved_at'] is None for e in events):
+ continue
+ cleared = max(e['resolved_at'] for e in events)
+ if (now - cleared).total_seconds() < older_than_hours * 3600:
+ continue
+ last = events[-1]
+ out.append({
+ 'ticket_id': tid, 'event_type': last['event_type'], 'target_name': last['target_name'],
+ 'target_detail': last['target_detail'], 'severity': last['severity'],
+ 'first_seen': events[0]['first_seen'].isoformat() + 'Z',
+ 'cleared_at': cleared.isoformat() + 'Z', 'events': len(events),
+ })
+ out.sort(key=lambda r: r['cleared_at'], reverse=True)
+ return out
+
+
def count_active_events() -> int:
"""Return count of all unresolved events (for pagination)."""
with get_conn() as conn:
diff --git a/monitor.py b/monitor.py
index 83af575..86f815a 100644
--- a/monitor.py
+++ b/monitor.py
@@ -408,6 +408,8 @@ class TicketClient:
self.comment_url = cfg.get('comment_url') or (
urljoin(self.url, 'api/ticket_comment_api.php') if self.url else '')
self.comments_denied = False
+ self.read_url = cfg.get('read_url') or (
+ urljoin(self.url, 'api/tickets_api.php') if self.url else '')
# Titles whose create matched a Closed ticket Tinker couldn't reopen: {title: (ticket_id, when)}
self._closed_matches: Dict[str, tuple] = {}
@@ -442,6 +444,41 @@ class TicketClient:
logger.error(f'Ticket comment failed for #{ticket_id}: {e}')
return False
+ # Statuses that mean someone still has a ticket on their list
+ OPEN_STATUSES = 'Open,Pending,In Progress'
+ _OPEN_TICKETS_MAX_PAGES = 20
+
+ def open_tickets(self) -> Optional[Dict[str, str]]:
+ """{ticket_id: status} for every ticket not yet closed, or None if Tinker can't say.
+
+ Read only: this needs the key's read scope and never changes a ticket.
+ """
+ if not self.api_key or not self.read_url:
+ return None
+ found: Dict[str, str] = {}
+ try:
+ for page in range(1, self._OPEN_TICKETS_MAX_PAGES + 1):
+ resp = requests.get(
+ self.read_url,
+ params={'status': self.OPEN_STATUSES, 'limit': 100, 'page': page},
+ headers={'Authorization': f'Bearer {self.api_key}'},
+ timeout=15,
+ )
+ resp.raise_for_status()
+ data = resp.json()
+ if not data.get('success'):
+ logger.warning(f'Unexpected ticket list response: {str(data)[:200]}')
+ return None
+ for t in data.get('tickets') or []:
+ found[str(t.get('ticket_id'))] = t.get('status') or ''
+ if page >= int(data.get('pages') or 1):
+ return found
+ logger.warning('Open ticket list is longer than expected; using the first pages only')
+ return found
+ except Exception as e:
+ logger.error(f'Reading open tickets failed: {e}')
+ return None
+
# How long to wait before asking again after a create matched an unreopenable Closed ticket.
# Each ask adds a "recurred" comment to that ticket, so once a poll would pile them up.
CLOSED_MATCH_RETRY_S = 24 * 3600
@@ -990,9 +1027,15 @@ UNIFI_UPDATE_STATE = 'unifi_update'
# Ports a host or device was last seen on, kept so a ticket for a dead host
# (whose link, and so its LLDP entry, is gone) can still say where it plugs in
KNOWN_UPLINKS_STATE = 'known_uplinks'
+# Which tickets are still open in Tinker, so the dashboard can flag ones left open
+# after their alert cleared. Refreshed this often; one list request per refresh.
+OPEN_TICKETS_STATE = 'open_tickets'
+OPEN_TICKETS_SYNC_S = 900
class NetworkMonitor:
+ _open_tickets_synced = 0.0
+
def __init__(self):
with open('config.json') as f:
self.cfg = json.load(f)
@@ -1245,6 +1288,20 @@ class NetworkMonitor:
elif self.tickets.comments_denied:
break
+ def _sync_open_tickets(self) -> None:
+ """Save which tickets are still open in Tinker, at most every OPEN_TICKETS_SYNC_S.
+
+ On a failed read the previous list stays, with its old checked_at, so the
+ dashboard can tell it is out of date rather than treat every ticket as closed.
+ """
+ now = time.time()
+ if now - self._open_tickets_synced < OPEN_TICKETS_SYNC_S:
+ return
+ self._open_tickets_synced = now
+ found = self.tickets.open_tickets()
+ if found is not None:
+ db.set_state(OPEN_TICKETS_STATE, {'checked_at': now, 'tickets': found})
+
# ------------------------------------------------------------------
# Interface monitoring (Prometheus)
# ------------------------------------------------------------------
@@ -2051,6 +2108,10 @@ class NetworkMonitor:
self._post_recovery_notes()
except Exception as e:
logger.error(f'Posting recovery notes failed: {e}', exc_info=True)
+ try:
+ self._sync_open_tickets()
+ except Exception as e:
+ logger.error(f'Syncing open tickets failed: {e}', exc_info=True)
# Housekeeping: deactivate expired suppressions and purge old resolved events
db.cleanup_expired_suppressions()
diff --git a/static/app.js b/static/app.js
index 1c9c995..d004f65 100644
--- a/static/app.js
+++ b/static/app.js
@@ -55,7 +55,10 @@ async function refreshAll() {
if (statusResult.status === 'fulfilled') {
const status = statusResult.value;
updateEventsTable(status.events || [], status.total_active);
- updateStatusBar(status.summary || {}, status.last_check || '', status.daemon_ok);
+ updateFollowUp(status.stale_tickets || []);
+ updateStatusBar(status.summary || {}, status.last_check || '', status.daemon_ok,
+ followUpCount(status.events || [], status.stale_tickets || []),
+ (status.stale_tickets || []).length > 0);
if (typeof window.gandalfNotifUpdate === 'function') window.gandalfNotifUpdate(status.events || []);
} else {
showToast('Status data unavailable', 'warning');
@@ -66,7 +69,14 @@ async function refreshAll() {
}
}
-function updateStatusBar(summary, lastCheck, daemonOk) {
+// Active alerts flagged stale or with a closed ticket, plus tickets left open after clearing
+function followUpCount(events, staleTickets) {
+ const flagged = events.filter(e => e.severity !== 'info' &&
+ ((e.stale_days !== null && e.stale_days !== undefined) || e.ticket_closed)).length;
+ return flagged + staleTickets.length;
+}
+
+function updateStatusBar(summary, lastCheck, daemonOk, followUps = 0, hasStaleTickets = false) {
const bar = document.querySelector('.status-chips');
if (!bar) return;
const chips = [];
@@ -74,6 +84,10 @@ function updateStatusBar(summary, lastCheck, daemonOk) {
if (summary.critical) chips.push(`● ${summary.critical} CRITICAL`);
if (summary.warning) chips.push(`● ${summary.warning} WARNING`);
if (!summary.critical && !summary.warning && daemonOk !== false) chips.push('✔ ALL SYSTEMS NOMINAL');
+ if (followUps) {
+ const href = hasStaleTickets ? '#followup' : '#events-table-wrap';
+ chips.push(`⏳ ${followUps} TO FOLLOW UP`);
+ }
bar.innerHTML = chips.join('');
const lc = document.getElementById('last-check');
@@ -469,7 +483,7 @@ function updateEventsTable(events, totalActive) {
const truncated = totalActive !== null && totalActive !== undefined && totalActive > active.length;
const countNotice = truncated
- ? ``
+ ? ``
: '';
const rows = active.map(e => {
@@ -485,9 +499,15 @@ function updateEventsTable(events, totalActive) {
const supBadge = e.is_suppressed
? `🔕 sup`
: '';
+ const staleBadge = (e.stale_days !== null && e.stale_days !== undefined)
+ ? `⏳ ${Number(e.stale_days)}d`
+ : '';
+ const closedBadge = e.ticket_closed
+ ? `ticket closed`
+ : '';
return `
- | ${e.severity}${supBadge} |
+ ${e.severity}${supBadge}${staleBadge}${closedBadge} |
${lt.escHtml(e.event_type.replace(/_/g,' '))} |
${lt.escHtml(e.target_name)} |
${lt.escHtml(e.target_detail || '-')} |
@@ -522,6 +542,28 @@ function updateEventsTable(events, totalActive) {
`;
}
+// ── Needs follow-up: tickets left open after their alert cleared ──────
+function updateFollowUp(tickets) {
+ const section = document.getElementById('followup');
+ const tbody = document.querySelector('#followup-table tbody');
+ if (!section || !tbody) return;
+ section.hidden = !tickets.length;
+ const badge = document.getElementById('followup-badge');
+ if (badge) badge.textContent = tickets.length;
+ const ticketBase = (typeof GANDALF_CONFIG !== 'undefined' && GANDALF_CONFIG.ticket_web_url)
+ ? GANDALF_CONFIG.ticket_web_url : 'http://t.lotusguild.org/ticket/';
+ tbody.innerHTML = tickets.map(t => `
+
+ | #${lt.escHtml(String(t.ticket_id))} |
+ ${lt.escHtml(t.target_name || '')} |
+ ${lt.escHtml((t.event_type || '').replace(/_/g, ' '))}${t.target_detail
+ ? ` ${lt.escHtml(t.target_detail)}` : ''} |
+ ${lt.time.ago(_toIso(t.cleared_at))} |
+ ${Number(t.events) || 0} |
+
`).join('');
+}
+
// ── Suppression modal ─────────────────────────────────────────────────
function openSuppressModal(type, name, detail) {
const modal = document.getElementById('suppress-modal');
diff --git a/static/style.css b/static/style.css
index 9732b88..131954a 100644
--- a/static/style.css
+++ b/static/style.css
@@ -229,6 +229,12 @@
.badge-resolved { color: var(--text-muted); border-color: var(--border-color); text-decoration: line-through; }
.badge-suppressed { font-size: .9em; padding: 0; border: none; color: var(--text-muted); }
.badge-purple { color: var(--accent-purple); border-color: var(--accent-purple); }
+.badge-stale { color: var(--amber, var(--orange)); border-color: transparent; margin-left: 4px; white-space: nowrap; }
+.badge-ticket-closed { color: var(--accent-purple); border-color: var(--accent-purple); margin-left: 4px; white-space: nowrap; }
+.chip-followup { text-decoration: none; cursor: pointer; }
+.chip-followup:hover, .chip-followup:focus-visible { background: rgba(255,107,0,.12); }
+.followup-intro { color: var(--text-muted); font-size: .85em; margin: 0 0 8px; }
+#followup-table td { white-space: nowrap; }
/* ── Table row state colors ───────────────────────────────────────── */
.lt-table tr.row-critical td { background: rgba(255,45,85,.04); }
diff --git a/templates/index.html b/templates/index.html
index 8c196c4..d2143d4 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -18,6 +18,10 @@
{% if daemon_ok and not summary.critical and not summary.warning %}
✔ ALL SYSTEMS NOMINAL
{% endif %}
+ {% set followups = (stale_tickets | length) + (stale_alerts or 0) %}
+ {% if followups %}
+ ⏳ {{ followups }} TO FOLLOW UP
+ {% endif %}
{{ last_check }}
@@ -129,6 +133,8 @@
{{ e.severity }}
{% if e.is_suppressed %}🔕 sup{% endif %}
+ {% if e.stale_days is not none %}⏳ {{ e.stale_days }}d{% endif %}
+ {% if e.ticket_closed %}ticket closed{% endif %}
|
{{ e.event_type | replace('_', ' ') }} |
{{ e.target_name }} |
@@ -177,6 +183,39 @@
+
+
+
+ These tickets are still open in Tinker, but their alert cleared over a day ago. Close them if the problem is fixed.
+
+
╚
+
╝
+
+
+
+ Tickets still open after their alert cleared
+
+ | Ticket | Target | Alert | Cleared | Alerts on ticket |
|---|
+
+
+ {% for t in stale_tickets %}
+
+ | #{{ t.ticket_id }} |
+ {{ t.target_name }} |
+ {{ t.event_type | replace('_', ' ') }}{% if t.target_detail %} {{ t.target_detail }}{% endif %} |
+ {{ t.cleared_at }} |
+ {{ t.events }} |
+
+ {% endfor %}
+
+
+
+