From d8c2acc4612b437042b9ff6822b5cf6fcf30e43f Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Sun, 15 Nov 2026 21:42:47 -0500 Subject: [PATCH] discord-bridge: check the gateway echo before retrying ambiguous webhook sends On 2026-11-15 21:34 an image relay failed with "read: connection reset by peer" after the request was sent. Patch 0001 deliberately doesn't retry those (a webhook post has no dedupe), so the image was dropped. Patch 0002: while a relay send is in flight the bridge watches the Discord gateway for its own post (same webhook, text and attachment count, created after the send started). On an ambiguous failure (reset, EOF, timeout waiting for the reply, Discord 5xx) it waits up to 6 s for that echo: if it shows up the send counts as delivered and is recorded (no resend, no copy back into Matrix); otherwise it retries with the 0001 backoff, checking again before each resend. build-patched.sh runs the new tests with -race. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PPmy3tPq869XDW4njjVaKA --- README.md | 10 +- discord-bridge/build-patched.sh | 2 +- ...ore-retrying-ambiguous-webhook-sends.patch | 437 ++++++++++++++++++ 3 files changed, 445 insertions(+), 4 deletions(-) create mode 100644 discord-bridge/patches/0002-check-gateway-echo-before-retrying-ambiguous-webhook-sends.patch diff --git a/README.md b/README.md index 2987569..bb46e80 100644 --- a/README.md +++ b/README.md @@ -418,9 +418,13 @@ channel webhook with their own display name. - `0001`: when a Matrix → Discord send fails before the request reaches Discord (DNS, connect, or `TLS handshake timeout`), the bridge retries after 2, 5, 10, 20, 30, 60, 60 and 60 s instead of dropping the message with "⚠ Your message may not have been bridged". The room's later messages wait - in order behind the retry. Failures after the request was sent are not retried, because a webhook - send has no dedupe and a retry could post the message twice. Edits, reactions and deletes are not - retried. + in order behind the retry. Edits, reactions and deletes are not retried. + - `0002`: when a relay (webhook) send fails after the request may already have reached Discord + (connection reset, EOF, timeout waiting for the reply, Discord 5xx), the bridge waits up to 6 s for + its own post to come back over the Discord gateway. If it does, the send counts as delivered (no + resend, no copy back into Matrix); if not, it retries with the 0001 backoff, checking again before + each resend. A webhook post has no dedupe, so this echo check is what keeps those retries from + double-posting. **Install (one time):** after this repo's `main` has reached `/opt/matrix-config` on LXC 151, run this on the Proxmox node hosting LXC 109 and 151. It restarts Synapse once and is safe to re-run. diff --git a/discord-bridge/build-patched.sh b/discord-bridge/build-patched.sh index b0b31f8..74aae61 100644 --- a/discord-bridge/build-patched.sh +++ b/discord-bridge/build-patched.sh @@ -25,7 +25,7 @@ for p in "$HERE"/patches/*.patch; do git apply "$p" done -go test -count=1 -run 'Retry|PreSend' . +go test -race -count=1 -run 'Retry|PreSend|Ambiguous|Echo' . COMMIT="$(git rev-parse HEAD)" MAUTRIX="$(go list -m -f '{{.Version}}' maunium.net/go/mautrix)" diff --git a/discord-bridge/patches/0002-check-gateway-echo-before-retrying-ambiguous-webhook-sends.patch b/discord-bridge/patches/0002-check-gateway-echo-before-retrying-ambiguous-webhook-sends.patch new file mode 100644 index 0000000..8a8a73f --- /dev/null +++ b/discord-bridge/patches/0002-check-gateway-echo-before-retrying-ambiguous-webhook-sends.patch @@ -0,0 +1,437 @@ +diff --git a/lotus_echo.go b/lotus_echo.go +new file mode 100644 +index 0000000..0cfdd95 +--- /dev/null ++++ b/lotus_echo.go +@@ -0,0 +1,118 @@ ++package main ++ ++// [lotus] Detect whether a webhook send that errored actually reached Discord. ++// ++// Discord echoes every webhook post back over the gateway as MESSAGE_CREATE. ++// While a relay send is in flight we register a waiter for its channel; the ++// gateway handler (user.go) offers every new message to the waiters before ++// queueing it for the portal. If a send fails ambiguously (the connection broke ++// after the request may have gone out), lotusRetrySend asks the waiter whether ++// the post showed up. If it did, the send is treated as successful and the ++// message ID recorded, so it is neither resent to Discord nor bridged back into ++// Matrix as a new message. ++// ++// The waiter is offered messages from the gateway goroutine, which keeps ++// running while the portal's own loop is blocked in the send. ++ ++import ( ++ "strings" ++ "sync" ++ "time" ++ ++ "github.com/bwmarrin/discordgo" ++) ++ ++// lotusEchoWait is how long to wait for the gateway echo after an ambiguous ++// failure. Echoes normally arrive well under a second after Discord accepts the ++// post. ++var lotusEchoWait = 6 * time.Second ++ ++type lotusEchoWaiter struct { ++ match func(*discordgo.Message) bool ++ ch chan *discordgo.Message ++} ++ ++var ( ++ lotusEchoMu sync.Mutex ++ lotusEchoWaiters = map[string][]*lotusEchoWaiter{} ++) ++ ++// lotusNoteGatewayMessage offers a newly created Discord message to any waiter ++// registered for its channel. Called for every MESSAGE_CREATE. ++func lotusNoteGatewayMessage(msg *discordgo.Message) { ++ if msg == nil || msg.WebhookID == "" { ++ return ++ } ++ lotusEchoMu.Lock() ++ defer lotusEchoMu.Unlock() ++ for _, w := range lotusEchoWaiters[msg.ChannelID] { ++ if w.match(msg) { ++ select { ++ case w.ch <- msg: ++ default: ++ } ++ } ++ } ++} ++ ++// lotusExpectEcho registers a waiter for channelID. check(wait) returns the ++// matching message if one has arrived, waiting up to lotusEchoWait when wait ++// is true. done must be called when the send is finished. ++func lotusExpectEcho(channelID string, match func(*discordgo.Message) bool) (check func(wait bool) (*discordgo.Message, bool), done func()) { ++ w := &lotusEchoWaiter{match: match, ch: make(chan *discordgo.Message, 1)} ++ lotusEchoMu.Lock() ++ lotusEchoWaiters[channelID] = append(lotusEchoWaiters[channelID], w) ++ lotusEchoMu.Unlock() ++ ++ check = func(wait bool) (*discordgo.Message, bool) { ++ if !wait { ++ select { ++ case msg := <-w.ch: ++ return msg, true ++ default: ++ return nil, false ++ } ++ } ++ timer := time.NewTimer(lotusEchoWait) ++ defer timer.Stop() ++ select { ++ case msg := <-w.ch: ++ return msg, true ++ case <-timer.C: ++ return nil, false ++ } ++ } ++ done = func() { ++ lotusEchoMu.Lock() ++ defer lotusEchoMu.Unlock() ++ list := lotusEchoWaiters[channelID] ++ for i, x := range list { ++ if x == w { ++ list = append(list[:i], list[i+1:]...) ++ break ++ } ++ } ++ if len(list) == 0 { ++ delete(lotusEchoWaiters, channelID) ++ } else { ++ lotusEchoWaiters[channelID] = list ++ } ++ } ++ return check, done ++} ++ ++// lotusWebhookEchoMatcher matches the gateway echo of a relay post: same ++// webhook, same text (ignoring surrounding whitespace, which Discord trims), ++// same number of attachments, created no earlier than shortly before the send ++// started. ++func lotusWebhookEchoMatcher(webhookID, content string, numFiles int, started time.Time) func(*discordgo.Message) bool { ++ notBefore := started.Add(-2 * time.Second) ++ content = strings.TrimSpace(content) ++ return func(msg *discordgo.Message) bool { ++ if msg.WebhookID != webhookID || strings.TrimSpace(msg.Content) != content || len(msg.Attachments) != numFiles { ++ return false ++ } ++ ts, err := discordgo.SnowflakeTimestamp(msg.ID) ++ return err != nil || !ts.Before(notBefore) ++ } ++} +diff --git a/lotus_echo_test.go b/lotus_echo_test.go +new file mode 100644 +index 0000000..6f5ad44 +--- /dev/null ++++ b/lotus_echo_test.go +@@ -0,0 +1,153 @@ ++package main ++ ++import ( ++ "errors" ++ "net" ++ "net/http" ++ "strconv" ++ "testing" ++ "time" ++ ++ "github.com/bwmarrin/discordgo" ++ "github.com/rs/zerolog" ++) ++ ++func resetErr() error { ++ return urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}) ++} ++ ++func TestIsAmbiguousSendError(t *testing.T) { ++ cases := []struct { ++ name string ++ err error ++ want bool ++ }{ ++ {"nil", nil, false}, ++ {"tls handshake timeout is pre-send", urlErr(tlsTimeout{}), false}, ++ {"read reset", resetErr(), true}, ++ {"awaiting headers", urlErr(errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), true}, ++ {"unexpected EOF", urlErr(errors.New("unexpected EOF")), true}, ++ {"discord 502", &discordgo.RESTError{Response: &http.Response{StatusCode: 502}}, true}, ++ {"discord 400", &discordgo.RESTError{Response: &http.Response{StatusCode: 400}}, false}, ++ {"other", errors.New("unknown message type"), false}, ++ } ++ for _, c := range cases { ++ if got := isAmbiguousSendError(c.err); got != c.want { ++ t.Errorf("%s: got %v want %v", c.name, got, c.want) ++ } ++ } ++} ++ ++func TestRetrySendAmbiguousWithoutCheckerIsNotRetried(t *testing.T) { ++ withNoSleep(t) ++ calls := 0 ++ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { ++ calls++ ++ return nil, resetErr() ++ }, nil) ++ if err == nil || calls != 1 { ++ t.Fatalf("calls=%d err=%v", calls, err) ++ } ++} ++ ++func TestRetrySendAmbiguousButDeliveredIsNotResent(t *testing.T) { ++ withNoSleep(t) ++ calls := 0 ++ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { ++ calls++ ++ return nil, resetErr() ++ }, func(wait bool) (*discordgo.Message, bool) { ++ return &discordgo.Message{ID: "echo"}, true ++ }) ++ if err != nil || msg == nil || msg.ID != "echo" || calls != 1 { ++ t.Fatalf("calls=%d msg=%v err=%v", calls, msg, err) ++ } ++} ++ ++func TestRetrySendAmbiguousNotDeliveredIsResent(t *testing.T) { ++ slept := withNoSleep(t) ++ calls := 0 ++ var waits []bool ++ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { ++ calls++ ++ if calls == 1 { ++ return nil, resetErr() ++ } ++ return &discordgo.Message{ID: "2"}, nil ++ }, func(wait bool) (*discordgo.Message, bool) { ++ waits = append(waits, wait) ++ return nil, false ++ }) ++ if err != nil || msg.ID != "2" || calls != 2 || len(*slept) != 1 { ++ t.Fatalf("calls=%d msg=%v err=%v slept=%v", calls, msg, err, *slept) ++ } ++ if len(waits) != 2 || !waits[0] || waits[1] { ++ t.Fatalf("waits=%v, want [true false]", waits) ++ } ++} ++ ++func TestRetrySendLateEchoDuringBackoffStopsResend(t *testing.T) { ++ withNoSleep(t) ++ calls := 0 ++ checks := 0 ++ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { ++ calls++ ++ return nil, resetErr() ++ }, func(wait bool) (*discordgo.Message, bool) { ++ checks++ ++ if checks == 2 { // the echo arrives during the backoff ++ return &discordgo.Message{ID: "late"}, true ++ } ++ return nil, false ++ }) ++ if err != nil || msg.ID != "late" || calls != 1 { ++ t.Fatalf("calls=%d msg=%v err=%v", calls, msg, err) ++ } ++} ++ ++func snowflakeAt(ts time.Time) string { ++ ms := ts.UnixMilli() - 1420070400000 ++ return strconv.FormatInt(ms<<22, 10) ++} ++ ++func TestEchoWaiterMatchesWebhookPost(t *testing.T) { ++ oldWait := lotusEchoWait ++ lotusEchoWait = 200 * time.Millisecond ++ t.Cleanup(func() { lotusEchoWait = oldWait }) ++ ++ start := time.Now() ++ check, done := lotusExpectEcho("chan1", lotusWebhookEchoMatcher("hook1", "hi", 1, start)) ++ defer done() ++ ++ if _, ok := check(false); ok { ++ t.Fatal("matched before any message") ++ } ++ // Wrong webhook, wrong content, wrong channel and an old message are ignored. ++ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan1", WebhookID: "other", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)}) ++ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan1", WebhookID: "hook1", Content: "nope", Attachments: make([]*discordgo.MessageAttachment, 1)}) ++ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan2", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)}) ++ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start.Add(-time.Minute)), ChannelID: "chan1", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)}) ++ if _, ok := check(true); ok { ++ t.Fatal("matched a non-matching message") ++ } ++ ++ want := snowflakeAt(start.Add(time.Second)) ++ go func() { ++ time.Sleep(20 * time.Millisecond) ++ lotusNoteGatewayMessage(&discordgo.Message{ID: want, ChannelID: "chan1", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)}) ++ }() ++ got, ok := check(true) ++ if !ok || got.ID != want { ++ t.Fatalf("got %v ok=%v", got, ok) ++ } ++} ++ ++func TestEchoWaiterDoneUnregisters(t *testing.T) { ++ _, done := lotusExpectEcho("chanX", func(*discordgo.Message) bool { return true }) ++ done() ++ lotusEchoMu.Lock() ++ defer lotusEchoMu.Unlock() ++ if _, ok := lotusEchoWaiters["chanX"]; ok { ++ t.Fatal("waiter still registered after done") ++ } ++} +diff --git a/lotus_retry.go b/lotus_retry.go +index 972b612..0c5e147 100644 +--- a/lotus_retry.go ++++ b/lotus_retry.go +@@ -74,17 +74,69 @@ func lotusRewindFiles(files []*discordgo.File) { + } + } + ++// isAmbiguousSendError reports whether err happened after the request may ++// already have reached Discord: the connection broke or timed out while ++// sending or waiting for the reply, or Discord answered with a 5xx. ++func isAmbiguousSendError(err error) bool { ++ if err == nil || isPreSendNetworkError(err) { ++ return false ++ } ++ var restErr *discordgo.RESTError ++ if errors.As(err, &restErr) { ++ return restErr.Response != nil && restErr.Response.StatusCode >= 500 ++ } ++ if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { ++ return true ++ } ++ var netErr net.Error ++ if errors.As(err, &netErr) { ++ return true ++ } ++ msg := err.Error() ++ return strings.Contains(msg, "connection reset") || ++ strings.Contains(msg, "broken pipe") || ++ strings.Contains(msg, "Client.Timeout exceeded") || ++ strings.Contains(msg, "server sent GOAWAY") || ++ strings.Contains(msg, "EOF") ++} ++ + // lotusRetrySend calls send until it succeeds, fails with an error that is not +-// a pre-send network error, or the retry budget runs out. +-func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error)) (T, error) { ++// retryable, or the retry budget runs out. ++// ++// Pre-send network errors are always retried. Ambiguous errors (see ++// isAmbiguousSendError) are retried only when checkDelivered is set: it is ++// called first (wait=true lets it wait briefly for evidence) and, if it finds ++// that the message did reach Discord, its result is returned as a success ++// instead of sending again. It is called again with wait=false just before ++// each resend. ++func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error), checkDelivered func(wait bool) (T, bool)) (T, error) { + res, err := send() +- for attempt := 0; attempt < len(lotusSendRetryDelays) && isPreSendNetworkError(err); attempt++ { ++ for attempt := 0; attempt < len(lotusSendRetryDelays); attempt++ { ++ switch { ++ case isPreSendNetworkError(err): ++ case checkDelivered != nil && isAmbiguousSendError(err): ++ if delivered, ok := checkDelivered(true); ok { ++ log.Info().Err(err).Int("attempts", attempt+1). ++ Msg("[lotus] Discord send errored but the message was delivered, not resending") ++ return delivered, nil ++ } ++ default: ++ return res, err ++ } + delay := lotusSendRetryDelays[attempt] + log.Warn().Err(err). + Int("attempt", attempt+1). + Dur("retry_in", delay). +- Msg("[lotus] Discord send failed before reaching Discord, retrying") ++ Msg("[lotus] Discord send failed and the message was not delivered, retrying") + lotusSleep(delay) ++ if checkDelivered != nil { ++ // A late echo of an earlier attempt can land during the backoff. ++ if delivered, ok := checkDelivered(false); ok { ++ log.Info().Int("attempts", attempt+1). ++ Msg("[lotus] earlier Discord send turned out to be delivered, not resending") ++ return delivered, nil ++ } ++ } + lotusRewindFiles(files) + res, err = send() + if err == nil { +diff --git a/lotus_retry_test.go b/lotus_retry_test.go +index efa262e..e35585b 100644 +--- a/lotus_retry_test.go ++++ b/lotus_retry_test.go +@@ -68,7 +68,7 @@ func TestRetrySendRecoversAndRewindsFiles(t *testing.T) { + return nil, urlErr(tlsTimeout{}) + } + return &discordgo.Message{ID: "42"}, nil +- }) ++ }, nil) + if err != nil || msg == nil || msg.ID != "42" { + t.Fatalf("got msg=%v err=%v", msg, err) + } +@@ -86,7 +86,7 @@ func TestRetrySendDoesNotRetryAfterSend(t *testing.T) { + _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { + calls++ + return nil, urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}) +- }) ++ }, nil) + if err == nil || calls != 1 { + t.Fatalf("calls=%d err=%v, want exactly one attempt", calls, err) + } +@@ -98,7 +98,7 @@ func TestRetrySendGivesUpAfterBudget(t *testing.T) { + _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { + calls++ + return nil, urlErr(tlsTimeout{}) +- }) ++ }, nil) + if err == nil { + t.Fatal("expected error after budget") + } +diff --git a/portal.go b/portal.go +index cc6da0d..57ba7f9 100644 +--- a/portal.go ++++ b/portal.go +@@ -1748,9 +1748,18 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) { + if !isWebhookSend { + msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) { + return sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...) +- }) ++ }, nil) + } else { + username, avatarURL := portal.getRelayUserMeta(sender) ++ // [lotus] watch for the gateway echo so an ambiguous failure can be ++ // checked instead of resent blindly (see lotus_echo.go) ++ echoChannel := channelID ++ if threadID != "" { ++ echoChannel = threadID ++ } ++ checkEcho, echoDone := lotusExpectEcho(echoChannel, ++ lotusWebhookEchoMatcher(portal.RelayWebhookID, sendReq.Content, len(sendReq.Files), time.Now())) ++ defer echoDone() + msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) { + return relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{ + Content: sendReq.Content, +@@ -1761,7 +1770,7 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) { + Embeds: sendReq.Embeds, + AllowedMentions: sendReq.AllowedMentions, + }) +- }) ++ }, checkEcho) + } + sender.handlePossible40002(err) + go portal.sendMessageMetrics(evt, err, "Error sending") +diff --git a/user.go b/user.go +index 8c0d6ec..16cd8c8 100644 +--- a/user.go ++++ b/user.go +@@ -704,6 +704,7 @@ func (user *User) eventHandler(rawEvt any) { + case *discordgo.RelationshipUpdate: + user.relationshipUpdateHandler(evt) + case *discordgo.MessageCreate: ++ lotusNoteGatewayMessage(evt.Message) // [lotus] see lotus_echo.go + user.pushPortalMessage(evt, "message create", evt.ChannelID, evt.GuildID) + case *discordgo.MessageDelete: + user.pushPortalMessage(evt, "message delete", evt.ChannelID, evt.GuildID) -- 2.47.3