From b3fec4af2a2a4883bdcff0c0971d2950eb21901e Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Sun, 15 Nov 2026 21:46:31 -0500 Subject: [PATCH] discord-bridge: check the gateway echo before retrying ambiguous webhook sends (#67) --- README.md | 10 +- discord-bridge/build-patched.sh | 2 +- ...ore-retrying-ambiguous-webhook-sends.patch | 437 ++++++++++++++++++ 3 files changed, 445 insertions(+), 4 deletions(-) create mode 100644 discord-bridge/patches/0002-check-gateway-echo-before-retrying-ambiguous-webhook-sends.patch diff --git a/README.md b/README.md index 2987569..bb46e80 100644 --- a/README.md +++ b/README.md @@ -418,9 +418,13 @@ channel webhook with their own display name. - `0001`: when a Matrix → Discord send fails before the request reaches Discord (DNS, connect, or `TLS handshake timeout`), the bridge retries after 2, 5, 10, 20, 30, 60, 60 and 60 s instead of dropping the message with "⚠ Your message may not have been bridged". The room's later messages wait - in order behind the retry. Failures after the request was sent are not retried, because a webhook - send has no dedupe and a retry could post the message twice. Edits, reactions and deletes are not - retried. + in order behind the retry. Edits, reactions and deletes are not retried. + - `0002`: when a relay (webhook) send fails after the request may already have reached Discord + (connection reset, EOF, timeout waiting for the reply, Discord 5xx), the bridge waits up to 6 s for + its own post to come back over the Discord gateway. If it does, the send counts as delivered (no + resend, no copy back into Matrix); if not, it retries with the 0001 backoff, checking again before + each resend. A webhook post has no dedupe, so this echo check is what keeps those retries from + double-posting. **Install (one time):** after this repo's `main` has reached `/opt/matrix-config` on LXC 151, run this on the Proxmox node hosting LXC 109 and 151. It restarts Synapse once and is safe to re-run. diff --git a/discord-bridge/build-patched.sh b/discord-bridge/build-patched.sh index b0b31f8..74aae61 100644 --- a/discord-bridge/build-patched.sh +++ b/discord-bridge/build-patched.sh @@ -25,7 +25,7 @@ for p in "$HERE"/patches/*.patch; do git apply "$p" done -go test -count=1 -run 'Retry|PreSend' . +go test -race -count=1 -run 'Retry|PreSend|Ambiguous|Echo' . COMMIT="$(git rev-parse HEAD)" MAUTRIX="$(go list -m -f '{{.Version}}' maunium.net/go/mautrix)" diff --git a/discord-bridge/patches/0002-check-gateway-echo-before-retrying-ambiguous-webhook-sends.patch b/discord-bridge/patches/0002-check-gateway-echo-before-retrying-ambiguous-webhook-sends.patch new file mode 100644 index 0000000..8a8a73f --- /dev/null +++ b/discord-bridge/patches/0002-check-gateway-echo-before-retrying-ambiguous-webhook-sends.patch @@ -0,0 +1,437 @@ +diff --git a/lotus_echo.go b/lotus_echo.go +new file mode 100644 +index 0000000..0cfdd95 +--- /dev/null ++++ b/lotus_echo.go +@@ -0,0 +1,118 @@ ++package main ++ ++// [lotus] Detect whether a webhook send that errored actually reached Discord. ++// ++// Discord echoes every webhook post back over the gateway as MESSAGE_CREATE. ++// While a relay send is in flight we register a waiter for its channel; the ++// gateway handler (user.go) offers every new message to the waiters before ++// queueing it for the portal. If a send fails ambiguously (the connection broke ++// after the request may have gone out), lotusRetrySend asks the waiter whether ++// the post showed up. If it did, the send is treated as successful and the ++// message ID recorded, so it is neither resent to Discord nor bridged back into ++// Matrix as a new message. ++// ++// The waiter is offered messages from the gateway goroutine, which keeps ++// running while the portal's own loop is blocked in the send. ++ ++import ( ++ "strings" ++ "sync" ++ "time" ++ ++ "github.com/bwmarrin/discordgo" ++) ++ ++// lotusEchoWait is how long to wait for the gateway echo after an ambiguous ++// failure. Echoes normally arrive well under a second after Discord accepts the ++// post. ++var lotusEchoWait = 6 * time.Second ++ ++type lotusEchoWaiter struct { ++ match func(*discordgo.Message) bool ++ ch chan *discordgo.Message ++} ++ ++var ( ++ lotusEchoMu sync.Mutex ++ lotusEchoWaiters = map[string][]*lotusEchoWaiter{} ++) ++ ++// lotusNoteGatewayMessage offers a newly created Discord message to any waiter ++// registered for its channel. Called for every MESSAGE_CREATE. ++func lotusNoteGatewayMessage(msg *discordgo.Message) { ++ if msg == nil || msg.WebhookID == "" { ++ return ++ } ++ lotusEchoMu.Lock() ++ defer lotusEchoMu.Unlock() ++ for _, w := range lotusEchoWaiters[msg.ChannelID] { ++ if w.match(msg) { ++ select { ++ case w.ch <- msg: ++ default: ++ } ++ } ++ } ++} ++ ++// lotusExpectEcho registers a waiter for channelID. check(wait) returns the ++// matching message if one has arrived, waiting up to lotusEchoWait when wait ++// is true. done must be called when the send is finished. ++func lotusExpectEcho(channelID string, match func(*discordgo.Message) bool) (check func(wait bool) (*discordgo.Message, bool), done func()) { ++ w := &lotusEchoWaiter{match: match, ch: make(chan *discordgo.Message, 1)} ++ lotusEchoMu.Lock() ++ lotusEchoWaiters[channelID] = append(lotusEchoWaiters[channelID], w) ++ lotusEchoMu.Unlock() ++ ++ check = func(wait bool) (*discordgo.Message, bool) { ++ if !wait { ++ select { ++ case msg := <-w.ch: ++ return msg, true ++ default: ++ return nil, false ++ } ++ } ++ timer := time.NewTimer(lotusEchoWait) ++ defer timer.Stop() ++ select { ++ case msg := <-w.ch: ++ return msg, true ++ case <-timer.C: ++ return nil, false ++ } ++ } ++ done = func() { ++ lotusEchoMu.Lock() ++ defer lotusEchoMu.Unlock() ++ list := lotusEchoWaiters[channelID] ++ for i, x := range list { ++ if x == w { ++ list = append(list[:i], list[i+1:]...) ++ break ++ } ++ } ++ if len(list) == 0 { ++ delete(lotusEchoWaiters, channelID) ++ } else { ++ lotusEchoWaiters[channelID] = list ++ } ++ } ++ return check, done ++} ++ ++// lotusWebhookEchoMatcher matches the gateway echo of a relay post: same ++// webhook, same text (ignoring surrounding whitespace, which Discord trims), ++// same number of attachments, created no earlier than shortly before the send ++// started. ++func lotusWebhookEchoMatcher(webhookID, content string, numFiles int, started time.Time) func(*discordgo.Message) bool { ++ notBefore := started.Add(-2 * time.Second) ++ content = strings.TrimSpace(content) ++ return func(msg *discordgo.Message) bool { ++ if msg.WebhookID != webhookID || strings.TrimSpace(msg.Content) != content || len(msg.Attachments) != numFiles { ++ return false ++ } ++ ts, err := discordgo.SnowflakeTimestamp(msg.ID) ++ return err != nil || !ts.Before(notBefore) ++ } ++} +diff --git a/lotus_echo_test.go b/lotus_echo_test.go +new file mode 100644 +index 0000000..6f5ad44 +--- /dev/null ++++ b/lotus_echo_test.go +@@ -0,0 +1,153 @@ ++package main ++ ++import ( ++ "errors" ++ "net" ++ "net/http" ++ "strconv" ++ "testing" ++ "time" ++ ++ "github.com/bwmarrin/discordgo" ++ "github.com/rs/zerolog" ++) ++ ++func resetErr() error { ++ return urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}) ++} ++ ++func TestIsAmbiguousSendError(t *testing.T) { ++ cases := []struct { ++ name string ++ err error ++ want bool ++ }{ ++ {"nil", nil, false}, ++ {"tls handshake timeout is pre-send", urlErr(tlsTimeout{}), false}, ++ {"read reset", resetErr(), true}, ++ {"awaiting headers", urlErr(errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), true}, ++ {"unexpected EOF", urlErr(errors.New("unexpected EOF")), true}, ++ {"discord 502", &discordgo.RESTError{Response: &http.Response{StatusCode: 502}}, true}, ++ {"discord 400", &discordgo.RESTError{Response: &http.Response{StatusCode: 400}}, false}, ++ {"other", errors.New("unknown message type"), false}, ++ } ++ for _, c := range cases { ++ if got := isAmbiguousSendError(c.err); got != c.want { ++ t.Errorf("%s: got %v want %v", c.name, got, c.want) ++ } ++ } ++} ++ ++func TestRetrySendAmbiguousWithoutCheckerIsNotRetried(t *testing.T) { ++ withNoSleep(t) ++ calls := 0 ++ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { ++ calls++ ++ return nil, resetErr() ++ }, nil) ++ if err == nil || calls != 1 { ++ t.Fatalf("calls=%d err=%v", calls, err) ++ } ++} ++ ++func TestRetrySendAmbiguousButDeliveredIsNotResent(t *testing.T) { ++ withNoSleep(t) ++ calls := 0 ++ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { ++ calls++ ++ return nil, resetErr() ++ }, func(wait bool) (*discordgo.Message, bool) { ++ return &discordgo.Message{ID: "echo"}, true ++ }) ++ if err != nil || msg == nil || msg.ID != "echo" || calls != 1 { ++ t.Fatalf("calls=%d msg=%v err=%v", calls, msg, err) ++ } ++} ++ ++func TestRetrySendAmbiguousNotDeliveredIsResent(t *testing.T) { ++ slept := withNoSleep(t) ++ calls := 0 ++ var waits []bool ++ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { ++ calls++ ++ if calls == 1 { ++ return nil, resetErr() ++ } ++ return &discordgo.Message{ID: "2"}, nil ++ }, func(wait bool) (*discordgo.Message, bool) { ++ waits = append(waits, wait) ++ return nil, false ++ }) ++ if err != nil || msg.ID != "2" || calls != 2 || len(*slept) != 1 { ++ t.Fatalf("calls=%d msg=%v err=%v slept=%v", calls, msg, err, *slept) ++ } ++ if len(waits) != 2 || !waits[0] || waits[1] { ++ t.Fatalf("waits=%v, want [true false]", waits) ++ } ++} ++ ++func TestRetrySendLateEchoDuringBackoffStopsResend(t *testing.T) { ++ withNoSleep(t) ++ calls := 0 ++ checks := 0 ++ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { ++ calls++ ++ return nil, resetErr() ++ }, func(wait bool) (*discordgo.Message, bool) { ++ checks++ ++ if checks == 2 { // the echo arrives during the backoff ++ return &discordgo.Message{ID: "late"}, true ++ } ++ return nil, false ++ }) ++ if err != nil || msg.ID != "late" || calls != 1 { ++ t.Fatalf("calls=%d msg=%v err=%v", calls, msg, err) ++ } ++} ++ ++func snowflakeAt(ts time.Time) string { ++ ms := ts.UnixMilli() - 1420070400000 ++ return strconv.FormatInt(ms<<22, 10) ++} ++ ++func TestEchoWaiterMatchesWebhookPost(t *testing.T) { ++ oldWait := lotusEchoWait ++ lotusEchoWait = 200 * time.Millisecond ++ t.Cleanup(func() { lotusEchoWait = oldWait }) ++ ++ start := time.Now() ++ check, done := lotusExpectEcho("chan1", lotusWebhookEchoMatcher("hook1", "hi", 1, start)) ++ defer done() ++ ++ if _, ok := check(false); ok { ++ t.Fatal("matched before any message") ++ } ++ // Wrong webhook, wrong content, wrong channel and an old message are ignored. ++ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan1", WebhookID: "other", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)}) ++ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan1", WebhookID: "hook1", Content: "nope", Attachments: make([]*discordgo.MessageAttachment, 1)}) ++ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan2", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)}) ++ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start.Add(-time.Minute)), ChannelID: "chan1", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)}) ++ if _, ok := check(true); ok { ++ t.Fatal("matched a non-matching message") ++ } ++ ++ want := snowflakeAt(start.Add(time.Second)) ++ go func() { ++ time.Sleep(20 * time.Millisecond) ++ lotusNoteGatewayMessage(&discordgo.Message{ID: want, ChannelID: "chan1", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)}) ++ }() ++ got, ok := check(true) ++ if !ok || got.ID != want { ++ t.Fatalf("got %v ok=%v", got, ok) ++ } ++} ++ ++func TestEchoWaiterDoneUnregisters(t *testing.T) { ++ _, done := lotusExpectEcho("chanX", func(*discordgo.Message) bool { return true }) ++ done() ++ lotusEchoMu.Lock() ++ defer lotusEchoMu.Unlock() ++ if _, ok := lotusEchoWaiters["chanX"]; ok { ++ t.Fatal("waiter still registered after done") ++ } ++} +diff --git a/lotus_retry.go b/lotus_retry.go +index 972b612..0c5e147 100644 +--- a/lotus_retry.go ++++ b/lotus_retry.go +@@ -74,17 +74,69 @@ func lotusRewindFiles(files []*discordgo.File) { + } + } + ++// isAmbiguousSendError reports whether err happened after the request may ++// already have reached Discord: the connection broke or timed out while ++// sending or waiting for the reply, or Discord answered with a 5xx. ++func isAmbiguousSendError(err error) bool { ++ if err == nil || isPreSendNetworkError(err) { ++ return false ++ } ++ var restErr *discordgo.RESTError ++ if errors.As(err, &restErr) { ++ return restErr.Response != nil && restErr.Response.StatusCode >= 500 ++ } ++ if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { ++ return true ++ } ++ var netErr net.Error ++ if errors.As(err, &netErr) { ++ return true ++ } ++ msg := err.Error() ++ return strings.Contains(msg, "connection reset") || ++ strings.Contains(msg, "broken pipe") || ++ strings.Contains(msg, "Client.Timeout exceeded") || ++ strings.Contains(msg, "server sent GOAWAY") || ++ strings.Contains(msg, "EOF") ++} ++ + // lotusRetrySend calls send until it succeeds, fails with an error that is not +-// a pre-send network error, or the retry budget runs out. +-func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error)) (T, error) { ++// retryable, or the retry budget runs out. ++// ++// Pre-send network errors are always retried. Ambiguous errors (see ++// isAmbiguousSendError) are retried only when checkDelivered is set: it is ++// called first (wait=true lets it wait briefly for evidence) and, if it finds ++// that the message did reach Discord, its result is returned as a success ++// instead of sending again. It is called again with wait=false just before ++// each resend. ++func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error), checkDelivered func(wait bool) (T, bool)) (T, error) { + res, err := send() +- for attempt := 0; attempt < len(lotusSendRetryDelays) && isPreSendNetworkError(err); attempt++ { ++ for attempt := 0; attempt < len(lotusSendRetryDelays); attempt++ { ++ switch { ++ case isPreSendNetworkError(err): ++ case checkDelivered != nil && isAmbiguousSendError(err): ++ if delivered, ok := checkDelivered(true); ok { ++ log.Info().Err(err).Int("attempts", attempt+1). ++ Msg("[lotus] Discord send errored but the message was delivered, not resending") ++ return delivered, nil ++ } ++ default: ++ return res, err ++ } + delay := lotusSendRetryDelays[attempt] + log.Warn().Err(err). + Int("attempt", attempt+1). + Dur("retry_in", delay). +- Msg("[lotus] Discord send failed before reaching Discord, retrying") ++ Msg("[lotus] Discord send failed and the message was not delivered, retrying") + lotusSleep(delay) ++ if checkDelivered != nil { ++ // A late echo of an earlier attempt can land during the backoff. ++ if delivered, ok := checkDelivered(false); ok { ++ log.Info().Int("attempts", attempt+1). ++ Msg("[lotus] earlier Discord send turned out to be delivered, not resending") ++ return delivered, nil ++ } ++ } + lotusRewindFiles(files) + res, err = send() + if err == nil { +diff --git a/lotus_retry_test.go b/lotus_retry_test.go +index efa262e..e35585b 100644 +--- a/lotus_retry_test.go ++++ b/lotus_retry_test.go +@@ -68,7 +68,7 @@ func TestRetrySendRecoversAndRewindsFiles(t *testing.T) { + return nil, urlErr(tlsTimeout{}) + } + return &discordgo.Message{ID: "42"}, nil +- }) ++ }, nil) + if err != nil || msg == nil || msg.ID != "42" { + t.Fatalf("got msg=%v err=%v", msg, err) + } +@@ -86,7 +86,7 @@ func TestRetrySendDoesNotRetryAfterSend(t *testing.T) { + _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { + calls++ + return nil, urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}) +- }) ++ }, nil) + if err == nil || calls != 1 { + t.Fatalf("calls=%d err=%v, want exactly one attempt", calls, err) + } +@@ -98,7 +98,7 @@ func TestRetrySendGivesUpAfterBudget(t *testing.T) { + _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { + calls++ + return nil, urlErr(tlsTimeout{}) +- }) ++ }, nil) + if err == nil { + t.Fatal("expected error after budget") + } +diff --git a/portal.go b/portal.go +index cc6da0d..57ba7f9 100644 +--- a/portal.go ++++ b/portal.go +@@ -1748,9 +1748,18 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) { + if !isWebhookSend { + msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) { + return sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...) +- }) ++ }, nil) + } else { + username, avatarURL := portal.getRelayUserMeta(sender) ++ // [lotus] watch for the gateway echo so an ambiguous failure can be ++ // checked instead of resent blindly (see lotus_echo.go) ++ echoChannel := channelID ++ if threadID != "" { ++ echoChannel = threadID ++ } ++ checkEcho, echoDone := lotusExpectEcho(echoChannel, ++ lotusWebhookEchoMatcher(portal.RelayWebhookID, sendReq.Content, len(sendReq.Files), time.Now())) ++ defer echoDone() + msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) { + return relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{ + Content: sendReq.Content, +@@ -1761,7 +1770,7 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) { + Embeds: sendReq.Embeds, + AllowedMentions: sendReq.AllowedMentions, + }) +- }) ++ }, checkEcho) + } + sender.handlePossible40002(err) + go portal.sendMessageMetrics(evt, err, "Error sending") +diff --git a/user.go b/user.go +index 8c0d6ec..16cd8c8 100644 +--- a/user.go ++++ b/user.go +@@ -704,6 +704,7 @@ func (user *User) eventHandler(rawEvt any) { + case *discordgo.RelationshipUpdate: + user.relationshipUpdateHandler(evt) + case *discordgo.MessageCreate: ++ lotusNoteGatewayMessage(evt.Message) // [lotus] see lotus_echo.go + user.pushPortalMessage(evt, "message create", evt.ChannelID, evt.GuildID) + case *discordgo.MessageDelete: + user.pushPortalMessage(evt, "message delete", evt.ChannelID, evt.GuildID)