discord-bridge: check the gateway echo before retrying ambiguous webhook sends (#67)
Lint / JS (eslint) (push) Successful in 6s
Lint / Shell (shellcheck) (push) Successful in 8s
Lint / No secrets in webhook configs (push) Successful in 5s
Lint / No Hookshot webhook IDs in README (push) Successful in 4s
Lint / Python (ruff) (push) Successful in 4s
Lint / Landing page is rendered (matrix (push) Successful in 5s
Lint / Secret scan (gitleaks) (push) Successful in 6s
Lint / Python deps (pip-audit) (push) Successful in 36s

This commit was merged in pull request #67.
This commit is contained in:
2026-11-15 21:46:31 -05:00
parent a57c91860b
commit b3fec4af2a
3 changed files with 445 additions and 4 deletions
+7 -3
View File
@@ -418,9 +418,13 @@ channel webhook with their own display name.
- `0001`: when a Matrix → Discord send fails before the request reaches Discord (DNS, connect, or
`TLS handshake timeout`), the bridge retries after 2, 5, 10, 20, 30, 60, 60 and 60 s instead of
dropping the message with "⚠ Your message may not have been bridged". The room's later messages wait
in order behind the retry. Failures after the request was sent are not retried, because a webhook
send has no dedupe and a retry could post the message twice. Edits, reactions and deletes are not
retried.
in order behind the retry. Edits, reactions and deletes are not retried.
- `0002`: when a relay (webhook) send fails after the request may already have reached Discord
(connection reset, EOF, timeout waiting for the reply, Discord 5xx), the bridge waits up to 6 s for
its own post to come back over the Discord gateway. If it does, the send counts as delivered (no
resend, no copy back into Matrix); if not, it retries with the 0001 backoff, checking again before
each resend. A webhook post has no dedupe, so this echo check is what keeps those retries from
double-posting.
**Install (one time):** after this repo's `main` has reached `/opt/matrix-config` on LXC 151, run this
on the Proxmox node hosting LXC 109 and 151. It restarts Synapse once and is safe to re-run.
+1 -1
View File
@@ -25,7 +25,7 @@ for p in "$HERE"/patches/*.patch; do
git apply "$p"
done
go test -count=1 -run 'Retry|PreSend' .
go test -race -count=1 -run 'Retry|PreSend|Ambiguous|Echo' .
COMMIT="$(git rev-parse HEAD)"
MAUTRIX="$(go list -m -f '{{.Version}}' maunium.net/go/mautrix)"
@@ -0,0 +1,437 @@
diff --git a/lotus_echo.go b/lotus_echo.go
new file mode 100644
index 0000000..0cfdd95
--- /dev/null
+++ b/lotus_echo.go
@@ -0,0 +1,118 @@
+package main
+
+// [lotus] Detect whether a webhook send that errored actually reached Discord.
+//
+// Discord echoes every webhook post back over the gateway as MESSAGE_CREATE.
+// While a relay send is in flight we register a waiter for its channel; the
+// gateway handler (user.go) offers every new message to the waiters before
+// queueing it for the portal. If a send fails ambiguously (the connection broke
+// after the request may have gone out), lotusRetrySend asks the waiter whether
+// the post showed up. If it did, the send is treated as successful and the
+// message ID recorded, so it is neither resent to Discord nor bridged back into
+// Matrix as a new message.
+//
+// The waiter is offered messages from the gateway goroutine, which keeps
+// running while the portal's own loop is blocked in the send.
+
+import (
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/bwmarrin/discordgo"
+)
+
+// lotusEchoWait is how long to wait for the gateway echo after an ambiguous
+// failure. Echoes normally arrive well under a second after Discord accepts the
+// post.
+var lotusEchoWait = 6 * time.Second
+
+type lotusEchoWaiter struct {
+ match func(*discordgo.Message) bool
+ ch chan *discordgo.Message
+}
+
+var (
+ lotusEchoMu sync.Mutex
+ lotusEchoWaiters = map[string][]*lotusEchoWaiter{}
+)
+
+// lotusNoteGatewayMessage offers a newly created Discord message to any waiter
+// registered for its channel. Called for every MESSAGE_CREATE.
+func lotusNoteGatewayMessage(msg *discordgo.Message) {
+ if msg == nil || msg.WebhookID == "" {
+ return
+ }
+ lotusEchoMu.Lock()
+ defer lotusEchoMu.Unlock()
+ for _, w := range lotusEchoWaiters[msg.ChannelID] {
+ if w.match(msg) {
+ select {
+ case w.ch <- msg:
+ default:
+ }
+ }
+ }
+}
+
+// lotusExpectEcho registers a waiter for channelID. check(wait) returns the
+// matching message if one has arrived, waiting up to lotusEchoWait when wait
+// is true. done must be called when the send is finished.
+func lotusExpectEcho(channelID string, match func(*discordgo.Message) bool) (check func(wait bool) (*discordgo.Message, bool), done func()) {
+ w := &lotusEchoWaiter{match: match, ch: make(chan *discordgo.Message, 1)}
+ lotusEchoMu.Lock()
+ lotusEchoWaiters[channelID] = append(lotusEchoWaiters[channelID], w)
+ lotusEchoMu.Unlock()
+
+ check = func(wait bool) (*discordgo.Message, bool) {
+ if !wait {
+ select {
+ case msg := <-w.ch:
+ return msg, true
+ default:
+ return nil, false
+ }
+ }
+ timer := time.NewTimer(lotusEchoWait)
+ defer timer.Stop()
+ select {
+ case msg := <-w.ch:
+ return msg, true
+ case <-timer.C:
+ return nil, false
+ }
+ }
+ done = func() {
+ lotusEchoMu.Lock()
+ defer lotusEchoMu.Unlock()
+ list := lotusEchoWaiters[channelID]
+ for i, x := range list {
+ if x == w {
+ list = append(list[:i], list[i+1:]...)
+ break
+ }
+ }
+ if len(list) == 0 {
+ delete(lotusEchoWaiters, channelID)
+ } else {
+ lotusEchoWaiters[channelID] = list
+ }
+ }
+ return check, done
+}
+
+// lotusWebhookEchoMatcher matches the gateway echo of a relay post: same
+// webhook, same text (ignoring surrounding whitespace, which Discord trims),
+// same number of attachments, created no earlier than shortly before the send
+// started.
+func lotusWebhookEchoMatcher(webhookID, content string, numFiles int, started time.Time) func(*discordgo.Message) bool {
+ notBefore := started.Add(-2 * time.Second)
+ content = strings.TrimSpace(content)
+ return func(msg *discordgo.Message) bool {
+ if msg.WebhookID != webhookID || strings.TrimSpace(msg.Content) != content || len(msg.Attachments) != numFiles {
+ return false
+ }
+ ts, err := discordgo.SnowflakeTimestamp(msg.ID)
+ return err != nil || !ts.Before(notBefore)
+ }
+}
diff --git a/lotus_echo_test.go b/lotus_echo_test.go
new file mode 100644
index 0000000..6f5ad44
--- /dev/null
+++ b/lotus_echo_test.go
@@ -0,0 +1,153 @@
+package main
+
+import (
+ "errors"
+ "net"
+ "net/http"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/bwmarrin/discordgo"
+ "github.com/rs/zerolog"
+)
+
+func resetErr() error {
+ return urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")})
+}
+
+func TestIsAmbiguousSendError(t *testing.T) {
+ cases := []struct {
+ name string
+ err error
+ want bool
+ }{
+ {"nil", nil, false},
+ {"tls handshake timeout is pre-send", urlErr(tlsTimeout{}), false},
+ {"read reset", resetErr(), true},
+ {"awaiting headers", urlErr(errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), true},
+ {"unexpected EOF", urlErr(errors.New("unexpected EOF")), true},
+ {"discord 502", &discordgo.RESTError{Response: &http.Response{StatusCode: 502}}, true},
+ {"discord 400", &discordgo.RESTError{Response: &http.Response{StatusCode: 400}}, false},
+ {"other", errors.New("unknown message type"), false},
+ }
+ for _, c := range cases {
+ if got := isAmbiguousSendError(c.err); got != c.want {
+ t.Errorf("%s: got %v want %v", c.name, got, c.want)
+ }
+ }
+}
+
+func TestRetrySendAmbiguousWithoutCheckerIsNotRetried(t *testing.T) {
+ withNoSleep(t)
+ calls := 0
+ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
+ calls++
+ return nil, resetErr()
+ }, nil)
+ if err == nil || calls != 1 {
+ t.Fatalf("calls=%d err=%v", calls, err)
+ }
+}
+
+func TestRetrySendAmbiguousButDeliveredIsNotResent(t *testing.T) {
+ withNoSleep(t)
+ calls := 0
+ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
+ calls++
+ return nil, resetErr()
+ }, func(wait bool) (*discordgo.Message, bool) {
+ return &discordgo.Message{ID: "echo"}, true
+ })
+ if err != nil || msg == nil || msg.ID != "echo" || calls != 1 {
+ t.Fatalf("calls=%d msg=%v err=%v", calls, msg, err)
+ }
+}
+
+func TestRetrySendAmbiguousNotDeliveredIsResent(t *testing.T) {
+ slept := withNoSleep(t)
+ calls := 0
+ var waits []bool
+ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
+ calls++
+ if calls == 1 {
+ return nil, resetErr()
+ }
+ return &discordgo.Message{ID: "2"}, nil
+ }, func(wait bool) (*discordgo.Message, bool) {
+ waits = append(waits, wait)
+ return nil, false
+ })
+ if err != nil || msg.ID != "2" || calls != 2 || len(*slept) != 1 {
+ t.Fatalf("calls=%d msg=%v err=%v slept=%v", calls, msg, err, *slept)
+ }
+ if len(waits) != 2 || !waits[0] || waits[1] {
+ t.Fatalf("waits=%v, want [true false]", waits)
+ }
+}
+
+func TestRetrySendLateEchoDuringBackoffStopsResend(t *testing.T) {
+ withNoSleep(t)
+ calls := 0
+ checks := 0
+ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
+ calls++
+ return nil, resetErr()
+ }, func(wait bool) (*discordgo.Message, bool) {
+ checks++
+ if checks == 2 { // the echo arrives during the backoff
+ return &discordgo.Message{ID: "late"}, true
+ }
+ return nil, false
+ })
+ if err != nil || msg.ID != "late" || calls != 1 {
+ t.Fatalf("calls=%d msg=%v err=%v", calls, msg, err)
+ }
+}
+
+func snowflakeAt(ts time.Time) string {
+ ms := ts.UnixMilli() - 1420070400000
+ return strconv.FormatInt(ms<<22, 10)
+}
+
+func TestEchoWaiterMatchesWebhookPost(t *testing.T) {
+ oldWait := lotusEchoWait
+ lotusEchoWait = 200 * time.Millisecond
+ t.Cleanup(func() { lotusEchoWait = oldWait })
+
+ start := time.Now()
+ check, done := lotusExpectEcho("chan1", lotusWebhookEchoMatcher("hook1", "hi", 1, start))
+ defer done()
+
+ if _, ok := check(false); ok {
+ t.Fatal("matched before any message")
+ }
+ // Wrong webhook, wrong content, wrong channel and an old message are ignored.
+ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan1", WebhookID: "other", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)})
+ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan1", WebhookID: "hook1", Content: "nope", Attachments: make([]*discordgo.MessageAttachment, 1)})
+ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan2", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)})
+ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start.Add(-time.Minute)), ChannelID: "chan1", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)})
+ if _, ok := check(true); ok {
+ t.Fatal("matched a non-matching message")
+ }
+
+ want := snowflakeAt(start.Add(time.Second))
+ go func() {
+ time.Sleep(20 * time.Millisecond)
+ lotusNoteGatewayMessage(&discordgo.Message{ID: want, ChannelID: "chan1", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)})
+ }()
+ got, ok := check(true)
+ if !ok || got.ID != want {
+ t.Fatalf("got %v ok=%v", got, ok)
+ }
+}
+
+func TestEchoWaiterDoneUnregisters(t *testing.T) {
+ _, done := lotusExpectEcho("chanX", func(*discordgo.Message) bool { return true })
+ done()
+ lotusEchoMu.Lock()
+ defer lotusEchoMu.Unlock()
+ if _, ok := lotusEchoWaiters["chanX"]; ok {
+ t.Fatal("waiter still registered after done")
+ }
+}
diff --git a/lotus_retry.go b/lotus_retry.go
index 972b612..0c5e147 100644
--- a/lotus_retry.go
+++ b/lotus_retry.go
@@ -74,17 +74,69 @@ func lotusRewindFiles(files []*discordgo.File) {
}
}
+// isAmbiguousSendError reports whether err happened after the request may
+// already have reached Discord: the connection broke or timed out while
+// sending or waiting for the reply, or Discord answered with a 5xx.
+func isAmbiguousSendError(err error) bool {
+ if err == nil || isPreSendNetworkError(err) {
+ return false
+ }
+ var restErr *discordgo.RESTError
+ if errors.As(err, &restErr) {
+ return restErr.Response != nil && restErr.Response.StatusCode >= 500
+ }
+ if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+ return true
+ }
+ var netErr net.Error
+ if errors.As(err, &netErr) {
+ return true
+ }
+ msg := err.Error()
+ return strings.Contains(msg, "connection reset") ||
+ strings.Contains(msg, "broken pipe") ||
+ strings.Contains(msg, "Client.Timeout exceeded") ||
+ strings.Contains(msg, "server sent GOAWAY") ||
+ strings.Contains(msg, "EOF")
+}
+
// lotusRetrySend calls send until it succeeds, fails with an error that is not
-// a pre-send network error, or the retry budget runs out.
-func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error)) (T, error) {
+// retryable, or the retry budget runs out.
+//
+// Pre-send network errors are always retried. Ambiguous errors (see
+// isAmbiguousSendError) are retried only when checkDelivered is set: it is
+// called first (wait=true lets it wait briefly for evidence) and, if it finds
+// that the message did reach Discord, its result is returned as a success
+// instead of sending again. It is called again with wait=false just before
+// each resend.
+func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error), checkDelivered func(wait bool) (T, bool)) (T, error) {
res, err := send()
- for attempt := 0; attempt < len(lotusSendRetryDelays) && isPreSendNetworkError(err); attempt++ {
+ for attempt := 0; attempt < len(lotusSendRetryDelays); attempt++ {
+ switch {
+ case isPreSendNetworkError(err):
+ case checkDelivered != nil && isAmbiguousSendError(err):
+ if delivered, ok := checkDelivered(true); ok {
+ log.Info().Err(err).Int("attempts", attempt+1).
+ Msg("[lotus] Discord send errored but the message was delivered, not resending")
+ return delivered, nil
+ }
+ default:
+ return res, err
+ }
delay := lotusSendRetryDelays[attempt]
log.Warn().Err(err).
Int("attempt", attempt+1).
Dur("retry_in", delay).
- Msg("[lotus] Discord send failed before reaching Discord, retrying")
+ Msg("[lotus] Discord send failed and the message was not delivered, retrying")
lotusSleep(delay)
+ if checkDelivered != nil {
+ // A late echo of an earlier attempt can land during the backoff.
+ if delivered, ok := checkDelivered(false); ok {
+ log.Info().Int("attempts", attempt+1).
+ Msg("[lotus] earlier Discord send turned out to be delivered, not resending")
+ return delivered, nil
+ }
+ }
lotusRewindFiles(files)
res, err = send()
if err == nil {
diff --git a/lotus_retry_test.go b/lotus_retry_test.go
index efa262e..e35585b 100644
--- a/lotus_retry_test.go
+++ b/lotus_retry_test.go
@@ -68,7 +68,7 @@ func TestRetrySendRecoversAndRewindsFiles(t *testing.T) {
return nil, urlErr(tlsTimeout{})
}
return &discordgo.Message{ID: "42"}, nil
- })
+ }, nil)
if err != nil || msg == nil || msg.ID != "42" {
t.Fatalf("got msg=%v err=%v", msg, err)
}
@@ -86,7 +86,7 @@ func TestRetrySendDoesNotRetryAfterSend(t *testing.T) {
_, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
calls++
return nil, urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")})
- })
+ }, nil)
if err == nil || calls != 1 {
t.Fatalf("calls=%d err=%v, want exactly one attempt", calls, err)
}
@@ -98,7 +98,7 @@ func TestRetrySendGivesUpAfterBudget(t *testing.T) {
_, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
calls++
return nil, urlErr(tlsTimeout{})
- })
+ }, nil)
if err == nil {
t.Fatal("expected error after budget")
}
diff --git a/portal.go b/portal.go
index cc6da0d..57ba7f9 100644
--- a/portal.go
+++ b/portal.go
@@ -1748,9 +1748,18 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) {
if !isWebhookSend {
msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) {
return sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...)
- })
+ }, nil)
} else {
username, avatarURL := portal.getRelayUserMeta(sender)
+ // [lotus] watch for the gateway echo so an ambiguous failure can be
+ // checked instead of resent blindly (see lotus_echo.go)
+ echoChannel := channelID
+ if threadID != "" {
+ echoChannel = threadID
+ }
+ checkEcho, echoDone := lotusExpectEcho(echoChannel,
+ lotusWebhookEchoMatcher(portal.RelayWebhookID, sendReq.Content, len(sendReq.Files), time.Now()))
+ defer echoDone()
msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) {
return relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{
Content: sendReq.Content,
@@ -1761,7 +1770,7 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) {
Embeds: sendReq.Embeds,
AllowedMentions: sendReq.AllowedMentions,
})
- })
+ }, checkEcho)
}
sender.handlePossible40002(err)
go portal.sendMessageMetrics(evt, err, "Error sending")
diff --git a/user.go b/user.go
index 8c0d6ec..16cd8c8 100644
--- a/user.go
+++ b/user.go
@@ -704,6 +704,7 @@ func (user *User) eventHandler(rawEvt any) {
case *discordgo.RelationshipUpdate:
user.relationshipUpdateHandler(evt)
case *discordgo.MessageCreate:
+ lotusNoteGatewayMessage(evt.Message) // [lotus] see lotus_echo.go
user.pushPortalMessage(evt, "message create", evt.ChannelID, evt.GuildID)
case *discordgo.MessageDelete:
user.pushPortalMessage(evt, "message delete", evt.ChannelID, evt.GuildID)