discord-bridge: retry Matrix→Discord sends that failed before reaching Discord #65

Merged
jared merged 1 commits from discord-bridge-send-retry into main 2026-11-14 14:31:24 -05:00
3 changed files with 301 additions and 0 deletions
+9
View File
@@ -412,6 +412,15 @@ channel webhook with their own display name.
- Database: `mautrix_discord` on LXC 109, role `mautrix_discord`, allowed from 10.10.10.29 in `pg_hba.conf`
- Service: `systemctl status mautrix-discord`, logs in `journalctl -u mautrix-discord` and `/var/log/mautrix-discord/`
- End-to-bridge encryption is allowed, so it can join encrypted rooms
- **Patched build.** The deployed binary is upstream v0.7.7 plus `discord-bridge/patches/`, built with
`discord-bridge/build-patched.sh` (deploy steps are in the script's header). `install.sh` still
downloads the stock release, so rebuild and redeploy the patched binary after any (re)install or upgrade.
- `0001`: when a Matrix → Discord send fails before the request reaches Discord (DNS, connect, or
`TLS handshake timeout`), the bridge retries after 2, 5, 10, 20, 30, 60, 60 and 60 s instead of
dropping the message with "⚠ Your message may not have been bridged". The room's later messages wait
in order behind the retry. Failures after the request was sent are not retried, because a webhook
send has no dedupe and a retry could post the message twice. Edits, reactions and deletes are not
retried.
**Install (one time):** after this repo's `main` has reached `/opt/matrix-config` on LXC 151, run this
on the Proxmox node hosting LXC 109 and 151. It restarts Synapse once and is safe to re-run.
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# Build mautrix-discord with the Lotus patches in discord-bridge/patches/.
#
# Needs: git, Go (version from upstream go.mod or newer), a C compiler and libolm-dev.
# Output: ./mautrix-discord-lotus (dynamically linked against libolm.so.3, which
# LXC 151 has from the libolm3 package).
#
# Deploy (on the Proxmox node hosting LXC 151):
# pct push 151 mautrix-discord-lotus /opt/mautrix-discord/mautrix-discord.new
# pct exec 151 -- bash -c 'cd /opt/mautrix-discord && cp -a mautrix-discord mautrix-discord.bak-$(date +%Y%m%d) \
# && chown mautrix-discord: mautrix-discord.new && chmod 755 mautrix-discord.new \
# && mv mautrix-discord.new mautrix-discord && systemctl restart mautrix-discord'
set -euo pipefail
VERSION="${VERSION:-v0.7.7}"
HERE="$(cd "$(dirname "$0")" && pwd)"
OUT="${OUT:-$PWD/mautrix-discord-lotus}"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
git clone -q --depth 1 --branch "$VERSION" https://github.com/mautrix/discord.git "$WORK/src"
cd "$WORK/src"
for p in "$HERE"/patches/*.patch; do
echo "applying $(basename "$p")"
git apply "$p"
done
go test -count=1 -run 'Retry|PreSend' .
COMMIT="$(git rev-parse HEAD)"
MAUTRIX="$(go list -m -f '{{.Version}}' maunium.net/go/mautrix)"
CGO_ENABLED=1 go build -o "$OUT" -ldflags "-s -w \
-X main.Tag=$VERSION \
-X main.Commit=$COMMIT+lotus \
-X 'main.BuildTime=$(date '+%b %_d %Y, %H:%M:%S')' \
-X 'maunium.net/go/mautrix.GoModVersion=$MAUTRIX'" .
"$OUT" --version
echo "built $OUT"
@@ -0,0 +1,253 @@
diff --git a/lotus_retry.go b/lotus_retry.go
new file mode 100644
index 0000000..972b612
--- /dev/null
+++ b/lotus_retry.go
@@ -0,0 +1,95 @@
+package main
+
+// [lotus] Retry Matrix→Discord sends that failed before reaching Discord.
+//
+// Upstream sends each message exactly once; a momentary network blip (for
+// example "net/http: TLS handshake timeout") drops the message and only posts
+// a "may not have been bridged" notice. We retry, but only for failures that
+// happen before the HTTP request is written (DNS, dial, TLS handshake). Those
+// cannot have reached Discord, so a retry can never post a duplicate.
+// Failures after the request was sent (read timeouts, resets, 5xx) are left
+// alone because webhook sends have no server-side dedupe.
+//
+// The retry runs inline in the portal's Matrix event loop, which handles a
+// room's events one at a time, so later messages queue behind it and ordering
+// is preserved.
+
+import (
+ "errors"
+ "io"
+ "net"
+ "strings"
+ "time"
+
+ "github.com/bwmarrin/discordgo"
+ "github.com/rs/zerolog"
+)
+
+var lotusSendRetryDelays = []time.Duration{
+ 2 * time.Second,
+ 5 * time.Second,
+ 10 * time.Second,
+ 20 * time.Second,
+ 30 * time.Second,
+ 60 * time.Second,
+ 60 * time.Second,
+ 60 * time.Second,
+}
+
+// lotusSleep is swapped out in tests.
+var lotusSleep = time.Sleep
+
+// isPreSendNetworkError reports whether err happened before the request could
+// have been written to Discord.
+func isPreSendNetworkError(err error) bool {
+ if err == nil {
+ return false
+ }
+ var restErr *discordgo.RESTError
+ if errors.As(err, &restErr) {
+ // Discord answered, so the request was delivered.
+ return false
+ }
+ var dnsErr *net.DNSError
+ if errors.As(err, &dnsErr) {
+ return true
+ }
+ var opErr *net.OpError
+ if errors.As(err, &opErr) && opErr.Op == "dial" {
+ return true
+ }
+ msg := err.Error()
+ return strings.Contains(msg, "TLS handshake timeout") ||
+ strings.Contains(msg, "proxyconnect")
+}
+
+func lotusRewindFiles(files []*discordgo.File) {
+ for _, f := range files {
+ if f == nil {
+ continue
+ }
+ if s, ok := f.Reader.(io.Seeker); ok {
+ _, _ = s.Seek(0, io.SeekStart)
+ }
+ }
+}
+
+// lotusRetrySend calls send until it succeeds, fails with an error that is not
+// a pre-send network error, or the retry budget runs out.
+func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error)) (T, error) {
+ res, err := send()
+ for attempt := 0; attempt < len(lotusSendRetryDelays) && isPreSendNetworkError(err); attempt++ {
+ delay := lotusSendRetryDelays[attempt]
+ log.Warn().Err(err).
+ Int("attempt", attempt+1).
+ Dur("retry_in", delay).
+ Msg("[lotus] Discord send failed before reaching Discord, retrying")
+ lotusSleep(delay)
+ lotusRewindFiles(files)
+ res, err = send()
+ if err == nil {
+ log.Info().Int("attempts", attempt+2).Msg("[lotus] Discord send succeeded after retry")
+ }
+ }
+ return res, err
+}
diff --git a/lotus_retry_test.go b/lotus_retry_test.go
new file mode 100644
index 0000000..efa262e
--- /dev/null
+++ b/lotus_retry_test.go
@@ -0,0 +1,108 @@
+package main
+
+import (
+ "bytes"
+ "errors"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "testing"
+ "time"
+
+ "github.com/bwmarrin/discordgo"
+ "github.com/rs/zerolog"
+)
+
+type tlsTimeout struct{}
+
+func (tlsTimeout) Error() string { return "net/http: TLS handshake timeout" }
+func (tlsTimeout) Timeout() bool { return true }
+func (tlsTimeout) Temporary() bool { return true }
+
+func urlErr(err error) error {
+ return &url.Error{Op: "Post", URL: "https://discord.com/api/v9/webhooks/1/x?wait=true", Err: err}
+}
+
+func TestIsPreSendNetworkError(t *testing.T) {
+ cases := []struct {
+ name string
+ err error
+ want bool
+ }{
+ {"nil", nil, false},
+ {"tls handshake timeout", urlErr(tlsTimeout{}), true},
+ {"dns", urlErr(&net.DNSError{Err: "no such host", Name: "discord.com"}), true},
+ {"dial refused", urlErr(&net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")}), true},
+ {"read reset after send", urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}), false},
+ {"response timeout", urlErr(errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), false},
+ {"discord rest error", &discordgo.RESTError{Response: &http.Response{StatusCode: 500}}, false},
+ {"other", errors.New("unknown message type"), false},
+ }
+ for _, c := range cases {
+ if got := isPreSendNetworkError(c.err); got != c.want {
+ t.Errorf("%s: got %v want %v", c.name, got, c.want)
+ }
+ }
+}
+
+func withNoSleep(t *testing.T) *[]time.Duration {
+ var slept []time.Duration
+ old := lotusSleep
+ lotusSleep = func(d time.Duration) { slept = append(slept, d) }
+ t.Cleanup(func() { lotusSleep = old })
+ return &slept
+}
+
+func TestRetrySendRecoversAndRewindsFiles(t *testing.T) {
+ slept := withNoSleep(t)
+ file := &discordgo.File{Name: "a.txt", Reader: bytes.NewReader([]byte("hello"))}
+ calls := 0
+ msg, err := lotusRetrySend(zerolog.Nop(), []*discordgo.File{file}, func() (*discordgo.Message, error) {
+ calls++
+ data, _ := io.ReadAll(file.Reader)
+ if string(data) != "hello" {
+ t.Fatalf("call %d read %q, file not rewound", calls, data)
+ }
+ if calls < 3 {
+ return nil, urlErr(tlsTimeout{})
+ }
+ return &discordgo.Message{ID: "42"}, nil
+ })
+ if err != nil || msg == nil || msg.ID != "42" {
+ t.Fatalf("got msg=%v err=%v", msg, err)
+ }
+ if calls != 3 {
+ t.Fatalf("calls=%d want 3", calls)
+ }
+ if len(*slept) != 2 || (*slept)[0] != 2*time.Second || (*slept)[1] != 5*time.Second {
+ t.Fatalf("slept=%v", *slept)
+ }
+}
+
+func TestRetrySendDoesNotRetryAfterSend(t *testing.T) {
+ withNoSleep(t)
+ calls := 0
+ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
+ calls++
+ return nil, urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")})
+ })
+ if err == nil || calls != 1 {
+ t.Fatalf("calls=%d err=%v, want exactly one attempt", calls, err)
+ }
+}
+
+func TestRetrySendGivesUpAfterBudget(t *testing.T) {
+ slept := withNoSleep(t)
+ calls := 0
+ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
+ calls++
+ return nil, urlErr(tlsTimeout{})
+ })
+ if err == nil {
+ t.Fatal("expected error after budget")
+ }
+ if calls != len(lotusSendRetryDelays)+1 || len(*slept) != len(lotusSendRetryDelays) {
+ t.Fatalf("calls=%d slept=%d", calls, len(*slept))
+ }
+}
diff --git a/portal.go b/portal.go
index db26a0e..cc6da0d 100644
--- a/portal.go
+++ b/portal.go
@@ -1743,18 +1743,24 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) {
sendReq.Nonce = generateNonce()
var msg *discordgo.Message
var err error
+ // [lotus] retry sends that failed before reaching Discord (see lotus_retry.go)
+ sendLog := portal.log.With().Str("event_id", evt.ID.String()).Logger()
if !isWebhookSend {
- msg, err = sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...)
+ msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) {
+ return sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...)
+ })
} else {
username, avatarURL := portal.getRelayUserMeta(sender)
- msg, err = relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{
- Content: sendReq.Content,
- Username: username,
- AvatarURL: avatarURL,
- Files: sendReq.Files,
- Components: sendReq.Components,
- Embeds: sendReq.Embeds,
- AllowedMentions: sendReq.AllowedMentions,
+ msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) {
+ return relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{
+ Content: sendReq.Content,
+ Username: username,
+ AvatarURL: avatarURL,
+ Files: sendReq.Files,
+ Components: sendReq.Components,
+ Embeds: sendReq.Embeds,
+ AllowedMentions: sendReq.AllowedMentions,
+ })
})
}
sender.handlePossible40002(err)