discord-bridge: retry Matrix→Discord sends that failed before reaching Discord #65
@@ -412,6 +412,15 @@ channel webhook with their own display name.
|
|||||||
- Database: `mautrix_discord` on LXC 109, role `mautrix_discord`, allowed from 10.10.10.29 in `pg_hba.conf`
|
- Database: `mautrix_discord` on LXC 109, role `mautrix_discord`, allowed from 10.10.10.29 in `pg_hba.conf`
|
||||||
- Service: `systemctl status mautrix-discord`, logs in `journalctl -u mautrix-discord` and `/var/log/mautrix-discord/`
|
- Service: `systemctl status mautrix-discord`, logs in `journalctl -u mautrix-discord` and `/var/log/mautrix-discord/`
|
||||||
- End-to-bridge encryption is allowed, so it can join encrypted rooms
|
- End-to-bridge encryption is allowed, so it can join encrypted rooms
|
||||||
|
- **Patched build.** The deployed binary is upstream v0.7.7 plus `discord-bridge/patches/`, built with
|
||||||
|
`discord-bridge/build-patched.sh` (deploy steps are in the script's header). `install.sh` still
|
||||||
|
downloads the stock release, so rebuild and redeploy the patched binary after any (re)install or upgrade.
|
||||||
|
- `0001`: when a Matrix → Discord send fails before the request reaches Discord (DNS, connect, or
|
||||||
|
`TLS handshake timeout`), the bridge retries after 2, 5, 10, 20, 30, 60, 60 and 60 s instead of
|
||||||
|
dropping the message with "⚠ Your message may not have been bridged". The room's later messages wait
|
||||||
|
in order behind the retry. Failures after the request was sent are not retried, because a webhook
|
||||||
|
send has no dedupe and a retry could post the message twice. Edits, reactions and deletes are not
|
||||||
|
retried.
|
||||||
|
|
||||||
**Install (one time):** after this repo's `main` has reached `/opt/matrix-config` on LXC 151, run this
|
**Install (one time):** after this repo's `main` has reached `/opt/matrix-config` on LXC 151, run this
|
||||||
on the Proxmox node hosting LXC 109 and 151. It restarts Synapse once and is safe to re-run.
|
on the Proxmox node hosting LXC 109 and 151. It restarts Synapse once and is safe to re-run.
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build mautrix-discord with the Lotus patches in discord-bridge/patches/.
|
||||||
|
#
|
||||||
|
# Needs: git, Go (version from upstream go.mod or newer), a C compiler and libolm-dev.
|
||||||
|
# Output: ./mautrix-discord-lotus (dynamically linked against libolm.so.3, which
|
||||||
|
# LXC 151 has from the libolm3 package).
|
||||||
|
#
|
||||||
|
# Deploy (on the Proxmox node hosting LXC 151):
|
||||||
|
# pct push 151 mautrix-discord-lotus /opt/mautrix-discord/mautrix-discord.new
|
||||||
|
# pct exec 151 -- bash -c 'cd /opt/mautrix-discord && cp -a mautrix-discord mautrix-discord.bak-$(date +%Y%m%d) \
|
||||||
|
# && chown mautrix-discord: mautrix-discord.new && chmod 755 mautrix-discord.new \
|
||||||
|
# && mv mautrix-discord.new mautrix-discord && systemctl restart mautrix-discord'
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
VERSION="${VERSION:-v0.7.7}"
|
||||||
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
OUT="${OUT:-$PWD/mautrix-discord-lotus}"
|
||||||
|
WORK="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$WORK"' EXIT
|
||||||
|
|
||||||
|
git clone -q --depth 1 --branch "$VERSION" https://github.com/mautrix/discord.git "$WORK/src"
|
||||||
|
cd "$WORK/src"
|
||||||
|
for p in "$HERE"/patches/*.patch; do
|
||||||
|
echo "applying $(basename "$p")"
|
||||||
|
git apply "$p"
|
||||||
|
done
|
||||||
|
|
||||||
|
go test -count=1 -run 'Retry|PreSend' .
|
||||||
|
|
||||||
|
COMMIT="$(git rev-parse HEAD)"
|
||||||
|
MAUTRIX="$(go list -m -f '{{.Version}}' maunium.net/go/mautrix)"
|
||||||
|
CGO_ENABLED=1 go build -o "$OUT" -ldflags "-s -w \
|
||||||
|
-X main.Tag=$VERSION \
|
||||||
|
-X main.Commit=$COMMIT+lotus \
|
||||||
|
-X 'main.BuildTime=$(date '+%b %_d %Y, %H:%M:%S')' \
|
||||||
|
-X 'maunium.net/go/mautrix.GoModVersion=$MAUTRIX'" .
|
||||||
|
|
||||||
|
"$OUT" --version
|
||||||
|
echo "built $OUT"
|
||||||
@@ -0,0 +1,253 @@
|
|||||||
|
diff --git a/lotus_retry.go b/lotus_retry.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..972b612
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/lotus_retry.go
|
||||||
|
@@ -0,0 +1,95 @@
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+// [lotus] Retry Matrix→Discord sends that failed before reaching Discord.
|
||||||
|
+//
|
||||||
|
+// Upstream sends each message exactly once; a momentary network blip (for
|
||||||
|
+// example "net/http: TLS handshake timeout") drops the message and only posts
|
||||||
|
+// a "may not have been bridged" notice. We retry, but only for failures that
|
||||||
|
+// happen before the HTTP request is written (DNS, dial, TLS handshake). Those
|
||||||
|
+// cannot have reached Discord, so a retry can never post a duplicate.
|
||||||
|
+// Failures after the request was sent (read timeouts, resets, 5xx) are left
|
||||||
|
+// alone because webhook sends have no server-side dedupe.
|
||||||
|
+//
|
||||||
|
+// The retry runs inline in the portal's Matrix event loop, which handles a
|
||||||
|
+// room's events one at a time, so later messages queue behind it and ordering
|
||||||
|
+// is preserved.
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "errors"
|
||||||
|
+ "io"
|
||||||
|
+ "net"
|
||||||
|
+ "strings"
|
||||||
|
+ "time"
|
||||||
|
+
|
||||||
|
+ "github.com/bwmarrin/discordgo"
|
||||||
|
+ "github.com/rs/zerolog"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+var lotusSendRetryDelays = []time.Duration{
|
||||||
|
+ 2 * time.Second,
|
||||||
|
+ 5 * time.Second,
|
||||||
|
+ 10 * time.Second,
|
||||||
|
+ 20 * time.Second,
|
||||||
|
+ 30 * time.Second,
|
||||||
|
+ 60 * time.Second,
|
||||||
|
+ 60 * time.Second,
|
||||||
|
+ 60 * time.Second,
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// lotusSleep is swapped out in tests.
|
||||||
|
+var lotusSleep = time.Sleep
|
||||||
|
+
|
||||||
|
+// isPreSendNetworkError reports whether err happened before the request could
|
||||||
|
+// have been written to Discord.
|
||||||
|
+func isPreSendNetworkError(err error) bool {
|
||||||
|
+ if err == nil {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ var restErr *discordgo.RESTError
|
||||||
|
+ if errors.As(err, &restErr) {
|
||||||
|
+ // Discord answered, so the request was delivered.
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ var dnsErr *net.DNSError
|
||||||
|
+ if errors.As(err, &dnsErr) {
|
||||||
|
+ return true
|
||||||
|
+ }
|
||||||
|
+ var opErr *net.OpError
|
||||||
|
+ if errors.As(err, &opErr) && opErr.Op == "dial" {
|
||||||
|
+ return true
|
||||||
|
+ }
|
||||||
|
+ msg := err.Error()
|
||||||
|
+ return strings.Contains(msg, "TLS handshake timeout") ||
|
||||||
|
+ strings.Contains(msg, "proxyconnect")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func lotusRewindFiles(files []*discordgo.File) {
|
||||||
|
+ for _, f := range files {
|
||||||
|
+ if f == nil {
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ if s, ok := f.Reader.(io.Seeker); ok {
|
||||||
|
+ _, _ = s.Seek(0, io.SeekStart)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// lotusRetrySend calls send until it succeeds, fails with an error that is not
|
||||||
|
+// a pre-send network error, or the retry budget runs out.
|
||||||
|
+func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error)) (T, error) {
|
||||||
|
+ res, err := send()
|
||||||
|
+ for attempt := 0; attempt < len(lotusSendRetryDelays) && isPreSendNetworkError(err); attempt++ {
|
||||||
|
+ delay := lotusSendRetryDelays[attempt]
|
||||||
|
+ log.Warn().Err(err).
|
||||||
|
+ Int("attempt", attempt+1).
|
||||||
|
+ Dur("retry_in", delay).
|
||||||
|
+ Msg("[lotus] Discord send failed before reaching Discord, retrying")
|
||||||
|
+ lotusSleep(delay)
|
||||||
|
+ lotusRewindFiles(files)
|
||||||
|
+ res, err = send()
|
||||||
|
+ if err == nil {
|
||||||
|
+ log.Info().Int("attempts", attempt+2).Msg("[lotus] Discord send succeeded after retry")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return res, err
|
||||||
|
+}
|
||||||
|
diff --git a/lotus_retry_test.go b/lotus_retry_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..efa262e
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/lotus_retry_test.go
|
||||||
|
@@ -0,0 +1,108 @@
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "bytes"
|
||||||
|
+ "errors"
|
||||||
|
+ "io"
|
||||||
|
+ "net"
|
||||||
|
+ "net/http"
|
||||||
|
+ "net/url"
|
||||||
|
+ "testing"
|
||||||
|
+ "time"
|
||||||
|
+
|
||||||
|
+ "github.com/bwmarrin/discordgo"
|
||||||
|
+ "github.com/rs/zerolog"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+type tlsTimeout struct{}
|
||||||
|
+
|
||||||
|
+func (tlsTimeout) Error() string { return "net/http: TLS handshake timeout" }
|
||||||
|
+func (tlsTimeout) Timeout() bool { return true }
|
||||||
|
+func (tlsTimeout) Temporary() bool { return true }
|
||||||
|
+
|
||||||
|
+func urlErr(err error) error {
|
||||||
|
+ return &url.Error{Op: "Post", URL: "https://discord.com/api/v9/webhooks/1/x?wait=true", Err: err}
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestIsPreSendNetworkError(t *testing.T) {
|
||||||
|
+ cases := []struct {
|
||||||
|
+ name string
|
||||||
|
+ err error
|
||||||
|
+ want bool
|
||||||
|
+ }{
|
||||||
|
+ {"nil", nil, false},
|
||||||
|
+ {"tls handshake timeout", urlErr(tlsTimeout{}), true},
|
||||||
|
+ {"dns", urlErr(&net.DNSError{Err: "no such host", Name: "discord.com"}), true},
|
||||||
|
+ {"dial refused", urlErr(&net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")}), true},
|
||||||
|
+ {"read reset after send", urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}), false},
|
||||||
|
+ {"response timeout", urlErr(errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), false},
|
||||||
|
+ {"discord rest error", &discordgo.RESTError{Response: &http.Response{StatusCode: 500}}, false},
|
||||||
|
+ {"other", errors.New("unknown message type"), false},
|
||||||
|
+ }
|
||||||
|
+ for _, c := range cases {
|
||||||
|
+ if got := isPreSendNetworkError(c.err); got != c.want {
|
||||||
|
+ t.Errorf("%s: got %v want %v", c.name, got, c.want)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func withNoSleep(t *testing.T) *[]time.Duration {
|
||||||
|
+ var slept []time.Duration
|
||||||
|
+ old := lotusSleep
|
||||||
|
+ lotusSleep = func(d time.Duration) { slept = append(slept, d) }
|
||||||
|
+ t.Cleanup(func() { lotusSleep = old })
|
||||||
|
+ return &slept
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRetrySendRecoversAndRewindsFiles(t *testing.T) {
|
||||||
|
+ slept := withNoSleep(t)
|
||||||
|
+ file := &discordgo.File{Name: "a.txt", Reader: bytes.NewReader([]byte("hello"))}
|
||||||
|
+ calls := 0
|
||||||
|
+ msg, err := lotusRetrySend(zerolog.Nop(), []*discordgo.File{file}, func() (*discordgo.Message, error) {
|
||||||
|
+ calls++
|
||||||
|
+ data, _ := io.ReadAll(file.Reader)
|
||||||
|
+ if string(data) != "hello" {
|
||||||
|
+ t.Fatalf("call %d read %q, file not rewound", calls, data)
|
||||||
|
+ }
|
||||||
|
+ if calls < 3 {
|
||||||
|
+ return nil, urlErr(tlsTimeout{})
|
||||||
|
+ }
|
||||||
|
+ return &discordgo.Message{ID: "42"}, nil
|
||||||
|
+ })
|
||||||
|
+ if err != nil || msg == nil || msg.ID != "42" {
|
||||||
|
+ t.Fatalf("got msg=%v err=%v", msg, err)
|
||||||
|
+ }
|
||||||
|
+ if calls != 3 {
|
||||||
|
+ t.Fatalf("calls=%d want 3", calls)
|
||||||
|
+ }
|
||||||
|
+ if len(*slept) != 2 || (*slept)[0] != 2*time.Second || (*slept)[1] != 5*time.Second {
|
||||||
|
+ t.Fatalf("slept=%v", *slept)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRetrySendDoesNotRetryAfterSend(t *testing.T) {
|
||||||
|
+ withNoSleep(t)
|
||||||
|
+ calls := 0
|
||||||
|
+ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
|
||||||
|
+ calls++
|
||||||
|
+ return nil, urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")})
|
||||||
|
+ })
|
||||||
|
+ if err == nil || calls != 1 {
|
||||||
|
+ t.Fatalf("calls=%d err=%v, want exactly one attempt", calls, err)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRetrySendGivesUpAfterBudget(t *testing.T) {
|
||||||
|
+ slept := withNoSleep(t)
|
||||||
|
+ calls := 0
|
||||||
|
+ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
|
||||||
|
+ calls++
|
||||||
|
+ return nil, urlErr(tlsTimeout{})
|
||||||
|
+ })
|
||||||
|
+ if err == nil {
|
||||||
|
+ t.Fatal("expected error after budget")
|
||||||
|
+ }
|
||||||
|
+ if calls != len(lotusSendRetryDelays)+1 || len(*slept) != len(lotusSendRetryDelays) {
|
||||||
|
+ t.Fatalf("calls=%d slept=%d", calls, len(*slept))
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/portal.go b/portal.go
|
||||||
|
index db26a0e..cc6da0d 100644
|
||||||
|
--- a/portal.go
|
||||||
|
+++ b/portal.go
|
||||||
|
@@ -1743,18 +1743,24 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) {
|
||||||
|
sendReq.Nonce = generateNonce()
|
||||||
|
var msg *discordgo.Message
|
||||||
|
var err error
|
||||||
|
+ // [lotus] retry sends that failed before reaching Discord (see lotus_retry.go)
|
||||||
|
+ sendLog := portal.log.With().Str("event_id", evt.ID.String()).Logger()
|
||||||
|
if !isWebhookSend {
|
||||||
|
- msg, err = sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...)
|
||||||
|
+ msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) {
|
||||||
|
+ return sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...)
|
||||||
|
+ })
|
||||||
|
} else {
|
||||||
|
username, avatarURL := portal.getRelayUserMeta(sender)
|
||||||
|
- msg, err = relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{
|
||||||
|
- Content: sendReq.Content,
|
||||||
|
- Username: username,
|
||||||
|
- AvatarURL: avatarURL,
|
||||||
|
- Files: sendReq.Files,
|
||||||
|
- Components: sendReq.Components,
|
||||||
|
- Embeds: sendReq.Embeds,
|
||||||
|
- AllowedMentions: sendReq.AllowedMentions,
|
||||||
|
+ msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) {
|
||||||
|
+ return relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{
|
||||||
|
+ Content: sendReq.Content,
|
||||||
|
+ Username: username,
|
||||||
|
+ AvatarURL: avatarURL,
|
||||||
|
+ Files: sendReq.Files,
|
||||||
|
+ Components: sendReq.Components,
|
||||||
|
+ Embeds: sendReq.Embeds,
|
||||||
|
+ AllowedMentions: sendReq.AllowedMentions,
|
||||||
|
+ })
|
||||||
|
})
|
||||||
|
}
|
||||||
|
sender.handlePossible40002(err)
|
||||||
Reference in New Issue
Block a user