diff --git a/README.md b/README.md index 1182976..bd5da41 100644 --- a/README.md +++ b/README.md @@ -412,6 +412,15 @@ channel webhook with their own display name. - Database: `mautrix_discord` on LXC 109, role `mautrix_discord`, allowed from 10.10.10.29 in `pg_hba.conf` - Service: `systemctl status mautrix-discord`, logs in `journalctl -u mautrix-discord` and `/var/log/mautrix-discord/` - End-to-bridge encryption is allowed, so it can join encrypted rooms +- **Patched build.** The deployed binary is upstream v0.7.7 plus `discord-bridge/patches/`, built with + `discord-bridge/build-patched.sh` (deploy steps are in the script's header). `install.sh` still + downloads the stock release, so rebuild and redeploy the patched binary after any (re)install or upgrade. + - `0001`: when a Matrix → Discord send fails before the request reaches Discord (DNS, connect, or + `TLS handshake timeout`), the bridge retries after 2, 5, 10, 20, 30, 60, 60 and 60 s instead of + dropping the message with "⚠ Your message may not have been bridged". The room's later messages wait + in order behind the retry. Failures after the request was sent are not retried, because a webhook + send has no dedupe and a retry could post the message twice. Edits, reactions and deletes are not + retried. **Install (one time):** after this repo's `main` has reached `/opt/matrix-config` on LXC 151, run this on the Proxmox node hosting LXC 109 and 151. It restarts Synapse once and is safe to re-run. diff --git a/discord-bridge/build-patched.sh b/discord-bridge/build-patched.sh new file mode 100644 index 0000000..b0b31f8 --- /dev/null +++ b/discord-bridge/build-patched.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Build mautrix-discord with the Lotus patches in discord-bridge/patches/. +# +# Needs: git, Go (version from upstream go.mod or newer), a C compiler and libolm-dev. +# Output: ./mautrix-discord-lotus (dynamically linked against libolm.so.3, which +# LXC 151 has from the libolm3 package). +# +# Deploy (on the Proxmox node hosting LXC 151): +# pct push 151 mautrix-discord-lotus /opt/mautrix-discord/mautrix-discord.new +# pct exec 151 -- bash -c 'cd /opt/mautrix-discord && cp -a mautrix-discord mautrix-discord.bak-$(date +%Y%m%d) \ +# && chown mautrix-discord: mautrix-discord.new && chmod 755 mautrix-discord.new \ +# && mv mautrix-discord.new mautrix-discord && systemctl restart mautrix-discord' +set -euo pipefail + +VERSION="${VERSION:-v0.7.7}" +HERE="$(cd "$(dirname "$0")" && pwd)" +OUT="${OUT:-$PWD/mautrix-discord-lotus}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +git clone -q --depth 1 --branch "$VERSION" https://github.com/mautrix/discord.git "$WORK/src" +cd "$WORK/src" +for p in "$HERE"/patches/*.patch; do + echo "applying $(basename "$p")" + git apply "$p" +done + +go test -count=1 -run 'Retry|PreSend' . + +COMMIT="$(git rev-parse HEAD)" +MAUTRIX="$(go list -m -f '{{.Version}}' maunium.net/go/mautrix)" +CGO_ENABLED=1 go build -o "$OUT" -ldflags "-s -w \ + -X main.Tag=$VERSION \ + -X main.Commit=$COMMIT+lotus \ + -X 'main.BuildTime=$(date '+%b %_d %Y, %H:%M:%S')' \ + -X 'maunium.net/go/mautrix.GoModVersion=$MAUTRIX'" . + +"$OUT" --version +echo "built $OUT" diff --git a/discord-bridge/patches/0001-retry-discord-sends-that-failed-before-sending.patch b/discord-bridge/patches/0001-retry-discord-sends-that-failed-before-sending.patch new file mode 100644 index 0000000..ed985dd --- /dev/null +++ b/discord-bridge/patches/0001-retry-discord-sends-that-failed-before-sending.patch @@ -0,0 +1,253 @@ +diff --git a/lotus_retry.go b/lotus_retry.go +new file mode 100644 +index 0000000..972b612 +--- /dev/null ++++ b/lotus_retry.go +@@ -0,0 +1,95 @@ ++package main ++ ++// [lotus] Retry Matrix→Discord sends that failed before reaching Discord. ++// ++// Upstream sends each message exactly once; a momentary network blip (for ++// example "net/http: TLS handshake timeout") drops the message and only posts ++// a "may not have been bridged" notice. We retry, but only for failures that ++// happen before the HTTP request is written (DNS, dial, TLS handshake). Those ++// cannot have reached Discord, so a retry can never post a duplicate. ++// Failures after the request was sent (read timeouts, resets, 5xx) are left ++// alone because webhook sends have no server-side dedupe. ++// ++// The retry runs inline in the portal's Matrix event loop, which handles a ++// room's events one at a time, so later messages queue behind it and ordering ++// is preserved. ++ ++import ( ++ "errors" ++ "io" ++ "net" ++ "strings" ++ "time" ++ ++ "github.com/bwmarrin/discordgo" ++ "github.com/rs/zerolog" ++) ++ ++var lotusSendRetryDelays = []time.Duration{ ++ 2 * time.Second, ++ 5 * time.Second, ++ 10 * time.Second, ++ 20 * time.Second, ++ 30 * time.Second, ++ 60 * time.Second, ++ 60 * time.Second, ++ 60 * time.Second, ++} ++ ++// lotusSleep is swapped out in tests. ++var lotusSleep = time.Sleep ++ ++// isPreSendNetworkError reports whether err happened before the request could ++// have been written to Discord. ++func isPreSendNetworkError(err error) bool { ++ if err == nil { ++ return false ++ } ++ var restErr *discordgo.RESTError ++ if errors.As(err, &restErr) { ++ // Discord answered, so the request was delivered. ++ return false ++ } ++ var dnsErr *net.DNSError ++ if errors.As(err, &dnsErr) { ++ return true ++ } ++ var opErr *net.OpError ++ if errors.As(err, &opErr) && opErr.Op == "dial" { ++ return true ++ } ++ msg := err.Error() ++ return strings.Contains(msg, "TLS handshake timeout") || ++ strings.Contains(msg, "proxyconnect") ++} ++ ++func lotusRewindFiles(files []*discordgo.File) { ++ for _, f := range files { ++ if f == nil { ++ continue ++ } ++ if s, ok := f.Reader.(io.Seeker); ok { ++ _, _ = s.Seek(0, io.SeekStart) ++ } ++ } ++} ++ ++// lotusRetrySend calls send until it succeeds, fails with an error that is not ++// a pre-send network error, or the retry budget runs out. ++func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error)) (T, error) { ++ res, err := send() ++ for attempt := 0; attempt < len(lotusSendRetryDelays) && isPreSendNetworkError(err); attempt++ { ++ delay := lotusSendRetryDelays[attempt] ++ log.Warn().Err(err). ++ Int("attempt", attempt+1). ++ Dur("retry_in", delay). ++ Msg("[lotus] Discord send failed before reaching Discord, retrying") ++ lotusSleep(delay) ++ lotusRewindFiles(files) ++ res, err = send() ++ if err == nil { ++ log.Info().Int("attempts", attempt+2).Msg("[lotus] Discord send succeeded after retry") ++ } ++ } ++ return res, err ++} +diff --git a/lotus_retry_test.go b/lotus_retry_test.go +new file mode 100644 +index 0000000..efa262e +--- /dev/null ++++ b/lotus_retry_test.go +@@ -0,0 +1,108 @@ ++package main ++ ++import ( ++ "bytes" ++ "errors" ++ "io" ++ "net" ++ "net/http" ++ "net/url" ++ "testing" ++ "time" ++ ++ "github.com/bwmarrin/discordgo" ++ "github.com/rs/zerolog" ++) ++ ++type tlsTimeout struct{} ++ ++func (tlsTimeout) Error() string { return "net/http: TLS handshake timeout" } ++func (tlsTimeout) Timeout() bool { return true } ++func (tlsTimeout) Temporary() bool { return true } ++ ++func urlErr(err error) error { ++ return &url.Error{Op: "Post", URL: "https://discord.com/api/v9/webhooks/1/x?wait=true", Err: err} ++} ++ ++func TestIsPreSendNetworkError(t *testing.T) { ++ cases := []struct { ++ name string ++ err error ++ want bool ++ }{ ++ {"nil", nil, false}, ++ {"tls handshake timeout", urlErr(tlsTimeout{}), true}, ++ {"dns", urlErr(&net.DNSError{Err: "no such host", Name: "discord.com"}), true}, ++ {"dial refused", urlErr(&net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")}), true}, ++ {"read reset after send", urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}), false}, ++ {"response timeout", urlErr(errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), false}, ++ {"discord rest error", &discordgo.RESTError{Response: &http.Response{StatusCode: 500}}, false}, ++ {"other", errors.New("unknown message type"), false}, ++ } ++ for _, c := range cases { ++ if got := isPreSendNetworkError(c.err); got != c.want { ++ t.Errorf("%s: got %v want %v", c.name, got, c.want) ++ } ++ } ++} ++ ++func withNoSleep(t *testing.T) *[]time.Duration { ++ var slept []time.Duration ++ old := lotusSleep ++ lotusSleep = func(d time.Duration) { slept = append(slept, d) } ++ t.Cleanup(func() { lotusSleep = old }) ++ return &slept ++} ++ ++func TestRetrySendRecoversAndRewindsFiles(t *testing.T) { ++ slept := withNoSleep(t) ++ file := &discordgo.File{Name: "a.txt", Reader: bytes.NewReader([]byte("hello"))} ++ calls := 0 ++ msg, err := lotusRetrySend(zerolog.Nop(), []*discordgo.File{file}, func() (*discordgo.Message, error) { ++ calls++ ++ data, _ := io.ReadAll(file.Reader) ++ if string(data) != "hello" { ++ t.Fatalf("call %d read %q, file not rewound", calls, data) ++ } ++ if calls < 3 { ++ return nil, urlErr(tlsTimeout{}) ++ } ++ return &discordgo.Message{ID: "42"}, nil ++ }) ++ if err != nil || msg == nil || msg.ID != "42" { ++ t.Fatalf("got msg=%v err=%v", msg, err) ++ } ++ if calls != 3 { ++ t.Fatalf("calls=%d want 3", calls) ++ } ++ if len(*slept) != 2 || (*slept)[0] != 2*time.Second || (*slept)[1] != 5*time.Second { ++ t.Fatalf("slept=%v", *slept) ++ } ++} ++ ++func TestRetrySendDoesNotRetryAfterSend(t *testing.T) { ++ withNoSleep(t) ++ calls := 0 ++ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { ++ calls++ ++ return nil, urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}) ++ }) ++ if err == nil || calls != 1 { ++ t.Fatalf("calls=%d err=%v, want exactly one attempt", calls, err) ++ } ++} ++ ++func TestRetrySendGivesUpAfterBudget(t *testing.T) { ++ slept := withNoSleep(t) ++ calls := 0 ++ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) { ++ calls++ ++ return nil, urlErr(tlsTimeout{}) ++ }) ++ if err == nil { ++ t.Fatal("expected error after budget") ++ } ++ if calls != len(lotusSendRetryDelays)+1 || len(*slept) != len(lotusSendRetryDelays) { ++ t.Fatalf("calls=%d slept=%d", calls, len(*slept)) ++ } ++} +diff --git a/portal.go b/portal.go +index db26a0e..cc6da0d 100644 +--- a/portal.go ++++ b/portal.go +@@ -1743,18 +1743,24 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) { + sendReq.Nonce = generateNonce() + var msg *discordgo.Message + var err error ++ // [lotus] retry sends that failed before reaching Discord (see lotus_retry.go) ++ sendLog := portal.log.With().Str("event_id", evt.ID.String()).Logger() + if !isWebhookSend { +- msg, err = sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...) ++ msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) { ++ return sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...) ++ }) + } else { + username, avatarURL := portal.getRelayUserMeta(sender) +- msg, err = relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{ +- Content: sendReq.Content, +- Username: username, +- AvatarURL: avatarURL, +- Files: sendReq.Files, +- Components: sendReq.Components, +- Embeds: sendReq.Embeds, +- AllowedMentions: sendReq.AllowedMentions, ++ msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) { ++ return relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{ ++ Content: sendReq.Content, ++ Username: username, ++ AvatarURL: avatarURL, ++ Files: sendReq.Files, ++ Components: sendReq.Components, ++ Embeds: sendReq.Embeds, ++ AllowedMentions: sendReq.AllowedMentions, ++ }) + }) + } + sender.handlePossible40002(err)