|
|
|
@@ -0,0 +1,253 @@
|
|
|
|
|
diff --git a/lotus_retry.go b/lotus_retry.go
|
|
|
|
|
new file mode 100644
|
|
|
|
|
index 0000000..972b612
|
|
|
|
|
--- /dev/null
|
|
|
|
|
+++ b/lotus_retry.go
|
|
|
|
|
@@ -0,0 +1,95 @@
|
|
|
|
|
+package main
|
|
|
|
|
+
|
|
|
|
|
+// [lotus] Retry Matrix→Discord sends that failed before reaching Discord.
|
|
|
|
|
+//
|
|
|
|
|
+// Upstream sends each message exactly once; a momentary network blip (for
|
|
|
|
|
+// example "net/http: TLS handshake timeout") drops the message and only posts
|
|
|
|
|
+// a "may not have been bridged" notice. We retry, but only for failures that
|
|
|
|
|
+// happen before the HTTP request is written (DNS, dial, TLS handshake). Those
|
|
|
|
|
+// cannot have reached Discord, so a retry can never post a duplicate.
|
|
|
|
|
+// Failures after the request was sent (read timeouts, resets, 5xx) are left
|
|
|
|
|
+// alone because webhook sends have no server-side dedupe.
|
|
|
|
|
+//
|
|
|
|
|
+// The retry runs inline in the portal's Matrix event loop, which handles a
|
|
|
|
|
+// room's events one at a time, so later messages queue behind it and ordering
|
|
|
|
|
+// is preserved.
|
|
|
|
|
+
|
|
|
|
|
+import (
|
|
|
|
|
+ "errors"
|
|
|
|
|
+ "io"
|
|
|
|
|
+ "net"
|
|
|
|
|
+ "strings"
|
|
|
|
|
+ "time"
|
|
|
|
|
+
|
|
|
|
|
+ "github.com/bwmarrin/discordgo"
|
|
|
|
|
+ "github.com/rs/zerolog"
|
|
|
|
|
+)
|
|
|
|
|
+
|
|
|
|
|
+var lotusSendRetryDelays = []time.Duration{
|
|
|
|
|
+ 2 * time.Second,
|
|
|
|
|
+ 5 * time.Second,
|
|
|
|
|
+ 10 * time.Second,
|
|
|
|
|
+ 20 * time.Second,
|
|
|
|
|
+ 30 * time.Second,
|
|
|
|
|
+ 60 * time.Second,
|
|
|
|
|
+ 60 * time.Second,
|
|
|
|
|
+ 60 * time.Second,
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+// lotusSleep is swapped out in tests.
|
|
|
|
|
+var lotusSleep = time.Sleep
|
|
|
|
|
+
|
|
|
|
|
+// isPreSendNetworkError reports whether err happened before the request could
|
|
|
|
|
+// have been written to Discord.
|
|
|
|
|
+func isPreSendNetworkError(err error) bool {
|
|
|
|
|
+ if err == nil {
|
|
|
|
|
+ return false
|
|
|
|
|
+ }
|
|
|
|
|
+ var restErr *discordgo.RESTError
|
|
|
|
|
+ if errors.As(err, &restErr) {
|
|
|
|
|
+ // Discord answered, so the request was delivered.
|
|
|
|
|
+ return false
|
|
|
|
|
+ }
|
|
|
|
|
+ var dnsErr *net.DNSError
|
|
|
|
|
+ if errors.As(err, &dnsErr) {
|
|
|
|
|
+ return true
|
|
|
|
|
+ }
|
|
|
|
|
+ var opErr *net.OpError
|
|
|
|
|
+ if errors.As(err, &opErr) && opErr.Op == "dial" {
|
|
|
|
|
+ return true
|
|
|
|
|
+ }
|
|
|
|
|
+ msg := err.Error()
|
|
|
|
|
+ return strings.Contains(msg, "TLS handshake timeout") ||
|
|
|
|
|
+ strings.Contains(msg, "proxyconnect")
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func lotusRewindFiles(files []*discordgo.File) {
|
|
|
|
|
+ for _, f := range files {
|
|
|
|
|
+ if f == nil {
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ if s, ok := f.Reader.(io.Seeker); ok {
|
|
|
|
|
+ _, _ = s.Seek(0, io.SeekStart)
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+// lotusRetrySend calls send until it succeeds, fails with an error that is not
|
|
|
|
|
+// a pre-send network error, or the retry budget runs out.
|
|
|
|
|
+func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error)) (T, error) {
|
|
|
|
|
+ res, err := send()
|
|
|
|
|
+ for attempt := 0; attempt < len(lotusSendRetryDelays) && isPreSendNetworkError(err); attempt++ {
|
|
|
|
|
+ delay := lotusSendRetryDelays[attempt]
|
|
|
|
|
+ log.Warn().Err(err).
|
|
|
|
|
+ Int("attempt", attempt+1).
|
|
|
|
|
+ Dur("retry_in", delay).
|
|
|
|
|
+ Msg("[lotus] Discord send failed before reaching Discord, retrying")
|
|
|
|
|
+ lotusSleep(delay)
|
|
|
|
|
+ lotusRewindFiles(files)
|
|
|
|
|
+ res, err = send()
|
|
|
|
|
+ if err == nil {
|
|
|
|
|
+ log.Info().Int("attempts", attempt+2).Msg("[lotus] Discord send succeeded after retry")
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ return res, err
|
|
|
|
|
+}
|
|
|
|
|
diff --git a/lotus_retry_test.go b/lotus_retry_test.go
|
|
|
|
|
new file mode 100644
|
|
|
|
|
index 0000000..efa262e
|
|
|
|
|
--- /dev/null
|
|
|
|
|
+++ b/lotus_retry_test.go
|
|
|
|
|
@@ -0,0 +1,108 @@
|
|
|
|
|
+package main
|
|
|
|
|
+
|
|
|
|
|
+import (
|
|
|
|
|
+ "bytes"
|
|
|
|
|
+ "errors"
|
|
|
|
|
+ "io"
|
|
|
|
|
+ "net"
|
|
|
|
|
+ "net/http"
|
|
|
|
|
+ "net/url"
|
|
|
|
|
+ "testing"
|
|
|
|
|
+ "time"
|
|
|
|
|
+
|
|
|
|
|
+ "github.com/bwmarrin/discordgo"
|
|
|
|
|
+ "github.com/rs/zerolog"
|
|
|
|
|
+)
|
|
|
|
|
+
|
|
|
|
|
+type tlsTimeout struct{}
|
|
|
|
|
+
|
|
|
|
|
+func (tlsTimeout) Error() string { return "net/http: TLS handshake timeout" }
|
|
|
|
|
+func (tlsTimeout) Timeout() bool { return true }
|
|
|
|
|
+func (tlsTimeout) Temporary() bool { return true }
|
|
|
|
|
+
|
|
|
|
|
+func urlErr(err error) error {
|
|
|
|
|
+ return &url.Error{Op: "Post", URL: "https://discord.com/api/v9/webhooks/1/x?wait=true", Err: err}
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func TestIsPreSendNetworkError(t *testing.T) {
|
|
|
|
|
+ cases := []struct {
|
|
|
|
|
+ name string
|
|
|
|
|
+ err error
|
|
|
|
|
+ want bool
|
|
|
|
|
+ }{
|
|
|
|
|
+ {"nil", nil, false},
|
|
|
|
|
+ {"tls handshake timeout", urlErr(tlsTimeout{}), true},
|
|
|
|
|
+ {"dns", urlErr(&net.DNSError{Err: "no such host", Name: "discord.com"}), true},
|
|
|
|
|
+ {"dial refused", urlErr(&net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")}), true},
|
|
|
|
|
+ {"read reset after send", urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")}), false},
|
|
|
|
|
+ {"response timeout", urlErr(errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), false},
|
|
|
|
|
+ {"discord rest error", &discordgo.RESTError{Response: &http.Response{StatusCode: 500}}, false},
|
|
|
|
|
+ {"other", errors.New("unknown message type"), false},
|
|
|
|
|
+ }
|
|
|
|
|
+ for _, c := range cases {
|
|
|
|
|
+ if got := isPreSendNetworkError(c.err); got != c.want {
|
|
|
|
|
+ t.Errorf("%s: got %v want %v", c.name, got, c.want)
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func withNoSleep(t *testing.T) *[]time.Duration {
|
|
|
|
|
+ var slept []time.Duration
|
|
|
|
|
+ old := lotusSleep
|
|
|
|
|
+ lotusSleep = func(d time.Duration) { slept = append(slept, d) }
|
|
|
|
|
+ t.Cleanup(func() { lotusSleep = old })
|
|
|
|
|
+ return &slept
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func TestRetrySendRecoversAndRewindsFiles(t *testing.T) {
|
|
|
|
|
+ slept := withNoSleep(t)
|
|
|
|
|
+ file := &discordgo.File{Name: "a.txt", Reader: bytes.NewReader([]byte("hello"))}
|
|
|
|
|
+ calls := 0
|
|
|
|
|
+ msg, err := lotusRetrySend(zerolog.Nop(), []*discordgo.File{file}, func() (*discordgo.Message, error) {
|
|
|
|
|
+ calls++
|
|
|
|
|
+ data, _ := io.ReadAll(file.Reader)
|
|
|
|
|
+ if string(data) != "hello" {
|
|
|
|
|
+ t.Fatalf("call %d read %q, file not rewound", calls, data)
|
|
|
|
|
+ }
|
|
|
|
|
+ if calls < 3 {
|
|
|
|
|
+ return nil, urlErr(tlsTimeout{})
|
|
|
|
|
+ }
|
|
|
|
|
+ return &discordgo.Message{ID: "42"}, nil
|
|
|
|
|
+ })
|
|
|
|
|
+ if err != nil || msg == nil || msg.ID != "42" {
|
|
|
|
|
+ t.Fatalf("got msg=%v err=%v", msg, err)
|
|
|
|
|
+ }
|
|
|
|
|
+ if calls != 3 {
|
|
|
|
|
+ t.Fatalf("calls=%d want 3", calls)
|
|
|
|
|
+ }
|
|
|
|
|
+ if len(*slept) != 2 || (*slept)[0] != 2*time.Second || (*slept)[1] != 5*time.Second {
|
|
|
|
|
+ t.Fatalf("slept=%v", *slept)
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func TestRetrySendDoesNotRetryAfterSend(t *testing.T) {
|
|
|
|
|
+ withNoSleep(t)
|
|
|
|
|
+ calls := 0
|
|
|
|
|
+ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
|
|
|
|
|
+ calls++
|
|
|
|
|
+ return nil, urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")})
|
|
|
|
|
+ })
|
|
|
|
|
+ if err == nil || calls != 1 {
|
|
|
|
|
+ t.Fatalf("calls=%d err=%v, want exactly one attempt", calls, err)
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func TestRetrySendGivesUpAfterBudget(t *testing.T) {
|
|
|
|
|
+ slept := withNoSleep(t)
|
|
|
|
|
+ calls := 0
|
|
|
|
|
+ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
|
|
|
|
|
+ calls++
|
|
|
|
|
+ return nil, urlErr(tlsTimeout{})
|
|
|
|
|
+ })
|
|
|
|
|
+ if err == nil {
|
|
|
|
|
+ t.Fatal("expected error after budget")
|
|
|
|
|
+ }
|
|
|
|
|
+ if calls != len(lotusSendRetryDelays)+1 || len(*slept) != len(lotusSendRetryDelays) {
|
|
|
|
|
+ t.Fatalf("calls=%d slept=%d", calls, len(*slept))
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
diff --git a/portal.go b/portal.go
|
|
|
|
|
index db26a0e..cc6da0d 100644
|
|
|
|
|
--- a/portal.go
|
|
|
|
|
+++ b/portal.go
|
|
|
|
|
@@ -1743,18 +1743,24 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) {
|
|
|
|
|
sendReq.Nonce = generateNonce()
|
|
|
|
|
var msg *discordgo.Message
|
|
|
|
|
var err error
|
|
|
|
|
+ // [lotus] retry sends that failed before reaching Discord (see lotus_retry.go)
|
|
|
|
|
+ sendLog := portal.log.With().Str("event_id", evt.ID.String()).Logger()
|
|
|
|
|
if !isWebhookSend {
|
|
|
|
|
- msg, err = sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...)
|
|
|
|
|
+ msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) {
|
|
|
|
|
+ return sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...)
|
|
|
|
|
+ })
|
|
|
|
|
} else {
|
|
|
|
|
username, avatarURL := portal.getRelayUserMeta(sender)
|
|
|
|
|
- msg, err = relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{
|
|
|
|
|
- Content: sendReq.Content,
|
|
|
|
|
- Username: username,
|
|
|
|
|
- AvatarURL: avatarURL,
|
|
|
|
|
- Files: sendReq.Files,
|
|
|
|
|
- Components: sendReq.Components,
|
|
|
|
|
- Embeds: sendReq.Embeds,
|
|
|
|
|
- AllowedMentions: sendReq.AllowedMentions,
|
|
|
|
|
+ msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) {
|
|
|
|
|
+ return relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{
|
|
|
|
|
+ Content: sendReq.Content,
|
|
|
|
|
+ Username: username,
|
|
|
|
|
+ AvatarURL: avatarURL,
|
|
|
|
|
+ Files: sendReq.Files,
|
|
|
|
|
+ Components: sendReq.Components,
|
|
|
|
|
+ Embeds: sendReq.Embeds,
|
|
|
|
|
+ AllowedMentions: sendReq.AllowedMentions,
|
|
|
|
|
+ })
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
sender.handlePossible40002(err)
|