discord-bridge: check the gateway echo before retrying ambiguous webhook sends (#67)
Lint / JS (eslint) (push) Successful in 6s
Lint / Shell (shellcheck) (push) Successful in 8s
Lint / No secrets in webhook configs (push) Successful in 5s
Lint / No Hookshot webhook IDs in README (push) Successful in 4s
Lint / Python (ruff) (push) Successful in 4s
Lint / Landing page is rendered (matrix (push) Successful in 5s
Lint / Secret scan (gitleaks) (push) Successful in 6s
Lint / Python deps (pip-audit) (push) Successful in 36s
Lint / JS (eslint) (push) Successful in 6s
Lint / Shell (shellcheck) (push) Successful in 8s
Lint / No secrets in webhook configs (push) Successful in 5s
Lint / No Hookshot webhook IDs in README (push) Successful in 4s
Lint / Python (ruff) (push) Successful in 4s
Lint / Landing page is rendered (matrix (push) Successful in 5s
Lint / Secret scan (gitleaks) (push) Successful in 6s
Lint / Python deps (pip-audit) (push) Successful in 36s
This commit was merged in pull request #67.
This commit is contained in:
@@ -418,9 +418,13 @@ channel webhook with their own display name.
|
|||||||
- `0001`: when a Matrix → Discord send fails before the request reaches Discord (DNS, connect, or
|
- `0001`: when a Matrix → Discord send fails before the request reaches Discord (DNS, connect, or
|
||||||
`TLS handshake timeout`), the bridge retries after 2, 5, 10, 20, 30, 60, 60 and 60 s instead of
|
`TLS handshake timeout`), the bridge retries after 2, 5, 10, 20, 30, 60, 60 and 60 s instead of
|
||||||
dropping the message with "⚠ Your message may not have been bridged". The room's later messages wait
|
dropping the message with "⚠ Your message may not have been bridged". The room's later messages wait
|
||||||
in order behind the retry. Failures after the request was sent are not retried, because a webhook
|
in order behind the retry. Edits, reactions and deletes are not retried.
|
||||||
send has no dedupe and a retry could post the message twice. Edits, reactions and deletes are not
|
- `0002`: when a relay (webhook) send fails after the request may already have reached Discord
|
||||||
retried.
|
(connection reset, EOF, timeout waiting for the reply, Discord 5xx), the bridge waits up to 6 s for
|
||||||
|
its own post to come back over the Discord gateway. If it does, the send counts as delivered (no
|
||||||
|
resend, no copy back into Matrix); if not, it retries with the 0001 backoff, checking again before
|
||||||
|
each resend. A webhook post has no dedupe, so this echo check is what keeps those retries from
|
||||||
|
double-posting.
|
||||||
|
|
||||||
**Install (one time):** after this repo's `main` has reached `/opt/matrix-config` on LXC 151, run this
|
**Install (one time):** after this repo's `main` has reached `/opt/matrix-config` on LXC 151, run this
|
||||||
on the Proxmox node hosting LXC 109 and 151. It restarts Synapse once and is safe to re-run.
|
on the Proxmox node hosting LXC 109 and 151. It restarts Synapse once and is safe to re-run.
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ for p in "$HERE"/patches/*.patch; do
|
|||||||
git apply "$p"
|
git apply "$p"
|
||||||
done
|
done
|
||||||
|
|
||||||
go test -count=1 -run 'Retry|PreSend' .
|
go test -race -count=1 -run 'Retry|PreSend|Ambiguous|Echo' .
|
||||||
|
|
||||||
COMMIT="$(git rev-parse HEAD)"
|
COMMIT="$(git rev-parse HEAD)"
|
||||||
MAUTRIX="$(go list -m -f '{{.Version}}' maunium.net/go/mautrix)"
|
MAUTRIX="$(go list -m -f '{{.Version}}' maunium.net/go/mautrix)"
|
||||||
|
|||||||
+437
@@ -0,0 +1,437 @@
|
|||||||
|
diff --git a/lotus_echo.go b/lotus_echo.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..0cfdd95
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/lotus_echo.go
|
||||||
|
@@ -0,0 +1,118 @@
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+// [lotus] Detect whether a webhook send that errored actually reached Discord.
|
||||||
|
+//
|
||||||
|
+// Discord echoes every webhook post back over the gateway as MESSAGE_CREATE.
|
||||||
|
+// While a relay send is in flight we register a waiter for its channel; the
|
||||||
|
+// gateway handler (user.go) offers every new message to the waiters before
|
||||||
|
+// queueing it for the portal. If a send fails ambiguously (the connection broke
|
||||||
|
+// after the request may have gone out), lotusRetrySend asks the waiter whether
|
||||||
|
+// the post showed up. If it did, the send is treated as successful and the
|
||||||
|
+// message ID recorded, so it is neither resent to Discord nor bridged back into
|
||||||
|
+// Matrix as a new message.
|
||||||
|
+//
|
||||||
|
+// The waiter is offered messages from the gateway goroutine, which keeps
|
||||||
|
+// running while the portal's own loop is blocked in the send.
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "strings"
|
||||||
|
+ "sync"
|
||||||
|
+ "time"
|
||||||
|
+
|
||||||
|
+ "github.com/bwmarrin/discordgo"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// lotusEchoWait is how long to wait for the gateway echo after an ambiguous
|
||||||
|
+// failure. Echoes normally arrive well under a second after Discord accepts the
|
||||||
|
+// post.
|
||||||
|
+var lotusEchoWait = 6 * time.Second
|
||||||
|
+
|
||||||
|
+type lotusEchoWaiter struct {
|
||||||
|
+ match func(*discordgo.Message) bool
|
||||||
|
+ ch chan *discordgo.Message
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+var (
|
||||||
|
+ lotusEchoMu sync.Mutex
|
||||||
|
+ lotusEchoWaiters = map[string][]*lotusEchoWaiter{}
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// lotusNoteGatewayMessage offers a newly created Discord message to any waiter
|
||||||
|
+// registered for its channel. Called for every MESSAGE_CREATE.
|
||||||
|
+func lotusNoteGatewayMessage(msg *discordgo.Message) {
|
||||||
|
+ if msg == nil || msg.WebhookID == "" {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ lotusEchoMu.Lock()
|
||||||
|
+ defer lotusEchoMu.Unlock()
|
||||||
|
+ for _, w := range lotusEchoWaiters[msg.ChannelID] {
|
||||||
|
+ if w.match(msg) {
|
||||||
|
+ select {
|
||||||
|
+ case w.ch <- msg:
|
||||||
|
+ default:
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// lotusExpectEcho registers a waiter for channelID. check(wait) returns the
|
||||||
|
+// matching message if one has arrived, waiting up to lotusEchoWait when wait
|
||||||
|
+// is true. done must be called when the send is finished.
|
||||||
|
+func lotusExpectEcho(channelID string, match func(*discordgo.Message) bool) (check func(wait bool) (*discordgo.Message, bool), done func()) {
|
||||||
|
+ w := &lotusEchoWaiter{match: match, ch: make(chan *discordgo.Message, 1)}
|
||||||
|
+ lotusEchoMu.Lock()
|
||||||
|
+ lotusEchoWaiters[channelID] = append(lotusEchoWaiters[channelID], w)
|
||||||
|
+ lotusEchoMu.Unlock()
|
||||||
|
+
|
||||||
|
+ check = func(wait bool) (*discordgo.Message, bool) {
|
||||||
|
+ if !wait {
|
||||||
|
+ select {
|
||||||
|
+ case msg := <-w.ch:
|
||||||
|
+ return msg, true
|
||||||
|
+ default:
|
||||||
|
+ return nil, false
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ timer := time.NewTimer(lotusEchoWait)
|
||||||
|
+ defer timer.Stop()
|
||||||
|
+ select {
|
||||||
|
+ case msg := <-w.ch:
|
||||||
|
+ return msg, true
|
||||||
|
+ case <-timer.C:
|
||||||
|
+ return nil, false
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ done = func() {
|
||||||
|
+ lotusEchoMu.Lock()
|
||||||
|
+ defer lotusEchoMu.Unlock()
|
||||||
|
+ list := lotusEchoWaiters[channelID]
|
||||||
|
+ for i, x := range list {
|
||||||
|
+ if x == w {
|
||||||
|
+ list = append(list[:i], list[i+1:]...)
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ if len(list) == 0 {
|
||||||
|
+ delete(lotusEchoWaiters, channelID)
|
||||||
|
+ } else {
|
||||||
|
+ lotusEchoWaiters[channelID] = list
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return check, done
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// lotusWebhookEchoMatcher matches the gateway echo of a relay post: same
|
||||||
|
+// webhook, same text (ignoring surrounding whitespace, which Discord trims),
|
||||||
|
+// same number of attachments, created no earlier than shortly before the send
|
||||||
|
+// started.
|
||||||
|
+func lotusWebhookEchoMatcher(webhookID, content string, numFiles int, started time.Time) func(*discordgo.Message) bool {
|
||||||
|
+ notBefore := started.Add(-2 * time.Second)
|
||||||
|
+ content = strings.TrimSpace(content)
|
||||||
|
+ return func(msg *discordgo.Message) bool {
|
||||||
|
+ if msg.WebhookID != webhookID || strings.TrimSpace(msg.Content) != content || len(msg.Attachments) != numFiles {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ ts, err := discordgo.SnowflakeTimestamp(msg.ID)
|
||||||
|
+ return err != nil || !ts.Before(notBefore)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/lotus_echo_test.go b/lotus_echo_test.go
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..6f5ad44
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/lotus_echo_test.go
|
||||||
|
@@ -0,0 +1,153 @@
|
||||||
|
+package main
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "errors"
|
||||||
|
+ "net"
|
||||||
|
+ "net/http"
|
||||||
|
+ "strconv"
|
||||||
|
+ "testing"
|
||||||
|
+ "time"
|
||||||
|
+
|
||||||
|
+ "github.com/bwmarrin/discordgo"
|
||||||
|
+ "github.com/rs/zerolog"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+func resetErr() error {
|
||||||
|
+ return urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")})
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestIsAmbiguousSendError(t *testing.T) {
|
||||||
|
+ cases := []struct {
|
||||||
|
+ name string
|
||||||
|
+ err error
|
||||||
|
+ want bool
|
||||||
|
+ }{
|
||||||
|
+ {"nil", nil, false},
|
||||||
|
+ {"tls handshake timeout is pre-send", urlErr(tlsTimeout{}), false},
|
||||||
|
+ {"read reset", resetErr(), true},
|
||||||
|
+ {"awaiting headers", urlErr(errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), true},
|
||||||
|
+ {"unexpected EOF", urlErr(errors.New("unexpected EOF")), true},
|
||||||
|
+ {"discord 502", &discordgo.RESTError{Response: &http.Response{StatusCode: 502}}, true},
|
||||||
|
+ {"discord 400", &discordgo.RESTError{Response: &http.Response{StatusCode: 400}}, false},
|
||||||
|
+ {"other", errors.New("unknown message type"), false},
|
||||||
|
+ }
|
||||||
|
+ for _, c := range cases {
|
||||||
|
+ if got := isAmbiguousSendError(c.err); got != c.want {
|
||||||
|
+ t.Errorf("%s: got %v want %v", c.name, got, c.want)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRetrySendAmbiguousWithoutCheckerIsNotRetried(t *testing.T) {
|
||||||
|
+ withNoSleep(t)
|
||||||
|
+ calls := 0
|
||||||
|
+ _, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
|
||||||
|
+ calls++
|
||||||
|
+ return nil, resetErr()
|
||||||
|
+ }, nil)
|
||||||
|
+ if err == nil || calls != 1 {
|
||||||
|
+ t.Fatalf("calls=%d err=%v", calls, err)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRetrySendAmbiguousButDeliveredIsNotResent(t *testing.T) {
|
||||||
|
+ withNoSleep(t)
|
||||||
|
+ calls := 0
|
||||||
|
+ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
|
||||||
|
+ calls++
|
||||||
|
+ return nil, resetErr()
|
||||||
|
+ }, func(wait bool) (*discordgo.Message, bool) {
|
||||||
|
+ return &discordgo.Message{ID: "echo"}, true
|
||||||
|
+ })
|
||||||
|
+ if err != nil || msg == nil || msg.ID != "echo" || calls != 1 {
|
||||||
|
+ t.Fatalf("calls=%d msg=%v err=%v", calls, msg, err)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRetrySendAmbiguousNotDeliveredIsResent(t *testing.T) {
|
||||||
|
+ slept := withNoSleep(t)
|
||||||
|
+ calls := 0
|
||||||
|
+ var waits []bool
|
||||||
|
+ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
|
||||||
|
+ calls++
|
||||||
|
+ if calls == 1 {
|
||||||
|
+ return nil, resetErr()
|
||||||
|
+ }
|
||||||
|
+ return &discordgo.Message{ID: "2"}, nil
|
||||||
|
+ }, func(wait bool) (*discordgo.Message, bool) {
|
||||||
|
+ waits = append(waits, wait)
|
||||||
|
+ return nil, false
|
||||||
|
+ })
|
||||||
|
+ if err != nil || msg.ID != "2" || calls != 2 || len(*slept) != 1 {
|
||||||
|
+ t.Fatalf("calls=%d msg=%v err=%v slept=%v", calls, msg, err, *slept)
|
||||||
|
+ }
|
||||||
|
+ if len(waits) != 2 || !waits[0] || waits[1] {
|
||||||
|
+ t.Fatalf("waits=%v, want [true false]", waits)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestRetrySendLateEchoDuringBackoffStopsResend(t *testing.T) {
|
||||||
|
+ withNoSleep(t)
|
||||||
|
+ calls := 0
|
||||||
|
+ checks := 0
|
||||||
|
+ msg, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
|
||||||
|
+ calls++
|
||||||
|
+ return nil, resetErr()
|
||||||
|
+ }, func(wait bool) (*discordgo.Message, bool) {
|
||||||
|
+ checks++
|
||||||
|
+ if checks == 2 { // the echo arrives during the backoff
|
||||||
|
+ return &discordgo.Message{ID: "late"}, true
|
||||||
|
+ }
|
||||||
|
+ return nil, false
|
||||||
|
+ })
|
||||||
|
+ if err != nil || msg.ID != "late" || calls != 1 {
|
||||||
|
+ t.Fatalf("calls=%d msg=%v err=%v", calls, msg, err)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func snowflakeAt(ts time.Time) string {
|
||||||
|
+ ms := ts.UnixMilli() - 1420070400000
|
||||||
|
+ return strconv.FormatInt(ms<<22, 10)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEchoWaiterMatchesWebhookPost(t *testing.T) {
|
||||||
|
+ oldWait := lotusEchoWait
|
||||||
|
+ lotusEchoWait = 200 * time.Millisecond
|
||||||
|
+ t.Cleanup(func() { lotusEchoWait = oldWait })
|
||||||
|
+
|
||||||
|
+ start := time.Now()
|
||||||
|
+ check, done := lotusExpectEcho("chan1", lotusWebhookEchoMatcher("hook1", "hi", 1, start))
|
||||||
|
+ defer done()
|
||||||
|
+
|
||||||
|
+ if _, ok := check(false); ok {
|
||||||
|
+ t.Fatal("matched before any message")
|
||||||
|
+ }
|
||||||
|
+ // Wrong webhook, wrong content, wrong channel and an old message are ignored.
|
||||||
|
+ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan1", WebhookID: "other", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)})
|
||||||
|
+ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan1", WebhookID: "hook1", Content: "nope", Attachments: make([]*discordgo.MessageAttachment, 1)})
|
||||||
|
+ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start), ChannelID: "chan2", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)})
|
||||||
|
+ lotusNoteGatewayMessage(&discordgo.Message{ID: snowflakeAt(start.Add(-time.Minute)), ChannelID: "chan1", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)})
|
||||||
|
+ if _, ok := check(true); ok {
|
||||||
|
+ t.Fatal("matched a non-matching message")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ want := snowflakeAt(start.Add(time.Second))
|
||||||
|
+ go func() {
|
||||||
|
+ time.Sleep(20 * time.Millisecond)
|
||||||
|
+ lotusNoteGatewayMessage(&discordgo.Message{ID: want, ChannelID: "chan1", WebhookID: "hook1", Content: "hi", Attachments: make([]*discordgo.MessageAttachment, 1)})
|
||||||
|
+ }()
|
||||||
|
+ got, ok := check(true)
|
||||||
|
+ if !ok || got.ID != want {
|
||||||
|
+ t.Fatalf("got %v ok=%v", got, ok)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func TestEchoWaiterDoneUnregisters(t *testing.T) {
|
||||||
|
+ _, done := lotusExpectEcho("chanX", func(*discordgo.Message) bool { return true })
|
||||||
|
+ done()
|
||||||
|
+ lotusEchoMu.Lock()
|
||||||
|
+ defer lotusEchoMu.Unlock()
|
||||||
|
+ if _, ok := lotusEchoWaiters["chanX"]; ok {
|
||||||
|
+ t.Fatal("waiter still registered after done")
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
diff --git a/lotus_retry.go b/lotus_retry.go
|
||||||
|
index 972b612..0c5e147 100644
|
||||||
|
--- a/lotus_retry.go
|
||||||
|
+++ b/lotus_retry.go
|
||||||
|
@@ -74,17 +74,69 @@ func lotusRewindFiles(files []*discordgo.File) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+// isAmbiguousSendError reports whether err happened after the request may
|
||||||
|
+// already have reached Discord: the connection broke or timed out while
|
||||||
|
+// sending or waiting for the reply, or Discord answered with a 5xx.
|
||||||
|
+func isAmbiguousSendError(err error) bool {
|
||||||
|
+ if err == nil || isPreSendNetworkError(err) {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ var restErr *discordgo.RESTError
|
||||||
|
+ if errors.As(err, &restErr) {
|
||||||
|
+ return restErr.Response != nil && restErr.Response.StatusCode >= 500
|
||||||
|
+ }
|
||||||
|
+ if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||||
|
+ return true
|
||||||
|
+ }
|
||||||
|
+ var netErr net.Error
|
||||||
|
+ if errors.As(err, &netErr) {
|
||||||
|
+ return true
|
||||||
|
+ }
|
||||||
|
+ msg := err.Error()
|
||||||
|
+ return strings.Contains(msg, "connection reset") ||
|
||||||
|
+ strings.Contains(msg, "broken pipe") ||
|
||||||
|
+ strings.Contains(msg, "Client.Timeout exceeded") ||
|
||||||
|
+ strings.Contains(msg, "server sent GOAWAY") ||
|
||||||
|
+ strings.Contains(msg, "EOF")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// lotusRetrySend calls send until it succeeds, fails with an error that is not
|
||||||
|
-// a pre-send network error, or the retry budget runs out.
|
||||||
|
-func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error)) (T, error) {
|
||||||
|
+// retryable, or the retry budget runs out.
|
||||||
|
+//
|
||||||
|
+// Pre-send network errors are always retried. Ambiguous errors (see
|
||||||
|
+// isAmbiguousSendError) are retried only when checkDelivered is set: it is
|
||||||
|
+// called first (wait=true lets it wait briefly for evidence) and, if it finds
|
||||||
|
+// that the message did reach Discord, its result is returned as a success
|
||||||
|
+// instead of sending again. It is called again with wait=false just before
|
||||||
|
+// each resend.
|
||||||
|
+func lotusRetrySend[T any](log zerolog.Logger, files []*discordgo.File, send func() (T, error), checkDelivered func(wait bool) (T, bool)) (T, error) {
|
||||||
|
res, err := send()
|
||||||
|
- for attempt := 0; attempt < len(lotusSendRetryDelays) && isPreSendNetworkError(err); attempt++ {
|
||||||
|
+ for attempt := 0; attempt < len(lotusSendRetryDelays); attempt++ {
|
||||||
|
+ switch {
|
||||||
|
+ case isPreSendNetworkError(err):
|
||||||
|
+ case checkDelivered != nil && isAmbiguousSendError(err):
|
||||||
|
+ if delivered, ok := checkDelivered(true); ok {
|
||||||
|
+ log.Info().Err(err).Int("attempts", attempt+1).
|
||||||
|
+ Msg("[lotus] Discord send errored but the message was delivered, not resending")
|
||||||
|
+ return delivered, nil
|
||||||
|
+ }
|
||||||
|
+ default:
|
||||||
|
+ return res, err
|
||||||
|
+ }
|
||||||
|
delay := lotusSendRetryDelays[attempt]
|
||||||
|
log.Warn().Err(err).
|
||||||
|
Int("attempt", attempt+1).
|
||||||
|
Dur("retry_in", delay).
|
||||||
|
- Msg("[lotus] Discord send failed before reaching Discord, retrying")
|
||||||
|
+ Msg("[lotus] Discord send failed and the message was not delivered, retrying")
|
||||||
|
lotusSleep(delay)
|
||||||
|
+ if checkDelivered != nil {
|
||||||
|
+ // A late echo of an earlier attempt can land during the backoff.
|
||||||
|
+ if delivered, ok := checkDelivered(false); ok {
|
||||||
|
+ log.Info().Int("attempts", attempt+1).
|
||||||
|
+ Msg("[lotus] earlier Discord send turned out to be delivered, not resending")
|
||||||
|
+ return delivered, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
lotusRewindFiles(files)
|
||||||
|
res, err = send()
|
||||||
|
if err == nil {
|
||||||
|
diff --git a/lotus_retry_test.go b/lotus_retry_test.go
|
||||||
|
index efa262e..e35585b 100644
|
||||||
|
--- a/lotus_retry_test.go
|
||||||
|
+++ b/lotus_retry_test.go
|
||||||
|
@@ -68,7 +68,7 @@ func TestRetrySendRecoversAndRewindsFiles(t *testing.T) {
|
||||||
|
return nil, urlErr(tlsTimeout{})
|
||||||
|
}
|
||||||
|
return &discordgo.Message{ID: "42"}, nil
|
||||||
|
- })
|
||||||
|
+ }, nil)
|
||||||
|
if err != nil || msg == nil || msg.ID != "42" {
|
||||||
|
t.Fatalf("got msg=%v err=%v", msg, err)
|
||||||
|
}
|
||||||
|
@@ -86,7 +86,7 @@ func TestRetrySendDoesNotRetryAfterSend(t *testing.T) {
|
||||||
|
_, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
|
||||||
|
calls++
|
||||||
|
return nil, urlErr(&net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset by peer")})
|
||||||
|
- })
|
||||||
|
+ }, nil)
|
||||||
|
if err == nil || calls != 1 {
|
||||||
|
t.Fatalf("calls=%d err=%v, want exactly one attempt", calls, err)
|
||||||
|
}
|
||||||
|
@@ -98,7 +98,7 @@ func TestRetrySendGivesUpAfterBudget(t *testing.T) {
|
||||||
|
_, err := lotusRetrySend(zerolog.Nop(), nil, func() (*discordgo.Message, error) {
|
||||||
|
calls++
|
||||||
|
return nil, urlErr(tlsTimeout{})
|
||||||
|
- })
|
||||||
|
+ }, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error after budget")
|
||||||
|
}
|
||||||
|
diff --git a/portal.go b/portal.go
|
||||||
|
index cc6da0d..57ba7f9 100644
|
||||||
|
--- a/portal.go
|
||||||
|
+++ b/portal.go
|
||||||
|
@@ -1748,9 +1748,18 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) {
|
||||||
|
if !isWebhookSend {
|
||||||
|
msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) {
|
||||||
|
return sess.ChannelMessageSendComplex(channelID, &sendReq, portal.RefererOptIfUser(sess, threadID)...)
|
||||||
|
- })
|
||||||
|
+ }, nil)
|
||||||
|
} else {
|
||||||
|
username, avatarURL := portal.getRelayUserMeta(sender)
|
||||||
|
+ // [lotus] watch for the gateway echo so an ambiguous failure can be
|
||||||
|
+ // checked instead of resent blindly (see lotus_echo.go)
|
||||||
|
+ echoChannel := channelID
|
||||||
|
+ if threadID != "" {
|
||||||
|
+ echoChannel = threadID
|
||||||
|
+ }
|
||||||
|
+ checkEcho, echoDone := lotusExpectEcho(echoChannel,
|
||||||
|
+ lotusWebhookEchoMatcher(portal.RelayWebhookID, sendReq.Content, len(sendReq.Files), time.Now()))
|
||||||
|
+ defer echoDone()
|
||||||
|
msg, err = lotusRetrySend(sendLog, sendReq.Files, func() (*discordgo.Message, error) {
|
||||||
|
return relayClient.WebhookThreadExecute(portal.RelayWebhookID, portal.RelayWebhookSecret, true, threadID, &discordgo.WebhookParams{
|
||||||
|
Content: sendReq.Content,
|
||||||
|
@@ -1761,7 +1770,7 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) {
|
||||||
|
Embeds: sendReq.Embeds,
|
||||||
|
AllowedMentions: sendReq.AllowedMentions,
|
||||||
|
})
|
||||||
|
- })
|
||||||
|
+ }, checkEcho)
|
||||||
|
}
|
||||||
|
sender.handlePossible40002(err)
|
||||||
|
go portal.sendMessageMetrics(evt, err, "Error sending")
|
||||||
|
diff --git a/user.go b/user.go
|
||||||
|
index 8c0d6ec..16cd8c8 100644
|
||||||
|
--- a/user.go
|
||||||
|
+++ b/user.go
|
||||||
|
@@ -704,6 +704,7 @@ func (user *User) eventHandler(rawEvt any) {
|
||||||
|
case *discordgo.RelationshipUpdate:
|
||||||
|
user.relationshipUpdateHandler(evt)
|
||||||
|
case *discordgo.MessageCreate:
|
||||||
|
+ lotusNoteGatewayMessage(evt.Message) // [lotus] see lotus_echo.go
|
||||||
|
user.pushPortalMessage(evt, "message create", evt.ChannelID, evt.GuildID)
|
||||||
|
case *discordgo.MessageDelete:
|
||||||
|
user.pushPortalMessage(evt, "message delete", evt.ChannelID, evt.GuildID)
|
||||||
Reference in New Issue
Block a user