From 26fb12988701211d40f72dc004c9d7997f538eae Mon Sep 17 00:00:00 2001 From: Connor Welsh Date: Fri, 21 Aug 2026 22:45:30 -0400 Subject: [PATCH] fix(tailscale): back off when the IPN bus read fails (#3142) --- core/internal/server/tailscale/manager.go | 22 ++++++++++++++---- .../internal/server/tailscale/manager_test.go | 23 +++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/core/internal/server/tailscale/manager.go b/core/internal/server/tailscale/manager.go index d6e97a888..a30496fe7 100644 --- a/core/internal/server/tailscale/manager.go +++ b/core/internal/server/tailscale/manager.go @@ -112,17 +112,13 @@ func (m *Manager) watchLoop(ctx context.Context) { m.updateState(&TailscaleState{Connected: false, BackendState: "Unreachable"}) unreachableSent = true } - select { - case <-ctx.Done(): + if !sleepBackoff(ctx, &backoff) { return - case <-time.After(backoff): } - backoff = min(backoff*2, 30*time.Second) continue } unreachableSent = false - backoff = time.Second log.Info("[Tailscale] Connected to IPN bus") m.markAvailable() @@ -133,6 +129,8 @@ func (m *Manager) watchLoop(ctx context.Context) { break } + backoff = time.Second + if notify.State == nil && notify.NetMap == nil { //nolint:staticcheck // NetMap is deprecated upstream but still the only activity signal on some platforms continue } @@ -143,9 +141,23 @@ func (m *Manager) watchLoop(ctx context.Context) { } watcher.Close() + + if !sleepBackoff(ctx, &backoff) { + return + } } } +func sleepBackoff(ctx context.Context, backoff *time.Duration) bool { + select { + case <-ctx.Done(): + return false + case <-time.After(*backoff): + } + *backoff = min(*backoff*2, 30*time.Second) + return true +} + // debounceLoop coalesces rapid bus notifications into a single Status RPC // per debounceWindow, since NetMap events can fire many times per second // on busy tailnets. diff --git a/core/internal/server/tailscale/manager_test.go b/core/internal/server/tailscale/manager_test.go index 87666e91b..afe980b6f 100644 --- a/core/internal/server/tailscale/manager_test.go +++ b/core/internal/server/tailscale/manager_test.go @@ -214,6 +214,29 @@ func TestWatchLoop_Reconnect(t *testing.T) { }, 3*time.Second, 50*time.Millisecond) } +func TestWatchLoop_BacksOffOnPersistentBusError(t *testing.T) { + var watchCalls atomic.Int32 + + client := &mockClient{ + watchFn: func(ctx context.Context, mask ipn.NotifyWatchOpt) (ipnBusWatcher, error) { + watchCalls.Add(1) + return newMockWatcher(ctx, nil, fmt.Errorf("json: cannot unmarshal object")), nil + }, + statusFn: func(ctx context.Context) (*ipnstate.Status, error) { + return runningStatus(), nil + }, + } + + m := newManager(client) + defer m.Close() + + time.Sleep(300 * time.Millisecond) + + calls := watchCalls.Load() + assert.LessOrEqual(t, int(calls), 3, + "a persistent bus read error should back off, not reconnect in a hot loop; got %d attempts in 300ms", calls) +} + func TestManager_Subscribe(t *testing.T) { client := &mockClient{ watchFn: func(ctx context.Context, mask ipn.NotifyWatchOpt) (ipnBusWatcher, error) {