e2e: add cmd probe recovery test (gost#837)

Verifies dead->alive->dead node transition: a node excluded by a failing cmd
probe resumes carrying traffic once the probe flips healthy, without restart.

tests/e2e/probe_test.go

@@ -100,6 +100,50 @@ func (s *ProbeSuite) TestCmdProbeFailover() {
 	}
 }
 
+// TestProbeRecovery verifies the dead→alive→dead transition that issue #837
+// describes: a node that fails is excluded, and once it recovers (probe flips
+// healthy) it resumes carrying traffic. Both relays stay up; only the cmd
+// probe flag files in the container flip, so no restart is needed.
+//
+// Phase 1: node-a marked dead, node-b healthy → every request goes to node-b.
+// Phase 2: recover node-a, kill node-b → every request must still succeed,
+// which is only possible if node-a's probe recovered it (otherwise both nodes
+// would be excluded and the request would fail).
+func (s *ProbeSuite) TestProbeRecovery() {
+	gostC, err := RunGostContainerWithPorts(s.ctx, SharedNetworkName, "testdata/probe/recovery.yaml", "8080/tcp")
+	s.Require().NoError(err)
+	defer gostC.Terminate(s.ctx)
+
+	// The node probes fire immediately at startup, so let both settle healthy
+	// before we start flipping flag files.
+	time.Sleep(1 * time.Second)
+
+	// Phase 1: knock node-a down, leave node-b up.
+	_, _, err = gostC.Exec(s.ctx, []string{"touch", "/tmp/a_down"})
+	s.Require().NoError(err)
+	time.Sleep(2500 * time.Millisecond) // > probe interval
+
+	for range 10 {
+		code, body := s.proxyRequest(gostC, "8080")
+		s.Require().Equal(0, code, "phase1: dead node-a must be excluded, all traffic via node-b")
+		s.Require().Contains(body, "hello-gost")
+	}
+
+	// Phase 2: recover node-a, knock node-b down. If node-a's probe did not
+	// revive it, the request would fail (no healthy candidate).
+	_, _, err = gostC.Exec(s.ctx, []string{"rm", "-f", "/tmp/a_down", "/tmp/b_down"})
+	s.Require().NoError(err)
+	_, _, err = gostC.Exec(s.ctx, []string{"touch", "/tmp/b_down"})
+	s.Require().NoError(err)
+	time.Sleep(2500 * time.Millisecond)
+
+	for range 10 {
+		code, body := s.proxyRequest(gostC, "8080")
+		s.Require().Equal(0, code, "phase2: recovered node-a must resume carrying traffic")
+		s.Require().Contains(body, "hello-gost")
+	}
+}
+
 func TestProbeSuite(t *testing.T) {
 	suite.Run(t, new(ProbeSuite))
 }