Skip to content

Commit 45746ee

Browse files
authored
feat(relay): add opt-in HTTP/2 health checks (QuantumNous#8059)
1 parent c046559 commit 45746ee

5 files changed

Lines changed: 159 additions & 0 deletions

File tree

‎.env.example‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,12 @@
6868
# RELAY_TIMEOUT=0
6969
# Relay HTTP 客户端空闲连接超时时间,单位秒,默认跟随 Go 标准库,设置为0表示不限制
7070
# RELAY_IDLE_CONN_TIMEOUT=90
71+
# HTTP/2 连接连续未收到任何帧多少秒后发送 PING;0(默认)关闭健康探测。
72+
# 适用于上游连接,包括代理与分片连接;与发给客户端的 SSE 心跳独立。
73+
# RELAY_HTTP2_PING_IDLE_TIMEOUT=0
74+
# 启用探测后等待 PING 应答的秒数;0 或负数使用 Go 默认值 15 秒。
75+
# 仅关闭未应答的连接,正常应答的慢请求可继续等待。
76+
# RELAY_HTTP2_PING_TIMEOUT=15
7177
# 等待上游返回响应头的超时时间,单位秒,默认 1800,设置为 0 表示不限制。
7278
# 仅约束「等待响应头」这一段;响应头返回之后的流式传输不受影响。
7379
# 注意:非流式请求通常要等上游生成完毕才会返回响应头,因此该值需留足余量。

‎common/constants.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,9 @@ var RelayTimeout int // unit is second
173173

174174
var RelayIdleConnTimeout int // unit is second
175175

176+
var RelayHTTP2PingIdleTimeout int // seconds without incoming frames before probing; 0 disables probes
177+
var RelayHTTP2PingTimeout int // seconds to wait for a PING acknowledgement
178+
176179
// RelayResponseHeaderTimeout limits how long the relay transport waits for the
177180
// upstream response headers after the request has been fully written.
178181
// 0 disables it (previous behaviour: wait forever).

‎common/init.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@ func InitEnv() {
113113
BatchUpdateInterval = GetEnvOrDefault("BATCH_UPDATE_INTERVAL", 5)
114114
RelayTimeout = GetEnvOrDefault("RELAY_TIMEOUT", 0)
115115
RelayIdleConnTimeout = GetEnvOrDefault("RELAY_IDLE_CONN_TIMEOUT", 90)
116+
RelayHTTP2PingIdleTimeout = GetEnvOrDefault("RELAY_HTTP2_PING_IDLE_TIMEOUT", 0)
117+
RelayHTTP2PingTimeout = GetEnvOrDefault("RELAY_HTTP2_PING_TIMEOUT", 15)
116118
RelayResponseHeaderTimeout = GetEnvOrDefault("RELAY_RESPONSE_HEADER_TIMEOUT", 1800)
117119
RelayMaxIdleConns = GetEnvOrDefault("RELAY_MAX_IDLE_CONNS", 500)
118120
RelayMaxIdleConnsPerHost = GetEnvOrDefault("RELAY_MAX_IDLE_CONNS_PER_HOST", 100)

‎service/http_client.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,14 @@ func newRelayHTTPTransport() *http.Transport {
9696
transport.MaxIdleConns = common.RelayMaxIdleConns
9797
transport.MaxIdleConnsPerHost = common.RelayMaxIdleConnsPerHost
9898
transport.IdleConnTimeout = time.Duration(common.RelayIdleConnTimeout) * time.Second
99+
if seconds := common.RelayHTTP2PingIdleTimeout; seconds > 0 {
100+
if transport.HTTP2 == nil {
101+
transport.HTTP2 = &http.HTTP2Config{}
102+
}
103+
// Probe the upstream connection, independently of downstream SSE heartbeats.
104+
transport.HTTP2.SendPingTimeout = time.Duration(min(seconds, maxTimeoutSeconds)) * time.Second
105+
transport.HTTP2.PingTimeout = time.Duration(min(max(common.RelayHTTP2PingTimeout, 0), maxTimeoutSeconds)) * time.Second
106+
}
99107
// Bound the wait for upstream response headers. Without it, an upstream that
100108
// accepts the connection but never responds (and never sends FIN/RST) parks the
101109
// goroutine forever, and every buffer that request owns -- the raw body read by

‎service/http_client_transport_test.go‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package service
22

33
import (
4+
"bytes"
5+
"context"
46
"crypto/tls"
57
"crypto/x509"
68
"encoding/pem"
@@ -12,25 +14,31 @@ import (
1214
"sync"
1315
"sync/atomic"
1416
"testing"
17+
"time"
1518

1619
"github.com/QuantumNous/new-api/common"
1720
"github.com/QuantumNous/new-api/relaykit/dto"
1821
"github.com/stretchr/testify/assert"
1922
"github.com/stretchr/testify/require"
23+
"golang.org/x/net/http2"
24+
"golang.org/x/net/http2/hpack"
2025
)
2126

2227
func withRelayHTTPTransportSettings(t *testing.T) {
2328
t.Helper()
2429
prevMaxIdle := common.RelayMaxIdleConns
2530
prevPerHost := common.RelayMaxIdleConnsPerHost
2631
prevTimeout := common.RelayIdleConnTimeout
32+
prevPingIdle, prevPingTimeout := common.RelayHTTP2PingIdleTimeout, common.RelayHTTP2PingTimeout
2733
common.RelayMaxIdleConns = 500
2834
common.RelayMaxIdleConnsPerHost = 100
2935
common.RelayIdleConnTimeout = 90
36+
common.RelayHTTP2PingIdleTimeout, common.RelayHTTP2PingTimeout = 1, 1
3037
t.Cleanup(func() {
3138
common.RelayMaxIdleConns = prevMaxIdle
3239
common.RelayMaxIdleConnsPerHost = prevPerHost
3340
common.RelayIdleConnTimeout = prevTimeout
41+
common.RelayHTTP2PingIdleTimeout, common.RelayHTTP2PingTimeout = prevPingIdle, prevPingTimeout
3442
})
3543
}
3644

@@ -569,3 +577,135 @@ func TestNormalizeHTTPTransportPolicyClampsWithoutPanic(t *testing.T) {
569577
assert.Equal(t, HTTPTransportPolicy{Protocol: dto.HTTPProtocolAuto, Shards: 1}, NormalizeHTTPTransportPolicy(dto.ChannelSettings{HTTP2ConnectionShards: -3}))
570578
assert.Equal(t, HTTPTransportPolicy{Protocol: dto.HTTPProtocolAuto, Shards: 8}, NormalizeHTTPTransportPolicy(dto.ChannelSettings{HTTP2ConnectionShards: 99}))
571579
}
580+
581+
// serveHTTP2ProbePeer can withhold PING acknowledgements without sending FIN or
582+
// RST, reproducing an upstream connection silently lost by an intermediary.
583+
func serveHTTP2ProbePeer(conn net.Conn, acknowledge bool, pings *atomic.Int32) error {
584+
defer conn.Close()
585+
if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
586+
return err
587+
}
588+
preface := make([]byte, len(http2.ClientPreface))
589+
if _, err := io.ReadFull(conn, preface); err != nil {
590+
return err
591+
}
592+
if string(preface) != http2.ClientPreface {
593+
return fmt.Errorf("unexpected HTTP/2 preface: %q", preface)
594+
}
595+
framer := http2.NewFramer(conn, conn)
596+
framer.ReadMetaHeaders = hpack.NewDecoder(4096, nil)
597+
if err := framer.WriteSettings(); err != nil {
598+
return err
599+
}
600+
var streamID uint32
601+
for {
602+
frame, err := framer.ReadFrame()
603+
if err == io.EOF {
604+
return nil
605+
}
606+
if err != nil {
607+
return err
608+
}
609+
switch frame := frame.(type) {
610+
case *http2.SettingsFrame:
611+
if !frame.IsAck() {
612+
if err := framer.WriteSettingsAck(); err != nil {
613+
return err
614+
}
615+
}
616+
case *http2.MetaHeadersFrame:
617+
streamID = frame.StreamID
618+
var block bytes.Buffer
619+
if err := hpack.NewEncoder(&block).WriteField(hpack.HeaderField{Name: ":status", Value: "200"}); err != nil {
620+
return err
621+
}
622+
if err := framer.WriteHeaders(http2.HeadersFrameParam{StreamID: streamID, BlockFragment: block.Bytes(), EndHeaders: true}); err != nil {
623+
return err
624+
}
625+
if err := framer.WriteData(streamID, false, []byte("start")); err != nil {
626+
return err
627+
}
628+
case *http2.PingFrame:
629+
if frame.IsAck() {
630+
continue
631+
}
632+
count := pings.Add(1)
633+
if acknowledge {
634+
if err := framer.WritePing(true, frame.Data); err != nil {
635+
return err
636+
}
637+
if count >= 2 && streamID != 0 {
638+
if err := framer.WriteData(streamID, true, []byte("end")); err != nil {
639+
return err
640+
}
641+
streamID = 0
642+
}
643+
}
644+
}
645+
}
646+
}
647+
648+
func TestRelayHTTP2HealthCheck(t *testing.T) {
649+
for _, tc := range []struct {
650+
name string
651+
idleTimeout int
652+
acknowledge bool
653+
}{
654+
{name: "disabled", idleTimeout: 0},
655+
{name: "healthy slow response", idleTimeout: 1, acknowledge: true},
656+
{name: "unresponsive connection", idleTimeout: 1},
657+
} {
658+
t.Run(tc.name, func(t *testing.T) {
659+
withRelayHTTPTransportSettings(t)
660+
common.RelayHTTP2PingIdleTimeout = tc.idleTimeout
661+
662+
var connections, pings atomic.Int32
663+
peerErrors := make(chan error, 4)
664+
server := httptest.NewUnstartedServer(nil)
665+
server.EnableHTTP2 = true
666+
server.Config.TLSNextProto = map[string]func(*http.Server, *tls.Conn, http.Handler){
667+
"h2": func(_ *http.Server, conn *tls.Conn, _ http.Handler) {
668+
connection := connections.Add(1)
669+
acknowledge := tc.acknowledge || connection > 1
670+
peerErrors <- serveHTTP2ProbePeer(conn, acknowledge, &pings)
671+
},
672+
}
673+
server.StartTLS()
674+
t.Cleanup(server.Close)
675+
client := newHTTPClientWithPolicyAndTLS(defaultHTTPTransportPolicy(), testTLSClientConfig(t, server))
676+
client.Timeout = 7 * time.Second
677+
if tc.idleTimeout == 0 {
678+
client.Timeout = 3 * time.Second
679+
}
680+
t.Cleanup(client.CloseIdleConnections)
681+
682+
resp, err := client.Get(server.URL)
683+
require.NoError(t, err)
684+
require.Equal(t, 2, resp.ProtoMajor)
685+
body, err := io.ReadAll(resp.Body)
686+
require.NoError(t, resp.Body.Close())
687+
if tc.idleTimeout == 0 {
688+
require.ErrorIs(t, err, context.DeadlineExceeded)
689+
assert.Zero(t, pings.Load(), "disabled probes must preserve the existing transport behavior")
690+
} else if tc.acknowledge {
691+
require.NoError(t, err, "a slow response must survive acknowledged health checks")
692+
assert.Equal(t, "startend", string(body))
693+
assert.GreaterOrEqual(t, pings.Load(), int32(2))
694+
assert.Equal(t, int32(1), connections.Load())
695+
} else {
696+
require.ErrorContains(t, err, "http2: client connection lost")
697+
assert.Equal(t, "start", string(body), "already delivered content must not be replayed")
698+
assert.Equal(t, int32(1), connections.Load())
699+
require.NoError(t, <-peerErrors, "the peer must observe the failed connection close")
700+
701+
resp, err = client.Get(server.URL)
702+
require.NoError(t, err)
703+
body, err = io.ReadAll(resp.Body)
704+
require.NoError(t, err)
705+
require.NoError(t, resp.Body.Close())
706+
assert.Equal(t, "startend", string(body))
707+
assert.Equal(t, int32(2), connections.Load(), "the next request must establish a fresh connection")
708+
}
709+
})
710+
}
711+
}

0 commit comments

Comments
 (0)