ZEROMAKE | keep codeing and thinking!
2018-08-010 | protocol

HTTP2 协议解析

前言

  • 最近工作都在做跟 http2 协议有关的东西,记录下协议的格式与资料。
  • 下篇(这篇)文章中会简略的写出一个支持高并发的 golanghttp2 转发器。

一、tcp 数据头特征

由于 HTTP2 依旧是客户端主动模式,所以协议头只有客户端到服务端的。

协议头的 ascii 模式
头为 32byte 的二进制可以 ascii 模式显示, 完成后直接开始 Frame 通信。

PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n

二、Frame 二进制格式描述

rfc 中定义了 10 种 Frame, 下面先把一些需要的静态类型声明

1
type FrameType uint8
2
3
const (
4
FrameData FrameType = 0x0
5
FrameHeaders FrameType = 0x1
6
FramePriority FrameType = 0x2
7
FrameRSTStream FrameType = 0x3
8
FrameSettings FrameType = 0x4
9
FramePushPromise FrameType = 0x5
10
FramePing FrameType = 0x6
11
FrameGoAway FrameType = 0x7
12
FrameWindowUpdate FrameType = 0x8
13
FrameContinuation FrameType = 0x9
14
)
15
16
type Flags uint8
17
18
func (f Flags) Has(v Flags) bool {
19
return (f & v) == v
20
}
21
22
const (
23
// Data Frame
24
FlagDataEndStream Flags = 0x1
25
FlagDataPadded Flags = 0x8
26
27
// Headers Frame
28
FlagHeadersEndStream Flags = 0x1
29
FlagHeadersEndHeaders Flags = 0x4
30
FlagHeadersPadded Flags = 0x8
31
FlagHeadersPriority Flags = 0x20
32
33
// Settings Frame
34
FlagSettingsAck Flags = 0x1
35
36
// Ping Frame
37
FlagPingAck Flags = 0x1
38
39
// Continuation Frame
40
FlagContinuationEndHeaders Flags = 0x4
41
42
FlagPushPromiseEndHeaders Flags = 0x4
43
FlagPushPromisePadded Flags = 0x8
44
)
45
46
// or (1<<31) - 1
47
const PadBit uint32 = 0x7fffffff

2.01 FRAME Format

rfc-frame

所有的 Frame 的头都是 9bit 长度。

1
+-----------------------------------------------+
2
| Length (24) |
3
+---------------+---------------+---------------+
4
| Type (8) | Flags (8) |
5
+-+-------------+---------------+-------------------------------+
6
|R| Stream Identifier (31) |
7
+=+=============================================================+
8
| Frame Payload (0...) ...
9
+---------------------------------------------------------------+
  • Length: 24bit 长度的转换为 int,范围在 2^142^24-1 之间。
  • Type: 8bit 的区分 Frame 的类型。
  • Flags: 8bit 通过位运算可以放置 8 个标识。
  • R: 1bit 占位必须为 0x0
  • Stream Identifier: 31bit 的无符号 int 。
  • Frame Payload: 长度为上面的 Length 的二进制。

解析示例

1
const (
2
frameHeaderLen = 9
3
)
4
type FrameHeader struct {
5
valid bool // caller can access []byte fields in the Frame
6
7
// Type is the 1 byte frame type. There are ten standard frame
8
// types, but extension frame types may be written by WriteRawFrame
9
// and will be returned by ReadFrame (as UnknownFrame).
10
Type FrameType
11
12
// Flags are the 1 byte of 8 potential bit flags per frame.
13
// They are specific to the frame type.
14
Flags Flags
15
16
// Length is the length of the frame, not including the 9 byte header.
17
// The maximum size is one byte less than 16MB (uint24), but only
18
// frames up to 16KB are allowed without peer agreement.
19
Length uint32
20
21
// StreamID is which stream this frame is for. Certain frames
22
// are not stream-specific, in which case this field is 0.
23
StreamID uint32
24
}
25
// 统一错误输出
26
type ErrCode uint32
27
28
const (
29
ErrCodeNo ErrCode = 0x0
30
ErrCodeProtocol ErrCode = 0x1
31
ErrCodeInternal ErrCode = 0x2
32
ErrCodeFlowControl ErrCode = 0x3
33
ErrCodeSettingsTimeout ErrCode = 0x4
34
ErrCodeStreamClosed ErrCode = 0x5
35
ErrCodeFrameSize ErrCode = 0x6
36
ErrCodeRefusedStream ErrCode = 0x7
37
ErrCodeCancel ErrCode = 0x8
38
ErrCodeCompression ErrCode = 0x9
39
ErrCodeConnect ErrCode = 0xa
40
ErrCodeEnhanceYourCalm ErrCode = 0xb
41
ErrCodeInadequateSecurity ErrCode = 0xc
42
ErrCodeHTTP11Required ErrCode = 0xd
43
)
44
45
var errCodeName = map[ErrCode]string{
46
ErrCodeNo: "NO_ERROR",
47
ErrCodeProtocol: "PROTOCOL_ERROR",
48
ErrCodeInternal: "INTERNAL_ERROR",
49
ErrCodeFlowControl: "FLOW_CONTROL_ERROR",
50
ErrCodeSettingsTimeout: "SETTINGS_TIMEOUT",
51
ErrCodeStreamClosed: "STREAM_CLOSED",
52
ErrCodeFrameSize: "FRAME_SIZE_ERROR",
53
ErrCodeRefusedStream: "REFUSED_STREAM",
54
ErrCodeCancel: "CANCEL",
55
ErrCodeCompression: "COMPRESSION_ERROR",
56
ErrCodeConnect: "CONNECT_ERROR",
57
ErrCodeEnhanceYourCalm: "ENHANCE_YOUR_CALM",
58
ErrCodeInadequateSecurity: "INADEQUATE_SECURITY",
59
ErrCodeHTTP11Required: "HTTP_1_1_REQUIRED",
60
}
61
62
func (e ErrCode) String() string {
63
if s, ok := errCodeName[e]; ok {
64
return s
65
}
66
return fmt.Sprintf("unknown error code 0x%x", uint32(e))
67
}
68
type connError struct {
69
Code ErrCode // the ConnectionError error code
70
Reason string // additional reason
71
}
72
73
func (e connError) Error() string {
74
return fmt.Sprintf("http2: connection error: %v: %v", e.Code, e.Reason)
75
}
76
77
type StreamError struct {
78
StreamID uint32
79
Code ErrCode
80
Cause error // optional additional detail
81
}
82
83
func streamError(id uint32, code ErrCode) StreamError {
84
return StreamError{StreamID: id, Code: code}
85
}
86
87
func (e StreamError) Error() string {
88
if e.Cause != nil {
89
return fmt.Sprintf("stream error: stream ID %d; %v; %v", e.StreamID, e.Code, e.Cause)
90
}
91
return fmt.Sprintf("stream error: stream ID %d; %v", e.StreamID, e.Code)
92
}
93
//
94
func readByte(p []byte) ([]byte, byte, error) {
95
if len(p) == 0 {
96
return nil, 0, io.ErrUnexpectedEOF
97
}
98
return p[1:], p[0], nil
99
}
100
101
func readUint32(p []byte) ([]byte, uint32, error) {
102
if len(p) < 4 {
103
return nil, 0, io.ErrUnexpectedEOF
104
}
105
return p[4:], binary.BigEndian.Uint32(p[:4]), nil
106
}
107
108
func readFrameHeader(buf []byte, r io.Reader) (*FrameHeader, error) {
109
// 读出 9 个字节的 Frame 的头。
110
_, err := io.ReadFull(r, buf[:frameHeaderLen])
111
if err != nil {
112
return nil, err
113
}
114
return &FrameHeader{
115
// 把长度解析为 uint32, 由于长度不足 32 bit 手动通过位操作
116
Length: (uint32(buf[0])<<16 | uint32(buf[1])<<8 | uint32(buf[2])),
117
// type 转为 uint8
118
Type: FrameType(buf[3]),
119
// flags 也为 uint8
120
Flags: Flags(buf[4]),
121
// StreamID 正好长度为 32bit 通过binary.BigEndian.Uint32进行转换。
122
StreamID: binary.BigEndian.Uint32(buf[5:]) & PadBit,
123
valid: true,
124
}, nil
125
}
126
127
func ReadFrame(r io.Reader) {
128
// buf可以考虑通过池子复用
129
buf := make([]byte, frameHeaderLen)
130
header, err := readFrameHeader(buf, r)
131
if err != nil {
132
return
133
}
134
payload := make([]byte, header.Length)
135
_, err = io.ReadFull(r, payload)
136
if err != nil {
137
return
138
}
139
// Frame Parser
140
}

2.02 DATA Format 0x0

rfc-data

1
+---------------+
2
|Pad Length? (8)|
3
+---------------+-----------------------------------------------+
4
| Data (*) ...
5
+---------------------------------------------------------------+
6
| Padding (*) ...
7
+---------------------------------------------------------------+
  • Pad Length: 8bit 可选,尾部 Padding 的填充字符的长度,可能用于对齐字节,需要 flag |= PADDED
  • Data: body 内容字节
  • Padding: 填充字节

解析示例

1
type DataFrame struct {
2
FrameHeader
3
Data []byte
4
}
5
func ParseDataFrame(fh *FrameHeader, payload []byte) (*DataFrame, error) {
6
if fh.StreamID == 0 {
7
return nil, ConnError{ErrCodeProtocol, "DATA frame with stream ID 0"}
8
}
9
f := &DataFrame{FrameHeader: *fh}
10
11
var padSize byte
12
if fh.Flags.Has(FlagDataPadded) {
13
var err error
14
payload, padSize, err = readByte(payload)
15
if err != nil {
16
return nil, err
17
}
18
}
19
if int(padSize) > len(payload) {
20
return nil, ConnError{ErrCodeProtocol, "pad size larger than data payload"}
21
}
22
f.Data = payload[:len(payload)-int(padSize)]
23
}

flags

  • END_STREAM: 0x1 会话流是否结束
  • PADDED: 0x8 是否有填充字节

2.03 HEADER Format 0x1

rfc-headers

1
+---------------+
2
|Pad Length? (8)|
3
+-+-------------+-----------------------------------------------+
4
|E| Stream Dependency? (31) |
5
+-+-------------+-----------------------------------------------+
6
| Weight? (8) |
7
+-+-------------+-----------------------------------------------+
8
| Header Block Fragment (*) ...
9
+---------------------------------------------------------------+
10
| Padding (*) ...
11
+---------------------------------------------------------------+
  • Pad Length: 8bit 可选,尾部 Padding 的填充字符的长度,可能用于对齐字节,需要 flag |= PADDED
  • E: 设置流是否为独占或依赖,PRIORITY 支持
  • Stream Dependency: 31bit 依赖的 stream identifier,需要 flag |= PRIORITY
  • Weight: 8bit 流的权重 1-255,需要 flag |= PRIORITY
  • Header Block Fragment: 压缩后的 Header 内容二进制
  • Padding: 填充字节

解析示例

1
type HeadersFrame struct {
2
FrameHeader
3
Priority PriorityParam
4
HeaderFragBuf []byte
5
}
6
func ParseHeadersFrame(fh *FrameHeader, p []byte) (*HeadersFrame, error) {
7
var err error
8
hf := &HeadersFrame{
9
FrameHeader: *fh,
10
}
11
if fh.StreamID == 0 {
12
return nil, ConnError{ErrCodeProtocol, "HEADERS frame with stream ID 0"}
13
}
14
var padLength uint8
15
if fh.Flags.Has(FlagHeadersPadded) {
16
if p, padLength, err = readByte(p); err != nil {
17
return nil, err
18
}
19
}
20
if fh.Flags.Has(FlagHeadersPriority) {
21
var v uint32
22
p, v, err = readUint32(p)
23
if err != nil {
24
return nil, err
25
}
26
hf.Priority.StreamDep = v & PadBit
27
hf.Priority.Exclusive = (v != hf.Priority.StreamDep) // high bit was set
28
p, hf.Priority.Weight, err = readByte(p)
29
if err != nil {
30
return nil, err
31
}
32
}
33
if len(p)-int(padLength) <= 0 {
34
return nil, streamError(fh.StreamID, ErrCodeProtocol)
35
}
36
hf.HeaderFragBuf = p[:len(p)-int(padLength)]
37
return hf, nil
38
}

flag

  • END_STREAM: 0x1 是否结束这次流会话(比如 GET 的 header)
  • END_HEADERS: 0x4 header 是否结束,没有结束需要读取下面的 header 分片
  • PADDED: 0x8 是否有填充
  • PRIORITY: 0x20 是否有流依赖

2.04 PRIORITY Format 0x2

rfc-priority

1
+-+-------------------------------------------------------------+
2
|E| Stream Dependency (31) |
3
+-+-------------+-----------------------------------------------+
4
| Weight (8) |
5
+-+-------------+
  • E: 设置流是否为独占或依赖,PRIORITY 支持
  • Stream Dependency: 31bit 依赖的 stream identifier
  • Weight: 8bit 流的权重 1-255

解析示例

1
type PriorityParam struct {
2
StreamDep uint32
3
4
Exclusive bool
5
6
Weight uint8
7
}
8
type PriorityFrame struct {
9
FrameHeader
10
PriorityParam
11
}
12
func ParsePriorityFrame(fh *FrameHeader, payload []byte) (*PriorityFrame, error) {
13
if fh.StreamID == 0 {
14
return nil, connError{
15
ErrCodeProtocol,
16
"PRIORITY frame with stream ID 0",
17
}
18
}
19
var payloadLength = len(payload)
20
if payloadLength != 5 {
21
return nil, connError{
22
ErrCodeFrameSize,
23
fmt.Sprintf(
24
"PRIORITY frame payload size was %d; want 5",
25
payloadLength,
26
),
27
}
28
}
29
v := binary.BigEndian.Uint32(payload[:4])
30
// E 不处理
31
streamID := v & PadBit // mask off high bit
32
return &PriorityFrame{
33
FrameHeader: *fh,
34
PriorityParam: PriorityParam{
35
Weight: payload[4],
36
StreamDep: streamID,
37
Exclusive: streamID != v, // was high bit set?
38
},
39
}, nil
40
}

2.05 RST_STREAM Fromat 0x3

1
+---------------------------------------------------------------+
2
| Error Code (32) |
3
+---------------------------------------------------------------+

解析示例

1
type RSTStreamFrame struct {
2
FrameHeader
3
ErrCode ErrCode
4
}
5
func ParseRSTStreamFrame(fh *FrameHeader, p []byte) (*RSTStreamFrame, error) {
6
if len(p) != 4 {
7
return nil, ConnectionError(ErrCodeFrameSize)
8
}
9
if fh.StreamID == 0 {
10
return nil, ConnectionError(ErrCodeProtocol)
11
}
12
return &RSTStreamFrame{*fh, ErrCode(binary.BigEndian.Uint32(p[:4]))}, nil
13
}

2.06 SETTINGS Format 0x4

rfc-settings

1
+-------------------------------+
2
| Identifier (16) |
3
+-------------------------------+-------------------------------+
4
| Value (32) |
5
+---------------------------------------------------------------+
6
+-------------------------------+
7
| Identifier (16) |
8
+-------------------------------+-------------------------------+
9
| Value (32) |
10
+---------------------------------------------------------------+
11
...
  • Identifier: 16 bit 长度的 key。
  • Value: 32 bit 长度的 value。
  • 以每一对为存在并且数量不限,有什么样设置见 SettingValues

解析示例

1
type SettingID uint16
2
type Setting struct {
3
ID SettingID
4
Val uint32
5
}
6
7
// SettingFrame setting frame
8
type SettingFrame struct {
9
FrameHeader
10
Settings map[SettingID]Setting
11
}
12
func ParserSettings(header *FrameHeader, payload []byte) (*SettingFrame, error) {
13
settings := map[SettingID]Setting{}
14
num := len(payload) / 6
15
for i := 0; i < num; i++ {
16
id := SettingID(binary.BigEndian.Uint16(payload[i*6 : i*6+2]))
17
s := Setting{
18
ID: id,
19
Val: binary.BigEndian.Uint32(payload[i*6+2 : i*6+6]),
20
}
21
settings[id] = s
22
}
23
return &SettingFrame{
24
*header,
25
settings,
26
}, nil
27
}

Flags

  • Ack: 0x1

2.07 PUSH_PROMISE Format 0x5

rfc-push_promise

1
+---------------+
2
|Pad Length? (8)|
3
+-+-------------+-----------------------------------------------+
4
|R| Promised Stream ID (31) |
5
+-+-----------------------------+-------------------------------+
6
| Header Block Fragment (*) ...
7
+---------------------------------------------------------------+
8
| Padding (*) ...
9
+---------------------------------------------------------------+
  • Pad Length: 8bit 填充大小,需要 flag := PADDED
  • R: 1bit 保留位
  • Promised Stream ID: 31bit 主动推送的下一个流 ID

解析示例

1
type PushPromiseFrame struct {
2
FrameHeader
3
PromiseID uint32
4
HeaderFragBuf []byte // not owned
5
}
6
func ParsePushPromise(fh *FrameHeader, p []byte) (*PushPromiseFrame, error) {
7
var err error
8
pp := &PushPromiseFrame{
9
FrameHeader: *fh,
10
}
11
if pp.StreamID == 0 {
12
return nil, ConnectionError(ErrCodeProtocol)
13
}
14
var padLength uint8
15
if fh.Flags.Has(FlagPushPromisePadded) {
16
if p, padLength, err = readByte(p); err != nil {
17
return nil, err
18
}
19
}
20
p, pp.PromiseID, err = readUint32(p)
21
if err != nil {
22
return nil, err
23
}
24
pp.PromiseID = pp.PromiseID & PadBit
25
var payloadLength = len(p)
26
if int(padLength) > payloadLength {
27
return nil, ConnectionError(ErrCodeProtocol)
28
}
29
pp.HeaderFragBuf = p[:payloadLength-int(padLength)]
30
return pp, nil
31
}

flag

  • END_HEADERS: 0x4 与 headers 下的作用相同
  • PADDED: 0x8 是否有填充

2.08 PING Format 0x6

rfc-ping

1
+---------------------------------------------------------------+
2
| |
3
| Opaque Data (64) |
4
| |
5
+---------------------------------------------------------------+
  • Opaque Data: 64bit 任意内容定长,用于各种协议 ping 时的自定义

解析示例

1
type PingFrame struct {
2
FrameHeader
3
Data [8]byte
4
}
5
func parsePingFrame(fh FrameHeader, payload []byte) (*PingFrame, error) {
6
if len(payload) != 8 {
7
return nil, ConnectionError(ErrCodeFrameSize)
8
}
9
if fh.StreamID != 0 {
10
return nil, ConnectionError(ErrCodeProtocol)
11
}
12
f := &PingFrame{FrameHeader: fh}
13
copy(f.Data[:], payload)
14
return f, nil
15
}

2.09 GOAWAY Format 0x7

rfc-goaway

1
+-+-------------------------------------------------------------+
2
|R| Last-Stream-ID (31) |
3
+-+-------------------------------------------------------------+
4
| Error Code (32) |
5
+---------------------------------------------------------------+
6
| Additional Debug Data (*) |
7
+---------------------------------------------------------------+
  • Last-Stream-ID: 31bit 最后的流 ID。
  • Error Code: 32bit 错误编码。
  • Additional Debug Data: 任意调试数据。

解析示例

1
type GoAwayFrame struct {
2
FrameHeader
3
LastStreamID uint32
4
ErrCode ErrCode
5
DebugData []byte
6
}
7
func parseGoAwayFrame(fh FrameHeader, p []byte) (*GoAwayFrame, error) {
8
if fh.StreamID != 0 {
9
return nil, ConnectionError(ErrCodeProtocol)
10
}
11
if len(p) < 8 {
12
return nil, ConnectionError(ErrCodeFrameSize)
13
}
14
return &GoAwayFrame{
15
FrameHeader: fh,
16
LastStreamID: binary.BigEndian.Uint32(p[:4]) & PadBit,
17
ErrCode: ErrCode(binary.BigEndian.Uint32(p[4:8])),
18
debugData: p[8:],
19
}, nil
20
}

2.10 WINDOWUPDATE Format 0x8

rfc-window_update

1
+-+-------------------------------------------------------------+
2
|R| Window Size Increment (31) |
3
+-+-------------------------------------------------------------+
  • R: 1bit 占位必须为 0x0
  • Window Size Increment: 31bit 的 uint32

解析示例

1
func ParserWindowUpdate(header FrameHeader, payload []byte) uint32 {
2
return binary.BigEndian.Uint32(payload[:4]) & PadBit
3
}

2.11 CONTINUATION Format 0x9

rfc-continuation

1
+---------------------------------------------------------------+
2
| Header Block Fragment (*) ...
3
+---------------------------------------------------------------+
  • Header Block Fragment: header 的分片压缩字节

解析示例

1
type ContinuationFrame struct {
2
FrameHeader
3
HeaderFragBuf []byte
4
}
5
6
func parseContinuationFrame(fh FrameHeader, p []byte) (*ContinuationFrame, error) {
7
if fh.StreamID == 0 {
8
return nil, connError{ErrCodeProtocol, "CONTINUATION frame with stream ID 0"}
9
}
10
return &ContinuationFrame{fh, p}, nil
11
}

flags

  • END_HEADERS: header 分片结束

三、Frame 功能逻辑描述

四、Frame.StreamID 是怎么完成 HTTP2 的多路复用功能

五、hpack 压缩 Header

六、分段 Header

七、HTTP2 中的 body 发送模式

八、参考资料