WebRTC H264组包流程
int VCMSessionInfo::InsertPacket(const VCMPacket& packet,
uint8_t* frame_buffer,
VCMDecodeErrorMode decode_error_mode,
const FrameData& frame_data)1、如果是空包（如fec包）
则更新空包最大和最小rtp序列号
void VCMSessionInfo::InformOfEmptyPacket(uint16_t seq_num) {
// Empty packets may be FEC or filler packets. They are sequential and
// follow the data packets, therefore, we should only keep track of the high
// and low sequence numbers and may assume that the packets in between are
// empty packets belonging to the same frame (timestamp).
if (empty_seq_num_high_ == -1)
empty_seq_num_high_ = seq_num;
else
empty_seq_num_high_ = LatestSequenceNumber(seq_num, empty_seq_num_high_);
if (empty_seq_num_low_ == -1 || IsNewerSequenceNumber(empty_seq_num_low_,
seq_num))
empty_seq_num_low_ = seq_num;
}
2、从集合typedef std::list PacketList;（包已按rtp序列号由小到大排好序）
尾部向前 查找待插包packet的位置
ReversePacketIterator rit = packets_.rbegin();
for (; rit != packets_.rend(); ++rit)
if (LatestSequenceNumber(packet.seqNum, (*rit).seqNum) == packet.seqNum)
break;
如果packetList中 已存在的包是
packet序号 1 2 4 5
如果此时待插入的包序号是3，那么
rit指针将指向序号为2的这个包
3、更新一帧 的 第一个和最后一个包的序号
if (packet.codec == kVideoCodecH264) {
frame_type_ = packet.frameType;
if (packet.isFirstPacket &&
(first_packet_seq_num_ == -1 ||
IsNewerSequenceNumber(first_packet_seq_num_, packet.seqNum))) {
first_packet_seq_num_ = packet.seqNum;
}
if (packet.markerBit &&
(last_packet_seq_num_ == -1 ||
IsNewerSequenceNumber(packet.seqNum, last_packet_seq_num_))) {
last_packet_seq_num_ = packet.seqNum;
}
}
4、将待插入的包插入到packets列表
//插入后packet_list_it 指向新插入的元素
PacketIterator packet_list_it = packets_.insert(rit.base(), packet);
此时packets_中包顺序为：
5、将包插入到帧frame_buffer中
//frame_buffer指向一帧的起始数据地址，packet_list_it是待插入包
//将包packet_it插入到帧frame_buffer对应的位置中（也就是frame_buffer中buffer对应位置的指针指向packet_it包数据，通过指针共享同一数据，避免拷贝）
size_t VCMSessionInfo::InsertBuffer(uint8_t* frame_buffer,
PacketIterator packet_it) {
VCMPacket& packet = *packet_it;
PacketIterator it;
// Calculate the offset into the frame buffer for this packet.
size_t offset = 0;
for (it = packets_.begin(); it != packet_it; ++it)
offset += (*it).sizeBytes;
// Set the data pointer to pointing to the start of this packet in the
// frame buffer.
const uint8_t* packet_buffer = packet.dataPtr;
packet.dataPtr = frame_buffer + offset;
// We handle H.264 STAP-A packets in a special way as we need to remove the
// two length bytes between each NAL unit, and potentially add start codes.
const size_t kH264NALHeaderLengthInBytes = 1;
const size_t kLengthFieldLength = 2;
if (packet.codecSpecificHeader.codec == kRtpVideoH264 &&
packet.codecSpecificHeader.codecHeader.H264.packetization_type ==
kH264StapA) {
size_t required_length = 0;
const uint8_t* nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes;
while (nalu_ptr < packet_buffer + packet.sizeBytes) {
size_t length = BufferToUWord16(nalu_ptr);
required_length +=
length + (packet.insertStartCode ? kH264StartCodeLengthBytes : 0);
nalu_ptr += kLengthFieldLength + length;
}
ShiftSubsequentPackets(packet_it, required_length);
nalu_ptr = packet_buffer + kH264NALHeaderLengthInBytes;
uint8_t* frame_buffer_ptr = frame_buffer + offset;
while (nalu_ptr < packet_buffer + packet.sizeBytes) {
size_t length = BufferToUWord16(nalu_ptr);
nalu_ptr += kLengthFieldLength;
frame_buffer_ptr += Insert(nalu_ptr,
length,
packet.insertStartCode,
const_cast<uint8_t*>(frame_buffer_ptr));
nalu_ptr += length;
}
packet.sizeBytes = required_length;
return packet.sizeBytes;
}
ShiftSubsequentPackets(
packet_it,
packet.sizeBytes +
(packet.insertStartCode ? kH264StartCodeLengthBytes : 0));
packet.sizeBytes = Insert(packet_buffer,
packet.sizeBytes,
packet.insertStartCode,
const_cast<uint8_t*>(packet.dataPtr));
return packet.sizeBytes;
}
本文福利， 免费领取C++音视频学习资料包、技术视频，内容包括（音视频开发，面试题，FFmpeg ，webRTC ，rtmp ，hls ，rtsp ，ffplay ，srs）↓↓↓↓↓↓见下面↓↓文章底部点击免费领取↓↓
函数流程分析：
（1）计算packets_中所有packet的大小和
size_t offset = 0;
for (it = packets_.begin(); it != packet_it; ++it)
offset += (*it).sizeBytes;
（2）帧给包3腾出空间和在帧中插入包
void VCMSessionInfo::ShiftSubsequentPackets(PacketIterator it,
int steps_to_shift) {
++it;
if (it == packets_.end())
return;
uint8_t* first_packet_ptr = const_cast<uint8_t*>((*it).dataPtr);
int shift_length = 0;
// Calculate the total move length and move the data pointers in advance.
for (; it != packets_.end(); ++it) {
shift_length += (*it).sizeBytes;
if ((*it).dataPtr != NULL)
(*it).dataPtr += steps_to_shift;
}
memmove(first_packet_ptr + steps_to_shift, first_packet_ptr, shift_length);
}
也就是统计帧中 包3 后边的包总共有多少字节数，然后将后续的包（4和5包）指向 帧 的其他位置，就是为了给包3腾出空间
插入前：
size_t VCMSessionInfo::Insert(const uint8_t* buffer,
size_t length,
bool insert_start_code,
uint8_t* frame_buffer) {
if (insert_start_code) {
const unsigned char startCode[] = {0, 0, 0, 1};
memcpy(frame_buffer, startCode, kH264StartCodeLengthBytes);
}
memcpy(frame_buffer + (insert_start_code ? kH264StartCodeLengthBytes : 0),
buffer,
length);
length += (insert_start_code ? kH264StartCodeLengthBytes : 0);
return length;
}
6、判断是否有完整的帧，也就是组包完成
void VCMSessionInfo::UpdateCompleteSession() {
if (HaveFirstPacket() && HaveLastPacket()) { //帧中有第一个包和最后一个包，最后一个包是根据rtp头中的m=1得到的
// Do we have all the packets in this session?
bool complete_session = true;
PacketIterator it = packets_.begin();
PacketIterator prev_it = it;
++it;
for (; it != packets_.end(); ++it) {
if (!InSequence(it, prev_it)) { //前后2个包的rtp序列号相差1
complete_session = false;
break;
}
prev_it = it;
}
complete_ = complete_session;
}
}
7、
if (decode_error_mode == kWithErrors)
decodable_ = true;
else if (decode_error_mode == kSelectiveErrors)
UpdateDecodableSession(frame_data);
如果VCMDecodeErrorMode decode_error_mode = decode_error_mode
就置可解码状态为decodable_ = true;
void VCMSessionInfo::UpdateDecodableSession(const FrameData& frame_data) {
// Irrelevant if session is already complete or decodable
if (complete_ || decodable_)
return;
// TODO(agalusza): Account for bursty loss.
// TODO(agalusza): Refine these values to better approximate optimal ones.
// Do not decode frames if the RTT is lower than this.
const int64_t kRttThreshold = 100;
// Do not decode frames if the number of packets is between these two
// thresholds.
const float kLowPacketPercentageThreshold = 0.2f;
const float kHighPacketPercentageThreshold = 0.8f;
if (frame_data.rtt_ms < kRttThreshold
|| frame_type_ == kVideoFrameKey
|| !HaveFirstPacket()
|| (NumPackets() <= kHighPacketPercentageThreshold
* frame_data.rolling_average_packets_per_frame
&& NumPackets() > kLowPacketPercentageThreshold
* frame_data.rolling_average_packets_per_frame))
return;
decodable_ = true;
}
如果VCMDecodeErrorMode decode_error_mode = kSelectiveErrors
，此时帧是可解码状态或者组包完成状态，那么直接返回
此时是关键帧或者rtt_ms < 100或者组包的总数在一个范围内也直接返回
否则 decodable_ = true
packet结构说明：
void VCMPacket::CopyCodecSpecifics(const RTPVideoHeader& videoHeader) {
if (markerBit) {
codecSpecificHeader.rotation = videoHeader.rotation;
}
switch (videoHeader.codec) {
case kRtpVideoVp8:
// Handle all packets within a frame as depending on the previous packet
// TODO(holmer): This should be changed to make fragments independent
// when the VP8 RTP receiver supports fragments.
if (isFirstPacket && markerBit)
completeNALU = kNaluComplete;
else if (isFirstPacket)
completeNALU = kNaluStart;
else if (markerBit)
completeNALU = kNaluEnd;
else
completeNALU = kNaluIncomplete;
codec = kVideoCodecVP8;
return;
case kRtpVideoH264:
isFirstPacket = videoHeader.isFirstPacket;
if (isFirstPacket)
insertStartCode = true;
if (isFirstPacket && markerBit) {
completeNALU = kNaluComplete;
} else if (isFirstPacket) {
completeNALU = kNaluStart;
} else if (markerBit) {
completeNALU = kNaluEnd;
} else {
completeNALU = kNaluIncomplete;
}
codec = kVideoCodecH264;
return;
case kRtpVideoGeneric:
case kRtpVideoNone:
codec = kVideoCodecUnknown;
return;
}
}
**VCMSessionInfo::MakeDecodable说明：**寻找可解码的nalu
size_t VCMSessionInfo::MakeDecodable() {
size_t return_length = 0;
if (packets_.empty()) {
return 0;
}
PacketIterator it = packets_.begin();
// Make sure we remove the first NAL unit if it's not decodable.
if ((*it).completeNALU == kNaluIncomplete ||
(*it).completeNALU == kNaluEnd) {
PacketIterator nalu_end = FindNaluEnd(it);
return_length += DeletePacketData(it, nalu_end);
it = nalu_end;
}
PacketIterator prev_it = it;
// Take care of the rest of the NAL units.
for (; it != packets_.end(); ++it) {
bool start_of_nalu = ((*it).completeNALU == kNaluStart ||
(*it).completeNALU == kNaluComplete);
if (!start_of_nalu && !InSequence(it, prev_it)) {
// Found a sequence number gap due to packet loss.
PacketIterator nalu_end = FindNaluEnd(it);
return_length += DeletePacketData(it, nalu_end);
it = nalu_end;
}
prev_it = it;
}
return return_length;
}
附nalu组包分片：
h264 的组合封包模式STAP-A的组包流程和上述差不多
如有一个 H.264 的 NALU 是这样的:
[00 0000 01 67 42 A0 1E 23 56 0E 2F ... ]
这是一个序列参数集 NAL 单元. [00 00 00 01] 是四个字节的开始码, 67 是 NALU 头, 42 开始的数据是 NALU 内容.封装成 RTP 包将如下:
[ RTPHeader ] [ 67 42 A0 1E 23 56 0E 2F ]
例如： 如有一个 H.264 的 NALU 是这样的:
[00 00 00 01 67 42 A0 1E 23 56 0E 2F...]
[00 00 00 01 68 42 12 2B 6A D6 FF E4...]
封装成RTP包，结果如下：
[RTP Header] [78(STAP-A头，一个字节)] [第一个NALU长度，两个字节] [67 42 A0 1E 23 56 0E 2F…] [第二个NALU长度，两个字节] [68 42 12 2B 6A D6 FF E4…]
本文福利， 免费领取C++音视频学习资料包、技术视频，内容包括（音视频开发，面试题，FFmpeg ，webRTC ，rtmp ，hls ，rtsp ，ffplay ，srs）↓↓↓↓↓↓见下面↓↓文章底部点击免费领取↓↓
本文含有隐藏内容，请 开通VIP 后查看