ffmpeg7.0 flv支持hdr
自从ffmpeg6.0应用enhance rtmp支持h265/av1的flv格式后，7.0迎来了flv的hdr能力。本文介绍ffmpeg7.0如何支持hdr in flv。
如果对enhance rtmp如何支持h265不了解，推荐详解Enhanced-RTMP支持H.265
1. enhance rtmp关于hdr
文档enhance-rtmp-v2.md中，metadata-frame章节规定相关支持HDR部分。
定义一个新的VideoPacketType.Metadata，其内容为AMF格式的metadata(AMF是一种简便的描述格式，非常节省内存)，具体内部有HDR相关的colorInfo数据(每次hdr参数更新的时候，就发送该metadata)。
type ColorInfo = {
colorConfig: {
// number of bits used to record the color channels for each pixel
bitDepth: number, // SHOULD be 8, 10 or 12
//
// colorPrimaries, transferCharacteristics and matrixCoefficients are defined
// in ISO/IEC 23091-4/ITU-T H.273. The values are an index into
// respective tables which are described in “Colour primaries”,
// "Transfer characteristics" and "Matrix coefficients" sections.
// It is RECOMMENDED to provide these values.
//
// indicates the chromaticity coordinates of the source color primaries
colorPrimaries: number, // enumeration [0-255]
// opto-electronic transfer characteristic function (ex. PQ, HLG)
transferCharacteristics: number, // enumeration [0-255]
// matrix coefficients used in deriving luma and chroma signals
matrixCoefficients: number, // enumeration [0-255]
},
hdrCll: {
//
// maximum value of the frame average light level
// (in 1 cd/m2) of the entire playback sequence
//
maxFall: number, // [0.0001-10000]
//
// maximum light level of any single pixel (in 1 cd/m2)
// of the entire playback sequence
//
maxCLL: number, // [0.0001-10000]
},
//
// The hdrMdcv object defines mastering display (i.e., where
// creative work is done during the mastering process) color volume (a.k.a., mdcv)
// metadata which describes primaries, white point and min/max luminance. The
// hdrMdcv object SHOULD be provided.
//
// Specification of the metadata along with its ranges adhere to the
// ST 2086:2018 - SMPTE Standard (except for minLuminance see
// comments below)
//
hdrMdcv: {
//
// Mastering display color volume (mdcv) xy Chromaticity Coordinates within CIE
// 1931 color space.
//
// Values SHALL be specified with four decimal places. The x coordinate SHALL
// be in the range [0.0001, 0.7400]. The y coordinate SHALL be
// in the range [0.0001, 0.8400].
//
redX: number,
redY: number,
greenX: number,
greenY: number,
blueX: number,
blueY: number,
whitePointX: number,
whitePointY: number,
//
// max/min display luminance of the mastering display (in 1 cd/m2 ie. nits)
//
// note: ST 2086:2018 - SMPTE Standard specifies minimum display mastering
// luminance in multiples of 0.0001 cd/m2.
//
// For consistency we specify all values
// in 1 cd/m2. Given that a hypothetical perfect screen has a peak brightness
// of 10,000 nits and a black level of .0005 nits we do not need to
// switch units to 0.0001 cd/m2 to increase resolution on the lower end of the
// minLuminance property. The ranges (in nits) mentioned below suffice
// the theoretical limit for Mastering Reference Displays and adhere to the
// SMPTE ST 2084 standard (a.k.a., PQ) which is capable of representing full gamut
// of luminance level.
//
maxLuminance: number, // [5-10000]
minLuminance: number, // [0.0001-5]
},
}
2. flv mux
在libavformat/flvenc.c中，新增flv_write_metadata_packet函数，用于添加metadata。
flv的数据结构:
typedef struct FLVContext {
.....
int metadata_pkt_written;//0: 还未写过，需要写；1: 写过了，不需要写。
} FLVContext;
在函数flv_write_metadata_packet中实现，见注释：
static void flv_write_metadata_packet(AVFormatContext *s, AVCodecParameters *par, unsigned int ts) {
FLVContext *flv = s->priv_data;
//不需要写入，就返回；可以通过设置这个标志变量来使能/去使能更新写metadata
if (flv->metadata_pkt_written)
return;
//支持h265, av1, vp9
if (par->codec_id == AV_CODEC_ID_HEVC || par->codec_id == AV_CODEC_ID_AV1 || par->codec_id == AV_CODEC_ID_VP9) {
......
//写入tag头，标识其为视频
avio_w8(pb, FLV_TAG_TYPE_VIDEO); //write video tag type
metadata_size_pos = avio_tell(pb);
avio_wb24(pb, 0 + flags_size);
put_timestamp(pb, ts); //ts = pkt->dts, gen
avio_wb24(pb, flv->reserved);
//根据enhance rtmp标志，写入FLV_IS_EX_HEADER标识，和fourCC的字段(hvc1 or av01 or vp09)
if (par->codec_id == AV_CODEC_ID_HEVC) {
avio_w8(pb, FLV_IS_EX_HEADER | PacketTypeMetadata| FLV_FRAME_VIDEO_INFO_CMD); // ExVideoTagHeader mode with PacketTypeMetadata
avio_write(pb, "hvc1", 4);
} else if (par->codec_id == AV_CODEC_ID_AV1 || par->codec_id == AV_CODEC_ID_VP9) {
avio_w8(pb, FLV_IS_EX_HEADER | PacketTypeMetadata| FLV_FRAME_VIDEO_INFO_CMD);
avio_write(pb, par->codec_id == AV_CODEC_ID_AV1 ? "av01" : "vp09", 4);
}
//下面为写入AMF格式的hdr相关的colorInfo数据
avio_w8(pb, AMF_DATA_TYPE_STRING);
put_amf_string(pb, "colorInfo");
avio_w8(pb, AMF_DATA_TYPE_OBJECT);
put_amf_string(pb, "colorConfig"); // colorConfig
avio_w8(pb, AMF_DATA_TYPE_OBJECT);
if (par->color_trc != AVCOL_TRC_UNSPECIFIED &&
par->color_trc < AVCOL_TRC_NB) {
put_amf_string(pb, "transferCharacteristics"); // color_trc
put_amf_double(pb, par->color_trc);
}
if (par->color_space != AVCOL_SPC_UNSPECIFIED &&
par->color_space < AVCOL_SPC_NB) {
put_amf_string(pb, "matrixCoefficients"); // colorspace
put_amf_double(pb, par->color_space);
}
if (par->color_primaries != AVCOL_PRI_UNSPECIFIED &&
par->color_primaries < AVCOL_PRI_NB) {
put_amf_string(pb, "colorPrimaries"); // color_primaries
put_amf_double(pb, par->color_primaries);
}
put_amf_string(pb, "");
avio_w8(pb, AMF_END_OF_OBJECT);
if (lightMetadata) {
put_amf_string(pb, "hdrCll");
avio_w8(pb, AMF_DATA_TYPE_OBJECT);
put_amf_string(pb, "maxFall");
put_amf_double(pb, lightMetadata->MaxFALL);
put_amf_string(pb, "maxCLL");
put_amf_double(pb, lightMetadata->MaxCLL);
put_amf_string(pb, "");
avio_w8(pb, AMF_END_OF_OBJECT);
}
if (displayMetadata && (displayMetadata->has_primaries || displayMetadata->has_luminance)) {
put_amf_string(pb, "hdrMdcv");
avio_w8(pb, AMF_DATA_TYPE_OBJECT);
if (displayMetadata->has_primaries) {
put_amf_string(pb, "redX");
put_amf_double(pb, av_q2d(displayMetadata->display_primaries[0][0]));
put_amf_string(pb, "redY");
put_amf_double(pb, av_q2d(displayMetadata->display_primaries[0][1]));
put_amf_string(pb, "greenX");
put_amf_double(pb, av_q2d(displayMetadata->display_primaries[1][0]));
put_amf_string(pb, "greenY");
put_amf_double(pb, av_q2d(displayMetadata->display_primaries[1][1]));
put_amf_string(pb, "blueX");
put_amf_double(pb, av_q2d(displayMetadata->display_primaries[2][0]));
put_amf_string(pb, "blueY");
put_amf_double(pb, av_q2d(displayMetadata->display_primaries[2][1]));
put_amf_string(pb, "whitePointX");
put_amf_double(pb, av_q2d(displayMetadata->white_point[0]));
put_amf_string(pb, "whitePointY");
put_amf_double(pb, av_q2d(displayMetadata->white_point[1]));
}
if (displayMetadata->has_luminance) {
put_amf_string(pb, "maxLuminance");
put_amf_double(pb, av_q2d(displayMetadata->max_luminance));
put_amf_string(pb, "minLuminance");
put_amf_double(pb, av_q2d(displayMetadata->min_luminance));
}
put_amf_string(pb, "");
avio_w8(pb, AMF_END_OF_OBJECT);
}
put_amf_string(pb, "");
avio_w8(pb, AMF_END_OF_OBJECT);
flv->metadata_pkt_written = 1;//标识写过了
}
}
其中HDR的数据来源为AVCodecParameters *par数据结构中的内容：
typedef struct AVCodecParameters {
....
/**
* Video only. Additional colorspace characteristics.
*/
enum AVColorRange color_range;
enum AVColorPrimaries color_primaries;
enum AVColorTransferCharacteristic color_trc;
enum AVColorSpace color_space;
enum AVChromaLocation chroma_location;
/*
* 类型: AV_PKT_DATA_CONTENT_LIGHT_LEVEL, 数据: AVContentLightMetadata* lightMetadata
* 类型: AV_PKT_DATA_MASTERING_DISPLAY_METADATA, 数据: AVMasteringDisplayMetadata *displayMetadata
*/
AVPacketSideData *coded_side_data;
....
}
3. flv demux
解析函数在libavformat/flvdec.c文件中，函数amf_parse_object中。
static int amf_parse_object(AVFormatContext *s, AVStream *astream,
AVStream *vstream, const char *key,
int64_t max_pos, int depth)
{
FLVMetaVideoColor *meta_video_color = flv->metaVideoColor;
......
if (meta_video_color) {
if (amf_type == AMF_DATA_TYPE_NUMBER ||
amf_type == AMF_DATA_TYPE_BOOL) {
if (!strcmp(key, "colorPrimaries")) {
meta_video_color->primaries = num_val;
} else if (!strcmp(key, "transferCharacteristics")) {
meta_video_color->transfer_characteristics = num_val;
} else if (!strcmp(key, "matrixCoefficients")) {
meta_video_color->matrix_coefficients = num_val;
} else if (!strcmp(key, "maxFall")) {
meta_video_color->max_fall = num_val;
} else if (!strcmp(key, "maxCLL")) {
meta_video_color->max_cll = num_val;
} else if (!strcmp(key, "redX")) {
meta_video_color->mastering_meta.r_x = num_val;
} else if (!strcmp(key, "redY")) {
meta_video_color->mastering_meta.r_y = num_val;
} else if (!strcmp(key, "greenX")) {
meta_video_color->mastering_meta.g_x = num_val;
} else if (!strcmp(key, "greenY")) {
meta_video_color->mastering_meta.g_y = num_val;
} else if (!strcmp(key, "blueX")) {
meta_video_color->mastering_meta.b_x = num_val;
} else if (!strcmp(key, "blueY")) {
meta_video_color->mastering_meta.b_y = num_val;
} else if (!strcmp(key, "whitePointX")) {
meta_video_color->mastering_meta.white_x = num_val;
} else if (!strcmp(key, "whitePointY")) {
meta_video_color->mastering_meta.white_y = num_val;
} else if (!strcmp(key, "maxLuminance")) {
meta_video_color->mastering_meta.max_luminance = num_val;
} else if (!strcmp(key, "minLuminance")) {
meta_video_color->mastering_meta.min_luminance = num_val;
}
}
}
......
}
4. 感谢Zhu Pengfei的提交
Author: Zhu Pengfei <411294962@qq.com>
Date: Mon Mar 4 21:52:04 2024 +0800
avformat/flvenc: support enhanced flv PacketTypeMetadata
Signed-off-by: Zhu Pengfei <411294962@qq.com>
Signed-off-by: Steven Liu <lq@chinaffmpeg.org>
``