优化音视频同步

1970f601 · xiongziliang · 0779a4be · 1970f601 · 1970f601 · 1970f601
Commit 1970f601 authored May 15, 2020 by xiongziliang
--- a/src/Common/Stamp.cpp
+++ b/src/Common/Stamp.cpp
@@ -42,8 +42,8 @@ void Stamp::setPlayBack(bool playback) {
    _playback = playback;
 }
-void Stamp::makeRelation(Stamp &other){
+void Stamp::syncTo(Stamp &other){
-    _related = &other;
+    _sync_master = &other;
 }
 void Stamp::revise(int64_t dts, int64_t pts, int64_t &dts_out, int64_t &pts_out,bool modifyStamp) {
@@ -53,23 +53,21 @@ void Stamp::revise(int64_t dts, int64_t pts, int64_t &dts_out, int64_t &pts_out,
        return;
    }
-    if(_related && _related->_last_dts){
+    if(_sync_master && _sync_master->_last_dts){
        //音视频dts当前时间差
-        int64_t dts_diff = _last_dts - _related->_last_dts;
+        int64_t dts_diff = _last_dts - _sync_master->_last_dts;
        if(ABS(dts_diff) < 5000){
            //如果绝对时间戳小于5秒，那么说明他们的起始时间戳是一致的，那么强制同步
            _last_relativeStamp = _relativeStamp;
-            _relativeStamp = _related->_relativeStamp + dts_diff;
+            _relativeStamp = _sync_master->_relativeStamp + dts_diff;
-            dts_out += dts_diff;
-            pts_out += dts_diff;
-//            DebugL << "音视频同步事件差:" << dts_diff;
        }
        //下次不用再强制同步
-        _related = nullptr;
+        _sync_master = nullptr;
    }
-    if(dts_out < 0){
+    if(dts_out < 0 || dts_out < _last_relativeStamp){
-        //相对时间戳小于0，那么说明是同步时间戳导致的,在这个过渡期内，我们一直返回上次的结果(目的是为了防止时间戳回退)
+        //相对时间戳小于0，或者小于上次的时间戳，
+        //那么说明是同步时间戳导致的,在这个过渡期内，我们一直返回上次的结果(目的是为了防止时间戳回退)
        pts_out = _last_relativeStamp + (pts_out - dts_out);
        dts_out = _last_relativeStamp;
    }

--- a/src/Common/Stamp.h
+++ b/src/Common/Stamp.h
@@ -69,9 +69,10 @@ public:
    void setPlayBack(bool playback = true);
    /**
-     * 产生关联,用于音视频同步用
+     * 音视频同步用，音频应该同步于视频(只修改音频时间戳)
+     * 因为音频时间戳修改后不影响播放速度
     */
-    void makeRelation(Stamp &other);
+    void syncTo(Stamp &other);
 private:
    void revise_l(int64_t dts, int64_t pts, int64_t &dts_out, int64_t &pts_out,bool modifyStamp = false);
@@ -81,7 +82,7 @@ private:
    int64_t _last_dts = 0;
    SmoothTicker _ticker;
    bool _playback = false;
-    Stamp *_related = nullptr;
+    Stamp *_sync_master = nullptr;
 };
 //dts生成器，

--- a/src/Extension/Frame.cpp
+++ b/src/Extension/Frame.cpp
@@ -76,8 +76,8 @@ Frame::Ptr Frame::getCacheAbleFrame(const Frame::Ptr &frame){
 }
 #define SWITCH_CASE(codec_id) case codec_id : return #codec_id
-const char *CodecInfo::getCodecName() {
+const char *getCodecName(CodecId codecId) {
-    switch (getCodecId()) {
+    switch (codecId) {
        SWITCH_CASE(CodecH264);
        SWITCH_CASE(CodecH265);
        SWITCH_CASE(CodecAAC);
@@ -88,8 +88,8 @@ const char *CodecInfo::getCodecName() {
    }
 }
-TrackType CodecInfo::getTrackType(){
+TrackType getTrackType(CodecId codecId){
-    switch (getCodecId()){
+    switch (codecId){
        case CodecH264:
        case CodecH265: return TrackVideo;
        case CodecAAC:
@@ -100,4 +100,11 @@ TrackType CodecInfo::getTrackType(){
    }
 }
+const char *CodecInfo::getCodecName() {
+    return mediakit::getCodecName(getCodecId());
+}
+TrackType CodecInfo::getTrackType() {
+    return mediakit::getTrackType(getCodecId());
+}
 }//namespace mediakit
--- a/src/Extension/Frame.h
+++ b/src/Extension/Frame.h
@@ -41,6 +41,16 @@ typedef enum {
 } TrackType;
 /**
+ * 获取编码器名称
+ */
+const char *getCodecName(CodecId codecId);
+/**
+ * 获取音视频类型
+ */
+TrackType getTrackType(CodecId codecId);
+/**
 * 编码信息的抽象接口
 */
 class CodecInfo {

--- a/src/Record/MP4Muxer.cpp
+++ b/src/Record/MP4Muxer.cpp
@@ -134,6 +134,26 @@ static uint8_t getObject(CodecId codecId){
    }
 }
+void MP4Muxer::stampSync(){
+    if(_codec_to_trackid.size() < 2){
+        return;
+    }
+    Stamp *audio = nullptr, *video = nullptr;
+    for(auto &pr : _codec_to_trackid){
+        switch (getTrackType((CodecId) pr.first)){
+            case TrackAudio : audio = &pr.second.stamp; break;
+            case TrackVideo : video = &pr.second.stamp; break;
+            default : break;
+        }
+    }
+    if(audio && video){
+        //音频时间戳同步于视频，因为音频时间戳被修改后不影响播放
+        audio->syncTo(*video);
+    }
+}
 void MP4Muxer::addTrack(const Track::Ptr &track) {
    auto mp4_object = getObject(track->getCodecId());
    if (!mp4_object) {
@@ -261,6 +281,9 @@ void MP4Muxer::addTrack(const Track::Ptr &track) {
        default: WarnL << "MP4录制不支持该编码格式:" << track->getCodecName(); break;
    }
+    //尝试音视频同步
+    stampSync();
 }
 }//namespace mediakit

--- a/src/Record/MP4Muxer.h
+++ b/src/Record/MP4Muxer.h
@@ -45,13 +45,14 @@ public:
 private:
    void openMP4();
    void closeMP4();
+    void stampSync();
 private:
-    struct track_info{
+    struct track_info {
        int track_id = -1;
        Stamp stamp;
    };
-    unordered_map<int,track_info> _codec_to_trackid;
+    unordered_map<int, track_info> _codec_to_trackid;
    List<Frame::Ptr> _frameCached;
    bool _started = false;
    bool _have_video = false;

--- a/src/Record/TsMuxer.cpp
+++ b/src/Record/TsMuxer.cpp
@@ -23,6 +23,26 @@ TsMuxer::~TsMuxer() {
    uninit();
 }
+void TsMuxer::stampSync(){
+    if(_codec_to_trackid.size() < 2){
+        return;
+    }
+    Stamp *audio = nullptr, *video = nullptr;
+    for(auto &pr : _codec_to_trackid){
+        switch (getTrackType((CodecId) pr.first)){
+            case TrackAudio : audio = &pr.second.stamp; break;
+            case TrackVideo : video = &pr.second.stamp; break;
+            default : break;
+        }
+    }
+    if(audio && video){
+        //音频时间戳同步于视频，因为音频时间戳被修改后不影响播放
+        audio->syncTo(*video);
+    }
+}
 void TsMuxer::addTrack(const Track::Ptr &track) {
    switch (track->getCodecId()) {
        case CodecH264: {
@@ -52,9 +72,11 @@ void TsMuxer::addTrack(const Track::Ptr &track) {
            break;
        }
-        default:
+        default: WarnL << "mpeg-ts 不支持该编码格式,已忽略:" << track->getCodecName(); break;
-            break;
    }
+    //尝试音视频同步
+    stampSync();
 }
 void TsMuxer::inputFrame(const Frame::Ptr &frame) {

--- a/src/Record/TsMuxer.h
+++ b/src/Record/TsMuxer.h
@@ -17,37 +17,59 @@
 #include "Util/File.h"
 #include "Common/MediaSink.h"
 #include "Common/Stamp.h"
 using namespace toolkit;
 namespace mediakit {
+//该类用于产生MPEG-TS
 class TsMuxer : public MediaSinkInterface {
 public:
    TsMuxer();
    virtual ~TsMuxer();
+    /**
+     * 添加音视频轨道
+     */
    void addTrack(const Track::Ptr &track) override;
+    /**
+     * 重置音视频轨道
+     */
    void resetTracks() override;
+    /**
+     * 输入帧数据
+     */
    void inputFrame(const Frame::Ptr &frame) override;
 protected:
+    /**
+     * 输出mpegts数据回调
+     * @param packet mpegts数据
+     * @param bytes mpegts数据长度
+     * @param timestamp 时间戳，单位毫秒
+     * @param is_idr_fast_packet 是否为关键帧的第一个TS包，用于确保ts切片第一帧为关键帧
+     */
    virtual void onTs(const void *packet, int bytes,uint32_t timestamp,bool is_idr_fast_packet) = 0;
 private:
    void init();
    void uninit();
+    //音视频时间戳同步用
+    void stampSync();
 private:
-    void  *_context = nullptr;
+    void *_context = nullptr;
-    char *_tsbuf[188];
+    char _tsbuf[188];
    uint32_t _timestamp = 0;
+    struct track_info {
-    struct track_info{
        int track_id = -1;
        Stamp stamp;
    };
-    unordered_map<int,track_info> _codec_to_trackid;
+    unordered_map<int, track_info> _codec_to_trackid;
    List<Frame::Ptr> _frameCached;
    bool _is_idr_fast_packet = false;
    bool _have_video = false;
 };
 }//namespace mediakit
 #endif //TSMUXER_H
\ No newline at end of file
--- a/src/Rtmp/FlvMuxer.cpp
+++ b/src/Rtmp/FlvMuxer.cpp
@@ -52,7 +52,7 @@ void FlvMuxer::start(const EventPoller::Ptr &poller,const RtmpMediaSource::Ptr &
    });
    //音频同步于视频
-    _stamp[0].makeRelation( _stamp[1]);
+    _stamp[0].syncTo(_stamp[1]);
    _ring_reader->setReadCB([weakSelf](const RtmpMediaSource::RingDataType &pkt){
        auto strongSelf = weakSelf.lock();
        if(!strongSelf){

--- a/src/Rtmp/RtmpSession.cpp
+++ b/src/Rtmp/RtmpSession.cpp
@@ -267,7 +267,7 @@ void RtmpSession::sendPlayResponse(const string &err,const RtmpMediaSource::Ptr 
    });
    //音频同步于视频
-    _stamp[0].makeRelation( _stamp[1]);
+    _stamp[0].syncTo(_stamp[1]);
    _pRingReader = src->getRing()->attach(getPoller());
    weak_ptr<RtmpSession> weakSelf = dynamic_pointer_cast<RtmpSession>(shared_from_this());
    _pRingReader->setReadCB([weakSelf](const RtmpMediaSource::RingDataType &pkt) {