完善simulcast接收及处理

a91f4d29 · ziyue · dcb91e3b · a91f4d29 · a91f4d29 · a91f4d29
Commit a91f4d29 authored Jun 24, 2021 by ziyue
--- a/webrtc/WebRtcTransport.cpp
+++ b/webrtc/WebRtcTransport.cpp
@@ -761,35 +761,38 @@ void WebRtcTransportImp::onRtp_l(const char *buf, size_t len, bool rtx) {
            return;
        }
 #endif
+        auto &ref = info->receiver[ssrc];
        if (!rtx) {
            //统计rtp接受情况，便于生成nack rtcp包
            info->nack_ctx[ssrc].received(seq);
            //时间戳转换成毫秒
            auto stamp_ms = ntohl(rtp->stamp) * uint64_t(1000) / info->plan_rtp->sample_rate;
            //统计rtp收到的情况，好做rr汇报
-            auto &ref = info->rtcp_context_recv[ssrc];
+            auto &cxt_ref = info->rtcp_context_recv[ssrc];
-            if (!ref) {
+            if (!cxt_ref) {
-                ref = std::make_shared<RtcpContext>(info->plan_rtp->sample_rate, true);
+                cxt_ref = std::make_shared<RtcpContext>(info->plan_rtp->sample_rate, true);
            }
-            ref->onRtp(seq, stamp_ms, len);
+            cxt_ref->onRtp(seq, stamp_ms, len);
            //修改ext id至统一
-            changeRtpExtId(*info, rtp, true, false);
+            string rid;
-        }
+            changeRtpExtId(*info, rtp, true, false, &rid);
-        //解析并排序rtp
+            if (!ref) {
-        auto &ref = info->receiver[ssrc];
+                ref = std::make_shared<RtpReceiverImp>([info, this, rid](RtpPacket::Ptr rtp) mutable {
-        if (!ref) {
+                    onSortedRtp(*info, rid, std::move(rtp));
-            ref = std::make_shared<RtpReceiverImp>([info, this](RtpPacket::Ptr rtp) mutable {
+                });
-                onSortedRtp(*info, std::move(rtp));
+                info->nack_ctx[ssrc].setOnNack([info, this, ssrc](const FCI_NACK &nack) mutable {
-            });
+                    onSendNack(*info, nack, ssrc);
-            info->nack_ctx[ssrc].setOnNack([info, this, ssrc](const FCI_NACK &nack) mutable {
+                });
-                onSendNack(*info, nack, ssrc);
+                //recv simulcast ssrc --> RtpPayloadInfo
-            });
+                _rtp_info_ssrc[ssrc] = std::make_pair(false, info);
-            //recv simulcast ssrc --> RtpPayloadInfo
+                InfoL << "receive rtp of ssrc:" << ssrc;
-            _rtp_info_ssrc[ssrc] = std::make_pair(false, info);
+            }
-            InfoL << "receive rtp of ssrc:" << ssrc;
        }
+        //解析并排序rtp
+        assert(ref);
        ref->inputRtp(info->media->type, info->plan_rtp->sample_rate, (uint8_t *) buf, len);
        return;
    }
@@ -809,9 +812,9 @@ void WebRtcTransportImp::onRtp_l(const char *buf, size_t len, bool rtx) {
    auto origin_seq = payload[0] << 8 | payload[1];
    rtp->seq = htons(origin_seq);
    if (info->offer_ssrc_rtp) {
-        //非simulcast
+        //非simulcast或音频
        rtp->ssrc = htonl(info->offer_ssrc_rtp);
-        TraceL << "received rtx rtp,ssrc: " << ssrc << ", seq:" << origin_seq;
+        TraceL << "received rtx rtp,ssrc: " << ssrc << ", seq:" << origin_seq << ", pt:" << (int)rtp->pt;
    } else {
        //todo simulcast下，辅码流通过rtx传输？
        //simulcast情况下，根据rtx的ssrc查找rtp的ssrc
@@ -834,7 +837,7 @@ void WebRtcTransportImp::onSendNack(RtpPayloadInfo &info, const FCI_NACK &nack, 
 ///////////////////////////////////////////////////////////////////
-void WebRtcTransportImp::onSortedRtp(RtpPayloadInfo &info, RtpPacket::Ptr rtp) {
+void WebRtcTransportImp::onSortedRtp(RtpPayloadInfo &info, const string &rid, RtpPacket::Ptr rtp) {
    if (info.media->type == TrackVideo && _pli_ticker.elapsedTime() > 2000) {
        //定期发送pli请求关键帧，方便非rtc等协议
        _pli_ticker.resetTime();
@@ -848,7 +851,24 @@ void WebRtcTransportImp::onSortedRtp(RtpPayloadInfo &info, RtpPacket::Ptr rtp) {
    }
    if (_push_src) {
-        _push_src->onWrite(std::move(rtp), false);
+        if (rtp->type == TrackAudio) {
+            //音频
+            for (auto &pr : _push_src_simulcast) {
+                pr.second->onWrite(rtp, false);
+            }
+        } else {
+            //视频
+            auto &src = _push_src_simulcast[rid];
+            if (!src) {
+                auto stream_id = rid.empty() ? _push_src->getId() : _push_src->getId() + "_" + rid;
+                auto src_imp = std::make_shared<RtspMediaSourceImp>(_push_src->getVhost(), _push_src->getApp(), stream_id);
+                src_imp->setSdp(_push_src->getSdp());
+                src_imp->setProtocolTranslation(_push_src->isRecording(Recorder::type_hls),_push_src->isRecording(Recorder::type_mp4));
+                src_imp->setListener(shared_from_this());
+                src = src_imp;
+            }
+            src->onWrite(std::move(rtp), false);
+        }
    }
 }

--- a/webrtc/WebRtcTransport.h
+++ b/webrtc/WebRtcTransport.h
@@ -358,7 +358,7 @@ private:
        unordered_map<uint32_t/*simulcast ssrc*/, std::shared_ptr<RtpReceiverImp> > receiver;
    };
-    void onSortedRtp(RtpPayloadInfo &info, RtpPacket::Ptr rtp);
+    void onSortedRtp(RtpPayloadInfo &info, const string &rid, RtpPacket::Ptr rtp);
    void onSendNack(RtpPayloadInfo &info, const FCI_NACK &nack, uint32_t ssrc);
    void changeRtpExtId(RtpPayloadInfo &info, const RtpHeader *header, bool is_recv, bool is_rtx = false, string *rid_ptr = nullptr) const;
@@ -380,6 +380,7 @@ private:
    Socket::Ptr _socket;
    //推流的rtsp源
    RtspMediaSource::Ptr _push_src;
+    unordered_map<string/*rid*/, RtspMediaSource::Ptr> _push_src_simulcast;
    //播放的rtsp源
    RtspMediaSource::Ptr _play_src;
    //播放rtsp源的reader对象

--- a/webrtc/offer-simulcast.sdp
+++ b/webrtc/offer-simulcast.sdp
-# chrome的sdp
 v=0
-o=- 403371946498103831 2 IN IP4 127.0.0.1
+o=- 1520777637155103417 2 IN IP4 127.0.0.1
 s=-
 t=0 0
 a=group:BUNDLE 0 1
@@ -9,10 +8,10 @@ a=msid-semantic: WMS
 m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126
 c=IN IP4 0.0.0.0
 a=rtcp:9 IN IP4 0.0.0.0
-a=ice-ufrag:pW4Z
+a=ice-ufrag:ma0v
-a=ice-pwd:S38S++HW3eTcPTyytsNI1XVp
+a=ice-pwd:H8+DMsvKNQE+Qz1uS7cZby1+
 a=ice-options:trickle
-a=fingerprint:sha-256 04:32:7B:56:7D:F7:D4:EC:65:7C:04:6C:F8:0B:03:F0:35:A9:1A:C3:43:3E:18:95:67:E6:0D:D1:EE:C9:16:8C
+a=fingerprint:sha-256 F1:30:4D:FE:6B:9A:BE:B4:31:65:30:C1:67:87:2F:74:23:7A:06:31:B0:49:DE:44:53:69:27:30:86:1F:E0:6C
 a=setup:actpass
 a=mid:0
 a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level
@@ -22,7 +21,7 @@ a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid
 a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
 a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
 a=sendrecv
-a=msid:- a3c6a137-1291-45cd-b985-07a9bd365452
+a=msid:- 7bf9dab3-79e9-4969-b3d2-44beae8b4286
 a=rtcp-mux
 a=rtpmap:111 opus/48000/2
 a=rtcp-fb:111 transport-cc
@@ -39,17 +38,17 @@ a=rtpmap:110 telephone-event/48000
 a=rtpmap:112 telephone-event/32000
 a=rtpmap:113 telephone-event/16000
 a=rtpmap:126 telephone-event/8000
-a=ssrc:3626257331 cname:JSFJMbaE9Pu5tevN
+a=ssrc:3454457472 cname:kbZgD5tgXGqTwvD1
-a=ssrc:3626257331 msid:- a3c6a137-1291-45cd-b985-07a9bd365452
+a=ssrc:3454457472 msid:- 7bf9dab3-79e9-4969-b3d2-44beae8b4286
-a=ssrc:3626257331 mslabel:-
+a=ssrc:3454457472 mslabel:-
-a=ssrc:3626257331 label:a3c6a137-1291-45cd-b985-07a9bd365452
+a=ssrc:3454457472 label:7bf9dab3-79e9-4969-b3d2-44beae8b4286
 m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 102 121 127 120 125 107 108 109 35 36 124 119 123 118 114 115 116
 c=IN IP4 0.0.0.0
 a=rtcp:9 IN IP4 0.0.0.0
-a=ice-ufrag:pW4Z
+a=ice-ufrag:ma0v
-a=ice-pwd:S38S++HW3eTcPTyytsNI1XVp
+a=ice-pwd:H8+DMsvKNQE+Qz1uS7cZby1+
 a=ice-options:trickle
-a=fingerprint:sha-256 04:32:7B:56:7D:F7:D4:EC:65:7C:04:6C:F8:0B:03:F0:35:A9:1A:C3:43:3E:18:95:67:E6:0D:D1:EE:C9:16:8C
+a=fingerprint:sha-256 F1:30:4D:FE:6B:9A:BE:B4:31:65:30:C1:67:87:2F:74:23:7A:06:31:B0:49:DE:44:53:69:27:30:86:1F:E0:6C
 a=setup:actpass
 a=mid:1
 a=extmap:14 urn:ietf:params:rtp-hdrext:toffset
@@ -64,7 +63,7 @@ a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid
 a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
 a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
 a=sendrecv
-a=msid:- 261e5384-9cf6-479d-9d59-aaf924d1a2ea
+a=msid:- fa2b9d11-98f5-4ea9-bf0a-6098069c6940
 a=rtcp-mux
 a=rtcp-rsize
 a=rtpmap:96 VP8/90000
@@ -143,7 +142,7 @@ a=rtcp-fb:124 transport-cc
 a=rtcp-fb:124 ccm fir
 a=rtcp-fb:124 nack
 a=rtcp-fb:124 nack pli
-a=fmtp:124 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f
+a=fmtp:124 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0032
 a=rtpmap:119 rtx/90000
 a=fmtp:119 apt=124
 a=rtpmap:123 H264/90000
@@ -152,7 +151,7 @@ a=rtcp-fb:123 transport-cc
 a=rtcp-fb:123 ccm fir
 a=rtcp-fb:123 nack
 a=rtcp-fb:123 nack pli
-a=fmtp:123 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f
+a=fmtp:123 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032
 a=rtpmap:118 rtx/90000
 a=fmtp:118 apt=123
 a=rtpmap:114 red/90000
@@ -162,94 +161,4 @@ a=rtpmap:116 ulpfec/90000
 a=rid:q send
 a=rid:h send
 a=rid:f send
 a=simulcast:send q;h;f
\ No newline at end of file
-#firefox的sdp
-v=0
-o=mozilla...THIS_IS_SDPARTA-88.0.1 3954544078885279475 0 IN IP4 0.0.0.0
-s=-
-t=0 0
-a=fingerprint:sha-256 9B:4F:D1:D2:A5:ED:08:BC:E8:D7:DD:D8:59:2C:E6:3D:19:F9:4C:67:9C:D9:9B:7B:C9:47:7A:3A:1F:05:C8:96
-a=group:BUNDLE 0 1
-a=ice-options:trickle
-a=msid-semantic:WMS *
-m=audio 9 UDP/TLS/RTP/SAVPF 109 9 0 8 101
-c=IN IP4 0.0.0.0
-a=sendrecv
-a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level
-a=extmap:2/recvonly urn:ietf:params:rtp-hdrext:csrc-audio-level
-a=extmap:3 urn:ietf:params:rtp-hdrext:sdes:mid
-a=fmtp:109 maxplaybackrate=48000;stereo=1;useinbandfec=1
-a=fmtp:101 0-15
-a=ice-pwd:92a9ced6d734f7ff2a45cde8b29572a9
-a=ice-ufrag:b986b945
-a=mid:0
-a=msid:- {ea61729a-c244-4c79-aeb7-b57765fefa26}
-a=rtcp-mux
-a=rtpmap:109 opus/48000/2
-a=rtpmap:9 G722/8000/1
-a=rtpmap:0 PCMU/8000
-a=rtpmap:8 PCMA/8000
-a=rtpmap:101 telephone-event/8000/1
-a=setup:actpass
-a=ssrc:3000327501 cname:{12e7c547-559e-46e2-94db-f1ad474c95dc}
-m=video 9 UDP/TLS/RTP/SAVPF 120 124 121 125 126 127 97 98
-c=IN IP4 0.0.0.0
-a=sendrecv
-a=extmap:3 urn:ietf:params:rtp-hdrext:sdes:mid
-a=extmap:4 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
-a=extmap:5 urn:ietf:params:rtp-hdrext:toffset
-a=extmap:6/recvonly http://www.webrtc.org/experiments/rtp-hdrext/playout-delay
-a=extmap:7 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
-a=extmap:8/sendonly urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
-a=extmap:9/sendonly urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
-a=fmtp:126 profile-level-id=42e01f;level-asymmetry-allowed=1;packetization-mode=1
-a=fmtp:97 profile-level-id=42e01f;level-asymmetry-allowed=1
-a=fmtp:120 max-fs=12288;max-fr=60
-a=fmtp:124 apt=120
-a=fmtp:121 max-fs=12288;max-fr=60
-a=fmtp:125 apt=121
-a=fmtp:127 apt=126
-a=fmtp:98 apt=97
-a=ice-pwd:92a9ced6d734f7ff2a45cde8b29572a9
-a=ice-ufrag:b986b945
-a=mid:1
-a=msid:- {3bfe1b80-20eb-4b42-b8b7-fac45fb281bf}
-a=rid:q send
-a=rid:h send
-a=rid:f send
-a=rtcp-fb:120 nack
-a=rtcp-fb:120 nack pli
-a=rtcp-fb:120 ccm fir
-a=rtcp-fb:120 goog-remb
-a=rtcp-fb:120 transport-cc
-a=rtcp-fb:121 nack
-a=rtcp-fb:121 nack pli
-a=rtcp-fb:121 ccm fir
-a=rtcp-fb:121 goog-remb
-a=rtcp-fb:121 transport-cc
-a=rtcp-fb:126 nack
-a=rtcp-fb:126 nack pli
-a=rtcp-fb:126 ccm fir
-a=rtcp-fb:126 goog-remb
-a=rtcp-fb:126 transport-cc
-a=rtcp-fb:97 nack
-a=rtcp-fb:97 nack pli
-a=rtcp-fb:97 ccm fir
-a=rtcp-fb:97 goog-remb
-a=rtcp-fb:97 transport-cc
-a=rtcp-mux
-a=rtcp-rsize
-a=rtpmap:120 VP8/90000
-a=rtpmap:124 rtx/90000
-a=rtpmap:121 VP9/90000
-a=rtpmap:125 rtx/90000
-a=rtpmap:126 H264/90000
-a=rtpmap:127 rtx/90000
-a=rtpmap:97 H264/90000
-a=rtpmap:98 rtx/90000
-a=setup:actpass
-a=simulcast:send q;h;f
-a=ssrc:2581133096 cname:{12e7c547-559e-46e2-94db-f1ad474c95dc}
-a=ssrc:773854125 cname:{12e7c547-559e-46e2-94db-f1ad474c95dc}
-a=ssrc:4100728001 cname:{12e7c547-559e-46e2-94db-f1ad474c95dc}
--- a/www/webrtc/index.html
+++ b/www/webrtc/index.html
@@ -108,7 +108,7 @@
                    element: document.getElementById('video'),// video 标签
                    debug: true,// 是否打印日志
                    zlmsdpUrl:document.getElementById('streamUrl').value,//流地址
-                    simulcast:false,//document.getElementById('simulcast').checked,
+                    simulcast:document.getElementById('simulcast').checked,
                    useCamera:document.getElementById('useCamera').checked,
                    audioEnable:document.getElementById('audioEnable').checked,
                    videoEnable:document.getElementById('videoEnable').checked,