rtsp/rtp 新增支持mjpeg编码 (#2166)

* Trying to send mjpeg via MultiMediaSourceMuxer

* Improved JPEGRtpEncoder::inputFrame code but still not working

* 优化代码

* 完善jpeg相关逻辑

* Micro fix

* FrameJPEG renamed to JPEGFrame according to ZLM style

* Modified  JPEGRtpEncoder::inputFrame and JPEGRtpEncoder::rtp_send_jpeg

* getVideoHeight(), getVideoWidth() and getVideoFps() in JPEGTrack

* mjpeg rtp打包避免内存拷贝/修复mjpeg rtp解包huffman_table size字段错误的bug

* 支持mjpeg pix type

* 优化性能

* add bom header

README.md

@@ -53,7 +53,7 @@
   - 服务器/客户端完整支持Basic/Digest方式的登录鉴权，全异步可配置化的鉴权接口
   - 支持H265编码
   - 服务器支持RTSP推流(包括`rtp over udp` `rtp over tcp`方式)
-  - 支持H264/H265/AAC/G711/OPUS编码，其他编码能转发但不能转协议
+  - 支持H264/H265/AAC/G711/OPUS/MJPEG编码，其他编码能转发但不能转协议
 
 - RTMP[S]
   - RTMP[S] 播放服务器，支持RTSP/MP4/HLS转RTMP

src/Extension/Factory.cpp

@@ -15,6 +15,7 @@
 #include "AACRtmp.h"
 #include "CommonRtmp.h"
 #include "H264Rtp.h"
+#include "JPEGRtp.h"
 #include "AACRtp.h"
 #include "H265Rtp.h"
 #include "CommonRtp.h"
@@ -22,6 +23,7 @@
 #include "Opus.h"
 #include "G711.h"
 #include "L16.h"
+#include "JPEG.h"
 #include "Util/base64.h"
 #include "Common/Parser.h"
 #include "Common/config.h"
@@ -89,6 +91,10 @@ Track::Ptr Factory::getTrackBySdp(const SdpTrack::Ptr &track) {
             return std::make_shared<H265Track>(vps, sps, pps, 0, 0, 0);
         }
 
+        case CodecJPEG : {
+            return std::make_shared<JPEGTrack>();
+        }
+
         default: {
             //其他codec不支持
             WarnL << "暂不支持该rtsp编码类型:" << track->getName();
@@ -113,6 +119,7 @@ Track::Ptr Factory::getTrackByAbstractTrack(const Track::Ptr& track) {
         case CodecOpus: return std::make_shared<OpusTrack>();
         case CodecH265: return std::make_shared<H265Track>();
         case CodecH264: return std::make_shared<H264Track>();
+        case CodecJPEG: return std::make_shared<JPEGTrack>();
 
         default: {
             //其他codec不支持
@@ -141,6 +148,7 @@ RtpCodec::Ptr Factory::getRtpEncoderByCodecId(CodecId codec_id, uint32_t sample_
             }
             return std::make_shared<CommonRtpEncoder>(codec_id, ssrc, mtu, sample_rate, pt, interleaved);
         }
+        case CodecJPEG: return std::make_shared<JPEGRtpEncoder>(ssrc, mtu, sample_rate, pt, interleaved);
         default: WarnL << "暂不支持该CodecId:" << codec_id; return nullptr;
     }
 }
@@ -172,6 +180,7 @@ RtpCodec::Ptr Factory::getRtpDecoderByTrack(const Track::Ptr &track) {
         case CodecOpus :
         case CodecG711A :
         case CodecG711U : return std::make_shared<CommonRtpDecoder>(track->getCodecId());
+        case CodecJPEG: return std::make_shared<JPEGRtpDecoder>();
         default : WarnL << "暂不支持该CodecId:" << track->getCodecName(); return nullptr;
     }
 }
@@ -210,6 +219,7 @@ Track::Ptr getTrackByCodecId(CodecId codecId, int sample_rate = 0, int channels
         case CodecOpus: return std::make_shared<OpusTrack>();
         case CodecG711A :
         case CodecG711U : return (sample_rate && channels && sample_bit) ? std::make_shared<G711Track>(codecId, sample_rate, channels, sample_bit) : nullptr;
+        case CodecJPEG : return std::make_shared<JPEGTrack>();
         default : WarnL << "暂不支持该CodecId:" << codecId; return nullptr;
     }
 }

src/Extension/Frame.h

@@ -38,7 +38,8 @@ typedef enum {
     XX(CodecL16,   TrackAudio, 6, "L16", PSI_STREAM_RESERVED)       \
     XX(CodecVP8,   TrackVideo, 7, "VP8", PSI_STREAM_VP8)            \
     XX(CodecVP9,   TrackVideo, 8, "VP9", PSI_STREAM_VP9)            \
-    XX(CodecAV1,   TrackVideo, 9, "AV1", PSI_STREAM_AV1)
+    XX(CodecAV1,   TrackVideo, 9, "AV1", PSI_STREAM_AV1)            \
+    XX(CodecJPEG,  TrackVideo, 10, "JPEG", PSI_STREAM_JPEG_2000)
 
 typedef enum {
     CodecInvalid = -1,

src/Extension/JPEG.cpp

+#include "JPEG.h"
+#include "Rtsp/Rtsp.h"
+#include "Util/util.h"
+
+using namespace toolkit;
+
+namespace mediakit {
+
+bool JPEGTrack::inputFrame(const Frame::Ptr &frame) {
+    if (!ready()) {
+        if ((_height & _width) > 0) {
+            if (_tmp == 0) _tmp = frame->dts();
+            else _fps = 1000.0 / (frame->dts() - _tmp);
+        } else getVideoResolution((uint8_t*)frame->data(), frame->size());
+        return false;
+    }
+    return VideoTrack::inputFrame(frame);
+}
+
+void JPEGTrack::getVideoResolution(const uint8_t *buf, int len) {
+    for (int i = 0; i < len - 8; i++) {
+        if (buf[i] != 0xff)
+            continue;
+        if (buf[i + 1] == 0xC0 /*SOF0*/) {
+            _height = buf[i + 5] * 256 + buf[i + 6];
+            _width = buf[i + 7] * 256 + buf[i + 8];
+            return;
+        }
+    }
+}
+
+class JPEGSdp : public Sdp {
+public:
+    JPEGSdp(int bitrate): Sdp(90000, Rtsp::PT_JPEG) {
+        _printer << "m=video 0 RTP/AVP " << (int)getPayloadType() << "\r\n";
+        if (bitrate) {
+            _printer << "b=AS:" << bitrate << "\r\n";
+        }
+        _printer << "a=control:trackID=" << (int)TrackVideo << "\r\n";
+    }
+
+    std::string getSdp() const { return _printer; }
+
+    CodecId getCodecId() const { return CodecJPEG; }
+
+private:
+    _StrPrinter _printer;
+};
+
+Sdp::Ptr JPEGTrack::getSdp() {
+    return std::make_shared<JPEGSdp>(getBitRate() / 1024);
+}
+} // namespace mediakit

src/Extension/JPEG.h

+#ifndef ZLMEDIAKIT_JPEG_H
+#define ZLMEDIAKIT_JPEG_H
+
+#include "Frame.h"
+#include "Track.h"
+
+namespace mediakit {
+
+class JPEGTrack : public VideoTrack {
+public:
+    using Ptr = std::shared_ptr<JPEGTrack>;
+
+    CodecId getCodecId() const override { return CodecJPEG; }
+    int getVideoHeight() const override { return _height; }
+    int getVideoWidth() const override { return _width; }
+    float getVideoFps() const override { return _fps; }
+    bool ready() override { return _fps > 0; }
+    bool inputFrame(const Frame::Ptr &frame) override;
+
+private:
+    Sdp::Ptr getSdp() override;
+    Track::Ptr clone() override { return std::make_shared<std::remove_reference<decltype(*this)>::type>(*this); }
+    void getVideoResolution(const uint8_t *buf, int len);
+
+private:
+    int _width = 0;
+    int _height = 0;
+    float _fps = 0;
+    uint64_t _tmp = 0;
+};
+
+class JPEGFrame : public Frame {
+public:
+    /**
+     *  JPEG/MJPEG帧
+     * @param buffer 帧数据
+     * @param dts 时间戳,单位毫秒
+     * @param pix_type pixel format type; AV_PIX_FMT_YUVJ422P || (AVCOL_RANGE_JPEG && AV_PIX_FMT_YUV422P) : 1; AV_PIX_FMT_YUVJ420P || (AVCOL_RANGE_JPEG && AV_PIX_FMT_YUV420P) : 0
+     * @param prefix_size JFIF头大小
+     */
+    JPEGFrame(toolkit::Buffer::Ptr buffer, uint64_t dts, uint8_t pix_type = 0, size_t prefix_size = 0) {
+        _buffer = std::move(buffer);
+        _dts = dts;
+        _pix_type = pix_type;
+        _prefix_size = prefix_size;
+    }
+    ~JPEGFrame() override = default;
+
+    uint64_t dts() const override { return _dts; }
+    size_t prefixSize() const override { return _prefix_size; }
+    bool keyFrame() const override { return true; }
+    bool configFrame() const override { return false; }
+    CodecId getCodecId() const override { return CodecJPEG; }
+
+    char *data() const override { return _buffer->data(); }
+    size_t size() const override { return _buffer->size(); }
+
+    uint8_t pixType() const {return _pix_type; }
+
+private:
+    uint8_t _pix_type;
+    size_t _prefix_size;
+    uint64_t _dts;
+    toolkit::Buffer::Ptr _buffer;
+};
+
+}//namespace mediakit
+
+#endif //ZLMEDIAKIT_JPEG_H

src/Extension/JPEGRtp.h

+#ifndef ZLMEDIAKIT_JPEGRTP_H
+#define ZLMEDIAKIT_JPEGRTP_H
+
+#include "Frame.h"
+#include "Rtsp/RtpCodec.h"
+
+namespace mediakit{
+
+/**
+ * RTP/JPEG specific private data.
+ */
+struct PayloadContext {
+    std::string frame;         ///< current frame buffer
+    uint32_t timestamp;      ///< current frame timestamp
+    int hdr_size;       ///< size of the current frame header
+    uint8_t qtables[128][128];
+    uint8_t qtables_len[128];
+};
+
+/**
+ * 通用 rtp解码类
+ */
+class JPEGRtpDecoder : public RtpCodec {
+public:
+    typedef std::shared_ptr <JPEGRtpDecoder> Ptr;
+
+    JPEGRtpDecoder();
+    ~JPEGRtpDecoder() override = default;
+
+    /**
+     * 返回编码类型ID
+     */
+    CodecId getCodecId() const override;
+
+    /**
+     * 输入rtp并解码
+     * @param rtp rtp数据包
+     * @param key_pos 此参数内部强制转换为false,请忽略之
+     */
+    bool inputRtp(const RtpPacket::Ptr &rtp, bool key_pos = false) override;
+
+private:
+    struct PayloadContext _ctx;
+};
+
+class JPEGRtpEncoder : public JPEGRtpDecoder, public RtpInfo {
+public:
+    using Ptr = std::shared_ptr<JPEGRtpEncoder>;
+
+    JPEGRtpEncoder(uint32_t ssrc, uint32_t mtu = 1400, uint32_t sample_rate = 90000, uint8_t payload_type = 96, uint8_t interleaved = TrackVideo * 2);
+    ~JPEGRtpEncoder() = default;
+
+    bool inputFrame(const Frame::Ptr &frame) override;
+
+private:
+    void rtpSendJpeg(const uint8_t *buf, int size, uint64_t pts, uint8_t type);
+};
+}//namespace mediakit
+#endif //ZLMEDIAKIT_JPEGRTP_H

src/Rtsp/Rtsp.h

@@ -48,7 +48,7 @@ typedef enum {
     XX(DVI4_22050, TrackAudio, 17, 22050, 1, CodecInvalid) \
     XX(G729, TrackAudio, 18, 8000, 1, CodecInvalid) \
     XX(CelB, TrackVideo, 25, 90000, 1, CodecInvalid) \
-    XX(JPEG, TrackVideo, 26, 90000, 1, CodecInvalid) \
+    XX(JPEG, TrackVideo, 26, 90000, 1, CodecJPEG) \
     XX(nv, TrackVideo, 28, 90000, 1, CodecInvalid) \
     XX(H261, TrackVideo, 31, 90000, 1, CodecInvalid) \
     XX(MPV, TrackVideo, 32, 90000, 1, CodecInvalid) \