HttpClient.cpp 10.5 KB
Newer Older
1
/*
2 3
* MIT License
*
xiongziliang committed
4
* Copyright (c) 2016-2019 xiongziliang <771730766@qq.com>
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
*
* This file is part of ZLMediaKit(https://github.com/xiongziliang/ZLMediaKit).
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
xiongziliang committed
26

27
#include <cstdlib>
xiongziliang committed
28
#include "HttpClient.h"
xiongziliang committed
29
#include "Common/config.h"
xiongziliang committed
30

xiongziliang committed
31
namespace mediakit {
xiongziliang committed
32 33


34
HttpClient::HttpClient() {
xiongziliang committed
35
}
36 37

HttpClient::~HttpClient() {
xiongziliang committed
38
}
39 40

void HttpClient::sendRequest(const string &strUrl, float fTimeOutSec) {
41
    _aliveTicker.resetTime();
42
    _url = strUrl;
43
    auto protocol = FindField(strUrl.data(), NULL, "://");
xiongziliang committed
44 45
    uint16_t defaultPort;
    bool isHttps;
xiongziliang committed
46
    if (strcasecmp(protocol.data(), "http") == 0) {
xiongziliang committed
47 48
        defaultPort = 80;
        isHttps = false;
xiongziliang committed
49
    } else if (strcasecmp(protocol.data(), "https") == 0) {
xiongziliang committed
50 51
        defaultPort = 443;
        isHttps = true;
52
    } else {
53
        auto strErr = StrPrinter << "非法的协议:" << protocol << endl;
xiongziliang committed
54 55
        throw std::invalid_argument(strErr);
    }
56

xiongziliang committed
57 58 59 60 61 62 63 64
    auto host = FindField(strUrl.data(), "://", "/");
    if (host.empty()) {
        host = FindField(strUrl.data(), "://", NULL);
    }
    _path = FindField(strUrl.data(), host.data(), NULL);
    if (_path.empty()) {
        _path = "/";
    }
65
    uint16_t port = atoi(FindField(host.data(), ":", NULL).data());
xiongziliang committed
66 67 68 69 70 71 72
    if (port <= 0) {
        //默认端口
        port = defaultPort;
    } else {
        //服务器域名
        host = FindField(host.data(), NULL, ":");
    }
73 74 75 76 77 78
    _header.emplace("Host", host);
    _header.emplace("Tools", "ZLMediaKit");
    _header.emplace("Connection", "keep-alive");
    _header.emplace("Accept", "*/*");
    _header.emplace("Accept-Language", "zh-CN,zh;q=0.8");
    _header.emplace("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36");
79 80

    if (_body && _body->remainSize()) {
81 82
        _header.emplace("Content-Length", to_string(_body->remainSize()));
        _header.emplace("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
xiongziliang committed
83 84 85 86 87
    }

    bool bChanged = (_lastHost != host + ":" + to_string(port)) || (_isHttps != isHttps);
    _lastHost = host + ":" + to_string(port);
    _isHttps = isHttps;
88
    _fTimeOutSec = fTimeOutSec;
89 90 91 92 93 94 95 96

    auto cookies = HttpCookieStorage::Instance().get(_lastHost,_path);
    _StrPrinter printer;
    for(auto &cookie : cookies){
        printer << cookie->getKey() << "=" << cookie->getVal() << ";";
    }
    if(!printer.empty()){
        printer.pop_back();
97
        _header.emplace("Cookie", printer);
98 99 100
    }


101
    if (!alive() || bChanged) {
102
        //InfoL << "reconnet:" << _lastHost;
103 104
        startConnect(host, port, fTimeOutSec);
    } else {
xiongziliang committed
105 106 107 108 109 110
        SockException ex;
        onConnect(ex);
    }
}


111
void HttpClient::onConnect(const SockException &ex) {
112
    _aliveTicker.resetTime();
113 114 115 116 117
    if (ex) {
        onDisconnect(ex);
        return;
    }

118 119 120
    //先假设http客户端只会接收一点点数据(只接受http头,节省内存)
    _sock->setReadBuffer(std::make_shared<BufferRaw>(1 * 1024));

121 122 123
    _totalBodySize = 0;
    _recvedBodySize = 0;
    HttpRequestSplitter::reset();
124
    _chunkedSplitter.reset();
125

126 127
    _StrPrinter printer;
    printer << _method + " " << _path + " HTTP/1.1\r\n";
xiongziliang committed
128
    for (auto &pr : _header) {
129
        printer << pr.first + ": ";
130
        printer << pr.second + "\r\n";
xiongziliang committed
131
    }
132
    send(printer << "\r\n");
xiongziliang committed
133
    onFlush();
xiongziliang committed
134
}
135 136 137

void HttpClient::onRecv(const Buffer::Ptr &pBuf) {
    _aliveTicker.resetTime();
xiongziliang committed
138
    HttpRequestSplitter::input(pBuf->data(), pBuf->size());
xiongziliang committed
139 140
}

141
void HttpClient::onErr(const SockException &ex) {
142
    if (ex.getErrCode() == Err_eof && _totalBodySize < 0) {
143 144
        //如果Content-Length未指定 但服务器断开链接
        //则认为本次http请求完成
145
        onResponseCompleted_l();
146
    }
147
    onDisconnect(ex);
xiongziliang committed
148 149
}

150 151
int64_t HttpClient::onRecvHeader(const char *data, uint64_t len) {
    _parser.Parse(data);
152 153 154 155 156 157
    if(_parser.Url() == "302" || _parser.Url() == "301"){
        auto newUrl = _parser["Location"];
        if(newUrl.empty()){
            shutdown(SockException(Err_shutdown,"未找到Location字段(跳转url)"));
            return 0;
        }
158 159 160 161 162 163
        if(onRedirectUrl(newUrl,_parser.Url() == "302")){
            HttpClient::clear();
            setMethod("GET");
            HttpClient::sendRequest(newUrl,_fTimeOutSec);
            return 0;
        }
164 165
    }

166
    checkCookie(_parser.getValues());
167
    _totalBodySize = onResponseHeader(_parser.Url(), _parser.getValues());
168

169 170 171
    if(!_parser["Content-Length"].empty()){
        //有Content-Length字段时忽略onResponseHeader的返回值
        _totalBodySize = atoll(_parser["Content-Length"].data());
172
    }
173

174
    if(_parser["Transfer-Encoding"] == "chunked"){
175 176 177
        //我们认为这种情况下后面应该有大量的数据过来,加大接收缓存提高性能
        _sock->setReadBuffer(std::make_shared<BufferRaw>(256 * 1024));

178 179 180 181 182 183 184 185 186 187 188 189 190
        //如果Transfer-Encoding字段等于chunked,则认为后续的content是不限制长度的
        _totalBodySize = -1;
        _chunkedSplitter = std::make_shared<HttpChunkedSplitter>([this](const char *data,uint64_t len){
            if(len > 0){
                auto recvedBodySize = _recvedBodySize + len;
                onResponseBody(data, len, recvedBodySize, INT64_MAX);
                _recvedBodySize = recvedBodySize;
            }else{
                onResponseCompleted_l();
            }
        });
    }

191
    if(_totalBodySize == 0){
192
        //后续没content,本次http请求结束
193 194 195
        onResponseCompleted_l();
        return 0;
    }
196

197 198
    //当_totalBodySize != 0时到达这里,代表后续有content
    //虽然我们在_totalBodySize >0 时知道content的确切大小,
199 200
    //但是由于我们没必要等content接收完毕才回调onRecvContent(因为这样浪费内存并且要多次拷贝数据)
    //所以返回-1代表我们接下来分段接收content
201
    _recvedBodySize = 0;
202 203 204 205 206 207 208
    if(_totalBodySize > 0){
        //根据_totalBodySize设置接收缓存大小
        _sock->setReadBuffer(std::make_shared<BufferRaw>(MIN(_totalBodySize + 1,256 * 1024)));
    }else{
        _sock->setReadBuffer(std::make_shared<BufferRaw>(256 * 1024));
    }

209 210 211 212
    return -1;
}

void HttpClient::onRecvContent(const char *data, uint64_t len) {
213 214 215 216
    if(_chunkedSplitter){
        _chunkedSplitter->input(data,len);
        return;
    }
217
    auto recvedBodySize = _recvedBodySize + len;
218 219 220 221 222 223 224 225 226 227
    if(_totalBodySize < 0){
        //不限长度的content,最大支持INT64_MAX个字节
        onResponseBody(data, len, recvedBodySize, INT64_MAX);
        _recvedBodySize = recvedBodySize;
        return;
    }

    //固定长度的content
    if ( recvedBodySize < _totalBodySize ) {
        //content还未接收完毕
228 229
        onResponseBody(data, len, recvedBodySize, _totalBodySize);
        _recvedBodySize = recvedBodySize;
230 231 232 233 234 235 236 237 238
        return;
    }

    //content接收完毕
    onResponseBody(data, _totalBodySize - _recvedBodySize, _totalBodySize, _totalBodySize);
    bool biggerThanExpected = recvedBodySize > _totalBodySize;
    onResponseCompleted_l();
    if(biggerThanExpected) {
        //声明的content数据比真实的小,那么我们只截取前面部分的并断开链接
xiongziliang committed
239
        shutdown(SockException(Err_shutdown, "http response content size bigger than expected"));
240
    }
xiongziliang committed
241
}
242

xiongziliang committed
243
void HttpClient::onFlush() {
244
    _aliveTicker.resetTime();
245
    while (_body && _body->remainSize() && !isSocketBusy()) {
246
        auto buffer = _body->readData();
247
        if (!buffer) {
248 249 250
            //数据发送结束或读取数据异常
            break;
        }
251
        if (send(buffer) <= 0) {
252 253 254 255 256 257 258 259
            //发送数据失败,不需要回滚数据,因为发送前已经通过isSocketBusy()判断socket可写
            //所以发送缓存区肯定未满,该buffer肯定已经写入socket
            break;
        }
    }
}

void HttpClient::onManager() {
260
    if (_aliveTicker.elapsedTime() > 3 * 1000 && _totalBodySize < 0 && !_chunkedSplitter) {
261 262
        //如果Content-Length未指定 但接收数据超时
        //则认为本次http请求完成
263
        onResponseCompleted_l();
264 265
    }

266
    if (_fTimeOutSec > 0 && _aliveTicker.elapsedTime() > _fTimeOutSec * 1000) {
267
        //超时
xiongziliang committed
268
        shutdown(SockException(Err_timeout, "http request timeout"));
269 270 271
    }
}

272
void HttpClient::onResponseCompleted_l() {
273 274
    _totalBodySize = 0;
    _recvedBodySize = 0;
275
    onResponseCompleted();
276 277
}

278
void HttpClient::checkCookie(HttpClient::HttpHeader &headers) {
279
    //Set-Cookie: IPTV_SERVER=8E03927B-CC8C-4389-BC00-31DBA7EC7B49;expires=Sun, Sep 23 2018 15:07:31 GMT;path=/index/api/
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
    for(auto it_set_cookie = headers.find("Set-Cookie") ; it_set_cookie != headers.end() ; ++it_set_cookie ){
        auto key_val = Parser::parseArgs(it_set_cookie->second,";","=");
        HttpCookie::Ptr cookie = std::make_shared<HttpCookie>();
        cookie->setHost(_lastHost);

        int index = 0;
        auto arg_vec = split(it_set_cookie->second, ";");
        for (string &key_val : arg_vec) {
            auto key = FindField(key_val.data(),NULL,"=");
            auto val = FindField(key_val.data(),"=", NULL);

            if(index++ == 0){
                cookie->setKeyVal(key,val);
                continue;
            }
295

296 297 298 299
            if(key == "path") {
                cookie->setPath(val);
                continue;
            }
300

301 302 303 304
            if(key == "expires"){
                cookie->setExpires(val,headers["Date"]);
                continue;
            }
305 306
        }

307 308
        if(!(*cookie)){
            //无效的cookie
309
            continue;
310
        }
311
        HttpCookieStorage::Instance().set(cookie);
312 313 314
    }
}

315

xiongziliang committed
316
} /* namespace mediakit */
xiongziliang committed
317