使用crtmpserver作为rtmp服务器进行直播或点播时，经常遇到直播或点播失败的问题，严重时可能会存在二次中有一次失败的现象，所以对代码进行详细分析，发现是握手协议存在bug.

握手协议简述

第一步: Client->Server: C0 + C1
第二步: Server->Client: S0 + S1 + S2
第三部: Client->Server: C2

生成S0 + S1 + S2时存在bug分析及解决

InboundRTMPProtocol::PerformHandshake负责生成S1 + S2响应报文。
S1和S2报文的长度均为1536，所有生成的buffer大小应该1536 + 1536 = 3072。
S1和S2的具体结构可参考crtmpserver提供的文档:
crtmpserver/docs/RTMPEHandshake.pdf。

bool InboundRTMPProtocol::PerformHandshake(IOBuffer &buffer, bool encrypted) {
    if (!ValidateClient(buffer)) {
        if (encrypted || _pProtocolHandler->ValidateHandshake()) {
            FATAL("Unable to validate client");
            return false;
        } else {
            WARN("Client not validated");
            _validationScheme = 0;
        }
    }

    //get the buffers
    uint8_t *pInputBuffer = GETIBPOINTER(buffer);
    if (_pOutputBuffer == NULL) {
        _pOutputBuffer = new uint8_t[3072];
    } else {
        delete[] _pOutputBuffer;
        _pOutputBuffer = new uint8_t[3072];
    }

    // timestamp(前4个字节，时间戳)
    EHTONLP(_pOutputBuffer, (uint32_t) time(NULL));

    // version(第5字节到第8字节，版本)
    EHTONLP(_pOutputBuffer + 4, (uint32_t) 0x00000000);

    // generate random data (第9到3072字节用随机数据填充)
    for (uint32_t i = 8; i < 3072; i++) {
        _pOutputBuffer[i] = rand() % 256;
    }

    // 随机打上印记
    for (uint32_t i = 0; i < 10; i++) {
        uint32_t index = rand() % (3072 - HTTP_HEADERS_SERVER_US_LEN);
        memcpy(_pOutputBuffer + index, HTTP_HEADERS_SERVER_US, HTTP_HEADERS_SERVER_US_LEN);
    }

    //**** FIRST 1536 bytes from server response ****//
    //compute DH key position
    uint32_t serverDHOffset = GetDHOffset(_pOutputBuffer, _validationScheme);
    uint32_t clientDHOffset = GetDHOffset(pInputBuffer, _validationScheme);

    //generate DH key
    DHWrapper dhWrapper(1024);

    if (!dhWrapper.Initialize()) {
        FATAL("Unable to initialize DH wrapper");
        return false;
    }

    if (!dhWrapper.CreateSharedKey(pInputBuffer + clientDHOffset, 128)) {
        FATAL("Unable to create shared key");
        return false;
    }

    if (!dhWrapper.CopyPublicKey(_pOutputBuffer + serverDHOffset, 128)) {
        FATAL("Couldn't write public key!");
        return false;
    }

    if (encrypted) {
        uint8_t secretKey[128];
        if (!dhWrapper.CopySharedKey(secretKey, sizeof (secretKey))) {
            FATAL("Unable to copy shared key");
            return false;
        }

        _pKeyIn = new RC4_KEY;
        _pKeyOut = new RC4_KEY;
        InitRC4Encryption(
                secretKey,
                (uint8_t*) & pInputBuffer[clientDHOffset],
                (uint8_t*) & _pOutputBuffer[serverDHOffset],
                _pKeyIn,
                _pKeyOut);

        //bring the keys to correct cursor
        uint8_t data[1536];
        RC4(_pKeyIn, 1536, data, data);
        RC4(_pKeyOut, 1536, data, data);
    }

    //generate the digest
    uint32_t serverDigestOffset = GetDigestOffset(_pOutputBuffer, _validationScheme);

    uint8_t *pTempBuffer = new uint8_t[1536 - 32];
    memcpy(pTempBuffer, _pOutputBuffer, serverDigestOffset);
    memcpy(pTempBuffer + serverDigestOffset, _pOutputBuffer + serverDigestOffset + 32,
            1536 - serverDigestOffset - 32);

    uint8_t *pTempHash = new uint8_t[512];
    HMACsha256(pTempBuffer, 1536 - 32, genuineFMSKey, 36, pTempHash);

    //put the digest in place
    memcpy(_pOutputBuffer + serverDigestOffset, pTempHash, 32);

    //cleanup
    delete[] pTempBuffer;
    delete[] pTempHash;


    //**** SECOND 1536 bytes from server response ****//
    //Compute the chalange index from the initial client request
    uint32_t keyChallengeIndex = GetDigestOffset(pInputBuffer, _validationScheme);

    //compute the key
    pTempHash = new uint8_t[512];
    HMACsha256(pInputBuffer + keyChallengeIndex, //pData
            32, //dataLength
            BaseRTMPProtocol::genuineFMSKey, //key
            68, //keyLength
            pTempHash //pResult
            );

    //generate the hash
    uint8_t *pLastHash = new uint8_t[512];
    HMACsha256(_pOutputBuffer + 1536, //pData
            1536 - 32, //dataLength
            pTempHash, //key
            32, //keyLength
            pLastHash //pResult
            );

    //put the hash where it belongs
    memcpy(_pOutputBuffer + 1536 * 2 - 32, pLastHash, 32);


    //cleanup
    delete[] pTempHash;
    delete[] pLastHash;
    //***** DONE BUILDING THE RESPONSE ***//


    //wire the response
    if (encrypted)
        _outputBuffer.ReadFromByte(6);
    else
        _outputBuffer.ReadFromByte(3);
    _outputBuffer.ReadFromBuffer(_pOutputBuffer, 3072);

    //final cleanup
    delete[] _pOutputBuffer;
    _pOutputBuffer = NULL;
    if (!buffer.IgnoreAll()) {
        FATAL("Unable to ignore input buffer");
        return false;
    }

    //signal outbound data
    if (!EnqueueForOutbound()) {
        FATAL("Unable to signal outbound data");
        return false;
    }

    //move to the next stage in the handshake
    _rtmpState = RTMP_STATE_SERVER_RESPONSE_SENT;

    return true;
}

以上代码中有一段在buffer随机打上印记的代码:

    // 随机打上印记
    for (uint32_t i = 0; i < 10; i++) {
        uint32_t index = rand() % (3072 - HTTP_HEADERS_SERVER_US_LEN);
        memcpy(_pOutputBuffer + index, HTTP_HEADERS_SERVER_US, HTTP_HEADERS_SERVER_US_LEN);
    }

其中HTTP_HEADERS_SERVER_US和HTTP_HEADERS_SERVER_US_LEN定义如下:

#define HTTP_HEADERS_SERVER_US "C++ RTMP Server (http://www.rtmpd.com)"
#define HTTP_HEADERS_SERVER_US_LEN 38

HTTP_HEADERS_SERVER_US_LEN为HTTP_HEADERS_SERVER_US的长度。

以上代码的主要作用是: 在S1+S1(1~3072字节)中随机插入10个C++ RTMP Server (http://www.rtmpd.com)字符串，作为标示。
执行完上述代码后，内存示意如下:

**随机插入10段`C++ RTMP Server (http://www.rtmpd.com)`之后的内存示意**

这里存在的bug是，因为是随机插入，所以有时会覆盖掉前8个字节(即timestamp+version)，这样会导致直播点播失败。

解决的方法:

    int nHeaderLen = strlen(HTTP_HEADERS_SERVER_US);

    for (uint32_t i = 0; i < 10; i++) 
    {
        uint32_t index = rand() % (3072 - 8 - nHeaderLen);
        memcpy(_pOutputBuffer + 8 + index, HTTP_HEADERS_SERVER_US, nHeaderLen);
    }

即插入10个随机字符串时，避开前8个字节。
其实还有一种更简单的方法, 就是将原有随机插入10个字符串的代码直接注释掉，那个只是提供个标示，其实木有啥卵用。