[FFmpeg-trac] #6453(avformat:new): RTMP handshake fails with some encoders

[FFmpeg]

#6453: RTMP handshake fails with some encoders
-------------------------------------+-------------------------------------
             Reporter:               |                    Owner:
  rubensanchez                       |                   Status:  new
                 Type:  defect       |                Component:  avformat
             Priority:  important    |               Resolution:
              Version:  git-master   |               Blocked By:
             Keywords:  RTMP,        |  Reproduced by developer:  1
  handshake                          |
             Blocking:               |
Analyzed by developer:  0            |
-------------------------------------+-------------------------------------

Comment (by rubensanchez):

 I implemented the following fix:


 {{{
 static int rtmp_server_handshake(URLContext *s, RTMPContext *rt)
 {
     uint8_t hs_s0s1[RTMP_HANDSHAKE_PACKET_SIZE + 1];
     uint8_t hs_c0c1[RTMP_HANDSHAKE_PACKET_SIZE + 1];
     uint8_t hs_c2[RTMP_HANDSHAKE_PACKET_SIZE + 1];
     uint8_t hs_s2[RTMP_HANDSHAKE_PACKET_SIZE];
     uint8_t dummy_uint;
     uint32_t hs_my_epoch;
     uint32_t temp       = 0;
     int randomidx       = 0;
     int inoutsize       = 0;
     int ret;

     /****************
      * Receive C0+C1
      ***************/
     ret = rtmp_receive_hs_packet(rt, &dummy_uint, &dummy_uint, hs_c0c1,
                                  RTMP_HANDSHAKE_PACKET_SIZE + 1);
     if (ret) {
         av_log(s, AV_LOG_ERROR, "RTMP Handshake C1 Error %d\n", ret);
         return ret;
     }
     // Check Version
     if (hs_c0c1[0] != 3) {
         av_log(s, AV_LOG_ERROR, "RTMP protocol version mismatch. Expected
 0x03 received %02x\n", hs_c0c1[0]);
         return AVERROR(EIO);
     }
     // Get client epoch and set our with the same value
     hs_my_epoch = AV_RB32(hs_c0c1 + 1);

     /*************
      * Send S0+S1
      ************/
     // Generate random data to send it on S0+S1
     for (randomidx = 9; randomidx < (RTMP_HANDSHAKE_PACKET_SIZE + 1);
          randomidx += 4)
         AV_WB32(hs_s0s1 + randomidx, av_get_random_seed());
     // Set the RTMP protocol code on S0+S1 (First byte)
     hs_s0s1[0] = 0x03;
     // Copy the random data from C1 to S1
     memcpy(hs_s0s1 + 1, hs_c0c1 + 1, RTMP_HANDSHAKE_PACKET_SIZE);
     AV_WB32(hs_s0s1 + 1, hs_my_epoch);
     AV_WB32(hs_s0s1 + 5, 0);
     inoutsize = ffurl_write(rt->stream, hs_s0s1,
                             RTMP_HANDSHAKE_PACKET_SIZE + 1);
     if (inoutsize != RTMP_HANDSHAKE_PACKET_SIZE + 1) {
         av_log(s, AV_LOG_ERROR, "RTMP Handshake S1 Error %d\n", ret);
         return AVERROR(EIO);
     }

     /***********
      * Send S2
      **********/
     // Get the S2 random data from C0+C1
     memcpy(hs_s2, hs_c0c1, RTMP_HANDSHAKE_PACKET_SIZE);
     ret = rtmp_send_hs_packet(rt, hs_my_epoch, 0, hs_s2,
                               RTMP_HANDSHAKE_PACKET_SIZE);
     if (ret) {
         av_log(s, AV_LOG_ERROR, "RTMP Handshake S2 Error\n");
         return ret;
     }

     /*************
      * Receive C2
      ************/

     ret = ffurl_read_complete(rt->stream, hs_c2,
                                     RTMP_HANDSHAKE_PACKET_SIZE + 1);
     if (ret <= 0)
         return AVERROR(EIO);
     if (ret != RTMP_HANDSHAKE_PACKET_SIZE + 1) {
         av_log(rt, AV_LOG_ERROR, "Erroneous Message size %d"
                " not following standard\n", (int)inoutsize);
         return AVERROR(EINVAL);
     }

     // Check timestamp and random data from C2
     temp  = AV_RB32(hs_c2 + 1);
     if (temp != hs_my_epoch)
         av_log(s, AV_LOG_WARNING,
                "Erroneous C2 Message epoch does not match up with C1
 epoch");
     if (memcmp(hs_c2 + 9, hs_c0c1 + 9,
                RTMP_HANDSHAKE_PACKET_SIZE - 8))
         av_log(s, AV_LOG_WARNING,
                "Erroneous C2 Message random does not match up\n");

     return 0;
 }
 }}}

--
Ticket URL: <https://trac.ffmpeg.org/ticket/6453#comment:1>
FFmpeg <https://ffmpeg.org>
FFmpeg issue tracker