[Libav-user] "Segmentation fault" When encoding H264 video frame

[Leandro Raffo]

2016-09-12 9:59 GMT-03:00 M N <assemblerx86 at yandex.com>:
> Hi,
>
>  My program crashes with "Segmentation Fault" error when it is in the avcodec_encode_video2() function, however, it is able to encode the audio (Or other non-H264 codecs) normally (If I choose to only encode frames in audio stream). Here is my code (Pastebin with highlighting: http://pastebin.com/V8j9J4vq):
>
> ****************************************
> #include <libavformat/avformat.h>
> #include <libavcodec/avcodec.h>
> #include <libavutil/avutil.h>
> #include <libavutil/rational.h>
>
> #include <stdio.h>
>
> int main()
> {
>     av_register_all();
>
>     //av_log_set_level(-8);
>
>
>     AVFormatContext *ps = avformat_alloc_context();
>
>     AVFormatContext *ps2 = NULL;//avformat_alloc_context();
>     AVOutputFormat *oF = av_guess_format("mp4", NULL, "video/mp4");
>
>
>     if(avformat_open_input(&ps, "vid.mp4", NULL, NULL) != 0)
>     {
>         printf("Failed to open input file.\n");
>         return -1;
>     }
>
>     avformat_alloc_output_context2(&ps2, oF, NULL, "vid2.mp4");
>
>     avformat_find_stream_info(ps, NULL);
>
>     AVCodecContext **pC = (AVCodecContext**)malloc(ps->nb_streams), **p2C = (AVCodecContext**)malloc(ps->nb_streams);
>
>     AVStream *oStream = NULL;
>     AVStream *iStream = NULL;
>
>     AVCodec *encoder = NULL;
>     AVCodec *decoder = NULL;
>
>     unsigned int i;
>
>     avio_open(&ps2->pb, "vid2.mp4", AVIO_FLAG_WRITE);
>
>     for(i = 0; i < ps->nb_streams; i++)
>     {
>         printf("%d\n", i);
>
>         iStream = ps->streams[i];
>
>         pC[i] = iStream->codec;
>
>         if(pC[i]->codec_type == AVMEDIA_TYPE_UNKNOWN) {
>                 printf("Skipping bad stream\n");
>                 continue;
>         }
>
>         oStream = avformat_new_stream(ps2, NULL);
>         //avcodec_parameters_copy(oStream->codecpar, iStream->codecpar);
>         p2C[i] = oStream->codec;
>
>         if(pC[i]->codec_type == AVMEDIA_TYPE_VIDEO || pC[i]->codec_type == AVMEDIA_TYPE_AUDIO)
>         {
>             encoder = avcodec_find_encoder(pC[i]->codec_id);
>             if (!encoder) {
>                 av_log(NULL, AV_LOG_FATAL, "Necessary encoder not found\n");
>                 return AVERROR_INVALIDDATA;
>             }
>
>             //AVCodecParameters *pars = avcodec_parameters_alloc();
>             //avcodec_parameters_from_context(pars, pC[i]);
>             //avcodec_parameters_to_context(p2C[i], pars);
>
>             if(pC[i]->codec_type == AVMEDIA_TYPE_VIDEO) {
>                                 p2C[i]->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
>
>                 p2C[i]->height = pC[i]->height;
>                 p2C[i]->width = pC[i]->width;
>                 p2C[i]->sample_aspect_ratio = pC[i]->sample_aspect_ratio;
>                 p2C[i]->gop_size = pC[i]->gop_size;
>                 //take first format from list of supported formats
>                 if (encoder->pix_fmts)
>                     p2C[i]->pix_fmt = encoder->pix_fmts[0];
>                 else
>                     p2C[i]->pix_fmt = pC[i]->pix_fmt;
>                 //video time_base can be set to whatever is handy and supported by encoder
>                 oStream->time_base = iStream->time_base;
>                 p2C[i]->time_base = pC[i]->time_base;
>
>             } else {
>                                 p2C[i]->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
>                 p2C[i]->sample_rate = pC[i]->sample_rate;
>                 p2C[i]->channel_layout = pC[i]->channel_layout;
>                 p2C[i]->channels = av_get_channel_layout_nb_channels(p2C[i]->channel_layout);
>                 // take first format from list of supported formats
>                 p2C[i]->sample_fmt = encoder->sample_fmts[0];
>                 oStream->time_base = (AVRational){1, p2C[i]->sample_rate};
>                 p2C[i]->time_base = (AVRational){1, p2C[i]->sample_rate};
>             }
>
>             //AVCodecParameters *par = avcodec_parameters_alloc();
>             //avcodec_parameters_from_context(par, pC[i]);
>             //avcodec_parameters_to_context(p2C[i], par);
>
>             decoder = avcodec_find_decoder(pC[i]->codec_id);
>             if(decoder == NULL) printf("Couldn't find decoder\n");
>
>             avcodec_open2(pC[i], decoder, NULL);
>             avcodec_open2(p2C[i], encoder, NULL);
>
>             //p2C-> = pC[i]->pts;
>         }
>         else if (pC[i]->codec_type == AVMEDIA_TYPE_UNKNOWN) {
>             av_log(NULL, AV_LOG_FATAL, "Elementary stream #%d is of unknown type, cannot proceed\n", i);
>             //return AVERROR_INVALIDDATA;
>         }
>         else
>         {
>             avcodec_copy_context(oStream->codec, iStream->codec);
>             printf("BUG\n");
>         }
>     }
>     printf("done\n");
>
>     int ret = avformat_write_header(ps2, NULL);
>     char err[200];
>     av_make_error_string(err, 200, ret);
>     printf("Write header %d: %s\n", ret, err);
>
>     unsigned int j = 0;
>     for(;; ++j)
>     {
>         AVPacket pkts;// = av_packet_alloc();
>         AVPacket pktr;// = av_packet_alloc();
>         AVFrame *rawFrame = av_frame_alloc();
>         av_init_packet(&pkts);
>         av_init_packet(&pktr);
>         if(av_read_frame(ps, &pkts) != 0) break;
>
>         int stream_index = pkts.stream_index;
>                 pkts.dts = av_rescale_q_rnd(pkts.dts, ps2->streams[stream_index]->codec->time_base, ps2->streams[stream_index]->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
>                 pkts.pts = av_rescale_q_rnd(pkts.pts, ps->streams[stream_index]->codec->time_base, ps2->streams[stream_index]->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
>                 pkts.duration = av_rescale_q(pkts.duration, ps2->streams[stream_index]->codec->time_base, ps2->streams[stream_index]->time_base);
>
>                 pktr.dts = av_rescale_q_rnd(pktr.dts, ps2->streams[stream_index]->codec->time_base, ps2->streams[stream_index]->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
>                 pktr.pts = av_rescale_q_rnd(pktr.pts, ps->streams[stream_index]->codec->time_base, ps2->streams[stream_index]->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
>                 pktr.duration = av_rescale_q(pktr.duration, ps2->streams[stream_index]->codec->time_base, ps2->streams[stream_index]->time_base);
>
>         int got_out;
>         if(pkts.stream_index == 0)
>         {
>
>             if(avcodec_decode_video2(pC[pkts.stream_index], rawFrame, &got_out, &pkts) >= 0)
>             {
>                 if(avcodec_encode_video2(p2C[pkts.stream_index], &pktr, rawFrame, &got_out) == 0)
>                 {
>                     if(av_interleaved_write_frame(ps2, &pktr) == 0)
>                     {
>                         printf("SUCCESSFULLY WROTE PACKET\n");
>                         //break;
>                     }
>                 } else printf("Failed to encode video\n");
>             } else printf("Failed to decode video\n");
>         }
>         else
>         {
>             if(avcodec_decode_audio4(pC[pkts.stream_index], rawFrame, &got_out, &pkts) >= 0)
>             {
>                 if(avcodec_encode_audio2(p2C[pkts.stream_index], &pktr, rawFrame, &got_out) == 0)
>                 {
>                     if(av_interleaved_write_frame(ps2, &pktr) == 0)
>                     {
>                         printf("SUCCESSFULLY WROTE PACKET\n");
>                         //break;
>                     }
>                 } else printf("Failed to encode audio\n");
>             } else printf("Failed to decode audio\n");
>         }
>         av_frame_free(&rawFrame);
>     }
>
>
>
>     if(av_write_trailer(ps2) == 0) printf("Wrote trailer\n");
> }
>
> ****************************************
>
> Thanks in advance.

Did you tried Valgrind?