[FFmpeg-devel] [PATCH WIPv2 2/2] avdev: add sdl2 device

Josh de Kock josh at itanimul.li
Sat May 21 20:35:04 CEST 2016


Hi Marton,

Sorry for not updating it after we last spoke, I lost my source for the
patch so I was force to do it again. Anyways, here it is. I'm using
SDL_Events for sending the packets, using SDL_WaitEvent to throttle it,
and Conditions to make sure that the queue doesn't build up. Is there
anything I missed?

There's still an issue with the window freezing up though, and on quit
there are "uncommitted CATransaction[s]" (an OSX thing, but I'm sure
that it would still impact other systems if it is a problem on OSX).

I spoke to some of the people in the #sdl IRC channel on freenode, and
they said that, even with most of the SDL functions in the separate
thread, it shouldn't work.

Any and all help would be much appreciated,

Josh

---
 libavdevice/Makefile     |   1 +
 libavdevice/alldevices.c |   1 +
 libavdevice/sdl2.c       | 458 +++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 460 insertions(+)
 create mode 100644 libavdevice/sdl2.c

diff --git a/libavdevice/Makefile b/libavdevice/Makefile
index 585827b..1c4b4d3 100644
--- a/libavdevice/Makefile
+++ b/libavdevice/Makefile
@@ -41,6 +41,7 @@ OBJS-$(CONFIG_PULSE_OUTDEV)              += pulse_audio_enc.o \
                                             pulse_audio_common.o
 OBJS-$(CONFIG_QTKIT_INDEV)               += qtkit.o
 OBJS-$(CONFIG_SDL_OUTDEV)                += sdl.o
+OBJS-$(CONFIG_SDL2_OUTDEV)               += sdl2.o
 OBJS-$(CONFIG_SNDIO_INDEV)               += sndio_dec.o sndio.o
 OBJS-$(CONFIG_SNDIO_OUTDEV)              += sndio_enc.o sndio.o
 OBJS-$(CONFIG_V4L2_INDEV)                += v4l2.o v4l2-common.o timefilter.o
diff --git a/libavdevice/alldevices.c b/libavdevice/alldevices.c
index 26aecf2..c0a9d9a 100644
--- a/libavdevice/alldevices.c
+++ b/libavdevice/alldevices.c
@@ -64,6 +64,7 @@ void avdevice_register_all(void)
     REGISTER_INOUTDEV(PULSE,            pulse);
     REGISTER_INDEV   (QTKIT,            qtkit);
     REGISTER_OUTDEV  (SDL,              sdl);
+    REGISTER_OUTDEV  (SDL2,             sdl2);
     REGISTER_INOUTDEV(SNDIO,            sndio);
     REGISTER_INOUTDEV(V4L2,             v4l2);
 //    REGISTER_INDEV   (V4L,              v4l
diff --git a/libavdevice/sdl2.c b/libavdevice/sdl2.c
new file mode 100644
index 0000000..8c3d49e
--- /dev/null
+++ b/libavdevice/sdl2.c
@@ -0,0 +1,458 @@
+/*
+ * Copyright (c) 2011 Stefano Sabatini
+ * Copyright (c) 2016 Josh de Kock
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * libSDL output device
+ */
+
+#include <SDL.h>
+#include <SDL_thread.h>
+
+#include "libavutil/avstring.h"
+#include "libavutil/imgutils.h"
+#include "libavutil/opt.h"
+#include "libavutil/parseutils.h"
+#include "libavutil/pixdesc.h"
+#include "libavutil/time.h"
+#include "avdevice.h"
+
+typedef struct {
+    AVClass *class;
+    SDL_Window *window;
+    SDL_Renderer *renderer;
+    char *window_title;
+    int window_width, window_height; /**< size of the window */
+    int window_fullscreen;
+    int window_borderless;
+
+    SDL_Texture *texture;
+    int texture_fmt;
+    SDL_Rect texture_rect;
+
+    int sdl_was_already_inited;
+    SDL_Thread *event_thread;
+    SDL_mutex *mutex;
+    SDL_cond *condition;
+    int init_ret; /* return code used to signal initialization errors */
+    int inited;
+    int quit;
+    int pkt_sent;
+} SDLContext;
+
+static const struct sdl_texture_pix_fmt_entry {
+    enum AVPixelFormat pix_fmt; int texture_fmt;
+} sdl_texture_pix_fmt_map[] = {
+    { AV_PIX_FMT_RGB8, SDL_PIXELFORMAT_RGB332 },
+    { AV_PIX_FMT_RGB444, SDL_PIXELFORMAT_RGB444 },
+    { AV_PIX_FMT_RGB555, SDL_PIXELFORMAT_RGB555 },
+    { AV_PIX_FMT_BGR555, SDL_PIXELFORMAT_BGR555 },
+    /* Not implemented in FFmpeg.
+    { AV_PIX_FMT_ARGB4444, SDL_PIXELFORMAT_ARGB4444 },
+    { AV_PIX_FMT_RGBA4444, SDL_PIXELFORMAT_RGBA4444 },
+    { AV_PIX_FMT_ABGR4444, SDL_PIXELFORMAT_ABGR4444 },
+    { AV_PIX_FMT_BGRA4444, SDL_PIXELFORMAT_BGRA4444 },
+    { AV_PIX_FMT_ARGB1555, SDL_PIXELFORMAT_ARGB1555 },
+    { AV_PIX_FMT_RGBA5551, SDL_PIXELFORMAT_RGBA5551 },
+    { AV_PIX_FMT_ABGR1555, SDL_PIXELFORMAT_ABGR1555 },
+    { AV_PIX_FMT_BGRA5551, SDL_PIXELFORMAT_BGRA5551 },
+     */
+    { AV_PIX_FMT_RGB565, SDL_PIXELFORMAT_RGB565 },
+    { AV_PIX_FMT_BGR565, SDL_PIXELFORMAT_BGR565 },
+    { AV_PIX_FMT_RGB24, SDL_PIXELFORMAT_RGB24 },
+    { AV_PIX_FMT_BGR24, SDL_PIXELFORMAT_BGR24 },
+    { AV_PIX_FMT_RGB24, SDL_PIXELFORMAT_RGB888 },
+    { AV_PIX_FMT_RGB0, SDL_PIXELFORMAT_RGBX8888 },
+    { AV_PIX_FMT_BGR24, SDL_PIXELFORMAT_BGR888 },
+    { AV_PIX_FMT_BGR0, SDL_PIXELFORMAT_BGRX8888 },
+    { AV_PIX_FMT_ARGB, SDL_PIXELFORMAT_ARGB8888 },
+    { AV_PIX_FMT_RGBA, SDL_PIXELFORMAT_RGBA8888 },
+    { AV_PIX_FMT_ABGR, SDL_PIXELFORMAT_ABGR8888 },
+    { AV_PIX_FMT_BGRA, SDL_PIXELFORMAT_BGRA8888 },
+    /* Not implemented
+    { AV_PIX_FMT_ARGB2101010, SDL_PIXELFORMAT_ARGB2101010 },
+     */
+    { AV_PIX_FMT_YUV420P, SDL_PIXELFORMAT_IYUV },
+    { AV_PIX_FMT_YUYV422, SDL_PIXELFORMAT_YUY2 },
+    { AV_PIX_FMT_UYVY422, SDL_PIXELFORMAT_UYVY },
+    { AV_PIX_FMT_NONE, 0 },
+};
+
+#define SDL_BASE_FLAGS (SDL_SWSURFACE|SDL_WINDOW_RESIZABLE)
+
+#define FF_PACKET_EVENT (SDL_USEREVENT)
+
+static void compute_texture_rect(AVFormatContext *s)
+{
+    AVRational sar, dar; /* sample and display aspect ratios */
+    SDLContext *sdl = s->priv_data;
+    AVStream *st = s->streams[0];
+    AVCodecParameters *par = s->streams[0]->codecpar;
+    SDL_Rect *texture_rect = &sdl->texture_rect;
+
+    /* compute texture width and height from the codec context information */
+    sar = st->sample_aspect_ratio.num ? st->sample_aspect_ratio : (AVRational){ 1, 1 };
+    dar = av_mul_q(sar, (AVRational){ par->width, par->height });
+
+    /* we suppose the screen has a 1/1 sample aspect ratio */
+    if (sdl->window_width && sdl->window_height) {
+        /* fit in the window */
+        if (av_cmp_q(dar, (AVRational){ sdl->window_width, sdl->window_height }) > 0) {
+            /* fit in width */
+            texture_rect->w = sdl->window_width;
+            texture_rect->h = av_rescale(texture_rect->w, dar.den, dar.num);
+        } else {
+            /* fit in height */
+            texture_rect->h = sdl->window_height;
+            texture_rect->w = av_rescale(texture_rect->h, dar.num, dar.den);
+        }
+    } else {
+        if (sar.num > sar.den) {
+            texture_rect->w = par->width;
+            texture_rect->h = av_rescale(texture_rect->w, dar.den, dar.num);
+        } else {
+            texture_rect->h = par->height;
+            texture_rect->w = av_rescale(texture_rect->h, dar.num, dar.den);
+        }
+        sdl->window_width  = texture_rect->w;
+        sdl->window_height = texture_rect->h;
+    }
+
+    texture_rect->x = (sdl->window_width  - texture_rect->w) / 2;
+    texture_rect->y = (sdl->window_height - texture_rect->h) / 2;
+}
+
+static int sdl2_write_trailer(AVFormatContext *s)
+{
+    SDLContext *sdl = s->priv_data;
+    sdl->quit = 1;
+
+    if (sdl->event_thread)
+        SDL_WaitThread(sdl->event_thread, NULL);
+    sdl->event_thread = NULL;
+    if (sdl->mutex)
+        SDL_DestroyMutex(sdl->mutex);
+    sdl->mutex = NULL;
+    if (sdl->condition)
+        SDL_DestroyCond(sdl->condition);
+    sdl->condition = NULL;
+
+    return 0;
+}
+
+static int event_thread(void *arg)
+{
+    AVFormatContext *s = arg;
+    SDLContext *sdl = s->priv_data;
+    AVCodecParameters *par = s->streams[0]->codecpar;
+
+    int flags = SDL_BASE_FLAGS | (sdl->window_fullscreen ? SDL_WINDOW_FULLSCREEN : 0) |
+                                 (sdl->window_borderless ? SDL_WINDOW_BORDERLESS : 0);
+
+    /* Initialization */
+    if (!sdl->inited){
+        if (SDL_Init(SDL_INIT_VIDEO) != 0) {
+            av_log(s, AV_LOG_ERROR, "Unable to initialize SDL: %s\n", SDL_GetError());
+            sdl->init_ret = AVERROR(EINVAL);
+            goto init_end;
+        }
+    }
+
+    compute_texture_rect(s);
+
+    if (SDL_CreateWindowAndRenderer(sdl->window_width, sdl->window_height,
+                                    flags, &sdl->window, &sdl->renderer) != 0){
+        av_log(sdl, AV_LOG_ERROR, "Couldn't create window and renderer: %s\n", SDL_GetError());
+        sdl->init_ret = AVERROR(EINVAL);
+        goto init_end;
+    }
+
+    SDL_SetWindowTitle(sdl->window, sdl->window_title);
+
+    sdl->texture = SDL_CreateTexture(sdl->renderer, sdl->texture_fmt, SDL_TEXTUREACCESS_STREAMING,
+                                     sdl->window_width, sdl->window_height);
+
+    if (!sdl->texture) {
+        av_log(sdl, AV_LOG_ERROR, "Unable to set create mode: %s\n", SDL_GetError());
+        sdl->init_ret = AVERROR(EINVAL);
+        goto init_end;
+    }
+
+    av_log(s, AV_LOG_VERBOSE, "w:%d h:%d fmt:%s -> w:%d h:%d\n",
+           par->width, par->height, av_get_pix_fmt_name(par->format),
+           sdl->window_width, sdl->window_height);
+
+init_end:
+
+    SDL_LockMutex(sdl->mutex);
+    sdl->inited = 1;
+    sdl->pkt_sent = 0;
+    SDL_UnlockMutex(sdl->mutex);
+    SDL_CondSignal(sdl->condition);
+
+    if (sdl->init_ret < 0)
+        return sdl->init_ret;
+
+    /* Main event loop. */
+    while (!sdl->quit) {
+        SDL_Event event;
+        if (!SDL_WaitEvent(&event)) {
+            av_log(s, AV_LOG_ERROR, "Error when getting SDL event: %s\n", SDL_GetError());
+            continue;
+        }
+        switch (event.type) {
+            case SDL_KEYDOWN:
+                switch (event.key.keysym.sym) {
+                    case SDLK_ESCAPE:
+                    case SDLK_q:
+                        sdl->quit = 1;
+                        break;
+                    default:
+                        break;
+                }
+                break;
+            case SDL_QUIT:
+                sdl->quit = 1;
+                break;
+            case SDL_WINDOWEVENT:
+                switch(event.window.event){
+                    case SDL_WINDOWEVENT_RESIZED:
+                    case SDL_WINDOWEVENT_SIZE_CHANGED:
+                        sdl->window_width  = event.window.data1;
+                        sdl->window_height = event.window.data2;
+                        SDL_LockMutex(sdl->mutex);
+                        compute_texture_rect(s);
+                        SDL_UnlockMutex(sdl->mutex);
+                        break;
+                    default:
+                        break;
+                }
+                break;
+            case FF_PACKET_EVENT:
+            {
+                AVPacket *pkt = event.user.data1;
+                sdl->pkt_sent = 1;
+                SDL_CondSignal(sdl->condition);
+
+                uint8_t *data[4];
+                int linesize[4];
+                av_image_fill_arrays(data, linesize, pkt->data, par->format, par->width, par->height, 1);
+                switch (sdl->texture_fmt) {
+                    case SDL_PIXELFORMAT_IYUV:
+                    case SDL_PIXELFORMAT_YUY2:
+                    case SDL_PIXELFORMAT_UYVY:
+                        SDL_UpdateYUVTexture(sdl->texture, NULL,
+                                             data[0], linesize[0],
+                                             data[1], linesize[1],
+                                             data[2], linesize[2]);
+                        break;
+                    case SDL_PIXELFORMAT_RGB332:
+                    case SDL_PIXELFORMAT_RGB444:
+                    case SDL_PIXELFORMAT_RGB555:
+                    case SDL_PIXELFORMAT_BGR555:
+                    /* Not implemented in FFmpeg.
+                    case SDL_PIXELFORMAT_ARGB4444:
+                    case SDL_PIXELFORMAT_RGBA4444:
+                    case SDL_PIXELFORMAT_ABGR4444:
+                    case SDL_PIXELFORMAT_BGRA4444:
+                    case SDL_PIXELFORMAT_ARGB1555:
+                    case SDL_PIXELFORMAT_RGBA5551:
+                    case SDL_PIXELFORMAT_ABGR1555:
+                    case SDL_PIXELFORMAT_BGRA5551:
+                     */
+                    case SDL_PIXELFORMAT_RGB565:
+                    case SDL_PIXELFORMAT_BGR565:
+                    case SDL_PIXELFORMAT_RGB24:
+                    case SDL_PIXELFORMAT_BGR24:
+                    case SDL_PIXELFORMAT_RGB888:
+                    case SDL_PIXELFORMAT_RGBX8888:
+                    case SDL_PIXELFORMAT_BGR888:
+                    case SDL_PIXELFORMAT_BGRX8888:
+                    case SDL_PIXELFORMAT_ARGB8888:
+                    case SDL_PIXELFORMAT_RGBA8888:
+                    case SDL_PIXELFORMAT_ABGR8888:
+                    case SDL_PIXELFORMAT_BGRA8888:
+                    /* Not implemented.
+                    case SDL_PIXELFORMAT_ARGB2101010:
+                     */
+                        SDL_UpdateTexture(sdl->texture, NULL, data[0], linesize[0]);
+                        break;
+                    default:
+                        av_log(NULL, AV_LOG_FATAL, "Unsupported pixel format\n");
+                        sdl->quit = 1;
+                        break;
+                }
+                SDL_RenderClear(sdl->renderer);
+                SDL_RenderCopy(sdl->renderer, sdl->texture, NULL, &sdl->texture_rect);
+                SDL_RenderPresent(sdl->renderer);
+                av_packet_free(&pkt);
+                break;
+            }
+            default:
+                break;
+        }
+    }
+
+    if (sdl->texture)
+        SDL_DestroyTexture(sdl->texture);
+    sdl->texture = NULL;
+
+    if (sdl->renderer)
+        SDL_DestroyRenderer(sdl->renderer);
+    sdl->renderer = NULL;
+
+    if (sdl->window)
+        SDL_DestroyWindow(sdl->window);
+    sdl->window = NULL;
+
+    if (!sdl->inited)
+        SDL_Quit();
+
+    return sdl->init_ret;
+}
+
+static int sdl2_write_header(AVFormatContext *s)
+{
+    SDLContext *sdl = s->priv_data;
+    AVCodecParameters *par = s->streams[0]->codecpar;
+    int i, ret = 0;
+
+    if (!sdl->window_title)
+        sdl->window_title = av_strdup(s->filename);
+
+    if (SDL_WasInit(SDL_INIT_VIDEO)) {
+        av_log(s, AV_LOG_WARNING,
+               "SDL video subsystem was already inited, you could have multiple SDL outputs. This may cause unknown behaviour.\n");
+        sdl->init_ret = AVERROR(EINVAL);
+        sdl->inited = 1;
+    }
+
+    if (   s->nb_streams > 1
+        || par->codec_type != AVMEDIA_TYPE_VIDEO
+        || par->codec_id   != AV_CODEC_ID_RAWVIDEO) {
+        av_log(s, AV_LOG_ERROR, "Only supports one rawvideo stream\n");
+        sdl->init_ret = AVERROR(EINVAL);
+        goto fail;
+    }
+
+    for (i = 0; sdl_texture_pix_fmt_map[i].pix_fmt != AV_PIX_FMT_NONE; i++) {
+        if (sdl_texture_pix_fmt_map[i].pix_fmt == par->format) {
+            sdl->texture_fmt = sdl_texture_pix_fmt_map[i].texture_fmt;
+            break;
+        }
+    }
+
+    if (!sdl->texture_fmt) {
+        av_log(s, AV_LOG_ERROR,
+               "Unsupported pixel format '%s'\n",
+               av_get_pix_fmt_name(par->format));
+        sdl->init_ret = AVERROR(EINVAL);
+        goto fail;
+    }
+
+    sdl->condition = SDL_CreateCond();
+    if (!sdl->condition) {
+        av_log(s, AV_LOG_ERROR, "Could not create SDL condition variable: %s\n", SDL_GetError());
+        ret = AVERROR_EXTERNAL;
+        goto fail;
+    }
+    sdl->mutex = SDL_CreateMutex();
+    if (!sdl->mutex) {
+        av_log(s, AV_LOG_ERROR, "Could not create SDL mutex: %s\n", SDL_GetError());
+        ret = AVERROR_EXTERNAL;
+        goto fail;
+    }
+    sdl->event_thread = SDL_CreateThread(event_thread, "event_thread", s);
+    if (!sdl->event_thread) {
+        av_log(s, AV_LOG_ERROR, "Could not create SDL event thread: %s\n", SDL_GetError());
+        ret = AVERROR_EXTERNAL;
+        goto fail;
+    }
+
+    /* Wait until the video system has been initiated. */
+    SDL_LockMutex(sdl->mutex);
+    while (!sdl->inited) {
+        SDL_CondWait(sdl->condition, sdl->mutex);
+    }
+
+    SDL_UnlockMutex(sdl->mutex);
+    if (sdl->init_ret < 0) {
+        ret = sdl->init_ret;
+        goto fail;
+    }
+    return 0;
+
+fail:
+    sdl2_write_trailer(s);
+    return ret;
+}
+
+static int sdl2_write_packet(AVFormatContext *s, AVPacket *pkt)
+{
+    SDLContext *sdl = s->priv_data;
+    if(sdl->quit)
+        return sdl2_write_trailer(s);
+
+    AVPacket *ref_pkt = av_packet_alloc();
+    av_packet_ref(ref_pkt, pkt);
+    SDL_Event event;
+    event.type = FF_PACKET_EVENT;
+    event.user.data1 = ref_pkt;
+    SDL_PushEvent(&event);
+
+    SDL_LockMutex(sdl->mutex);
+    while(!sdl->pkt_sent)
+        SDL_CondWait(sdl->condition, sdl->mutex);
+    sdl->pkt_sent = 0;
+    SDL_UnlockMutex(sdl->mutex);
+    return 0;
+}
+
+#define OFFSET(x) offsetof(SDLContext,x)
+
+static const AVOption options[] = {
+    { "window_title", "set SDL window title",           OFFSET(window_title), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, AV_OPT_FLAG_ENCODING_PARAM },
+    { "window_size",  "set SDL window forced size",     OFFSET(window_width), AV_OPT_TYPE_IMAGE_SIZE, { .str = NULL }, 0, 0, AV_OPT_FLAG_ENCODING_PARAM },
+    { "window_fullscreen", "set SDL window fullscreen", OFFSET(window_fullscreen), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, AV_OPT_FLAG_ENCODING_PARAM },
+    { "window_borderless", "set SDL window border off", OFFSET(window_fullscreen), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, AV_OPT_FLAG_ENCODING_PARAM },
+    { NULL },
+};
+
+static const AVClass sdl2_class = {
+    .class_name = "sdl2 outdev",
+    .item_name  = av_default_item_name,
+    .option     = options,
+    .version    = LIBAVUTIL_VERSION_INT,
+    .category   = AV_CLASS_CATEGORY_DEVICE_VIDEO_OUTPUT,
+};
+
+AVOutputFormat ff_sdl2_muxer = {
+    .name           = "sdl2",
+    .long_name      = NULL_IF_CONFIG_SMALL("SDL2 output device"),
+    .priv_data_size = sizeof(SDLContext),
+    .audio_codec    = AV_CODEC_ID_NONE,
+    .video_codec    = AV_CODEC_ID_RAWVIDEO,
+    .write_header   = sdl2_write_header,
+    .write_packet   = sdl2_write_packet,
+    .write_trailer  = sdl2_write_trailer,
+    .flags          = AVFMT_NOFILE | AVFMT_VARIABLE_FPS | AVFMT_NOTIMESTAMPS,
+    .priv_class     = &sdl2_class,
+};
\ No newline at end of file
-- 
2.6.4 (Apple Git-63)



More information about the ffmpeg-devel mailing list