[FFmpeg-devel] [PATCH v1] fftools/ffmpeg: Add support for per frame rotation and flip

Jun Li junli1026 at gmail.com
Sun May 12 08:57:01 EEST 2019


On Sat, May 11, 2019 at 10:47 PM Jun Li <junli1026 at gmail.com> wrote:

> Fix #6945
> Current implementaion for autorotate works fine for stream
> level rotataion but no support for frame level operation
> and frame flip. This patch is for adding flip support and
> per frame operations.
> ---
>  fftools/cmdutils.c        |  9 ++---
>  fftools/cmdutils.h        |  2 +-
>  fftools/ffmpeg.c          | 21 +++++++++-
>  fftools/ffmpeg.h          |  2 +
>  fftools/ffmpeg_filter.c   | 81 ++++++++++++++++++++++++++++++++++++---
>  fftools/ffplay.c          | 28 +++++++++++---
>  libavutil/display.c       | 14 +++++++
>  libavutil/display.h       | 14 +++++++
>  libavutil/tests/display.c |  8 ++++
>  tests/ref/fate/display    |  4 ++
>  10 files changed, 166 insertions(+), 17 deletions(-)
>
> diff --git a/fftools/cmdutils.c b/fftools/cmdutils.c
> index 9cfbc45c2b..b8bb904419 100644
> --- a/fftools/cmdutils.c
> +++ b/fftools/cmdutils.c
> @@ -2172,13 +2172,12 @@ void *grow_array(void *array, int elem_size, int
> *size, int new_size)
>      return array;
>  }
>
> -double get_rotation(AVStream *st)
> +double get_rotation_hflip(int32_t display_matrix[9], int* hflip)
>  {
> -    uint8_t* displaymatrix = av_stream_get_side_data(st,
> -
>  AV_PKT_DATA_DISPLAYMATRIX, NULL);
>      double theta = 0;
> -    if (displaymatrix)
> -        theta = -av_display_rotation_get((int32_t*) displaymatrix);
> +
> +    if (display_matrix)
> +        theta = -av_display_rotation_hflip_get(display_matrix, hflip);
>
>      theta -= 360*floor(theta/360 + 0.9/360);
>
> diff --git a/fftools/cmdutils.h b/fftools/cmdutils.h
> index 6e2e0a2acb..dce44ed6e1 100644
> --- a/fftools/cmdutils.h
> +++ b/fftools/cmdutils.h
> @@ -643,6 +643,6 @@ void *grow_array(void *array, int elem_size, int
> *size, int new_size);
>      char name[128];\
>      av_get_channel_layout_string(name, sizeof(name), 0, ch_layout);
>
> -double get_rotation(AVStream *st);
> +double get_rotation_hflip(int32_t display_matrix[9], int* hflip);
>
>  #endif /* FFTOOLS_CMDUTILS_H */
> diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
> index 01f04103cf..9ea1aaa930 100644
> --- a/fftools/ffmpeg.c
> +++ b/fftools/ffmpeg.c
> @@ -2126,6 +2126,24 @@ static int
> ifilter_has_all_input_formats(FilterGraph *fg)
>      return 1;
>  }
>
> +static int ifilter_display_matrix_need_from_frame(InputFilter *ifilter,
> AVFrame* frame)
> +{
> +    int32_t* stream_matrix =
> (int32_t*)av_stream_get_side_data(ifilter->ist->st,
> +
> AV_PKT_DATA_DISPLAYMATRIX, NULL);
> +    // if we already have display matrix from stream, use it instead of
> extracting
> +    // from frame.
> +    if (stream_matrix) {
> +        memcpy(ifilter->display_matrix, stream_matrix, 4 * 9);
> +        return 0;
> +    }
> +
> +    // cleanup the display matrix, may be from last frame
> +    memset(ifilter->display_matrix, 0, 4 * 9);
> +    av_display_rotation_set(ifilter->display_matrix, 0);
> +
> +    return !!av_dict_get(frame->metadata, "Orientation", NULL, 0);
> +}
> +
>  static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
>  {
>      FilterGraph *fg = ifilter->graph;
> @@ -2141,7 +2159,8 @@ static int ifilter_send_frame(InputFilter *ifilter,
> AVFrame *frame)
>                         ifilter->channel_layout != frame->channel_layout;
>          break;
>      case AVMEDIA_TYPE_VIDEO:
> -        need_reinit |= ifilter->width  != frame->width ||
> +        need_reinit |= ifilter_display_matrix_need_from_frame(ifilter,
> frame) ||
> +                       ifilter->width  != frame->width ||
>                         ifilter->height != frame->height;
>          break;
>      }
> diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
> index eb1eaf6363..8331720663 100644
> --- a/fftools/ffmpeg.h
> +++ b/fftools/ffmpeg.h
> @@ -251,6 +251,8 @@ typedef struct InputFilter {
>      int channels;
>      uint64_t channel_layout;
>
> +    int32_t display_matrix[9];
> +
>      AVBufferRef *hw_frames_ctx;
>
>      int eof;
> diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
> index 72838de1e2..1894f30561 100644
> --- a/fftools/ffmpeg_filter.c
> +++ b/fftools/ffmpeg_filter.c
> @@ -328,6 +328,7 @@ static void init_input_filter(FilterGraph *fg,
> AVFilterInOut *in)
>      fg->inputs[fg->nb_inputs - 1]->format = -1;
>      fg->inputs[fg->nb_inputs - 1]->type = ist->st->codecpar->codec_type;
>      fg->inputs[fg->nb_inputs - 1]->name = describe_filter_link(fg, in, 1);
> +    av_display_rotation_set(fg->inputs[fg->nb_inputs -
> 1]->display_matrix, 0);
>
>      fg->inputs[fg->nb_inputs - 1]->frame_queue = av_fifo_alloc(8 *
> sizeof(AVFrame*));
>      if (!fg->inputs[fg->nb_inputs - 1]->frame_queue)
> @@ -807,22 +808,43 @@ static int configure_input_video_filter(FilterGraph
> *fg, InputFilter *ifilter,
>      last_filter = ifilter->filter;
>
>      if (ist->autorotate) {
> -        double theta = get_rotation(ist->st);
> +        int hflip = 0;
> +        double theta = get_rotation_hflip(ifilter->display_matrix,
> &hflip);
>
> -        if (fabs(theta - 90) < 1.0) {
> +        if (fabs(theta) < 1.0) {
> +            if (hflip)
> +                ret = insert_filter(&last_filter, &pad_idx, "hflip",
> NULL);
> +        } else if (fabs(theta - 90) < 1.0) {
>              ret = insert_filter(&last_filter, &pad_idx, "transpose",
> "clock");
> -        } else if (fabs(theta - 180) < 1.0) {
> -            ret = insert_filter(&last_filter, &pad_idx, "hflip", NULL);
>              if (ret < 0)
>                  return ret;
> -            ret = insert_filter(&last_filter, &pad_idx, "vflip", NULL);
> +            if (hflip)
> +                ret = insert_filter(&last_filter, &pad_idx, "hflip",
> NULL);
> +        } else if (fabs(theta - 180) < 1.0) {
> +            if (hflip) { // rotate 180 and hflip equals vflip
> +                ret = insert_filter(&last_filter, &pad_idx, "vflip",
> NULL);
> +            } else {
> +                ret = insert_filter(&last_filter, &pad_idx, "hflip",
> NULL);
> +                if (ret < 0)
> +                    return ret;
> +                ret = insert_filter(&last_filter, &pad_idx, "vflip",
> NULL);
> +            }
>          } else if (fabs(theta - 270) < 1.0) {
>              ret = insert_filter(&last_filter, &pad_idx, "transpose",
> "cclock");
> +            if (ret < 0)
> +                return ret;
> +            if (hflip)
> +                ret = insert_filter(&last_filter, &pad_idx, "hflip",
> NULL);
>          } else if (fabs(theta) > 1.0) {
>              char rotate_buf[64];
>              snprintf(rotate_buf, sizeof(rotate_buf), "%f*PI/180", theta);
>              ret = insert_filter(&last_filter, &pad_idx, "rotate",
> rotate_buf);
> +            if (ret < 0)
> +                return ret;
> +            if (hflip)
> +                ret = insert_filter(&last_filter, &pad_idx, "hflip",
> NULL);
>          }
> +
>          if (ret < 0)
>              return ret;
>      }
> @@ -1182,6 +1204,53 @@ fail:
>      return ret;
>  }
>
> +static void set_display_matrix_from_frame(const AVFrame *frame, int32_t
> m[9])
> +{
> +    AVDictionaryEntry *entry = NULL;
> +    int orientation = 0;
> +
> +    // read exif orientation data
> +    entry = av_dict_get(frame->metadata, "Orientation", NULL, 0);
> +    if (entry) {
> +        orientation = atoi(entry->value);
> +        memset(m, 0, 4 * 9);
> +        switch (orientation)
> +        {
> +            case 1: // horizontal (normal)
> +                av_display_rotation_set(m, 0.0);
> +                break;
> +            case 2: // mirror horizontal
> +                av_display_rotation_set(m, 0.0);
> +                av_display_matrix_flip(m, 1, 0);
> +                break;
> +            case 3: // rotate 180
> +                av_display_rotation_set(m, 180.0);
> +                break;
> +            case 4: // mirror vertical
> +                av_display_rotation_set(m, 0.0);
> +                av_display_matrix_flip(m, 0, 1);
> +                break;
> +            case 5: // mirror horizontal and rotate 270 CW
> +                av_display_rotation_set(m, 270.0);
> +                av_display_matrix_flip(m, 0, 1);
> +                break;
> +            case 6: // rotate 90 CW
> +                av_display_rotation_set(m, 90.0);
> +                break;
> +            case 7: // mirror horizontal and rotate 90 CW
> +                av_display_rotation_set(m, 90.0);
> +                av_display_matrix_flip(m, 0, 1);
> +                break;
> +            case 8: // rotate 270 CW
> +                av_display_rotation_set(m, 270.0);
> +                break;
> +            default:
> +                av_display_rotation_set(m, 0.0);
> +                break;
> +        }
> +    }
> +}
> +
>  int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame
> *frame)
>  {
>      av_buffer_unref(&ifilter->hw_frames_ctx);
> @@ -1202,6 +1271,8 @@ int ifilter_parameters_from_frame(InputFilter
> *ifilter, const AVFrame *frame)
>              return AVERROR(ENOMEM);
>      }
>
> +    set_display_matrix_from_frame(frame, ifilter->display_matrix);
> +
>      return 0;
>  }
>
> diff --git a/fftools/ffplay.c b/fftools/ffplay.c
> index 8f050e16e6..717a9a830c 100644
> --- a/fftools/ffplay.c
> +++ b/fftools/ffplay.c
> @@ -1914,19 +1914,37 @@ static int configure_video_filters(AVFilterGraph
> *graph, VideoState *is, const c
>  } while (0)
>
>      if (autorotate) {
> -        double theta  = get_rotation(is->video_st);
> -
> -        if (fabs(theta - 90) < 1.0) {
> +        int hflip = 0;
> +        double theta = 0.0;
> +        int32_t* display_matrix =
> (int32_t*)av_stream_get_side_data(is->video_st,
> +
> AV_PKT_DATA_DISPLAYMATRIX, NULL);
> +        if (display_matrix)
> +             theta  = get_rotation_hflip(display_matrix, &hflip);
> +
> +        if (fabs(theta) < 1.0) {
> +            if (hflip)
> +                INSERT_FILT("hflip", NULL);
> +        } else if (fabs(theta - 90) < 1.0) {
>              INSERT_FILT("transpose", "clock");
> +            if (hflip)
> +                INSERT_FILT("hflip", NULL);
>          } else if (fabs(theta - 180) < 1.0) {
> -            INSERT_FILT("hflip", NULL);
> -            INSERT_FILT("vflip", NULL);
> +            if (hflip) { // rotate 180 and hflip equals vflip
> +                INSERT_FILT("vflip", NULL);
> +            } else {
> +                INSERT_FILT("hflip", NULL);
> +                INSERT_FILT("vflip", NULL);
> +            }
>          } else if (fabs(theta - 270) < 1.0) {
>              INSERT_FILT("transpose", "cclock");
> +            if (hflip)
> +                INSERT_FILT("hflip", NULL);
>          } else if (fabs(theta) > 1.0) {
>              char rotate_buf[64];
>              snprintf(rotate_buf, sizeof(rotate_buf), "%f*PI/180", theta);
>              INSERT_FILT("rotate", rotate_buf);
> +            if (hflip)
> +                INSERT_FILT("hflip", NULL);
>          }
>      }
>
> diff --git a/libavutil/display.c b/libavutil/display.c
> index a0076e067b..855de4a006 100644
> --- a/libavutil/display.c
> +++ b/libavutil/display.c
> @@ -71,3 +71,17 @@ void av_display_matrix_flip(int32_t matrix[9], int
> hflip, int vflip)
>          for (i = 0; i < 9; i++)
>              matrix[i] *= flip[i % 3];
>  }
> +
> +double av_display_rotation_hflip_get(const int32_t matrix[9], int *hflip)
> +{
> +    int32_t m[9];
> +    *hflip = 0;
> +    memcpy(m, matrix, sizeof(m));
> +
> +    if (m[0] > 0 && m[4] < 0 || m[0] < 0 && m[4] > 0 ||
> +        m[1] > 0 && m[3] > 0 || m[1] < 0 && m[3] < 0) {
> +        *hflip = 1;
> +        av_display_matrix_flip(m, 1, 0);
> +    }
> +    return av_display_rotation_get(m);
> +}
> \ No newline at end of file
> diff --git a/libavutil/display.h b/libavutil/display.h
> index 515adad795..23d71a0bc5 100644
> --- a/libavutil/display.h
> +++ b/libavutil/display.h
> @@ -106,6 +106,20 @@ void av_display_rotation_set(int32_t matrix[9],
> double angle);
>   */
>  void av_display_matrix_flip(int32_t matrix[9], int hflip, int vflip);
>
> +/**
> + * Extract the rotation component and hflip status of the transformation
> matrix.
> + *
> + * @param matrix the transformation matrix
> + * @param hflip wille be set to 1 if the matrix need horizontal flipped
> + * @return the angle (in degrees) by which the transformation rotates the
> frame
> + *         counterclockwise. The angle will be in range [-180.0, 180.0],
> + *         or NaN if the matrix is singular.
> + *
> + * @note floating point numbers are inherently inexact, so callers are
> + *       recommended to round the return value to nearest integer before
> use.
> + */
> +double av_display_rotation_hflip_get(const int32_t matrix[9], int* hflip);
> +
>  /**
>   * @}
>   * @}
> diff --git a/libavutil/tests/display.c b/libavutil/tests/display.c
> index 893ebb5543..65a0971e7b 100644
> --- a/libavutil/tests/display.c
> +++ b/libavutil/tests/display.c
> @@ -35,6 +35,8 @@ static void print_matrix(int32_t matrix[9])
>  int main(void)
>  {
>      int32_t matrix[9];
> +    int hflip = 0;
> +    double degree;
>
>      // Set the matrix to 90 degrees
>      av_display_rotation_set(matrix, 90);
> @@ -56,6 +58,12 @@ int main(void)
>      print_matrix(matrix);
>      printf("degrees: %f\n", av_display_rotation_get(matrix));
>
> +    // flip vertical
> +    av_display_matrix_flip(matrix, 0, 1);
> +    print_matrix(matrix);
> +    degree = av_display_rotation_hflip_get(matrix, &hflip);
> +    printf("degrees: %f, hflip: %i\n", degree, hflip);
> +
>      return 0;
>
>  }
> diff --git a/tests/ref/fate/display b/tests/ref/fate/display
> index 251e7e0cdf..14b4c7db7c 100644
> --- a/tests/ref/fate/display
> +++ b/tests/ref/fate/display
> @@ -14,3 +14,7 @@ degrees: 135.000000
>  -46340 -46340 0
>  0 0 1073741824
>  degrees: -135.000000
> +-46340 -46340 0
> +-46340 46340 0
> +0 0 1073741824
> +degrees: 45.000000, hflip: 1
> --
> 2.17.1


This change is to address the feedback of last patch using 1st frame's
orientation as stream rotation.
Thanks again for review. I added the support for per frame rotation.

(I know it involves too many file changes and not easy to read. Sorry about
that I tried to
 separate them but failed since due to some dependencies).

Best Regards,
Jun


More information about the ffmpeg-devel mailing list