August 2019 - ffmpeg-cvslog - lists.ffmpeg.org

avfilter/vf_v360: add cubemap 1x6 layout
by Paul B Mahol 31 Aug '19

31 Aug '19

ffmpeg | branch: master | Paul B Mahol <onemda(a)gmail.com> | Sat Aug 31 14:52:32 2019 +0200| [c79d6728a75528b7fa77035b089b456a953b8e3f] | committer: Paul B Mahol avfilter/vf_v360: add cubemap 1x6 layout > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=c79d6728a75528b7f… --- doc/filters.texi | 3 +- libavfilter/vf_v360.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/doc/filters.texi b/doc/filters.texi index 2271749e49..4a222796c7 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -17932,7 +17932,8 @@ Equirectangular projection. @item c3x2 @item c6x1 -Cubemap with 3x2/6x1 layout. +@item c1x6 +Cubemap with 3x2/6x1/1x6 layout. Format specific options: diff --git a/libavfilter/vf_v360.c b/libavfilter/vf_v360.c index 9b7f0c109e..78e843c0c1 100644 --- a/libavfilter/vf_v360.c +++ b/libavfilter/vf_v360.c @@ -49,6 +49,7 @@ enum Projections { FLAT, DUAL_FISHEYE, FACEBOOK, + CUBEMAP_1_6, NB_PROJECTIONS, }; @@ -139,6 +140,7 @@ static const AVOption v360_options[] = { { "eac", "equi-angular", 0, AV_OPT_TYPE_CONST, {.i64=EQUIANGULAR}, 0, 0, FLAGS, "in" }, { "dfisheye", "dual fisheye", 0, AV_OPT_TYPE_CONST, {.i64=DUAL_FISHEYE}, 0, 0, FLAGS, "in" }, { "fb", "facebook's 360 format", 0, AV_OPT_TYPE_CONST, {.i64=FACEBOOK}, 0, 0, FLAGS, "in" }, + { "c1x6", "cubemap1x6", 0, AV_OPT_TYPE_CONST, {.i64=CUBEMAP_1_6}, 0, 0, FLAGS, "in" }, { "output", "set output projection", OFFSET(out), AV_OPT_TYPE_INT, {.i64=CUBEMAP_3_2}, 0, NB_PROJECTIONS-1, FLAGS, "out" }, { "e", "equirectangular", 0, AV_OPT_TYPE_CONST, {.i64=EQUIRECTANGULAR}, 0, 0, FLAGS, "out" }, { "c3x2", "cubemap3x2", 0, AV_OPT_TYPE_CONST, {.i64=CUBEMAP_3_2}, 0, 0, FLAGS, "out" }, @@ -146,6 +148,7 @@ static const AVOption v360_options[] = { { "eac", "equi-angular", 0, AV_OPT_TYPE_CONST, {.i64=EQUIANGULAR}, 0, 0, FLAGS, "out" }, { "flat", "regular video", 0, AV_OPT_TYPE_CONST, {.i64=FLAT}, 0, 0, FLAGS, "out" }, { "fb", "facebook's 360 format", 0, AV_OPT_TYPE_CONST, {.i64=FACEBOOK}, 0, 0, FLAGS, "out" }, + { "c1x6", "cubemap1x6", 0, AV_OPT_TYPE_CONST, {.i64=CUBEMAP_1_6}, 0, 0, FLAGS, "out" }, { "interp", "set interpolation method", OFFSET(interp), AV_OPT_TYPE_INT, {.i64=BILINEAR}, 0, NB_INTERP_METHODS-1, FLAGS, "interp" }, { "near", "nearest neighbour", 0, AV_OPT_TYPE_CONST, {.i64=NEAREST}, 0, 0, FLAGS, "interp" }, { "nearest", "nearest neighbour", 0, AV_OPT_TYPE_CONST, {.i64=NEAREST}, 0, 0, FLAGS, "interp" }, @@ -1165,6 +1168,34 @@ static void xyz_to_cube3x2(const V360Context *s, } /** + * Calculate 3D coordinates on sphere for corresponding frame position in cubemap1x6 format. + * + * @param s filter context + * @param i horizontal position on frame [0, height) + * @param j vertical position on frame [0, width) + * @param width frame width + * @param height frame height + * @param vec coordinates on sphere + */ +static void cube1x6_to_xyz(const V360Context *s, + int i, int j, int width, int height, + float *vec) +{ + const float ew = width; + const float eh = height / 6.f; + + const int face = floorf(j / eh); + + const int v_shift = ceilf(eh * face); + const int ehi = ceilf(eh * (face + 1)) - v_shift; + + const float uf = 2.f * i / ew - 1.f; + const float vf = 2.f * (j - v_shift) / ehi - 1.f; + + cube_to_xyz(s, uf, vf, face, vec); +} + +/** * Calculate 3D coordinates on sphere for corresponding frame position in cubemap6x1 format. * * @param s filter context @@ -1193,6 +1224,84 @@ static void cube6x1_to_xyz(const V360Context *s, } /** + * Calculate frame position in cubemap1x6 format for corresponding 3D coordinates on sphere. + * + * @param s filter context + * @param vec coordinates on sphere + * @param width frame width + * @param height frame height + * @param us horizontal coordinates for interpolation window + * @param vs vertical coordinates for interpolation window + * @param du horizontal relative coordinate + * @param dv vertical relative coordinate + */ +static void xyz_to_cube1x6(const V360Context *s, + const float *vec, int width, int height, + uint16_t us[4][4], uint16_t vs[4][4], float *du, float *dv) +{ + const float eh = height / 6.f; + const int ewi = width; + float uf, vf; + int ui, vi; + int ehi; + int i, j; + int direction, face; + + xyz_to_cube(s, vec, &uf, &vf, &direction); + + uf *= (1.f - s->in_pad); + vf *= (1.f - s->in_pad); + + face = s->in_cubemap_face_order[direction]; + ehi = ceilf(eh * (face + 1)) - ceilf(eh * face); + + uf = 0.5f * ewi * (uf + 1.f); + vf = 0.5f * ehi * (vf + 1.f); + + ui = floorf(uf); + vi = floorf(vf); + + *du = uf - ui; + *dv = vf - vi; + + for (i = -1; i < 3; i++) { + for (j = -1; j < 3; j++) { + int new_ui = ui + j; + int new_vi = vi + i; + int v_shift; + int new_ehi; + + if (new_ui >= 0 && new_ui < ewi && new_vi >= 0 && new_vi < ehi) { + face = s->in_cubemap_face_order[direction]; + + v_shift = ceilf(eh * face); + } else { + uf = 2.f * new_ui / ewi - 1.f; + vf = 2.f * new_vi / ehi - 1.f; + + uf /= (1.f - s->in_pad); + vf /= (1.f - s->in_pad); + + process_cube_coordinates(s, uf, vf, direction, &uf, &vf, &face); + + uf *= (1.f - s->in_pad); + vf *= (1.f - s->in_pad); + + v_shift = ceilf(eh * face); + new_ehi = ceilf(eh * (face + 1)) - v_shift; + + new_ui = av_clip(roundf(0.5f * ewi * (uf + 1.f)), 0, ewi - 1); + new_vi = av_clip(roundf(0.5f * new_ehi * (vf + 1.f)), 0, new_ehi - 1); + } + + + us[i + 1][j + 1] = new_ui; + vs[i + 1][j + 1] = v_shift + new_vi; + } + } +} + +/** * Calculate frame position in cubemap6x1 format for corresponding 3D coordinates on sphere. * * @param s filter context @@ -1918,6 +2027,12 @@ static int config_output(AVFilterLink *outlink) wf = inlink->w / 3.f * 4.f; hf = inlink->h; break; + case CUBEMAP_1_6: + in_transform = xyz_to_cube1x6; + err = prepare_cube_in(ctx); + wf = inlink->w * 4.f; + hf = inlink->h / 3.f; + break; case CUBEMAP_6_1: in_transform = xyz_to_cube6x1; err = prepare_cube_in(ctx); @@ -1967,6 +2082,12 @@ static int config_output(AVFilterLink *outlink) w = roundf(wf / 4.f * 3.f); h = roundf(hf); break; + case CUBEMAP_1_6: + out_transform = cube1x6_to_xyz; + err = prepare_cube_out(ctx); + w = roundf(wf / 4.f); + h = roundf(hf * 3.f); + break; case CUBEMAP_6_1: out_transform = cube6x1_to_xyz; err = prepare_cube_out(ctx);

1 0

avcodec/vorbisdec: Check get_vlc2() failure
by Michael Niedermayer 30 Aug '19

30 Aug '19

ffmpeg | branch: master | Michael Niedermayer <michael(a)niedermayer.cc> | Mon Jul 8 01:18:05 2019 +0200| [07b948fe60789064d7c784d47b8fe798a9a4d2b9] | committer: Michael Niedermayer avcodec/vorbisdec: Check get_vlc2() failure Fixes: out of array read Fixes: 16510/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_VORBIS_fuzzer-5754510382727168 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer <michael(a)niedermayer.cc> > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=07b948fe60789064d… --- libavcodec/vorbisdec.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/libavcodec/vorbisdec.c b/libavcodec/vorbisdec.c index 1045d574b1..104dc86a13 100644 --- a/libavcodec/vorbisdec.c +++ b/libavcodec/vorbisdec.c @@ -1442,7 +1442,7 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc, int vqbook = vr->books[vqclass][pass]; if (vqbook >= 0 && vc->codebooks[vqbook].codevectors) { - unsigned coffs; + int coffs; unsigned dim = vc->codebooks[vqbook].dimensions; unsigned step = FASTDIV(vr->partition_size << 1, dim << 1); vorbis_codebook codebook = vc->codebooks[vqbook]; @@ -1451,14 +1451,20 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc, voffs = voffset+j*vlen; for (k = 0; k < step; ++k) { - coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3) * dim; + coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3); + if (coffs < 0) + return coffs; + coffs *= dim; for (l = 0; l < dim; ++l) vec[voffs + k + l * step] += codebook.codevectors[coffs + l]; } } else if (vr_type == 1) { voffs = voffset + j * vlen; for (k = 0; k < step; ++k) { - coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3) * dim; + coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3); + if (coffs < 0) + return coffs; + coffs *= dim; for (l = 0; l < dim; ++l, ++voffs) { vec[voffs]+=codebook.codevectors[coffs+l]; @@ -1471,13 +1477,19 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc, if (dim == 2) { for (k = 0; k < step; ++k) { - coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3) * 2; + coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3); + if (coffs < 0) + return coffs; + coffs *= 2; vec[voffs + k ] += codebook.codevectors[coffs ]; vec[voffs + k + vlen] += codebook.codevectors[coffs + 1]; } } else if (dim == 4) { for (k = 0; k < step; ++k, voffs += 2) { - coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3) * 4; + coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3); + if (coffs < 0) + return coffs; + coffs *= 4; vec[voffs ] += codebook.codevectors[coffs ]; vec[voffs + 1 ] += codebook.codevectors[coffs + 2]; vec[voffs + vlen ] += codebook.codevectors[coffs + 1]; @@ -1485,7 +1497,10 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc, } } else for (k = 0; k < step; ++k) { - coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3) * dim; + coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3); + if (coffs < 0) + return coffs; + coffs *= dim; for (l = 0; l < dim; l += 2, voffs++) { vec[voffs ] += codebook.codevectors[coffs + l ]; vec[voffs + vlen] += codebook.codevectors[coffs + l + 1]; @@ -1502,7 +1517,10 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc, unsigned voffs_mod = voffset - voffs_div * ch; for (k = 0; k < step; ++k) { - coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3) * dim; + coffs = get_vlc2(gb, codebook.vlc.table, codebook.nb_bits, 3); + if (coffs < 0) + return coffs; + coffs *= dim; for (l = 0; l < dim; ++l) { vec[voffs_div + voffs_mod * vlen] += codebook.codevectors[coffs + l];

1 0

avfilter/vf_stereo3d: improve dubois anaglyphs
by Paul B Mahol 30 Aug '19

30 Aug '19

ffmpeg | branch: master | Paul B Mahol <onemda(a)gmail.com> | Fri Aug 30 22:35:51 2019 +0200| [62459d6584ada67b3fb5f38ec86904784c2ab331] | committer: Paul B Mahol avfilter/vf_stereo3d: improve dubois anaglyphs > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=62459d6584ada67b3… --- libavfilter/vf_stereo3d.c | 12 ++++++------ tests/ref/fate/filter-stereo3d-sbsl-arcd | 10 +++++----- tests/ref/fate/filter-stereo3d-sbsl-aybd | 10 +++++----- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/libavfilter/vf_stereo3d.c b/libavfilter/vf_stereo3d.c index 8b22f880ca..0e39fb3931 100644 --- a/libavfilter/vf_stereo3d.c +++ b/libavfilter/vf_stereo3d.c @@ -100,9 +100,9 @@ static const int ana_coeff[][3][6] = { { 0, 0, 0, 0, 65536, 0}, { 0, 0, 0, 0, 0, 65536}}, [ANAGLYPH_RC_DUBOIS] = - {{29891, 32800, 11559, -2849, -5763, -102}, - {-2627, -2479, -1033, 24804, 48080, -1209}, - { -997, -1350, -358, -4729, -7403, 80373}}, + {{29884, 32768, 11534, -2818, -5767, -131}, + {-2621, -2490, -1049, 24773, 48103, -1180}, + { -983, -1376, -328, -4719, -7406, 80347}}, [ANAGLYPH_GM_GRAY] = {{ 0, 0, 0, 19595, 38470, 7471}, {19595, 38470, 7471, 0, 0, 0}, @@ -132,9 +132,9 @@ static const int ana_coeff[][3][6] = { { 0, 0, 0, 0, 65536, 0}, { 0, 0, 65536, 0, 0, 0}}, [ANAGLYPH_YB_DUBOIS] = - {{65535,-12650,18451, -987, -7590, -1049}, - {-1604, 56032, 4196, 370, 3826, -1049}, - {-2345,-10676, 1358, 5801, 11416, 56217}}, + {{69599,-13435,19595, -1048, -8061, -1114}, + {-1704, 59507, 4456, 393, 4063, -1114}, + {-2490,-11338, 1442, 6160, 12124, 59703}}, }; typedef struct Stereo3DContext { diff --git a/tests/ref/fate/filter-stereo3d-sbsl-arcd b/tests/ref/fate/filter-stereo3d-sbsl-arcd index b5f11e278e..20f080b973 100644 --- a/tests/ref/fate/filter-stereo3d-sbsl-arcd +++ b/tests/ref/fate/filter-stereo3d-sbsl-arcd @@ -3,8 +3,8 @@ #codec_id 0: rawvideo #dimensions 0: 176x288 #sar 0: 0/1 -0, 0, 0, 1, 152064, 0xa0261570 -0, 1, 1, 1, 152064, 0x678403c8 -0, 2, 2, 1, 152064, 0x1087e7b6 -0, 3, 3, 1, 152064, 0xa3909df3 -0, 4, 4, 1, 152064, 0x87e4c4d4 +0, 0, 0, 1, 152064, 0x34b1fb03 +0, 1, 1, 1, 152064, 0x2045eadd +0, 2, 2, 1, 152064, 0x1266cdde +0, 3, 3, 1, 152064, 0xc9f083bd +0, 4, 4, 1, 152064, 0x0701ab16 diff --git a/tests/ref/fate/filter-stereo3d-sbsl-aybd b/tests/ref/fate/filter-stereo3d-sbsl-aybd index 0aa4e8bc2c..277e66bdca 100644 --- a/tests/ref/fate/filter-stereo3d-sbsl-aybd +++ b/tests/ref/fate/filter-stereo3d-sbsl-aybd @@ -3,8 +3,8 @@ #codec_id 0: rawvideo #dimensions 0: 176x288 #sar 0: 0/1 -0, 0, 0, 1, 152064, 0x7dc98468 -0, 1, 1, 1, 152064, 0xf72db6c9 -0, 2, 2, 1, 152064, 0x1630f53f -0, 3, 3, 1, 152064, 0xc1765599 -0, 4, 4, 1, 152064, 0x12e35db1 +0, 0, 0, 1, 152064, 0xf576742c +0, 1, 1, 1, 152064, 0x2ce36ae4 +0, 2, 2, 1, 152064, 0x2dc99849 +0, 3, 3, 1, 152064, 0x940413b6 +0, 4, 4, 1, 152064, 0x48a600d3

1 0

avcodec/vaapi_encode: Simplify code with av_clip_int8()
by Michael Niedermayer 30 Aug '19

30 Aug '19

ffmpeg | branch: master | Michael Niedermayer <michael(a)niedermayer.cc> | Wed Aug 28 19:26:49 2019 +0200| [9b57b90c4c2752426c6552a87a96821ff9708fac] | committer: Michael Niedermayer avcodec/vaapi_encode: Simplify code with av_clip_int8() Reviewed-by: "Li, Zhong" <zhong.li(a)intel.com> Signed-off-by: Michael Niedermayer <michael(a)niedermayer.cc> > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=9b57b90c4c2752426… --- libavcodec/vaapi_encode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c index 2fb43cf1a4..3be9159d37 100644 --- a/libavcodec/vaapi_encode.c +++ b/libavcodec/vaapi_encode.c @@ -480,7 +480,7 @@ static int vaapi_encode_issue(AVCodecContext *avctx, .width = roi->right - roi->left, .height = roi->bottom - roi->top, }, - .roi_value = av_clip_c(v, INT8_MIN, INT8_MAX), + .roi_value = av_clip_int8(v), }; }

1 0

dnn: export operand info in python script and load in c code
by Guo, Yejun 30 Aug '19

30 Aug '19

ffmpeg | branch: master | Guo, Yejun <yejun.guo(a)intel.com> | Tue Aug 20 16:50:34 2019 +0800| [83e0b71f66f2f0cc293305e2bd138d87660a8c5b] | committer: Pedro Arthur dnn: export operand info in python script and load in c code Signed-off-by: Guo, Yejun <yejun.guo(a)intel.com> Signed-off-by: Pedro Arthur <bygrandao(a)gmail.com> > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=83e0b71f66f2f0cc2… --- libavfilter/dnn/dnn_backend_native.c | 49 +++++++++++--- libavfilter/dnn/dnn_backend_native.h | 2 +- libavfilter/dnn_interface.h | 2 +- tools/python/convert_from_tensorflow.py | 111 +++++++++++++++++++++++++++++--- 4 files changed, 142 insertions(+), 22 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c index 5d393537a8..8b05bec293 100644 --- a/libavfilter/dnn/dnn_backend_native.c +++ b/libavfilter/dnn/dnn_backend_native.c @@ -72,7 +72,6 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) ConvolutionalParams *conv_params; DepthToSpaceParams *depth_to_space_params; LayerPadParams *pad_params; - int32_t operand_index = 0; model = av_malloc(sizeof(DNNModel)); if (!model){ @@ -93,9 +92,10 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) } model->model = (void *)network; - avio_seek(model_file_context, file_size - 4, SEEK_SET); + avio_seek(model_file_context, file_size - 8, SEEK_SET); network->layers_num = (int32_t)avio_rl32(model_file_context); - dnn_size = 4; + network->operands_num = (int32_t)avio_rl32(model_file_context); + dnn_size = 8; avio_seek(model_file_context, 0, SEEK_SET); network->layers = av_mallocz(network->layers_num * sizeof(Layer)); @@ -105,11 +105,6 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) return NULL; } - /** - * Operands should be read from model file, the whole change will be huge. - * to make things step by step, we first mock the operands, instead of reading from model file. - */ - network->operands_num = network->layers_num + 1; network->operands = av_mallocz(network->operands_num * sizeof(DnnOperand)); if (!network->operands){ avio_closep(&model_file_context); @@ -120,8 +115,6 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) for (layer = 0; layer < network->layers_num; ++layer){ layer_type = (int32_t)avio_rl32(model_file_context); dnn_size += 4; - network->layers[layer].input_operand_indexes[0] = operand_index++; - network->layers[layer].output_operand_index = operand_index; switch (layer_type){ case CONV: conv_params = av_malloc(sizeof(ConvolutionalParams)); @@ -162,6 +155,9 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) for (i = 0; i < conv_params->output_num; ++i){ conv_params->biases[i] = av_int2float(avio_rl32(model_file_context)); } + network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context); + network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context); + dnn_size += 8; network->layers[layer].type = CONV; network->layers[layer].params = conv_params; break; @@ -174,6 +170,9 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) } depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context); dnn_size += 4; + network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context); + network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context); + dnn_size += 8; network->layers[layer].type = DEPTH_TO_SPACE; network->layers[layer].params = depth_to_space_params; break; @@ -191,6 +190,9 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) pad_params->paddings[i][1] = avio_rl32(model_file_context); dnn_size += 8; } + network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context); + network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context); + dnn_size += 8; network->layers[layer].type = MIRROR_PAD; network->layers[layer].params = pad_params; break; @@ -201,6 +203,33 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) } } + for (int32_t i = 0; i < network->operands_num; ++i){ + DnnOperand *oprd; + int32_t name_len; + int32_t operand_index = (int32_t)avio_rl32(model_file_context); + dnn_size += 4; + + oprd = &network->operands[operand_index]; + name_len = (int32_t)avio_rl32(model_file_context); + dnn_size += 4; + + avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name)); + dnn_size += name_len; + + oprd->type = (int32_t)avio_rl32(model_file_context); + dnn_size += 4; + + oprd->data_type = (int32_t)avio_rl32(model_file_context); + dnn_size += 4; + + for (int32_t dim = 0; dim < 4; ++dim) { + oprd->dims[dim] = (int32_t)avio_rl32(model_file_context); + dnn_size += 4; + } + + oprd->isNHWC = 1; + } + avio_closep(&model_file_context); if (dnn_size != file_size){ diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h index 87b439472d..08e7d15331 100644 --- a/libavfilter/dnn/dnn_backend_native.h +++ b/libavfilter/dnn/dnn_backend_native.h @@ -36,7 +36,7 @@ typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc; typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; -typedef enum {DOT_INPUT, DOT_INTERMEDIATE, DOT_OUTPUT} DNNOperandType; +typedef enum {DOT_INPUT = 1, DOT_OUTPUT = 2, DOT_INTERMEDIATE = DOT_INPUT | DOT_INPUT} DNNOperandType; typedef struct Layer{ DNNLayerType type; diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h index c24df0e961..057005f47e 100644 --- a/libavfilter/dnn_interface.h +++ b/libavfilter/dnn_interface.h @@ -32,7 +32,7 @@ typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType; typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType; -typedef enum {DNN_FLOAT, DNN_UINT8} DNNDataType; +typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType; typedef struct DNNInputData{ void *data; diff --git a/tools/python/convert_from_tensorflow.py b/tools/python/convert_from_tensorflow.py index cbc76a9f03..bab11a5aab 100644 --- a/tools/python/convert_from_tensorflow.py +++ b/tools/python/convert_from_tensorflow.py @@ -23,6 +23,37 @@ import sys, struct __all__ = ['convert_from_tensorflow'] +class Operand(object): + IOTYPE_INPUT = 1 + IOTYPE_OUTPUT = 2 + IOTYPE_INTERMEDIATE = IOTYPE_INPUT | IOTYPE_OUTPUT + DTYPE_FLOAT = 1 + DTYPE_UINT8 = 4 + index = 0 + def __init__(self, name, dtype, dims): + self.name = name + self.dtype = dtype + self.dims = dims + self.iotype = 0 + self.used_count = 0 + self.index = Operand.index + Operand.index = Operand.index + 1 + self.iotype2str = {Operand.IOTYPE_INPUT: 'in', Operand.IOTYPE_OUTPUT: 'out', Operand.IOTYPE_INTERMEDIATE: 'inout'} + self.dtype2str = {Operand.DTYPE_FLOAT: 'DT_FLOAT', Operand.DTYPE_UINT8: 'DT_UINT8'} + + def add_iotype(self, iotype): + self.iotype = self.iotype | iotype + if iotype == Operand.IOTYPE_INPUT: + self.used_count = self.used_count + 1 + + def __str__(self): + return "{}: (name: {}, iotype: {}, dtype: {}, dims: ({},{},{},{}) used_count: {})".format(self.index, + self.name, self.iotype2str[self.iotype], self.dtype2str[self.dtype], + self.dims[0], self.dims[1], self.dims[2], self.dims[3], self.used_count) + + def __lt__(self, other): + return self.index < other.index + class TFConverter: def __init__(self, graph_def, nodes, outfile, dump4tb): self.graph_def = graph_def @@ -37,8 +68,28 @@ class TFConverter: self.conv_paddings = {'VALID':0, 'SAME':1} self.converted_nodes = set() self.conv2d_scope_names = set() + self.conv2d_scopename_inputname_dict = {} self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3} self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2} + self.name_operand_dict = {} + + + def add_operand(self, name, type): + node = self.name_node_dict[name] + if name not in self.name_operand_dict: + dtype = node.attr['dtype'].type + if dtype == 0: + dtype = node.attr['T'].type + dims = [-1,-1,-1,-1] + if 'shape' in node.attr: + dims[0] = node.attr['shape'].shape.dim[0].size + dims[1] = node.attr['shape'].shape.dim[1].size + dims[2] = node.attr['shape'].shape.dim[2].size + dims[3] = node.attr['shape'].shape.dim[3].size + operand = Operand(name, dtype, dims) + self.name_operand_dict[name] = operand; + self.name_operand_dict[name].add_iotype(type) + return self.name_operand_dict[name].index def dump_for_tensorboard(self): @@ -60,11 +111,10 @@ class TFConverter: # the BiasAdd name is possible be changed into the output name, # if activation is None, and BiasAdd.next is the last op which is Identity if conv2d_scope_name + '/BiasAdd' in self.edges: - activation = self.edges[conv2d_scope_name + '/BiasAdd'][0] - activation = activation.op + anode = self.edges[conv2d_scope_name + '/BiasAdd'][0] else: - activation = 'None' - return knode, bnode, dnode, activation + anode = None + return knode, bnode, dnode, anode def dump_conv2d_to_file(self, node, f): @@ -73,16 +123,21 @@ class TFConverter: self.converted_nodes.add(node.name) scope_name = TFConverter.get_scope_name(node.name) - #knode for kernel, bnode for bias, dnode for dilation - knode, bnode, dnode, activation = self.get_conv2d_params(scope_name) + #knode for kernel, bnode for bias, dnode for dilation, anode for activation + knode, bnode, dnode, anode = self.get_conv2d_params(scope_name) if dnode is not None: dilation = struct.unpack('i', dnode.attr['value'].tensor.tensor_content[0:4])[0] else: dilation = 1 + if anode is not None: + activation = anode.op + else: + activation = 'None' + padding = node.attr['padding'].s.decode("utf-8") - # conv2d with dilation > 1 generates tens of nodes, not easy to parse them, so use tricky. + # conv2d with dilation > 1 generates tens of nodes, not easy to parse them, so use this tricky method. if dilation > 1 and scope_name + '/stack' in self.name_node_dict: if self.name_node_dict[scope_name + '/stack'].op == "Const": padding = 'SAME' @@ -107,6 +162,15 @@ class TFConverter: bias = btensor.tensor_content f.write(bias) + input_name = self.conv2d_scopename_inputname_dict[scope_name] + input_operand_index = self.add_operand(input_name, Operand.IOTYPE_INPUT) + + if anode is not None: + output_operand_index = self.add_operand(anode.name, Operand.IOTYPE_OUTPUT) + else: + output_operand_index = self.add_operand(self.edges[bnode.name][0].name, Operand.IOTYPE_OUTPUT) + np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f) + def dump_depth2space_to_file(self, node, f): assert(node.op == 'DepthToSpace') @@ -114,6 +178,9 @@ class TFConverter: block_size = node.attr['block_size'].i np.array([self.op2code[node.op], block_size], dtype=np.uint32).tofile(f) self.converted_nodes.add(node.name) + input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT) + output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT) + np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f) def dump_mirrorpad_to_file(self, node, f): @@ -127,6 +194,9 @@ class TFConverter: paddings = pnode.attr['value'].tensor.tensor_content f.write(paddings) self.converted_nodes.add(node.name) + input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT) + output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT) + np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f) def dump_layers_to_file(self, f): @@ -147,10 +217,21 @@ class TFConverter: self.dump_mirrorpad_to_file(node, f) + def dump_operands_to_file(self, f): + operands = sorted(self.name_operand_dict.values()) + for operand in operands: + #print('{}'.format(operand)) + np.array([operand.index, len(operand.name)], dtype=np.uint32).tofile(f) + f.write(operand.name.encode('utf-8')) + np.array([operand.iotype, operand.dtype], dtype=np.uint32).tofile(f) + np.array([operand.dims[0], operand.dims[1], operand.dims[2], operand.dims[3]], dtype=np.uint32).tofile(f) + + def dump_to_file(self): with open(self.outfile, 'wb') as f: self.dump_layers_to_file(f) - np.array([self.layer_number], dtype=np.uint32).tofile(f) + self.dump_operands_to_file(f) + np.array([self.layer_number, len(self.name_operand_dict)], dtype=np.uint32).tofile(f) def generate_name_node_dict(self): @@ -212,19 +293,29 @@ class TFConverter: return name[0:index] - def generate_conv2d_scope_names(self): + def generate_conv2d_scope_info(self): + # conv2d is a sub block in graph, get the scope name for node in self.nodes: if node.op == 'Conv2D': scope = TFConverter.get_scope_name(node.name) self.conv2d_scope_names.add(scope) + # get the input name to the conv2d sub block + for node in self.nodes: + scope = TFConverter.get_scope_name(node.name) + if scope in self.conv2d_scope_names: + if node.op == 'Conv2D' or node.op == 'Shape': + for inp in node.input: + if TFConverter.get_scope_name(inp) != scope: + self.conv2d_scopename_inputname_dict[scope] = inp + def run(self): self.generate_name_node_dict() self.generate_output_names() self.remove_identity() self.generate_edges() - self.generate_conv2d_scope_names() + self.generate_conv2d_scope_info() if self.dump4tb: self.dump_for_tensorboard()

1 0

dnn: change .model file format to put layer number at the end of file
by Guo, Yejun 30 Aug '19

30 Aug '19

ffmpeg | branch: master | Guo, Yejun <yejun.guo(a)intel.com> | Thu Aug 29 13:53:41 2019 +0800| [2d5e39c13e500f96a18729ac16472ff69f8d2cfe] | committer: Pedro Arthur dnn: change .model file format to put layer number at the end of file currently, the layer number is at the beginning of the .model file, so we have to scan twice in python script, the first scan to get the layer number. Only one scan needed after put the layer number at the end of .model file. Signed-off-by: Guo, Yejun <yejun.guo(a)intel.com> Signed-off-by: Pedro Arthur <bygrandao(a)gmail.com> > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=2d5e39c13e500f96a… --- libavfilter/dnn/dnn_backend_native.c | 2 ++ tools/python/convert_from_tensorflow.py | 12 +----------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c index daa4f5008c..5d393537a8 100644 --- a/libavfilter/dnn/dnn_backend_native.c +++ b/libavfilter/dnn/dnn_backend_native.c @@ -93,8 +93,10 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) } model->model = (void *)network; + avio_seek(model_file_context, file_size - 4, SEEK_SET); network->layers_num = (int32_t)avio_rl32(model_file_context); dnn_size = 4; + avio_seek(model_file_context, 0, SEEK_SET); network->layers = av_mallocz(network->layers_num * sizeof(Layer)); if (!network->layers){ diff --git a/tools/python/convert_from_tensorflow.py b/tools/python/convert_from_tensorflow.py index 34454b86de..cbc76a9f03 100644 --- a/tools/python/convert_from_tensorflow.py +++ b/tools/python/convert_from_tensorflow.py @@ -129,15 +129,6 @@ class TFConverter: self.converted_nodes.add(node.name) - def generate_layer_number(self): - # in current hard code implementation, the layer number is the first data written to the native model file - # it is not easy to know it at the beginning time in the general converter, so first do a dry run for compatibility - # will be refined later. - with open('/tmp/tmp.model', 'wb') as f: - self.dump_layers_to_file(f) - self.converted_nodes.clear() - - def dump_layers_to_file(self, f): for node in self.nodes: if node.name in self.converted_nodes: @@ -157,10 +148,9 @@ class TFConverter: def dump_to_file(self): - self.generate_layer_number() with open(self.outfile, 'wb') as f: - np.array([self.layer_number], dtype=np.uint32).tofile(f) self.dump_layers_to_file(f) + np.array([self.layer_number], dtype=np.uint32).tofile(f) def generate_name_node_dict(self):

1 0

dnn: introduce dnn operand (in c code) to hold operand infos within network
by Guo, Yejun 30 Aug '19

30 Aug '19

ffmpeg | branch: master | Guo, Yejun <yejun.guo(a)intel.com> | Thu Aug 29 13:53:33 2019 +0800| [09a455a24649af36e8eea81029be7a410201be4c] | committer: Pedro Arthur dnn: introduce dnn operand (in c code) to hold operand infos within network the info can be saved in dnn operand object without regenerating again and again, and it is also needed for layer split/merge, and for memory reuse. to make things step by step, this patch just focuses on c code, the change within python script will be added later. Signed-off-by: Guo, Yejun <yejun.guo(a)intel.com> Signed-off-by: Pedro Arthur <bygrandao(a)gmail.com> > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=09a455a24649af36e… --- libavfilter/dnn/dnn_backend_native.c | 226 ++++++++++++------------- libavfilter/dnn/dnn_backend_native.h | 54 +++++- libavfilter/dnn/dnn_backend_native_layer_pad.c | 24 ++- libavfilter/dnn/dnn_backend_native_layer_pad.h | 4 +- tests/dnn/Makefile | 2 +- tests/dnn/dnn-layer-pad-test.c | 60 +++++-- 6 files changed, 236 insertions(+), 134 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c index d52abc66e3..daa4f5008c 100644 --- a/libavfilter/dnn/dnn_backend_native.c +++ b/libavfilter/dnn/dnn_backend_native.c @@ -30,77 +30,30 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) { ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; - InputParams *input_params; - ConvolutionalParams *conv_params; - DepthToSpaceParams *depth_to_space_params; - LayerPadParams *pad_params; - int cur_width, cur_height, cur_channels; - int32_t layer; - if (network->layers_num <= 0 || network->layers[0].type != INPUT){ + if (network->layers_num <= 0 || network->operands_num <= 0) return DNN_ERROR; - } - else{ - input_params = (InputParams *)network->layers[0].params; - input_params->width = cur_width = input->width; - input_params->height = cur_height = input->height; - input_params->channels = cur_channels = input->channels; - if (input->data){ - av_freep(&input->data); - } - av_assert0(input->dt == DNN_FLOAT); - network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); - if (!network->layers[0].output){ - return DNN_ERROR; - } - } - - for (layer = 1; layer < network->layers_num; ++layer){ - switch (network->layers[layer].type){ - case CONV: - conv_params = (ConvolutionalParams *)network->layers[layer].params; - if (conv_params->input_num != cur_channels){ - return DNN_ERROR; - } - cur_channels = conv_params->output_num; - - if (conv_params->padding_method == VALID) { - int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; - cur_height -= pad_size; - cur_width -= pad_size; - } - break; - case DEPTH_TO_SPACE: - depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; - if (cur_channels % (depth_to_space_params->block_size * depth_to_space_params->block_size) != 0){ - return DNN_ERROR; - } - cur_channels = cur_channels / (depth_to_space_params->block_size * depth_to_space_params->block_size); - cur_height *= depth_to_space_params->block_size; - cur_width *= depth_to_space_params->block_size; - break; - case MIRROR_PAD: - pad_params = (LayerPadParams *)network->layers[layer].params; - cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1]; - cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1]; - cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1]; - break; - default: - return DNN_ERROR; - } - if (network->layers[layer].output){ - av_freep(&network->layers[layer].output); - } - - if (cur_height <= 0 || cur_width <= 0) - return DNN_ERROR; - network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); - if (!network->layers[layer].output){ - return DNN_ERROR; - } - } + av_assert0(input->dt == DNN_FLOAT); + + /** + * as the first step, suppose network->operands[0] is the input operand. + */ + network->operands[0].dims[0] = 1; + network->operands[0].dims[1] = input->height; + network->operands[0].dims[2] = input->width; + network->operands[0].dims[3] = input->channels; + network->operands[0].type = DOT_INPUT; + network->operands[0].data_type = DNN_FLOAT; + network->operands[0].isNHWC = 1; + + av_freep(&network->operands[0].data); + network->operands[0].length = calculate_operand_data_length(&network->operands[0]); + network->operands[0].data = av_malloc(network->operands[0].length); + if (!network->operands[0].data) + return DNN_ERROR; + input->data = network->operands[0].data; return DNN_SUCCESS; } @@ -119,6 +72,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) ConvolutionalParams *conv_params; DepthToSpaceParams *depth_to_space_params; LayerPadParams *pad_params; + int32_t operand_index = 0; model = av_malloc(sizeof(DNNModel)); if (!model){ @@ -131,7 +85,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) } file_size = avio_size(model_file_context); - network = av_malloc(sizeof(ConvolutionalNetwork)); + network = av_mallocz(sizeof(ConvolutionalNetwork)); if (!network){ avio_closep(&model_file_context); av_freep(&model); @@ -139,32 +93,33 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) } model->model = (void *)network; - network->layers_num = 1 + (int32_t)avio_rl32(model_file_context); + network->layers_num = (int32_t)avio_rl32(model_file_context); dnn_size = 4; - network->layers = av_malloc(network->layers_num * sizeof(Layer)); + network->layers = av_mallocz(network->layers_num * sizeof(Layer)); if (!network->layers){ - av_freep(&network); avio_closep(&model_file_context); - av_freep(&model); + ff_dnn_free_model_native(&model); return NULL; } - for (layer = 0; layer < network->layers_num; ++layer){ - network->layers[layer].output = NULL; - network->layers[layer].params = NULL; - } - network->layers[0].type = INPUT; - network->layers[0].params = av_malloc(sizeof(InputParams)); - if (!network->layers[0].params){ + /** + * Operands should be read from model file, the whole change will be huge. + * to make things step by step, we first mock the operands, instead of reading from model file. + */ + network->operands_num = network->layers_num + 1; + network->operands = av_mallocz(network->operands_num * sizeof(DnnOperand)); + if (!network->operands){ avio_closep(&model_file_context); ff_dnn_free_model_native(&model); return NULL; } - for (layer = 1; layer < network->layers_num; ++layer){ + for (layer = 0; layer < network->layers_num; ++layer){ layer_type = (int32_t)avio_rl32(model_file_context); dnn_size += 4; + network->layers[layer].input_operand_indexes[0] = operand_index++; + network->layers[layer].output_operand_index = operand_index; switch (layer_type){ case CONV: conv_params = av_malloc(sizeof(ConvolutionalParams)); @@ -258,14 +213,35 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) #define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x))) -static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height) +static int convolve(DnnOperand *operands, const int32_t *input_operand_indexes, int32_t output_operand_index, const ConvolutionalParams *conv_params) { + float *output; + int32_t input_operand_index = input_operand_indexes[0]; + int number = operands[input_operand_index].dims[0]; + int height = operands[input_operand_index].dims[1]; + int width = operands[input_operand_index].dims[2]; + int channel = operands[input_operand_index].dims[3]; + const float *input = operands[input_operand_index].data; + int radius = conv_params->kernel_size >> 1; int src_linesize = width * conv_params->input_num; int filter_linesize = conv_params->kernel_size * conv_params->input_num; int filter_size = conv_params->kernel_size * filter_linesize; int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0; + DnnOperand *output_operand = &operands[output_operand_index]; + output_operand->dims[0] = number; + output_operand->dims[1] = height - pad_size * 2; + output_operand->dims[2] = width - pad_size * 2; + output_operand->dims[3] = conv_params->output_num; + output_operand->length = calculate_operand_data_length(output_operand); + output_operand->data = av_realloc(output_operand->data, output_operand->length); + if (!output_operand->data) + return -1; + output = output_operand->data; + + av_assert0(channel == conv_params->input_num); + for (int y = pad_size; y < height - pad_size; ++y) { for (int x = pad_size; x < width - pad_size; ++x) { for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) { @@ -311,16 +287,36 @@ static void convolve(const float *input, float *output, const ConvolutionalParam output += conv_params->output_num; } } + return 0; } -static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels) +static int depth_to_space(DnnOperand *operands, const int32_t *input_operand_indexes, int32_t output_operand_index, int block_size) { + float *output; + int32_t input_operand_index = input_operand_indexes[0]; + int number = operands[input_operand_index].dims[0]; + int height = operands[input_operand_index].dims[1]; + int width = operands[input_operand_index].dims[2]; + int channels = operands[input_operand_index].dims[3]; + const float *input = operands[input_operand_index].data; + int y, x, by, bx, ch; int new_channels = channels / (block_size * block_size); int output_linesize = width * channels; int by_linesize = output_linesize / block_size; int x_linesize = new_channels * block_size; + DnnOperand *output_operand = &operands[output_operand_index]; + output_operand->dims[0] = number; + output_operand->dims[1] = height * block_size; + output_operand->dims[2] = width * block_size; + output_operand->dims[3] = new_channels; + output_operand->length = calculate_operand_data_length(output_operand); + output_operand->data = av_realloc(output_operand->data, output_operand->length); + if (!output_operand->data) + return -1; + output = output_operand->data; + for (y = 0; y < height; ++y){ for (x = 0; x < width; ++x){ for (by = 0; by < block_size; ++by){ @@ -334,58 +330,38 @@ static void depth_to_space(const float *input, float *output, int block_size, in } output += output_linesize; } + return 0; } DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output) { ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model; - int cur_width, cur_height, cur_channels; int32_t layer; - InputParams *input_params; ConvolutionalParams *conv_params; DepthToSpaceParams *depth_to_space_params; LayerPadParams *pad_params; - if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){ + if (network->layers_num <= 0 || network->operands_num <= 0) + return DNN_ERROR; + if (!network->operands[0].data) return DNN_ERROR; - } - else{ - input_params = (InputParams *)network->layers[0].params; - cur_width = input_params->width; - cur_height = input_params->height; - cur_channels = input_params->channels; - } - for (layer = 1; layer < network->layers_num; ++layer){ - if (!network->layers[layer].output){ - return DNN_ERROR; - } + for (layer = 0; layer < network->layers_num; ++layer){ switch (network->layers[layer].type){ case CONV: conv_params = (ConvolutionalParams *)network->layers[layer].params; - convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height); - cur_channels = conv_params->output_num; - if (conv_params->padding_method == VALID) { - int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; - cur_height -= pad_size; - cur_width -= pad_size; - } + convolve(network->operands, network->layers[layer].input_operand_indexes, + network->layers[layer].output_operand_index, conv_params); break; case DEPTH_TO_SPACE: depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; - depth_to_space(network->layers[layer - 1].output, network->layers[layer].output, - depth_to_space_params->block_size, cur_width, cur_height, cur_channels); - cur_height *= depth_to_space_params->block_size; - cur_width *= depth_to_space_params->block_size; - cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size; + depth_to_space(network->operands, network->layers[layer].input_operand_indexes, + network->layers[layer].output_operand_index, depth_to_space_params->block_size); break; case MIRROR_PAD: pad_params = (LayerPadParams *)network->layers[layer].params; - dnn_execute_layer_pad(network->layers[layer - 1].output, network->layers[layer].output, - pad_params, 1, cur_height, cur_width, cur_channels); - cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1]; - cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1]; - cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1]; + dnn_execute_layer_pad(network->operands, network->layers[layer].input_operand_indexes, + network->layers[layer].output_operand_index, pad_params); break; case INPUT: return DNN_ERROR; @@ -395,14 +371,24 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output // native mode does not support multiple outputs yet if (nb_output > 1) return DNN_ERROR; - outputs[0].data = network->layers[network->layers_num - 1].output; - outputs[0].height = cur_height; - outputs[0].width = cur_width; - outputs[0].channels = cur_channels; + + /** + * as the first step, suppose network->operands[network->operands_num - 1] is the output operand. + */ + outputs[0].data = network->operands[network->operands_num - 1].data; + outputs[0].height = network->operands[network->operands_num - 1].dims[1]; + outputs[0].width = network->operands[network->operands_num - 1].dims[2]; + outputs[0].channels = network->operands[network->operands_num - 1].dims[3]; return DNN_SUCCESS; } +int32_t calculate_operand_data_length(DnnOperand* operand) +{ + // currently, we just support DNN_FLOAT + return operand->dims[0] * operand->dims[1] * operand->dims[2] * operand->dims[3] * sizeof(float); +} + void ff_dnn_free_model_native(DNNModel **model) { ConvolutionalNetwork *network; @@ -413,7 +399,6 @@ void ff_dnn_free_model_native(DNNModel **model) { network = (ConvolutionalNetwork *)(*model)->model; for (layer = 0; layer < network->layers_num; ++layer){ - av_freep(&network->layers[layer].output); if (network->layers[layer].type == CONV){ conv_params = (ConvolutionalParams *)network->layers[layer].params; av_freep(&conv_params->kernel); @@ -422,6 +407,11 @@ void ff_dnn_free_model_native(DNNModel **model) av_freep(&network->layers[layer].params); } av_freep(&network->layers); + + for (uint32_t operand = 0; operand < network->operands_num; ++operand) + av_freep(&network->operands[operand].data); + av_freep(&network->operands); + av_freep(&network); av_freep(model); } diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h index b6f95330d2..87b439472d 100644 --- a/libavfilter/dnn/dnn_backend_native.h +++ b/libavfilter/dnn/dnn_backend_native.h @@ -36,12 +36,60 @@ typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc; typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; +typedef enum {DOT_INPUT, DOT_INTERMEDIATE, DOT_OUTPUT} DNNOperandType; + typedef struct Layer{ DNNLayerType type; - float *output; + /** + * a layer can have multiple inputs and one output. + * 4 is just a big enough number for input operands (increase it if necessary), + * do not use 'int32_t *input_operand_indexes', so we don't worry about mem leaks. + */ + int32_t input_operand_indexes[4]; + int32_t output_operand_index; void *params; } Layer; +typedef struct DnnOperand{ + /** + * there are two memory layouts, NHWC or NCHW, so we use dims, + * dims[0] is Number. + */ + int32_t dims[4]; + + /** + * input/output/intermediate operand of the network + */ + DNNOperandType type; + + /** + * support different kinds of data type such as float, half float, int8 etc, + * first support float now. + */ + DNNDataType data_type; + + /** + * NHWC if 1, otherwise NCHW. + * let's first support NHWC only, this flag is for extensive usage. + */ + int8_t isNHWC; + + /** + * to avoid possible memory leak, do not use char *name + */ + char name[128]; + + /** + * data pointer with data length in bytes. + * usedNumbersLeft is only valid for intermediate operand, + * it means how many layers still depend on this operand, + * todo: the memory can be reused when usedNumbersLeft is zero. + */ + void *data; + int32_t length; + int32_t usedNumbersLeft; +}DnnOperand; + typedef struct ConvolutionalParams{ int32_t input_num, output_num, kernel_size; DNNActivationFunc activation; @@ -63,6 +111,8 @@ typedef struct DepthToSpaceParams{ typedef struct ConvolutionalNetwork{ Layer *layers; int32_t layers_num; + DnnOperand *operands; + int32_t operands_num; } ConvolutionalNetwork; DNNModel *ff_dnn_load_model_native(const char *model_filename); @@ -71,4 +121,6 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output void ff_dnn_free_model_native(DNNModel **model); +int32_t calculate_operand_data_length(DnnOperand *operand); + #endif diff --git a/libavfilter/dnn/dnn_backend_native_layer_pad.c b/libavfilter/dnn/dnn_backend_native_layer_pad.c index 5417d73ad1..c2905a75ea 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_pad.c +++ b/libavfilter/dnn/dnn_backend_native_layer_pad.c @@ -48,12 +48,21 @@ static int after_get_buddy(int given, int border, LayerPadModeParam mode) } } -void dnn_execute_layer_pad(const float *input, float *output, const LayerPadParams *params, int number, int height, int width, int channel) +int dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_indexes, int32_t output_operand_index, + const LayerPadParams *params) { int32_t before_paddings; int32_t after_paddings; + float* output; // suppose format is <N, H, W, C> + int32_t input_operand_index = input_operand_indexes[0]; + int number = operands[input_operand_index].dims[0]; + int height = operands[input_operand_index].dims[1]; + int width = operands[input_operand_index].dims[2]; + int channel = operands[input_operand_index].dims[3]; + const float *input = operands[input_operand_index].data; + int new_number = number + params->paddings[0][0] + params->paddings[0][1]; int new_height = height + params->paddings[1][0] + params->paddings[1][1]; int new_width = width + params->paddings[2][0] + params->paddings[2][1]; @@ -67,6 +76,17 @@ void dnn_execute_layer_pad(const float *input, float *output, const LayerPadPara int new_wc_stride = new_c_stride * new_width; int new_hwc_stride = new_wc_stride * new_height; + DnnOperand *output_operand = &operands[output_operand_index]; + output_operand->dims[0] = new_number; + output_operand->dims[1] = new_height; + output_operand->dims[2] = new_width; + output_operand->dims[3] = new_channel; + output_operand->length = calculate_operand_data_length(output_operand); + output_operand->data = av_realloc(output_operand->data, output_operand->length); + if (!output_operand->data) + return -1; + output = output_operand->data; + // copy the original data for (int n = 0; n < number; n++) { for (int h = 0; h < height; h++) { @@ -208,4 +228,6 @@ void dnn_execute_layer_pad(const float *input, float *output, const LayerPadPara } } } + + return 0; } diff --git a/libavfilter/dnn/dnn_backend_native_layer_pad.h b/libavfilter/dnn/dnn_backend_native_layer_pad.h index 0fbe652202..7cc8213521 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_pad.h +++ b/libavfilter/dnn/dnn_backend_native_layer_pad.h @@ -26,6 +26,7 @@ #define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_PAD_H #include <stdint.h> +#include "dnn_backend_native.h" typedef enum {LPMP_CONSTANT, LPMP_REFLECT, LPMP_SYMMETRIC} LayerPadModeParam; @@ -35,6 +36,7 @@ typedef struct LayerPadParams{ float constant_values; } LayerPadParams; -void dnn_execute_layer_pad(const float *input, float *output, const LayerPadParams *params, int number, int height, int width, int channel); +int dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_indexes, int32_t output_operand_index, + const LayerPadParams *params); #endif diff --git a/tests/dnn/Makefile b/tests/dnn/Makefile index 0e050ea218..fabed7510f 100644 --- a/tests/dnn/Makefile +++ b/tests/dnn/Makefile @@ -5,7 +5,7 @@ DNNTESTPROGS := $(DNNTESTPROGS:%=$(DNNTESTSDIR)/%-test$(EXESUF)) -include $(wildcard $(DNNTESTOBJS:.o=.d)) $(DNNTESTPROGS): %$(EXESUF): %.o $(FF_STATIC_DEP_LIBS) - $(LD) $(LDFLAGS) $(LDEXEFLAGS) $(LD_O) $(filter %.o,$^) $(FF_STATIC_DEP_LIBS) $(ELIBS) + $(LD) $(LDFLAGS) $(LDEXEFLAGS) $(LD_O) $(filter %.o,$^) $(FF_STATIC_DEP_LIBS) $(EXTRALIBS-avcodec) $(EXTRALIBS-avfilter) $(EXTRALIBS-avformat) $(EXTRALIBS-avutil) $(EXTRALIBS-swresample) $(EXTRALIBS) testclean:: $(RM) $(addprefix $(DNNTESTSDIR)/,$(CLEANSUFFIXES) *-test$(EXESUF)) diff --git a/tests/dnn/dnn-layer-pad-test.c b/tests/dnn/dnn-layer-pad-test.c index 28a49eb55a..1fb2be1590 100644 --- a/tests/dnn/dnn-layer-pad-test.c +++ b/tests/dnn/dnn-layer-pad-test.c @@ -44,6 +44,8 @@ static int test_with_mode_symmetric(void) */ LayerPadParams params; + DnnOperand operands[2]; + int32_t input_indexes[1]; float input[1*4*4*3] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47 }; @@ -57,8 +59,7 @@ static int test_with_mode_symmetric(void) 27.0, 28.0, 29.0, 24.0, 25.0, 26.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 33.0, 34.0, 35.0, 30.0, 31.0, 32.0, 18.0, 19.0, 20.0, 15.0, 16.0, 17.0, 12.0, 13.0, 14.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 21.0, 22.0, 23.0, 18.0, 19.0, 20.0 }; - float output[1*9*9*3]; - memset(output, 0, sizeof(output)); + float *output; params.mode = LPMP_SYMMETRIC; params.paddings[0][0] = 0; @@ -70,15 +71,26 @@ static int test_with_mode_symmetric(void) params.paddings[3][0] = 0; params.paddings[3][1] = 0; - dnn_execute_layer_pad(input, output, &params, 1, 4, 4, 3); + operands[0].data = input; + operands[0].dims[0] = 1; + operands[0].dims[1] = 4; + operands[0].dims[2] = 4; + operands[0].dims[3] = 3; + operands[1].data = NULL; - for (int i = 0; i < sizeof(output) / sizeof(float); i++) { + input_indexes[0] = 0; + dnn_execute_layer_pad(operands, input_indexes, 1, &params); + + output = operands[1].data; + for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) { if (fabs(output[i] - expected_output[i]) > EPSON) { printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]); + av_freep(&output); return 1; } } + av_freep(&output); return 0; } @@ -102,6 +114,8 @@ static int test_with_mode_reflect(void) */ LayerPadParams params; + DnnOperand operands[2]; + int32_t input_indexes[1]; float input[3*2*2*3] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35 }; @@ -110,8 +124,7 @@ static int test_with_mode_reflect(void) 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0 }; - float output[6*2*2*3]; - memset(output, 0, sizeof(output)); + float *output; params.mode = LPMP_REFLECT; params.paddings[0][0] = 1; @@ -123,15 +136,26 @@ static int test_with_mode_reflect(void) params.paddings[3][0] = 0; params.paddings[3][1] = 0; - dnn_execute_layer_pad(input, output, &params, 3, 2, 2, 3); + operands[0].data = input; + operands[0].dims[0] = 3; + operands[0].dims[1] = 2; + operands[0].dims[2] = 2; + operands[0].dims[3] = 3; + operands[1].data = NULL; + + input_indexes[0] = 0; + dnn_execute_layer_pad(operands, input_indexes, 1, &params); - for (int i = 0; i < sizeof(output) / sizeof(float); i++) { + output = operands[1].data; + for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) { if (fabs(output[i] - expected_output[i]) > EPSON) { printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]); + av_freep(&output); return 1; } } + av_freep(&output); return 0; } @@ -155,6 +179,8 @@ static int test_with_mode_constant(void) */ LayerPadParams params; + DnnOperand operands[2]; + int32_t input_indexes[1]; float input[1*2*2*3] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; @@ -163,8 +189,7 @@ static int test_with_mode_constant(void) 728.0, 728.0, 0.0, 1.0, 2.0, 728.0, 728.0, 728.0, 3.0, 4.0, 5.0, 728.0, 728.0, 728.0, 6.0, 7.0, 8.0, 728.0, 728.0, 728.0, 9.0, 10.0, 11.0, 728.0, 728.0 }; - float output[1*3*2*6]; - memset(output, 0, sizeof(output)); + float *output; params.mode = LPMP_CONSTANT; params.constant_values = 728; @@ -177,15 +202,26 @@ static int test_with_mode_constant(void) params.paddings[3][0] = 1; params.paddings[3][1] = 2; - dnn_execute_layer_pad(input, output, &params, 1, 2, 2, 3); + operands[0].data = input; + operands[0].dims[0] = 3; + operands[0].dims[1] = 2; + operands[0].dims[2] = 2; + operands[0].dims[3] = 3; + operands[1].data = NULL; + + input_indexes[0] = 0; + dnn_execute_layer_pad(operands, input_indexes, 1, &params); - for (int i = 0; i < sizeof(output) / sizeof(float); i++) { + output = operands[1].data; + for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) { if (fabs(output[i] - expected_output[i]) > EPSON) { printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]); + av_freep(&output); return 1; } } + av_freep(&output); return 0; }

1 0

avfilter/vf_v360: add facebook's format
by Eugene Lyapustin 30 Aug '19

30 Aug '19

ffmpeg | branch: master | Eugene Lyapustin <unishifft(a)gmail.com> | Sun Aug 18 20:26:36 2019 +0300| [20a12448aaf140cf1ec06ee22e1ab4ebaea5c9ba] | committer: Paul B Mahol avfilter/vf_v360: add facebook's format > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=20a12448aaf140cf1… --- doc/filters.texi | 3 + libavfilter/vf_v360.c | 162 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) diff --git a/doc/filters.texi b/doc/filters.texi index 7759fe177b..2271749e49 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -18024,6 +18024,9 @@ No padding. Default value is @b{@samp{0}}. @end table +@item fb +Facebook's 360 format. + @end table @item interp diff --git a/libavfilter/vf_v360.c b/libavfilter/vf_v360.c index aad1afd129..9b7f0c109e 100644 --- a/libavfilter/vf_v360.c +++ b/libavfilter/vf_v360.c @@ -48,6 +48,7 @@ enum Projections { EQUIANGULAR, FLAT, DUAL_FISHEYE, + FACEBOOK, NB_PROJECTIONS, }; @@ -137,12 +138,14 @@ static const AVOption v360_options[] = { { "c6x1", "cubemap6x1", 0, AV_OPT_TYPE_CONST, {.i64=CUBEMAP_6_1}, 0, 0, FLAGS, "in" }, { "eac", "equi-angular", 0, AV_OPT_TYPE_CONST, {.i64=EQUIANGULAR}, 0, 0, FLAGS, "in" }, { "dfisheye", "dual fisheye", 0, AV_OPT_TYPE_CONST, {.i64=DUAL_FISHEYE}, 0, 0, FLAGS, "in" }, + { "fb", "facebook's 360 format", 0, AV_OPT_TYPE_CONST, {.i64=FACEBOOK}, 0, 0, FLAGS, "in" }, { "output", "set output projection", OFFSET(out), AV_OPT_TYPE_INT, {.i64=CUBEMAP_3_2}, 0, NB_PROJECTIONS-1, FLAGS, "out" }, { "e", "equirectangular", 0, AV_OPT_TYPE_CONST, {.i64=EQUIRECTANGULAR}, 0, 0, FLAGS, "out" }, { "c3x2", "cubemap3x2", 0, AV_OPT_TYPE_CONST, {.i64=CUBEMAP_3_2}, 0, 0, FLAGS, "out" }, { "c6x1", "cubemap6x1", 0, AV_OPT_TYPE_CONST, {.i64=CUBEMAP_6_1}, 0, 0, FLAGS, "out" }, { "eac", "equi-angular", 0, AV_OPT_TYPE_CONST, {.i64=EQUIANGULAR}, 0, 0, FLAGS, "out" }, { "flat", "regular video", 0, AV_OPT_TYPE_CONST, {.i64=FLAT}, 0, 0, FLAGS, "out" }, + { "fb", "facebook's 360 format", 0, AV_OPT_TYPE_CONST, {.i64=FACEBOOK}, 0, 0, FLAGS, "out" }, { "interp", "set interpolation method", OFFSET(interp), AV_OPT_TYPE_INT, {.i64=BILINEAR}, 0, NB_INTERP_METHODS-1, FLAGS, "interp" }, { "near", "nearest neighbour", 0, AV_OPT_TYPE_CONST, {.i64=NEAREST}, 0, 0, FLAGS, "interp" }, { "nearest", "nearest neighbour", 0, AV_OPT_TYPE_CONST, {.i64=NEAREST}, 0, 0, FLAGS, "interp" }, @@ -1647,6 +1650,153 @@ static void xyz_to_dfisheye(const V360Context *s, } /** + * Calculate 3D coordinates on sphere for corresponding frame position in facebook's format. + * + * @param s filter context + * @param i horizontal position on frame [0, height) + * @param j vertical position on frame [0, width) + * @param width frame width + * @param height frame height + * @param vec coordinates on sphere + */ +static void fb_to_xyz(const V360Context *s, + int i, int j, int width, int height, + float *vec) +{ + const float scale = 0.99f; + float l_x, l_y, l_z; + + if (i < 4 * width / 5) { + const float theta_range = M_PI / 4.f; + + const int ew = 4 * width / 5; + const int eh = height; + + const float phi = ((2.f * i) / ew - 1.f) * M_PI / scale; + const float theta = ((2.f * j) / eh - 1.f) * theta_range / scale; + + const float sin_phi = sinf(phi); + const float cos_phi = cosf(phi); + const float sin_theta = sinf(theta); + const float cos_theta = cosf(theta); + + l_x = cos_theta * sin_phi; + l_y = -sin_theta; + l_z = -cos_theta * cos_phi; + } else { + const int ew = width / 5; + const int eh = height / 2; + + float uf, vf; + float norm; + + if (j < eh) { // UP + uf = 2.f * (i - 4 * ew) / ew - 1.f; + vf = 2.f * (j ) / eh - 1.f; + + uf /= scale; + vf /= scale; + + l_x = uf; + l_y = 1.f; + l_z = -vf; + } else { // DOWN + uf = 2.f * (i - 4 * ew) / ew - 1.f; + vf = 2.f * (j - eh) / eh - 1.f; + + uf /= scale; + vf /= scale; + + l_x = uf; + l_y = -1.f; + l_z = vf; + } + + norm = sqrtf(l_x * l_x + l_y * l_y + l_z * l_z); + + l_x /= norm; + l_y /= norm; + l_z /= norm; + } + + vec[0] = l_x; + vec[1] = l_y; + vec[2] = l_z; +} + +/** + * Calculate frame position in facebook's format for corresponding 3D coordinates on sphere. + * + * @param s filter context + * @param vec coordinates on sphere + * @param width frame width + * @param height frame height + * @param us horizontal coordinates for interpolation window + * @param vs vertical coordinates for interpolation window + * @param du horizontal relative coordinate + * @param dv vertical relative coordinate + */ +static void xyz_to_fb(const V360Context *s, + const float *vec, int width, int height, + uint16_t us[4][4], uint16_t vs[4][4], float *du, float *dv) +{ + const float scale = 0.99f; + + const float phi = atan2f(vec[0], -vec[2]); + const float theta = asinf(-vec[1]); + const float theta_range = M_PI / 4.f; + + int ew, eh; + int u_shift, v_shift; + float uf, vf; + int ui, vi; + int i, j; + + if (theta > -theta_range && theta < theta_range) { + ew = 4 * width / 5; + eh = height; + + u_shift = 0; + v_shift = 0; + + uf = (phi / M_PI * scale + 1.f) * ew / 2.f; + vf = (theta / theta_range * scale + 1.f) * eh / 2.f; + } else { + ew = width / 5; + eh = height / 2; + + u_shift = 4 * ew; + + if (theta < 0.f) { // UP + uf = vec[0] / vec[1]; + vf = -vec[2] / vec[1]; + v_shift = 0; + } else { // DOWN + uf = -vec[0] / vec[1]; + vf = -vec[2] / vec[1]; + v_shift = eh; + } + + uf = 0.5f * ew * (uf * scale + 1.f); + vf = 0.5f * eh * (vf * scale + 1.f); + } + + ui = floorf(uf); + vi = floorf(vf); + + *du = uf - ui; + *dv = vf - vi; + + for (i = -1; i < 3; i++) { + for (j = -1; j < 3; j++) { + us[i + 1][j + 1] = u_shift + av_clip(ui + j, 0, ew - 1); + vs[i + 1][j + 1] = v_shift + av_clip(vi + i, 0, eh - 1); + } + } + +} + +/** * Calculate rotation matrix for yaw/pitch/roll angles. */ static inline void calculate_rotation_matrix(float yaw, float pitch, float roll, @@ -1789,6 +1939,12 @@ static int config_output(AVFilterLink *outlink) wf = inlink->w; hf = inlink->h; break; + case FACEBOOK: + in_transform = xyz_to_fb; + err = 0; + wf = inlink->w / 5.f * 4.f; + hf = inlink->h; + break; default: av_log(ctx, AV_LOG_ERROR, "Specified input format is not handled.\n"); return AVERROR_BUG; @@ -1832,6 +1988,12 @@ static int config_output(AVFilterLink *outlink) case DUAL_FISHEYE: av_log(ctx, AV_LOG_ERROR, "Dual fisheye format is not accepted as output.\n"); return AVERROR(EINVAL); + case FACEBOOK: + out_transform = fb_to_xyz; + err = 0; + w = roundf(wf / 4.f * 5.f); + h = roundf(hf); + break; default: av_log(ctx, AV_LOG_ERROR, "Specified output format is not handled.\n"); return AVERROR_BUG;

1 0

avutil/file: add more check befor destory the buffer
by Jun Zhao 30 Aug '19

30 Aug '19

ffmpeg | branch: master | Jun Zhao <barryjzhao(a)tencent.com> | Fri Aug 30 09:09:16 2019 +0800| [95780f4dcbf89f511e5d0c19035221cea0826d7d] | committer: Jun Zhao avutil/file: add more check befor destory the buffer add more check befor destory the buffer Reviewed-by: Michael Niedermayer <michael(a)niedermayer.cc> Signed-off-by: Jun Zhao <barryjzhao(a)tencent.com> > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=95780f4dcbf89f511… --- libavutil/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavutil/file.c b/libavutil/file.c index b91c2fa391..f228b723ec 100644 --- a/libavutil/file.c +++ b/libavutil/file.c @@ -143,7 +143,7 @@ out: void av_file_unmap(uint8_t *bufptr, size_t size) { - if (!size) + if (!size || !bufptr) return; #if HAVE_MMAP munmap(bufptr, size);

1 0

avutil/file: always set *size to zero if *bufptr is NULL
by Jun Zhao 30 Aug '19

30 Aug '19

ffmpeg | branch: master | Jun Zhao <barryjzhao(a)tencent.com> | Thu Aug 29 00:15:52 2019 +0800| [7ce15b1d439400cba1aa09e9ee80aacbd84eb988] | committer: Jun Zhao avutil/file: always set *size to zero if *bufptr is NULL Always set *size to zero if *bufptr is NULL, it's more make sence. fix #8095 Reviewed-by: Michael Niedermayer <michael(a)niedermayer.cc> Signed-off-by: Jun Zhao <barryjzhao(a)tencent.com> > http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=7ce15b1d439400cba… --- libavutil/file.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libavutil/file.c b/libavutil/file.c index d946085b28..b91c2fa391 100644 --- a/libavutil/file.c +++ b/libavutil/file.c @@ -60,6 +60,7 @@ int av_file_map(const char *filename, uint8_t **bufptr, size_t *size, off_t off_size; char errbuf[128]; *bufptr = NULL; + *size = 0; if (fd < 0) { err = AVERROR(errno); @@ -97,6 +98,7 @@ int av_file_map(const char *filename, uint8_t **bufptr, size_t *size, av_strerror(err, errbuf, sizeof(errbuf)); av_log(&file_log_ctx, AV_LOG_ERROR, "Error occurred in mmap(): %s\n", errbuf); close(fd); + *size = 0; return err; } *bufptr = ptr; @@ -108,6 +110,7 @@ int av_file_map(const char *filename, uint8_t **bufptr, size_t *size, if (!mh) { av_log(&file_log_ctx, AV_LOG_ERROR, "Error occurred in CreateFileMapping()\n"); close(fd); + *size = 0; return -1; } @@ -116,6 +119,7 @@ int av_file_map(const char *filename, uint8_t **bufptr, size_t *size, if (!ptr) { av_log(&file_log_ctx, AV_LOG_ERROR, "Error occurred in MapViewOfFile()\n"); close(fd); + *size = 0; return -1; } @@ -126,6 +130,7 @@ int av_file_map(const char *filename, uint8_t **bufptr, size_t *size, if (!*bufptr) { av_log(&file_log_ctx, AV_LOG_ERROR, "Memory allocation error occurred\n"); close(fd); + *size = 0; return AVERROR(ENOMEM); } read(fd, *bufptr, *size);

1 0