[FFmpeg-devel] [PATCH v2] Add multiple padding method in dnn native
Xuewei Meng
xwmeng96 at gmail.com
Wed May 15 11:40:30 EEST 2019
Guo, Yejun <yejun.guo at intel.com> 于2019年5月15日周三 下午2:21写道：
>
>
> > -----Original Message-----
> > From: ffmpeg-devel [mailto:ffmpeg-devel-bounces at ffmpeg.org] On Behalf Of
> > Steven Liu
> > Sent: Wednesday, May 15, 2019 10:38 AM
> > To: FFmpeg development discussions and patches <ffmpeg-devel at ffmpeg.org>
> > Cc: Xuewei Meng <xwmeng96 at gmail.com>
> > Subject: Re: [FFmpeg-devel] [PATCH v2] Add multiple padding method in dnn
> > native
> >
> > Xuewei Meng <xwmeng96 at gmail.com> 于2019年5月11日周六 上午11:11
> > 写道：
> > >
> > > ---
> > > libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++--------
> > > libavfilter/dnn_backend_native.h | 3 ++
> > > 2 files changed, 43 insertions(+), 12 deletions(-)
>
> @xuewei, we still need to mention the impact of sr filter, and explain why
> same_clamp_to_edge is needed.
>
> There are three padding methods in this patch, VALID, SAME and
SAME_CLAMP_TO_EDGE. The 'VALID' and 'SAME' options are tensorflow supported
padding methods. And the third one, 'SAME_CLAMP_TO_EDGE', is suggested by
sr filter. As this method can keep the output with the same size as
original input, and gives a slight better result as mentioned by Pedro
Arthur. So we keep this option in dnn native mode.
> > >
> > > diff --git a/libavfilter/dnn_backend_native.c
> > b/libavfilter/dnn_backend_native.c
> > > index 06fbdf368b..171a756385 100644
> > > --- a/libavfilter/dnn_backend_native.c
> > > +++ b/libavfilter/dnn_backend_native.c
> > > @@ -61,6 +61,12 @@ static DNNReturnType set_input_output_native(void
> > *model, DNNInputData *input, c
> > > return DNN_ERROR;
> > > }
> > > cur_channels = conv_params->output_num;
> > > +
> > > + if(conv_params->padding_method == VALID){
> > > + int pad_size = conv_params->kernel_size - 1;
> > > + cur_height -= pad_size;
> > > + cur_width -= pad_size;
> > > + }
> > > break;
> > > case DEPTH_TO_SPACE:
> > > depth_to_space_params = (DepthToSpaceParams
> > *)network->layers[layer].params;
> > > @@ -77,6 +83,10 @@ static DNNReturnType set_input_output_native(void
> > *model, DNNInputData *input, c
> > > if (network->layers[layer].output){
> > > av_freep(&network->layers[layer].output);
> > > }
> > > +
> > > + if(cur_height <= 0 || cur_width <= 0)
> > > + return DNN_ERROR;
> > > +
> > > network->layers[layer].output = av_malloc(cur_height *
> > cur_width * cur_channels * sizeof(float));
> > > if (!network->layers[layer].output){
> > > return DNN_ERROR;
> > > @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const
> > char *model_filename)
> > > ff_dnn_free_model_native(&model);
> > > return NULL;
> > > }
> > > + conv_params->padding_method =
> > (int32_t)avio_rl32(model_file_context);
> > > conv_params->activation =
> > (int32_t)avio_rl32(model_file_context);
> > > conv_params->input_num =
> > (int32_t)avio_rl32(model_file_context);
> > > conv_params->output_num =
> > (int32_t)avio_rl32(model_file_context);
> > > conv_params->kernel_size =
> > (int32_t)avio_rl32(model_file_context);
> > > kernel_size = conv_params->input_num *
> > conv_params->output_num *
> > > conv_params->kernel_size *
> > conv_params->kernel_size;
> > > - dnn_size += 16 + (kernel_size + conv_params->output_num
> > << 2);
> > > + dnn_size += 20 + (kernel_size + conv_params->output_num
> > << 2);
> > > if (dnn_size > file_size || conv_params->input_num <= 0 ||
> > > conv_params->output_num <= 0 ||
> > conv_params->kernel_size <= 0){
> > > avio_closep(&model_file_context);
> > > @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const
> > char *model_filename)
> > >
> > > static void convolve(const float *input, float *output, const
> > ConvolutionalParams *conv_params, int width, int height)
> > > {
> > > - int y, x, n_filter, ch, kernel_y, kernel_x;
> > > int radius = conv_params->kernel_size >> 1;
> > > int src_linesize = width * conv_params->input_num;
> > > int filter_linesize = conv_params->kernel_size *
> > conv_params->input_num;
> > > int filter_size = conv_params->kernel_size * filter_linesize;
> > > + int pad_size = (conv_params->padding_method == VALID) ?
> > (conv_params->kernel_size - 1) / 2 : 0;
> > >
> > > - for (y = 0; y < height; ++y){
> > > - for (x = 0; x < width; ++x){
> > > - for (n_filter = 0; n_filter < conv_params->output_num;
> > ++n_filter){
> > > + for (int y = pad_size; y < height - pad_size; ++y){
> > > + for (int x = pad_size; x < width - pad_size; ++x){
> > > + for (int n_filter = 0; n_filter < conv_params->output_num;
> > ++n_filter){
> > > output[n_filter] = conv_params->biases[n_filter];
> > > - for (ch = 0; ch < conv_params->input_num; ++ch){
> > > - for (kernel_y = 0; kernel_y <
> > conv_params->kernel_size; ++kernel_y){
> > > - for (kernel_x = 0; kernel_x <
> > conv_params->kernel_size; ++kernel_x){
> > > - output[n_filter] +=
> > input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize +
> > > -
> > CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num +
> ch]
> > *
> > > -
> > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
> > > -
> > kernel_x * conv_params->input_num + ch];
> > > +
> > > + for (int ch = 0; ch < conv_params->input_num; ++ch){
> > > + for (int kernel_y = 0; kernel_y <
> > conv_params->kernel_size; ++kernel_y){
> > > + for (int kernel_x = 0; kernel_x <
> > conv_params->kernel_size; ++kernel_x){
> > > + float input_pel;
> > > + if(conv_params->padding_method ==
> > SAME_CLAMP_TO_EDGE){
> > > + int y_pos = CLAMP_TO_EDGE(y +
> > kernel_y - radius, height);
> > > + int x_pos = CLAMP_TO_EDGE(x +
> > kernel_x - radius, width);
> > > + input_pel = input[y_pos *
> > src_linesize + x_pos * conv_params->input_num + ch];
> > > + }else{
> > > + int y_pos = y + kernel_y - radius;
> > > + int x_pos = x + kernel_x - radius;
> > > + input_pel = (x_pos < 0 || x_pos >=
> > width || y_pos < 0 || y_pos >= height) ? 0.0 :
> > > + input[y_pos
> > * src_linesize + x_pos * conv_params->input_num + ch];
> > > + }
> > > +
> > > +
> > > + output[n_filter] += input_pel *
> > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
> > > +
> > kernel_x * conv_params->input_num + ch];
> > > }
> > > }
> > > }
> > > @@ -305,6 +328,11 @@ DNNReturnType
> > ff_dnn_execute_model_native(const DNNModel *model, DNNData *output
> > > conv_params = (ConvolutionalParams
> > *)network->layers[layer].params;
> > > convolve(network->layers[layer - 1].output,
> > network->layers[layer].output, conv_params, cur_width, cur_height);
> > > cur_channels = conv_params->output_num;
> > > + if(conv_params->padding_method == VALID){
> > > + int pad_size = conv_params->kernel_size - 1;
> > > + cur_height -= pad_size;
> > > + cur_width -= pad_size;
> > > + }
> > > break;
> > > case DEPTH_TO_SPACE:
> > > depth_to_space_params = (DepthToSpaceParams
> > *)network->layers[layer].params;
> > > diff --git a/libavfilter/dnn_backend_native.h
> > b/libavfilter/dnn_backend_native.h
> > > index e13a68a168..d70cd16387 100644
> > > --- a/libavfilter/dnn_backend_native.h
> > > +++ b/libavfilter/dnn_backend_native.h
> > > @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE}
> > DNNLayerType;
> > >
> > > typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc;
> > >
> > > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE}
> > DNNConvPaddingParam;
> > > +
> > > typedef struct Layer{
> > > DNNLayerType type;
> > > float *output;
> > > @@ -43,6 +45,7 @@ typedef struct Layer{
> > > typedef struct ConvolutionalParams{
> > > int32_t input_num, output_num, kernel_size;
> > > DNNActivationFunc activation;
> > > + DNNConvPaddingParam padding_method;
> > > float *kernel;
> > > float *biases;
> > > } ConvolutionalParams;
> > > --
> > > 2.17.1
> > >
> > > _______________________________________________
> > > ffmpeg-devel mailing list
> > > ffmpeg-devel at ffmpeg.org
> > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> > >
> > > To unsubscribe, visit link above, or email
> > > ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
> >
> >
> > The https://github.com/HighVoltageRocknRoll/sr has loss of
> > communication，and the project
> > https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i
> > think the pull request cannot be merge.
> > 1. So i recommend Xuewei fork the project to his github, and merge the
> > pr to his fork project, and modify the sr document of
> > libavfilter/vf_sr.c. makes GSoC derain mentor project continue.
>
> I prefer this one.
>
> >
> > 2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code
> > for the derain.
> >
> > Comments welcome.
> >
> > Thanks
> >
> > Steven
> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel at ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
>
More information about the ffmpeg-devel
mailing list