FFmpeg
dnn_backend_torch.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2024
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN Torch backend implementation.
24  */
25 
26 #include <torch/torch.h>
27 #include <torch/script.h>
28 
29 extern "C" {
30 #include "../internal.h"
31 #include "dnn_io_proc.h"
32 #include "dnn_backend_common.h"
33 #include "libavutil/opt.h"
34 #include "libavutil/mem.h"
35 #include "queue.h"
36 #include "safe_queue.h"
37 }
38 
39 typedef struct THModel {
42  torch::jit::Module *jit_model;
46 } THModel;
47 
48 typedef struct THInferRequest {
49  torch::Tensor *output;
50  torch::Tensor *input_tensor;
52 
53 typedef struct THRequestItem {
58 
59 
60 #define OFFSET(x) offsetof(THOptions, x)
61 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
62 static const AVOption dnn_th_options[] = {
63  { "optimize", "turn on graph executor optimization", OFFSET(optimize), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS},
64  { NULL }
65 };
66 
67 static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
68 {
69  THModel *th_model = (THModel *)task->model;
70  DnnContext *ctx = th_model->ctx;
71  LastLevelTaskItem *lltask = (LastLevelTaskItem *)av_malloc(sizeof(*lltask));
72  if (!lltask) {
73  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for LastLevelTaskItem\n");
74  return AVERROR(ENOMEM);
75  }
76  task->inference_todo = 1;
77  task->inference_done = 0;
78  lltask->task = task;
79  if (ff_queue_push_back(lltask_queue, lltask) < 0) {
80  av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");
81  av_freep(&lltask);
82  return AVERROR(ENOMEM);
83  }
84  return 0;
85 }
86 
87 static void th_free_request(THInferRequest *request)
88 {
89  if (!request)
90  return;
91  if (request->output) {
92  delete(request->output);
93  request->output = NULL;
94  }
95  if (request->input_tensor) {
96  delete(request->input_tensor);
97  request->input_tensor = NULL;
98  }
99  return;
100 }
101 
103 {
104  THRequestItem *item;
105  if (!arg || !*arg) {
106  return;
107  }
108  item = *arg;
110  av_freep(&item->infer_request);
111  av_freep(&item->lltask);
113  av_freep(arg);
114 }
115 
116 static void dnn_free_model_th(DNNModel **model)
117 {
118  THModel *th_model;
119  if (!model || !*model)
120  return;
121 
122  th_model = (THModel *) (*model)->model;
123  while (ff_safe_queue_size(th_model->request_queue) != 0) {
125  destroy_request_item(&item);
126  }
128 
129  while (ff_queue_size(th_model->lltask_queue) != 0) {
131  av_freep(&item);
132  }
133  ff_queue_destroy(th_model->lltask_queue);
134 
135  while (ff_queue_size(th_model->task_queue) != 0) {
136  TaskItem *item = (TaskItem *)ff_queue_pop_front(th_model->task_queue);
137  av_frame_free(&item->in_frame);
138  av_frame_free(&item->out_frame);
139  av_freep(&item);
140  }
141  ff_queue_destroy(th_model->task_queue);
142  delete th_model->jit_model;
143  av_freep(&th_model);
144  av_freep(model);
145 }
146 
147 static int get_input_th(void *model, DNNData *input, const char *input_name)
148 {
149  input->dt = DNN_FLOAT;
150  input->order = DCO_RGB;
151  input->layout = DL_NCHW;
152  input->dims[0] = 1;
153  input->dims[1] = 3;
154  input->dims[2] = -1;
155  input->dims[3] = -1;
156  return 0;
157 }
158 
159 static void deleter(void *arg)
160 {
161  av_freep(&arg);
162 }
163 
164 static int fill_model_input_th(THModel *th_model, THRequestItem *request)
165 {
166  LastLevelTaskItem *lltask = NULL;
167  TaskItem *task = NULL;
168  THInferRequest *infer_request = NULL;
169  DNNData input = { 0 };
170  DnnContext *ctx = th_model->ctx;
171  int ret, width_idx, height_idx, channel_idx;
172 
173  lltask = (LastLevelTaskItem *)ff_queue_pop_front(th_model->lltask_queue);
174  if (!lltask) {
175  ret = AVERROR(EINVAL);
176  goto err;
177  }
178  request->lltask = lltask;
179  task = lltask->task;
180  infer_request = request->infer_request;
181 
182  ret = get_input_th(th_model, &input, NULL);
183  if ( ret != 0) {
184  goto err;
185  }
186  width_idx = dnn_get_width_idx_by_layout(input.layout);
187  height_idx = dnn_get_height_idx_by_layout(input.layout);
188  channel_idx = dnn_get_channel_idx_by_layout(input.layout);
189  input.dims[height_idx] = task->in_frame->height;
190  input.dims[width_idx] = task->in_frame->width;
191  input.data = av_malloc(input.dims[height_idx] * input.dims[width_idx] *
192  input.dims[channel_idx] * sizeof(float));
193  if (!input.data)
194  return AVERROR(ENOMEM);
195  infer_request->input_tensor = new torch::Tensor();
196  infer_request->output = new torch::Tensor();
197 
198  switch (th_model->model->func_type) {
199  case DFT_PROCESS_FRAME:
200  input.scale = 255;
201  if (task->do_ioproc) {
202  if (th_model->model->frame_pre_proc != NULL) {
203  th_model->model->frame_pre_proc(task->in_frame, &input, th_model->model->filter_ctx);
204  } else {
206  }
207  }
208  break;
209  default:
210  avpriv_report_missing_feature(NULL, "model function type %d", th_model->model->func_type);
211  break;
212  }
213  *infer_request->input_tensor = torch::from_blob(input.data,
214  {1, input.dims[channel_idx], input.dims[height_idx], input.dims[width_idx]},
215  deleter, torch::kFloat32);
216  return 0;
217 
218 err:
219  th_free_request(infer_request);
220  return ret;
221 }
222 
223 static int th_start_inference(void *args)
224 {
225  THRequestItem *request = (THRequestItem *)args;
226  THInferRequest *infer_request = NULL;
227  LastLevelTaskItem *lltask = NULL;
228  TaskItem *task = NULL;
229  THModel *th_model = NULL;
230  DnnContext *ctx = NULL;
231  std::vector<torch::jit::IValue> inputs;
232  torch::NoGradGuard no_grad;
233 
234  if (!request) {
235  av_log(NULL, AV_LOG_ERROR, "THRequestItem is NULL\n");
236  return AVERROR(EINVAL);
237  }
238  infer_request = request->infer_request;
239  lltask = request->lltask;
240  task = lltask->task;
241  th_model = (THModel *)task->model;
242  ctx = th_model->ctx;
243 
244  if (ctx->torch_option.optimize)
245  torch::jit::setGraphExecutorOptimize(true);
246  else
247  torch::jit::setGraphExecutorOptimize(false);
248 
249  if (!infer_request->input_tensor || !infer_request->output) {
250  av_log(ctx, AV_LOG_ERROR, "input or output tensor is NULL\n");
251  return DNN_GENERIC_ERROR;
252  }
253  inputs.push_back(*infer_request->input_tensor);
254 
255  *infer_request->output = th_model->jit_model->forward(inputs).toTensor();
256 
257  return 0;
258 }
259 
260 static void infer_completion_callback(void *args) {
261  THRequestItem *request = (THRequestItem*)args;
262  LastLevelTaskItem *lltask = request->lltask;
263  TaskItem *task = lltask->task;
264  DNNData outputs = { 0 };
265  THInferRequest *infer_request = request->infer_request;
266  THModel *th_model = (THModel *)task->model;
267  torch::Tensor *output = infer_request->output;
268 
269  c10::IntArrayRef sizes = output->sizes();
270  outputs.order = DCO_RGB;
271  outputs.layout = DL_NCHW;
272  outputs.dt = DNN_FLOAT;
273  if (sizes.size() == 4) {
274  // 4 dimensions: [batch_size, channel, height, width]
275  // this format of data is normally used for video frame SR
276  outputs.dims[0] = sizes.at(0); // N
277  outputs.dims[1] = sizes.at(1); // C
278  outputs.dims[2] = sizes.at(2); // H
279  outputs.dims[3] = sizes.at(3); // W
280  } else {
281  avpriv_report_missing_feature(th_model->ctx, "Support of this kind of model");
282  goto err;
283  }
284 
285  switch (th_model->model->func_type) {
286  case DFT_PROCESS_FRAME:
287  if (task->do_ioproc) {
288  outputs.scale = 255;
289  outputs.data = output->data_ptr();
290  if (th_model->model->frame_post_proc != NULL) {
291  th_model->model->frame_post_proc(task->out_frame, &outputs, th_model->model->filter_ctx);
292  } else {
293  ff_proc_from_dnn_to_frame(task->out_frame, &outputs, th_model->ctx);
294  }
295  } else {
298  }
299  break;
300  default:
301  avpriv_report_missing_feature(th_model->ctx, "model function type %d", th_model->model->func_type);
302  goto err;
303  }
304  task->inference_done++;
305  av_freep(&request->lltask);
306 err:
307  th_free_request(infer_request);
308 
309  if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) {
310  destroy_request_item(&request);
311  av_log(th_model->ctx, AV_LOG_ERROR, "Unable to push back request_queue when failed to start inference.\n");
312  }
313 }
314 
315 static int execute_model_th(THRequestItem *request, Queue *lltask_queue)
316 {
317  THModel *th_model = NULL;
318  LastLevelTaskItem *lltask;
319  TaskItem *task = NULL;
320  int ret = 0;
321 
322  if (ff_queue_size(lltask_queue) == 0) {
323  destroy_request_item(&request);
324  return 0;
325  }
326 
327  lltask = (LastLevelTaskItem *)ff_queue_peek_front(lltask_queue);
328  if (lltask == NULL) {
329  av_log(NULL, AV_LOG_ERROR, "Failed to get LastLevelTaskItem\n");
330  ret = AVERROR(EINVAL);
331  goto err;
332  }
333  task = lltask->task;
334  th_model = (THModel *)task->model;
335 
336  ret = fill_model_input_th(th_model, request);
337  if ( ret != 0) {
338  goto err;
339  }
340  if (task->async) {
341  avpriv_report_missing_feature(th_model->ctx, "LibTorch async");
342  } else {
343  ret = th_start_inference((void *)(request));
344  if (ret != 0) {
345  goto err;
346  }
347  infer_completion_callback(request);
348  return (task->inference_done == task->inference_todo) ? 0 : DNN_GENERIC_ERROR;
349  }
350 
351 err:
352  th_free_request(request->infer_request);
353  if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) {
354  destroy_request_item(&request);
355  }
356  return ret;
357 }
358 
359 static int get_output_th(void *model, const char *input_name, int input_width, int input_height,
360  const char *output_name, int *output_width, int *output_height)
361 {
362  int ret = 0;
363  THModel *th_model = (THModel*) model;
364  DnnContext *ctx = th_model->ctx;
365  TaskItem task = { 0 };
366  THRequestItem *request = NULL;
367  DNNExecBaseParams exec_params = {
368  .input_name = input_name,
369  .output_names = &output_name,
370  .nb_output = 1,
371  .in_frame = NULL,
372  .out_frame = NULL,
373  };
374  ret = ff_dnn_fill_gettingoutput_task(&task, &exec_params, th_model, input_height, input_width, ctx);
375  if ( ret != 0) {
376  goto err;
377  }
378 
379  ret = extract_lltask_from_task(&task, th_model->lltask_queue);
380  if ( ret != 0) {
381  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
382  goto err;
383  }
384 
385  request = (THRequestItem*) ff_safe_queue_pop_front(th_model->request_queue);
386  if (!request) {
387  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
388  ret = AVERROR(EINVAL);
389  goto err;
390  }
391 
392  ret = execute_model_th(request, th_model->lltask_queue);
393  *output_width = task.out_frame->width;
394  *output_height = task.out_frame->height;
395 
396 err:
397  av_frame_free(&task.out_frame);
398  av_frame_free(&task.in_frame);
399  return ret;
400 }
401 
403 {
404  THInferRequest *request = (THInferRequest *)av_malloc(sizeof(THInferRequest));
405  if (!request) {
406  return NULL;
407  }
408  request->input_tensor = NULL;
409  request->output = NULL;
410  return request;
411 }
412 
414 {
415  DNNModel *model = NULL;
416  THModel *th_model = NULL;
417  THRequestItem *item = NULL;
418  const char *device_name = ctx->device ? ctx->device : "cpu";
419 
420  model = (DNNModel *)av_mallocz(sizeof(DNNModel));
421  if (!model) {
422  return NULL;
423  }
424 
425  th_model = (THModel *)av_mallocz(sizeof(THModel));
426  if (!th_model) {
427  av_freep(&model);
428  return NULL;
429  }
430  th_model->model = model;
431  model->model = th_model;
432  th_model->ctx = ctx;
433 
434  c10::Device device = c10::Device(device_name);
435  if (!device.is_cpu()) {
436  av_log(ctx, AV_LOG_ERROR, "Not supported device:\"%s\"\n", device_name);
437  goto fail;
438  }
439 
440  try {
441  th_model->jit_model = new torch::jit::Module;
442  (*th_model->jit_model) = torch::jit::load(ctx->model_filename);
443  } catch (const c10::Error& e) {
444  av_log(ctx, AV_LOG_ERROR, "Failed to load torch model\n");
445  goto fail;
446  }
447 
448  th_model->request_queue = ff_safe_queue_create();
449  if (!th_model->request_queue) {
450  goto fail;
451  }
452 
453  item = (THRequestItem *)av_mallocz(sizeof(THRequestItem));
454  if (!item) {
455  goto fail;
456  }
457  item->lltask = NULL;
459  if (!item->infer_request) {
460  av_log(NULL, AV_LOG_ERROR, "Failed to allocate memory for Torch inference request\n");
461  goto fail;
462  }
465  item->exec_module.args = item;
466 
467  if (ff_safe_queue_push_back(th_model->request_queue, item) < 0) {
468  goto fail;
469  }
470  item = NULL;
471 
472  th_model->task_queue = ff_queue_create();
473  if (!th_model->task_queue) {
474  goto fail;
475  }
476 
477  th_model->lltask_queue = ff_queue_create();
478  if (!th_model->lltask_queue) {
479  goto fail;
480  }
481 
482  model->get_input = &get_input_th;
483  model->get_output = &get_output_th;
484  model->filter_ctx = filter_ctx;
485  model->func_type = func_type;
486  return model;
487 
488 fail:
489  if (item) {
490  destroy_request_item(&item);
491  av_freep(&item);
492  }
493  dnn_free_model_th(&model);
494  return NULL;
495 }
496 
497 static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)
498 {
499  THModel *th_model = (THModel *)model->model;
500  DnnContext *ctx = th_model->ctx;
501  TaskItem *task;
502  THRequestItem *request;
503  int ret = 0;
504 
505  ret = ff_check_exec_params(ctx, DNN_TH, model->func_type, exec_params);
506  if (ret != 0) {
507  av_log(ctx, AV_LOG_ERROR, "exec parameter checking fail.\n");
508  return ret;
509  }
510 
511  task = (TaskItem *)av_malloc(sizeof(TaskItem));
512  if (!task) {
513  av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
514  return AVERROR(ENOMEM);
515  }
516 
517  ret = ff_dnn_fill_task(task, exec_params, th_model, 0, 1);
518  if (ret != 0) {
519  av_freep(&task);
520  av_log(ctx, AV_LOG_ERROR, "unable to fill task.\n");
521  return ret;
522  }
523 
524  ret = ff_queue_push_back(th_model->task_queue, task);
525  if (ret < 0) {
526  av_freep(&task);
527  av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
528  return ret;
529  }
530 
531  ret = extract_lltask_from_task(task, th_model->lltask_queue);
532  if (ret != 0) {
533  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
534  return ret;
535  }
536 
537  request = (THRequestItem *)ff_safe_queue_pop_front(th_model->request_queue);
538  if (!request) {
539  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
540  return AVERROR(EINVAL);
541  }
542 
543  return execute_model_th(request, th_model->lltask_queue);
544 }
545 
547 {
548  THModel *th_model = (THModel *)model->model;
549  return ff_dnn_get_result_common(th_model->task_queue, in, out);
550 }
551 
552 static int dnn_flush_th(const DNNModel *model)
553 {
554  THModel *th_model = (THModel *)model->model;
555  THRequestItem *request;
556 
557  if (ff_queue_size(th_model->lltask_queue) == 0)
558  // no pending task need to flush
559  return 0;
560 
561  request = (THRequestItem *)ff_safe_queue_pop_front(th_model->request_queue);
562  if (!request) {
563  av_log(th_model->ctx, AV_LOG_ERROR, "unable to get infer request.\n");
564  return AVERROR(EINVAL);
565  }
566 
567  return execute_model_th(request, th_model->lltask_queue);
568 }
569 
570 extern const DNNModule ff_dnn_backend_torch = {
571  .clazz = DNN_DEFINE_CLASS(dnn_th),
572  .load_model = dnn_load_model_th,
573  .execute_model = dnn_execute_model_th,
574  .get_result = dnn_get_result_th,
575  .flush = dnn_flush_th,
576  .free_model = dnn_free_model_th,
577 };
THRequestItem::lltask
LastLevelTaskItem * lltask
Definition: dnn_backend_torch.cpp:55
THModel::lltask_queue
Queue * lltask_queue
Definition: dnn_backend_torch.cpp:45
THRequestItem::infer_request
THInferRequest * infer_request
Definition: dnn_backend_torch.cpp:54
THModel::ctx
DnnContext * ctx
Definition: dnn_backend_torch.cpp:40
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
ff_safe_queue_pop_front
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
Definition: safe_queue.c:105
out
FILE * out
Definition: movenc.c:55
deleter
static void deleter(void *arg)
Definition: dnn_backend_torch.cpp:159
FLAGS
#define FLAGS
Definition: dnn_backend_torch.cpp:61
THModel
Definition: dnn_backend_torch.cpp:39
DNNAsyncExecModule
Common Async Execution Mechanism for the DNN Backends.
Definition: dnn_backend_common.h:65
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
ff_queue_pop_front
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition: queue.c:151
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:30
ff_queue_size
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Definition: queue.c:88
DNN_GENERIC_ERROR
#define DNN_GENERIC_ERROR
Definition: dnn_interface.h:33
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:160
LastLevelTaskItem
Definition: dnn_backend_common.h:57
ff_dnn_backend_torch
const DNNModule ff_dnn_backend_torch
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:374
AVFrame::width
int width
Definition: frame.h:446
SafeQueue
Double-ended queue with mutex locks ensuring data consistency while multithreading.
Definition: safe_queue.c:46
dnn_execute_model_th
static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_backend_torch.cpp:497
AVOption
AVOption.
Definition: opt.h:357
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:108
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:77
dnn_io_proc.h
TaskItem
Definition: dnn_backend_common.h:43
DNNAsyncExecModule::callback
void(* callback)(void *args)
Completion Callback for the backend.
Definition: dnn_backend_common.h:77
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:30
get_input_th
static int get_input_th(void *model, DNNData *input, const char *input_name)
Definition: dnn_backend_torch.cpp:147
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:97
ff_queue_create
Queue * ff_queue_create(void)
Create a Queue instance.
Definition: queue.c:47
dnn_get_width_idx_by_layout
static int dnn_get_width_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:194
TaskItem::model
void * model
Definition: dnn_backend_common.h:44
fail
#define fail()
Definition: checkasm.h:182
DnnContext
Definition: dnn_interface.h:141
filter_ctx
static FilteringContext * filter_ctx
Definition: transcode.c:52
Queue
Linear double-ended data structure.
Definition: queue.c:33
ff_queue_push_back
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
Definition: queue.c:130
THModel::jit_model
torch::jit::Module * jit_model
Definition: dnn_backend_torch.cpp:42
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
LastLevelTaskItem::task
TaskItem * task
Definition: dnn_backend_common.h:58
destroy_request_item
static void destroy_request_item(THRequestItem **arg)
Definition: dnn_backend_torch.cpp:102
th_create_inference_request
static THInferRequest * th_create_inference_request(void)
Definition: dnn_backend_torch.cpp:402
ff_queue_destroy
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
Definition: queue.c:72
DNNData
Definition: dnn_interface.h:65
DNNModule::clazz
const AVClass clazz
Definition: dnn_interface.h:174
ff_dnn_fill_gettingoutput_task
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
Definition: dnn_backend_common.c:156
ctx
AVFormatContext * ctx
Definition: movenc.c:49
TaskItem::inference_todo
uint32_t inference_todo
Definition: dnn_backend_common.h:52
DL_NCHW
@ DL_NCHW
Definition: dnn_interface.h:61
dnn_load_model_th
static DNNModel * dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
Definition: dnn_backend_torch.cpp:413
arg
const char * arg
Definition: jacosubdec.c:67
if
if(ret)
Definition: filter_design.txt:179
ff_safe_queue_size
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
Definition: safe_queue.c:80
ff_proc_from_frame_to_dnn
int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:182
THRequestItem::exec_module
DNNAsyncExecModule exec_module
Definition: dnn_backend_torch.cpp:56
NULL
#define NULL
Definition: coverity.c:32
sizes
static const int sizes[][2]
Definition: img2dec.c:59
ff_safe_queue_create
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
Definition: safe_queue.c:52
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:111
ff_dnn_async_module_cleanup
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
Definition: dnn_backend_common.c:86
infer_completion_callback
static void infer_completion_callback(void *args)
Definition: dnn_backend_torch.cpp:260
TaskItem::in_frame
AVFrame * in_frame
Definition: dnn_backend_common.h:45
extract_lltask_from_task
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
Definition: dnn_backend_torch.cpp:67
inputs
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
Definition: filter_design.txt:243
THInferRequest::output
torch::Tensor * output
Definition: dnn_backend_torch.cpp:49
TaskItem::async
uint8_t async
Definition: dnn_backend_common.h:49
TaskItem::inference_done
uint32_t inference_done
Definition: dnn_backend_common.h:53
queue.h
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:99
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
ff_safe_queue_destroy
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
Definition: safe_queue.c:69
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:37
dnn_get_result_th
static DNNAsyncStatusType dnn_get_result_th(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_backend_torch.cpp:546
ff_dnn_fill_task
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
Definition: dnn_backend_common.c:50
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
DNN_DEFINE_CLASS
#define DNN_DEFINE_CLASS(fname)
Definition: dnn_backend_common.h:39
THRequestItem
Definition: dnn_backend_torch.cpp:53
ff_safe_queue_push_back
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
Definition: safe_queue.c:95
th_start_inference
static int th_start_inference(void *args)
Definition: dnn_backend_torch.cpp:223
THInferRequest::input_tensor
torch::Tensor * input_tensor
Definition: dnn_backend_torch.cpp:50
DNNAsyncExecModule::start_inference
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
Definition: dnn_backend_common.h:70
DNNAsyncExecModule::args
void * args
Argument for the execution functions.
Definition: dnn_backend_common.h:83
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:256
safe_queue.h
THInferRequest
Definition: dnn_backend_torch.cpp:48
outputs
static const AVFilterPad outputs[]
Definition: af_aap.c:311
ret
ret
Definition: filter_design.txt:187
get_output_th
static int get_output_th(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_backend_torch.cpp:359
DNNModel::get_input
int(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:102
TaskItem::out_frame
AVFrame * out_frame
Definition: dnn_backend_common.h:46
AVFrame::height
int height
Definition: frame.h:446
dnn_backend_common.h
dnn_th_options
static const AVOption dnn_th_options[]
Definition: dnn_backend_torch.cpp:62
execute_model_th
static int execute_model_th(THRequestItem *request, Queue *lltask_queue)
Definition: dnn_backend_torch.cpp:315
OFFSET
#define OFFSET(x)
Definition: dnn_backend_torch.cpp:60
AV_OPT_TYPE_INT
@ AV_OPT_TYPE_INT
Definition: opt.h:245
THModel::model
DNNModel * model
Definition: dnn_backend_torch.cpp:41
ff_dnn_get_result_common
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
Definition: dnn_backend_common.c:136
ff_queue_peek_front
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
Definition: queue.c:93
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:42
AVFilterContext
An instance of a filter.
Definition: avfilter.h:407
DNNModel
Definition: dnn_interface.h:93
DNN_TH
@ DNN_TH
Definition: dnn_interface.h:35
mem.h
dnn_get_height_idx_by_layout
static int dnn_get_height_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:199
dnn_flush_th
static int dnn_flush_th(const DNNModel *model)
Definition: dnn_backend_torch.cpp:552
THModel::task_queue
Queue * task_queue
Definition: dnn_backend_torch.cpp:44
dnn_get_channel_idx_by_layout
static int dnn_get_channel_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:204
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
DNNExecBaseParams
Definition: dnn_interface.h:76
dnn_free_model_th
static void dnn_free_model_th(DNNModel **model)
Definition: dnn_backend_torch.cpp:116
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
TaskItem::do_ioproc
uint8_t do_ioproc
Definition: dnn_backend_common.h:50
DNNModel::get_output
int(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:104
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
DNNModule
Definition: dnn_interface.h:173
fill_model_input_th
static int fill_model_input_th(THModel *th_model, THRequestItem *request)
Definition: dnn_backend_torch.cpp:164
THModel::request_queue
SafeQueue * request_queue
Definition: dnn_backend_torch.cpp:43
DNNModel::model
void * model
Definition: dnn_interface.h:95
ff_proc_from_dnn_to_frame
int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:42
th_free_request
static void th_free_request(THInferRequest *request)
Definition: dnn_backend_torch.cpp:87