onnx.hpp 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335
  1. // This file is part of OpenCV project.
  2. // It is subject to the license terms in the LICENSE file found in the top-level directory
  3. // of this distribution and at http://opencv.org/license.html.
  4. //
  5. // Copyright (C) 2020-2021 Intel Corporation
  6. #ifndef OPENCV_GAPI_INFER_ONNX_HPP
  7. #define OPENCV_GAPI_INFER_ONNX_HPP
  8. #include <unordered_map>
  9. #include <string>
  10. #include <array>
  11. #include <tuple> // tuple, tuple_size
  12. #include <opencv2/gapi/opencv_includes.hpp>
  13. #include <opencv2/gapi/util/any.hpp>
  14. #include <opencv2/core/cvdef.h> // GAPI_EXPORTS
  15. #include <opencv2/gapi/gkernel.hpp> // GKernelPackage
  16. #include <opencv2/gapi/infer.hpp> // Generic
  17. namespace cv {
  18. namespace gapi {
  19. /**
  20. * @brief This namespace contains G-API ONNX Runtime backend functions, structures, and symbols.
  21. */
  22. namespace onnx {
  23. GAPI_EXPORTS cv::gapi::GBackend backend();
  24. enum class TraitAs: int {
  25. TENSOR, //!< G-API traits an associated cv::Mat as a raw tensor
  26. // and passes dimensions as-is
  27. IMAGE //!< G-API traits an associated cv::Mat as an image so
  28. // creates an "image" blob (NCHW/NHWC, etc)
  29. };
  30. using PostProc = std::function<void(const std::unordered_map<std::string, cv::Mat> &,
  31. std::unordered_map<std::string, cv::Mat> &)>;
  32. namespace detail {
  33. /**
  34. * @brief This structure contains description of inference parameters
  35. * which is specific to ONNX models.
  36. */
  37. struct ParamDesc {
  38. std::string model_path; //!< Path to model.
  39. // NB: nun_* may differ from topology's real input/output port numbers
  40. // (e.g. topology's partial execution)
  41. std::size_t num_in; //!< How many inputs are defined in the operation
  42. std::size_t num_out; //!< How many outputs are defined in the operation
  43. // NB: Here order follows the `Net` API
  44. std::vector<std::string> input_names; //!< Names of input network layers.
  45. std::vector<std::string> output_names; //!< Names of output network layers.
  46. using ConstInput = std::pair<cv::Mat, TraitAs>;
  47. std::unordered_map<std::string, ConstInput> const_inputs; //!< Map with pair of name of network layer and ConstInput which will be associated with this.
  48. std::vector<cv::Scalar> mean; //!< Mean values for preprocessing.
  49. std::vector<cv::Scalar> stdev; //!< Standard deviation values for preprocessing.
  50. std::vector<cv::GMatDesc> out_metas; //!< Out meta information about your output (type, dimension).
  51. PostProc custom_post_proc; //!< Post processing function.
  52. std::vector<bool> normalize; //!< Vector of bool values that enabled or disabled normalize of input data.
  53. std::vector<std::string> names_to_remap; //!< Names of output layers that will be processed in PostProc function.
  54. bool is_generic;
  55. // TODO: Needs to modify the rest of ParamDesc accordingly to support
  56. // both generic and non-generic options without duplication
  57. // (as it was done for the OV IE backend)
  58. // These values are pushed into the respective vector<> fields above
  59. // when the generic infer parameters are unpacked (see GONNXBackendImpl::unpackKernel)
  60. std::unordered_map<std::string, std::pair<cv::Scalar, cv::Scalar> > generic_mstd;
  61. std::unordered_map<std::string, bool> generic_norm;
  62. };
  63. } // namespace detail
  64. template<typename Net>
  65. struct PortCfg {
  66. using In = std::array
  67. < std::string
  68. , std::tuple_size<typename Net::InArgs>::value >;
  69. using Out = std::array
  70. < std::string
  71. , std::tuple_size<typename Net::OutArgs>::value >;
  72. using NormCoefs = std::array
  73. < cv::Scalar
  74. , std::tuple_size<typename Net::InArgs>::value >;
  75. using Normalize = std::array
  76. < bool
  77. , std::tuple_size<typename Net::InArgs>::value >;
  78. };
  79. /**
  80. * Contains description of inference parameters and kit of functions that
  81. * fill this parameters.
  82. */
  83. template<typename Net> class Params {
  84. public:
  85. /** @brief Class constructor.
  86. Constructs Params based on model information and sets default values for other
  87. inference description parameters.
  88. @param model Path to model (.onnx file).
  89. */
  90. Params(const std::string &model) {
  91. desc.model_path = model;
  92. desc.num_in = std::tuple_size<typename Net::InArgs>::value;
  93. desc.num_out = std::tuple_size<typename Net::OutArgs>::value;
  94. desc.is_generic = false;
  95. };
  96. /** @brief Specifies sequence of network input layers names for inference.
  97. The function is used to associate data of graph inputs with input layers of
  98. network topology. Number of names has to match the number of network inputs. If a network
  99. has only one input layer, there is no need to call it as the layer is
  100. associated with input automatically but this doesn't prevent you from
  101. doing it yourself. Count of names has to match to number of network inputs.
  102. @param layer_names std::array<std::string, N> where N is the number of inputs
  103. as defined in the @ref G_API_NET. Contains names of input layers.
  104. @return the reference on modified object.
  105. */
  106. Params<Net>& cfgInputLayers(const typename PortCfg<Net>::In &layer_names) {
  107. desc.input_names.assign(layer_names.begin(), layer_names.end());
  108. return *this;
  109. }
  110. /** @brief Specifies sequence of output layers names for inference.
  111. The function is used to associate data of graph outputs with output layers of
  112. network topology. If a network has only one output layer, there is no need to call it
  113. as the layer is associated with output automatically but this doesn't prevent
  114. you from doing it yourself. Count of names has to match to number of network
  115. outputs or you can set your own output but for this case you have to
  116. additionally use @ref cfgPostProc function.
  117. @param layer_names std::array<std::string, N> where N is the number of outputs
  118. as defined in the @ref G_API_NET. Contains names of output layers.
  119. @return the reference on modified object.
  120. */
  121. Params<Net>& cfgOutputLayers(const typename PortCfg<Net>::Out &layer_names) {
  122. desc.output_names.assign(layer_names.begin(), layer_names.end());
  123. return *this;
  124. }
  125. /** @brief Sets a constant input.
  126. The function is used to set constant input. This input has to be
  127. a prepared tensor since preprocessing is disabled for this case. You should
  128. provide name of network layer which will receive provided data.
  129. @param layer_name Name of network layer.
  130. @param data cv::Mat that contains data which will be associated with network layer.
  131. @param hint Type of input (TENSOR).
  132. @return the reference on modified object.
  133. */
  134. Params<Net>& constInput(const std::string &layer_name,
  135. const cv::Mat &data,
  136. TraitAs hint = TraitAs::TENSOR) {
  137. desc.const_inputs[layer_name] = {data, hint};
  138. return *this;
  139. }
  140. /** @brief Specifies mean value and standard deviation for preprocessing.
  141. The function is used to set mean value and standard deviation for preprocessing
  142. of input data.
  143. @param m std::array<cv::Scalar, N> where N is the number of inputs
  144. as defined in the @ref G_API_NET. Contains mean values.
  145. @param s std::array<cv::Scalar, N> where N is the number of inputs
  146. as defined in the @ref G_API_NET. Contains standard deviation values.
  147. @return the reference on modified object.
  148. */
  149. Params<Net>& cfgMeanStd(const typename PortCfg<Net>::NormCoefs &m,
  150. const typename PortCfg<Net>::NormCoefs &s) {
  151. desc.mean.assign(m.begin(), m.end());
  152. desc.stdev.assign(s.begin(), s.end());
  153. return *this;
  154. }
  155. /** @brief Configures graph output and provides the post processing function from user.
  156. The function is used when you work with networks with dynamic outputs.
  157. Since we can't know dimensions of inference result needs provide them for
  158. construction of graph output. This dimensions can differ from inference result.
  159. So you have to provide @ref PostProc function that gets information from inference
  160. result and fill output which is constructed by dimensions from out_metas.
  161. @param out_metas Out meta information about your output (type, dimension).
  162. @param remap_function Post processing function, which has two parameters. First is onnx
  163. result, second is graph output. Both parameters is std::map that contain pair of
  164. layer's name and cv::Mat.
  165. @return the reference on modified object.
  166. */
  167. Params<Net>& cfgPostProc(const std::vector<cv::GMatDesc> &out_metas,
  168. const PostProc &remap_function) {
  169. desc.out_metas = out_metas;
  170. desc.custom_post_proc = remap_function;
  171. return *this;
  172. }
  173. /** @overload
  174. Function with a rvalue parameters.
  175. @param out_metas rvalue out meta information about your output (type, dimension).
  176. @param remap_function rvalue post processing function, which has two parameters. First is onnx
  177. result, second is graph output. Both parameters is std::map that contain pair of
  178. layer's name and cv::Mat.
  179. @return the reference on modified object.
  180. */
  181. Params<Net>& cfgPostProc(std::vector<cv::GMatDesc> &&out_metas,
  182. PostProc &&remap_function) {
  183. desc.out_metas = std::move(out_metas);
  184. desc.custom_post_proc = std::move(remap_function);
  185. return *this;
  186. }
  187. /** @overload
  188. The function has additional parameter names_to_remap. This parameter provides
  189. information about output layers which will be used for inference and post
  190. processing function.
  191. @param out_metas Out meta information.
  192. @param remap_function Post processing function.
  193. @param names_to_remap Names of output layers. network's inference will
  194. be done on these layers. Inference's result will be processed in post processing
  195. function using these names.
  196. @return the reference on modified object.
  197. */
  198. Params<Net>& cfgPostProc(const std::vector<cv::GMatDesc> &out_metas,
  199. const PostProc &remap_function,
  200. const std::vector<std::string> &names_to_remap) {
  201. desc.out_metas = out_metas;
  202. desc.custom_post_proc = remap_function;
  203. desc.names_to_remap = names_to_remap;
  204. return *this;
  205. }
  206. /** @overload
  207. Function with a rvalue parameters and additional parameter names_to_remap.
  208. @param out_metas rvalue out meta information.
  209. @param remap_function rvalue post processing function.
  210. @param names_to_remap rvalue names of output layers. network's inference will
  211. be done on these layers. Inference's result will be processed in post processing
  212. function using these names.
  213. @return the reference on modified object.
  214. */
  215. Params<Net>& cfgPostProc(std::vector<cv::GMatDesc> &&out_metas,
  216. PostProc &&remap_function,
  217. std::vector<std::string> &&names_to_remap) {
  218. desc.out_metas = std::move(out_metas);
  219. desc.custom_post_proc = std::move(remap_function);
  220. desc.names_to_remap = std::move(names_to_remap);
  221. return *this;
  222. }
  223. /** @brief Specifies normalize parameter for preprocessing.
  224. The function is used to set normalize parameter for preprocessing of input data.
  225. @param normalizations std::array<cv::Scalar, N> where N is the number of inputs
  226. as defined in the @ref G_API_NET. Сontains bool values that enabled or disabled
  227. normalize of input data.
  228. @return the reference on modified object.
  229. */
  230. Params<Net>& cfgNormalize(const typename PortCfg<Net>::Normalize &normalizations) {
  231. desc.normalize.assign(normalizations.begin(), normalizations.end());
  232. return *this;
  233. }
  234. // BEGIN(G-API's network parametrization API)
  235. GBackend backend() const { return cv::gapi::onnx::backend(); }
  236. std::string tag() const { return Net::tag(); }
  237. cv::util::any params() const { return { desc }; }
  238. // END(G-API's network parametrization API)
  239. protected:
  240. detail::ParamDesc desc;
  241. };
  242. /*
  243. * @brief This structure provides functions for generic network type that
  244. * fill inference parameters.
  245. * @see struct Generic
  246. */
  247. template<>
  248. class Params<cv::gapi::Generic> {
  249. public:
  250. /** @brief Class constructor.
  251. Constructs Params based on input information and sets default values for other
  252. inference description parameters.
  253. @param tag string tag of the network for which these parameters are intended.
  254. @param model_path path to model file (.onnx file).
  255. */
  256. Params(const std::string& tag, const std::string& model_path)
  257. : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {} }, m_tag(tag) {}
  258. void cfgMeanStdDev(const std::string &layer,
  259. const cv::Scalar &m,
  260. const cv::Scalar &s) {
  261. desc.generic_mstd[layer] = std::make_pair(m, s);
  262. }
  263. void cfgNormalize(const std::string &layer, bool flag) {
  264. desc.generic_norm[layer] = flag;
  265. }
  266. // BEGIN(G-API's network parametrization API)
  267. GBackend backend() const { return cv::gapi::onnx::backend(); }
  268. std::string tag() const { return m_tag; }
  269. cv::util::any params() const { return { desc }; }
  270. // END(G-API's network parametrization API)
  271. protected:
  272. detail::ParamDesc desc;
  273. std::string m_tag;
  274. };
  275. } // namespace onnx
  276. } // namespace gapi
  277. } // namespace cv
  278. #endif // OPENCV_GAPI_INFER_HPP