dnn.hpp 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746
  1. /*M///////////////////////////////////////////////////////////////////////////////////////
  2. //
  3. // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
  4. //
  5. // By downloading, copying, installing or using the software you agree to this license.
  6. // If you do not agree to this license, do not download, install,
  7. // copy or use the software.
  8. //
  9. //
  10. // License Agreement
  11. // For Open Source Computer Vision Library
  12. //
  13. // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
  14. // Third party copyrights are property of their respective owners.
  15. //
  16. // Redistribution and use in source and binary forms, with or without modification,
  17. // are permitted provided that the following conditions are met:
  18. //
  19. // * Redistribution's of source code must retain the above copyright notice,
  20. // this list of conditions and the following disclaimer.
  21. //
  22. // * Redistribution's in binary form must reproduce the above copyright notice,
  23. // this list of conditions and the following disclaimer in the documentation
  24. // and/or other materials provided with the distribution.
  25. //
  26. // * The name of the copyright holders may not be used to endorse or promote products
  27. // derived from this software without specific prior written permission.
  28. //
  29. // This software is provided by the copyright holders and contributors "as is" and
  30. // any express or implied warranties, including, but not limited to, the implied
  31. // warranties of merchantability and fitness for a particular purpose are disclaimed.
  32. // In no event shall the Intel Corporation or contributors be liable for any direct,
  33. // indirect, incidental, special, exemplary, or consequential damages
  34. // (including, but not limited to, procurement of substitute goods or services;
  35. // loss of use, data, or profits; or business interruption) however caused
  36. // and on any theory of liability, whether in contract, strict liability,
  37. // or tort (including negligence or otherwise) arising in any way out of
  38. // the use of this software, even if advised of the possibility of such damage.
  39. //
  40. //M*/
  41. #ifndef OPENCV_DNN_DNN_HPP
  42. #define OPENCV_DNN_DNN_HPP
  43. #include <vector>
  44. #include <opencv2/core.hpp>
  45. #if !defined CV_DOXYGEN && !defined CV_DNN_DONT_ADD_EXPERIMENTAL_NS
  46. #define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_v2 {
  47. #define CV__DNN_EXPERIMENTAL_NS_END }
  48. namespace cv { namespace dnn { namespace experimental_dnn_v2 { } using namespace experimental_dnn_v2; }}
  49. #else
  50. #define CV__DNN_EXPERIMENTAL_NS_BEGIN
  51. #define CV__DNN_EXPERIMENTAL_NS_END
  52. #endif
  53. #include <opencv2/dnn/dict.hpp>
  54. namespace cv {
  55. namespace dnn {
  56. CV__DNN_EXPERIMENTAL_NS_BEGIN
  57. //! @addtogroup dnn
  58. //! @{
  59. typedef std::vector<int> MatShape;
  60. /**
  61. * @brief Enum of computation backends supported by layers.
  62. */
  63. enum Backend
  64. {
  65. DNN_BACKEND_DEFAULT,
  66. DNN_BACKEND_HALIDE
  67. };
  68. /**
  69. * @brief Enum of target devices for computations.
  70. */
  71. enum Target
  72. {
  73. DNN_TARGET_CPU,
  74. DNN_TARGET_OPENCL
  75. };
  76. /** @brief This class provides all data needed to initialize layer.
  77. *
  78. * It includes dictionary with scalar params (which can be readed by using Dict interface),
  79. * blob params #blobs and optional meta information: #name and #type of layer instance.
  80. */
  81. class CV_EXPORTS LayerParams : public Dict
  82. {
  83. public:
  84. //TODO: Add ability to name blob params
  85. std::vector<Mat> blobs; //!< List of learned parameters stored as blobs.
  86. String name; //!< Name of the layer instance (optional, can be used internal purposes).
  87. String type; //!< Type name which was used for creating layer by layer factory (optional).
  88. };
  89. /**
  90. * @brief Derivatives of this class encapsulates functions of certain backends.
  91. */
  92. class BackendNode
  93. {
  94. public:
  95. BackendNode(int backendId);
  96. virtual ~BackendNode(); //!< Virtual destructor to make polymorphism.
  97. int backendId; //!< Backend identifier.
  98. };
  99. /**
  100. * @brief Derivatives of this class wraps cv::Mat for different backends and targets.
  101. */
  102. class BackendWrapper
  103. {
  104. public:
  105. BackendWrapper(int backendId, int targetId);
  106. /**
  107. * @brief Wrap cv::Mat for specific backend and target.
  108. * @param[in] targetId Target identifier.
  109. * @param[in] m cv::Mat for wrapping.
  110. *
  111. * Make CPU->GPU data transfer if it's require for the target.
  112. */
  113. BackendWrapper(int targetId, const cv::Mat& m);
  114. /**
  115. * @brief Make wrapper for reused cv::Mat.
  116. * @param[in] base Wrapper of cv::Mat that will be reused.
  117. * @param[in] shape Specific shape.
  118. *
  119. * Initialize wrapper from another one. It'll wrap the same host CPU
  120. * memory and mustn't allocate memory on device(i.e. GPU). It might
  121. * has different shape. Use in case of CPU memory reusing for reuse
  122. * associented memory on device too.
  123. */
  124. BackendWrapper(const Ptr<BackendWrapper>& base, const MatShape& shape);
  125. virtual ~BackendWrapper(); //!< Virtual destructor to make polymorphism.
  126. /**
  127. * @brief Transfer data to CPU host memory.
  128. */
  129. virtual void copyToHost() = 0;
  130. /**
  131. * @brief Indicate that an actual data is on CPU.
  132. */
  133. virtual void setHostDirty() = 0;
  134. int backendId; //!< Backend identifier.
  135. int targetId; //!< Target identifier.
  136. };
  137. class CV_EXPORTS ActivationLayer;
  138. class CV_EXPORTS BatchNormLayer;
  139. class CV_EXPORTS ScaleLayer;
  140. /** @brief This interface class allows to build new Layers - are building blocks of networks.
  141. *
  142. * Each class, derived from Layer, must implement allocate() methods to declare own outputs and forward() to compute outputs.
  143. * Also before using the new layer into networks you must register your layer by using one of @ref dnnLayerFactory "LayerFactory" macros.
  144. */
  145. class CV_EXPORTS_W Layer : public Algorithm
  146. {
  147. public:
  148. //! List of learned parameters must be stored here to allow read them by using Net::getParam().
  149. CV_PROP_RW std::vector<Mat> blobs;
  150. /** @brief Computes and sets internal parameters according to inputs, outputs and blobs.
  151. * @param[in] input vector of already allocated input blobs
  152. * @param[out] output vector of already allocated output blobs
  153. *
  154. * If this method is called after network has allocated all memory for input and output blobs
  155. * and before inferencing.
  156. */
  157. virtual void finalize(const std::vector<Mat*> &input, std::vector<Mat> &output);
  158. /** @brief Given the @p input blobs, computes the output @p blobs.
  159. * @param[in] input the input blobs.
  160. * @param[out] output allocated output blobs, which will store results of the computation.
  161. * @param[out] internals allocated internal blobs
  162. */
  163. virtual void forward(std::vector<Mat*> &input, std::vector<Mat> &output, std::vector<Mat> &internals) = 0;
  164. /** @brief @overload */
  165. CV_WRAP void finalize(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs);
  166. /** @brief @overload */
  167. CV_WRAP std::vector<Mat> finalize(const std::vector<Mat> &inputs);
  168. /** @brief @overload */
  169. CV_WRAP void forward(const std::vector<Mat> &inputs, CV_IN_OUT std::vector<Mat> &outputs,
  170. CV_IN_OUT std::vector<Mat> &internals);
  171. /** @brief Allocates layer and computes output. */
  172. CV_WRAP void run(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs,
  173. CV_IN_OUT std::vector<Mat> &internals);
  174. /** @brief Returns index of input blob into the input array.
  175. * @param inputName label of input blob
  176. *
  177. * Each layer input and output can be labeled to easily identify them using "%<layer_name%>[.output_name]" notation.
  178. * This method maps label of input blob to its index into input vector.
  179. */
  180. virtual int inputNameToIndex(String inputName);
  181. /** @brief Returns index of output blob in output array.
  182. * @see inputNameToIndex()
  183. */
  184. virtual int outputNameToIndex(String outputName);
  185. /**
  186. * @brief Ask layer if it support specific backend for doing computations.
  187. * @param[in] backendId computation backend identifier.
  188. * @see Backend
  189. */
  190. virtual bool supportBackend(int backendId);
  191. /**
  192. * @brief Returns Halide backend node.
  193. * @param[in] inputs Input Halide buffers.
  194. * @see BackendNode, BackendWrapper
  195. *
  196. * Input buffers should be exactly the same that will be used in forward invocations.
  197. * Despite we can use Halide::ImageParam based on input shape only,
  198. * it helps prevent some memory management issues (if something wrong,
  199. * Halide tests will be failed).
  200. */
  201. virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs);
  202. /**
  203. * @brief Automatic Halide scheduling based on layer hyper-parameters.
  204. * @param[in] node Backend node with Halide functions.
  205. * @param[in] inputs Blobs that will be used in forward invocations.
  206. * @param[in] outputs Blobs that will be used in forward invocations.
  207. * @param[in] targetId Target identifier
  208. * @see BackendNode, Target
  209. *
  210. * Layer don't use own Halide::Func members because we can have applied
  211. * layers fusing. In this way the fused function should be scheduled.
  212. */
  213. virtual void applyHalideScheduler(Ptr<BackendNode>& node,
  214. const std::vector<Mat*> &inputs,
  215. const std::vector<Mat> &outputs,
  216. int targetId) const;
  217. /**
  218. * @brief Implement layers fusing.
  219. * @param[in] node Backend node of bottom layer.
  220. * @see BackendNode
  221. *
  222. * Actual for graph-based backends. If layer attached successfully,
  223. * returns non-empty cv::Ptr to node of the same backend.
  224. * Fuse only over the last function.
  225. */
  226. virtual Ptr<BackendNode> tryAttach(const Ptr<BackendNode>& node);
  227. /**
  228. * @brief Tries to attach to the layer the subsequent activation layer, i.e. do the layer fusion in a partial case.
  229. * @param[in] layer The subsequent activation layer.
  230. *
  231. * Returns true if the activation layer has been attached successfully.
  232. */
  233. virtual bool setActivation(const Ptr<ActivationLayer>& layer);
  234. /**
  235. * @brief Tries to attach to the layer the subsequent batch normalization layer, i.e. do the layer fusion in a partial case.
  236. * @param[in] layer The subsequent batch normalization layer.
  237. *
  238. * Returns true if the batch normalization layer has been attached successfully.
  239. */
  240. virtual bool setBatchNorm(const Ptr<BatchNormLayer>& layer);
  241. /**
  242. * @brief Tries to attach to the layer the subsequent scaling layer, i.e. do the layer fusion in a partial case.
  243. * @param[in] layer The subsequent scaling layer.
  244. *
  245. * Returns true if the scaling layer has been attached successfully.
  246. */
  247. virtual bool setScale(const Ptr<ScaleLayer>& layer);
  248. /**
  249. * @brief "Deattaches" all the layers, attached to particular layer.
  250. */
  251. virtual void unsetAttached();
  252. virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
  253. const int requiredOutputs,
  254. std::vector<MatShape> &outputs,
  255. std::vector<MatShape> &internals) const;
  256. virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
  257. const std::vector<MatShape> &outputs) const {(void)inputs; (void)outputs; return 0;}
  258. CV_PROP String name; //!< Name of the layer instance, can be used for logging or other internal purposes.
  259. CV_PROP String type; //!< Type name which was used for creating layer by layer factory.
  260. CV_PROP int preferableTarget; //!< prefer target for layer forwarding
  261. Layer();
  262. explicit Layer(const LayerParams &params); //!< Initializes only #name, #type and #blobs fields.
  263. void setParamsFrom(const LayerParams &params); //!< Initializes only #name, #type and #blobs fields.
  264. virtual ~Layer();
  265. };
  266. /** @brief This class allows to create and manipulate comprehensive artificial neural networks.
  267. *
  268. * Neural network is presented as directed acyclic graph (DAG), where vertices are Layer instances,
  269. * and edges specify relationships between layers inputs and outputs.
  270. *
  271. * Each network layer has unique integer id and unique string name inside its network.
  272. * LayerId can store either layer name or layer id.
  273. *
  274. * This class supports reference counting of its instances, i. e. copies point to the same instance.
  275. */
  276. class CV_EXPORTS_W_SIMPLE Net
  277. {
  278. public:
  279. CV_WRAP Net(); //!< Default constructor.
  280. CV_WRAP ~Net(); //!< Destructor frees the net only if there aren't references to the net anymore.
  281. /** Returns true if there are no layers in the network. */
  282. CV_WRAP bool empty() const;
  283. /** @brief Adds new layer to the net.
  284. * @param name unique name of the adding layer.
  285. * @param type typename of the adding layer (type must be registered in LayerRegister).
  286. * @param params parameters which will be used to initialize the creating layer.
  287. * @returns unique identifier of created layer, or -1 if a failure will happen.
  288. */
  289. int addLayer(const String &name, const String &type, LayerParams &params);
  290. /** @brief Adds new layer and connects its first input to the first output of previously added layer.
  291. * @see addLayer()
  292. */
  293. int addLayerToPrev(const String &name, const String &type, LayerParams &params);
  294. /** @brief Converts string name of the layer to the integer identifier.
  295. * @returns id of the layer, or -1 if the layer wasn't found.
  296. */
  297. CV_WRAP int getLayerId(const String &layer);
  298. CV_WRAP std::vector<String> getLayerNames() const;
  299. /** @brief Container for strings and integers. */
  300. typedef DictValue LayerId;
  301. /** @brief Returns pointer to layer with specified id or name which the network use. */
  302. CV_WRAP Ptr<Layer> getLayer(LayerId layerId);
  303. /** @brief Returns pointers to input layers of specific layer. */
  304. std::vector<Ptr<Layer> > getLayerInputs(LayerId layerId); // FIXIT: CV_WRAP
  305. /** @brief Delete layer for the network (not implemented yet) */
  306. CV_WRAP void deleteLayer(LayerId layer);
  307. /** @brief Connects output of the first layer to input of the second layer.
  308. * @param outPin descriptor of the first layer output.
  309. * @param inpPin descriptor of the second layer input.
  310. *
  311. * Descriptors have the following template <DFN>&lt;layer_name&gt;[.input_number]</DFN>:
  312. * - the first part of the template <DFN>layer_name</DFN> is sting name of the added layer.
  313. * If this part is empty then the network input pseudo layer will be used;
  314. * - the second optional part of the template <DFN>input_number</DFN>
  315. * is either number of the layer input, either label one.
  316. * If this part is omitted then the first layer input will be used.
  317. *
  318. * @see setNetInputs(), Layer::inputNameToIndex(), Layer::outputNameToIndex()
  319. */
  320. CV_WRAP void connect(String outPin, String inpPin);
  321. /** @brief Connects #@p outNum output of the first layer to #@p inNum input of the second layer.
  322. * @param outLayerId identifier of the first layer
  323. * @param inpLayerId identifier of the second layer
  324. * @param outNum number of the first layer output
  325. * @param inpNum number of the second layer input
  326. */
  327. void connect(int outLayerId, int outNum, int inpLayerId, int inpNum);
  328. /** @brief Sets outputs names of the network input pseudo layer.
  329. *
  330. * Each net always has special own the network input pseudo layer with id=0.
  331. * This layer stores the user blobs only and don't make any computations.
  332. * In fact, this layer provides the only way to pass user data into the network.
  333. * As any other layer, this layer can label its outputs and this function provides an easy way to do this.
  334. */
  335. CV_WRAP void setInputsNames(const std::vector<String> &inputBlobNames);
  336. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  337. * @param outputName name for layer which output is needed to get
  338. * @return blob for first output of specified layer.
  339. * @details By default runs forward pass for the whole network.
  340. */
  341. CV_WRAP Mat forward(const String& outputName = String());
  342. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  343. * @param outputBlobs contains all output blobs for specified layer.
  344. * @param outputName name for layer which output is needed to get
  345. * @details If @p outputName is empty, runs forward pass for the whole network.
  346. */
  347. CV_WRAP void forward(std::vector<Mat>& outputBlobs, const String& outputName = String());
  348. /** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames.
  349. * @param outputBlobs contains blobs for first outputs of specified layers.
  350. * @param outBlobNames names for layers which outputs are needed to get
  351. */
  352. CV_WRAP void forward(std::vector<Mat>& outputBlobs,
  353. const std::vector<String>& outBlobNames);
  354. /** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames.
  355. * @param outputBlobs contains all output blobs for each layer specified in @p outBlobNames.
  356. * @param outBlobNames names for layers which outputs are needed to get
  357. */
  358. CV_WRAP void forward(std::vector<std::vector<Mat> >& outputBlobs,
  359. const std::vector<String>& outBlobNames);
  360. //TODO:
  361. /** @brief Optimized forward.
  362. * @warning Not implemented yet.
  363. * @details Makes forward only those layers which weren't changed after previous forward().
  364. */
  365. void forwardOpt(LayerId toLayer);
  366. /** @overload */
  367. void forwardOpt(const std::vector<LayerId> &toLayers);
  368. /**
  369. * @brief Compile Halide layers.
  370. * @param[in] scheduler Path to YAML file with scheduling directives.
  371. * @see setPreferableBackend
  372. *
  373. * Schedule layers that support Halide backend. Then compile them for
  374. * specific target. For layers that not represented in scheduling file
  375. * or if no manual scheduling used at all, automatic scheduling will be applied.
  376. */
  377. CV_WRAP void setHalideScheduler(const String& scheduler);
  378. /**
  379. * @brief Ask network to use specific computation backend where it supported.
  380. * @param[in] backendId backend identifier.
  381. * @see Backend
  382. */
  383. CV_WRAP void setPreferableBackend(int backendId);
  384. /**
  385. * @brief Ask network to make computations on specific target device.
  386. * @param[in] targetId target identifier.
  387. * @see Target
  388. */
  389. CV_WRAP void setPreferableTarget(int targetId);
  390. /** @brief Sets the new value for the layer output blob
  391. * @param name descriptor of the updating layer output blob.
  392. * @param blob new blob.
  393. * @see connect(String, String) to know format of the descriptor.
  394. * @note If updating blob is not empty then @p blob must have the same shape,
  395. * because network reshaping is not implemented yet.
  396. */
  397. CV_WRAP void setInput(const Mat &blob, const String& name = "");
  398. /** @brief Sets the new value for the learned param of the layer.
  399. * @param layer name or id of the layer.
  400. * @param numParam index of the layer parameter in the Layer::blobs array.
  401. * @param blob the new value.
  402. * @see Layer::blobs
  403. * @note If shape of the new blob differs from the previous shape,
  404. * then the following forward pass may fail.
  405. */
  406. CV_WRAP void setParam(LayerId layer, int numParam, const Mat &blob);
  407. /** @brief Returns parameter blob of the layer.
  408. * @param layer name or id of the layer.
  409. * @param numParam index of the layer parameter in the Layer::blobs array.
  410. * @see Layer::blobs
  411. */
  412. CV_WRAP Mat getParam(LayerId layer, int numParam = 0);
  413. /** @brief Returns indexes of layers with unconnected outputs.
  414. */
  415. CV_WRAP std::vector<int> getUnconnectedOutLayers() const;
  416. /** @brief Returns input and output shapes for all layers in loaded model;
  417. * preliminary inferencing isn't necessary.
  418. * @param netInputShapes shapes for all input blobs in net input layer.
  419. * @param layersIds output parameter for layer IDs.
  420. * @param inLayersShapes output parameter for input layers shapes;
  421. * order is the same as in layersIds
  422. * @param outLayersShapes output parameter for output layers shapes;
  423. * order is the same as in layersIds
  424. */
  425. CV_WRAP void getLayersShapes(const std::vector<MatShape>& netInputShapes,
  426. CV_OUT std::vector<int>& layersIds,
  427. CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes,
  428. CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const;
  429. /** @overload */
  430. CV_WRAP void getLayersShapes(const MatShape& netInputShape,
  431. CV_OUT std::vector<int>& layersIds,
  432. CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes,
  433. CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const;
  434. /** @brief Returns input and output shapes for layer with specified
  435. * id in loaded model; preliminary inferencing isn't necessary.
  436. * @param netInputShape shape input blob in net input layer.
  437. * @param layerId id for layer.
  438. * @param inLayerShapes output parameter for input layers shapes;
  439. * order is the same as in layersIds
  440. * @param outLayerShapes output parameter for output layers shapes;
  441. * order is the same as in layersIds
  442. */
  443. void getLayerShapes(const MatShape& netInputShape,
  444. const int layerId,
  445. CV_OUT std::vector<MatShape>& inLayerShapes,
  446. CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP
  447. /** @overload */
  448. void getLayerShapes(const std::vector<MatShape>& netInputShapes,
  449. const int layerId,
  450. CV_OUT std::vector<MatShape>& inLayerShapes,
  451. CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP
  452. /** @brief Computes FLOP for whole loaded model with specified input shapes.
  453. * @param netInputShapes vector of shapes for all net inputs.
  454. * @returns computed FLOP.
  455. */
  456. CV_WRAP int64 getFLOPS(const std::vector<MatShape>& netInputShapes) const;
  457. /** @overload */
  458. CV_WRAP int64 getFLOPS(const MatShape& netInputShape) const;
  459. /** @overload */
  460. CV_WRAP int64 getFLOPS(const int layerId,
  461. const std::vector<MatShape>& netInputShapes) const;
  462. /** @overload */
  463. CV_WRAP int64 getFLOPS(const int layerId,
  464. const MatShape& netInputShape) const;
  465. /** @brief Returns list of types for layer used in model.
  466. * @param layersTypes output parameter for returning types.
  467. */
  468. CV_WRAP void getLayerTypes(CV_OUT std::vector<String>& layersTypes) const;
  469. /** @brief Returns count of layers of specified type.
  470. * @param layerType type.
  471. * @returns count of layers
  472. */
  473. CV_WRAP int getLayersCount(const String& layerType) const;
  474. /** @brief Computes bytes number which are requered to store
  475. * all weights and intermediate blobs for model.
  476. * @param netInputShapes vector of shapes for all net inputs.
  477. * @param weights output parameter to store resulting bytes for weights.
  478. * @param blobs output parameter to store resulting bytes for intermediate blobs.
  479. */
  480. void getMemoryConsumption(const std::vector<MatShape>& netInputShapes,
  481. CV_OUT size_t& weights, CV_OUT size_t& blobs) const; // FIXIT: CV_WRAP
  482. /** @overload */
  483. CV_WRAP void getMemoryConsumption(const MatShape& netInputShape,
  484. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  485. /** @overload */
  486. CV_WRAP void getMemoryConsumption(const int layerId,
  487. const std::vector<MatShape>& netInputShapes,
  488. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  489. /** @overload */
  490. CV_WRAP void getMemoryConsumption(const int layerId,
  491. const MatShape& netInputShape,
  492. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  493. /** @brief Computes bytes number which are requered to store
  494. * all weights and intermediate blobs for each layer.
  495. * @param netInputShapes vector of shapes for all net inputs.
  496. * @param layerIds output vector to save layer IDs.
  497. * @param weights output parameter to store resulting bytes for weights.
  498. * @param blobs output parameter to store resulting bytes for intermediate blobs.
  499. */
  500. void getMemoryConsumption(const std::vector<MatShape>& netInputShapes,
  501. CV_OUT std::vector<int>& layerIds,
  502. CV_OUT std::vector<size_t>& weights,
  503. CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP
  504. /** @overload */
  505. void getMemoryConsumption(const MatShape& netInputShape,
  506. CV_OUT std::vector<int>& layerIds,
  507. CV_OUT std::vector<size_t>& weights,
  508. CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP
  509. /** @brief Enables or disables layer fusion in the network.
  510. * @param fusion true to enable the fusion, false to disable. The fusion is enabled by default.
  511. */
  512. CV_WRAP void enableFusion(bool fusion);
  513. /** @brief Returns overall time for inference and timings (in ticks) for layers.
  514. * Indexes in returned vector correspond to layers ids. Some layers can be fused with others,
  515. * in this case zero ticks count will be return for that skipped layers.
  516. * @param timings vector for tick timings for all layers.
  517. * @return overall ticks for model inference.
  518. */
  519. CV_WRAP int64 getPerfProfile(CV_OUT std::vector<double>& timings);
  520. private:
  521. struct Impl;
  522. Ptr<Impl> impl;
  523. };
  524. /**
  525. * @deprecated Deprecated as external interface. Will be for internal needs only.
  526. * @brief Small interface class for loading trained serialized models of different dnn-frameworks. */
  527. class CV_EXPORTS_W Importer : public Algorithm
  528. {
  529. public:
  530. /** @brief Adds loaded layers into the @p net and sets connections between them. */
  531. CV_DEPRECATED CV_WRAP virtual void populateNet(Net net) = 0;
  532. virtual ~Importer();
  533. };
  534. /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
  535. * @param cfgFile path to the .cfg file with text description of the network architecture.
  536. * @param darknetModel path to the .weights file with learned network.
  537. * @returns Network object that ready to do forward, throw an exception in failure cases.
  538. * @details This is shortcut consisting from DarknetImporter and Net::populateNet calls.
  539. */
  540. CV_EXPORTS_W Net readNetFromDarknet(const String &cfgFile, const String &darknetModel = String());
  541. /**
  542. * @deprecated Use @ref readNetFromCaffe instead.
  543. * @brief Creates the importer of <a href="http://caffe.berkeleyvision.org">Caffe</a> framework network.
  544. * @param prototxt path to the .prototxt file with text description of the network architecture.
  545. * @param caffeModel path to the .caffemodel file with learned network.
  546. * @returns Pointer to the created importer, NULL in failure cases.
  547. */
  548. CV_DEPRECATED CV_EXPORTS_W Ptr<Importer> createCaffeImporter(const String &prototxt, const String &caffeModel = String());
  549. /** @brief Reads a network model stored in Caffe model files.
  550. * @details This is shortcut consisting from createCaffeImporter and Net::populateNet calls.
  551. */
  552. CV_EXPORTS_W Net readNetFromCaffe(const String &prototxt, const String &caffeModel = String());
  553. /** @brief Reads a network model stored in Tensorflow model file.
  554. * @details This is shortcut consisting from createTensorflowImporter and Net::populateNet calls.
  555. */
  556. CV_EXPORTS_W Net readNetFromTensorflow(const String &model, const String &config = String());
  557. /** @brief Reads a network model stored in Torch model file.
  558. * @details This is shortcut consisting from createTorchImporter and Net::populateNet calls.
  559. */
  560. CV_EXPORTS_W Net readNetFromTorch(const String &model, bool isBinary = true);
  561. /**
  562. * @deprecated Use @ref readNetFromTensorflow instead.
  563. * @brief Creates the importer of <a href="http://www.tensorflow.org">TensorFlow</a> framework network.
  564. * @param model path to the .pb file with binary protobuf description of the network architecture.
  565. * @returns Pointer to the created importer, NULL in failure cases.
  566. */
  567. CV_DEPRECATED CV_EXPORTS_W Ptr<Importer> createTensorflowImporter(const String &model);
  568. /**
  569. * @deprecated Use @ref readNetFromTorch instead.
  570. * @brief Creates the importer of <a href="http://torch.ch">Torch7</a> framework network.
  571. * @param filename path to the file, dumped from Torch by using torch.save() function.
  572. * @param isBinary specifies whether the network was serialized in ascii mode or binary.
  573. * @returns Pointer to the created importer, NULL in failure cases.
  574. *
  575. * @warning Torch7 importer is experimental now, you need explicitly set CMake `opencv_dnn_BUILD_TORCH_IMPORTER` flag to compile its.
  576. *
  577. * @note Ascii mode of Torch serializer is more preferable, because binary mode extensively use `long` type of C language,
  578. * which has various bit-length on different systems.
  579. *
  580. * The loading file must contain serialized <a href="https://github.com/torch/nn/blob/master/doc/module.md">nn.Module</a> object
  581. * with importing network. Try to eliminate a custom objects from serialazing data to avoid importing errors.
  582. *
  583. * List of supported layers (i.e. object instances derived from Torch nn.Module class):
  584. * - nn.Sequential
  585. * - nn.Parallel
  586. * - nn.Concat
  587. * - nn.Linear
  588. * - nn.SpatialConvolution
  589. * - nn.SpatialMaxPooling, nn.SpatialAveragePooling
  590. * - nn.ReLU, nn.TanH, nn.Sigmoid
  591. * - nn.Reshape
  592. * - nn.SoftMax, nn.LogSoftMax
  593. *
  594. * Also some equivalents of these classes from cunn, cudnn, and fbcunn may be successfully imported.
  595. */
  596. CV_DEPRECATED CV_EXPORTS_W Ptr<Importer> createTorchImporter(const String &filename, bool isBinary = true);
  597. /** @brief Loads blob which was serialized as torch.Tensor object of Torch7 framework.
  598. * @warning This function has the same limitations as createTorchImporter().
  599. */
  600. CV_EXPORTS_W Mat readTorchBlob(const String &filename, bool isBinary = true);
  601. /** @brief Creates 4-dimensional blob from image. Optionally resizes and crops @p image from center,
  602. * subtract @p mean values, scales values by @p scalefactor, swap Blue and Red channels.
  603. * @param image input image (with 1- or 3-channels).
  604. * @param size spatial size for output image
  605. * @param mean scalar with mean values which are subtracted from channels. Values are intended
  606. * to be in (mean-R, mean-G, mean-B) order if @p image has BGR ordering and @p swapRB is true.
  607. * @param scalefactor multiplier for @p image values.
  608. * @param swapRB flag which indicates that swap first and last channels
  609. * in 3-channel image is necessary.
  610. * @param crop flag which indicates whether image will be cropped after resize or not
  611. * @details if @p crop is true, input image is resized so one side after resize is equal to corresponing
  612. * dimension in @p size and another one is equal or larger. Then, crop from the center is performed.
  613. * If @p crop is false, direct resize without cropping and preserving aspect ratio is performed.
  614. * @returns 4-dimansional Mat with NCHW dimensions order.
  615. */
  616. CV_EXPORTS_W Mat blobFromImage(const Mat& image, double scalefactor=1.0, const Size& size = Size(),
  617. const Scalar& mean = Scalar(), bool swapRB=true, bool crop=true);
  618. /** @brief Creates 4-dimensional blob from series of images. Optionally resizes and
  619. * crops @p images from center, subtract @p mean values, scales values by @p scalefactor,
  620. * swap Blue and Red channels.
  621. * @param images input images (all with 1- or 3-channels).
  622. * @param size spatial size for output image
  623. * @param mean scalar with mean values which are subtracted from channels. Values are intended
  624. * to be in (mean-R, mean-G, mean-B) order if @p image has BGR ordering and @p swapRB is true.
  625. * @param scalefactor multiplier for @p images values.
  626. * @param swapRB flag which indicates that swap first and last channels
  627. * in 3-channel image is necessary.
  628. * @param crop flag which indicates whether image will be cropped after resize or not
  629. * @details if @p crop is true, input image is resized so one side after resize is equal to corresponing
  630. * dimension in @p size and another one is equal or larger. Then, crop from the center is performed.
  631. * If @p crop is false, direct resize without cropping and preserving aspect ratio is performed.
  632. * @returns 4-dimansional Mat with NCHW dimensions order.
  633. */
  634. CV_EXPORTS_W Mat blobFromImages(const std::vector<Mat>& images, double scalefactor=1.0,
  635. Size size = Size(), const Scalar& mean = Scalar(), bool swapRB=true, bool crop=true);
  636. /** @brief Convert all weights of Caffe network to half precision floating point.
  637. * @param src Path to origin model from Caffe framework contains single
  638. * precision floating point weights (usually has `.caffemodel` extension).
  639. * @param dst Path to destination model with updated weights.
  640. *
  641. * @note Shrinked model has no origin float32 weights so it can't be used
  642. * in origin Caffe framework anymore. However the structure of data
  643. * is taken from NVidia's Caffe fork: https://github.com/NVIDIA/caffe.
  644. * So the resulting model may be used there.
  645. */
  646. CV_EXPORTS_W void shrinkCaffeModel(const String& src, const String& dst);
  647. //! @}
  648. CV__DNN_EXPERIMENTAL_NS_END
  649. }
  650. }
  651. #include <opencv2/dnn/layer.hpp>
  652. #include <opencv2/dnn/dnn.inl.hpp>
  653. #endif /* OPENCV_DNN_DNN_HPP */