TensorRT 8.4.0
NvInfer.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: LicenseRef-NvidiaProprietary
4 *
5 * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
6 * property and proprietary rights in and to this material, related
7 * documentation and any modifications thereto. Any use, reproduction,
8 * disclosure or distribution of this material and related documentation
9 * without an express license agreement from NVIDIA CORPORATION or
10 * its affiliates is strictly prohibited.
11 */
12
13#ifndef NV_INFER_H
14#define NV_INFER_H
15
16#include "NvInferLegacyDims.h"
17#include "NvInferRuntime.h"
18
27//
30
36
42namespace nvinfer1
43{
44
52enum class LayerType : int32_t
53{
54 kCONVOLUTION = 0,
56 kACTIVATION = 2,
57 kPOOLING = 3,
58 kLRN = 4,
59 kSCALE = 5,
60 kSOFTMAX = 6,
61 kDECONVOLUTION = 7,
62 kCONCATENATION = 8,
63 kELEMENTWISE = 9,
64 kPLUGIN = 10,
65 kUNARY = 11,
66 kPADDING = 12,
67 kSHUFFLE = 13,
68 kREDUCE = 14,
69 kTOPK = 15,
70 kGATHER = 16,
71 kMATRIX_MULTIPLY = 17,
72 kRAGGED_SOFTMAX = 18,
73 kCONSTANT = 19,
74 kRNN_V2 = 20,
75 kIDENTITY = 21,
76 kPLUGIN_V2 = 22,
77 kSLICE = 23,
78 kSHAPE = 24,
79 kPARAMETRIC_RELU = 25,
80 kRESIZE = 26,
81 kTRIP_LIMIT = 27,
82 kRECURRENCE = 28,
83 kITERATOR = 29,
84 kLOOP_OUTPUT = 30,
85 kSELECT = 31,
86 kFILL = 32,
87 kQUANTIZE = 33,
88 kDEQUANTIZE = 34,
89 kCONDITION = 35,
92 kSCATTER = 38,
93 kEINSUM = 39,
94 kASSERTION = 40,
95};
96
102template <>
103constexpr inline int32_t EnumMax<LayerType>() noexcept
104{
105 return 41;
106}
107
114using TensorFormats = uint32_t;
115
121enum class ActivationType : int32_t
122{
123 kRELU = 0,
124 kSIGMOID = 1,
125 kTANH = 2,
126 kLEAKY_RELU = 3,
127 kELU = 4,
128 kSELU = 5,
129 kSOFTSIGN = 6,
130 kSOFTPLUS = 7,
131 kCLIP = 8,
132 kHARD_SIGMOID = 9,
133 kSCALED_TANH = 10,
135};
136
137namespace impl
138{
144template <>
146{
147 static constexpr int32_t kVALUE = 12;
148};
149} // namespace impl
150
166class ITensor : public INoCopy
167{
168public:
181 void setName(const char* name) noexcept
182 {
183 mImpl->setName(name);
184 }
185
193 const char* getName() const noexcept
194 {
195 return mImpl->getName();
196 }
197
212 void setDimensions(Dims dimensions) noexcept
213 {
214 mImpl->setDimensions(dimensions);
215 }
216
225 Dims getDimensions() const noexcept
226 {
227 return mImpl->getDimensions();
228 }
229
240 void setType(DataType type) noexcept
241 {
242 mImpl->setType(type);
243 }
244
252 DataType getType() const noexcept
253 {
254 return mImpl->getType();
255 }
256
267 bool setDynamicRange(float min, float max) noexcept
268 {
269 return mImpl->setDynamicRange(min, max);
270 }
271
275 bool isNetworkInput() const noexcept
276 {
277 return mImpl->isNetworkInput();
278 }
279
283 bool isNetworkOutput() const noexcept
284 {
285 return mImpl->isNetworkOutput();
286 }
287
305 void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
306 {
307 mImpl->setBroadcastAcrossBatch(broadcastAcrossBatch);
308 }
309
321 bool getBroadcastAcrossBatch() const noexcept
322 {
323 return mImpl->getBroadcastAcrossBatch();
324 }
325
332 {
333 return mImpl->getLocation();
334 }
335
346 void setLocation(TensorLocation location) noexcept
347 {
348 mImpl->setLocation(location);
349 }
350
356 bool dynamicRangeIsSet() const noexcept
357 {
358 return mImpl->dynamicRangeIsSet();
359 }
360
364 void resetDynamicRange() noexcept
365 {
366 mImpl->resetDynamicRange();
367 }
368
374 float getDynamicRangeMin() const noexcept
375 {
376 return mImpl->getDynamicRangeMin();
377 }
378
384 float getDynamicRangeMax() const noexcept
385 {
386 return mImpl->getDynamicRangeMax();
387 }
388
403 void setAllowedFormats(TensorFormats formats) noexcept
404 {
405 mImpl->setAllowedFormats(formats);
406 }
407
417 {
418 return mImpl->getAllowedFormats();
419 }
420
451 bool isShapeTensor() const noexcept
452 {
453 return mImpl->isShapeTensor();
454 }
455
474 bool isExecutionTensor() const noexcept
475 {
476 return mImpl->isExecutionTensor();
477 }
478
479protected:
480 apiv::VTensor* mImpl;
481 virtual ~ITensor() noexcept = default;
482};
483
491class ILayer : public INoCopy
492{
493public:
499 LayerType getType() const noexcept
500 {
501 return mLayer->getType();
502 }
503
511 void setName(const char* name) noexcept
512 {
513 mLayer->setName(name);
514 }
515
519
522 const char* getName() const noexcept
523 {
524 return mLayer->getName();
525 }
526
530 int32_t getNbInputs() const noexcept
531 {
532 return mLayer->getNbInputs();
533 }
534
543 ITensor* getInput(int32_t index) const noexcept
544 {
545 return mLayer->getInput(index);
546 }
547
551 int32_t getNbOutputs() const noexcept
552 {
553 return mLayer->getNbOutputs();
554 }
555
562 ITensor* getOutput(int32_t index) const noexcept
563 {
564 return mLayer->getOutput(index);
565 }
566
579 void setInput(int32_t index, ITensor& tensor) noexcept
580 {
581 return mLayer->setInput(index, tensor);
582 }
583
607 void setPrecision(DataType dataType) noexcept
608 {
609 mLayer->setPrecision(dataType);
610 }
611
619 DataType getPrecision() const noexcept
620 {
621 return mLayer->getPrecision();
622 }
623
631 bool precisionIsSet() const noexcept
632 {
633 return mLayer->precisionIsSet();
634 }
635
641 void resetPrecision() noexcept
642 {
643 mLayer->resetPrecision();
644 }
645
679 void setOutputType(int32_t index, DataType dataType) noexcept
680 {
681 mLayer->setOutputType(index, dataType);
682 }
683
693 DataType getOutputType(int32_t index) const noexcept
694 {
695 return mLayer->getOutputType(index);
696 }
697
706 bool outputTypeIsSet(int32_t index) const noexcept
707 {
708 return mLayer->outputTypeIsSet(index);
709 }
710
718 void resetOutputType(int32_t index) noexcept
719 {
720 return mLayer->resetOutputType(index);
721 }
722
723protected:
724 virtual ~ILayer() noexcept = default;
725 apiv::VLayer* mLayer;
726};
727
950enum class PaddingMode : int32_t
951{
954 kSAME_UPPER = 2,
955 kSAME_LOWER = 3,
957 kCAFFE_ROUND_UP = 5
958};
959
960namespace impl
961{
967template <>
969{
970 static constexpr int32_t kVALUE = 6;
971};
972} // namespace impl
973
987{
988public:
998 TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
999 {
1000 mImpl->setKernelSize(kernelSize);
1001 }
1002
1011 {
1012 return mImpl->getKernelSize();
1013 }
1014
1022 void setNbOutputMaps(int32_t nbOutputMaps) noexcept
1023 {
1024 mImpl->setNbOutputMaps(nbOutputMaps);
1025 }
1026
1032 int32_t getNbOutputMaps() const noexcept
1033 {
1034 return mImpl->getNbOutputMaps();
1035 }
1036
1048 TRT_DEPRECATED void setStride(DimsHW stride) noexcept
1049 {
1050 mImpl->setStride(stride);
1051 }
1052
1059 {
1060 return mImpl->getStride();
1061 }
1062
1078 TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
1079 {
1080 return mImpl->setPadding(padding);
1081 }
1082
1091 {
1092 return mImpl->getPadding();
1093 }
1094
1110 void setNbGroups(int32_t nbGroups) noexcept
1111 {
1112 mImpl->setNbGroups(nbGroups);
1113 }
1114
1120 int32_t getNbGroups() const noexcept
1121 {
1122 return mImpl->getNbGroups();
1123 }
1124
1134 void setKernelWeights(Weights weights) noexcept
1135 {
1136 mImpl->setKernelWeights(weights);
1137 }
1138
1144 Weights getKernelWeights() const noexcept
1145 {
1146 return mImpl->getKernelWeights();
1147 }
1148
1159 void setBiasWeights(Weights weights) noexcept
1160 {
1161 mImpl->setBiasWeights(weights);
1162 }
1163
1169 Weights getBiasWeights() const noexcept
1170 {
1171 return mImpl->getBiasWeights();
1172 }
1173
1185 TRT_DEPRECATED void setDilation(DimsHW dilation) noexcept
1186 {
1187 return mImpl->setDilation(dilation);
1188 }
1189
1198 {
1199 return mImpl->getDilation();
1200 }
1201
1214 void setPrePadding(Dims padding) noexcept
1215 {
1216 mImpl->setPrePadding(padding);
1217 }
1218
1224 Dims getPrePadding() const noexcept
1225 {
1226 return mImpl->getPrePadding();
1227 }
1228
1241 void setPostPadding(Dims padding) noexcept
1242 {
1243 mImpl->setPostPadding(padding);
1244 }
1245
1251 Dims getPostPadding() const noexcept
1252 {
1253 return mImpl->getPostPadding();
1254 }
1255
1265 void setPaddingMode(PaddingMode paddingMode) noexcept
1266 {
1267 mImpl->setPaddingMode(paddingMode);
1268 }
1269
1278 {
1279 return mImpl->getPaddingMode();
1280 }
1281
1290 void setKernelSizeNd(Dims kernelSize) noexcept
1291 {
1292 mImpl->setKernelSizeNd(kernelSize);
1293 }
1294
1300 Dims getKernelSizeNd() const noexcept
1301 {
1302 return mImpl->getKernelSizeNd();
1303 }
1304
1315 void setStrideNd(Dims stride) noexcept
1316 {
1317 mImpl->setStrideNd(stride);
1318 }
1319
1325 Dims getStrideNd() const noexcept
1326 {
1327 return mImpl->getStrideNd();
1328 }
1329
1343 void setPaddingNd(Dims padding) noexcept
1344 {
1345 mImpl->setPaddingNd(padding);
1346 }
1347
1355 Dims getPaddingNd() const noexcept
1356 {
1357 return mImpl->getPaddingNd();
1358 }
1359
1369 void setDilationNd(Dims dilation) noexcept
1370 {
1371 mImpl->setDilationNd(dilation);
1372 }
1373
1379 Dims getDilationNd() const noexcept
1380 {
1381 return mImpl->getDilationNd();
1382 }
1383
1405 using ILayer::setInput;
1406
1407protected:
1408 virtual ~IConvolutionLayer() noexcept = default;
1409 apiv::VConvolutionLayer* mImpl;
1410};
1411
1444{
1445public:
1453 void setNbOutputChannels(int32_t nbOutputs) noexcept
1454 {
1455 mImpl->setNbOutputChannels(nbOutputs);
1456 }
1457
1463 int32_t getNbOutputChannels() const noexcept
1464 {
1465 return mImpl->getNbOutputChannels();
1466 }
1467
1473 void setKernelWeights(Weights weights) noexcept
1474 {
1475 mImpl->setKernelWeights(weights);
1476 }
1477
1483 Weights getKernelWeights() const noexcept
1484 {
1485 return mImpl->getKernelWeights();
1486 }
1487
1495 void setBiasWeights(Weights weights) noexcept
1496 {
1497 mImpl->setBiasWeights(weights);
1498 }
1499
1505 Weights getBiasWeights() const noexcept
1506 {
1507 return mImpl->getBiasWeights();
1508 }
1509
1531 using ILayer::setInput;
1532
1533protected:
1534 virtual ~IFullyConnectedLayer() noexcept = default;
1535 apiv::VFullyConnectedLayer* mImpl;
1536};
1537
1552{
1553public:
1562 {
1563 mImpl->setActivationType(type);
1564 }
1565
1572 {
1573 return mImpl->getActivationType();
1574 }
1575
1586 void setAlpha(float alpha) noexcept
1587 {
1588 mImpl->setAlpha(alpha);
1589 }
1590
1600 void setBeta(float beta) noexcept
1601 {
1602 mImpl->setBeta(beta);
1603 }
1604
1609 float getAlpha() const noexcept
1610 {
1611 return mImpl->getAlpha();
1612 }
1613
1618 float getBeta() const noexcept
1619 {
1620 return mImpl->getBeta();
1621 }
1622
1623protected:
1624 virtual ~IActivationLayer() noexcept = default;
1625 apiv::VActivationLayer* mImpl;
1626};
1627
1633enum class PoolingType : int32_t
1634{
1635 kMAX = 0, // Maximum over elements
1636 kAVERAGE = 1, // Average over elements. If the tensor is padded, the count includes the padding
1637 kMAX_AVERAGE_BLEND = 2 // Blending between max and average pooling: (1-blendFactor)*maxPool + blendFactor*avgPool
1638};
1639
1640namespace impl
1641{
1647template <>
1649{
1650 static constexpr int32_t kVALUE = 3;
1651};
1652} // namespace impl
1653
1665class IPoolingLayer : public ILayer
1666{
1667public:
1675 void setPoolingType(PoolingType type) noexcept
1676 {
1677 mImpl->setPoolingType(type);
1678 }
1679
1686 {
1687 return mImpl->getPoolingType();
1688 }
1689
1699 TRT_DEPRECATED void setWindowSize(DimsHW windowSize) noexcept
1700 {
1701 mImpl->setWindowSize(windowSize);
1702 }
1703
1712 {
1713 return mImpl->getWindowSize();
1714 }
1715
1727 TRT_DEPRECATED void setStride(DimsHW stride) noexcept
1728 {
1729 mImpl->setStride(stride);
1730 }
1731
1740 {
1741 return mImpl->getStride();
1742 }
1743
1755 TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
1756 {
1757 mImpl->setPadding(padding);
1758 }
1759
1770 {
1771 return mImpl->getPadding();
1772 }
1773
1784 void setBlendFactor(float blendFactor) noexcept
1785 {
1786 mImpl->setBlendFactor(blendFactor);
1787 }
1788
1797 float getBlendFactor() const noexcept
1798 {
1799 return mImpl->getBlendFactor();
1800 }
1801
1814 void setAverageCountExcludesPadding(bool exclusive) noexcept
1815 {
1816 mImpl->setAverageCountExcludesPadding(exclusive);
1817 }
1818
1826 {
1827 return mImpl->getAverageCountExcludesPadding();
1828 }
1829
1843 void setPrePadding(Dims padding) noexcept
1844 {
1845 mImpl->setPrePadding(padding);
1846 }
1847
1853 Dims getPrePadding() const noexcept
1854 {
1855 return mImpl->getPrePadding();
1856 }
1857
1871 void setPostPadding(Dims padding) noexcept
1872 {
1873 mImpl->setPostPadding(padding);
1874 }
1875
1881 Dims getPostPadding() const noexcept
1882 {
1883 return mImpl->getPostPadding();
1884 }
1885
1894 void setPaddingMode(PaddingMode paddingMode) noexcept
1895 {
1896 mImpl->setPaddingMode(paddingMode);
1897 }
1898
1906 {
1907 return mImpl->getPaddingMode();
1908 }
1909
1918 void setWindowSizeNd(Dims windowSize) noexcept
1919 {
1920 mImpl->setWindowSizeNd(windowSize);
1921 }
1922
1928 Dims getWindowSizeNd() const noexcept
1929 {
1930 return mImpl->getWindowSizeNd();
1931 }
1932
1943 void setStrideNd(Dims stride) noexcept
1944 {
1945 mImpl->setStrideNd(stride);
1946 }
1947
1953 Dims getStrideNd() const noexcept
1954 {
1955 return mImpl->getStrideNd();
1956 }
1957
1972 void setPaddingNd(Dims padding) noexcept
1973 {
1974 mImpl->setPaddingNd(padding);
1975 }
1976
1984 Dims getPaddingNd() const noexcept
1985 {
1986 return mImpl->getPaddingNd();
1987 }
1988
1989protected:
1990 virtual ~IPoolingLayer() noexcept = default;
1991 apiv::VPoolingLayer* mImpl;
1992};
1993
2003class ILRNLayer : public ILayer
2004{
2005public:
2015 void setWindowSize(int32_t windowSize) noexcept
2016 {
2017 mImpl->setWindowSize(windowSize);
2018 }
2019
2025 int32_t getWindowSize() const noexcept
2026 {
2027 return mImpl->getWindowSize();
2028 }
2029
2036 void setAlpha(float alpha) noexcept
2037 {
2038 mImpl->setAlpha(alpha);
2039 }
2040
2046 float getAlpha() const noexcept
2047 {
2048 return mImpl->getAlpha();
2049 }
2050
2057 void setBeta(float beta) noexcept
2058 {
2059 mImpl->setBeta(beta);
2060 }
2061
2067 float getBeta() const noexcept
2068 {
2069 return mImpl->getBeta();
2070 }
2071
2078 void setK(float k) noexcept
2079 {
2080 mImpl->setK(k);
2081 }
2082
2088 float getK() const noexcept
2089 {
2090 return mImpl->getK();
2091 }
2092
2093protected:
2094 virtual ~ILRNLayer() noexcept = default;
2095 apiv::VLRNLayer* mImpl;
2096};
2097
2103enum class ScaleMode : int32_t
2104{
2105 kUNIFORM = 0,
2106 kCHANNEL = 1,
2107 kELEMENTWISE = 2
2108};
2109
2115template <>
2116constexpr inline int32_t EnumMax<ScaleMode>() noexcept
2117{
2118 return 3;
2119}
2120
2147class IScaleLayer : public ILayer
2148{
2149public:
2155 void setMode(ScaleMode mode) noexcept
2156 {
2157 mImpl->setMode(mode);
2158 }
2159
2165 ScaleMode getMode() const noexcept
2166 {
2167 return mImpl->getMode();
2168 }
2169
2175 void setShift(Weights shift) noexcept
2176 {
2177 mImpl->setShift(shift);
2178 }
2179
2185 Weights getShift() const noexcept
2186 {
2187 return mImpl->getShift();
2188 }
2189
2195 void setScale(Weights scale) noexcept
2196 {
2197 mImpl->setScale(scale);
2198 }
2199
2205 Weights getScale() const noexcept
2206 {
2207 return mImpl->getScale();
2208 }
2209
2215 void setPower(Weights power) noexcept
2216 {
2217 mImpl->setPower(power);
2218 }
2219
2225 Weights getPower() const noexcept
2226 {
2227 return mImpl->getPower();
2228 }
2229
2240 int32_t getChannelAxis() const noexcept
2241 {
2242 return mImpl->getChannelAxis();
2243 }
2244
2261 void setChannelAxis(int32_t channelAxis) noexcept
2262 {
2263 mImpl->setChannelAxis(channelAxis);
2264 }
2265
2266protected:
2267 virtual ~IScaleLayer() noexcept = default;
2268 apiv::VScaleLayer* mImpl;
2269};
2270
2287class ISoftMaxLayer : public ILayer
2288{
2289public:
2322 void setAxes(uint32_t axes) noexcept
2323 {
2324 mImpl->setAxes(axes);
2325 }
2326
2332 uint32_t getAxes() const noexcept
2333 {
2334 return mImpl->getAxes();
2335 }
2336
2337protected:
2338 virtual ~ISoftMaxLayer() noexcept = default;
2339 apiv::VSoftMaxLayer* mImpl;
2340};
2341
2355{
2356public:
2367 void setAxis(int32_t axis) noexcept
2368 {
2369 mImpl->setAxis(axis);
2370 }
2371
2377 int32_t getAxis() const noexcept
2378 {
2379 return mImpl->getAxis();
2380 }
2381
2382protected:
2383 virtual ~IConcatenationLayer() noexcept = default;
2384 apiv::VConcatenationLayer* mImpl;
2385};
2386
2395{
2396public:
2408 TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
2409 {
2410 mImpl->setKernelSize(kernelSize);
2411 }
2412
2421 {
2422 return mImpl->getKernelSize();
2423 }
2424
2432 void setNbOutputMaps(int32_t nbOutputMaps) noexcept
2433 {
2434 mImpl->setNbOutputMaps(nbOutputMaps);
2435 }
2436
2442 int32_t getNbOutputMaps() const noexcept
2443 {
2444 return mImpl->getNbOutputMaps();
2445 }
2446
2458 TRT_DEPRECATED void setStride(DimsHW stride) noexcept
2459 {
2460 mImpl->setStride(stride);
2461 }
2462
2471 {
2472 return mImpl->getStride();
2473 }
2474
2490 TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
2491 {
2492 mImpl->setPadding(padding);
2493 }
2494
2505 {
2506 return mImpl->getPadding();
2507 }
2508
2524 void setNbGroups(int32_t nbGroups) noexcept
2525 {
2526 mImpl->setNbGroups(nbGroups);
2527 }
2528
2534 int32_t getNbGroups() const noexcept
2535 {
2536 return mImpl->getNbGroups();
2537 }
2538
2548 void setKernelWeights(Weights weights) noexcept
2549 {
2550 mImpl->setKernelWeights(weights);
2551 }
2552
2558 Weights getKernelWeights() const noexcept
2559 {
2560 return mImpl->getKernelWeights();
2561 }
2562
2573 void setBiasWeights(Weights weights) noexcept
2574 {
2575 mImpl->setBiasWeights(weights);
2576 }
2577
2583 Weights getBiasWeights() const noexcept
2584 {
2585 return mImpl->getBiasWeights();
2586 }
2587
2601 void setPrePadding(Dims padding) noexcept
2602 {
2603 mImpl->setPrePadding(padding);
2604 }
2605
2611 Dims getPrePadding() const noexcept
2612 {
2613 return mImpl->getPrePadding();
2614 }
2615
2629 void setPostPadding(Dims padding) noexcept
2630 {
2631 mImpl->setPostPadding(padding);
2632 }
2633
2639 Dims getPostPadding() const noexcept
2640 {
2641 return mImpl->getPostPadding();
2642 }
2643
2653 void setPaddingMode(PaddingMode paddingMode) noexcept
2654 {
2655 mImpl->setPaddingMode(paddingMode);
2656 }
2657
2666 {
2667 return mImpl->getPaddingMode();
2668 }
2669
2680 void setKernelSizeNd(Dims kernelSize) noexcept
2681 {
2682 mImpl->setKernelSizeNd(kernelSize);
2683 }
2684
2690 Dims getKernelSizeNd() const noexcept
2691 {
2692 return mImpl->getKernelSizeNd();
2693 }
2694
2707 void setStrideNd(Dims stride) noexcept
2708 {
2709 mImpl->setStrideNd(stride);
2710 }
2711
2717 Dims getStrideNd() const noexcept
2718 {
2719 return mImpl->getStrideNd();
2720 }
2721
2735 void setPaddingNd(Dims padding) noexcept
2736 {
2737 mImpl->setPaddingNd(padding);
2738 }
2739
2747 Dims getPaddingNd() const noexcept
2748 {
2749 return mImpl->getPaddingNd();
2750 }
2751
2773 using ILayer::setInput;
2774
2781 void setDilationNd(Dims dilation) noexcept
2782 {
2783 mImpl->setDilationNd(dilation);
2784 }
2785
2791 Dims getDilationNd() const noexcept
2792 {
2793 return mImpl->getDilationNd();
2794 }
2795
2796protected:
2797 virtual ~IDeconvolutionLayer() noexcept = default;
2798 apiv::VDeconvolutionLayer* mImpl;
2799};
2800
2814enum class ElementWiseOperation : int32_t
2815{
2816 kSUM = 0,
2817 kPROD = 1,
2818 kMAX = 2,
2819 kMIN = 3,
2820 kSUB = 4,
2821 kDIV = 5,
2822 kPOW = 6,
2823 kFLOOR_DIV = 7,
2824 kAND = 8,
2825 kOR = 9,
2826 kXOR = 10,
2827 kEQUAL = 11,
2828 kGREATER = 12,
2829 kLESS = 13
2830};
2831
2832namespace impl
2833{
2839template <>
2841{
2842 static constexpr int32_t kVALUE = 14;
2843};
2844} // namespace impl
2845
2866{
2867public:
2878 {
2879 return mImpl->setOperation(op);
2880 }
2881
2890 {
2891 return mImpl->getOperation();
2892 }
2893
2894protected:
2895 apiv::VElementWiseLayer* mImpl;
2896 virtual ~IElementWiseLayer() noexcept = default;
2897};
2898
2904enum class GatherMode : int32_t
2905{
2906 kDEFAULT = 0,
2907 kELEMENT = 1,
2908 kND = 2
2909};
2910
2916template <>
2917constexpr inline int32_t EnumMax<GatherMode>() noexcept
2918{
2919 return 3;
2920}
2921
3000class IGatherLayer : public ILayer
3001{
3002public:
3012 void setGatherAxis(int32_t axis) noexcept
3013 {
3014 mImpl->setGatherAxis(axis);
3015 }
3016
3023 int32_t getGatherAxis() const noexcept
3024 {
3025 return mImpl->getGatherAxis();
3026 }
3027
3044 void setNbElementWiseDims(int32_t elementWiseDims) noexcept
3045 {
3046 mImpl->setNbElementWiseDims(elementWiseDims);
3047 }
3048
3054 int32_t getNbElementWiseDims() const noexcept
3055 {
3056 return mImpl->getNbElementWiseDims();
3057 }
3058
3064 void setMode(GatherMode mode) noexcept
3065 {
3066 mImpl->setMode(mode);
3067 }
3068
3074 GatherMode getMode() const noexcept
3075 {
3076 return mImpl->getMode();
3077 }
3078
3079protected:
3080 apiv::VGatherLayer* mImpl;
3081 virtual ~IGatherLayer() noexcept = default;
3082};
3083
3163enum class RNNOperation : int32_t
3164{
3165 kRELU = 0,
3166 kTANH = 1,
3167 kLSTM = 2,
3168 kGRU = 3
3169};
3170
3176template <>
3177constexpr inline int32_t EnumMax<RNNOperation>() noexcept
3178{
3179 return 4;
3180}
3181
3189enum class RNNDirection : int32_t
3190{
3191 kUNIDIRECTION = 0,
3192 kBIDIRECTION = 1
3193};
3194
3200template <>
3201constexpr inline int32_t EnumMax<RNNDirection>() noexcept
3202{
3203 return 2;
3204}
3205
3221enum class RNNInputMode : int32_t
3222{
3223 kLINEAR = 0,
3224 kSKIP = 1
3225};
3226
3232template <>
3233constexpr inline int32_t EnumMax<RNNInputMode>() noexcept
3234{
3235 return 2;
3236}
3237
3245enum class RNNGateType : int32_t
3246{
3247 kINPUT = 0,
3248 kOUTPUT = 1,
3249 kFORGET = 2,
3250 kUPDATE = 3,
3251 kRESET = 4,
3252 kCELL = 5,
3253 kHIDDEN = 6
3254};
3255
3261template <>
3262constexpr inline int32_t EnumMax<RNNGateType>() noexcept
3263{
3264 return 7;
3265}
3266
3279{
3280public:
3281 int32_t getLayerCount() const noexcept
3282 {
3283 return mImpl->getLayerCount();
3284 }
3285 int32_t getHiddenSize() const noexcept
3286 {
3287 return mImpl->getHiddenSize();
3288 }
3289 int32_t getMaxSeqLength() const noexcept
3290 {
3291 return mImpl->getMaxSeqLength();
3292 }
3293 int32_t getDataLength() const noexcept
3294 {
3295 return mImpl->getDataLength();
3296 }
3297
3312 void setSequenceLengths(ITensor& seqLengths) noexcept
3313 {
3314 return mImpl->setSequenceLengths(seqLengths);
3315 }
3316
3324 ITensor* getSequenceLengths() const noexcept
3325 {
3326 return mImpl->getSequenceLengths();
3327 }
3328
3334 void setOperation(RNNOperation op) noexcept
3335 {
3336 mImpl->setOperation(op);
3337 }
3338
3345 {
3346 return mImpl->getOperation();
3347 }
3348
3354 void setInputMode(RNNInputMode op) noexcept
3355 {
3356 mImpl->setInputMode(op);
3357 }
3358
3365 {
3366 return mImpl->getInputMode();
3367 }
3368
3379 void setDirection(RNNDirection op) noexcept
3380 {
3381 mImpl->setDirection(op);
3382 }
3383
3390 {
3391 return mImpl->getDirection();
3392 }
3393
3448 void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept
3449 {
3450 mImpl->setWeightsForGate(layerIndex, gate, isW, weights);
3451 }
3452
3458 Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
3459 {
3460 return mImpl->getWeightsForGate(layerIndex, gate, isW);
3461 }
3462
3483 void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept
3484 {
3485 mImpl->setBiasForGate(layerIndex, gate, isW, bias);
3486 }
3487
3493 Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
3494 {
3495 return mImpl->getBiasForGate(layerIndex, gate, isW);
3496 }
3497
3510 void setHiddenState(ITensor& hidden) noexcept
3511 {
3512 mImpl->setHiddenState(hidden);
3513 }
3514
3520 ITensor* getHiddenState() const noexcept
3521 {
3522 return mImpl->getHiddenState();
3523 }
3524
3539 void setCellState(ITensor& cell) noexcept
3540 {
3541 mImpl->setCellState(cell);
3542 }
3543
3549 ITensor* getCellState() const noexcept
3550 {
3551 return mImpl->getCellState();
3552 }
3553
3554protected:
3555 apiv::VRNNv2Layer* mImpl;
3556 virtual ~IRNNv2Layer() noexcept = default;
3557};
3558
3569{
3570public:
3577 {
3578 return mImpl->getPlugin();
3579 }
3580
3581protected:
3582 apiv::VPluginV2Layer* mImpl;
3583 virtual ~IPluginV2Layer() noexcept = default;
3584};
3585
3599enum class UnaryOperation : int32_t
3600{
3601 kEXP = 0,
3602 kLOG = 1,
3603 kSQRT = 2,
3604 kRECIP = 3,
3605 kABS = 4,
3606 kNEG = 5,
3607 kSIN = 6,
3608 kCOS = 7,
3609 kTAN = 8,
3610 kSINH = 9,
3611 kCOSH = 10,
3612 kASIN = 11,
3613 kACOS = 12,
3614 kATAN = 13,
3615 kASINH = 14,
3616 kACOSH = 15,
3617 kATANH = 16,
3618 kCEIL = 17,
3619 kFLOOR = 18,
3620 kERF = 19,
3621 kNOT = 20,
3622 kSIGN = 21,
3623 kROUND = 22
3624};
3625
3631template <>
3632constexpr inline int32_t EnumMax<UnaryOperation>() noexcept
3633{
3634 return 23;
3635}
3636
3644class IUnaryLayer : public ILayer
3645{
3646public:
3655 {
3656 mImpl->setOperation(op);
3657 }
3658
3665 {
3666 return mImpl->getOperation();
3667 }
3668
3669protected:
3670 apiv::VUnaryLayer* mImpl;
3671 virtual ~IUnaryLayer() noexcept = default;
3672};
3673
3692enum class ReduceOperation : int32_t
3693{
3694 kSUM = 0,
3695 kPROD = 1,
3696 kMAX = 2,
3697 kMIN = 3,
3698 kAVG = 4
3699};
3700
3706template <>
3707constexpr inline int32_t EnumMax<ReduceOperation>() noexcept
3708{
3709 return 5;
3710}
3711
3719class IReduceLayer : public ILayer
3720{
3721public:
3728 {
3729 mImpl->setOperation(op);
3730 }
3731
3738 {
3739 return mImpl->getOperation();
3740 }
3741
3747 void setReduceAxes(uint32_t reduceAxes) noexcept
3748 {
3749 mImpl->setReduceAxes(reduceAxes);
3750 }
3751
3757 uint32_t getReduceAxes() const noexcept
3758 {
3759 return mImpl->getReduceAxes();
3760 }
3761
3767 void setKeepDimensions(bool keepDimensions) noexcept
3768 {
3769 mImpl->setKeepDimensions(keepDimensions);
3770 }
3771
3777 bool getKeepDimensions() const noexcept
3778 {
3779 return mImpl->getKeepDimensions();
3780 }
3781
3782protected:
3783 apiv::VReduceLayer* mImpl;
3784 virtual ~IReduceLayer() noexcept = default;
3785};
3786
3797class IPaddingLayer : public ILayer
3798{
3799public:
3809 TRT_DEPRECATED void setPrePadding(DimsHW padding) noexcept
3810 {
3811 mImpl->setPrePadding(padding);
3812 }
3813
3822 {
3823 return mImpl->getPrePadding();
3824 }
3825
3835 TRT_DEPRECATED void setPostPadding(DimsHW padding) noexcept
3836 {
3837 mImpl->setPostPadding(padding);
3838 }
3839
3848 {
3849 return mImpl->getPostPadding();
3850 }
3851
3861 void setPrePaddingNd(Dims padding) noexcept
3862 {
3863 mImpl->setPrePaddingNd(padding);
3864 }
3865
3873 Dims getPrePaddingNd() const noexcept
3874 {
3875 return mImpl->getPrePaddingNd();
3876 }
3877
3887 void setPostPaddingNd(Dims padding) noexcept
3888 {
3889 mImpl->setPostPaddingNd(padding);
3890 }
3891
3899 Dims getPostPaddingNd() const noexcept
3900 {
3901 return mImpl->getPostPaddingNd();
3902 }
3903
3904protected:
3905 apiv::VPaddingLayer* mImpl;
3906 virtual ~IPaddingLayer() noexcept = default;
3907};
3908
3910{
3917 int32_t order[Dims::MAX_DIMS];
3918};
3919
3932class IShuffleLayer : public ILayer
3933{
3934public:
3944 void setFirstTranspose(Permutation permutation) noexcept
3945 {
3946 mImpl->setFirstTranspose(permutation);
3947 }
3948
3957 {
3958 return mImpl->getFirstTranspose();
3959 }
3960
3981 void setReshapeDimensions(Dims dimensions) noexcept
3982 {
3983 mImpl->setReshapeDimensions(dimensions);
3984 }
3985
3995 {
3996 return mImpl->getReshapeDimensions();
3997 }
3998
4004 //
4027 using ILayer::setInput;
4028
4041 void setSecondTranspose(Permutation permutation) noexcept
4042 {
4043 mImpl->setSecondTranspose(permutation);
4044 }
4045
4054 {
4055 return mImpl->getSecondTranspose();
4056 }
4057
4069 void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
4070 {
4071 return mImpl->setZeroIsPlaceholder(zeroIsPlaceholder);
4072 }
4073
4082 bool getZeroIsPlaceholder() const noexcept
4083 {
4084 return mImpl->getZeroIsPlaceholder();
4085 }
4086
4087protected:
4088 apiv::VShuffleLayer* mImpl;
4089 virtual ~IShuffleLayer() noexcept = default;
4090};
4091
4097enum class SliceMode : int32_t
4098{
4099 kDEFAULT = 0,
4100 kWRAP = 1,
4101 kCLAMP = 2,
4102 kFILL = 3,
4103 kREFLECT = 4,
4106};
4107
4113template <>
4114constexpr inline int32_t EnumMax<SliceMode>() noexcept
4115{
4116 return 5;
4117}
4118
4154class ISliceLayer : public ILayer
4155{
4156public:
4166 void setStart(Dims start) noexcept
4167 {
4168 mImpl->setStart(start);
4169 }
4170
4181 Dims getStart() const noexcept
4182 {
4183 return mImpl->getStart();
4184 }
4185
4195 void setSize(Dims size) noexcept
4196 {
4197 return mImpl->setSize(size);
4198 }
4199
4210 Dims getSize() const noexcept
4211 {
4212 return mImpl->getSize();
4213 }
4214
4224 void setStride(Dims stride) noexcept
4225 {
4226 mImpl->setStride(stride);
4227 }
4228
4239 Dims getStride() const noexcept
4240 {
4241 return mImpl->getStride();
4242 }
4243
4249 void setMode(SliceMode mode) noexcept
4250 {
4251 mImpl->setMode(mode);
4252 }
4253
4259 SliceMode getMode() const noexcept
4260 {
4261 return mImpl->getMode();
4262 }
4263
4285 using ILayer::setInput;
4286
4287protected:
4288 apiv::VSliceLayer* mImpl;
4289 virtual ~ISliceLayer() noexcept = default;
4290};
4291
4304class IShapeLayer : public ILayer
4305{
4306protected:
4307 apiv::VShapeLayer* mImpl;
4308 virtual ~IShapeLayer() noexcept = default;
4309};
4310
4316enum class TopKOperation : int32_t
4317{
4318 kMAX = 0,
4319 kMIN = 1,
4320};
4321
4327template <>
4328constexpr inline int32_t EnumMax<TopKOperation>() noexcept
4329{
4330 return 2;
4331}
4332
4340class ITopKLayer : public ILayer
4341{
4342public:
4348 void setOperation(TopKOperation op) noexcept
4349 {
4350 mImpl->setOperation(op);
4351 }
4352
4359 {
4360 return mImpl->getOperation();
4361 }
4362
4370 void setK(int32_t k) noexcept
4371 {
4372 mImpl->setK(k);
4373 }
4374
4380 int32_t getK() const noexcept
4381 {
4382 return mImpl->getK();
4383 }
4384
4390 void setReduceAxes(uint32_t reduceAxes) noexcept
4391 {
4392 mImpl->setReduceAxes(reduceAxes);
4393 }
4394
4400 uint32_t getReduceAxes() const noexcept
4401 {
4402 return mImpl->getReduceAxes();
4403 }
4404
4405protected:
4406 apiv::VTopKLayer* mImpl;
4407 virtual ~ITopKLayer() noexcept = default;
4408};
4409
4416enum class MatrixOperation : int32_t
4417{
4421 kNONE,
4422
4424 kTRANSPOSE,
4425
4436 kVECTOR
4437};
4438
4444template <>
4445constexpr inline int32_t EnumMax<MatrixOperation>() noexcept
4446{
4447 return 3;
4448}
4449
4476{
4477public:
4484 void setOperation(int32_t index, MatrixOperation op) noexcept
4485 {
4486 mImpl->setOperation(index, op);
4487 }
4488
4496 MatrixOperation getOperation(int32_t index) const noexcept
4497 {
4498 return mImpl->getOperation(index);
4499 }
4500
4501protected:
4502 apiv::VMatrixMultiplyLayer* mImpl;
4503 virtual ~IMatrixMultiplyLayer() noexcept = default;
4504};
4505
4521{
4522protected:
4523 apiv::VRaggedSoftMaxLayer* mImpl;
4524 virtual ~IRaggedSoftMaxLayer() noexcept = default;
4525};
4526
4551{
4552protected:
4553 apiv::VIdentityLayer* mImpl;
4554 virtual ~IIdentityLayer() noexcept = default;
4555};
4556
4566{
4567public:
4577 void setWeights(Weights weights) noexcept
4578 {
4579 mImpl->setWeights(weights);
4580 }
4581
4587 Weights getWeights() const noexcept
4588 {
4589 return mImpl->getWeights();
4590 }
4591
4599 void setDimensions(Dims dimensions) noexcept
4600 {
4601 mImpl->setDimensions(dimensions);
4602 }
4603
4611 Dims getDimensions() const noexcept
4612 {
4613 return mImpl->getDimensions();
4614 }
4615
4616protected:
4617 apiv::VConstantLayer* mImpl;
4618 virtual ~IConstantLayer() noexcept = default;
4619};
4620
4631{
4632protected:
4633 apiv::VParametricReLULayer* mImpl;
4634 virtual ~IParametricReLULayer() noexcept = default;
4635};
4636
4642enum class ResizeMode : int32_t
4643{
4644 kNEAREST = 0,
4645 kLINEAR = 1
4646};
4647
4648namespace impl
4649{
4655template <>
4657{
4658 static constexpr int32_t kVALUE = 2;
4659};
4660} // namespace impl
4661
4670{
4683 kALIGN_CORNERS = 0,
4684
4691 kASYMMETRIC = 1,
4692
4699 kHALF_PIXEL = 2,
4700};
4701
4702namespace impl
4703{
4709template <>
4711{
4712 static constexpr int32_t kVALUE = 3;
4713};
4714} // namespace impl
4715
4723enum class ResizeSelector : int32_t
4724{
4726 kFORMULA = 0,
4727
4729 kUPPER = 1,
4730};
4731
4732namespace impl
4733{
4739template <>
4741{
4742 static constexpr int32_t kVALUE = 2;
4743};
4744} // namespace impl
4745
4753enum class ResizeRoundMode : int32_t
4754{
4756 kHALF_UP = 0,
4757
4759 kHALF_DOWN = 1,
4760
4762 kFLOOR = 2,
4763
4765 kCEIL = 3,
4766};
4767
4768namespace impl
4769{
4775template <>
4777{
4778 static constexpr int32_t kVALUE = 4;
4779};
4780} // namespace impl
4781
4818class IResizeLayer : public ILayer
4819{
4820public:
4839 void setOutputDimensions(Dims dimensions) noexcept
4840 {
4841 return mImpl->setOutputDimensions(dimensions);
4842 }
4843
4849 Dims getOutputDimensions() const noexcept
4850 {
4851 return mImpl->getOutputDimensions();
4852 }
4853
4879 void setScales(const float* scales, int32_t nbScales) noexcept
4880 {
4881 mImpl->setScales(scales, nbScales);
4882 }
4883
4898 int32_t getScales(int32_t size, float* scales) const noexcept
4899 {
4900 return mImpl->getScales(size, scales);
4901 }
4902
4910 void setResizeMode(ResizeMode resizeMode) noexcept
4911 {
4912 mImpl->setResizeMode(resizeMode);
4913 }
4914
4920 ResizeMode getResizeMode() const noexcept
4921 {
4922 return mImpl->getResizeMode();
4923 }
4924
4936 TRT_DEPRECATED void setAlignCorners(bool alignCorners) noexcept
4937 {
4938 mImpl->setAlignCorners(alignCorners);
4939 }
4940
4948 TRT_DEPRECATED bool getAlignCorners() const noexcept
4949 {
4950 return mImpl->getAlignCorners();
4951 }
4952
4972 using ILayer::setInput;
4973
4984 {
4985 mImpl->setCoordinateTransformation(coordTransform);
4986 }
4987
4994 {
4995 return mImpl->getCoordinateTransformation();
4996 }
4997
5009 {
5010 mImpl->setSelectorForSinglePixel(selector);
5011 }
5012
5019 {
5020 return mImpl->getSelectorForSinglePixel();
5021 }
5022
5033 {
5034 mImpl->setNearestRounding(value);
5035 }
5036
5043 {
5044 return mImpl->getNearestRounding();
5045 }
5046
5047protected:
5048 virtual ~IResizeLayer() noexcept = default;
5049 apiv::VResizeLayer* mImpl;
5050};
5051
5053enum class LoopOutput : int32_t
5054{
5056 kLAST_VALUE = 0,
5057
5059 kCONCATENATE = 1,
5060
5062 kREVERSE = 2
5063};
5064
5070template <>
5071constexpr inline int32_t EnumMax<LoopOutput>() noexcept
5072{
5073 return 3;
5074}
5075
5077enum class TripLimit : int32_t
5078{
5079
5080 kCOUNT = 0,
5081 kWHILE = 1
5082};
5083
5089template <>
5090constexpr inline int32_t EnumMax<TripLimit>() noexcept
5091{
5092 return 2;
5093}
5094
5095class ILoop;
5096
5098{
5099public:
5101 ILoop* getLoop() const noexcept
5102 {
5103 return mBoundary->getLoop();
5104 }
5105
5106protected:
5107 virtual ~ILoopBoundaryLayer() noexcept = default;
5108 apiv::VLoopBoundaryLayer* mBoundary;
5109};
5110
5117{
5118public:
5121 {
5122 return mBoundary->getConditional();
5123 }
5124
5125protected:
5126 virtual ~IIfConditionalBoundaryLayer() noexcept = default;
5127 apiv::VConditionalBoundaryLayer* mBoundary;
5128};
5129
5134{
5135public:
5136protected:
5137 virtual ~IConditionLayer() noexcept = default;
5138 apiv::VConditionLayer* mImpl;
5139};
5140
5147{
5148public:
5149protected:
5150 virtual ~IIfConditionalOutputLayer() noexcept = default;
5151 apiv::VConditionalOutputLayer* mImpl;
5152};
5153
5158{
5159public:
5160protected:
5161 virtual ~IIfConditionalInputLayer() noexcept = default;
5162 apiv::VConditionalInputLayer* mImpl;
5163};
5164
5187{
5188public:
5199 {
5200 return mImpl->setCondition(condition);
5201 }
5202
5214 IIfConditionalOutputLayer* addOutput(ITensor& trueSubgraphOutput, ITensor& falseSubgraphOutput) noexcept
5215 {
5216 return mImpl->addOutput(trueSubgraphOutput, falseSubgraphOutput);
5217 }
5218
5227 {
5228 return mImpl->addInput(input);
5229 }
5230
5239 void setName(const char* name) noexcept
5240 {
5241 mImpl->setName(name);
5242 }
5243
5249 const char* getName() const noexcept
5250 {
5251 return mImpl->getName();
5252 }
5253
5254protected:
5255 virtual ~IIfConditional() noexcept = default;
5256 apiv::VIfConditional* mImpl;
5257};
5258
5259
5261{
5262public:
5268 //
5281 using ILayer::setInput;
5282
5283protected:
5284 virtual ~IRecurrenceLayer() noexcept = default;
5285 apiv::VRecurrenceLayer* mImpl;
5286};
5287
5306{
5307public:
5308 LoopOutput getLoopOutput() const noexcept
5309 {
5310 return mImpl->getLoopOutput();
5311 }
5312
5325 void setAxis(int32_t axis) noexcept
5326 {
5327 mImpl->setAxis(axis);
5328 }
5329
5331 int32_t getAxis() const noexcept
5332 {
5333 return mImpl->getAxis();
5334 }
5335
5341 //
5356 using ILayer::setInput;
5357
5358protected:
5359 virtual ~ILoopOutputLayer() noexcept = default;
5360 apiv::VLoopOutputLayer* mImpl;
5361};
5362
5364{
5365public:
5366 TripLimit getTripLimit() const noexcept
5367 {
5368 return mImpl->getTripLimit();
5369 }
5370
5371protected:
5372 virtual ~ITripLimitLayer() noexcept = default;
5373 apiv::VTripLimitLayer* mImpl;
5374};
5375
5377{
5378public:
5380 void setAxis(int32_t axis) noexcept
5381 {
5382 mImpl->setAxis(axis);
5383 }
5384
5386 int32_t getAxis() const noexcept
5387 {
5388 return mImpl->getAxis();
5389 }
5390
5396 void setReverse(bool reverse) noexcept
5397 {
5398 mImpl->setReverse(reverse);
5399 }
5400
5402 bool getReverse() const noexcept
5403 {
5404 return mImpl->getReverse();
5405 }
5406
5407protected:
5408 virtual ~IIteratorLayer() noexcept = default;
5409 apiv::VIteratorLayer* mImpl;
5410};
5411
5417class ILoop : public INoCopy
5418{
5419public:
5426 IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept
5427 {
5428 return mImpl->addRecurrence(initialValue);
5429 }
5430
5448 {
5449 return mImpl->addTripLimit(tensor, limit);
5450 }
5451
5460 IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept
5461 {
5462 return mImpl->addIterator(tensor, axis, reverse);
5463 }
5464
5472 ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept
5473 {
5474 return mImpl->addLoopOutput(tensor, outputKind, axis);
5475 }
5476
5485 void setName(const char* name) noexcept
5486 {
5487 mImpl->setName(name);
5488 }
5489
5495 const char* getName() const noexcept
5496 {
5497 return mImpl->getName();
5498 }
5499
5500protected:
5501 virtual ~ILoop() noexcept = default;
5502 apiv::VLoop* mImpl;
5503};
5504
5508class ISelectLayer : public ILayer
5509{
5510protected:
5511 virtual ~ISelectLayer() noexcept = default;
5512 apiv::VSelectLayer* mImpl;
5513};
5514
5530{
5531public:
5540 void setMessage(const char* message) noexcept
5541 {
5542 mImpl->setMessage(message);
5543 }
5544
5550 const char* getMessage() const noexcept
5551 {
5552 return mImpl->getMessage();
5553 }
5554
5555protected:
5556 virtual ~IAssertionLayer() noexcept = default;
5557
5558 apiv::VAssertionLayer* mImpl;
5559};
5560
5568enum class FillOperation : int32_t
5569{
5570 kLINSPACE = 0,
5571 kRANDOM_UNIFORM = 1
5572};
5573
5579template <>
5580constexpr inline int32_t EnumMax<FillOperation>() noexcept
5581{
5582 return 2;
5583}
5584
5610class IFillLayer : public ILayer
5611{
5612public:
5621 //
5622 void setDimensions(Dims dimensions) noexcept
5623 {
5624 mImpl->setDimensions(dimensions);
5625 }
5626
5637 Dims getDimensions() const noexcept
5638 {
5639 return mImpl->getDimensions();
5640 }
5641
5647 void setOperation(FillOperation op) noexcept
5648 {
5649 mImpl->setOperation(op);
5650 }
5651
5658 {
5659 return mImpl->getOperation();
5660 }
5661
5674 //
5675 void setAlpha(double alpha) noexcept
5676 {
5677 mImpl->setAlpha(alpha);
5678 }
5679
5690 double getAlpha() const noexcept
5691 {
5692 return mImpl->getAlpha();
5693 }
5694
5708 void setBeta(double beta) noexcept
5709 {
5710 mImpl->setBeta(beta);
5711 }
5712
5723 double getBeta() const noexcept
5724 {
5725 return mImpl->getBeta();
5726 }
5727
5754 using ILayer::setInput;
5755
5756protected:
5757 virtual ~IFillLayer() noexcept = default;
5758 apiv::VFillLayer* mImpl;
5759};
5760
5819{
5820public:
5829 int32_t getAxis() const noexcept
5830 {
5831 return mImpl->getAxis();
5832 }
5840 void setAxis(int32_t axis) noexcept
5841 {
5842 mImpl->setAxis(axis);
5843 }
5844
5845protected:
5846 virtual ~IQuantizeLayer() noexcept = default;
5847 apiv::VQuantizeLayer* mImpl;
5848};
5849
5906{
5907public:
5916 int32_t getAxis() const noexcept
5917 {
5918 return mImpl->getAxis();
5919 }
5927 void setAxis(int32_t axis) noexcept
5928 {
5929 mImpl->setAxis(axis);
5930 }
5931
5932protected:
5933 virtual ~IDequantizeLayer() noexcept = default;
5934 apiv::VDequantizeLayer* mImpl;
5935};
5936
5972class IEinsumLayer : public ILayer
5973{
5974public:
5984 bool setEquation(const char* equation) noexcept
5985 {
5986 return mImpl->setEquation(equation);
5987 }
5988
5994 const char* getEquation() const noexcept
5995 {
5996 return mImpl->getEquation();
5997 }
5998
5999protected:
6000 virtual ~IEinsumLayer() noexcept = default;
6001 apiv::VEinsumLayer* mImpl;
6002};
6003
6009enum class ScatterMode : int32_t
6010{
6011 kELEMENT = 0,
6012 kND = 1,
6013};
6014
6020template <>
6021constexpr inline int32_t EnumMax<ScatterMode>() noexcept
6022{
6023 return 2;
6024}
6025
6082class IScatterLayer : public ILayer
6083{
6084public:
6090 void setMode(ScatterMode mode) noexcept
6091 {
6092 mImpl->setMode(mode);
6093 }
6094
6100 ScatterMode getMode() const noexcept
6101 {
6102 return mImpl->getMode();
6103 }
6104
6110 void setAxis(int32_t axis) noexcept
6111 {
6112 mImpl->setAxis(axis);
6113 }
6114
6118 int32_t getAxis() const noexcept
6119 {
6120 return mImpl->getAxis();
6121 }
6122
6123protected:
6124 apiv::VScatterLayer* mImpl;
6125 virtual ~IScatterLayer() noexcept = default;
6126}; // class IScatterLayer
6127
6148{
6149public:
6150 virtual ~INetworkDefinition() noexcept = default;
6151
6187 ITensor* addInput(const char* name, DataType type, Dims dimensions) noexcept
6188 {
6189 return mImpl->addInput(name, type, dimensions);
6190 }
6191
6201 void markOutput(ITensor& tensor) noexcept
6202 {
6203 mImpl->markOutput(tensor);
6204 }
6205
6225 ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
6226 {
6227 return mImpl->addConvolution(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
6228 }
6229
6248 ITensor& input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept
6249 {
6250 return mImpl->addFullyConnected(input, nbOutputs, kernelWeights, biasWeights);
6251 }
6252
6268 {
6269 return mImpl->addActivation(input, type);
6270 }
6271
6287 {
6288 return mImpl->addPooling(input, type, windowSize);
6289 }
6290
6305 ILRNLayer* addLRN(ITensor& input, int32_t window, float alpha, float beta, float k) noexcept
6306 {
6307 return mImpl->addLRN(input, window, alpha, beta, k);
6308 }
6309
6332 IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
6333 {
6334 return mImpl->addScale(input, mode, shift, scale, power);
6335 }
6336
6346 {
6347 return mImpl->addSoftMax(input);
6348 }
6349
6362 IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept
6363 {
6364 return mImpl->addConcatenation(inputs, nbInputs);
6365 }
6366
6386 ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
6387 {
6388 return mImpl->addDeconvolution(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
6389 }
6390
6414 {
6415 return mImpl->addElementWise(input1, input2, op);
6416 }
6417
6435 IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept
6436 {
6437 return mImpl->addUnary(input, operation);
6438 }
6439
6452 TRT_DEPRECATED IPaddingLayer* addPadding(ITensor& input, DimsHW prePadding, DimsHW postPadding) noexcept
6453 {
6454 return mImpl->addPadding(input, prePadding, postPadding);
6455 }
6456
6467 {
6468 return mImpl->addShuffle(input);
6469 }
6470
6478 int32_t getNbLayers() const noexcept
6479 {
6480 return mImpl->getNbLayers();
6481 }
6482
6492 ILayer* getLayer(int32_t index) const noexcept
6493 {
6494 return mImpl->getLayer(index);
6495 }
6496
6504 int32_t getNbInputs() const noexcept
6505 {
6506 return mImpl->getNbInputs();
6507 }
6508
6520 ITensor* getInput(int32_t index) const noexcept
6521 {
6522 return mImpl->getInput(index);
6523 }
6524
6534 int32_t getNbOutputs() const noexcept
6535 {
6536 return mImpl->getNbOutputs();
6537 }
6538
6550 ITensor* getOutput(int32_t index) const noexcept
6551 {
6552 return mImpl->getOutput(index);
6553 }
6554
6562 TRT_DEPRECATED void destroy() noexcept
6563 {
6564 delete this;
6565 }
6566
6590 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
6591 {
6592 return mImpl->addReduce(input, operation, reduceAxes, keepDimensions);
6593 }
6594
6623 ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
6624 {
6625 return mImpl->addTopK(input, op, k, reduceAxes);
6626 }
6627
6639 IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept
6640 {
6641 return mImpl->addGather(data, indices, axis);
6642 }
6643
6656 {
6657 return mImpl->addGatherV2(data, indices, mode);
6658 }
6659
6674 {
6675 return mImpl->addRaggedSoftMax(input, bounds);
6676 }
6677
6695 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept
6696 {
6697 return mImpl->addMatrixMultiply(input0, op0, input1, op1);
6698 }
6699
6720 IConstantLayer* addConstant(Dims dimensions, Weights weights) noexcept
6721 {
6722 return mImpl->addConstant(dimensions, weights);
6723 }
6724
6790 ITensor& input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) noexcept
6791 {
6792 return mImpl->addRNNv2(input, layerCount, hiddenSize, maxSeqLen, op);
6793 }
6794
6805 {
6806 return mImpl->addIdentity(input);
6807 }
6808
6819 void removeTensor(ITensor& tensor) noexcept
6820 {
6821 mImpl->removeTensor(tensor);
6822 }
6823
6831 void unmarkOutput(ITensor& tensor) noexcept
6832 {
6833 mImpl->unmarkOutput(tensor);
6834 }
6835
6850 IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept
6851 {
6852 return mImpl->addPluginV2(inputs, nbInputs, plugin);
6853 }
6854
6869 ISliceLayer* addSlice(ITensor& input, Dims start, Dims size, Dims stride) noexcept
6870 {
6871 return mImpl->addSlice(input, start, size, stride);
6872 }
6873
6891 void setName(const char* name) noexcept
6892 {
6893 mImpl->setName(name);
6894 }
6895
6905 const char* getName() const noexcept
6906 {
6907 return mImpl->getName();
6908 }
6909
6923 IShapeLayer* addShape(ITensor& input) noexcept
6924 {
6925 return mImpl->addShape(input);
6926 }
6927
6942 bool hasImplicitBatchDimension() const noexcept
6943 {
6944 return mImpl->hasImplicitBatchDimension();
6945 }
6946
6960 bool markOutputForShapes(ITensor& tensor) noexcept
6961 {
6962 return mImpl->markOutputForShapes(tensor);
6963 }
6964
6972 bool unmarkOutputForShapes(ITensor& tensor) noexcept
6973 {
6974 return mImpl->unmarkOutputForShapes(tensor);
6975 }
6976
6991 {
6992 return mImpl->addParametricReLU(input, slope);
6993 }
6994
7013 ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7014 {
7015 return mImpl->addConvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7016 }
7017
7032 IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims windowSize) noexcept
7033 {
7034 return mImpl->addPoolingNd(input, type, windowSize);
7035 }
7036
7051 //
7055 ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
7056 {
7057 return mImpl->addDeconvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
7058 }
7059
7091 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
7092 {
7093 return mImpl->addScaleNd(input, mode, shift, scale, power, channelAxis);
7094 }
7095
7107 {
7108 return mImpl->addResize(input);
7109 }
7110
7124 {
7125 return mImpl->hasExplicitPrecision();
7126 }
7127
7139 ILoop* addLoop() noexcept
7140 {
7141 return mImpl->addLoop();
7142 }
7143
7179 ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept
7180 {
7181 return mImpl->addSelect(condition, thenInput, elseInput);
7182 }
7183
7196 IAssertionLayer* addAssertion(ITensor& condition, const char* message) noexcept
7197 {
7198 return mImpl->addAssertion(condition, message);
7199 }
7200
7214 IFillLayer* addFill(Dims dimensions, FillOperation op) noexcept
7215 {
7216 return mImpl->addFill(dimensions, op);
7217 }
7218
7231 TRT_DEPRECATED IPaddingLayer* addPaddingNd(ITensor& input, Dims prePadding, Dims postPadding) noexcept
7232 {
7233 return mImpl->addPaddingNd(input, prePadding, postPadding);
7234 }
7235
7251 bool setWeightsName(Weights weights, const char* name) noexcept
7252 {
7253 return mImpl->setWeightsName(weights, name);
7254 }
7255
7267 //
7270 void setErrorRecorder(IErrorRecorder* recorder) noexcept
7271 {
7272 mImpl->setErrorRecorder(recorder);
7273 }
7274
7286 {
7287 return mImpl->getErrorRecorder();
7288 }
7289
7305 {
7306 return mImpl->addDequantize(input, scale);
7307 }
7308
7324 IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept
7325 {
7326 return mImpl->addScatter(data, indices, updates, mode);
7327 }
7328
7343 IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale) noexcept
7344 {
7345 return mImpl->addQuantize(input, scale);
7346 }
7347
7359 {
7360 return mImpl->addIfConditional();
7361 }
7362
7372 IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, const char* equation) noexcept
7373 {
7374 return mImpl->addEinsum(inputs, nbInputs, equation);
7375 }
7376
7377protected:
7378 apiv::VNetworkDefinition* mImpl;
7379};
7380
7386enum class CalibrationAlgoType : int32_t
7387{
7392};
7393
7399template <>
7400constexpr inline int32_t EnumMax<CalibrationAlgoType>() noexcept
7401{
7402 return 4;
7403}
7404
7417{
7418public:
7424 virtual int32_t getBatchSize() const noexcept = 0;
7425
7439 virtual bool getBatch(void* bindings[], const char* names[], int32_t nbBindings) noexcept = 0;
7440
7455 virtual const void* readCalibrationCache(std::size_t& length) noexcept = 0;
7456
7465 virtual void writeCalibrationCache(const void* ptr, std::size_t length) noexcept = 0;
7466
7472 virtual CalibrationAlgoType getAlgorithm() noexcept = 0;
7473
7474 virtual ~IInt8Calibrator() noexcept = default;
7475};
7476
7482{
7483public:
7488 {
7490 }
7491
7492 virtual ~IInt8EntropyCalibrator() noexcept = default;
7493};
7494
7500{
7501public:
7506 {
7508 }
7509
7510 virtual ~IInt8EntropyCalibrator2() noexcept = default;
7511};
7512
7517{
7518public:
7523 {
7525 }
7526
7527 virtual ~IInt8MinMaxCalibrator() noexcept = default;
7528};
7529
7535{
7536public:
7541 {
7543 }
7544
7551 virtual double getQuantile() const noexcept = 0;
7552
7559 virtual double getRegressionCutoff() const noexcept = 0;
7560
7573 virtual const void* readHistogramCache(std::size_t& length) noexcept = 0;
7574
7583 virtual void writeHistogramCache(const void* ptr, std::size_t length) noexcept = 0;
7584
7585 virtual ~IInt8LegacyCalibrator() noexcept = default;
7586};
7587
7599{
7600public:
7605 {
7606 return mImpl->getTensorFormat();
7607 }
7608
7612 DataType getDataType() const noexcept
7613 {
7614 return mImpl->getDataType();
7615 }
7616
7620 Dims getStrides() const noexcept
7621 {
7622 return mImpl->getStrides();
7623 }
7624
7625protected:
7626 virtual ~IAlgorithmIOInfo() noexcept = default;
7627 apiv::VAlgorithmIOInfo* mImpl;
7628};
7629
7642{
7643public:
7647 int64_t getImplementation() const noexcept
7648 {
7649 return mImpl->getImplementation();
7650 }
7651
7655 int64_t getTactic() const noexcept
7656 {
7657 return mImpl->getTactic();
7658 }
7659
7660protected:
7661 virtual ~IAlgorithmVariant() noexcept = default;
7662 apiv::VAlgorithmVariant* mImpl;
7663};
7664
7674{
7675public:
7680 const char* getName() const noexcept
7681 {
7682 return mImpl->getName();
7683 }
7684
7691 Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
7692 {
7693 return mImpl->getDimensions(index, select);
7694 }
7695
7699 int32_t getNbInputs() const noexcept
7700 {
7701 return mImpl->getNbInputs();
7702 }
7703
7707 int32_t getNbOutputs() const noexcept
7708 {
7709 return mImpl->getNbOutputs();
7710 }
7711
7712protected:
7713 virtual ~IAlgorithmContext() noexcept = default;
7714 apiv::VAlgorithmContext* mImpl;
7715};
7716
7726class IAlgorithm : public INoCopy
7727{
7728public:
7739 TRT_DEPRECATED const IAlgorithmIOInfo& getAlgorithmIOInfo(int32_t index) const noexcept
7740 {
7741 return mImpl->getAlgorithmIOInfo(index);
7742 }
7743
7748 {
7749 return mImpl->getAlgorithmVariant();
7750 }
7751
7755 float getTimingMSec() const noexcept
7756 {
7757 return mImpl->getTimingMSec();
7758 }
7759
7763 std::size_t getWorkspaceSize() const noexcept
7764 {
7765 return mImpl->getWorkspaceSize();
7766 }
7767
7776 const IAlgorithmIOInfo* getAlgorithmIOInfoByIndex(int32_t index) const noexcept
7777 {
7778 return mImpl->getAlgorithmIOInfoByIndex(index);
7779 }
7780
7781protected:
7782 virtual ~IAlgorithm() noexcept = default;
7783 apiv::VAlgorithm* mImpl;
7784}; // IAlgorithm
7785
7795{
7796public:
7811 virtual int32_t selectAlgorithms(const IAlgorithmContext& context, const IAlgorithm* const* choices,
7812 int32_t nbChoices, int32_t* selection) noexcept
7813 = 0;
7824 virtual void reportAlgorithms(const IAlgorithmContext* const* algoContexts, const IAlgorithm* const* algoChoices,
7825 int32_t nbAlgorithms) noexcept
7826 = 0;
7827
7828 virtual ~IAlgorithmSelector() noexcept = default;
7829};
7830
7837using QuantizationFlags = uint32_t;
7838
7846enum class QuantizationFlag : int32_t
7847{
7852};
7853
7859template <>
7860constexpr inline int32_t EnumMax<QuantizationFlag>() noexcept
7861{
7862 return 1;
7863}
7864
7871using BuilderFlags = uint32_t;
7872
7880enum class BuilderFlag : int32_t
7881{
7882 kFP16 = 0,
7883 kINT8 = 1,
7884 kDEBUG = 2,
7885 kGPU_FALLBACK = 3,
7886
7898
7899 kREFIT = 5,
7901
7905 kTF32 = 7,
7906
7908 kSPARSE_WEIGHTS = 8,
7909
7916 kSAFETY_SCOPE = 9,
7917
7920
7924
7928 kDIRECT_IO = 12,
7929
7932};
7933
7939template <>
7940constexpr inline int32_t EnumMax<BuilderFlag>() noexcept
7941{
7942 return 14;
7943}
7944
7955class ITimingCache : public INoCopy
7956{
7957public:
7958 virtual ~ITimingCache() noexcept = default;
7959
7969 nvinfer1::IHostMemory* serialize() const noexcept
7970 {
7971 return mImpl->serialize();
7972 }
7973
7993 bool combine(const ITimingCache& inputCache, bool ignoreMismatch) noexcept
7994 {
7995 return mImpl->combine(inputCache, ignoreMismatch);
7996 }
7997
8003 bool reset() noexcept
8004 {
8005 return mImpl->reset();
8006 }
8007
8008protected:
8009 apiv::VTimingCache* mImpl;
8010};
8011
8019enum class MemoryPoolType : int32_t
8020{
8028 kWORKSPACE = 0,
8029
8037
8043 kDLA_LOCAL_DRAM = 2,
8044
8050 kDLA_GLOBAL_DRAM = 3,
8051};
8052
8058template <>
8059constexpr inline int32_t EnumMax<MemoryPoolType>() noexcept
8060{
8061 return 4;
8062}
8063
8072{
8073public:
8074 virtual ~IBuilderConfig() noexcept = default;
8075
8086 virtual void setMinTimingIterations(int32_t minTiming) noexcept
8087 {
8088 mImpl->setMinTimingIterations(minTiming);
8089 }
8090
8098 virtual int32_t getMinTimingIterations() const noexcept
8099 {
8100 return mImpl->getMinTimingIterations();
8101 }
8102
8111 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept
8112 {
8113 mImpl->setAvgTimingIterations(avgTiming);
8114 }
8115
8123 int32_t getAvgTimingIterations() const noexcept
8124 {
8125 return mImpl->getAvgTimingIterations();
8126 }
8127
8136 void setEngineCapability(EngineCapability capability) noexcept
8137 {
8138 mImpl->setEngineCapability(capability);
8139 }
8140
8149 {
8150 return mImpl->getEngineCapability();
8151 }
8152
8158 void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept
8159 {
8160 mImpl->setInt8Calibrator(calibrator);
8161 }
8162
8167 {
8168 return mImpl->getInt8Calibrator();
8169 }
8170
8181 TRT_DEPRECATED void setMaxWorkspaceSize(std::size_t workspaceSize) noexcept
8182 {
8183 mImpl->setMaxWorkspaceSize(workspaceSize);
8184 }
8185
8198 TRT_DEPRECATED std::size_t getMaxWorkspaceSize() const noexcept
8199 {
8200 return mImpl->getMaxWorkspaceSize();
8201 }
8202
8215 void setFlags(BuilderFlags builderFlags) noexcept
8216 {
8217 mImpl->setFlags(builderFlags);
8218 }
8219
8227 BuilderFlags getFlags() const noexcept
8228 {
8229 return mImpl->getFlags();
8230 }
8231
8239 void clearFlag(BuilderFlag builderFlag) noexcept
8240 {
8241 mImpl->clearFlag(builderFlag);
8242 }
8243
8251 void setFlag(BuilderFlag builderFlag) noexcept
8252 {
8253 mImpl->setFlag(builderFlag);
8254 }
8255
8263 bool getFlag(BuilderFlag builderFlag) const noexcept
8264 {
8265 return mImpl->getFlag(builderFlag);
8266 }
8267
8278 void setDeviceType(const ILayer* layer, DeviceType deviceType) noexcept
8279 {
8280 mImpl->setDeviceType(layer, deviceType);
8281 }
8282
8287 DeviceType getDeviceType(const ILayer* layer) const noexcept
8288 {
8289 return mImpl->getDeviceType(layer);
8290 }
8291
8297 bool isDeviceTypeSet(const ILayer* layer) const noexcept
8298 {
8299 return mImpl->isDeviceTypeSet(layer);
8300 }
8301
8307 void resetDeviceType(const ILayer* layer) noexcept
8308 {
8309 mImpl->resetDeviceType(layer);
8310 }
8311
8316 bool canRunOnDLA(const ILayer* layer) const noexcept
8317 {
8318 return mImpl->canRunOnDLA(layer);
8319 }
8320
8331 void setDLACore(int32_t dlaCore) noexcept
8332 {
8333 mImpl->setDLACore(dlaCore);
8334 }
8335
8342 int32_t getDLACore() const noexcept
8343 {
8344 return mImpl->getDLACore();
8345 }
8346
8352 void setDefaultDeviceType(DeviceType deviceType) noexcept
8353 {
8354 mImpl->setDefaultDeviceType(deviceType);
8355 }
8356
8363 {
8364 return mImpl->getDefaultDeviceType();
8365 }
8366
8372 void reset() noexcept
8373 {
8374 mImpl->reset();
8375 }
8376
8386 TRT_DEPRECATED void destroy() noexcept
8387 {
8388 delete this;
8389 }
8390
8398 void setProfileStream(const cudaStream_t stream) noexcept
8399 {
8400 return mImpl->setProfileStream(stream);
8401 }
8402
8410 cudaStream_t getProfileStream() const noexcept
8411 {
8412 return mImpl->getProfileStream();
8413 }
8414
8426 int32_t addOptimizationProfile(const IOptimizationProfile* profile) noexcept
8427 {
8428 return mImpl->addOptimizationProfile(profile);
8429 }
8430
8439 int32_t getNbOptimizationProfiles() const noexcept
8440 {
8441 return mImpl->getNbOptimizationProfiles();
8442 }
8443
8452 {
8453 mImpl->setProfilingVerbosity(verbosity);
8454 }
8455
8465 {
8466 return mImpl->getProfilingVerbosity();
8467 }
8468
8474 {
8475 mImpl->setAlgorithmSelector(selector);
8476 }
8477
8482 {
8483 return mImpl->getAlgorithmSelector();
8484 }
8485
8496 bool setCalibrationProfile(const IOptimizationProfile* profile) noexcept
8497 {
8498 return mImpl->setCalibrationProfile(profile);
8499 }
8500
8507 {
8508 return mImpl->getCalibrationProfile();
8509 }
8510
8524 {
8525 mImpl->setQuantizationFlags(flags);
8526 }
8527
8536 {
8537 return mImpl->getQuantizationFlags();
8538 }
8539
8548 {
8549 mImpl->clearQuantizationFlag(flag);
8550 }
8551
8560 {
8561 mImpl->setQuantizationFlag(flag);
8562 }
8563
8571 bool getQuantizationFlag(QuantizationFlag flag) const noexcept
8572 {
8573 return mImpl->getQuantizationFlag(flag);
8574 }
8575
8596 bool setTacticSources(TacticSources tacticSources) noexcept
8597 {
8598 return mImpl->setTacticSources(tacticSources);
8599 }
8600
8612 {
8613 return mImpl->getTacticSources();
8614 }
8615
8630 nvinfer1::ITimingCache* createTimingCache(const void* blob, std::size_t size) const noexcept
8631 {
8632 return mImpl->createTimingCache(blob, size);
8633 }
8634
8653 bool setTimingCache(const ITimingCache& cache, bool ignoreMismatch) noexcept
8654 {
8655 return mImpl->setTimingCache(cache, ignoreMismatch);
8656 }
8657
8664 {
8665 return mImpl->getTimingCache();
8666 }
8667
8695 void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
8696 {
8697 mImpl->setMemoryPoolLimit(pool, poolSize);
8698 }
8699
8714 std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
8715 {
8716 return mImpl->getMemoryPoolLimit(pool);
8717 }
8718
8719protected:
8720 apiv::VBuilderConfig* mImpl;
8721};
8722
8730
8741{
8747 kEXPLICIT_BATCH = 0,
8748
8762};
8763
8769template <>
8770constexpr inline int32_t EnumMax<NetworkDefinitionCreationFlag>() noexcept
8771{
8772 return 2;
8773}
8774
8782class IBuilder : public INoCopy
8783{
8784public:
8785 virtual ~IBuilder() noexcept = default;
8786
8795 void setMaxBatchSize(int32_t batchSize) noexcept
8796 {
8797 mImpl->setMaxBatchSize(batchSize);
8798 }
8799
8808 int32_t getMaxBatchSize() const noexcept
8809 {
8810 return mImpl->getMaxBatchSize();
8811 }
8812
8816 bool platformHasFastFp16() const noexcept
8817 {
8818 return mImpl->platformHasFastFp16();
8819 }
8820
8824 bool platformHasFastInt8() const noexcept
8825 {
8826 return mImpl->platformHasFastInt8();
8827 }
8828
8836 TRT_DEPRECATED void destroy() noexcept
8837 {
8838 delete this;
8839 }
8840
8848 int32_t getMaxDLABatchSize() const noexcept
8849 {
8850 return mImpl->getMaxDLABatchSize();
8851 }
8852
8856 int32_t getNbDLACores() const noexcept
8857 {
8858 return mImpl->getNbDLACores();
8859 }
8860
8872 void setGpuAllocator(IGpuAllocator* allocator) noexcept
8873 {
8874 mImpl->setGpuAllocator(allocator);
8875 }
8876
8883 {
8884 return mImpl->createBuilderConfig();
8885 }
8886
8898 INetworkDefinition& network, IBuilderConfig& config) noexcept
8899 {
8900 return mImpl->buildEngineWithConfig(network, config);
8901 }
8902
8915 {
8916 return mImpl->createNetworkV2(flags);
8917 }
8918
8929 {
8930 return mImpl->createOptimizationProfile();
8931 }
8932
8944 //
8947 void setErrorRecorder(IErrorRecorder* recorder) noexcept
8948 {
8949 mImpl->setErrorRecorder(recorder);
8950 }
8951
8963 {
8964 return mImpl->getErrorRecorder();
8965 }
8966
8970 void reset() noexcept
8971 {
8972 mImpl->reset();
8973 }
8974
8978 bool platformHasTf32() const noexcept
8979 {
8980 return mImpl->platformHasTf32();
8981 }
8982
8998 {
8999 return mImpl->buildSerializedNetwork(network, config);
9000 }
9001
9021 bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept
9022 {
9023 return mImpl->isNetworkSupported(network, config);
9024 }
9025
9031 ILogger* getLogger() const noexcept
9032 {
9033 return mImpl->getLogger();
9034 }
9035
9045 bool setMaxThreads(int32_t maxThreads) noexcept
9046 {
9047 return mImpl->setMaxThreads(maxThreads);
9048 }
9049
9059 int32_t getMaxThreads() const noexcept
9060 {
9061 return mImpl->getMaxThreads();
9062 }
9063
9064protected:
9065 apiv::VBuilder* mImpl;
9066};
9067
9068} // namespace nvinfer1
9069
9074extern "C" TENSORRTAPI void* createInferBuilder_INTERNAL(void* logger, int32_t version) noexcept;
9075
9076namespace nvinfer1
9077{
9078namespace
9079{
9080
9088inline IBuilder* createInferBuilder(ILogger& logger) noexcept
9089{
9090 return static_cast<IBuilder*>(createInferBuilder_INTERNAL(&logger, NV_TENSORRT_VERSION));
9091}
9092
9093} // namespace
9094
9106 nvinfer1::EngineCapability capability) noexcept;
9107
9108} // namespace nvinfer1
9109
9110#endif // NV_INFER_H
#define TENSORRTAPI
Definition: NvInferRuntimeCommon.h:54
#define NV_TENSORRT_VERSION
Definition: NvInferRuntimeCommon.h:73
#define TRT_DEPRECATED
Definition: NvInferRuntimeCommon.h:40
#define TRT_DEPRECATED_ENUM
Definition: NvInferRuntimeCommon.h:41
Definition: NvInferRuntimeCommon.h:153
static constexpr int32_t MAX_DIMS
The maximum rank (number of dimensions) supported for a tensor.
Definition: NvInferRuntimeCommon.h:156
Descriptor for two-dimensional spatial data.
Definition: NvInferLegacyDims.h:64
An Activation layer in a network definition.
Definition: NvInfer.h:1552
void setBeta(float beta) noexcept
Set the beta parameter (must be finite).
Definition: NvInfer.h:1600
void setActivationType(ActivationType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1561
ActivationType getActivationType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1571
float getAlpha() const noexcept
Get the alpha parameter.
Definition: NvInfer.h:1609
virtual ~IActivationLayer() noexcept=default
float getBeta() const noexcept
Get the beta parameter.
Definition: NvInfer.h:1618
void setAlpha(float alpha) noexcept
Set the alpha parameter (must be finite).
Definition: NvInfer.h:1586
Describes the context and requirements, that could be fulfilled by one or more instances of IAlgorith...
Definition: NvInfer.h:7674
const char * getName() const noexcept
Return name of the algorithm node. This is a unique identifier for the IAlgorithmContext.
Definition: NvInfer.h:7680
int32_t getNbOutputs() const noexcept
Return number of outputs of the algorithm.
Definition: NvInfer.h:7707
int32_t getNbInputs() const noexcept
Return number of inputs of the algorithm.
Definition: NvInfer.h:7699
virtual ~IAlgorithmContext() noexcept=default
Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept
Get the minimum / optimum / maximum dimensions for input or output tensor.
Definition: NvInfer.h:7691
Describes a variation of execution of a layer. An algorithm is represented by IAlgorithmVariant and t...
Definition: NvInfer.h:7727
std::size_t getWorkspaceSize() const noexcept
The size of the GPU temporary memory in bytes which the algorithm uses at execution time.
Definition: NvInfer.h:7763
float getTimingMSec() const noexcept
The time in milliseconds to execute the algorithm.
Definition: NvInfer.h:7755
virtual ~IAlgorithm() noexcept=default
TRT_DEPRECATED const IAlgorithmIOInfo & getAlgorithmIOInfo(int32_t index) const noexcept
Returns the format of an Algorithm input or output. Algorithm inputs are incrementally numbered first...
Definition: NvInfer.h:7739
const IAlgorithmVariant & getAlgorithmVariant() const noexcept
Returns the algorithm variant.
Definition: NvInfer.h:7747
const IAlgorithmIOInfo * getAlgorithmIOInfoByIndex(int32_t index) const noexcept
Returns the format of an Algorithm input or output. Algorithm inputs are incrementally numbered first...
Definition: NvInfer.h:7776
Carries information about input or output of the algorithm. IAlgorithmIOInfo for all the input and ou...
Definition: NvInfer.h:7599
virtual ~IAlgorithmIOInfo() noexcept=default
Dims getStrides() const noexcept
Return strides of the input/output tensor of algorithm.
Definition: NvInfer.h:7620
DataType getDataType() const noexcept
Return DataType of the input/output of algorithm.
Definition: NvInfer.h:7612
TensorFormat getTensorFormat() const noexcept
Return TensorFormat of the input/output of algorithm.
Definition: NvInfer.h:7604
Interface implemented by application for selecting and reporting algorithms of a layer provided by th...
Definition: NvInfer.h:7795
virtual int32_t selectAlgorithms(const IAlgorithmContext &context, const IAlgorithm *const *choices, int32_t nbChoices, int32_t *selection) noexcept=0
Select Algorithms for a layer from the given list of algorithm choices.
virtual void reportAlgorithms(const IAlgorithmContext *const *algoContexts, const IAlgorithm *const *algoChoices, int32_t nbAlgorithms) noexcept=0
Called by TensorRT to report choices it made.
virtual ~IAlgorithmSelector() noexcept=default
provides a unique 128-bit identifier, which along with the input and output information denotes the v...
Definition: NvInfer.h:7642
virtual ~IAlgorithmVariant() noexcept=default
int64_t getTactic() const noexcept
Return tactic of the algorithm.
Definition: NvInfer.h:7655
int64_t getImplementation() const noexcept
Return implementation of the algorithm.
Definition: NvInfer.h:7647
An assertion layer in a network.
Definition: NvInfer.h:5530
void setMessage(const char *message) noexcept
Set the message to print if the assertion fails.
Definition: NvInfer.h:5540
virtual ~IAssertionLayer() noexcept=default
const char * getMessage() const noexcept
Return the assertion message.
Definition: NvInfer.h:5550
Holds properties for configuring a builder to produce an engine.
Definition: NvInfer.h:8072
DeviceType getDeviceType(const ILayer *layer) const noexcept
Get the device that this layer executes on.
Definition: NvInfer.h:8287
void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept
Set the memory size for the memory pool.
Definition: NvInfer.h:8695
void setQuantizationFlag(QuantizationFlag flag) noexcept
Set a single quantization flag.
Definition: NvInfer.h:8559
const IOptimizationProfile * getCalibrationProfile() noexcept
Get the current calibration profile.
Definition: NvInfer.h:8506
void setInt8Calibrator(IInt8Calibrator *calibrator) noexcept
Set Int8 Calibration interface.
Definition: NvInfer.h:8158
void clearQuantizationFlag(QuantizationFlag flag) noexcept
clear a quantization flag.
Definition: NvInfer.h:8547
void setDeviceType(const ILayer *layer, DeviceType deviceType) noexcept
Set the device that this layer must execute on.
Definition: NvInfer.h:8278
bool setTacticSources(TacticSources tacticSources) noexcept
Set tactic sources.
Definition: NvInfer.h:8596
bool getQuantizationFlag(QuantizationFlag flag) const noexcept
Returns true if the quantization flag is set.
Definition: NvInfer.h:8571
std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept
Get the memory size limit of the memory pool.
Definition: NvInfer.h:8714
int32_t getDLACore() const noexcept
Get the DLA core that the engine executes on.
Definition: NvInfer.h:8342
void setEngineCapability(EngineCapability capability) noexcept
Configure the builder to target specified EngineCapability flow.
Definition: NvInfer.h:8136
bool getFlag(BuilderFlag builderFlag) const noexcept
Returns true if the build mode flag is set.
Definition: NvInfer.h:8263
void setQuantizationFlags(QuantizationFlags flags) noexcept
Set the quantization flags.
Definition: NvInfer.h:8523
virtual void setAvgTimingIterations(int32_t avgTiming) noexcept
Set the number of averaging iterations used when timing layers.
Definition: NvInfer.h:8111
void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept
Set verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:8451
nvinfer1::ITimingCache * createTimingCache(const void *blob, std::size_t size) const noexcept
Create timing cache.
Definition: NvInfer.h:8630
IAlgorithmSelector * getAlgorithmSelector() const noexcept
Get Algorithm Selector.
Definition: NvInfer.h:8481
int32_t getNbOptimizationProfiles() const noexcept
Get number of optimization profiles.
Definition: NvInfer.h:8439
QuantizationFlags getQuantizationFlags() const noexcept
Get the quantization flags.
Definition: NvInfer.h:8535
bool setCalibrationProfile(const IOptimizationProfile *profile) noexcept
Add a calibration profile.
Definition: NvInfer.h:8496
virtual int32_t getMinTimingIterations() const noexcept
Query the number of minimization iterations.
Definition: NvInfer.h:8098
void reset() noexcept
Resets the builder configuration to defaults.
Definition: NvInfer.h:8372
TRT_DEPRECATED void destroy() noexcept
Delete this IBuilderConfig.
Definition: NvInfer.h:8386
EngineCapability getEngineCapability() const noexcept
Query EngineCapability flow configured for the builder.
Definition: NvInfer.h:8148
void setAlgorithmSelector(IAlgorithmSelector *selector) noexcept
Set Algorithm Selector.
Definition: NvInfer.h:8473
TRT_DEPRECATED void setMaxWorkspaceSize(std::size_t workspaceSize) noexcept
Set the maximum workspace size.
Definition: NvInfer.h:8181
TRT_DEPRECATED std::size_t getMaxWorkspaceSize() const noexcept
Get the maximum workspace size.
Definition: NvInfer.h:8198
DeviceType getDefaultDeviceType() const noexcept
Get the default DeviceType which was set by setDefaultDeviceType.
Definition: NvInfer.h:8362
BuilderFlags getFlags() const noexcept
Get the build mode flags for this builder config. Defaults to 0.
Definition: NvInfer.h:8227
void setFlags(BuilderFlags builderFlags) noexcept
Set the build mode flags to turn on builder options for this network.
Definition: NvInfer.h:8215
TacticSources getTacticSources() const noexcept
Get tactic sources.
Definition: NvInfer.h:8611
bool setTimingCache(const ITimingCache &cache, bool ignoreMismatch) noexcept
Attach a timing cache to IBuilderConfig.
Definition: NvInfer.h:8653
bool canRunOnDLA(const ILayer *layer) const noexcept
Checks if a layer can run on DLA.
Definition: NvInfer.h:8316
void setDLACore(int32_t dlaCore) noexcept
Sets the DLA core used by the network.
Definition: NvInfer.h:8331
int32_t addOptimizationProfile(const IOptimizationProfile *profile) noexcept
Add an optimization profile.
Definition: NvInfer.h:8426
const nvinfer1::ITimingCache * getTimingCache() const noexcept
Get the pointer to the timing cache from current IBuilderConfig.
Definition: NvInfer.h:8663
bool isDeviceTypeSet(const ILayer *layer) const noexcept
whether the DeviceType has been explicitly set for this layer
Definition: NvInfer.h:8297
void clearFlag(BuilderFlag builderFlag) noexcept
clear a single build mode flag.
Definition: NvInfer.h:8239
apiv::VBuilderConfig * mImpl
Definition: NvInfer.h:8720
int32_t getAvgTimingIterations() const noexcept
Query the number of averaging iterations.
Definition: NvInfer.h:8123
void setDefaultDeviceType(DeviceType deviceType) noexcept
Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on...
Definition: NvInfer.h:8352
void setFlag(BuilderFlag builderFlag) noexcept
Set a single build mode flag.
Definition: NvInfer.h:8251
virtual ~IBuilderConfig() noexcept=default
cudaStream_t getProfileStream() const noexcept
Get the cuda stream that is used to profile this network.
Definition: NvInfer.h:8410
IInt8Calibrator * getInt8Calibrator() const noexcept
Get Int8 Calibration interface.
Definition: NvInfer.h:8166
ProfilingVerbosity getProfilingVerbosity() const noexcept
Get verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Definition: NvInfer.h:8464
void resetDeviceType(const ILayer *layer) noexcept
reset the DeviceType for this layer
Definition: NvInfer.h:8307
void setProfileStream(const cudaStream_t stream) noexcept
Set the cuda stream that is used to profile this network.
Definition: NvInfer.h:8398
Builds an engine from a network definition.
Definition: NvInfer.h:8783
int32_t getMaxDLABatchSize() const noexcept
Get the maximum batch size DLA can support. For any tensor the total volume of index dimensions combi...
Definition: NvInfer.h:8848
int32_t getNbDLACores() const noexcept
Return the number of DLA engines available to this builder.
Definition: NvInfer.h:8856
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:8962
apiv::VBuilder * mImpl
Definition: NvInfer.h:9065
ILogger * getLogger() const noexcept
get the logger with which the builder was created
Definition: NvInfer.h:9031
bool isNetworkSupported(INetworkDefinition const &network, IBuilderConfig const &config) const noexcept
Checks that a network is within the scope of the IBuilderConfig settings.
Definition: NvInfer.h:9021
bool platformHasTf32() const noexcept
Determine whether the platform has TF32 support.
Definition: NvInfer.h:8978
int32_t getMaxThreads() const noexcept
get the maximum number of threads that can be used by the builder.
Definition: NvInfer.h:9059
TRT_DEPRECATED void destroy() noexcept
Destroy this object.
Definition: NvInfer.h:8836
int32_t getMaxBatchSize() const noexcept
Get the maximum batch size.
Definition: NvInfer.h:8808
nvinfer1::IOptimizationProfile * createOptimizationProfile() noexcept
Create a new optimization profile.
Definition: NvInfer.h:8928
bool platformHasFastFp16() const noexcept
Determine whether the platform has fast native fp16.
Definition: NvInfer.h:8816
void setGpuAllocator(IGpuAllocator *allocator) noexcept
Set the GPU allocator.
Definition: NvInfer.h:8872
nvinfer1::INetworkDefinition * createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept
Create a network definition object.
Definition: NvInfer.h:8914
nvinfer1::IBuilderConfig * createBuilderConfig() noexcept
Create a builder configuration object.
Definition: NvInfer.h:8882
void reset() noexcept
Resets the builder state to default values.
Definition: NvInfer.h:8970
bool setMaxThreads(int32_t maxThreads) noexcept
Set the maximum number of threads.
Definition: NvInfer.h:9045
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:8947
bool platformHasFastInt8() const noexcept
Determine whether the platform has fast native int8.
Definition: NvInfer.h:8824
nvinfer1::IHostMemory * buildSerializedNetwork(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds and serializes a network for the given INetworkDefinition and IBuilderConfig.
Definition: NvInfer.h:8997
virtual ~IBuilder() noexcept=default
TRT_DEPRECATED nvinfer1::ICudaEngine * buildEngineWithConfig(INetworkDefinition &network, IBuilderConfig &config) noexcept
Builds an engine for the given INetworkDefinition and given IBuilderConfig.
Definition: NvInfer.h:8897
A concatenation layer in a network definition.
Definition: NvInfer.h:2355
void setAxis(int32_t axis) noexcept
Set the axis along which concatenation occurs.
Definition: NvInfer.h:2367
int32_t getAxis() const noexcept
Get the axis along which concatenation occurs.
Definition: NvInfer.h:2377
virtual ~IConcatenationLayer() noexcept=default
Definition: NvInfer.h:5134
virtual ~IConditionLayer() noexcept=default
Layer that represents a constant value.
Definition: NvInfer.h:4566
void setWeights(Weights weights) noexcept
Set the weights for the layer.
Definition: NvInfer.h:4577
Weights getWeights() const noexcept
Get the weights for the layer.
Definition: NvInfer.h:4587
apiv::VConstantLayer * mImpl
Definition: NvInfer.h:4617
void setDimensions(Dims dimensions) noexcept
Set the dimensions for the layer.
Definition: NvInfer.h:4599
virtual ~IConstantLayer() noexcept=default
Dims getDimensions() const noexcept
Get the dimensions for the layer.
Definition: NvInfer.h:4611
A convolution layer in a network definition.
Definition: NvInfer.h:987
TRT_DEPRECATED DimsHW getPadding() const noexcept
Get the padding of the convolution. If the padding is asymmetric, the pre-padding is returned.
Definition: NvInfer.h:1090
TRT_DEPRECATED DimsHW getStride() const noexcept
Get the stride of the convolution.
Definition: NvInfer.h:1058
TRT_DEPRECATED DimsHW getDilation() const noexcept
Get the dilation for a convolution.
Definition: NvInfer.h:1197
void setStrideNd(Dims stride) noexcept
Set the multi-dimension stride of the convolution.
Definition: NvInfer.h:1315
void setKernelSizeNd(Dims kernelSize) noexcept
Set the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1290
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1224
Weights getBiasWeights() const noexcept
Get the bias weights for the convolution.
Definition: NvInfer.h:1169
int32_t getNbGroups() const noexcept
Get the number of groups of the convolution.
Definition: NvInfer.h:1120
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1265
TRT_DEPRECATED void setStride(DimsHW stride) noexcept
Get the stride of the convolution.
Definition: NvInfer.h:1048
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the convolution.
Definition: NvInfer.h:1355
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the convolution.
Definition: NvInfer.h:1325
TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
Set the padding of the convolution.
Definition: NvInfer.h:1078
Weights getKernelWeights() const noexcept
Get the kernel weights of the convolution.
Definition: NvInfer.h:1144
void setPaddingNd(Dims padding) noexcept
Set the multi-dimension padding of the convolution.
Definition: NvInfer.h:1343
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1379
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the convolution.
Definition: NvInfer.h:1134
void setDilationNd(Dims dilation) noexcept
Set the multi-dimension dilation of the convolution.
Definition: NvInfer.h:1369
Dims getPostPadding() const noexcept
Get the post-padding.
Definition: NvInfer.h:1251
TRT_DEPRECATED void setDilation(DimsHW dilation) noexcept
Set the dilation for a convolution.
Definition: NvInfer.h:1185
void setNbGroups(int32_t nbGroups) noexcept
Set the number of groups for a convolution.
Definition: NvInfer.h:1110
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1277
int32_t getNbOutputMaps() const noexcept
Get the number of output maps for the convolution.
Definition: NvInfer.h:1032
void setPrePadding(Dims padding) noexcept
Set the multi-dimension pre-padding of the convolution.
Definition: NvInfer.h:1214
virtual ~IConvolutionLayer() noexcept=default
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the convolution.
Definition: NvInfer.h:1159
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the convolution.
Definition: NvInfer.h:1300
void setNbOutputMaps(int32_t nbOutputMaps) noexcept
Set the number of output maps for the convolution.
Definition: NvInfer.h:1022
TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
Set the HW kernel size of the convolution.
Definition: NvInfer.h:998
void setPostPadding(Dims padding) noexcept
Set the multi-dimension post-padding of the convolution.
Definition: NvInfer.h:1241
TRT_DEPRECATED DimsHW getKernelSize() const noexcept
Get the HW kernel size of the convolution.
Definition: NvInfer.h:1010
An engine for executing inference on a built network, with functionally unsafe features.
Definition: NvInferRuntime.h:1342
A deconvolution layer in a network definition.
Definition: NvInfer.h:2395
void setBiasWeights(Weights weights) noexcept
Set the bias weights for the deconvolution.
Definition: NvInfer.h:2573
TRT_DEPRECATED void setStride(DimsHW stride) noexcept
Set the stride of the deconvolution.
Definition: NvInfer.h:2458
void setNbOutputMaps(int32_t nbOutputMaps) noexcept
Set the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2432
Weights getKernelWeights() const noexcept
Get the kernel weights for the deconvolution.
Definition: NvInfer.h:2558
void setPrePadding(Dims padding) noexcept
Set the multi-dimension pre-padding of the deconvolution.
Definition: NvInfer.h:2601
TRT_DEPRECATED DimsHW getPadding() const noexcept
Get the padding of the deconvolution.
Definition: NvInfer.h:2504
Dims getStrideNd() const noexcept
Get the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2717
int32_t getNbGroups() const noexcept
Get the number of groups for a deconvolution.
Definition: NvInfer.h:2534
Dims getDilationNd() const noexcept
Get the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2791
Weights getBiasWeights() const noexcept
Get the bias weights for the deconvolution.
Definition: NvInfer.h:2583
void setKernelWeights(Weights weights) noexcept
Set the kernel weights for the deconvolution.
Definition: NvInfer.h:2548
void setDilationNd(Dims dilation) noexcept
Set the multi-dimension dilation of the deconvolution.
Definition: NvInfer.h:2781
TRT_DEPRECATED DimsHW getStride() const noexcept
Get the stride of the deconvolution.
Definition: NvInfer.h:2470
TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
Set the padding of the deconvolution.
Definition: NvInfer.h:2490
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:2639
Dims getKernelSizeNd() const noexcept
Get the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2690
void setKernelSizeNd(Dims kernelSize) noexcept
Set the multi-dimension kernel size of the deconvolution.
Definition: NvInfer.h:2680
virtual ~IDeconvolutionLayer() noexcept=default
TRT_DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept
Set the HW kernel size of the convolution.
Definition: NvInfer.h:2408
void setStrideNd(Dims stride) noexcept
Set the multi-dimension stride of the deconvolution.
Definition: NvInfer.h:2707
Dims getPaddingNd() const noexcept
Get the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2747
void setNbGroups(int32_t nbGroups) noexcept
Set the number of groups for a deconvolution.
Definition: NvInfer.h:2524
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:2653
void setPaddingNd(Dims padding) noexcept
Set the multi-dimension padding of the deconvolution.
Definition: NvInfer.h:2735
TRT_DEPRECATED DimsHW getKernelSize() const noexcept
Get the HW kernel size of the deconvolution.
Definition: NvInfer.h:2420
void setPostPadding(Dims padding) noexcept
Set the multi-dimension post-padding of the deconvolution.
Definition: NvInfer.h:2629
int32_t getNbOutputMaps() const noexcept
Get the number of output feature maps for the deconvolution.
Definition: NvInfer.h:2442
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:2611
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:2665
A Dequantize layer in a network definition.
Definition: NvInfer.h:5906
virtual ~IDequantizeLayer() noexcept=default
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5916
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5927
An Einsum layer in a network.
Definition: NvInfer.h:5973
virtual ~IEinsumLayer() noexcept=default
bool setEquation(const char *equation) noexcept
Set the equation. The equation is a comma-separated list of subscript labels, where each label refers...
Definition: NvInfer.h:5984
const char * getEquation() const noexcept
Return the equation.
Definition: NvInfer.h:5994
A elementwise layer in a network definition.
Definition: NvInfer.h:2866
virtual ~IElementWiseLayer() noexcept=default
apiv::VElementWiseLayer * mImpl
Definition: NvInfer.h:2895
ElementWiseOperation getOperation() const noexcept
Get the binary operation for the layer.
Definition: NvInfer.h:2889
void setOperation(ElementWiseOperation op) noexcept
Set the binary operation for the layer.
Definition: NvInfer.h:2877
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1661
Generate an output tensor with specified mode.
Definition: NvInfer.h:5611
FillOperation getOperation() const noexcept
Get the fill operation for the layer.
Definition: NvInfer.h:5657
void setOperation(FillOperation op) noexcept
Set the fill operation for the layer.
Definition: NvInfer.h:5647
void setDimensions(Dims dimensions) noexcept
Set the output tensor's dimensions.
Definition: NvInfer.h:5622
void setBeta(double beta) noexcept
Set the beta parameter.
Definition: NvInfer.h:5708
double getAlpha() const noexcept
Get the value of alpha parameter.
Definition: NvInfer.h:5690
void setAlpha(double alpha) noexcept
Set the alpha parameter.
Definition: NvInfer.h:5675
Dims getDimensions() const noexcept
Get the output tensor's dimensions.
Definition: NvInfer.h:5637
double getBeta() const noexcept
Get the value of beta parameter.
Definition: NvInfer.h:5723
virtual ~IFillLayer() noexcept=default
A fully connected layer in a network definition. This layer expects an input tensor of three or more ...
Definition: NvInfer.h:1444
virtual ~IFullyConnectedLayer() noexcept=default
void setNbOutputChannels(int32_t nbOutputs) noexcept
Set the number of output channels K from the fully connected layer.
Definition: NvInfer.h:1453
Weights getKernelWeights() const noexcept
Get the kernel weights.
Definition: NvInfer.h:1483
void setBiasWeights(Weights weights) noexcept
Set the bias weights.
Definition: NvInfer.h:1495
void setKernelWeights(Weights weights) noexcept
Set the kernel weights, given as a KxC matrix in row-major order.
Definition: NvInfer.h:1473
int32_t getNbOutputChannels() const noexcept
Get the number of output channels K from the fully connected layer.
Definition: NvInfer.h:1463
Weights getBiasWeights() const noexcept
Get the bias weights.
Definition: NvInfer.h:1505
A Gather layer in a network definition. Supports several kinds of gathering.
Definition: NvInfer.h:3001
void setGatherAxis(int32_t axis) noexcept
Set the axis used by GatherMode::kELEMENTS and GatherMode::kDEFAULT The axis must be less than the nu...
Definition: NvInfer.h:3012
void setNbElementWiseDims(int32_t elementWiseDims) noexcept
Set the number of leading dimensions of indices tensor to be handled elementwise. The gathering of in...
Definition: NvInfer.h:3044
apiv::VGatherLayer * mImpl
Definition: NvInfer.h:3080
int32_t getNbElementWiseDims() const noexcept
Get the number of leading dimensions of indices tensor to be handled elementwise.
Definition: NvInfer.h:3054
void setMode(GatherMode mode) noexcept
Set the gather mode.
Definition: NvInfer.h:3064
int32_t getGatherAxis() const noexcept
Get the axis to gather on.
Definition: NvInfer.h:3023
GatherMode getMode() const noexcept
Get the gather mode.
Definition: NvInfer.h:3074
virtual ~IGatherLayer() noexcept=default
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:1334
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:144
A layer that represents the identity function.
Definition: NvInfer.h:4551
apiv::VIdentityLayer * mImpl
Definition: NvInfer.h:4553
virtual ~IIdentityLayer() noexcept=default
Definition: NvInfer.h:5117
IIfConditional * getConditional() const noexcept
Return pointer to the IIfConditional associated with this boundary layer.
Definition: NvInfer.h:5120
virtual ~IIfConditionalBoundaryLayer() noexcept=default
Definition: NvInfer.h:5187
IIfConditionalInputLayer * addInput(ITensor &input) noexcept
Add an If-conditional input.
Definition: NvInfer.h:5226
virtual ~IIfConditional() noexcept=default
void setName(const char *name) noexcept
Set the name of the conditional.
Definition: NvInfer.h:5239
IConditionLayer * setCondition(ITensor &condition) noexcept
Set the condition tensor for this If-Conditional construct.
Definition: NvInfer.h:5198
IIfConditionalOutputLayer * addOutput(ITensor &trueSubgraphOutput, ITensor &falseSubgraphOutput) noexcept
Add an If-conditional output.
Definition: NvInfer.h:5214
const char * getName() const noexcept
Return the name of the conditional.
Definition: NvInfer.h:5249
Definition: NvInfer.h:5158
virtual ~IIfConditionalInputLayer() noexcept=default
Definition: NvInfer.h:5147
virtual ~IIfConditionalOutputLayer() noexcept=default
Application-implemented interface for calibration.
Definition: NvInfer.h:7417
virtual int32_t getBatchSize() const noexcept=0
Get the batch size used for calibration batches.
Definition: NvInfer.h:7500
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7505
virtual ~IInt8EntropyCalibrator2() noexcept=default
Definition: NvInfer.h:7482
virtual ~IInt8EntropyCalibrator() noexcept=default
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7487
Definition: NvInfer.h:7535
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7540
virtual double getQuantile() const noexcept=0
The quantile (between 0 and 1) that will be used to select the region maximum when the quantile metho...
Definition: NvInfer.h:7517
CalibrationAlgoType getAlgorithm() noexcept override
Definition: NvInfer.h:7522
virtual ~IInt8MinMaxCalibrator() noexcept=default
Definition: NvInfer.h:5377
virtual ~IIteratorLayer() noexcept=default
void setReverse(bool reverse) noexcept
Definition: NvInfer.h:5396
bool getReverse() const noexcept
True if and only if reversing input.
Definition: NvInfer.h:5402
int32_t getAxis() const noexcept
Get axis being iterated over.
Definition: NvInfer.h:5386
void setAxis(int32_t axis) noexcept
Set axis to iterate over.
Definition: NvInfer.h:5380
A LRN layer in a network definition.
Definition: NvInfer.h:2004
int32_t getWindowSize() const noexcept
Get the LRN window size.
Definition: NvInfer.h:2025
float getAlpha() const noexcept
Get the LRN alpha value.
Definition: NvInfer.h:2046
void setWindowSize(int32_t windowSize) noexcept
Set the LRN window size.
Definition: NvInfer.h:2015
void setK(float k) noexcept
Set the LRN K value.
Definition: NvInfer.h:2078
void setAlpha(float alpha) noexcept
Set the LRN alpha value.
Definition: NvInfer.h:2036
void setBeta(float beta) noexcept
Set the LRN beta value.
Definition: NvInfer.h:2057
virtual ~ILRNLayer() noexcept=default
float getBeta() const noexcept
Get the LRN beta value.
Definition: NvInfer.h:2067
float getK() const noexcept
Get the LRN K value.
Definition: NvInfer.h:2088
Base class for all layer classes in a network definition.
Definition: NvInfer.h:492
bool precisionIsSet() const noexcept
whether the computational precision has been set for this layer
Definition: NvInfer.h:631
void setPrecision(DataType dataType) noexcept
Set the computational precision of this layer.
Definition: NvInfer.h:607
void resetPrecision() noexcept
reset the computational precision for this layer
Definition: NvInfer.h:641
int32_t getNbInputs() const noexcept
Get the number of inputs of a layer.
Definition: NvInfer.h:530
DataType getOutputType(int32_t index) const noexcept
get the output type of this layer
Definition: NvInfer.h:693
DataType getPrecision() const noexcept
get the computational precision of this layer
Definition: NvInfer.h:619
int32_t getNbOutputs() const noexcept
Get the number of outputs of a layer.
Definition: NvInfer.h:551
void setName(const char *name) noexcept
Set the name of a layer.
Definition: NvInfer.h:511
bool outputTypeIsSet(int32_t index) const noexcept
whether the output type has been set for this layer
Definition: NvInfer.h:706
ITensor * getOutput(int32_t index) const noexcept
Get the layer output corresponding to the given index.
Definition: NvInfer.h:562
void setInput(int32_t index, ITensor &tensor) noexcept
Replace an input of this layer with a specific tensor.
Definition: NvInfer.h:579
void resetOutputType(int32_t index) noexcept
reset the output type for this layer
Definition: NvInfer.h:718
ITensor * getInput(int32_t index) const noexcept
Get the layer input corresponding to the given index.
Definition: NvInfer.h:543
void setOutputType(int32_t index, DataType dataType) noexcept
Set the output type of this layer.
Definition: NvInfer.h:679
LayerType getType() const noexcept
Return the type of a layer.
Definition: NvInfer.h:499
const char * getName() const noexcept
Return the name of a layer.
Definition: NvInfer.h:522
virtual ~ILayer() noexcept=default
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntimeCommon.h:1472
Definition: NvInfer.h:5098
virtual ~ILoopBoundaryLayer() noexcept=default
ILoop * getLoop() const noexcept
Return pointer to ILoop associated with this boundary layer.
Definition: NvInfer.h:5101
Definition: NvInfer.h:5418
ITripLimitLayer * addTripLimit(ITensor &tensor, TripLimit limit) noexcept
Add a trip-count limiter, based on the given tensor.
Definition: NvInfer.h:5447
IIteratorLayer * addIterator(ITensor &tensor, int32_t axis=0, bool reverse=false) noexcept
Return layer that subscripts tensor by loop iteration.
Definition: NvInfer.h:5460
ILoopOutputLayer * addLoopOutput(ITensor &tensor, LoopOutput outputKind, int32_t axis=0) noexcept
Make an output for this loop, based on the given tensor.
Definition: NvInfer.h:5472
virtual ~ILoop() noexcept=default
const char * getName() const noexcept
Return the name of the loop.
Definition: NvInfer.h:5495
void setName(const char *name) noexcept
Set the name of the loop.
Definition: NvInfer.h:5485
IRecurrenceLayer * addRecurrence(ITensor &initialValue) noexcept
Create a recurrence layer for this loop with initialValue as its first input.
Definition: NvInfer.h:5426
Definition: NvInfer.h:5306
virtual ~ILoopOutputLayer() noexcept=default
int32_t getAxis() const noexcept
Get axis being concatenated over.
Definition: NvInfer.h:5331
LoopOutput getLoopOutput() const noexcept
Definition: NvInfer.h:5308
void setAxis(int32_t axis) noexcept
Set where to insert the contenation axis. Ignored if getLoopOutput() is kLAST_VALUE.
Definition: NvInfer.h:5325
Layer that represents a Matrix Multiplication.
Definition: NvInfer.h:4476
apiv::VMatrixMultiplyLayer * mImpl
Definition: NvInfer.h:4502
virtual ~IMatrixMultiplyLayer() noexcept=default
MatrixOperation getOperation(int32_t index) const noexcept
Get the operation for an input tensor.
Definition: NvInfer.h:4496
void setOperation(int32_t index, MatrixOperation op) noexcept
Set the operation for an input tensor.
Definition: NvInfer.h:4484
A network definition for input to the builder.
Definition: NvInfer.h:6148
IPluginV2Layer * addPluginV2(ITensor *const *inputs, int32_t nbInputs, IPluginV2 &plugin) noexcept
Add a plugin layer to the network using the IPluginV2 interface.
Definition: NvInfer.h:6850
IConvolutionLayer * addConvolutionNd(ITensor &input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension convolution layer to the network.
Definition: NvInfer.h:7012
IDequantizeLayer * addDequantize(ITensor &input, ITensor &scale) noexcept
Add a dequantization layer to the network.
Definition: NvInfer.h:7304
IConcatenationLayer * addConcatenation(ITensor *const *inputs, int32_t nbInputs) noexcept
Add a concatenation layer to the network.
Definition: NvInfer.h:6362
TRT_DEPRECATED IDeconvolutionLayer * addDeconvolution(ITensor &input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a deconvolution layer to the network.
Definition: NvInfer.h:6385
IGatherLayer * addGatherV2(ITensor &data, ITensor &indices, GatherMode mode)
Add gather with specified mode, axis=0 and nbElementWiseDims=0.
Definition: NvInfer.h:6655
IShuffleLayer * addShuffle(ITensor &input) noexcept
Add a shuffle layer to the network.
Definition: NvInfer.h:6466
ILRNLayer * addLRN(ITensor &input, int32_t window, float alpha, float beta, float k) noexcept
Add a LRN layer to the network.
Definition: NvInfer.h:6305
ITopKLayer * addTopK(ITensor &input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept
Add a TopK layer to the network.
Definition: NvInfer.h:6623
IAssertionLayer * addAssertion(ITensor &condition, const char *message) noexcept
Add an assertion layer to the network.
Definition: NvInfer.h:7196
IScaleLayer * addScaleNd(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
Add a multi-dimension scale layer to the network.
Definition: NvInfer.h:7090
TRT_DEPRECATED bool hasExplicitPrecision() const noexcept
True if network is an explicit precision network.
Definition: NvInfer.h:7123
IParametricReLULayer * addParametricReLU(ITensor &input, ITensor &slope) noexcept
Add a parametric ReLU layer to the network.
Definition: NvInfer.h:6990
ITensor * getOutput(int32_t index) const noexcept
Get the output tensor specified by the given index.
Definition: NvInfer.h:6550
ITensor * getInput(int32_t index) const noexcept
Get the input tensor specified by the given index.
Definition: NvInfer.h:6520
bool unmarkOutputForShapes(ITensor &tensor) noexcept
Undo markOutputForShapes.
Definition: NvInfer.h:6972
IFillLayer * addFill(Dims dimensions, FillOperation op) noexcept
Add a fill layer to the network.
Definition: NvInfer.h:7214
ILoop * addLoop() noexcept
Add a loop to the network.
Definition: NvInfer.h:7139
IDeconvolutionLayer * addDeconvolutionNd(ITensor &input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a multi-dimension deconvolution layer to the network.
Definition: NvInfer.h:7054
IActivationLayer * addActivation(ITensor &input, ActivationType type) noexcept
Add an activation layer to the network.
Definition: NvInfer.h:6267
virtual ~INetworkDefinition() noexcept=default
void setName(const char *name) noexcept
Sets the name of the network.
Definition: NvInfer.h:6891
ILayer * getLayer(int32_t index) const noexcept
Get the layer specified by the given index.
Definition: NvInfer.h:6492
TRT_DEPRECATED IRNNv2Layer * addRNNv2(ITensor &input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) noexcept
Add an layerCount deep RNN layer to the network with hiddenSize internal states that can take a batch...
Definition: NvInfer.h:6789
TRT_DEPRECATED IFullyConnectedLayer * addFullyConnected(ITensor &input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept
Add a fully connected layer to the network.
Definition: NvInfer.h:6247
IIfConditional * addIfConditional() noexcept
Add an If-conditional layer to the network.
Definition: NvInfer.h:7358
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInfer.h:7285
int32_t getNbInputs() const noexcept
Get the number of inputs in the network.
Definition: NvInfer.h:6504
bool hasImplicitBatchDimension() const noexcept
Query whether the network was created with an implicit batch dimension.
Definition: NvInfer.h:6942
IReduceLayer * addReduce(ITensor &input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
Add a reduce layer to the network.
Definition: NvInfer.h:6589
IUnaryLayer * addUnary(ITensor &input, UnaryOperation operation) noexcept
Add a unary layer to the network.
Definition: NvInfer.h:6435
const char * getName() const noexcept
Returns the name associated with the network.
Definition: NvInfer.h:6905
void removeTensor(ITensor &tensor) noexcept
remove a tensor from the network definition.
Definition: NvInfer.h:6819
ISelectLayer * addSelect(ITensor &condition, ITensor &thenInput, ITensor &elseInput) noexcept
Add a select layer to the network.
Definition: NvInfer.h:7179
IScatterLayer * addScatter(ITensor &data, ITensor &indices, ITensor &updates, ScatterMode mode) noexcept
Add a Scatter layer to the network with specified mode and axis=0.
Definition: NvInfer.h:7324
int32_t getNbLayers() const noexcept
Get the number of layers in the network.
Definition: NvInfer.h:6478
apiv::VNetworkDefinition * mImpl
Definition: NvInfer.h:7378
IPoolingLayer * addPoolingNd(ITensor &input, PoolingType type, Dims windowSize) noexcept
Add a multi-dimension pooling layer to the network.
Definition: NvInfer.h:7032
bool markOutputForShapes(ITensor &tensor) noexcept
Enable tensor's value to be computed by IExecutionContext::getShapeBinding.
Definition: NvInfer.h:6960
TRT_DEPRECATED IPaddingLayer * addPadding(ITensor &input, DimsHW prePadding, DimsHW postPadding) noexcept
Add a padding layer to the network.
Definition: NvInfer.h:6452
IScaleLayer * addScale(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
Add a Scale layer to the network.
Definition: NvInfer.h:6332
void unmarkOutput(ITensor &tensor) noexcept
unmark a tensor as a network output.
Definition: NvInfer.h:6831
IIdentityLayer * addIdentity(ITensor &input) noexcept
Add an identity layer.
Definition: NvInfer.h:6804
TRT_DEPRECATED IConvolutionLayer * addConvolution(ITensor &input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
Add a convolution layer to the network.
Definition: NvInfer.h:6224
IQuantizeLayer * addQuantize(ITensor &input, ITensor &scale) noexcept
Add a quantization layer to the network.
Definition: NvInfer.h:7343
IElementWiseLayer * addElementWise(ITensor &input1, ITensor &input2, ElementWiseOperation op) noexcept
Add an elementwise layer to the network.
Definition: NvInfer.h:6413
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInfer.h:7270
ISliceLayer * addSlice(ITensor &input, Dims start, Dims size, Dims stride) noexcept
Add a slice layer to the network.
Definition: NvInfer.h:6869
IConstantLayer * addConstant(Dims dimensions, Weights weights) noexcept
Add a constant layer to the network.
Definition: NvInfer.h:6720
IRaggedSoftMaxLayer * addRaggedSoftMax(ITensor &input, ITensor &bounds) noexcept
Add a RaggedSoftMax layer to the network.
Definition: NvInfer.h:6673
IShapeLayer * addShape(ITensor &input) noexcept
Add a shape layer to the network.
Definition: NvInfer.h:6923
IGatherLayer * addGather(ITensor &data, ITensor &indices, int32_t axis) noexcept
Add gather with mode GatherMode::kDEFAULT and specified axis and nbElementWiseDims=0.
Definition: NvInfer.h:6639
IEinsumLayer * addEinsum(ITensor *const *inputs, int32_t nbInputs, const char *equation) noexcept
Add an Einsum layer to the network.
Definition: NvInfer.h:7372
TRT_DEPRECATED IPoolingLayer * addPooling(ITensor &input, PoolingType type, DimsHW windowSize) noexcept
Add a pooling layer to the network.
Definition: NvInfer.h:6286
TRT_DEPRECATED void destroy() noexcept
Destroy this INetworkDefinition object.
Definition: NvInfer.h:6562
IResizeLayer * addResize(ITensor &input) noexcept
Add a resize layer to the network.
Definition: NvInfer.h:7106
IMatrixMultiplyLayer * addMatrixMultiply(ITensor &input0, MatrixOperation op0, ITensor &input1, MatrixOperation op1) noexcept
Add a MatrixMultiply layer to the network.
Definition: NvInfer.h:6694
ISoftMaxLayer * addSoftMax(ITensor &input) noexcept
Add a SoftMax layer to the network.
Definition: NvInfer.h:6345
void markOutput(ITensor &tensor) noexcept
Mark a tensor as a network output.
Definition: NvInfer.h:6201
bool setWeightsName(Weights weights, const char *name) noexcept
Associate a name with all current uses of the given weights.
Definition: NvInfer.h:7251
int32_t getNbOutputs() const noexcept
Get the number of outputs in the network.
Definition: NvInfer.h:6534
TRT_DEPRECATED IPaddingLayer * addPaddingNd(ITensor &input, Dims prePadding, Dims postPadding) noexcept
Add a padding layer to the network. Only 2D padding is currently supported.
Definition: NvInfer.h:7231
Forward declaration of IEngineInspector for use by other interfaces.
Definition: NvInferRuntime.h:43
Optimization profile for dynamic input dimensions and shape tensors.
Definition: NvInferRuntime.h:1123
Layer that represents a padding operation.
Definition: NvInfer.h:3798
Dims getPostPaddingNd() const noexcept
Get the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3899
TRT_DEPRECATED DimsHW getPrePadding() const noexcept
Get the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3821
TRT_DEPRECATED void setPostPadding(DimsHW padding) noexcept
Set the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3835
virtual ~IPaddingLayer() noexcept=default
TRT_DEPRECATED void setPrePadding(DimsHW padding) noexcept
Set the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3809
Dims getPrePaddingNd() const noexcept
Get the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3873
TRT_DEPRECATED DimsHW getPostPadding() const noexcept
Get the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3847
void setPostPaddingNd(Dims padding) noexcept
Set the padding that is applied at the end of the tensor.
Definition: NvInfer.h:3887
void setPrePaddingNd(Dims padding) noexcept
Set the padding that is applied at the start of the tensor.
Definition: NvInfer.h:3861
apiv::VPaddingLayer * mImpl
Definition: NvInfer.h:3905
Layer that represents a parametric ReLU operation.
Definition: NvInfer.h:4631
apiv::VParametricReLULayer * mImpl
Definition: NvInfer.h:4633
virtual ~IParametricReLULayer() noexcept=default
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:1206
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:373
Layer type for pluginV2.
Definition: NvInfer.h:3569
virtual ~IPluginV2Layer() noexcept=default
apiv::VPluginV2Layer * mImpl
Definition: NvInfer.h:3582
IPluginV2 & getPlugin() noexcept
Get the plugin for the layer.
Definition: NvInfer.h:3576
A Pooling layer in a network definition.
Definition: NvInfer.h:1666
TRT_DEPRECATED DimsHW getStride() const noexcept
Get the stride for pooling.
Definition: NvInfer.h:1739
PoolingType getPoolingType() const noexcept
Get the type of activation to be performed.
Definition: NvInfer.h:1685
void setWindowSizeNd(Dims windowSize) noexcept
Set the multi-dimension window size for pooling.
Definition: NvInfer.h:1918
PaddingMode getPaddingMode() const noexcept
Get the padding mode.
Definition: NvInfer.h:1905
Dims getPostPadding() const noexcept
Get the padding.
Definition: NvInfer.h:1881
TRT_DEPRECATED void setStride(DimsHW stride) noexcept
Set the stride for pooling.
Definition: NvInfer.h:1727
bool getAverageCountExcludesPadding() const noexcept
Get whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1825
Dims getPrePadding() const noexcept
Get the pre-padding.
Definition: NvInfer.h:1853
void setPoolingType(PoolingType type) noexcept
Set the type of activation to be performed.
Definition: NvInfer.h:1675
void setPaddingMode(PaddingMode paddingMode) noexcept
Set the padding mode.
Definition: NvInfer.h:1894
TRT_DEPRECATED void setWindowSize(DimsHW windowSize) noexcept
Set the window size for pooling.
Definition: NvInfer.h:1699
Dims getWindowSizeNd() const noexcept
Get the multi-dimension window size for pooling.
Definition: NvInfer.h:1928
void setAverageCountExcludesPadding(bool exclusive) noexcept
Set whether average pooling uses as a denominator the overlap area between the window and the unpadde...
Definition: NvInfer.h:1814
void setPaddingNd(Dims padding) noexcept
Set the multi-dimension padding for pooling.
Definition: NvInfer.h:1972
void setPrePadding(Dims padding) noexcept
Set the multi-dimension pre-padding for pooling.
Definition: NvInfer.h:1843
float getBlendFactor() const noexcept
Get the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1797
Dims getStrideNd() const noexcept
Get the multi-dimension stride for pooling.
Definition: NvInfer.h:1953
virtual ~IPoolingLayer() noexcept=default
Dims getPaddingNd() const noexcept
Get the multi-dimension padding for pooling.
Definition: NvInfer.h:1984
TRT_DEPRECATED DimsHW getWindowSize() const noexcept
Get the window size for pooling.
Definition: NvInfer.h:1711
TRT_DEPRECATED DimsHW getPadding() const noexcept
Get the padding for pooling.
Definition: NvInfer.h:1769
void setStrideNd(Dims stride) noexcept
Set the multi-dimension stride for pooling.
Definition: NvInfer.h:1943
void setPostPadding(Dims padding) noexcept
Set the multi-dimension post-padding for pooling.
Definition: NvInfer.h:1871
TRT_DEPRECATED void setPadding(DimsHW padding) noexcept
Set the padding for pooling.
Definition: NvInfer.h:1755
void setBlendFactor(float blendFactor) noexcept
Set the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
Definition: NvInfer.h:1784
A Quantize layer in a network definition.
Definition: NvInfer.h:5819
void setAxis(int32_t axis) noexcept
Set the quantization axis.
Definition: NvInfer.h:5840
int32_t getAxis() const noexcept
Get the quantization axis.
Definition: NvInfer.h:5829
virtual ~IQuantizeLayer() noexcept=default
An RNN layer in a network definition, version 2.
Definition: NvInfer.h:3279
void setDirection(RNNDirection op) noexcept
Set the direction of the RNN layer.
Definition: NvInfer.h:3379
void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept
Set the bias parameters for an individual gate in the RNN.
Definition: NvInfer.h:3483
void setCellState(ITensor &cell) noexcept
Set the initial cell state of the LSTM with the provided cell ITensor.
Definition: NvInfer.h:3539
int32_t getDataLength() const noexcept
Get the maximum data length of the RNN.
Definition: NvInfer.h:3293
void setInputMode(RNNInputMode op) noexcept
Set the input mode of the RNN layer.
Definition: NvInfer.h:3354
Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
Get the bias parameters for an individual gate in the RNN.
Definition: NvInfer.h:3493
void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept
Set the weight parameters for an individual gate in the RNN.
Definition: NvInfer.h:3448
void setOperation(RNNOperation op) noexcept
Set the operation of the RNN layer.
Definition: NvInfer.h:3334
Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept
Get the weight parameters for an individual gate in the RNN.
Definition: NvInfer.h:3458
RNNDirection getDirection() const noexcept
Get the direction of the RNN layer.
Definition: NvInfer.h:3389
void setSequenceLengths(ITensor &seqLengths) noexcept
Specify individual sequence lengths in the batch with the ITensor pointed to by seqLengths.
Definition: NvInfer.h:3312
RNNInputMode getInputMode() const noexcept
Get the input mode of the RNN layer.
Definition: NvInfer.h:3364
ITensor * getHiddenState() const noexcept
Get the initial hidden state of the RNN.
Definition: NvInfer.h:3520
ITensor * getCellState() const noexcept
Get the initial cell state of the RNN.
Definition: NvInfer.h:3549
int32_t getMaxSeqLength() const noexcept
Get the maximum sequence length of the RNN.
Definition: NvInfer.h:3289
int32_t getLayerCount() const noexcept
Get the layer count of the RNN.
Definition: NvInfer.h:3281
apiv::VRNNv2Layer * mImpl
Definition: NvInfer.h:3555
ITensor * getSequenceLengths() const noexcept
Get the sequence lengths specified for the RNN.
Definition: NvInfer.h:3324
RNNOperation getOperation() const noexcept
Get the operation of the RNN layer.
Definition: NvInfer.h:3344
virtual ~IRNNv2Layer() noexcept=default
int32_t getHiddenSize() const noexcept
Get the hidden size of the RNN.
Definition: NvInfer.h:3285
void setHiddenState(ITensor &hidden) noexcept
Set the initial hidden state of the RNN with the provided hidden ITensor.
Definition: NvInfer.h:3510
A RaggedSoftmax layer in a network definition.
Definition: NvInfer.h:4521
apiv::VRaggedSoftMaxLayer * mImpl
Definition: NvInfer.h:4523
virtual ~IRaggedSoftMaxLayer() noexcept=default
Definition: NvInfer.h:5261
virtual ~IRecurrenceLayer() noexcept=default
Layer that represents a reduction across a non-bool tensor.
Definition: NvInfer.h:3720
void setKeepDimensions(bool keepDimensions) noexcept
Set the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:3767
void setOperation(ReduceOperation op) noexcept
Set the reduce operation for the layer.
Definition: NvInfer.h:3727
ReduceOperation getOperation() const noexcept
Get the reduce operation for the layer.
Definition: NvInfer.h:3737
virtual ~IReduceLayer() noexcept=default
uint32_t getReduceAxes() const noexcept
Get the axes over which to reduce for the layer.
Definition: NvInfer.h:3757
void setReduceAxes(uint32_t reduceAxes) noexcept
Set the axes over which to reduce.
Definition: NvInfer.h:3747
apiv::VReduceLayer * mImpl
Definition: NvInfer.h:3783
bool getKeepDimensions() const noexcept
Get the boolean that specifies whether or not to keep the reduced dimensions for the layer.
Definition: NvInfer.h:3777
A resize layer in a network definition.
Definition: NvInfer.h:4819
void setSelectorForSinglePixel(ResizeSelector selector) noexcept
Set coordinate selector function when resized to single pixel.
Definition: NvInfer.h:5008
void setOutputDimensions(Dims dimensions) noexcept
Set the output dimensions.
Definition: NvInfer.h:4839
void setNearestRounding(ResizeRoundMode value) noexcept
Set rounding mode for nearest neighbor resize.
Definition: NvInfer.h:5032
virtual ~IResizeLayer() noexcept=default
int32_t getScales(int32_t size, float *scales) const noexcept
Copies resize scales to scales[0, ..., nbScales-1], where nbScales is the number of scales that were ...
Definition: NvInfer.h:4898
ResizeMode getResizeMode() const noexcept
Get resize mode for an input tensor.
Definition: NvInfer.h:4920
void setResizeMode(ResizeMode resizeMode) noexcept
Set resize mode for an input tensor.
Definition: NvInfer.h:4910
ResizeSelector getSelectorForSinglePixel() const noexcept
Get the coordinate selector function when resized to single pixel.
Definition: NvInfer.h:5018
TRT_DEPRECATED bool getAlignCorners() const noexcept
True if align corners has been set.
Definition: NvInfer.h:4948
void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept
Set coordinate transformation function.
Definition: NvInfer.h:4983
void setScales(const float *scales, int32_t nbScales) noexcept
Set the resize scales.
Definition: NvInfer.h:4879
Dims getOutputDimensions() const noexcept
Get the output dimensions.
Definition: NvInfer.h:4849
ResizeRoundMode getNearestRounding() const noexcept
Get rounding mode for nearest neighbor resize.
Definition: NvInfer.h:5042
TRT_DEPRECATED void setAlignCorners(bool alignCorners) noexcept
Set whether to align corners while resizing.
Definition: NvInfer.h:4936
ResizeCoordinateTransformation getCoordinateTransformation() const noexcept
Get coordinate transformation function.
Definition: NvInfer.h:4993
A Scale layer in a network definition.
Definition: NvInfer.h:2148
Weights getScale() const noexcept
Get the scale value.
Definition: NvInfer.h:2205
Weights getPower() const noexcept
Get the power value.
Definition: NvInfer.h:2225
void setScale(Weights scale) noexcept
Set the scale value.
Definition: NvInfer.h:2195
void setPower(Weights power) noexcept
Set the power value.
Definition: NvInfer.h:2215
ScaleMode getMode() const noexcept
Get the scale mode.
Definition: NvInfer.h:2165
void setShift(Weights shift) noexcept
Set the shift value.
Definition: NvInfer.h:2175
void setChannelAxis(int32_t channelAxis) noexcept
Set the channel axis.
Definition: NvInfer.h:2261
Weights getShift() const noexcept
Get the shift value.
Definition: NvInfer.h:2185
virtual ~IScaleLayer() noexcept=default
void setMode(ScaleMode mode) noexcept
Set the scale mode.
Definition: NvInfer.h:2155
int32_t getChannelAxis() const noexcept
Get the channel axis.
Definition: NvInfer.h:2240
A scatter layer in a network definition. Supports several kinds of scattering.
Definition: NvInfer.h:6083
void setMode(ScatterMode mode) noexcept
Set the scatter mode.
Definition: NvInfer.h:6090
apiv::VScatterLayer * mImpl
Definition: NvInfer.h:6124
void setAxis(int32_t axis) noexcept
Set the axis used by ScatterMode::kELEMENTS.
Definition: NvInfer.h:6110
int32_t getAxis() const noexcept
Get the axis.
Definition: NvInfer.h:6118
ScatterMode getMode() const noexcept
Get the scatter mode.
Definition: NvInfer.h:6100
virtual ~IScatterLayer() noexcept=default
Definition: NvInfer.h:5509
virtual ~ISelectLayer() noexcept=default
Layer type for getting shape of a tensor.
Definition: NvInfer.h:4305
virtual ~IShapeLayer() noexcept=default
apiv::VShapeLayer * mImpl
Definition: NvInfer.h:4307
Layer type for shuffling data.
Definition: NvInfer.h:3933
apiv::VShuffleLayer * mImpl
Definition: NvInfer.h:4088
void setReshapeDimensions(Dims dimensions) noexcept
Set the reshaped dimensions.
Definition: NvInfer.h:3981
void setFirstTranspose(Permutation permutation) noexcept
Set the permutation applied by the first transpose operation.
Definition: NvInfer.h:3944
void setSecondTranspose(Permutation permutation) noexcept
Set the permutation applied by the second transpose operation.
Definition: NvInfer.h:4041
Dims getReshapeDimensions() const noexcept
Get the reshaped dimensions.
Definition: NvInfer.h:3994
Permutation getFirstTranspose() const noexcept
Get the permutation applied by the first transpose operation.
Definition: NvInfer.h:3956
virtual ~IShuffleLayer() noexcept=default
Permutation getSecondTranspose() const noexcept
Get the permutation applied by the second transpose operation.
Definition: NvInfer.h:4053
bool getZeroIsPlaceholder() const noexcept
Get meaning of 0 in reshape dimensions.
Definition: NvInfer.h:4082
void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept
Set meaning of 0 in reshape dimensions.
Definition: NvInfer.h:4069
Slices an input tensor into an output tensor based on the offset and strides.
Definition: NvInfer.h:4155
void setMode(SliceMode mode) noexcept
Set the slice mode.
Definition: NvInfer.h:4249
apiv::VSliceLayer * mImpl
Definition: NvInfer.h:4288
virtual ~ISliceLayer() noexcept=default
void setStride(Dims stride) noexcept
Set the stride for computing the output slice data.
Definition: NvInfer.h:4224
void setStart(Dims start) noexcept
Set the start offset that the slice layer uses to create the output slice.
Definition: NvInfer.h:4166
Dims getStart() const noexcept
Get the start offset for the slice layer.
Definition: NvInfer.h:4181
void setSize(Dims size) noexcept
Set the dimensions of the output slice.
Definition: NvInfer.h:4195
Dims getSize() const noexcept
Get dimensions of the output slice.
Definition: NvInfer.h:4210
SliceMode getMode() const noexcept
Get the slice mode.
Definition: NvInfer.h:4259
Dims getStride() const noexcept
Get the stride for the output slice.
Definition: NvInfer.h:4239
A Softmax layer in a network definition.
Definition: NvInfer.h:2288
void setAxes(uint32_t axes) noexcept
Set the axis along which softmax is computed. Currently, only one axis can be set.
Definition: NvInfer.h:2322
uint32_t getAxes() const noexcept
Get the axis along which softmax occurs.
Definition: NvInfer.h:2332
virtual ~ISoftMaxLayer() noexcept=default
A tensor in a network definition.
Definition: NvInfer.h:167
bool setDynamicRange(float min, float max) noexcept
Set dynamic range for the tensor.
Definition: NvInfer.h:267
void setAllowedFormats(TensorFormats formats) noexcept
Set allowed formats for this tensor. By default all formats are allowed. Shape tensors (for which isS...
Definition: NvInfer.h:403
TensorLocation getLocation() const noexcept
Get the storage location of a tensor.
Definition: NvInfer.h:331
void setName(const char *name) noexcept
Set the tensor name.
Definition: NvInfer.h:181
void resetDynamicRange() noexcept
Undo effect of setDynamicRange.
Definition: NvInfer.h:364
bool isExecutionTensor() const noexcept
Whether the tensor is an execution tensor.
Definition: NvInfer.h:474
void setType(DataType type) noexcept
Set the data type of a tensor.
Definition: NvInfer.h:240
bool dynamicRangeIsSet() const noexcept
Query whether dynamic range is set.
Definition: NvInfer.h:356
void setLocation(TensorLocation location) noexcept
Set the storage location of a tensor.
Definition: NvInfer.h:346
bool isShapeTensor() const noexcept
Whether the tensor is a shape tensor.
Definition: NvInfer.h:451
float getDynamicRangeMax() const noexcept
Get maximum of dynamic range.
Definition: NvInfer.h:384
bool isNetworkInput() const noexcept
Whether the tensor is a network input.
Definition: NvInfer.h:275
bool isNetworkOutput() const noexcept
Whether the tensor is a network output.
Definition: NvInfer.h:283
bool getBroadcastAcrossBatch() const noexcept
Check if tensor is broadcast across the batch.
Definition: NvInfer.h:321
void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept
Set whether to enable broadcast of tensor across the batch.
Definition: NvInfer.h:305
DataType getType() const noexcept
Get the data type of a tensor.
Definition: NvInfer.h:252
apiv::VTensor * mImpl
Definition: NvInfer.h:480
float getDynamicRangeMin() const noexcept
Get minimum of dynamic range.
Definition: NvInfer.h:374
virtual ~ITensor() noexcept=default
void setDimensions(Dims dimensions) noexcept
Set the dimensions of a tensor.
Definition: NvInfer.h:212
const char * getName() const noexcept
Get the tensor name.
Definition: NvInfer.h:193
Dims getDimensions() const noexcept
Get the dimensions of a tensor.
Definition: NvInfer.h:225
TensorFormats getAllowedFormats() const noexcept
Get a bitmask of TensorFormat values that the tensor supports. For a shape tensor,...
Definition: NvInfer.h:416
Class to handle tactic timing info collected from builder.
Definition: NvInfer.h:7956
bool combine(const ITimingCache &inputCache, bool ignoreMismatch) noexcept
Combine input timing cache into local instance.
Definition: NvInfer.h:7993
virtual ~ITimingCache() noexcept=default
apiv::VTimingCache * mImpl
Definition: NvInfer.h:8009
bool reset() noexcept
Empty the timing cache.
Definition: NvInfer.h:8003
Layer that represents a TopK reduction.
Definition: NvInfer.h:4341
void setK(int32_t k) noexcept
Set the k value for the layer.
Definition: NvInfer.h:4370
void setReduceAxes(uint32_t reduceAxes) noexcept
Set which axes to reduce for the layer.
Definition: NvInfer.h:4390
TopKOperation getOperation() const noexcept
Get the operation for the layer.
Definition: NvInfer.h:4358
apiv::VTopKLayer * mImpl
Definition: NvInfer.h:4406
void setOperation(TopKOperation op) noexcept
Set the operation for the layer.
Definition: NvInfer.h:4348
int32_t getK() const noexcept
Get the k value for the layer.
Definition: NvInfer.h:4380
uint32_t getReduceAxes() const noexcept
Get the axes to reduce for the layer.
Definition: NvInfer.h:4400
virtual ~ITopKLayer() noexcept=default
Definition: NvInfer.h:5364
TripLimit getTripLimit() const noexcept
Definition: NvInfer.h:5366
virtual ~ITripLimitLayer() noexcept=default
Layer that represents an unary operation.
Definition: NvInfer.h:3645
void setOperation(UnaryOperation op) noexcept
Set the unary operation for the layer.
Definition: NvInfer.h:3654
apiv::VUnaryLayer * mImpl
Definition: NvInfer.h:3670
UnaryOperation getOperation() const noexcept
Get the unary operation for the layer.
Definition: NvInfer.h:3664
virtual ~IUnaryLayer() noexcept=default
An array of weights used as a layer parameter.
Definition: NvInferRuntime.h:126
The TensorRT API version 1 namespace.
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:1304
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:4724
@ kFORMULA
Use formula to map the original index.
@ kUPPER
Select the upper left pixel.
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:69
nvinfer1::IPluginRegistry * getBuilderPluginRegistry(nvinfer1::EngineCapability capability) noexcept
Return the plugin registry for the given capability or nullptr if no registry exists.
MemoryPoolType
The type for memory pools used by TensorRT.
Definition: NvInfer.h:8020
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:2104
@ kUNIFORM
Identical coefficients across all elements of the tensor.
@ kCHANNEL
Per-channel coefficients.
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:7837
constexpr int32_t EnumMax< RNNDirection >() noexcept
Definition: NvInfer.h:3201
constexpr int32_t EnumMax< BuilderFlag >() noexcept
Definition: NvInfer.h:7940
constexpr int32_t EnumMax< LayerType >() noexcept
Definition: NvInfer.h:103
constexpr int32_t EnumMax< RNNGateType >() noexcept
Definition: NvInfer.h:3262
constexpr int32_t EnumMax< CalibrationAlgoType >() noexcept
Definition: NvInfer.h:7400
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:3600
@ kCOSH
Hyperbolic cosine.
@ kACOSH
Inverse hyperbolic cosine.
@ kERF
Gauss error function.
@ kACOS
Inverse cosine.
@ kABS
Absolute value.
@ kSINH
Hyperbolic sine.
@ kROUND
Round to nearest even for float datatype.
@ kATANH
Inverse hyperbolic tangent.
@ kASINH
Inverse hyperbolic sine.
@ kSIGN
Sign, If input > 0, output 1; if input < 0, output -1; if input == 0, output 0.
@ kEXP
Exponentiation.
@ kATAN
Inverse tangent.
constexpr int32_t EnumMax< ReduceOperation >() noexcept
Definition: NvInfer.h:3707
constexpr int32_t EnumMax< TripLimit >() noexcept
Definition: NvInfer.h:5090
constexpr int32_t EnumMax< RNNInputMode >() noexcept
Definition: NvInfer.h:3233
RNNInputMode
Enumerates the RNN input modes that may occur with an RNN layer.
Definition: NvInfer.h:3222
@ kSKIP
No operation is performed on the first recurrent layer.
@ kLINEAR
Perform the normal matrix multiplication in the first recurrent layer.
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:122
@ kSELU
Selu activation: x>0 ? beta * x : beta * (alpha*exp(x) - alpha)
@ kTANH
TanH activation.
@ kSCALED_TANH
Scaled tanh activation: alpha*tanh(beta*x)
@ kRELU
Rectified linear activation.
@ kELU
Elu activation: x>=0 ? x : alpha * (exp(x) - 1).
@ kLEAKY_RELU
LeakyRelu activation: x>=0 ? x : alpha * x.
@ kSOFTSIGN
Softsign activation: x / (1+|x|)
@ kHARD_SIGMOID
Hard sigmoid activation: max(0, min(1, alpha*x+beta))
@ kTHRESHOLDED_RELU
Thresholded ReLU activation: x>alpha ? x : 0.
@ kSIGMOID
Sigmoid activation.
@ kCLIP
Clip activation: max(alpha, min(beta, x))
@ kSOFTPLUS
Parametric softplus activation: alpha*log(exp(beta*x)+1)
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:5569
@ kLINSPACE
Generate evenly spaced numbers over a specified interval.
@ kRANDOM_UNIFORM
Generate a tensor with random values drawn from a uniform distribution.
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4754
@ kHALF_UP
Round half up.
@ kHALF_DOWN
Round half down.
RNNGateType
Identifies an individual gate within an RNN cell.
Definition: NvInfer.h:3246
@ kUPDATE
Update gate (z).
@ kHIDDEN
Hidden gate (h).
@ kFORGET
Forget gate (f).
@ kOUTPUT
Output gate (o).
@ kRESET
Reset gate (r).
@ kINPUT
Input gate (i).
@ kCELL
Cell gate (c).
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:951
@ kSAME_LOWER
Use SAME padding, with prePadding >= postPadding.
@ kEXPLICIT_ROUND_DOWN
Use explicit padding, rounding output size down.
@ kCAFFE_ROUND_DOWN
Use CAFFE padding, rounding output size down, uses prePadding value.
@ kEXPLICIT_ROUND_UP
Use explicit padding, rounding output size up.
@ kCAFFE_ROUND_UP
Use CAFFE padding, rounding output size up, uses prePadding value.
@ kSAME_UPPER
Use SAME padding, with prePadding <= postPadding.
TripLimit
Enum that describes kinds of trip limits.
Definition: NvInfer.h:5078
@ kWHILE
Tensor is a scalar of type kBOOL. Loop terminates when value is false.
@ kCOUNT
Tensor is scalar of type kINT32 that contains the trip count.
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:8729
constexpr int32_t EnumMax< GatherMode >() noexcept
Definition: NvInfer.h:2917
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:114
uint32_t BuilderFlags
Represents one or more QuantizationFlag values using binary OR operations, e.g., 1U << BuilderFlag::k...
Definition: NvInfer.h:7871
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:602
constexpr int32_t EnumMax< ScaleMode >() noexcept
Definition: NvInfer.h:2116
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:7387
LayerType
The type values of layer classes.
Definition: NvInfer.h:53
@ kRAGGED_SOFTMAX
Ragged softmax layer.
@ kDECONVOLUTION
Deconvolution layer.
@ kREDUCE
Reduce layer.
@ kASSERTION
Assertion layer.
@ kTOPK
TopK layer.
@ kRESIZE
Resize Layer.
@ kPADDING
Padding layer.
@ kMATRIX_MULTIPLY
Matrix multiply layer.
@ kCONDITION
Condition layer.
@ kCONDITIONAL_INPUT
Conditional Input layer.
@ kIDENTITY
Identity layer.
@ kQUANTIZE
Quantize layer.
@ kSCATTER
Scatter layer.
@ kCONVOLUTION
Convolution layer.
@ kPARAMETRIC_RELU
Parametric ReLU layer.
@ kCONCATENATION
Concatenation layer.
@ kSLICE
Slice layer.
@ kEINSUM
Einsum layer.
@ kSOFTMAX
SoftMax layer.
@ kSHAPE
Shape layer.
@ kFULLY_CONNECTED
Fully connected layer.
@ kRECURRENCE
Loop Recurrence layer.
@ kDEQUANTIZE
Dequantize layer.
@ kSHUFFLE
Shuffle layer.
@ kITERATOR
Loop Iterator layer.
@ kPOOLING
Pooling layer.
@ kTRIP_LIMIT
Loop Trip limit layer.
@ kSCALE
Scale layer.
@ kGATHER
Gather layer.
@ kUNARY
UnaryOp operation Layer.
@ kACTIVATION
Activation layer.
@ kELEMENTWISE
Elementwise layer.
@ kSELECT
Select layer.
@ kPLUGIN_V2
PluginV2 layer.
@ kLOOP_OUTPUT
Loop output layer.
@ kCONDITIONAL_OUTPUT
Conditional Output layer.
@ kCONSTANT
Constant layer.
@ kRNN_V2
RNNv2 layer.
@ kFILL
Fill layer.
@ kPLUGIN
Plugin layer.
SliceMode
Controls how ISliceLayer handles out of bounds coordinates.
Definition: NvInfer.h:4098
@ kCLAMP
Out of bounds indices are clamped to bounds.
@ kWRAP
Coordinates wrap around periodically.
constexpr int32_t EnumMax< QuantizationFlag >() noexcept
Definition: NvInfer.h:7860
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2905
@ kDEFAULT
Similar to ONNX Gather.
@ kELEMENT
Similar to ONNX GatherElements.
@ kND
Similar to ONNX GatherND.
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:114
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:1316
ResizeMode
Enumerates various modes of resize in the resize layer. Resize mode set using setResizeMode().
Definition: NvInfer.h:4643
@ kNEAREST
ND (0 < N <= 8) nearest neighbor resizing.
constexpr int32_t EnumMax< SliceMode >() noexcept
Definition: NvInfer.h:4114
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:8741
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2815
@ kSUB
Subtract the second element from the first.
@ kSUM
Sum of the two elements.
@ kPROD
Product of the two elements.
@ kFLOOR_DIV
Floor division of the first element by the second.
@ kEQUAL
Check if two elements are equal.
@ kAND
Logical AND of two elements.
@ kOR
Logical OR of two elements.
@ kMIN
Minimum of the two elements.
@ kPOW
The first element to the power of the second element.
@ kLESS
Check if element in first tensor is less than corresponding element in second tensor.
@ kGREATER
Check if element in first tensor is greater than corresponding element in second tensor.
@ kXOR
Logical XOR of two elements.
@ kDIV
Divide the first element by the second.
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:7847
RNNDirection
Enumerates the RNN direction that may be performed by an RNN layer.
Definition: NvInfer.h:3190
@ kBIDIRECTION
Network iterates from first to last and vice versa and outputs concatenated.
@ kUNIDIRECTION
Network iterations from first input to last input.
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:7881
@ kDEBUG
Enable debugging of layers via synchronizing after every layer.
@ kGPU_FALLBACK
Enable layers marked to execute on GPU if layer cannot execute on DLA.
@ kSPARSE_WEIGHTS
Allow the builder to examine weights and use optimized functions when weights have suitable sparsity.
@ kFP16
Enable FP16 layer selection, with FP32 fallback.
@ kINT8
Enable Int8 layer selection, with FP32 fallback with FP16 fallback if kFP16 also specified.
@ kDISABLE_TIMING_CACHE
Disable reuse of timing information across identical layers.
@ kREFIT
Enable building a refittable engine.
@ kOBEY_PRECISION_CONSTRAINTS
Require that layers execute in specified precisions. Build fails otherwise.
@ kREJECT_EMPTY_ALGORITHMS
Fail if IAlgorithmSelector::selectAlgorithms returns an empty set of algorithms.
constexpr int32_t EnumMax< TopKOperation >() noexcept
Definition: NvInfer.h:4328
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:184
constexpr int32_t EnumMax< MemoryPoolType >() noexcept
Definition: NvInfer.h:8059
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:4317
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:3693
constexpr int32_t EnumMax< LoopOutput >() noexcept
Definition: NvInfer.h:5071
RNNOperation
Enumerates the RNN operations that may be performed by an RNN layer.
Definition: NvInfer.h:3164
@ kGRU
Three-gate network consisting of Gated Recurrent Units.
@ kLSTM
Four-gate LSTM network w/o peephole connections.
constexpr int32_t EnumMax< NetworkDefinitionCreationFlag >() noexcept
Definition: NvInfer.h:8770
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:6010
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:4417
@ kTRANSPOSE
Like kNONE, but transpose the matrix dimensions.
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:4670
constexpr int32_t EnumMax< UnaryOperation >() noexcept
Definition: NvInfer.h:3632
LoopOutput
Enum that describes kinds of loop outputs.
Definition: NvInfer.h:5054
@ kLAST_VALUE
Output value is value of tensor for last iteration.
@ kCONCATENATE
Output value is concatenation of values of tensor for each iteration, in forward order.
@ kREVERSE
Output value is concatenation of values of tensor for each iteration, in reverse order.
constexpr int32_t EnumMax< MatrixOperation >() noexcept
Definition: NvInfer.h:4445
constexpr int32_t EnumMax< RNNOperation >() noexcept
Definition: NvInfer.h:3177
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1634
constexpr int32_t EnumMax< FillOperation >() noexcept
Definition: NvInfer.h:5580
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:216
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:1083
constexpr int32_t EnumMax< ScatterMode >() noexcept
Definition: NvInfer.h:6021
Definition: NvInfer.h:3910
Declaration of EnumMaxImpl struct to store maximum number of elements in an enumeration type.
Definition: NvInferRuntimeCommon.h:99