TensorRT 8.4.0
NvInferRuntimeCommon.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: LicenseRef-NvidiaProprietary
4 *
5 * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
6 * property and proprietary rights in and to this material, related
7 * documentation and any modifications thereto. Any use, reproduction,
8 * disclosure or distribution of this material and related documentation
9 * without an express license agreement from NVIDIA CORPORATION or
10 * its affiliates is strictly prohibited.
11 */
12
13#ifndef NV_INFER_RUNTIME_COMMON_H
14#define NV_INFER_RUNTIME_COMMON_H
15
16#include "NvInferVersion.h"
17#include <cstddef>
18#include <cstdint>
19#include <cuda_runtime_api.h>
20
21// Items that are marked as deprecated will be removed in a future release.
22#if __cplusplus >= 201402L
23#define TRT_DEPRECATED [[deprecated]]
24#if __GNUC__ < 6
25#define TRT_DEPRECATED_ENUM
26#else
27#define TRT_DEPRECATED_ENUM TRT_DEPRECATED
28#endif
29#ifdef _MSC_VER
30#define TRT_DEPRECATED_API __declspec(dllexport)
31#else
32#define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default")))
33#endif
34#else
35#ifdef _MSC_VER
36#define TRT_DEPRECATED
37#define TRT_DEPRECATED_ENUM
38#define TRT_DEPRECATED_API __declspec(dllexport)
39#else
40#define TRT_DEPRECATED __attribute__((deprecated))
41#define TRT_DEPRECATED_ENUM
42#define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default")))
43#endif
44#endif
45
46// Defines which symbols are exported
47#ifdef TENSORRT_BUILD_LIB
48#ifdef _MSC_VER
49#define TENSORRTAPI __declspec(dllexport)
50#else
51#define TENSORRTAPI __attribute__((visibility("default")))
52#endif
53#else
54#define TENSORRTAPI
55#endif
56#define TRTNOEXCEPT
62
63// forward declare some CUDA types to avoid an include dependency
64
65extern "C"
66{
68 struct cublasContext;
70 struct cudnnContext;
71}
72
73#define NV_TENSORRT_VERSION nvinfer1::kNV_TENSORRT_VERSION_IMPL
79namespace nvinfer1
80{
81
82static constexpr int32_t kNV_TENSORRT_VERSION_IMPL
83 = (NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + NV_TENSORRT_PATCH; // major, minor, patch
84
86using char_t = char;
89
91class IErrorRecorder;
93class IGpuAllocator;
94
95namespace impl
96{
98template <typename T>
100} // namespace impl
101
103template <typename T>
104constexpr int32_t EnumMax() noexcept
105{
107}
108
113enum class DataType : int32_t
114{
116 kFLOAT = 0,
117
119 kHALF = 1,
120
122 kINT8 = 2,
123
125 kINT32 = 3,
126
128 kBOOL = 4
129};
130
131namespace impl
132{
134template <>
136{
137 // Declaration of kVALUE that represents maximum number of elements in DataType enum
138 static constexpr int32_t kVALUE = 5;
139};
140} // namespace impl
141
153{
154public:
156 static constexpr int32_t MAX_DIMS{8};
158 int32_t nbDims;
160 int32_t d[MAX_DIMS];
161};
162
168using Dims = Dims32;
169
183enum class TensorFormat : int32_t
184{
192 kLINEAR = 0,
193
200 kCHW2 = 1,
201
208 kHWC8 = 2,
209
225 kCHW4 = 3,
226
237 kCHW16 = 4,
238
248 kCHW32 = 5,
249
256 kDHWC8 = 6,
257
264 kCDHW32 = 7,
265
268 kHWC = 8,
269
278 kDLA_LINEAR = 9,
279
292 kDLA_HWC4 = 10,
293
300 kHWC16 = 11
301};
302
309
310namespace impl
311{
313template <>
315{
317 static constexpr int32_t kVALUE = 12;
318};
319} // namespace impl
320
332{
340 float scale;
341};
342
349enum class PluginVersion : uint8_t
350{
352 kV2 = 0,
354 kV2_EXT = 1,
356 kV2_IOEXT = 2,
358 kV2_DYNAMICEXT = 3,
359};
360
373{
374public:
385 virtual int32_t getTensorRTVersion() const noexcept
386 {
387 return NV_TENSORRT_VERSION;
388 }
389
402 virtual AsciiChar const* getPluginType() const noexcept = 0;
403
416 virtual AsciiChar const* getPluginVersion() const noexcept = 0;
417
431 virtual int32_t getNbOutputs() const noexcept = 0;
432
448 virtual Dims getOutputDimensions(int32_t index, Dims const* inputs, int32_t nbInputDims) noexcept = 0;
449
472 virtual bool supportsFormat(DataType type, PluginFormat format) const noexcept = 0;
473
505 virtual void configureWithFormat(Dims const* inputDims, int32_t nbInputs, Dims const* outputDims, int32_t nbOutputs,
506 DataType type, PluginFormat format, int32_t maxBatchSize) noexcept
507 = 0;
508
520 virtual int32_t initialize() noexcept = 0;
521
534 virtual void terminate() noexcept = 0;
535
550 virtual size_t getWorkspaceSize(int32_t maxBatchSize) const noexcept = 0;
551
568 virtual int32_t enqueue(int32_t batchSize, void const* const* inputs, void* const* outputs, void* workspace,
569 cudaStream_t stream) noexcept
570 = 0;
571
582 virtual size_t getSerializationSize() const noexcept = 0;
583
597 virtual void serialize(void* buffer) const noexcept = 0;
598
607 virtual void destroy() noexcept = 0;
608
623 virtual IPluginV2* clone() const noexcept = 0;
624
639 virtual void setPluginNamespace(AsciiChar const* pluginNamespace) noexcept = 0;
640
649 virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
650
651 // @cond SuppressDoxyWarnings
652 IPluginV2() = default;
653 virtual ~IPluginV2() noexcept = default;
654// @endcond
655
656protected:
657// @cond SuppressDoxyWarnings
658 IPluginV2(IPluginV2 const&) = default;
659 IPluginV2(IPluginV2&&) = default;
660 IPluginV2& operator=(IPluginV2 const&) & = default;
661 IPluginV2& operator=(IPluginV2&&) & = default;
662// @endcond
663};
664
676{
677public:
694 int32_t index, nvinfer1::DataType const* inputTypes, int32_t nbInputs) const noexcept
695 = 0;
696
713 int32_t outputIndex, bool const* inputIsBroadcasted, int32_t nbInputs) const noexcept
714 = 0;
715
734 virtual bool canBroadcastInputAcrossBatch(int32_t inputIndex) const noexcept = 0;
735
770 virtual void configurePlugin(Dims const* inputDims, int32_t nbInputs, Dims const* outputDims, int32_t nbOutputs,
771 DataType const* inputTypes, DataType const* outputTypes, bool const* inputIsBroadcast,
772 bool const* outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) noexcept
773 = 0;
774
775 IPluginV2Ext() = default;
776 ~IPluginV2Ext() override = default;
777
801 virtual void attachToContext(
802 cudnnContext* /*cudnn*/, cublasContext* /*cublas*/, IGpuAllocator* /*allocator*/) noexcept
803 {
804 }
805
819 virtual void detachFromContext() noexcept {}
820
833 IPluginV2Ext* clone() const noexcept override = 0;
834
835protected:
836 // @cond SuppressDoxyWarnings
837 IPluginV2Ext(IPluginV2Ext const&) = default;
838 IPluginV2Ext(IPluginV2Ext&&) = default;
839 IPluginV2Ext& operator=(IPluginV2Ext const&) & = default;
840 IPluginV2Ext& operator=(IPluginV2Ext&&) & = default;
841// @endcond
842
854 int32_t getTensorRTVersion() const noexcept override
855 {
856 return static_cast<int32_t>((static_cast<uint32_t>(PluginVersion::kV2_EXT) << 24U)
857 | (static_cast<uint32_t>(NV_TENSORRT_VERSION) & 0xFFFFFFU));
858 }
859
863 void configureWithFormat(Dims const* /*inputDims*/, int32_t /*nbInputs*/, Dims const* /*outputDims*/,
864 int32_t /*nbOutputs*/, DataType /*type*/, PluginFormat /*format*/, int32_t /*maxBatchSize*/) noexcept override
865 {
866 }
867};
868
879{
880public:
898 virtual void configurePlugin(
899 PluginTensorDesc const* in, int32_t nbInput, PluginTensorDesc const* out, int32_t nbOutput) noexcept
900 = 0;
901
940 int32_t pos, PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) const noexcept
941 = 0;
942
943 // @cond SuppressDoxyWarnings
944 IPluginV2IOExt() = default;
945 ~IPluginV2IOExt() override = default;
946// @endcond
947
948protected:
949// @cond SuppressDoxyWarnings
950 IPluginV2IOExt(IPluginV2IOExt const&) = default;
951 IPluginV2IOExt(IPluginV2IOExt&&) = default;
952 IPluginV2IOExt& operator=(IPluginV2IOExt const&) & = default;
953 IPluginV2IOExt& operator=(IPluginV2IOExt&&) & = default;
954// @endcond
955
967 int32_t getTensorRTVersion() const noexcept override
968 {
969 return static_cast<int32_t>((static_cast<uint32_t>(PluginVersion::kV2_IOEXT) << 24U)
970 | (static_cast<uint32_t>(NV_TENSORRT_VERSION) & 0xFFFFFFU));
971 }
972
973private:
974 // Following are obsolete base class methods, and must not be implemented or used.
975
976 void configurePlugin(Dims const*, int32_t, Dims const*, int32_t, DataType const*, DataType const*, bool const*,
977 bool const*, PluginFormat, int32_t) noexcept final
978 {
979 }
980
981 bool supportsFormat(DataType, PluginFormat) const noexcept final
982 {
983 return false;
984 }
985};
986
991
992enum class PluginFieldType : int32_t
993{
995 kFLOAT16 = 0,
997 kFLOAT32 = 1,
999 kFLOAT64 = 2,
1001 kINT8 = 3,
1003 kINT16 = 4,
1005 kINT32 = 5,
1007 kCHAR = 6,
1009 kDIMS = 7,
1011 kUNKNOWN = 8
1012};
1013
1022{
1023public:
1031 void const* data;
1040 int32_t length;
1041
1042 PluginField(AsciiChar const* const name_ = nullptr, void const* const data_ = nullptr,
1043 PluginFieldType const type_ = PluginFieldType::kUNKNOWN, int32_t const length_ = 0) noexcept
1044 : name(name_)
1045 , data(data_)
1046 , type(type_)
1047 , length(length_)
1048 {
1049 }
1050};
1051
1054{
1056 int32_t nbFields;
1059};
1060
1068
1070{
1071public:
1079 virtual int32_t getTensorRTVersion() const noexcept
1080 {
1081 return NV_TENSORRT_VERSION;
1082 }
1083
1096 virtual AsciiChar const* getPluginName() const noexcept = 0;
1097
1110 virtual AsciiChar const* getPluginVersion() const noexcept = 0;
1111
1122 virtual PluginFieldCollection const* getFieldNames() noexcept = 0;
1123
1133 virtual IPluginV2* createPlugin(AsciiChar const* name, PluginFieldCollection const* fc) noexcept = 0;
1134
1144 virtual IPluginV2* deserializePlugin(AsciiChar const* name, void const* serialData, size_t serialLength) noexcept
1145 = 0;
1146
1159 virtual void setPluginNamespace(AsciiChar const* pluginNamespace) noexcept = 0;
1160
1173 virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
1174
1175 IPluginCreator() = default;
1176 virtual ~IPluginCreator() = default;
1177
1178protected:
1179// @cond SuppressDoxyWarnings
1180 IPluginCreator(IPluginCreator const&) = default;
1181 IPluginCreator(IPluginCreator&&) = default;
1182 IPluginCreator& operator=(IPluginCreator const&) & = default;
1183 IPluginCreator& operator=(IPluginCreator&&) & = default;
1184// @endcond
1185};
1186
1204
1206{
1207public:
1219 virtual bool registerCreator(IPluginCreator& creator, AsciiChar const* const pluginNamespace) noexcept = 0;
1220
1229 virtual IPluginCreator* const* getPluginCreatorList(int32_t* const numCreators) const noexcept = 0;
1230
1242 virtual IPluginCreator* getPluginCreator(AsciiChar const* const pluginName, AsciiChar const* const pluginVersion,
1243 AsciiChar const* const pluginNamespace = "") noexcept
1244 = 0;
1245
1246 // @cond SuppressDoxyWarnings
1247 IPluginRegistry() = default;
1248 IPluginRegistry(IPluginRegistry const&) = delete;
1249 IPluginRegistry(IPluginRegistry&&) = delete;
1250 IPluginRegistry& operator=(IPluginRegistry const&) & = delete;
1251 IPluginRegistry& operator=(IPluginRegistry&&) & = delete;
1252// @endcond
1253
1254protected:
1255 virtual ~IPluginRegistry() noexcept = default;
1256
1257public:
1267 //
1274 virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
1275
1291 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1292
1308 virtual bool deregisterCreator(IPluginCreator const& creator) noexcept = 0;
1309};
1310
1311enum class AllocatorFlag : int32_t
1312{
1313 kRESIZABLE = 0,
1314};
1315
1316namespace impl
1317{
1319template <>
1321{
1322 static constexpr int32_t kVALUE = 1;
1323};
1324} // namespace impl
1325
1326using AllocatorFlags = uint32_t;
1327
1334{
1335public:
1357 virtual void* allocate(uint64_t const size, uint64_t const alignment, AllocatorFlags const flags) noexcept = 0;
1358
1377 TRT_DEPRECATED virtual void free(void* const memory) noexcept = 0;
1378
1383 virtual ~IGpuAllocator() = default;
1384 IGpuAllocator() = default;
1385
1419 virtual void* reallocate(void* /*baseAddr*/, uint64_t /*alignment*/, uint64_t /*newSize*/) noexcept
1420 {
1421 return nullptr;
1422 }
1423
1444 virtual bool deallocate(void* const memory) noexcept
1445 {
1446 this->free(memory);
1447 return true;
1448 }
1449
1450protected:
1451// @cond SuppressDoxyWarnings
1452 IGpuAllocator(IGpuAllocator const&) = default;
1453 IGpuAllocator(IGpuAllocator&&) = default;
1454 IGpuAllocator& operator=(IGpuAllocator const&) & = default;
1455 IGpuAllocator& operator=(IGpuAllocator&&) & = default;
1456// @endcond
1457};
1458
1472{
1473public:
1479 enum class Severity : int32_t
1480 {
1482 kINTERNAL_ERROR = 0,
1484 kERROR = 1,
1486 kWARNING = 2,
1488 kINFO = 3,
1490 kVERBOSE = 4,
1491 };
1492
1505 virtual void log(Severity severity, AsciiChar const* msg) noexcept = 0;
1506
1507 ILogger() = default;
1508 virtual ~ILogger() = default;
1509
1510protected:
1511// @cond SuppressDoxyWarnings
1512 ILogger(ILogger const&) = default;
1513 ILogger(ILogger&&) = default;
1514 ILogger& operator=(ILogger const&) & = default;
1515 ILogger& operator=(ILogger&&) & = default;
1516// @endcond
1517};
1518
1519namespace impl
1520{
1522template <>
1523struct EnumMaxImpl<ILogger::Severity>
1524{
1526 static constexpr int32_t kVALUE = 5;
1527};
1528} // namespace impl
1529
1535enum class ErrorCode : int32_t
1536{
1540 kSUCCESS = 0,
1541
1546
1551 kINTERNAL_ERROR = 2,
1552
1558
1566 kINVALID_CONFIG = 4,
1567
1574
1580
1588
1597
1610 kINVALID_STATE = 9,
1611
1622 kUNSUPPORTED_STATE = 10,
1623
1624};
1625
1626namespace impl
1627{
1629template <>
1631{
1633 static constexpr int32_t kVALUE = 11;
1634};
1635} // namespace impl
1636
1661{
1662public:
1666 using ErrorDesc = char const*;
1667
1671 static constexpr size_t kMAX_DESC_LENGTH{127U};
1672
1676 using RefCount = int32_t;
1677
1678 IErrorRecorder() = default;
1679 virtual ~IErrorRecorder() noexcept = default;
1680
1681 // Public API used to retrieve information from the error recorder.
1682
1701 virtual int32_t getNbErrors() const noexcept = 0;
1702
1720 virtual ErrorCode getErrorCode(int32_t errorIdx) const noexcept = 0;
1721
1741 virtual ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept = 0;
1742
1757 virtual bool hasOverflowed() const noexcept = 0;
1758
1773 virtual void clear() noexcept = 0;
1774
1775 // API used by TensorRT to report Error information to the application.
1776
1797 virtual bool reportError(ErrorCode val, ErrorDesc desc) noexcept = 0;
1798
1815 virtual RefCount incRefCount() noexcept = 0;
1816
1833 virtual RefCount decRefCount() noexcept = 0;
1834
1835protected:
1836 // @cond SuppressDoxyWarnings
1837 IErrorRecorder(IErrorRecorder const&) = default;
1838 IErrorRecorder(IErrorRecorder&&) = default;
1839 IErrorRecorder& operator=(IErrorRecorder const&) & = default;
1840 IErrorRecorder& operator=(IErrorRecorder&&) & = default;
1841 // @endcond
1842}; // class IErrorRecorder
1843} // namespace nvinfer1
1844
1850extern "C" TENSORRTAPI int32_t getInferLibVersion() noexcept;
1851
1852#endif // NV_INFER_RUNTIME_COMMON_H
#define TENSORRTAPI
Definition: NvInferRuntimeCommon.h:54
int32_t getInferLibVersion() noexcept
Return the library version number.
#define NV_TENSORRT_VERSION
Definition: NvInferRuntimeCommon.h:73
#define TRT_DEPRECATED
Definition: NvInferRuntimeCommon.h:40
#define NV_TENSORRT_MINOR
TensorRT minor version.
Definition: NvInferVersion.h:23
#define NV_TENSORRT_MAJOR
TensorRT major version.
Definition: NvInferVersion.h:22
#define NV_TENSORRT_PATCH
TensorRT patch version.
Definition: NvInferVersion.h:24
Definition: NvInferRuntimeCommon.h:153
int32_t nbDims
The rank (number of dimensions).
Definition: NvInferRuntimeCommon.h:158
static constexpr int32_t MAX_DIMS
The maximum rank (number of dimensions) supported for a tensor.
Definition: NvInferRuntimeCommon.h:156
int32_t d[MAX_DIMS]
The extent of each dimension.
Definition: NvInferRuntimeCommon.h:160
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1661
virtual ~IErrorRecorder() noexcept=default
char const * ErrorDesc
Definition: NvInferRuntimeCommon.h:1666
int32_t RefCount
Definition: NvInferRuntimeCommon.h:1676
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:1334
virtual bool deallocate(void *const memory) noexcept
Definition: NvInferRuntimeCommon.h:1444
virtual void * reallocate(void *, uint64_t, uint64_t) noexcept
Definition: NvInferRuntimeCommon.h:1419
virtual ~IGpuAllocator()=default
virtual void * allocate(uint64_t const size, uint64_t const alignment, AllocatorFlags const flags) noexcept=0
virtual TRT_DEPRECATED void free(void *const memory) noexcept=0
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntimeCommon.h:1472
virtual ~ILogger()=default
Severity
Definition: NvInferRuntimeCommon.h:1480
virtual void log(Severity severity, AsciiChar const *msg) noexcept=0
Plugin creator class for user implemented layers.
Definition: NvInferRuntimeCommon.h:1070
virtual int32_t getTensorRTVersion() const noexcept
Return the version of the API the plugin creator was compiled with.
Definition: NvInferRuntimeCommon.h:1079
virtual AsciiChar const * getPluginName() const noexcept=0
Return the plugin name.
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:1206
virtual bool registerCreator(IPluginCreator &creator, AsciiChar const *const pluginNamespace) noexcept=0
Register a plugin creator. Returns false if one with same type is already registered.
virtual IPluginCreator * getPluginCreator(AsciiChar const *const pluginName, AsciiChar const *const pluginVersion, AsciiChar const *const pluginNamespace="") noexcept=0
Return plugin creator based on plugin name, version, and namespace associated with plugin during netw...
virtual IPluginCreator *const * getPluginCreatorList(int32_t *const numCreators) const noexcept=0
Return all the registered plugin creators and the number of registered plugin creators....
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:676
~IPluginV2Ext() override=default
virtual bool canBroadcastInputAcrossBatch(int32_t inputIndex) const noexcept=0
Return true if plugin can use input that is broadcast across batch without replication.
void configureWithFormat(Dims const *, int32_t, Dims const *, int32_t, DataType, PluginFormat, int32_t) noexcept override
Derived classes should not implement this. In a C++11 API it would be override final.
Definition: NvInferRuntimeCommon.h:863
virtual bool isOutputBroadcastAcrossBatch(int32_t outputIndex, bool const *inputIsBroadcasted, int32_t nbInputs) const noexcept=0
Return true if output tensor is broadcast across a batch.
IPluginV2Ext * clone() const noexcept override=0
Clone the plugin object. This copies over internal plugin parameters as well and returns a new plugin...
virtual void configurePlugin(Dims const *inputDims, int32_t nbInputs, Dims const *outputDims, int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes, bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) noexcept=0
Configure the layer with input and output data types.
virtual void detachFromContext() noexcept
Detach the plugin object from its execution context.
Definition: NvInferRuntimeCommon.h:819
virtual void attachToContext(cudnnContext *, cublasContext *, IGpuAllocator *) noexcept
Attach the plugin object to an execution context and grant the plugin the access to some context reso...
Definition: NvInferRuntimeCommon.h:801
virtual nvinfer1::DataType getOutputDataType(int32_t index, nvinfer1::DataType const *inputTypes, int32_t nbInputs) const noexcept=0
Return the DataType of the plugin output at the requested index.
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:373
virtual AsciiChar const * getPluginType() const noexcept=0
Return the plugin type. Should match the plugin name returned by the corresponding plugin creator.
virtual void terminate() noexcept=0
Release resources acquired during plugin layer initialization. This is called when the engine is dest...
virtual int32_t getTensorRTVersion() const noexcept
Return the API version with which this plugin was built.
Definition: NvInferRuntimeCommon.h:385
virtual void setPluginNamespace(AsciiChar const *pluginNamespace) noexcept=0
Set the namespace that this plugin object belongs to. Ideally, all plugin objects from the same plugi...
virtual void configureWithFormat(Dims const *inputDims, int32_t nbInputs, Dims const *outputDims, int32_t nbOutputs, DataType type, PluginFormat format, int32_t maxBatchSize) noexcept=0
Configure the layer.
virtual void serialize(void *buffer) const noexcept=0
Serialize the layer.
virtual void destroy() noexcept=0
Destroy the plugin object. This will be called when the network, builder or engine is destroyed.
virtual AsciiChar const * getPluginVersion() const noexcept=0
Return the plugin version. Should match the plugin version returned by the corresponding plugin creat...
virtual Dims getOutputDimensions(int32_t index, Dims const *inputs, int32_t nbInputDims) noexcept=0
Get the dimension of an output tensor.
virtual IPluginV2 * clone() const noexcept=0
Clone the plugin object. This copies over internal plugin parameters and returns a new plugin object ...
virtual int32_t enqueue(int32_t batchSize, void const *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) noexcept=0
Execute the layer.
virtual size_t getSerializationSize() const noexcept=0
Find the size of the serialization buffer required.
virtual size_t getWorkspaceSize(int32_t maxBatchSize) const noexcept=0
Find the workspace size required by the layer.
virtual int32_t getNbOutputs() const noexcept=0
Get the number of outputs from the layer.
virtual AsciiChar const * getPluginNamespace() const noexcept=0
Return the namespace of the plugin object.
virtual int32_t initialize() noexcept=0
Initialize the layer for execution. This is called when the engine is created.
virtual bool supportsFormat(DataType type, PluginFormat format) const noexcept=0
Check format support.
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:879
int32_t getTensorRTVersion() const noexcept override
Return the API version with which this plugin was built. The upper byte is reserved by TensorRT and i...
Definition: NvInferRuntimeCommon.h:967
virtual void configurePlugin(PluginTensorDesc const *in, int32_t nbInput, PluginTensorDesc const *out, int32_t nbOutput) noexcept=0
Configure the layer.
virtual bool supportsFormatCombination(int32_t pos, PluginTensorDesc const *inOut, int32_t nbInputs, int32_t nbOutputs) const noexcept=0
Return true if plugin supports the format and datatype for the input/output indexed by pos.
Structure containing plugin attribute field names and associated data This information can be parsed ...
Definition: NvInferRuntimeCommon.h:1022
AsciiChar const * name
Plugin field attribute name.
Definition: NvInferRuntimeCommon.h:1027
PluginField(AsciiChar const *const name_=nullptr, void const *const data_=nullptr, PluginFieldType const type_=PluginFieldType::kUNKNOWN, int32_t const length_=0) noexcept
Definition: NvInferRuntimeCommon.h:1042
void const * data
Plugin field attribute data.
Definition: NvInferRuntimeCommon.h:1031
int32_t length
Number of data entries in the Plugin attribute.
Definition: NvInferRuntimeCommon.h:1040
PluginFieldType type
Plugin field attribute type.
Definition: NvInferRuntimeCommon.h:1036
The TensorRT API version 1 namespace.
ErrorCode
Error codes that can be returned by TensorRT during execution.
Definition: NvInferRuntimeCommon.h:1536
PluginFieldType
Definition: NvInferRuntimeCommon.h:993
@ kUNKNOWN
Unknown field type.
@ kFLOAT32
FP32 field type.
@ kCHAR
char field type.
@ kINT16
INT16 field type.
@ kDIMS
nvinfer1::Dims field type.
@ kFLOAT64
FP64 field type.
@ kFLOAT16
FP16 field type.
char_t AsciiChar
AsciiChar is the type used by TensorRT to represent valid ASCII characters.
Definition: NvInferRuntimeCommon.h:88
char char_t
char_t is the type used by TensorRT to represent all valid characters.
Definition: NvInferRuntimeCommon.h:86
@ kV2_DYNAMICEXT
IPluginV2DynamicExt.
@ kV2_IOEXT
IPluginV2IOExt.
@ kV2_EXT
IPluginV2Ext.
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:114
@ kFLOAT
32-bit floating point format.
@ kBOOL
8-bit boolean. 0 = false, 1 = true, other values undefined.
@ kHALF
IEEE 16-bit floating-point format.
@ kINT8
8-bit integer representing a quantized floating-point value.
@ kINT32
Signed 32-bit integer format.
TensorFormat PluginFormat
PluginFormat is reserved for backward compatibility.
Definition: NvInferRuntimeCommon.h:308
@ kINT8
Enable Int8 layer selection, with FP32 fallback with FP16 fallback if kFP16 also specified.
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:184
constexpr int32_t EnumMax() noexcept
Maximum number of elements in an enumeration type.
Definition: NvInferRuntimeCommon.h:104
AllocatorFlag
Definition: NvInferRuntimeCommon.h:1312
@ kRESIZABLE
TensorRT may call realloc() on this allocation.
uint32_t AllocatorFlags
Definition: NvInferRuntimeCommon.h:1326
Definition of plugin versions.
Plugin field collection struct.
Definition: NvInferRuntimeCommon.h:1054
PluginField const * fields
Pointer to PluginField entries.
Definition: NvInferRuntimeCommon.h:1058
int32_t nbFields
Number of PluginField entries.
Definition: NvInferRuntimeCommon.h:1056
Fields that a plugin might see for an input or output.
Definition: NvInferRuntimeCommon.h:332
DataType type
Definition: NvInferRuntimeCommon.h:336
Dims dims
Dimensions.
Definition: NvInferRuntimeCommon.h:334
TensorFormat format
Tensor format.
Definition: NvInferRuntimeCommon.h:338
float scale
Scale for INT8 data type.
Definition: NvInferRuntimeCommon.h:340
Declaration of EnumMaxImpl struct to store maximum number of elements in an enumeration type.
Definition: NvInferRuntimeCommon.h:99