diff --git a/backends/cuda/Makefile.am b/backends/cuda/Makefile.am index 342b869e2..85f1999c3 100644 --- a/backends/cuda/Makefile.am +++ b/backends/cuda/Makefile.am @@ -231,7 +231,7 @@ tracer_cudart.c: $(srcdir)/gen_cudart.rb $(srcdir)/tracer_cudart_helpers.include SRC_DIR=$(srcdir) $(RUBY) $< > $@ tracer_cudart.map: $(srcdir)/gen_cudart_linker_script.rb $(CUDART_MODEL) - SRC_DIR=$(srcdir) $(RUBY) $< libcudart.so.12 > $@ + SRC_DIR=$(srcdir) $(RUBY) $< libcudart.so.13 > $@ EXTRA_DIST += \ gen_cuda.rb \ @@ -297,7 +297,7 @@ libcudart_la_DEPENDENCIES = tracer_cudart.map libcudarttracepoints.la libcudart_la_CPPFLAGS = -I$(top_srcdir)/utils -I$(top_srcdir)/utils/include -I$(srcdir)/include -I./ libcudart_la_CFLAGS = -Wall -Wextra $(WERROR) $(LIBFFI_CFLAGS) $(LTTNG_UST_CFLAGS) libcudart_la_LDFLAGS = $(LTTNG_UST_LIBS) -ldl -lpthread $(LIBFFI_LIBS) -libcudart_la_LDFLAGS += -version-number 12:1:55 -Wl,--version-script,tracer_cudart.map +libcudart_la_LDFLAGS += -version-number 13:0:48 -Wl,--version-script,tracer_cudart.map libcudart_la_LIBADD = libcudarttracepoints.la tmplibdir = $(libdir)/tmp diff --git a/backends/cuda/cuda_meta_parameters.yaml b/backends/cuda/cuda_meta_parameters.yaml index fe2275fb0..d2f083804 100644 --- a/backends/cuda/cuda_meta_parameters.yaml +++ b/backends/cuda/cuda_meta_parameters.yaml @@ -838,3 +838,243 @@ meta_parameters: cuGetExportTable: - [InScalar, pExportTableId] - [OutScalar, ppExportTable] + cuCheckpointProcessCheckpoint: + - [InScalar, args] + cuCheckpointProcessGetRestoreThreadId: + - [OutScalar, tid] + cuCheckpointProcessGetState: + - [OutScalar, state] + cuCheckpointProcessLock: + - [InScalar, args] + cuCheckpointProcessRestore: + - [InScalar, args] + - [InArray, args->gpuPairs, args->gpuPairsCount] + cuCheckpointProcessUnlock: + - [InScalar, args] + cuCtxCreate_v4: + - [OutScalar, pctx] + - [InScalar, ctxCreateParams] + - [InArray, ctxCreateParams->execAffinityParams, ctxCreateParams->numExecAffinityParams] + - [InScalar, ctxCreateParams->cigParams] + cuCtxFromGreenCtx: + - [OutScalar, pContext] + cuCtxGetDevice_v2: + - [OutScalar, device] + cuCtxGetDevResource: + - [OutScalar, resource] + cuDeviceGetDevResource: + - [OutScalar, resource] + cuDeviceGetHostAtomicCapabilities: + - [OutArray, capabilities, count] + - [InArray, operations, count] + cuDeviceGetP2PAtomicCapabilities: + - [OutArray, capabilities, count] + - [InArray, operations, count] + cuDeviceRegisterAsyncNotification: + - [OutScalar, callback] + cuDevResourceGenerateDesc: + - [OutScalar, phDesc] + - [InArray, resources, nbResources] + cuDevSmResourceSplitByCount: + - [InOutScalar, nbGroups] + - [OutArray, result, nbGroups] + - [InScalar, input] + - [OutScalar, remaining] + cuEventElapsedTime_v2: + - [OutScalar, pMilliseconds] + cuFuncGetName: + - [OutPtrString, name] + cuFuncGetParamInfo: + - [OutScalar, paramOffset] + - [OutScalar, paramSize] + cuFuncIsLoaded: + - [OutScalar, state] + cuGraphAddDependencies_v2: + - [InArray, from, numDependencies] + - [InArray, to, numDependencies] + - [InArray, edgeData, numDependencies] + cuGraphAddNode: + - [OutScalar, phGraphNode] + - [InArray, dependencies, numDependencies] + - [InScalar, nodeParams] + cuGraphAddNode_v2: + - [OutScalar, phGraphNode] + - [InArray, dependencies, numDependencies] + - [InArray, dependencyData, numDependencies] + - [InScalar, nodeParams] + cuGraphConditionalHandleCreate: + - [OutScalar, pHandle_out] + cuGraphExecNodeSetParams: + - [InScalar, nodeParams] + cuGraphGetEdges_v2: + - [InOutScalar, numEdges] + - [OutArray, from, numEdges] + - [OutArray, to, numEdges] + - [OutArray, edgeData, numEdges] + cuGraphNodeGetDependencies_v2: + - [InOutScalar, numDependencies] + - [OutArray, dependencies, numDependencies] + - [OutArray, edgeData, numDependencies] + cuGraphNodeGetDependentNodes_v2: + - [InOutScalar, numDependentNodes] + - [OutArray, dependentNodes, numDependentNodes] + - [OutArray, edgeData, numDependentNodes] + cuGraphNodeSetParams: + - [InScalar, nodeParams] + cuGraphRemoveDependencies_v2: + - [InArray, from, numDependencies] + - [InArray, to, numDependencies] + - [InArray, edgeData, numDependencies] + cuGreenCtxCreate: + - [OutScalar, phCtx] + cuGreenCtxGetDevResource: + - [OutScalar, resource] + cuGreenCtxGetId: + - [OutScalar, greenCtxId] + cuGreenCtxStreamCreate: + - [OutScalar, phStream] + cuKernelGetLibrary: + - [OutScalar, pLib] + cuKernelGetName: + - [OutPtrString, name] + cuKernelGetParamInfo: + - [OutScalar, paramOffset] + - [OutScalar, paramSize] + cuLibraryEnumerateKernels: + - [OutArray, kernels, numKernels] + cuLibraryGetKernelCount: + - [OutScalar, count] + cuLogsCurrent: + - [OutScalar, iterator_out] + cuLogsDumpToFile: + - [InScalar, iterator] + - [InString, pathToFile] + cuLogsDumpToMemory: + - [InScalar, iterator] + - [InOutScalar, size] + - [OutArray, buffer, size] + cuLogsRegisterCallback: + - [OutScalar, callback_out] + cuMemBatchDecompressAsync: + - [InArray, paramsArray, count] + - [OutScalar, errorIndex] + cuMemBatchDecompressAsync_ptsz: + - [InArray, paramsArray, count] + - [OutScalar, errorIndex] + cuMemcpy3DBatchAsync: + - [InArray, opList, numOps] + - [OutScalar, failIdx] + cuMemcpy3DBatchAsync_ptsz: + - [InArray, opList, numOps] + - [OutScalar, failIdx] + cuMemcpy3DBatchAsync_v2: + - [InArray, opList, numOps] + cuMemcpy3DBatchAsync_v2_ptsz: + - [InArray, opList, numOps] + cuMemcpyBatchAsync: + - [InArray, dsts, count] + - [InArray, srcs, count] + - [InArray, sizes, count] + - [InArray, attrs, numAttrs] + - [InArray, attrsIdxs, numAttrs] + - [OutScalar, failIdx] + cuMemcpyBatchAsync_ptsz: + - [InArray, dsts, count] + - [InArray, srcs, count] + - [InArray, sizes, count] + - [InArray, attrs, numAttrs] + - [InArray, attrsIdxs, numAttrs] + - [OutScalar, failIdx] + cuMemcpyBatchAsync_v2: + - [InArray, dsts, count] + - [InArray, srcs, count] + - [InArray, sizes, count] + - [InArray, attrs, numAttrs] + - [InArray, attrsIdxs, numAttrs] + cuMemcpyBatchAsync_v2_ptsz: + - [InArray, dsts, count] + - [InArray, srcs, count] + - [InArray, sizes, count] + - [InArray, attrs, numAttrs] + - [InArray, attrsIdxs, numAttrs] + cuMemDiscardAndPrefetchBatchAsync: + - [InArray, dptrs, count] + - [InArray, sizes, count] + - [InArray, prefetchLocs, numPrefetchLocs] + - [InArray, prefetchLocIdxs, numPrefetchLocs] + cuMemDiscardAndPrefetchBatchAsync_ptsz: + - [InArray, dptrs, count] + - [InArray, sizes, count] + - [InArray, prefetchLocs, numPrefetchLocs] + - [InArray, prefetchLocIdxs, numPrefetchLocs] + cuMemDiscardBatchAsync: + - [InArray, dptrs, count] + - [InArray, sizes, count] + cuMemDiscardBatchAsync_ptsz: + - [InArray, dptrs, count] + - [InArray, sizes, count] + cuMemGetDefaultMemPool: + - [OutScalar, pool_out] + - [InScalar, location] + cuMemGetMemPool: + - [OutScalar, pool] + - [InScalar, location] + cuMemPrefetchBatchAsync: + - [InArray, dptrs, count] + - [InArray, sizes, count] + - [InArray, prefetchLocs, numPrefetchLocs] + - [InArray, prefetchLocIdxs, numPrefetchLocs] + cuMemPrefetchBatchAsync_ptsz: + - [InArray, dptrs, count] + - [InArray, sizes, count] + - [InArray, prefetchLocs, numPrefetchLocs] + - [InArray, prefetchLocIdxs, numPrefetchLocs] + cuMemSetMemPool: + - [InScalar, location] + cuModuleEnumerateFunctions: + - [OutArray, functions, numFunctions] + cuModuleGetFunctionCount: + - [OutScalar, count] + cuStreamBeginCaptureToGraph: + - [InArray, dependencies, numDependencies] + - [InArray, dependencyData, numDependencies] + cuStreamBeginCaptureToGraph_ptsz: + - [InArray, dependencies, numDependencies] + - [InArray, dependencyData, numDependencies] + cuStreamGetCaptureInfo_v3: + - [OutScalar, captureStatus_out] + - [OutScalar, id_out] + - [OutScalar, graph_out] + - [OutScalar, numDependencies_out] + - [OutArrayByRef, dependencies_out, numDependencies_out] + - [OutArrayByRef, edgeData_out, numDependencies_out] + cuStreamGetCaptureInfo_v3_ptsz: + - [OutScalar, captureStatus_out] + - [OutScalar, id_out] + - [OutScalar, graph_out] + - [OutScalar, numDependencies_out] + - [OutArrayByRef, dependencies_out, numDependencies_out] + - [OutArrayByRef, edgeData_out, numDependencies_out] + cuStreamGetCtx_v2: + - [OutScalar, pCtx] + - [OutScalar, pGreenCtx] + cuStreamGetCtx_v2_ptsz: + - [OutScalar, pCtx] + - [OutScalar, pGreenCtx] + cuStreamGetDevice: + - [OutScalar, device] + cuStreamGetDevice_ptsz: + - [OutScalar, device] + cuStreamGetGreenCtx: + - [OutScalar, phCtx] + cuStreamUpdateCaptureDependencies_v2: + - [InArray, dependencies, numDependencies] + - [InArray, dependencyData, numDependencies] + cuStreamUpdateCaptureDependencies_v2_ptsz: + - [InArray, dependencies, numDependencies] + - [InArray, dependencyData, numDependencies] + cuTensorMapEncodeIm2colWide: + - [OutScalar, tensorMap] + - [InArray, globalDim, tensorRank] + - [InArray, globalStrides, tensorRank] + - [InArray, elementStrides, tensorRank] diff --git a/backends/cuda/cuda_model.rb b/backends/cuda/cuda_model.rb index a02a0b065..d809c751d 100644 --- a/backends/cuda/cuda_model.rb +++ b/backends/cuda/cuda_model.rb @@ -167,7 +167,8 @@ def upper_snake_case(str) } profiling_start_no_stream = profiling_start.call('NULL') -profiling_start_stream = profiling_start.call('hStream') +profiling_start_hstream = profiling_start.call('hStream') +profiling_start_stream = profiling_start.call('stream') profiling_start_config = <(m) { m.name != 'CU_COREDUMP_LIGHTWEIGHT_FLAGS' }, + fix_values: ->(v) { v.gsub('1u <<', '1 <<') } + ) end def print_cuda_object(object) @@ -13,10 +17,17 @@ def print_cuda_object(object) puts <<~EOF module CUDA CU_COMPUTE_ACCELERATED_TARGET_BASE = 0x10000 + CU_COMPUTE_FAMILY_TARGET_BASE = 0x20000 CU_TARGET_COMPUTE_90 = 90 + CU_TARGET_COMPUTE_100 = 100 + CU_TARGET_COMPUTE_103 = 103 + CU_TARGET_COMPUTE_110 = 110 + CU_TARGET_COMPUTE_120 = 120 + CU_TARGET_COMPUTE_121 = 121 CU_TENSOR_MAP_NUM_QWORDS = 16 CU_IPC_HANDLE_SIZE = 64 CUDA_IPC_HANDLE_SIZE = 64 + RESOURCE_ABI_EXTERNAL_BYTES = 48 extend FFI::Library module Handle @@ -76,11 +87,13 @@ def print_struct(name, struct) typedef :uint64, #{to_ffi_name('CUsurfObject')} typedef :uint64, #{to_ffi_name('CUmemGenericAllocationHandle_v1')} typedef #{to_ffi_name('CUmemGenericAllocationHandle_v1')}, #{to_ffi_name('CUmemGenericAllocationHandle')} + typedef :uint64, #{to_ffi_name('CUgraphConditionalHandle')} EOF $all_types.each do |t| if t.type.is_a? YAMLCAst::Enum enum = $all_enums.find { |e| t.type.name == e.name } + enum ||= t.type print_enum(t.name, enum) elsif $objects.include?(t.name) print_cuda_object(t.name) diff --git a/backends/cuda/gen_cuda_library_base.rb b/backends/cuda/gen_cuda_library_base.rb index 058c72fa4..092ede546 100644 --- a/backends/cuda/gen_cuda_library_base.rb +++ b/backends/cuda/gen_cuda_library_base.rb @@ -89,7 +89,6 @@ def to_name_space(name) $all_types.each do |t| if t.type.is_a? YAMLCAst::Enum - $all_enums.find { |e| t.type.name == e.name } $all_enum_names.push t.name elsif t.type.is_a? YAMLCAst::Struct $all_struct_names.push t.name @@ -110,7 +109,12 @@ def to_ffi else to_ffi_name(type.name) end - length_ = length.is_a?(String) ? length.gsub('sizeof(CUlaunchAttributeID)', '4') : length + length_ = if length.is_a?(String) + length.gsub('sizeof(CUlaunchAttributeID)', '4') + .gsub('sizeof(CUcheckpointGpuPair*)', '8') + else + length + end [t, length_] end end diff --git a/backends/cuda/gen_cudart.rb b/backends/cuda/gen_cudart.rb index 12f010e06..2bddd199f 100644 --- a/backends/cuda/gen_cudart.rb +++ b/backends/cuda/gen_cudart.rb @@ -105,5 +105,5 @@ end $cudart_commands.each do |c| - puts "__asm__(\".symver #{c.name},#{c.name}@@libcudart.so.12, remove\");" + puts "__asm__(\".symver #{c.name},#{c.name}@@libcudart.so.13, remove\");" end diff --git a/backends/cuda/include/crt/host_defines.h b/backends/cuda/include/crt/host_defines.h index 71180be3b..5629b3ebf 100644 --- a/backends/cuda/include/crt/host_defines.h +++ b/backends/cuda/include/crt/host_defines.h @@ -1,5 +1,5 @@ /* - * Copyright 1993-2022 NVIDIA Corporation. All rights reserved. + * Copyright 1993-2025 NVIDIA Corporation. All rights reserved. * * NOTICE TO LICENSEE: * @@ -60,6 +60,14 @@ #if !defined(__HOST_DEFINES_H__) #define __HOST_DEFINES_H__ +#if defined(__CUDACC__) && !defined(__CUDACC_RTC__) && !defined(__CUDADEVRT_INTERNAL__) && !defined(_ALLOW_UNSUPPORTED_LIBCPP) +#include +#if ((defined(_MSC_VER ) && (defined(_M_X64) || defined(_M_AMD64))) ||\ + (defined(__x86_64__) || defined(__amd64__))) && defined(_LIBCPP_VERSION) && !(defined(__HORIZON__) || defined(__ANDROID__) || defined(__QNX__)) +#error "libc++ is not supported on x86 system" +#endif +#endif + /* CUDA JIT mode (__CUDACC_RTC__) also uses GNU style attributes */ #if defined(__GNUC__) || (defined(__PGIC__) && defined(__linux__)) || defined(__CUDA_LIBDEVICE__) || defined(__CUDACC_RTC__) @@ -69,24 +77,18 @@ #define __no_return__ \ __attribute__((noreturn)) - -#if defined(__CUDACC__) || defined(__CUDA_ARCH__) || defined(__CUDA_LIBDEVICE__) -/* gcc allows users to define attributes with underscores, - e.g., __attribute__((__noinline__)). - Consider a non-CUDA source file (e.g. .cpp) that has the - above attribute specification, and includes this header file. In that case, - defining __noinline__ as below would cause a gcc compilation error. - Hence, only define __noinline__ when the code is being processed - by a CUDA compiler component. -*/ -#define __noinline__ \ - __attribute__((noinline)) -#endif /* __CUDACC__ || __CUDA_ARCH__ || __CUDA_LIBDEVICE__ */ - + +#undef __forceinline__ #define __forceinline__ \ __inline__ __attribute__((always_inline)) +#define __inline_hint__ \ + __attribute__((nv_inline_hint)) #define __align__(n) \ __attribute__((aligned(n))) +#define __maxnreg__(a) \ + __attribute__((maxnreg(a))) +#define __local_maxnreg__(a) \ + __attribute__((local_maxnreg(a))) #define __thread__ \ __thread #define __import__ @@ -116,12 +118,16 @@ __inline #define __no_return__ \ __declspec(noreturn) -#define __noinline__ \ - __declspec(noinline) #define __forceinline__ \ __forceinline +#define __inline_hint__ \ + __declspec(nv_inline_hint) #define __align__(n) \ __declspec(align(n)) +#define __maxnreg__(n) \ + __declspec(maxnreg(n)) +#define __local_maxnreg__(n) \ + __declspec(local_maxnreg(n)) #define __thread__ \ __declspec(thread) #define __import__ \ @@ -213,6 +219,18 @@ #define __global__ \ __location__(global) #endif /* defined(__CUDACC__) || !defined(__global__) */ +#if defined(__CUDACC__) || !defined(__tile_global__) +#define __tile_global__ \ + __location__(tile_global) +#endif /* defined(__CUDACC__) || !defined(__tile_global__) */ +#if defined(__CUDACC__) || !defined(__tile__) +#define __tile__ \ + __location__(tile) +#endif /* defined(__CUDACC__) || !defined(__tile__) */ +#if defined(__CUDACC__) || !defined(__tile_builtin__) +#define __tile_builtin__ \ + __location__(tile_builtin) +#endif /* defined(__CUDACC__) || !defined(__tile_builtin__) */ #if defined(__CUDACC__) || !defined(__shared__) #define __shared__ \ __location__(shared) @@ -225,7 +243,10 @@ #define __managed__ \ __location__(managed) #endif /* defined(__CUDACC__) || !defined(__managed__) */ - +#if defined(__CUDACC__) || !defined(__nv_pure__) +#define __nv_pure__ \ + __location__(nv_pure) +#endif /* defined(__CUDACC__) || !defined(__nv_pure__) */ #if !defined(__CUDACC__) #define __device_builtin__ #define __device_builtin_texture_type__ @@ -253,8 +274,37 @@ #endif /* defined(_MSC_VER) */ #endif /* defined(__CUDACC__) || !defined(__cluster_dims__) */ +#if defined(__CUDACC__) || !defined(__block_size__) +#if defined(_MSC_VER) +#define __block_size__(...) \ + __declspec(__block_size__(__VA_ARGS__)) + +#else /* !defined(_MSC_VER) */ +#define __block_size__(...) \ + __attribute__((block_size(__VA_ARGS__))) +#endif /* defined(_MSC_VER) */ +#endif /* defined(__CUDACC__) || !defined(__block_size__) */ + #define __CUDA_ARCH_HAS_FEATURE__(_FEAT) __CUDA_ARCH_FEAT_##_FEAT +#ifndef __CUDA_ARCH_FAMILY_SPECIFIC__ +#define __CUDA_HAS_ARCH_FAMILY_SPECIFIC(N) 0 +#else /* defined(__CUDA_ARCH_FAMILY_SPECIFIC__) */ +#if (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1010) +#define __CUDA_HAS_ARCH_FAMILY_SPECIFIC(N) (__CUDA_ARCH_FAMILY_SPECIFIC__ == (N##0)) +#else +#define __CUDA_HAS_ARCH_FAMILY_SPECIFIC(N) \ + ( (((N) == 101) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1010)) || \ + (((N) != 101) && (__CUDA_ARCH_FAMILY_SPECIFIC__ >= (N##0) && __CUDA_ARCH_FAMILY_SPECIFIC__ < ((N##0) + 100)))) +#endif /* defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1010) */ +#endif /* __CUDA_ARCH_FAMILY_SPECIFIC__ */ + +#ifndef __CUDA_ARCH_SPECIFIC__ +#define __CUDA_HAS_ARCH_SPECIFIC(N) 0 +#else +#define __CUDA_HAS_ARCH_SPECIFIC(N) (__CUDA_ARCH_SPECIFIC__ == (N##0)) +#endif + #endif /* !__HOST_DEFINES_H__ */ #if defined(__UNDEF_CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS_HOST_DEFINES_H__) diff --git a/backends/cuda/include/cuda.h b/backends/cuda/include/cuda.h index a773d7747..30f40f73b 100644 --- a/backends/cuda/include/cuda.h +++ b/backends/cuda/include/cuda.h @@ -1,5 +1,5 @@ /* - * Copyright 1993-2022 NVIDIA Corporation. All rights reserved. + * Copyright 1993-2023 NVIDIA Corporation. All rights reserved. * * NOTICE TO LICENSEE: * @@ -87,8 +87,7 @@ typedef uint64_t cuuint64_t; #endif #define cuDeviceTotalMem cuDeviceTotalMem_v2 -#define cuCtxCreate cuCtxCreate_v2 -#define cuCtxCreate_v3 cuCtxCreate_v3 +#define cuCtxCreate cuCtxCreate_v4 #define cuModuleGetGlobal cuModuleGetGlobal_v2 #define cuMemGetInfo cuMemGetInfo_v2 #define cuMemAlloc cuMemAlloc_v2 @@ -97,6 +96,11 @@ typedef uint64_t cuuint64_t; #define cuMemGetAddressRange cuMemGetAddressRange_v2 #define cuMemAllocHost cuMemAllocHost_v2 #define cuMemHostGetDevicePointer cuMemHostGetDevicePointer_v2 +#define cuMemAdvise cuMemAdvise_v2 +#define cuMemPrefetchAsync __CUDA_API_PTSZ(cuMemPrefetchAsync_v2) +#define cuMemPrefetchBatchAsync __CUDA_API_PTSZ(cuMemPrefetchBatchAsync) +#define cuMemDiscardBatchAsync __CUDA_API_PTSZ(cuMemDiscardBatchAsync) +#define cuMemDiscardAndPrefetchBatchAsync __CUDA_API_PTSZ(cuMemDiscardAndPrefetchBatchAsync) #define cuMemcpyHtoD __CUDA_API_PTDS(cuMemcpyHtoD_v2) #define cuMemcpyDtoH __CUDA_API_PTDS(cuMemcpyDtoH_v2) #define cuMemcpyDtoD __CUDA_API_PTDS(cuMemcpyDtoD_v2) @@ -115,6 +119,8 @@ typedef uint64_t cuuint64_t; #define cuMemcpyDtoDAsync __CUDA_API_PTSZ(cuMemcpyDtoDAsync_v2) #define cuMemcpy2DAsync __CUDA_API_PTSZ(cuMemcpy2DAsync_v2) #define cuMemcpy3DAsync __CUDA_API_PTSZ(cuMemcpy3DAsync_v2) +#define cuMemcpyBatchAsync __CUDA_API_PTSZ(cuMemcpyBatchAsync_v2) +#define cuMemcpy3DBatchAsync __CUDA_API_PTSZ(cuMemcpy3DBatchAsync_v2) #define cuMemsetD8 __CUDA_API_PTDS(cuMemsetD8_v2) #define cuMemsetD16 __CUDA_API_PTDS(cuMemsetD16_v2) #define cuMemsetD32 __CUDA_API_PTDS(cuMemsetD32_v2) @@ -143,7 +149,8 @@ typedef uint64_t cuuint64_t; #define cuDevicePrimaryCtxRelease cuDevicePrimaryCtxRelease_v2 #define cuDevicePrimaryCtxReset cuDevicePrimaryCtxReset_v2 #define cuDevicePrimaryCtxSetFlags cuDevicePrimaryCtxSetFlags_v2 -#define cuDeviceGetUuid_v2 cuDeviceGetUuid_v2 + +#define cuDeviceGetUuid cuDeviceGetUuid_v2 #define cuIpcOpenMemHandle cuIpcOpenMemHandle_v2 #define cuGraphInstantiate cuGraphInstantiateWithFlags @@ -155,13 +162,23 @@ typedef uint64_t cuuint64_t; #define cuGraphKernelNodeSetParams cuGraphKernelNodeSetParams_v2 #define cuGraphExecKernelNodeSetParams cuGraphExecKernelNodeSetParams_v2 +#define cuGraphGetEdges cuGraphGetEdges_v2 +#define cuGraphNodeGetDependencies cuGraphNodeGetDependencies_v2 +#define cuGraphNodeGetDependentNodes cuGraphNodeGetDependentNodes_v2 +#define cuGraphAddDependencies cuGraphAddDependencies_v2 +#define cuGraphRemoveDependencies cuGraphRemoveDependencies_v2 +#define cuGraphAddNode cuGraphAddNode_v2 + +#define cuEventElapsedTime cuEventElapsedTime_v2 + #define cuStreamWriteValue32 __CUDA_API_PTSZ(cuStreamWriteValue32_v2) #define cuStreamWaitValue32 __CUDA_API_PTSZ(cuStreamWaitValue32_v2) #define cuStreamWriteValue64 __CUDA_API_PTSZ(cuStreamWriteValue64_v2) #define cuStreamWaitValue64 __CUDA_API_PTSZ(cuStreamWaitValue64_v2) #define cuStreamBatchMemOp __CUDA_API_PTSZ(cuStreamBatchMemOp_v2) -#define cuStreamGetCaptureInfo __CUDA_API_PTSZ(cuStreamGetCaptureInfo_v2) -#define cuStreamGetCaptureInfo_v2 __CUDA_API_PTSZ(cuStreamGetCaptureInfo_v2) + +#define cuStreamGetCaptureInfo __CUDA_API_PTSZ(cuStreamGetCaptureInfo_v3) +#define cuStreamUpdateCaptureDependencies __CUDA_API_PTSZ(cuStreamUpdateCaptureDependencies_v2) #if defined(__CUDA_API_PER_THREAD_DEFAULT_STREAM) #define cuMemcpy __CUDA_API_PTDS(cuMemcpy) @@ -170,8 +187,6 @@ typedef uint64_t cuuint64_t; #define cuMemcpyPeerAsync __CUDA_API_PTSZ(cuMemcpyPeerAsync) #define cuMemcpy3DPeer __CUDA_API_PTDS(cuMemcpy3DPeer) #define cuMemcpy3DPeerAsync __CUDA_API_PTSZ(cuMemcpy3DPeerAsync) - #define cuMemPrefetchAsync __CUDA_API_PTSZ(cuMemPrefetchAsync) - #define cuMemsetD8Async __CUDA_API_PTSZ(cuMemsetD8Async) #define cuMemsetD16Async __CUDA_API_PTSZ(cuMemsetD16Async) #define cuMemsetD32Async __CUDA_API_PTSZ(cuMemsetD32Async) @@ -182,11 +197,12 @@ typedef uint64_t cuuint64_t; #define cuStreamGetPriority __CUDA_API_PTSZ(cuStreamGetPriority) #define cuStreamGetId __CUDA_API_PTSZ(cuStreamGetId) #define cuStreamGetFlags __CUDA_API_PTSZ(cuStreamGetFlags) + #define cuStreamGetDevice __CUDA_API_PTSZ(cuStreamGetDevice) #define cuStreamGetCtx __CUDA_API_PTSZ(cuStreamGetCtx) + #define cuStreamGetCtx_v2 __CUDA_API_PTSZ(cuStreamGetCtx_v2) #define cuStreamWaitEvent __CUDA_API_PTSZ(cuStreamWaitEvent) #define cuStreamEndCapture __CUDA_API_PTSZ(cuStreamEndCapture) #define cuStreamIsCapturing __CUDA_API_PTSZ(cuStreamIsCapturing) - #define cuStreamUpdateCaptureDependencies __CUDA_API_PTSZ(cuStreamUpdateCaptureDependencies) #define cuStreamAddCallback __CUDA_API_PTSZ(cuStreamAddCallback) #define cuStreamAttachMemAsync __CUDA_API_PTSZ(cuStreamAttachMemAsync) #define cuStreamQuery __CUDA_API_PTSZ(cuStreamQuery) @@ -199,6 +215,7 @@ typedef uint64_t cuuint64_t; #define cuGraphicsMapResources __CUDA_API_PTSZ(cuGraphicsMapResources) #define cuGraphicsUnmapResources __CUDA_API_PTSZ(cuGraphicsUnmapResources) + #define cuLaunchCooperativeKernel __CUDA_API_PTSZ(cuLaunchCooperativeKernel) #define cuSignalExternalSemaphoresAsync __CUDA_API_PTSZ(cuSignalExternalSemaphoresAsync) @@ -215,8 +232,13 @@ typedef uint64_t cuuint64_t; #define cuMemFreeAsync __CUDA_API_PTSZ(cuMemFreeAsync) #define cuMemAllocAsync __CUDA_API_PTSZ(cuMemAllocAsync) #define cuMemAllocFromPoolAsync __CUDA_API_PTSZ(cuMemAllocFromPoolAsync) + + #define cuStreamBeginCaptureToGraph __CUDA_API_PTSZ(cuStreamBeginCaptureToGraph) + #endif +#define cuMemBatchDecompressAsync __CUDA_API_PTSZ(cuMemBatchDecompressAsync) + /** * \file cuda.h * \brief Header file for the CUDA Toolkit application programming interface. @@ -238,7 +260,7 @@ typedef uint64_t cuuint64_t; /** * CUDA API version number */ -#define CUDA_VERSION 12010 +#define CUDA_VERSION 13000 #ifdef __cplusplus extern "C" { @@ -257,7 +279,7 @@ typedef CUdeviceptr_v2 CUdeviceptr; /**< CUDA device po typedef int CUdevice_v1; /**< CUDA device */ typedef CUdevice_v1 CUdevice; /**< CUDA device */ -typedef struct CUctx_st *CUcontext; /**< CUDA context */ +typedef struct CUctx_st *CUcontext; /**< A regular context handle */ typedef struct CUmod_st *CUmodule; /**< CUDA module */ typedef struct CUfunc_st *CUfunction; /**< CUDA function */ typedef struct CUlib_st *CUlibrary; /**< CUDA library */ @@ -280,6 +302,15 @@ typedef struct CUgraphNode_st *CUgraphNode; /**< CUDA graph nod typedef struct CUgraphExec_st *CUgraphExec; /**< CUDA executable graph */ typedef struct CUmemPoolHandle_st *CUmemoryPool; /**< CUDA memory pool */ typedef struct CUuserObject_st *CUuserObject; /**< CUDA user object for graphs */ +typedef cuuint64_t CUgraphConditionalHandle; /**< CUDA graph conditional handle */ +typedef struct CUgraphDeviceUpdatableNode_st *CUgraphDeviceNode; /**< CUDA graph device node handle */ +typedef struct CUasyncCallbackEntry_st *CUasyncCallbackHandle; /**< CUDA async notification callback handle */ +/*! + * \typedef typedef struct CUgreenCtx_st* CUgreenCtx + * A green context handle. This handle can be used safely from only one CPU thread at a time. + * Created via ::cuGreenCtxCreate + */ +typedef struct CUgreenCtx_st *CUgreenCtx; #ifndef CU_UUID_HAS_BEEN_DEFINED #define CU_UUID_HAS_BEEN_DEFINED @@ -293,6 +324,17 @@ typedef struct CUuuid_st { /**< CUDA definition o */ #define CU_IPC_HANDLE_SIZE 64 +/** + * Fabric handle - An opaque handle representing a memory allocation + * that can be exported to processes in same or different nodes. For IPC + * between processes on different nodes they must be connected via the + * NVSwitch fabric. + */ +typedef struct CUmemFabricHandle_st { + unsigned char data[CU_IPC_HANDLE_SIZE]; +} CUmemFabricHandle_v1; +typedef CUmemFabricHandle_v1 CUmemFabricHandle; + /** * CUDA IPC event handle */ @@ -344,7 +386,7 @@ typedef enum CUctx_flags_enum { CU_CTX_LMEM_RESIZE_TO_MAX = 0x10, /**< Keep local memory allocation after launch */ CU_CTX_COREDUMP_ENABLE = 0x20, /**< Trigger coredumps from exceptions in this context */ CU_CTX_USER_COREDUMP_ENABLE= 0x40, /**< Enable user pipe to trigger coredumps in this context */ - CU_CTX_SYNC_MEMOPS = 0x80, /**< Force synchronous blocking on cudaMemcpy/cudaMemset */ + CU_CTX_SYNC_MEMOPS = 0x80, /**< Ensure synchronous memory operations on this context will synchronize */ CU_CTX_FLAGS_MASK = 0xFF } CUctx_flags; @@ -488,7 +530,7 @@ typedef enum CUstreamBatchMemOpType_enum { } CUstreamBatchMemOpType; /** - * Flags for ::cuStreamMemoryBarrier + * Flags for ::CUstreamBatchMemOpParams::memoryBarrier */ typedef enum CUstreamMemoryBarrier_flags_enum { CU_STREAM_MEMORY_BARRIER_TYPE_SYS = 0x0, /**< System-wide memory barrier. */ @@ -499,7 +541,7 @@ typedef enum CUstreamMemoryBarrier_flags_enum { * Per-operation parameters for ::cuStreamBatchMemOp */ typedef union CUstreamBatchMemOpParams_union { - CUstreamBatchMemOpType operation; + CUstreamBatchMemOpType operation; /**< Operation. This is the first field of all the union elemets and acts as a TAG to determine which union member is valid.*/ struct CUstreamMemOpWaitValueParams_st { CUstreamBatchMemOpType operation; CUdeviceptr address; @@ -507,9 +549,9 @@ typedef union CUstreamBatchMemOpParams_union { cuuint32_t value; cuuint64_t value64; }; - unsigned int flags; + unsigned int flags; /**< See ::CUstreamWaitValue_flags.*/ CUdeviceptr alias; /**< For driver internal use. Initial value is unimportant. */ - } waitValue; + } waitValue; /**< Params for ::CU_STREAM_MEM_OP_WAIT_VALUE_32 and ::CU_STREAM_MEM_OP_WAIT_VALUE_64 operations.*/ struct CUstreamMemOpWriteValueParams_st { CUstreamBatchMemOpType operation; CUdeviceptr address; @@ -517,27 +559,44 @@ typedef union CUstreamBatchMemOpParams_union { cuuint32_t value; cuuint64_t value64; }; - unsigned int flags; - CUdeviceptr alias; /**< For driver internal use. Initial value is unimportant. */ - } writeValue; + unsigned int flags; /**< See ::CUstreamWriteValue_flags.*/ + CUdeviceptr alias; /**< For driver internal use. Initial value is unimportant. */ + } writeValue; /**< Params for ::CU_STREAM_MEM_OP_WRITE_VALUE_32 and ::CU_STREAM_MEM_OP_WRITE_VALUE_64 operations.*/ struct CUstreamMemOpFlushRemoteWritesParams_st { CUstreamBatchMemOpType operation; - unsigned int flags; - } flushRemoteWrites; + unsigned int flags; /**< Must be 0.*/ + } flushRemoteWrites; /**< Params for ::CU_STREAM_MEM_OP_FLUSH_REMOTE_WRITES operations.*/ struct CUstreamMemOpMemoryBarrierParams_st { /**< Only supported in the _v2 API */ CUstreamBatchMemOpType operation; - unsigned int flags; - } memoryBarrier; + unsigned int flags; /**< See ::CUstreamMemoryBarrier_flags */ + } memoryBarrier; /**< Params for ::CU_STREAM_MEM_OP_BARRIER operations.*/ cuuint64_t pad[6]; } CUstreamBatchMemOpParams_v1; typedef CUstreamBatchMemOpParams_v1 CUstreamBatchMemOpParams; -typedef struct CUDA_BATCH_MEM_OP_NODE_PARAMS_st { +/** + * Batch memory operation node parameters + * + * Used in the legacy ::cuGraphAddBatchMemOpNode api. + * New code should use ::cuGraphAddNode() + */ +typedef struct CUDA_BATCH_MEM_OP_NODE_PARAMS_v1_st { CUcontext ctx; unsigned int count; CUstreamBatchMemOpParams *paramArray; unsigned int flags; -} CUDA_BATCH_MEM_OP_NODE_PARAMS; +} CUDA_BATCH_MEM_OP_NODE_PARAMS_v1; +typedef CUDA_BATCH_MEM_OP_NODE_PARAMS_v1 CUDA_BATCH_MEM_OP_NODE_PARAMS; + +/** + * Batch memory operation node parameters + */ +typedef struct CUDA_BATCH_MEM_OP_NODE_PARAMS_v2_st { + CUcontext ctx; /**< Context to use for the operations. */ + unsigned int count; /**< Number of operations in paramArray. */ + CUstreamBatchMemOpParams *paramArray; /**< Array of batch memory operations. */ + unsigned int flags; /**< Flags to control the node. */ +} CUDA_BATCH_MEM_OP_NODE_PARAMS_v2; /** * Occupancy calculator flag @@ -555,45 +614,89 @@ typedef enum CUstreamUpdateCaptureDependencies_flags_enum { CU_STREAM_SET_CAPTURE_DEPENDENCIES = 0x1 /**< Replace the dependency set with the new nodes */ } CUstreamUpdateCaptureDependencies_flags; +/** +* Types of async notification that can be sent +*/ +typedef enum CUasyncNotificationType_enum { + CU_ASYNC_NOTIFICATION_TYPE_OVER_BUDGET = 0x1 /**< Sent when the process has exceeded its device memory budget */ +} CUasyncNotificationType; + +/** +* Information passed to the user via the async notification callback +*/ +typedef struct CUasyncNotificationInfo_st { + CUasyncNotificationType type; /**< The type of notification being sent */ + union { + struct { + unsigned long long bytesOverBudget; /**< The number of bytes that the process has allocated above its device memory budget */ + } overBudget; /**< Information about notifications of type \p CU_ASYNC_NOTIFICATION_TYPE_OVER_BUDGET */ + } info; /**< Information about the notification. \p type must be checked in order to interpret this field. */ +} CUasyncNotificationInfo; + +/** + * CUDA async notification callback + * \param info Information describing what actions to take as a result of this notification. + * \param userData Pointer to user defined data provided at callback registration. + * \param callback The callback handle associated with this specific callback. + */ +typedef void (*CUasyncCallback)(CUasyncNotificationInfo *info, void *userData, CUasyncCallbackHandle callback); + /** * Array formats */ typedef enum CUarray_format_enum { - CU_AD_FORMAT_UNSIGNED_INT8 = 0x01, /**< Unsigned 8-bit integers */ - CU_AD_FORMAT_UNSIGNED_INT16 = 0x02, /**< Unsigned 16-bit integers */ - CU_AD_FORMAT_UNSIGNED_INT32 = 0x03, /**< Unsigned 32-bit integers */ - CU_AD_FORMAT_SIGNED_INT8 = 0x08, /**< Signed 8-bit integers */ - CU_AD_FORMAT_SIGNED_INT16 = 0x09, /**< Signed 16-bit integers */ - CU_AD_FORMAT_SIGNED_INT32 = 0x0a, /**< Signed 32-bit integers */ - CU_AD_FORMAT_HALF = 0x10, /**< 16-bit floating point */ - CU_AD_FORMAT_FLOAT = 0x20, /**< 32-bit floating point */ - CU_AD_FORMAT_NV12 = 0xb0, /**< 8-bit YUV planar format, with 4:2:0 sampling */ - CU_AD_FORMAT_UNORM_INT8X1 = 0xc0, /**< 1 channel unsigned 8-bit normalized integer */ - CU_AD_FORMAT_UNORM_INT8X2 = 0xc1, /**< 2 channel unsigned 8-bit normalized integer */ - CU_AD_FORMAT_UNORM_INT8X4 = 0xc2, /**< 4 channel unsigned 8-bit normalized integer */ - CU_AD_FORMAT_UNORM_INT16X1 = 0xc3, /**< 1 channel unsigned 16-bit normalized integer */ - CU_AD_FORMAT_UNORM_INT16X2 = 0xc4, /**< 2 channel unsigned 16-bit normalized integer */ - CU_AD_FORMAT_UNORM_INT16X4 = 0xc5, /**< 4 channel unsigned 16-bit normalized integer */ - CU_AD_FORMAT_SNORM_INT8X1 = 0xc6, /**< 1 channel signed 8-bit normalized integer */ - CU_AD_FORMAT_SNORM_INT8X2 = 0xc7, /**< 2 channel signed 8-bit normalized integer */ - CU_AD_FORMAT_SNORM_INT8X4 = 0xc8, /**< 4 channel signed 8-bit normalized integer */ - CU_AD_FORMAT_SNORM_INT16X1 = 0xc9, /**< 1 channel signed 16-bit normalized integer */ - CU_AD_FORMAT_SNORM_INT16X2 = 0xca, /**< 2 channel signed 16-bit normalized integer */ - CU_AD_FORMAT_SNORM_INT16X4 = 0xcb, /**< 4 channel signed 16-bit normalized integer */ - CU_AD_FORMAT_BC1_UNORM = 0x91, /**< 4 channel unsigned normalized block-compressed (BC1 compression) format */ - CU_AD_FORMAT_BC1_UNORM_SRGB = 0x92, /**< 4 channel unsigned normalized block-compressed (BC1 compression) format with sRGB encoding*/ - CU_AD_FORMAT_BC2_UNORM = 0x93, /**< 4 channel unsigned normalized block-compressed (BC2 compression) format */ - CU_AD_FORMAT_BC2_UNORM_SRGB = 0x94, /**< 4 channel unsigned normalized block-compressed (BC2 compression) format with sRGB encoding*/ - CU_AD_FORMAT_BC3_UNORM = 0x95, /**< 4 channel unsigned normalized block-compressed (BC3 compression) format */ - CU_AD_FORMAT_BC3_UNORM_SRGB = 0x96, /**< 4 channel unsigned normalized block-compressed (BC3 compression) format with sRGB encoding*/ - CU_AD_FORMAT_BC4_UNORM = 0x97, /**< 1 channel unsigned normalized block-compressed (BC4 compression) format */ - CU_AD_FORMAT_BC4_SNORM = 0x98, /**< 1 channel signed normalized block-compressed (BC4 compression) format */ - CU_AD_FORMAT_BC5_UNORM = 0x99, /**< 2 channel unsigned normalized block-compressed (BC5 compression) format */ - CU_AD_FORMAT_BC5_SNORM = 0x9a, /**< 2 channel signed normalized block-compressed (BC5 compression) format */ - CU_AD_FORMAT_BC6H_UF16 = 0x9b, /**< 3 channel unsigned half-float block-compressed (BC6H compression) format */ - CU_AD_FORMAT_BC6H_SF16 = 0x9c, /**< 3 channel signed half-float block-compressed (BC6H compression) format */ - CU_AD_FORMAT_BC7_UNORM = 0x9d, /**< 4 channel unsigned normalized block-compressed (BC7 compression) format */ - CU_AD_FORMAT_BC7_UNORM_SRGB = 0x9e /**< 4 channel unsigned normalized block-compressed (BC7 compression) format with sRGB encoding */ + CU_AD_FORMAT_UNSIGNED_INT8 = 0x01, /**< Unsigned 8-bit integers */ + CU_AD_FORMAT_UNSIGNED_INT16 = 0x02, /**< Unsigned 16-bit integers */ + CU_AD_FORMAT_UNSIGNED_INT32 = 0x03, /**< Unsigned 32-bit integers */ + CU_AD_FORMAT_SIGNED_INT8 = 0x08, /**< Signed 8-bit integers */ + CU_AD_FORMAT_SIGNED_INT16 = 0x09, /**< Signed 16-bit integers */ + CU_AD_FORMAT_SIGNED_INT32 = 0x0a, /**< Signed 32-bit integers */ + CU_AD_FORMAT_HALF = 0x10, /**< 16-bit floating point */ + CU_AD_FORMAT_FLOAT = 0x20, /**< 32-bit floating point */ + CU_AD_FORMAT_NV12 = 0xb0, /**< 8-bit YUV planar format, with 4:2:0 sampling */ + CU_AD_FORMAT_UNORM_INT8X1 = 0xc0, /**< 1 channel unsigned 8-bit normalized integer */ + CU_AD_FORMAT_UNORM_INT8X2 = 0xc1, /**< 2 channel unsigned 8-bit normalized integer */ + CU_AD_FORMAT_UNORM_INT8X4 = 0xc2, /**< 4 channel unsigned 8-bit normalized integer */ + CU_AD_FORMAT_UNORM_INT16X1 = 0xc3, /**< 1 channel unsigned 16-bit normalized integer */ + CU_AD_FORMAT_UNORM_INT16X2 = 0xc4, /**< 2 channel unsigned 16-bit normalized integer */ + CU_AD_FORMAT_UNORM_INT16X4 = 0xc5, /**< 4 channel unsigned 16-bit normalized integer */ + CU_AD_FORMAT_SNORM_INT8X1 = 0xc6, /**< 1 channel signed 8-bit normalized integer */ + CU_AD_FORMAT_SNORM_INT8X2 = 0xc7, /**< 2 channel signed 8-bit normalized integer */ + CU_AD_FORMAT_SNORM_INT8X4 = 0xc8, /**< 4 channel signed 8-bit normalized integer */ + CU_AD_FORMAT_SNORM_INT16X1 = 0xc9, /**< 1 channel signed 16-bit normalized integer */ + CU_AD_FORMAT_SNORM_INT16X2 = 0xca, /**< 2 channel signed 16-bit normalized integer */ + CU_AD_FORMAT_SNORM_INT16X4 = 0xcb, /**< 4 channel signed 16-bit normalized integer */ + CU_AD_FORMAT_BC1_UNORM = 0x91, /**< 4 channel unsigned normalized block-compressed (BC1 compression) format */ + CU_AD_FORMAT_BC1_UNORM_SRGB = 0x92, /**< 4 channel unsigned normalized block-compressed (BC1 compression) format with sRGB encoding*/ + CU_AD_FORMAT_BC2_UNORM = 0x93, /**< 4 channel unsigned normalized block-compressed (BC2 compression) format */ + CU_AD_FORMAT_BC2_UNORM_SRGB = 0x94, /**< 4 channel unsigned normalized block-compressed (BC2 compression) format with sRGB encoding*/ + CU_AD_FORMAT_BC3_UNORM = 0x95, /**< 4 channel unsigned normalized block-compressed (BC3 compression) format */ + CU_AD_FORMAT_BC3_UNORM_SRGB = 0x96, /**< 4 channel unsigned normalized block-compressed (BC3 compression) format with sRGB encoding*/ + CU_AD_FORMAT_BC4_UNORM = 0x97, /**< 1 channel unsigned normalized block-compressed (BC4 compression) format */ + CU_AD_FORMAT_BC4_SNORM = 0x98, /**< 1 channel signed normalized block-compressed (BC4 compression) format */ + CU_AD_FORMAT_BC5_UNORM = 0x99, /**< 2 channel unsigned normalized block-compressed (BC5 compression) format */ + CU_AD_FORMAT_BC5_SNORM = 0x9a, /**< 2 channel signed normalized block-compressed (BC5 compression) format */ + CU_AD_FORMAT_BC6H_UF16 = 0x9b, /**< 3 channel unsigned half-float block-compressed (BC6H compression) format */ + CU_AD_FORMAT_BC6H_SF16 = 0x9c, /**< 3 channel signed half-float block-compressed (BC6H compression) format */ + CU_AD_FORMAT_BC7_UNORM = 0x9d, /**< 4 channel unsigned normalized block-compressed (BC7 compression) format */ + CU_AD_FORMAT_BC7_UNORM_SRGB = 0x9e, /**< 4 channel unsigned normalized block-compressed (BC7 compression) format with sRGB encoding */ + CU_AD_FORMAT_P010 = 0x9f, /**< 10-bit YUV planar format, with 4:2:0 sampling */ + CU_AD_FORMAT_P016 = 0xa1, /**< 16-bit YUV planar format, with 4:2:0 sampling */ + CU_AD_FORMAT_NV16 = 0xa2, /**< 8-bit YUV planar format, with 4:2:2 sampling */ + CU_AD_FORMAT_P210 = 0xa3, /**< 10-bit YUV planar format, with 4:2:2 sampling */ + CU_AD_FORMAT_P216 = 0xa4, /**< 16-bit YUV planar format, with 4:2:2 sampling */ + CU_AD_FORMAT_YUY2 = 0xa5, /**< 2 channel, 8-bit YUV packed planar format, with 4:2:2 sampling */ + CU_AD_FORMAT_Y210 = 0xa6, /**< 2 channel, 10-bit YUV packed planar format, with 4:2:2 sampling */ + CU_AD_FORMAT_Y216 = 0xa7, /**< 2 channel, 16-bit YUV packed planar format, with 4:2:2 sampling */ + CU_AD_FORMAT_AYUV = 0xa8, /**< 4 channel, 8-bit YUV packed planar format, with 4:4:4 sampling */ + CU_AD_FORMAT_Y410 = 0xa9, /**< 10-bit YUV packed planar format, with 4:4:4 sampling */ + CU_AD_FORMAT_Y416 = 0xb1, /**< 4 channel, 12-bit YUV packed planar format, with 4:4:4 sampling */ + CU_AD_FORMAT_Y444_PLANAR8 = 0xb2, /**< 3 channel 8-bit YUV planar format, with 4:4:4 sampling */ + CU_AD_FORMAT_Y444_PLANAR10 = 0xb3, /**< 3 channel 10-bit YUV planar format, with 4:4:4 sampling */ + CU_AD_FORMAT_YUV444_8bit_SemiPlanar = 0xb4, /**< 3 channel 8-bit YUV semi-planar format, with 4:4:4 sampling */ + CU_AD_FORMAT_YUV444_16bit_SemiPlanar = 0xb5, /**< 3 channel 16-bit YUV semi-planar format, with 4:4:4 sampling */ + CU_AD_FORMAT_UNORM_INT_101010_2 = 0x50, /**< 4 channel unorm R10G10B10A2 RGB format */ + CU_AD_FORMAT_MAX = 0x7FFFFFFF } CUarray_format; /** @@ -708,7 +811,7 @@ typedef enum CUdevice_attribute_enum { CU_DEVICE_ATTRIBUTE_MANAGED_MEMORY = 83, /**< Device can allocate managed memory on this system */ CU_DEVICE_ATTRIBUTE_MULTI_GPU_BOARD = 84, /**< Device is on a multi-GPU board */ CU_DEVICE_ATTRIBUTE_MULTI_GPU_BOARD_GROUP_ID = 85, /**< Unique id for a group of devices on the same multi-GPU board */ - CU_DEVICE_ATTRIBUTE_HOST_NATIVE_ATOMIC_SUPPORTED = 86, /**< Link between the device and the host supports native atomic operations (this is a placeholder attribute, and is not supported on any current hardware)*/ + CU_DEVICE_ATTRIBUTE_HOST_NATIVE_ATOMIC_SUPPORTED = 86, /**< Link between the device and the host supports all native atomic operations */ CU_DEVICE_ATTRIBUTE_SINGLE_TO_DOUBLE_PRECISION_PERF_RATIO = 87, /**< Ratio of single precision performance (in floating-point operations per second) to double precision performance */ CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS = 88, /**< Device supports coherently accessing pageable memory without calling cudaHostRegister on it */ CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS = 89, /**< Device can coherently access managed memory concurrently with the CPU */ @@ -751,8 +854,26 @@ typedef enum CUdevice_attribute_enum { CU_DEVICE_ATTRIBUTE_IPC_EVENT_SUPPORTED = 125, /**< Device supports IPC Events. */ CU_DEVICE_ATTRIBUTE_MEM_SYNC_DOMAIN_COUNT = 126, /**< Number of memory domains the device supports. */ CU_DEVICE_ATTRIBUTE_TENSOR_MAP_ACCESS_SUPPORTED = 127, /**< Device supports accessing memory using Tensor Map. */ + CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_FABRIC_SUPPORTED = 128, /**< Device supports exporting memory to a fabric handle with cuMemExportToShareableHandle() or requested with cuMemCreate() */ CU_DEVICE_ATTRIBUTE_UNIFIED_FUNCTION_POINTERS = 129, /**< Device supports unified function pointers. */ + CU_DEVICE_ATTRIBUTE_NUMA_CONFIG = 130, /**< NUMA configuration of a device: value is of type ::CUdeviceNumaConfig enum */ + CU_DEVICE_ATTRIBUTE_NUMA_ID = 131, /**< NUMA node ID of the GPU memory */ CU_DEVICE_ATTRIBUTE_MULTICAST_SUPPORTED = 132, /**< Device supports switch multicast and reduction operations. */ + CU_DEVICE_ATTRIBUTE_MPS_ENABLED = 133, /**< Indicates if contexts created on this device will be shared via MPS */ + CU_DEVICE_ATTRIBUTE_HOST_NUMA_ID = 134, /**< NUMA ID of the host node closest to the device. Returns -1 when system does not support NUMA. */ + CU_DEVICE_ATTRIBUTE_D3D12_CIG_SUPPORTED = 135, /**< Device supports CIG with D3D12. */ + CU_DEVICE_ATTRIBUTE_MEM_DECOMPRESS_ALGORITHM_MASK = 136, /**< The returned valued shall be interpreted as a bitmask, where the individual bits are described by the ::CUmemDecompressAlgorithm enum. */ + CU_DEVICE_ATTRIBUTE_MEM_DECOMPRESS_MAXIMUM_LENGTH = 137, /**< The returned valued is the maximum length in bytes of a single decompress operation that is allowed. */ + CU_DEVICE_ATTRIBUTE_VULKAN_CIG_SUPPORTED = 138, /**< Device supports CIG with Vulkan. */ + CU_DEVICE_ATTRIBUTE_GPU_PCI_DEVICE_ID = 139, /**< The combined 16-bit PCI device ID and 16-bit PCI vendor ID. */ + CU_DEVICE_ATTRIBUTE_GPU_PCI_SUBSYSTEM_ID = 140, /**< The combined 16-bit PCI subsystem ID and 16-bit PCI subsystem vendor ID. */ + CU_DEVICE_ATTRIBUTE_HOST_NUMA_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED = 141, /**< Device supports HOST_NUMA location with the virtual memory management APIs like ::cuMemCreate, ::cuMemMap and related APIs */ + CU_DEVICE_ATTRIBUTE_HOST_NUMA_MEMORY_POOLS_SUPPORTED = 142, /**< Device supports HOST_NUMA location with the ::cuMemAllocAsync and ::cuMemPool family of APIs */ + CU_DEVICE_ATTRIBUTE_HOST_NUMA_MULTINODE_IPC_SUPPORTED = 143, /**< Device supports HOST_NUMA location IPC between nodes in a multi-node system. */ + CU_DEVICE_ATTRIBUTE_HOST_MEMORY_POOLS_SUPPORTED = 144, /**< Device suports HOST location with the ::cuMemAllocAsync and ::cuMemPool family of APIs */ + CU_DEVICE_ATTRIBUTE_HOST_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED = 145, /**< Device supports HOST location with the virtual memory management APIs like ::cuMemCreate, ::cuMemMap and related APIs */ + CU_DEVICE_ATTRIBUTE_HOST_ALLOC_DMA_BUF_SUPPORTED = 146, /**< Device supports page-locked host memory buffer sharing with dma_buf mechanism. */ + CU_DEVICE_ATTRIBUTE_ONLY_PARTIAL_HOST_NATIVE_ATOMIC_SUPPORTED = 147, /**< Link between the device and the host supports only some native atomic operations */ CU_DEVICE_ATTRIBUTE_MAX } CUdevice_attribute; @@ -793,11 +914,11 @@ typedef enum CUpointer_attribute_enum { CU_POINTER_ATTRIBUTE_ALLOWED_HANDLE_TYPES = 14, /**< Bitmask of allowed ::CUmemAllocationHandleType for this allocation **/ CU_POINTER_ATTRIBUTE_IS_GPU_DIRECT_RDMA_CAPABLE = 15, /**< 1 if the memory this pointer is referencing can be used with the GPUDirect RDMA API **/ CU_POINTER_ATTRIBUTE_ACCESS_FLAGS = 16, /**< Returns the access flags the device associated with the current context has on the corresponding memory referenced by the pointer given */ - CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE = 17 /**< Returns the mempool handle for the allocation if it was allocated from a mempool. Otherwise returns NULL. **/ - , + CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE = 17, /**< Returns the mempool handle for the allocation if it was allocated from a mempool. Otherwise returns NULL. **/ CU_POINTER_ATTRIBUTE_MAPPING_SIZE = 18, /**< Size of the actual underlying mapping that the pointer belongs to **/ CU_POINTER_ATTRIBUTE_MAPPING_BASE_ADDR = 19, /**< The start address of the mapping that the pointer belongs to **/ CU_POINTER_ATTRIBUTE_MEMORY_BLOCK_ID = 20 /**< A process-wide unique id corresponding to the physical allocation the pointer belongs to **/ + , CU_POINTER_ATTRIBUTE_IS_HW_DECOMPRESS_CAPABLE = 21 /**< Returns in \p *data a boolean that indicates whether the pointer points to memory that is capable to be used for hardware accelerated decompression. */ } CUpointer_attribute; /** @@ -862,12 +983,18 @@ typedef enum CUfunction_attribute_enum { * The maximum size in bytes of dynamically-allocated shared memory that can be used by * this function. If the user-specified dynamic shared memory size is larger than this * value, the launch will fail. + * The default value of this attribute is + * ::CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK - ::CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, + * except when ::CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES is greater than + * ::CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, then the default value of this attribute is 0. + * The value can be increased to ::CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN - + * ::CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES. * See ::cuFuncSetAttribute, ::cuKernelSetAttribute */ CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES = 8, /** - * On devices where the L1 cache and shared memory use the same hardware resources, + * On devices where the L1 cache and shared memory use the same hardware resources, * this sets the shared memory carveout preference, in percent of the total shared memory. * Refer to ::CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR. * This is only a hint, and the driver can choose a different ratio if required to execute the function. @@ -958,6 +1085,8 @@ typedef enum CUfunc_cache_enum { } CUfunc_cache; /** + * \deprecated + * * Shared memory configurations */ typedef enum CUsharedconfig_enum { @@ -1007,10 +1136,14 @@ typedef enum CUmem_advise_enum { } CUmem_advise; typedef enum CUmem_range_attribute_enum { - CU_MEM_RANGE_ATTRIBUTE_READ_MOSTLY = 1, /**< Whether the range will mostly be read and only occasionally be written to */ - CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION = 2, /**< The preferred location of the range */ - CU_MEM_RANGE_ATTRIBUTE_ACCESSED_BY = 3, /**< Memory range has ::CU_MEM_ADVISE_SET_ACCESSED_BY set for specified device */ - CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION = 4 /**< The last location to which the range was prefetched */ + CU_MEM_RANGE_ATTRIBUTE_READ_MOSTLY = 1, /**< Whether the range will mostly be read and only occasionally be written to */ + CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION = 2, /**< The preferred location of the range */ + CU_MEM_RANGE_ATTRIBUTE_ACCESSED_BY = 3, /**< Memory range has ::CU_MEM_ADVISE_SET_ACCESSED_BY set for specified device */ + CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION = 4, /**< The last location to which the range was prefetched */ + CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION_TYPE = 5, /**< The preferred location type of the range */ + CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION_ID = 6, /**< The preferred location id of the range */ + CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION_TYPE = 7, /**< The last location type to which the range was prefetched */ + CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION_ID = 8 /**< The last location id to which the range was prefetched */ } CUmem_range_attribute; /** @@ -1332,6 +1465,53 @@ typedef enum CUjit_option_enum */ CU_JIT_POSITION_INDEPENDENT_CODE = 30, + /** + * This option hints to the JIT compiler the minimum number of CTAs from the + * kernel’s grid to be mapped to a SM. This option is ignored when used together + * with ::CU_JIT_MAX_REGISTERS or ::CU_JIT_THREADS_PER_BLOCK. + * Optimizations based on this option need ::CU_JIT_MAX_THREADS_PER_BLOCK to + * be specified as well. For kernels already using PTX directive .minnctapersm, + * this option will be ignored by default. Use ::CU_JIT_OVERRIDE_DIRECTIVE_VALUES + * to let this option take precedence over the PTX directive. + * Option type: unsigned int\n + * Applies to: compiler only + */ + CU_JIT_MIN_CTA_PER_SM = 31, + + /** + * Maximum number threads in a thread block, computed as the product of + * the maximum extent specifed for each dimension of the block. This limit + * is guaranteed not to be exeeded in any invocation of the kernel. Exceeding + * the the maximum number of threads results in runtime error or kernel launch + * failure. For kernels already using PTX directive .maxntid, this option will + * be ignored by default. Use ::CU_JIT_OVERRIDE_DIRECTIVE_VALUES to let this + * option take precedence over the PTX directive. + * Option type: int\n + * Applies to: compiler only + */ + CU_JIT_MAX_THREADS_PER_BLOCK = 32, + + /** + * This option lets the values specified using ::CU_JIT_MAX_REGISTERS, + * ::CU_JIT_THREADS_PER_BLOCK, ::CU_JIT_MAX_THREADS_PER_BLOCK and + * ::CU_JIT_MIN_CTA_PER_SM take precedence over any PTX directives. + * (0: Disable, default; 1: Enable) + * Option type: int\n + * Applies to: compiler only + */ + CU_JIT_OVERRIDE_DIRECTIVE_VALUES = 33, + + /** + * This option specifies the maximum number of concurrent threads to use + * when running compiler optimizations. If the specified value is 1, the + * option will be ignored. If the specified value is 0, the number of + * threads will match the number of CPUs on the underlying machine. + * Otherwise, if the option is N, then up to N threads will be used. + * Option type: unsigned int\n + * Applies to: compiler only + */ + CU_JIT_SPLIT_COMPILE = 34, + CU_JIT_NUM_OPTIONS } CUjit_option; @@ -1341,32 +1521,63 @@ typedef enum CUjit_option_enum */ #define CU_COMPUTE_ACCELERATED_TARGET_BASE 0x10000 +/* + * Indicates that compute device class supports family features. + */ +#define CU_COMPUTE_FAMILY_TARGET_BASE 0x20000 + /** * Online compilation targets */ typedef enum CUjit_target_enum { - CU_TARGET_COMPUTE_30 = 30, /**< Compute device class 3.0 */ - CU_TARGET_COMPUTE_32 = 32, /**< Compute device class 3.2 */ - CU_TARGET_COMPUTE_35 = 35, /**< Compute device class 3.5 */ - CU_TARGET_COMPUTE_37 = 37, /**< Compute device class 3.7 */ - CU_TARGET_COMPUTE_50 = 50, /**< Compute device class 5.0 */ - CU_TARGET_COMPUTE_52 = 52, /**< Compute device class 5.2 */ - CU_TARGET_COMPUTE_53 = 53, /**< Compute device class 5.3 */ - CU_TARGET_COMPUTE_60 = 60, /**< Compute device class 6.0.*/ - CU_TARGET_COMPUTE_61 = 61, /**< Compute device class 6.1.*/ - CU_TARGET_COMPUTE_62 = 62, /**< Compute device class 6.2.*/ - CU_TARGET_COMPUTE_70 = 70, /**< Compute device class 7.0.*/ - CU_TARGET_COMPUTE_72 = 72, /**< Compute device class 7.2.*/ - CU_TARGET_COMPUTE_75 = 75, /**< Compute device class 7.5.*/ - CU_TARGET_COMPUTE_80 = 80, /**< Compute device class 8.0.*/ - CU_TARGET_COMPUTE_86 = 86, /**< Compute device class 8.6.*/ - CU_TARGET_COMPUTE_87 = 87, /**< Compute device class 8.7.*/ - CU_TARGET_COMPUTE_89 = 89, /**< Compute device class 8.9.*/ - CU_TARGET_COMPUTE_90 = 90, /**< Compute device class 9.0.*/ + CU_TARGET_COMPUTE_30 = 30, /**< Compute device class 3.0 */ + CU_TARGET_COMPUTE_32 = 32, /**< Compute device class 3.2 */ + CU_TARGET_COMPUTE_35 = 35, /**< Compute device class 3.5 */ + CU_TARGET_COMPUTE_37 = 37, /**< Compute device class 3.7 */ + CU_TARGET_COMPUTE_50 = 50, /**< Compute device class 5.0 */ + CU_TARGET_COMPUTE_52 = 52, /**< Compute device class 5.2 */ + CU_TARGET_COMPUTE_53 = 53, /**< Compute device class 5.3 */ + CU_TARGET_COMPUTE_60 = 60, /**< Compute device class 6.0.*/ + CU_TARGET_COMPUTE_61 = 61, /**< Compute device class 6.1.*/ + CU_TARGET_COMPUTE_62 = 62, /**< Compute device class 6.2.*/ + CU_TARGET_COMPUTE_70 = 70, /**< Compute device class 7.0.*/ + CU_TARGET_COMPUTE_72 = 72, /**< Compute device class 7.2.*/ + CU_TARGET_COMPUTE_75 = 75, /**< Compute device class 7.5.*/ + CU_TARGET_COMPUTE_80 = 80, /**< Compute device class 8.0.*/ + CU_TARGET_COMPUTE_86 = 86, /**< Compute device class 8.6.*/ + CU_TARGET_COMPUTE_87 = 87, /**< Compute device class 8.7.*/ + CU_TARGET_COMPUTE_89 = 89, /**< Compute device class 8.9.*/ + CU_TARGET_COMPUTE_90 = 90, /**< Compute device class 9.0.*/ + CU_TARGET_COMPUTE_100 = 100, /**< Compute device class 10.0.*/ + CU_TARGET_COMPUTE_110 = 110, /**< Compute device class 11.0.*/ + CU_TARGET_COMPUTE_103 = 103, /**< Compute device class 10.3.*/ + CU_TARGET_COMPUTE_120 = 120, /**< Compute device class 12.0.*/ + CU_TARGET_COMPUTE_121 = 121, /**< Compute device class 12.1.*/ /**< Compute device class 9.0. with accelerated features.*/ - CU_TARGET_COMPUTE_90A = CU_COMPUTE_ACCELERATED_TARGET_BASE + CU_TARGET_COMPUTE_90 + CU_TARGET_COMPUTE_90A = CU_COMPUTE_ACCELERATED_TARGET_BASE + CU_TARGET_COMPUTE_90, + /**< Compute device class 10.0. with accelerated features.*/ + CU_TARGET_COMPUTE_100A = CU_COMPUTE_ACCELERATED_TARGET_BASE + CU_TARGET_COMPUTE_100, + /**< Compute device class 11.0 with accelerated features.*/ + CU_TARGET_COMPUTE_110A = CU_COMPUTE_ACCELERATED_TARGET_BASE + CU_TARGET_COMPUTE_110, + /**< Compute device class 10.3. with accelerated features.*/ + CU_TARGET_COMPUTE_103A = CU_COMPUTE_ACCELERATED_TARGET_BASE + CU_TARGET_COMPUTE_103, + /**< Compute device class 12.0. with accelerated features.*/ + CU_TARGET_COMPUTE_120A = CU_COMPUTE_ACCELERATED_TARGET_BASE + CU_TARGET_COMPUTE_120, + /**< Compute device class 12.1. with accelerated features.*/ + CU_TARGET_COMPUTE_121A = CU_COMPUTE_ACCELERATED_TARGET_BASE + CU_TARGET_COMPUTE_121, + + /**< Compute device class 10.x with family features.*/ + CU_TARGET_COMPUTE_100F = CU_COMPUTE_FAMILY_TARGET_BASE + CU_TARGET_COMPUTE_100, + /**< Compute device class 11.0 with family features.*/ + CU_TARGET_COMPUTE_110F = CU_COMPUTE_FAMILY_TARGET_BASE + CU_TARGET_COMPUTE_110, + /**< Compute device class 10.3. with family features.*/ + CU_TARGET_COMPUTE_103F = CU_COMPUTE_FAMILY_TARGET_BASE + CU_TARGET_COMPUTE_103, + /**< Compute device class 12.0. with family features.*/ + CU_TARGET_COMPUTE_120F = CU_COMPUTE_FAMILY_TARGET_BASE + CU_TARGET_COMPUTE_120, + /**< Compute device class 12.1. with family features.*/ + CU_TARGET_COMPUTE_121F = CU_COMPUTE_FAMILY_TARGET_BASE + CU_TARGET_COMPUTE_121, } CUjit_target; /** @@ -1482,6 +1693,9 @@ typedef enum CUlimit_enum { CU_LIMIT_DEV_RUNTIME_PENDING_LAUNCH_COUNT = 0x04, /**< GPU device runtime pending launch count */ CU_LIMIT_MAX_L2_FETCH_GRANULARITY = 0x05, /**< A value between 0 and 128 that indicates the maximum fetch granularity of L2 (in Bytes). This is a hint */ CU_LIMIT_PERSISTING_L2_CACHE_SIZE = 0x06, /**< A size in bytes for L2 persisting lines cache size */ + CU_LIMIT_SHMEM_SIZE = 0x07, /**< A maximum size in bytes of shared memory available to CUDA kernels on a CIG context. Can only be queried, cannot be set */ + CU_LIMIT_CIG_ENABLED = 0x08, /**< A non-zero value indicates this CUDA context is a CIG-enabled context. Can only be queried, cannot be set */ + CU_LIMIT_CIG_SHMEM_FALLBACK_ENABLED = 0x09, /**< When set to zero, CUDA will fail to launch a kernel on a CIG context, instead of using the fallback path, if the kernel uses more shared memory than available */ CU_LIMIT_MAX } CUlimit; @@ -1558,7 +1772,8 @@ typedef struct CUDA_KERNEL_NODE_PARAMS_st { /** * GPU kernel node parameters - */typedef struct CUDA_KERNEL_NODE_PARAMS_v2_st { + */ +typedef struct CUDA_KERNEL_NODE_PARAMS_v2_st { CUfunction func; /**< Kernel to launch */ unsigned int gridDimX; /**< Width of grid in blocks */ unsigned int gridDimY; /**< Height of grid in blocks */ @@ -1574,6 +1789,24 @@ typedef struct CUDA_KERNEL_NODE_PARAMS_st { } CUDA_KERNEL_NODE_PARAMS_v2; typedef CUDA_KERNEL_NODE_PARAMS_v2 CUDA_KERNEL_NODE_PARAMS; +/** + * GPU kernel node parameters + */ +typedef struct CUDA_KERNEL_NODE_PARAMS_v3_st { + CUfunction func; /**< Kernel to launch */ + unsigned int gridDimX; /**< Width of grid in blocks */ + unsigned int gridDimY; /**< Height of grid in blocks */ + unsigned int gridDimZ; /**< Depth of grid in blocks */ + unsigned int blockDimX; /**< X dimension of each thread block */ + unsigned int blockDimY; /**< Y dimension of each thread block */ + unsigned int blockDimZ; /**< Z dimension of each thread block */ + unsigned int sharedMemBytes; /**< Dynamic shared-memory size per thread block in bytes */ + void **kernelParams; /**< Array of pointers to kernel parameters */ + void **extra; /**< Extra options */ + CUkernel kern; /**< Kernel to launch, will only be referenced if func is NULL */ + CUcontext ctx; /**< Context for the kernel task to run in. The value NULL will indicate the current context should be used by the api. This field is ignored if func is set. */ +} CUDA_KERNEL_NODE_PARAMS_v3; + /** * Memset node parameters */ @@ -1587,6 +1820,19 @@ typedef struct CUDA_MEMSET_NODE_PARAMS_st { } CUDA_MEMSET_NODE_PARAMS_v1; typedef CUDA_MEMSET_NODE_PARAMS_v1 CUDA_MEMSET_NODE_PARAMS; +/** + * Memset node parameters + */ +typedef struct CUDA_MEMSET_NODE_PARAMS_v2_st { + CUdeviceptr dst; /**< Destination device pointer */ + size_t pitch; /**< Pitch of destination device pointer. Unused if height is 1 */ + unsigned int value; /**< Value to be set */ + unsigned int elementSize; /**< Size of each element in bytes. Must be 1, 2, or 4. */ + size_t width; /**< Width of the row in elements */ + size_t height; /**< Number of rows */ + CUcontext ctx; /**< Context on which to run the node */ +} CUDA_MEMSET_NODE_PARAMS_v2; + /** * Host node parameters */ @@ -1596,6 +1842,61 @@ typedef struct CUDA_HOST_NODE_PARAMS_st { } CUDA_HOST_NODE_PARAMS_v1; typedef CUDA_HOST_NODE_PARAMS_v1 CUDA_HOST_NODE_PARAMS; +/** + * Host node parameters + */ +typedef struct CUDA_HOST_NODE_PARAMS_v2_st { + CUhostFn fn; /**< The function to call when the node executes */ + void* userData; /**< Argument to pass to the function */ +} CUDA_HOST_NODE_PARAMS_v2; + +/** + * Conditional node handle flags + */ +#define CU_GRAPH_COND_ASSIGN_DEFAULT 0x1 /**< Default value is applied when graph is launched. */ + +/** + * Conditional node types + */ +typedef enum CUgraphConditionalNodeType_enum { + CU_GRAPH_COND_TYPE_IF = 0, /**< Conditional 'if/else' Node. Body[0] executed if condition is non-zero. If \p size == 2, an optional ELSE graph is created and this is executed if the condition is zero. */ + CU_GRAPH_COND_TYPE_WHILE = 1, /**< Conditional 'while' Node. Body executed repeatedly while condition value is non-zero. */ + CU_GRAPH_COND_TYPE_SWITCH = 2, /**< Conditional 'switch' Node. Body[n] is executed once, where 'n' is the value of the condition. If the condition does not match a body index, no body is launched. */ +} CUgraphConditionalNodeType; + +/** + * Conditional node parameters + */ +typedef struct CUDA_CONDITIONAL_NODE_PARAMS { + CUgraphConditionalHandle handle; /**< Conditional node handle. + Handles must be created in advance of creating the node + using ::cuGraphConditionalHandleCreate. */ + CUgraphConditionalNodeType type; /**< Type of conditional node. */ + unsigned int size; /**< Size of graph output array. Allowed values are 1 for CU_GRAPH_COND_TYPE_WHILE, 1 or 2 + for CU_GRAPH_COND_TYPE_IF, or any value greater than zero for CU_GRAPH_COND_TYPE_SWITCH. */ + CUgraph *phGraph_out; /**< CUDA-owned array populated with conditional node child graphs during creation of the node. + Valid for the lifetime of the conditional node. + The contents of the graph(s) are subject to the following constraints: + + - Allowed node types are kernel nodes, empty nodes, child graphs, memsets, + memcopies, and conditionals. This applies recursively to child graphs and conditional bodies. + - All kernels, including kernels in nested conditionals or child graphs at any level, + must belong to the same CUDA context. + + These graphs may be populated using graph node creation APIs or ::cuStreamBeginCaptureToGraph. + + CU_GRAPH_COND_TYPE_IF: + phGraph_out[0] is executed when the condition is non-zero. If \p size == 2, phGraph_out[1] will + be executed when the condition is zero. + CU_GRAPH_COND_TYPE_WHILE: + phGraph_out[0] is executed as long as the condition is non-zero. + CU_GRAPH_COND_TYPE_SWITCH: + phGraph_out[n] is executed when the condition is equal to n. If the condition >= \p size, + no body graph is executed. + */ + CUcontext ctx; /**< Context on which to run the node. Must match context used to create the handle and all body nodes. */ +} CUDA_CONDITIONAL_NODE_PARAMS; + /** * Graph node types */ @@ -1612,9 +1913,85 @@ typedef enum CUgraphNodeType_enum { CU_GRAPH_NODE_TYPE_EXT_SEMAS_WAIT = 9, /**< External semaphore wait node */ CU_GRAPH_NODE_TYPE_MEM_ALLOC = 10,/**< Memory Allocation Node */ CU_GRAPH_NODE_TYPE_MEM_FREE = 11,/**< Memory Free Node */ - CU_GRAPH_NODE_TYPE_BATCH_MEM_OP = 12 /**< Batch MemOp Node */ + CU_GRAPH_NODE_TYPE_BATCH_MEM_OP = 12,/**< Batch MemOp Node + See ::cuStreamBatchMemOp and ::CUstreamBatchMemOpType for what these nodes can do. */ + CU_GRAPH_NODE_TYPE_CONDITIONAL = 13,/**< Conditional Node + + May be used to implement a conditional execution path or loop + inside of a graph. The graph(s) contained within the body of the conditional node + can be selectively executed or iterated upon based on the value of a conditional + variable. + + Handles must be created in advance of creating the node + using ::cuGraphConditionalHandleCreate. + + The following restrictions apply to graphs which contain conditional nodes: + The graph cannot be used in a child node. + Only one instantiation of the graph may exist at any point in time. + The graph cannot be cloned. + + To set the control value, supply a default value when creating the handle and/or + call ::cudaGraphSetConditional from device code.*/ } CUgraphNodeType; +/** + * Type annotations that can be applied to graph edges as part of ::CUgraphEdgeData. + */ +typedef enum CUgraphDependencyType_enum { + CU_GRAPH_DEPENDENCY_TYPE_DEFAULT = 0, /**< This is an ordinary dependency. */ + CU_GRAPH_DEPENDENCY_TYPE_PROGRAMMATIC = 1 /**< This dependency type allows the downstream node to + use \c cudaGridDependencySynchronize(). It may only be used + between kernel nodes, and must be used with either the + ::CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC or + ::CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER outgoing port. */ +} CUgraphDependencyType; + +/** + * This port activates when the kernel has finished executing. + */ +#define CU_GRAPH_KERNEL_NODE_PORT_DEFAULT 0 +/** + * This port activates when all blocks of the kernel have performed cudaTriggerProgrammaticLaunchCompletion() + * or have terminated. It must be used with edge type ::CU_GRAPH_DEPENDENCY_TYPE_PROGRAMMATIC. See also + * ::CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_EVENT. + */ +#define CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC 1 +/** + * This port activates when all blocks of the kernel have begun execution. See also + * ::CU_LAUNCH_ATTRIBUTE_LAUNCH_COMPLETION_EVENT. + */ +#define CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER 2 + +/** + * Optional annotation for edges in a CUDA graph. Note, all edges implicitly have annotations and + * default to a zero-initialized value if not specified. A zero-initialized struct indicates a + * standard full serialization of two nodes with memory visibility. + */ +typedef struct CUgraphEdgeData_st { + unsigned char from_port; /**< This indicates when the dependency is triggered from the upstream + node on the edge. The meaning is specfic to the node type. A value + of 0 in all cases means full completion of the upstream node, with + memory visibility to the downstream node or portion thereof + (indicated by \c to_port). +
+ Only kernel nodes define non-zero ports. A kernel node + can use the following output port types: + ::CU_GRAPH_KERNEL_NODE_PORT_DEFAULT, ::CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC, + or ::CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER. */ + unsigned char to_port; /**< This indicates what portion of the downstream node is dependent on + the upstream node or portion thereof (indicated by \c from_port). The + meaning is specific to the node type. A value of 0 in all cases means + the entirety of the downstream node is dependent on the upstream work. +
+ Currently no node types define non-zero ports. Accordingly, this field + must be set to zero. */ + unsigned char type; /**< This should be populated with a value from ::CUgraphDependencyType. (It + is typed as char due to compiler-specific layout of bitfields.) See + ::CUgraphDependencyType. */ + unsigned char reserved[5]; /**< These bytes are unused and must be zeroed. This ensures + compatibility if additional fields are added in the future. */ +} CUgraphEdgeData; + /** * Graph instantiation results */ @@ -1624,7 +2001,8 @@ typedef enum CUgraphInstantiateResult_enum CUDA_GRAPH_INSTANTIATE_ERROR = 1, /**< Instantiation failed for an unexpected reason which is described in the return value of the function */ CUDA_GRAPH_INSTANTIATE_INVALID_STRUCTURE = 2, /**< Instantiation failed due to invalid structure, such as cycles */ CUDA_GRAPH_INSTANTIATE_NODE_OPERATION_NOT_SUPPORTED = 3, /**< Instantiation for device launch failed because the graph contained an unsupported operation */ - CUDA_GRAPH_INSTANTIATE_MULTIPLE_CTXS_NOT_SUPPORTED = 4 /**< Instantiation for device launch failed due to the nodes belonging to different contexts */ + CUDA_GRAPH_INSTANTIATE_MULTIPLE_CTXS_NOT_SUPPORTED = 4, /**< Instantiation for device launch failed due to the nodes belonging to different contexts */ + CUDA_GRAPH_INSTANTIATE_CONDITIONAL_HANDLE_UNUSED = 5, /**< One or more conditional handles are not associated with conditional nodes */ } CUgraphInstantiateResult; /** @@ -1654,35 +2032,70 @@ typedef enum CUclusterSchedulingPolicy_enum { CU_CLUSTER_SCHEDULING_POLICY_LOAD_BALANCING = 2 /**< allow the hardware to load-balance the blocks in a cluster to the SMs */ } CUclusterSchedulingPolicy; +/** + * Memory Synchronization Domain + * + * A kernel can be launched in a specified memory synchronization domain that affects all memory operations issued by + * that kernel. A memory barrier issued in one domain will only order memory operations in that domain, thus eliminating + * latency increase from memory barriers ordering unrelated traffic. + * + * By default, kernels are launched in domain 0. Kernel launched with ::CU_LAUNCH_MEM_SYNC_DOMAIN_REMOTE will have a + * different domain ID. User may also alter the domain ID with ::CUlaunchMemSyncDomainMap for a specific stream / + * graph node / kernel launch. See ::CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN, ::cuStreamSetAttribute, ::cuLaunchKernelEx, + * ::cuGraphKernelNodeSetAttribute. + * + * Memory operations done in kernels launched in different domains are considered system-scope distanced. In other + * words, a GPU scoped memory synchronization is not sufficient for memory order to be observed by kernels in another + * memory synchronization domain even if they are on the same GPU. + */ typedef enum CUlaunchMemSyncDomain_enum { - CU_LAUNCH_MEM_SYNC_DOMAIN_DEFAULT = 0, - CU_LAUNCH_MEM_SYNC_DOMAIN_REMOTE = 1 + CU_LAUNCH_MEM_SYNC_DOMAIN_DEFAULT = 0, /**< Launch kernels in the default domain */ + CU_LAUNCH_MEM_SYNC_DOMAIN_REMOTE = 1 /**< Launch kernels in the remote domain */ } CUlaunchMemSyncDomain; +/** + * Memory Synchronization Domain map + * + * See ::cudaLaunchMemSyncDomain. + * + * By default, kernels are launched in domain 0. Kernel launched with ::CU_LAUNCH_MEM_SYNC_DOMAIN_REMOTE will have a + * different domain ID. User may also alter the domain ID with ::CUlaunchMemSyncDomainMap for a specific stream / + * graph node / kernel launch. See ::CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP. + * + * Domain ID range is available through ::CU_DEVICE_ATTRIBUTE_MEM_SYNC_DOMAIN_COUNT. + */ typedef struct CUlaunchMemSyncDomainMap_st { - unsigned char default_; - unsigned char remote; + unsigned char default_; /**< The default domain ID to use for designated kernels */ + unsigned char remote; /**< The remote domain ID to use for designated kernels */ } CUlaunchMemSyncDomainMap; +/** + * Launch attributes enum; used as id field of ::CUlaunchAttribute + */ typedef enum CUlaunchAttributeID_enum { CU_LAUNCH_ATTRIBUTE_IGNORE = 0 /**< Ignored entry, for convenient composition */ - , CU_LAUNCH_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 /**< Valid for streams, graph nodes, launches. */ - , CU_LAUNCH_ATTRIBUTE_COOPERATIVE = 2 /**< Valid for graph nodes, launches. */ - , CU_LAUNCH_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 /**< Valid for streams. */ - , CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION = 4 /**< Valid for graph nodes, launches. */ - , CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 /**< Valid for graph nodes, launches. */ + , CU_LAUNCH_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 /**< Valid for streams, graph nodes, launches. See + ::CUlaunchAttributeValue::accessPolicyWindow. */ + , CU_LAUNCH_ATTRIBUTE_COOPERATIVE = 2 /**< Valid for graph nodes, launches. See + ::CUlaunchAttributeValue::cooperative. */ + , CU_LAUNCH_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 /**< Valid for streams. See + ::CUlaunchAttributeValue::syncPolicy. */ + , CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION = 4 /**< Valid for graph nodes, launches. See ::CUlaunchAttributeValue::clusterDim. */ + , CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 /**< Valid for graph nodes, launches. See ::CUlaunchAttributeValue::clusterSchedulingPolicyPreference. */ , CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION = 6 /**< Valid for launches. Setting - programmaticStreamSerializationAllowed to non-0 - signals that the kernel will use programmatic - means to resolve its stream dependency, so that - the CUDA runtime should opportunistically allow - the grid's execution to overlap with the previous - kernel in the stream, if that kernel requests the - overlap. The dependent launches can choose to wait - on the dependency using the programmatic sync - (cudaGridDependencySynchronize() or equivalent PTX - instructions). */ - , CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_EVENT = 7 /**< Valid for launches. Event recorded through this + ::CUlaunchAttributeValue::programmaticStreamSerializationAllowed + to non-0 signals that the kernel will use programmatic + means to resolve its stream dependency, so that the + CUDA runtime should opportunistically allow the grid's + execution to overlap with the previous kernel in the + stream, if that kernel requests the overlap. The + dependent launches can choose to wait on the + dependency using the programmatic sync + (cudaGridDependencySynchronize() or equivalent PTX + instructions). */ + , CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_EVENT = 7 /**< Valid for launches. Set + ::CUlaunchAttributeValue::programmaticEvent to + record the event. Event recorded through this launch attribute is guaranteed to only trigger after all block in the associated kernel trigger the event. A block can trigger the event through @@ -1700,47 +2113,239 @@ typedef enum CUlaunchAttributeID_enum { may only observe the event trigger long after the associated kernel has completed. This recording type is primarily meant for establishing - programmatic dependency between device tasks. The - event supplied must not be an interprocess or + programmatic dependency between device tasks. Note + also this type of dependency allows, but does not + guarantee, concurrent execution of tasks. +
+ The event supplied must not be an interprocess or interop event. The event must disable timing (i.e. - created with ::CU_EVENT_DISABLE_TIMING flag set). + must be created with the ::CU_EVENT_DISABLE_TIMING + flag set). */ - , CU_LAUNCH_ATTRIBUTE_PRIORITY = 8 /**< Valid for streams, graph nodes, launches. */ - , CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 - , CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 -#ifdef __CUDA_API_VERSION_INTERNAL + , CU_LAUNCH_ATTRIBUTE_PRIORITY = 8 /**< Valid for streams, graph nodes, launches. See + ::CUlaunchAttributeValue::priority. */ + , CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 /**< Valid for streams, graph nodes, launches. See + ::CUlaunchAttributeValue::memSyncDomainMap. */ + , CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 /**< Valid for streams, graph nodes, launches. See + ::CUlaunchAttributeValue::memSyncDomain. */ + , CU_LAUNCH_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 /**< Valid for graph nodes, launches. Set + ::CUlaunchAttributeValue::preferredClusterDim + to allow the kernel launch to specify a preferred substitute + cluster dimension. Blocks may be grouped according to either + the dimensions specified with this attribute (grouped into a + "preferred substitute cluster"), or the one specified with + ::CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION attribute (grouped + into a "regular cluster"). The cluster dimensions of a + "preferred substitute cluster" shall be an integer multiple + greater than zero of the regular cluster dimensions. The + device will attempt - on a best-effort basis - to group + thread blocks into preferred clusters over grouping them + into regular clusters. When it deems necessary (primarily + when the device temporarily runs out of physical resources + to launch the larger preferred clusters), the device may + switch to launch the regular clusters instead to attempt to + utilize as much of the physical device resources as possible. +
+ Each type of cluster will have its enumeration / coordinate + setup as if the grid consists solely of its type of cluster. + For example, if the preferred substitute cluster dimensions + double the regular cluster dimensions, there might be + simultaneously a regular cluster indexed at (1,0,0), and a + preferred cluster indexed at (1,0,0). In this example, the + preferred substitute cluster (1,0,0) replaces regular + clusters (2,0,0) and (3,0,0) and groups their blocks. +
+ This attribute will only take effect when a regular cluster + dimension has been specified. The preferred substitute + cluster dimension must be an integer multiple greater than + zero of the regular cluster dimension and must divide the + grid. It must also be no more than `maxBlocksPerCluster`, if + it is set in the kernel's `__launch_bounds__`. Otherwise it + must be less than the maximum value the driver can support. + Otherwise, setting this attribute to a value physically + unable to fit on any particular device is permitted. */ + , CU_LAUNCH_ATTRIBUTE_LAUNCH_COMPLETION_EVENT = 12 /**< Valid for launches. Set + ::CUlaunchAttributeValue::launchCompletionEvent to record the + event. +
+ Nominally, the event is triggered once all blocks of the kernel + have begun execution. Currently this is a best effort. If a kernel + B has a launch completion dependency on a kernel A, B may wait + until A is complete. Alternatively, blocks of B may begin before + all blocks of A have begun, for example if B can claim execution + resources unavailable to A (e.g. they run on different GPUs) or + if B is a higher priority than A. + Exercise caution if such an ordering inversion could lead + to deadlock. +
+ A launch completion event is nominally similar to a programmatic + event with \c triggerAtBlockStart set except that it is not + visible to \c cudaGridDependencySynchronize() and can be used with + compute capability less than 9.0. +
+ The event supplied must not be an interprocess or interop + event. The event must disable timing (i.e. must be created + with the ::CU_EVENT_DISABLE_TIMING flag set). */ + , CU_LAUNCH_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 /**< Valid for graph nodes, launches. This attribute is graphs-only, + and passing it to a launch in a non-capturing stream will result + in an error. +
+ ::CUlaunchAttributeValue::deviceUpdatableKernelNode::deviceUpdatable can + only be set to 0 or 1. Setting the field to 1 indicates that the + corresponding kernel node should be device-updatable. On success, a handle + will be returned via + ::CUlaunchAttributeValue::deviceUpdatableKernelNode::devNode which can be + passed to the various device-side update functions to update the node's + kernel parameters from within another kernel. For more information on the + types of device updates that can be made, as well as the relevant limitations + thereof, see ::cudaGraphKernelNodeUpdatesApply. +
+ Nodes which are device-updatable have additional restrictions compared to + regular kernel nodes. Firstly, device-updatable nodes cannot be removed + from their graph via ::cuGraphDestroyNode. Additionally, once opted-in + to this functionality, a node cannot opt out, and any attempt to set the + deviceUpdatable attribute to 0 will result in an error. Device-updatable + kernel nodes also cannot have their attributes copied to/from another kernel + node via ::cuGraphKernelNodeCopyAttributes. Graphs containing one or more + device-updatable nodes also do not allow multiple instantiation, and neither + the graph nor its instantiated version can be passed to ::cuGraphExecUpdate. +
+ If a graph contains device-updatable nodes and updates those nodes from the device + from within the graph, the graph must be uploaded with ::cuGraphUpload before it + is launched. For such a graph, if host-side executable graph updates are made to the + device-updatable nodes, the graph must be uploaded before it is launched again. */ + , CU_LAUNCH_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 /**< Valid for launches. On devices where the L1 cache and shared memory use the + same hardware resources, setting ::CUlaunchAttributeValue::sharedMemCarveout to a + percentage between 0-100 signals the CUDA driver to set the shared memory carveout + preference, in percent of the total shared memory for that kernel launch. + This attribute takes precedence over ::CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT. + This is only a hint, and the CUDA driver can choose a different configuration if + required for the launch. */ + , CU_LAUNCH_ATTRIBUTE_NVLINK_UTIL_CENTRIC_SCHEDULING = 16 /**< Valid for streams, graph nodes, launches. This attribute is a hint to the CUDA runtime that the + launch should attempt to make the kernel maximize its NVLINK utilization. +
+ When possible to honor this hint, CUDA will assume each block in the grid launch will carry out an even amount + of NVLINK traffic, and make a best-effort attempt to adjust the kernel launch based on that assumption. +
+ This attribute is a hint only. CUDA makes no functional or performance guarantee. Its applicability can be + affected by many different factors, including driver version (i.e. CUDA doesn't guarantee the performance + characteristics will be maintained between driver versions or a driver update could alter or regress + previously observed perf characteristics.) It also doesn't guarantee a successful result, i.e. applying + the attribute may not improve the performance of either the targeted kernel or the encapsulating application. +
+ Valid values for ::CUlaunchAttributeValue::nvlinkUtilCentricScheduling are 0 (disabled) and 1 (enabled). + */ +#if defined(__CUDA_API_VERSION_INTERNAL) && !defined(__CUDA_API_VERSION_INTERNAL_ODR) , CU_LAUNCH_ATTRIBUTE_MAX #endif } CUlaunchAttributeID; +/** + * Launch attributes union; used as value field of ::CUlaunchAttribute + */ typedef union CUlaunchAttributeValue_union { - char pad[64]; /**< Pad to 64 bytes */ - CUaccessPolicyWindow accessPolicyWindow; /**< Attribute ::CUaccessPolicyWindow. */ - int cooperative; /**< Nonzero indicates a cooperative kernel (see ::cuLaunchCooperativeKernel). */ - CUsynchronizationPolicy syncPolicy; /**< ::CUsynchronizationPolicy for work queued up in this stream */ + char pad[64]; /* Pad to 64 bytes */ + CUaccessPolicyWindow accessPolicyWindow; /**< Value of launch attribute ::CU_LAUNCH_ATTRIBUTE_ACCESS_POLICY_WINDOW. */ + int cooperative; /**< Value of launch attribute ::CU_LAUNCH_ATTRIBUTE_COOPERATIVE. Nonzero indicates a cooperative + kernel (see ::cuLaunchCooperativeKernel). */ + CUsynchronizationPolicy syncPolicy; /**< Value of launch attribute + ::CU_LAUNCH_ATTRIBUTE_SYNCHRONIZATION_POLICY. ::CUsynchronizationPolicy for + work queued up in this stream */ + + /** + * Value of launch attribute ::CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION that + * represents the desired cluster dimensions for the kernel. Opaque type + * with the following fields: + * - \p x - The X dimension of the cluster, in blocks. Must be a divisor + * of the grid X dimension. + * - \p y - The Y dimension of the cluster, in blocks. Must be a divisor + * of the grid Y dimension. + * - \p z - The Z dimension of the cluster, in blocks. Must be a divisor + * of the grid Z dimension. + */ struct { unsigned int x; unsigned int y; unsigned int z; - } clusterDim; /**< Cluster dimensions for the kernel node. */ - CUclusterSchedulingPolicy clusterSchedulingPolicyPreference; /**< Cluster scheduling policy preference for the kernel node. */ - int programmaticStreamSerializationAllowed; + } clusterDim; + CUclusterSchedulingPolicy clusterSchedulingPolicyPreference; /**< Value of launch attribute + ::CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE. Cluster + scheduling policy preference for the kernel. */ + int programmaticStreamSerializationAllowed; /**< Value of launch attribute + ::CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION. */ + /** + * Value of launch attribute ::CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_EVENT + * with the following fields: + * - \p CUevent event - Event to fire when all blocks trigger it. + * - \p Event record flags, see ::cuEventRecordWithFlags. Does not accept :CU_EVENT_RECORD_EXTERNAL. + * - \p triggerAtBlockStart - If this is set to non-0, each block launch will automatically trigger the event. + */ struct { CUevent event; - int flags; /* Does not accept ::CU_EVENT_RECORD_EXTERNAL */ + int flags; int triggerAtBlockStart; } programmaticEvent; - int priority; /**< Execution priority of the kernel. */ - CUlaunchMemSyncDomainMap memSyncDomainMap; - CUlaunchMemSyncDomain memSyncDomain; + /** + * Value of launch attribute ::CU_LAUNCH_ATTRIBUTE_LAUNCH_COMPLETION_EVENT + * with the following fields: + * - \p CUevent event - Event to fire when the last block launches + * - \p int flags; - Event record flags, see ::cuEventRecordWithFlags. Does not accept ::CU_EVENT_RECORD_EXTERNAL. + */ + struct { + CUevent event; + int flags; + } launchCompletionEvent; + int priority; /**< Value of launch attribute ::CU_LAUNCH_ATTRIBUTE_PRIORITY. Execution priority of the kernel. */ + CUlaunchMemSyncDomainMap memSyncDomainMap; /**< Value of launch attribute + ::CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP. See + ::CUlaunchMemSyncDomainMap. */ + CUlaunchMemSyncDomain memSyncDomain; /**< Value of launch attribute + ::CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN. See::CUlaunchMemSyncDomain */ + /** + * Value of launch attribute ::CU_LAUNCH_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION + * that represents the desired preferred cluster dimensions for the kernel. + * Opaque type with the following fields: + * - \p x - The X dimension of the preferred cluster, in blocks. Must + * be a divisor of the grid X dimension, and must be a + * multiple of the \p x field of ::CUlaunchAttributeValue::clusterDim. + * - \p y - The Y dimension of the preferred cluster, in blocks. Must + * be a divisor of the grid Y dimension, and must be a + * multiple of the \p y field of ::CUlaunchAttributeValue::clusterDim. + * - \p z - The Z dimension of the preferred cluster, in blocks. Must be + * equal to the \p z field of ::CUlaunchAttributeValue::clusterDim. + */ + struct { + unsigned int x; + unsigned int y; + unsigned int z; + } preferredClusterDim; + + /** + * Value of launch attribute ::CU_LAUNCH_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE. + * with the following fields: + * - \p int deviceUpdatable - Whether or not the resulting kernel node should be device-updatable. + * - \p CUgraphDeviceNode devNode - Returns a handle to pass to the various device-side update functions. + */ + struct { + int deviceUpdatable; + CUgraphDeviceNode devNode; + } deviceUpdatableKernelNode; + unsigned int sharedMemCarveout; /**< Value of launch attribute ::CU_LAUNCH_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT. */ + unsigned int nvlinkUtilCentricScheduling; } CUlaunchAttributeValue; +/** + * Launch attribute + */ typedef struct CUlaunchAttribute_st { - CUlaunchAttributeID id; + CUlaunchAttributeID id; /**< Attribute to set */ char pad[8 - sizeof(CUlaunchAttributeID)]; - CUlaunchAttributeValue value; + CUlaunchAttributeValue value; /**< Value of the attribute */ } CUlaunchAttribute; +/** + * CUDA extensible launch configuration + */ typedef struct CUlaunchConfig_st { unsigned int gridDimX; /**< Width of grid in blocks */ unsigned int gridDimY; /**< Height of grid in blocks */ @@ -1750,8 +2355,8 @@ typedef struct CUlaunchConfig_st { unsigned int blockDimZ; /**< Z dimension of each thread block */ unsigned int sharedMemBytes; /**< Dynamic shared-memory size per thread block in bytes */ CUstream hStream; /**< Stream identifier */ - CUlaunchAttribute *attrs; /**< nullable if numAttrs == 0 */ - unsigned int numAttrs; /**< number of attributes populated in attrs */ + CUlaunchAttribute *attrs; /**< List of attributes; nullable if ::CUlaunchConfig::numAttrs == 0 */ + unsigned int numAttrs; /**< Number of attributes populated in ::CUlaunchConfig::attrs */ } CUlaunchConfig; typedef CUlaunchAttributeID CUkernelNodeAttrID; @@ -1762,6 +2367,9 @@ typedef CUlaunchAttributeID CUkernelNodeAttrID; #define CU_KERNEL_NODE_ATTRIBUTE_PRIORITY CU_LAUNCH_ATTRIBUTE_PRIORITY #define CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP #define CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN +#define CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION CU_LAUNCH_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION +#define CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE CU_LAUNCH_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE +#define CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT CU_LAUNCH_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT typedef CUlaunchAttributeValue CUkernelNodeAttrValue_v1; typedef CUkernelNodeAttrValue_v1 CUkernelNodeAttrValue; @@ -1844,6 +2452,30 @@ typedef struct CUexecAffinityParam_st { */ typedef CUexecAffinityParam_v1 CUexecAffinityParam; +typedef enum CUcigDataType_enum { + CIG_DATA_TYPE_D3D12_COMMAND_QUEUE = 0x1, /** D3D12 Command Queue Handle */ + CIG_DATA_TYPE_NV_BLOB = 0x2, /** Nvidia specific data blob used for Vulkan and other NV clients*/ +} CUcigDataType; + +/** +* CIG Context Create Params +*/ +typedef struct CUctxCigParam_st { + CUcigDataType sharedDataType; + void* sharedData; +} CUctxCigParam; + +/** +* Params for creating CUDA context +* Exactly one of execAffinityParams and cigParams +* must be non-NULL. +*/ +typedef struct CUctxCreateParams_st { + CUexecAffinityParam *execAffinityParams; + int numExecAffinityParams; + CUctxCigParam *cigParams; +} CUctxCreateParams; + /** * Library options to be specified with ::cuLibraryLoadData() or ::cuLibraryLoadFromFile() */ @@ -1891,8 +2523,8 @@ typedef enum cudaError_enum { CUDA_ERROR_INVALID_VALUE = 1, /** - * The API call failed because it was unable to allocate enough memory to - * perform the requested operation. + * The API call failed because it was unable to allocate enough memory or + * other resources to perform the requested operation. */ CUDA_ERROR_OUT_OF_MEMORY = 2, @@ -1943,6 +2575,14 @@ typedef enum cudaError_enum { */ CUDA_ERROR_STUB_LIBRARY = 34, + /** + * This indicates that the API call requires a newer CUDA driver than the one + * currently installed. Users should install an updated NVIDIA CUDA driver + * to allow the API call to succeed. + */ + CUDA_ERROR_CALL_REQUIRES_NEWER_DRIVER = 36, + + /** * This indicates that requested CUDA device is unavailable at the current * time. Devices are often unavailable due to use of @@ -1981,6 +2621,8 @@ typedef enum cudaError_enum { * ::cuCtxDestroy() invoked on it). This can also be returned if a user * mixes different API versions (i.e. 3010 context with 3020 API calls). * See ::cuCtxGetApiVersion() for more details. + * This can also be returned if the green context passed to an API call + * was not converted to a ::CUcontext using ::cuCtxFromGreenCtx API. */ CUDA_ERROR_INVALID_CONTEXT = 201, @@ -2113,6 +2755,17 @@ typedef enum cudaError_enum { */ CUDA_ERROR_UNSUPPORTED_DEVSIDE_SYNC = 225, + /** + * This indicates that an exception occurred on the device that is now + * contained by the GPU's error containment capability. Common causes are - + * a. Certain types of invalid accesses of peer GPU memory over nvlink + * b. Certain classes of hardware errors + * This leaves the process in an inconsistent state and any further CUDA + * work will return the same error. To continue using CUDA, the process must + * be terminated and relaunched. + */ + CUDA_ERROR_CONTAINED = 226, + /** * This indicates that the device kernel source is invalid. This includes * compilation/linker errors encountered in device code or user error. @@ -2151,6 +2804,14 @@ typedef enum cudaError_enum { */ CUDA_ERROR_ILLEGAL_STATE = 401, + /** + * This indicates an attempt was made to introspect an object in a way that + * would discard semantically important information. This is either due to + * the object using funtionality newer than the API version used to + * introspect it or omission of optional return arguments. + */ + CUDA_ERROR_LOSSY_QUERY = 402, + /** * This indicates that a named symbol was not found. Examples of symbols * are global/constant variable names, driver function names, texture names, @@ -2321,6 +2982,14 @@ typedef enum cudaError_enum { */ CUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE = 720, + /** + * An exception occurred on the device while exiting a kernel using tensor memory: the + * tensor memory was not completely deallocated. This leaves the process in an inconsistent + * state and any further CUDA work will return the same error. To continue using CUDA, the + * process must be terminated and relaunched. + */ + CUDA_ERROR_TENSOR_MEMORY_LEAK = 721, + /** * This error indicates that the attempted operation is not permitted. */ @@ -2479,6 +3148,30 @@ typedef enum cudaError_enum { */ CUDA_ERROR_INVALID_CLUSTER_SIZE = 912, + /** + * Indiciates a function handle is not loaded when calling an API that requires + * a loaded function. + */ + CUDA_ERROR_FUNCTION_NOT_LOADED = 913, + + /** + * This error indicates one or more resources passed in are not valid resource + * types for the operation. + */ + CUDA_ERROR_INVALID_RESOURCE_TYPE = 914, + + /** + * This error indicates one or more resources are insufficient or non-applicable for + * the operation. + */ + CUDA_ERROR_INVALID_RESOURCE_CONFIGURATION = 915, + + /** + * This error indicates that an error happened during the key rotation + * sequence. + */ + CUDA_ERROR_KEY_ROTATION = 916, + /** * This indicates that an unknown internal error has occurred. */ @@ -2491,11 +3184,46 @@ typedef enum cudaError_enum { typedef enum CUdevice_P2PAttribute_enum { CU_DEVICE_P2P_ATTRIBUTE_PERFORMANCE_RANK = 0x01, /**< A relative value indicating the performance of the link between two devices */ CU_DEVICE_P2P_ATTRIBUTE_ACCESS_SUPPORTED = 0x02, /**< P2P Access is enable */ - CU_DEVICE_P2P_ATTRIBUTE_NATIVE_ATOMIC_SUPPORTED = 0x03, /**< Atomic operation over the link supported */ + CU_DEVICE_P2P_ATTRIBUTE_NATIVE_ATOMIC_SUPPORTED = 0x03, /**< All CUDA-valid atomic operation over the link are supported */ CU_DEVICE_P2P_ATTRIBUTE_ACCESS_ACCESS_SUPPORTED = 0x04, /**< \deprecated use CU_DEVICE_P2P_ATTRIBUTE_CUDA_ARRAY_ACCESS_SUPPORTED instead */ CU_DEVICE_P2P_ATTRIBUTE_CUDA_ARRAY_ACCESS_SUPPORTED = 0x04 /**< Accessing CUDA arrays over the link supported */ + , + CU_DEVICE_P2P_ATTRIBUTE_ONLY_PARTIAL_NATIVE_ATOMIC_SUPPORTED = 0x05 /**< Only some CUDA-valid atomic operations over the link are supported. */ } CUdevice_P2PAttribute; +/** + * CUDA-valid Atomic Operations + */ +typedef enum CUatomicOperation_enum { + CU_ATOMIC_OPERATION_INTEGER_ADD = 0, + CU_ATOMIC_OPERATION_INTEGER_MIN = 1, + CU_ATOMIC_OPERATION_INTEGER_MAX = 2, + CU_ATOMIC_OPERATION_INTEGER_INCREMENT = 3, + CU_ATOMIC_OPERATION_INTEGER_DECREMENT = 4, + CU_ATOMIC_OPERATION_AND = 5, + CU_ATOMIC_OPERATION_OR = 6, + CU_ATOMIC_OPERATION_XOR = 7, + CU_ATOMIC_OPERATION_EXCHANGE = 8, + CU_ATOMIC_OPERATION_CAS = 9, + CU_ATOMIC_OPERATION_FLOAT_ADD = 10, + CU_ATOMIC_OPERATION_FLOAT_MIN = 11, + CU_ATOMIC_OPERATION_FLOAT_MAX = 12, + CU_ATOMIC_OPERATION_MAX +} CUatomicOperation; + +/** + * CUDA-valid Atomic Operation capabilities + */ +typedef enum CUatomicOperationCapability_enum { + CU_ATOMIC_CAPABILITY_SIGNED = 1u<<0, + CU_ATOMIC_CAPABILITY_UNSIGNED = 1u<<1, + CU_ATOMIC_CAPABILITY_REDUCTION = 1u<<2, + CU_ATOMIC_CAPABILITY_SCALAR_32 = 1u<<3, + CU_ATOMIC_CAPABILITY_SCALAR_64 = 1u<<4, + CU_ATOMIC_CAPABILITY_SCALAR_128 = 1u<<5, + CU_ATOMIC_CAPABILITY_VECTOR_32x4 = 1u<<6 +} CUatomicOperationCapability; + /** * CUDA stream callback * \param hStream The stream the callback was added to, as passed to ::cuStreamAddCallback. May be NULL. @@ -2667,6 +3395,16 @@ typedef struct CUDA_MEMCPY3D_PEER_st { } CUDA_MEMCPY3D_PEER_v1; typedef CUDA_MEMCPY3D_PEER_v1 CUDA_MEMCPY3D_PEER; +/** + * Memcpy node parameters + */ +typedef struct CUDA_MEMCPY_NODE_PARAMS_st { + int flags; /**< Must be zero */ + int reserved; /**< Must be zero */ + CUcontext copyCtx; /**< Context on which to run the node */ + CUDA_MEMCPY3D copyParams; /**< Parameters for the memory copy */ +} CUDA_MEMCPY_NODE_PARAMS; + /** * Array descriptor */ @@ -2855,13 +3593,13 @@ typedef CUDA_RESOURCE_VIEW_DESC_v1 CUDA_RESOURCE_VIEW_DESC; #define CU_TENSOR_MAP_NUM_QWORDS 16 /** - * Tensor map descriptor. Requires compiler support for aligning to 64 bytes. + * Tensor map descriptor. Requires compiler support for aligning to 128 bytes. */ typedef struct CUtensorMap_st { -#if __cplusplus >= 201103L - alignas(64) +#if defined(__cplusplus) && (__cplusplus >= 201103L) + alignas(128) #elif __STDC_VERSION__ >= 201112L - _Alignas(64) + _Alignas(128) #endif cuuint64_t opaque[CU_TENSOR_MAP_NUM_QWORDS]; } CUtensorMap; @@ -2882,7 +3620,10 @@ typedef enum CUtensorMapDataType_enum { CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, CU_TENSOR_MAP_DATA_TYPE_FLOAT32_FTZ, CU_TENSOR_MAP_DATA_TYPE_TFLOAT32, - CU_TENSOR_MAP_DATA_TYPE_TFLOAT32_FTZ + CU_TENSOR_MAP_DATA_TYPE_TFLOAT32_FTZ, + CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, + CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, + CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B } CUtensorMapDataType; /** @@ -2901,7 +3642,10 @@ typedef enum CUtensorMapSwizzle_enum { CU_TENSOR_MAP_SWIZZLE_NONE = 0, CU_TENSOR_MAP_SWIZZLE_32B, CU_TENSOR_MAP_SWIZZLE_64B, - CU_TENSOR_MAP_SWIZZLE_128B + CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B, + CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B_FLIP_8B, + CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B } CUtensorMapSwizzle; /** @@ -2922,6 +3666,14 @@ typedef enum CUtensorMapFloatOOBfill_enum { CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA } CUtensorMapFloatOOBfill; +/** + * Tensor map Im2Col wide mode + */ +typedef enum CUtensorMapIm2ColWideMode_enum { + CU_TENSOR_MAP_IM2COL_WIDE_MODE_W = 0, + CU_TENSOR_MAP_IM2COL_WIDE_MODE_W128 +} CUtensorMapIm2ColWideMode; + /** * GPU Direct v3 tokens */ @@ -2993,7 +3745,11 @@ typedef enum CUexternalMemoryHandleType_enum { /** * Handle is an NvSciBuf object */ - CU_EXTERNAL_MEMORY_HANDLE_TYPE_NVSCIBUF = 8 + CU_EXTERNAL_MEMORY_HANDLE_TYPE_NVSCIBUF = 8, + /** + * Handle is a dma_buf file descriptor + */ + CU_EXTERNAL_MEMORY_HANDLE_TYPE_DMABUF_FD = 9 } CUexternalMemoryHandleType; /** @@ -3342,6 +4098,15 @@ typedef struct CUDA_EXT_SEM_SIGNAL_NODE_PARAMS_st { } CUDA_EXT_SEM_SIGNAL_NODE_PARAMS_v1; typedef CUDA_EXT_SEM_SIGNAL_NODE_PARAMS_v1 CUDA_EXT_SEM_SIGNAL_NODE_PARAMS; +/** + * Semaphore signal node parameters + */ +typedef struct CUDA_EXT_SEM_SIGNAL_NODE_PARAMS_v2_st { + CUexternalSemaphore* extSemArray; /**< Array of external semaphore handles. */ + const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS* paramsArray; /**< Array of external semaphore signal parameters. */ + unsigned int numExtSems; /**< Number of handles and parameters supplied in extSemArray and paramsArray. */ +} CUDA_EXT_SEM_SIGNAL_NODE_PARAMS_v2; + /** * Semaphore wait node parameters */ @@ -3352,6 +4117,15 @@ typedef struct CUDA_EXT_SEM_WAIT_NODE_PARAMS_st { } CUDA_EXT_SEM_WAIT_NODE_PARAMS_v1; typedef CUDA_EXT_SEM_WAIT_NODE_PARAMS_v1 CUDA_EXT_SEM_WAIT_NODE_PARAMS; +/** + * Semaphore wait node parameters + */ +typedef struct CUDA_EXT_SEM_WAIT_NODE_PARAMS_v2_st { + CUexternalSemaphore* extSemArray; /**< Array of external semaphore handles. */ + const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS* paramsArray; /**< Array of external semaphore wait parameters. */ + unsigned int numExtSems; /**< Number of handles and parameters supplied in extSemArray and paramsArray. */ +} CUDA_EXT_SEM_WAIT_NODE_PARAMS_v2; + typedef unsigned long long CUmemGenericAllocationHandle_v1; typedef CUmemGenericAllocationHandle_v1 CUmemGenericAllocationHandle; @@ -3363,6 +4137,7 @@ typedef enum CUmemAllocationHandleType_enum { CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR = 0x1, /**< Allows a file descriptor to be used for exporting. Permitted only on POSIX systems. (int) */ CU_MEM_HANDLE_TYPE_WIN32 = 0x2, /**< Allows a Win32 NT handle to be used for exporting. (HANDLE) */ CU_MEM_HANDLE_TYPE_WIN32_KMT = 0x4, /**< Allows a Win32 KMT handle to be used for exporting. (D3DKMT_HANDLE) */ + CU_MEM_HANDLE_TYPE_FABRIC = 0x8, /**< Allows a fabric handle to be used for exporting. (CUmemFabricHandle)*/ CU_MEM_HANDLE_TYPE_MAX = 0x7FFFFFFF } CUmemAllocationHandleType; @@ -3380,9 +4155,13 @@ typedef enum CUmemAccess_flags_enum { * Specifies the type of location */ typedef enum CUmemLocationType_enum { - CU_MEM_LOCATION_TYPE_INVALID = 0x0, - CU_MEM_LOCATION_TYPE_DEVICE = 0x1, /**< Location is a device location, thus id is a device ordinal */ - CU_MEM_LOCATION_TYPE_MAX = 0x7FFFFFFF + CU_MEM_LOCATION_TYPE_INVALID = 0x0, + CU_MEM_LOCATION_TYPE_NONE = 0x0, /**< Location is unspecified. This is used when creating a managed memory pool to indicate no preferred location for the pool */ + CU_MEM_LOCATION_TYPE_DEVICE = 0x1, /**< Location is a device location, thus id is a device ordinal */ + CU_MEM_LOCATION_TYPE_HOST = 0x2, /**< Location is host, id is ignored */ + CU_MEM_LOCATION_TYPE_HOST_NUMA = 0x3, /**< Location is a host NUMA node, thus id is a host NUMA node id */ + CU_MEM_LOCATION_TYPE_HOST_NUMA_CURRENT = 0x4, /**< Location is a host NUMA node of the current thread, id is ignored */ + CU_MEM_LOCATION_TYPE_MAX = 0x7FFFFFFF } CUmemLocationType; /** @@ -3395,6 +4174,9 @@ typedef enum CUmemAllocationType_enum { * location while the application is actively using it */ CU_MEM_ALLOCATION_TYPE_PINNED = 0x1, + /** This allocation type is managed memory + */ + CU_MEM_ALLOCATION_TYPE_MANAGED = 0x2, CU_MEM_ALLOCATION_TYPE_MAX = 0x7FFFFFFF } CUmemAllocationType; @@ -3415,6 +4197,13 @@ typedef enum CUmemRangeHandleType_enum CU_MEM_RANGE_HANDLE_TYPE_MAX = 0x7FFFFFFF } CUmemRangeHandleType; +/** +* Flag for requesting handle type for address range. +*/ +typedef enum CUmemRangeFlags_enum { + CU_MEM_RANGE_FLAG_DMA_BUF_MAPPING_TYPE_PCIE = 0x1 /**< Indicates that DMA_BUF handle should be mapped via PCIe BAR1 */ +} CUmemRangeFlags; + /** * Sparse subresource types */ @@ -3504,6 +4293,11 @@ typedef enum CUmemAllocationCompType_enum { * This flag if set indicates that the memory will be used as a tile pool. */ #define CU_MEM_CREATE_USAGE_TILE_POOL 0x1 +/** + * This flag, if set, indicates that the memory will be used as a buffer for + * hardware accelerated decompression. + */ +#define CU_MEM_CREATE_USAGE_HW_DECOMPRESS 0x2 /** * Specifies the allocation properties for a allocation. @@ -3690,6 +4484,12 @@ typedef enum CUmemPool_attribute_enum { CU_MEMPOOL_ATTR_USED_MEM_HIGH } CUmemPool_attribute; +/** + * This flag, if set, indicates that the memory will be used as a buffer for + * hardware accelerated decompression. + */ +#define CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS 0x2 + /** * Specifies the properties of allocations made from the pool. */ @@ -3704,7 +4504,9 @@ typedef struct CUmemPoolProps_st { * processes. In all other cases, this field is required to be zero. */ void *win32SecurityAttributes; - unsigned char reserved[64]; /**< reserved for future use, must be 0 */ + size_t maxSize; /**< Maximum pool size. When set to 0, defaults to a system dependent value. */ + unsigned short usage; /**< Bitmask indicating intended usage for the pool. */ + unsigned char reserved[54]; /**< reserved for future use, must be 0 */ } CUmemPoolProps_v1; typedef CUmemPoolProps_v1 CUmemPoolProps; @@ -3717,51 +4519,272 @@ typedef struct CUmemPoolPtrExportData_st { typedef CUmemPoolPtrExportData_v1 CUmemPoolPtrExportData; /** - * Memory allocation node parameters + * Flags to specify for copies within a batch. For more details see ::cuMemcpyBatchAsync. */ -typedef struct CUDA_MEM_ALLOC_NODE_PARAMS_st { +typedef enum CUmemcpyFlags_enum { + CU_MEMCPY_FLAG_DEFAULT = 0x0, + /** - * in: location where the allocation should reside (specified in ::location). - * ::handleTypes must be ::CU_MEM_HANDLE_TYPE_NONE. IPC is not supported. - */ - CUmemPoolProps poolProps; - const CUmemAccessDesc *accessDescs; /**< in: array of memory access descriptors. Used to describe peer GPU access */ - size_t accessDescCount; /**< in: number of memory access descriptors. Must not exceed the number of GPUs. */ - size_t bytesize; /**< in: size in bytes of the requested allocation */ - CUdeviceptr dptr; /**< out: address of the allocation returned by CUDA */ -} CUDA_MEM_ALLOC_NODE_PARAMS; + * Hint to the driver to try and overlap the copy with compute work on the SMs. + */ + CU_MEMCPY_FLAG_PREFER_OVERLAP_WITH_COMPUTE = 0x1 +} CUmemcpyFlags; -typedef enum CUgraphMem_attribute_enum { +/** + * These flags allow applications to convey the source access ordering CUDA must maintain. + * The destination will always be accessed in stream order. + */ +typedef enum CUmemcpySrcAccessOrder_enum { /** - * (value type = cuuint64_t) - * Amount of memory, in bytes, currently associated with graphs + * Default invalid. */ - CU_GRAPH_MEM_ATTR_USED_MEM_CURRENT, + CU_MEMCPY_SRC_ACCESS_ORDER_INVALID = 0x0, /** - * (value type = cuuint64_t) - * High watermark of memory, in bytes, associated with graphs since the - * last time it was reset. High watermark can only be reset to zero. + * Indicates that access to the source pointer must be in stream order. */ - CU_GRAPH_MEM_ATTR_USED_MEM_HIGH, + CU_MEMCPY_SRC_ACCESS_ORDER_STREAM = 0x1, /** - * (value type = cuuint64_t) - * Amount of memory, in bytes, currently allocated for use by - * the CUDA graphs asynchronous allocator. + * Indicates that access to the source pointer can be out of stream order and + * all accesses must be complete before the API call returns. This flag is suited for + * ephemeral sources (ex., stack variables) when it's known that no prior operations + * in the stream can be accessing the memory and also that the lifetime of the memory + * is limited to the scope that the source variable was declared in. Specifying + * this flag allows the driver to optimize the copy and removes the need for the user + * to synchronize the stream after the API call. */ - CU_GRAPH_MEM_ATTR_RESERVED_MEM_CURRENT, + CU_MEMCPY_SRC_ACCESS_ORDER_DURING_API_CALL = 0x2, /** - * (value type = cuuint64_t) - * High watermark of memory, in bytes, currently allocated for use by - * the CUDA graphs asynchronous allocator. + * Indicates that access to the source pointer can be out of stream order and the accesses + * can happen even after the API call returns. This flag is suited for host pointers + * allocated outside CUDA (ex., via malloc) when it's known that no prior operations + * in the stream can be accessing the memory. Specifying this flag allows the driver + * to optimize the copy on certain platforms. */ - CU_GRAPH_MEM_ATTR_RESERVED_MEM_HIGH -} CUgraphMem_attribute; + CU_MEMCPY_SRC_ACCESS_ORDER_ANY = 0x3, + + CU_MEMCPY_SRC_ACCESS_ORDER_MAX = 0x7FFFFFFF +} CUmemcpySrcAccessOrder; /** - * If set, each kernel launched as part of ::cuLaunchCooperativeKernelMultiDevice only + * Attributes specific to copies within a batch. For more details on usage see ::cuMemcpyBatchAsync. + */ +typedef struct CUmemcpyAttributes_st { + CUmemcpySrcAccessOrder srcAccessOrder; /**< Source access ordering to be observed for copies with this attribute. */ + CUmemLocation srcLocHint; /**< Hint location for the source operand. Ignored when the pointers are not managed memory or memory allocated outside CUDA. */ + CUmemLocation dstLocHint; /**< Hint location for the destination operand. Ignored when the pointers are not managed memory or memory allocated outside CUDA. */ + unsigned int flags; /**< Additional flags for copies with this attribute. See ::CUmemcpyFlags */ +} CUmemcpyAttributes_v1; +typedef CUmemcpyAttributes_v1 CUmemcpyAttributes; + +/** + * These flags allow applications to convey the operand type for individual copies specified in ::cuMemcpy3DBatchAsync. + */ +typedef enum CUmemcpy3DOperandType_enum { + CU_MEMCPY_OPERAND_TYPE_POINTER = 0x1, /**< Memcpy operand is a valid pointer. */ + CU_MEMCPY_OPERAND_TYPE_ARRAY = 0x2, /**< Memcpy operand is a CUarray. */ + CU_MEMCPY_OPERAND_TYPE_MAX = 0x7FFFFFFF +} CUmemcpy3DOperandType; + +/** + * Struct representing offset into a CUarray in elements + */ +typedef struct CUoffset3D_st { + size_t x; + size_t y; + size_t z; +} CUoffset3D_v1; +typedef CUoffset3D_v1 CUoffset3D; + +/** + * Struct representing width/height/depth of a CUarray in elements + */ +typedef struct CUextent3D_st { + size_t width; + size_t height; + size_t depth; +} CUextent3D_v1; +typedef CUextent3D_v1 CUextent3D; + +/** + * Struct representing an operand for copy with ::cuMemcpy3DBatchAsync + */ +typedef struct CUmemcpy3DOperand_st { + CUmemcpy3DOperandType type; + union { + /** + * Struct representing an operand when ::CUmemcpy3DOperand::type is ::CU_MEMCPY_OPERAND_TYPE_POINTER + */ + struct { + CUdeviceptr ptr; + size_t rowLength; /**< Length of each row in elements. */ + size_t layerHeight; /**< Height of each layer in elements. */ + CUmemLocation locHint; /**< Hint location for the operand. Ignored when the pointers are not managed memory or memory allocated outside CUDA. */ + } ptr; + + /** + * Struct representing an operand when ::CUmemcpy3DOperand::type is ::CU_MEMCPY_OPERAND_TYPE_ARRAY + */ + struct { + CUarray array; + CUoffset3D offset; + } array; + } op; +} CUmemcpy3DOperand_v1; +typedef CUmemcpy3DOperand_v1 CUmemcpy3DOperand; + +typedef struct CUDA_MEMCPY3D_BATCH_OP_st { + CUmemcpy3DOperand src; /**< Source memcpy operand. */ + CUmemcpy3DOperand dst; /**< Destination memcpy operand. */ + CUextent3D extent; /**< Extents of the memcpy between src and dst. The width, height and depth components must not be 0.*/ + CUmemcpySrcAccessOrder srcAccessOrder; /**< Source access ordering to be observed for copy from src to dst. */ + unsigned int flags; /**< Additional flags for copies with this attribute. See ::CUmemcpyFlags */ +} CUDA_MEMCPY3D_BATCH_OP_v1; +typedef CUDA_MEMCPY3D_BATCH_OP_v1 CUDA_MEMCPY3D_BATCH_OP; + +/** + * Memory allocation node parameters + */ +typedef struct CUDA_MEM_ALLOC_NODE_PARAMS_v1_st { + /** + * in: location where the allocation should reside (specified in ::location). + * ::handleTypes must be ::CU_MEM_HANDLE_TYPE_NONE. IPC is not supported. + */ + CUmemPoolProps poolProps; + const CUmemAccessDesc *accessDescs; /**< in: array of memory access descriptors. Used to describe peer GPU access */ + size_t accessDescCount; /**< in: number of memory access descriptors. Must not exceed the number of GPUs. */ + size_t bytesize; /**< in: size in bytes of the requested allocation */ + CUdeviceptr dptr; /**< out: address of the allocation returned by CUDA */ +} CUDA_MEM_ALLOC_NODE_PARAMS_v1; +typedef CUDA_MEM_ALLOC_NODE_PARAMS_v1 CUDA_MEM_ALLOC_NODE_PARAMS; + +/** + * Memory allocation node parameters + */ +typedef struct CUDA_MEM_ALLOC_NODE_PARAMS_v2_st { + /** + * in: location where the allocation should reside (specified in ::location). + * ::handleTypes must be ::CU_MEM_HANDLE_TYPE_NONE. IPC is not supported. + */ + CUmemPoolProps poolProps; + const CUmemAccessDesc *accessDescs; /**< in: array of memory access descriptors. Used to describe peer GPU access */ + size_t accessDescCount; /**< in: number of memory access descriptors. Must not exceed the number of GPUs. */ + size_t bytesize; /**< in: size in bytes of the requested allocation */ + CUdeviceptr dptr; /**< out: address of the allocation returned by CUDA */ +} CUDA_MEM_ALLOC_NODE_PARAMS_v2; + +/** + * Memory free node parameters + */ +typedef struct CUDA_MEM_FREE_NODE_PARAMS_st { + CUdeviceptr dptr; /**< in: the pointer to free */ +} CUDA_MEM_FREE_NODE_PARAMS; + +typedef enum CUgraphMem_attribute_enum { + /** + * (value type = cuuint64_t) + * Amount of memory, in bytes, currently associated with graphs + */ + CU_GRAPH_MEM_ATTR_USED_MEM_CURRENT, + + /** + * (value type = cuuint64_t) + * High watermark of memory, in bytes, associated with graphs since the + * last time it was reset. High watermark can only be reset to zero. + */ + CU_GRAPH_MEM_ATTR_USED_MEM_HIGH, + + /** + * (value type = cuuint64_t) + * Amount of memory, in bytes, currently allocated for use by + * the CUDA graphs asynchronous allocator. + */ + CU_GRAPH_MEM_ATTR_RESERVED_MEM_CURRENT, + + /** + * (value type = cuuint64_t) + * High watermark of memory, in bytes, currently allocated for use by + * the CUDA graphs asynchronous allocator. + */ + CU_GRAPH_MEM_ATTR_RESERVED_MEM_HIGH +} CUgraphMem_attribute; + +/** + * Child graph node ownership + */ +typedef enum CUgraphChildGraphNodeOwnership_enum { + CU_GRAPH_CHILD_GRAPH_OWNERSHIP_CLONE = 0, /**< Default behavior for a child graph node. Child graph is cloned + into the parent and memory allocation/free nodes can't be present + in the child graph. */ + CU_GRAPH_CHILD_GRAPH_OWNERSHIP_MOVE = 1, /**< The child graph is moved to the parent. The handle to the child graph + is owned by the parent and will be destroyed when the parent is + destroyed. + + The following restrictions apply to child graphs after they have been moved: + Cannot be independently instantiated or destroyed; + Cannot be added as a child graph of a separate parent graph; + Cannot be used as an argument to cuGraphExecUpdate; + Cannot have additional memory allocation or free nodes added. */ +} CUgraphChildGraphNodeOwnership; + +/** + * Child graph node parameters + */ +typedef struct CUDA_CHILD_GRAPH_NODE_PARAMS_st { + CUgraph graph; /**< The child graph to clone into the node for node creation, or + * a handle to the graph owned by the node for node query. + * The graph must not contain conditional nodes. Graphs + * containing memory allocation or memory free nodes must + * set the ownership to be moved to the parent. + */ + CUgraphChildGraphNodeOwnership ownership; /**< The ownership relationship of the child graph node. */ +} CUDA_CHILD_GRAPH_NODE_PARAMS; + +/** + * Event record node parameters + */ +typedef struct CUDA_EVENT_RECORD_NODE_PARAMS_st { + CUevent event; /**< The event to record when the node executes */ +} CUDA_EVENT_RECORD_NODE_PARAMS; + +/** + * Event wait node parameters + */ +typedef struct CUDA_EVENT_WAIT_NODE_PARAMS_st { + CUevent event; /**< The event to wait on from the node */ +} CUDA_EVENT_WAIT_NODE_PARAMS; + +/** + * Graph node parameters. See ::cuGraphAddNode. + */ +typedef struct CUgraphNodeParams_st { + CUgraphNodeType type; /**< Type of the node */ + int reserved0[3]; /**< Reserved. Must be zero. */ + + union { + long long reserved1[29]; /**< Padding. Unused bytes must be zero. */ + CUDA_KERNEL_NODE_PARAMS_v3 kernel; /**< Kernel node parameters. */ + CUDA_MEMCPY_NODE_PARAMS memcpy; /**< Memcpy node parameters. */ + CUDA_MEMSET_NODE_PARAMS_v2 memset; /**< Memset node parameters. */ + CUDA_HOST_NODE_PARAMS_v2 host; /**< Host node parameters. */ + CUDA_CHILD_GRAPH_NODE_PARAMS graph; /**< Child graph node parameters. */ + CUDA_EVENT_WAIT_NODE_PARAMS eventWait; /**< Event wait node parameters. */ + CUDA_EVENT_RECORD_NODE_PARAMS eventRecord; /**< Event record node parameters. */ + CUDA_EXT_SEM_SIGNAL_NODE_PARAMS_v2 extSemSignal; /**< External semaphore signal node parameters. */ + CUDA_EXT_SEM_WAIT_NODE_PARAMS_v2 extSemWait; /**< External semaphore wait node parameters. */ + CUDA_MEM_ALLOC_NODE_PARAMS_v2 alloc; /**< Memory allocation node parameters. */ + CUDA_MEM_FREE_NODE_PARAMS free; /**< Memory free node parameters. */ + CUDA_BATCH_MEM_OP_NODE_PARAMS_v2 memOp; /**< MemOp node parameters. */ + CUDA_CONDITIONAL_NODE_PARAMS conditional; /**< Conditional node parameters. */ + }; + + long long reserved2; /**< Reserved bytes. Must be zero. */ +} CUgraphNodeParams; + +/** + * If set, each kernel launched as part of ::cuLaunchCooperativeKernelMultiDevice only * waits for prior work in the stream corresponding to that GPU to complete before the * kernel begins execution. */ @@ -3830,6 +4853,12 @@ typedef enum CUgraphMem_attribute_enum { */ #define CUDA_ARRAY3D_DEFERRED_MAPPING 0x80 +/** + * This flag indicates that the CUDA array will be used for hardware accelerated + * video encode/decode operations. + */ +#define CUDA_ARRAY3D_VIDEO_ENCODE_DECODE 0x100 + /** * Override the texref format with a format inferred from the array. * Flag for ::cuTexRefSetArray() @@ -3867,6 +4896,11 @@ typedef enum CUgraphMem_attribute_enum { */ #define CU_TRSF_SEAMLESS_CUBEMAP 0x40 +/** + * Launch with the required block dimension. + */ +#define CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM 1 + /** * C++ compile time constant for CU_LAUNCH_PARAM_END */ @@ -3974,8 +5008,9 @@ typedef enum CUgraphDebugDot_flags_enum { CU_GRAPH_DEBUG_DOT_FLAGS_HANDLES = 1<<10, /**< Adds node handles and every kernel function handle to output */ CU_GRAPH_DEBUG_DOT_FLAGS_MEM_ALLOC_NODE_PARAMS = 1<<11, /**< Adds memory alloc node parameters to output */ CU_GRAPH_DEBUG_DOT_FLAGS_MEM_FREE_NODE_PARAMS = 1<<12, /**< Adds memory free node parameters to output */ - CU_GRAPH_DEBUG_DOT_FLAGS_BATCH_MEM_OP_NODE_PARAMS = 1<<13 /**< Adds batch mem op node parameters to output */ - , CU_GRAPH_DEBUG_DOT_FLAGS_EXTRA_TOPO_INFO = 1<<14 /**< Adds edge numbering information */ + CU_GRAPH_DEBUG_DOT_FLAGS_BATCH_MEM_OP_NODE_PARAMS = 1<<13, /**< Adds batch mem op node parameters to output */ + CU_GRAPH_DEBUG_DOT_FLAGS_EXTRA_TOPO_INFO = 1<<14, /**< Adds edge numbering information */ + CU_GRAPH_DEBUG_DOT_FLAGS_CONDITIONAL_NODE_PARAMS = 1<<15 /**< Adds conditional node parameters to output */ } CUgraphDebugDot_flags; /** @@ -3997,12 +5032,75 @@ typedef enum CUuserObjectRetain_flags_enum { */ typedef enum CUgraphInstantiate_flags_enum { CUDA_GRAPH_INSTANTIATE_FLAG_AUTO_FREE_ON_LAUNCH = 1 /**< Automatically free memory allocated in a graph before relaunching. */ - , CUDA_GRAPH_INSTANTIATE_FLAG_UPLOAD = 2 /**< Automatically upload the graph after instantiaton. */ - , CUDA_GRAPH_INSTANTIATE_FLAG_DEVICE_LAUNCH = 4 /**< Instantiate the graph to be launchable from the device. */ + , CUDA_GRAPH_INSTANTIATE_FLAG_UPLOAD = 2 /**< Automatically upload the graph after instantiation. Only supported by + ::cuGraphInstantiateWithParams. The upload will be performed using the + stream provided in \p instantiateParams. */ + , CUDA_GRAPH_INSTANTIATE_FLAG_DEVICE_LAUNCH = 4 /**< Instantiate the graph to be launchable from the device. This flag can only + be used on platforms which support unified addressing. This flag cannot be + used in conjunction with CUDA_GRAPH_INSTANTIATE_FLAG_AUTO_FREE_ON_LAUNCH. */ , CUDA_GRAPH_INSTANTIATE_FLAG_USE_NODE_PRIORITY = 8 /**< Run the graph using the per-node priority attributes rather than the priority of the stream it is launched into. */ } CUgraphInstantiate_flags; +/** + * CUDA device NUMA configuration + */ +typedef enum CUdeviceNumaConfig_enum { + CU_DEVICE_NUMA_CONFIG_NONE = 0, /**< The GPU is not a NUMA node */ + CU_DEVICE_NUMA_CONFIG_NUMA_NODE, /**< The GPU is a NUMA node, CU_DEVICE_ATTRIBUTE_NUMA_ID contains its NUMA ID */ +} CUdeviceNumaConfig; + +/** + * CUDA Process States + */ +typedef enum CUprocessState_enum { + CU_PROCESS_STATE_RUNNING = 0, /**< Default process state */ + CU_PROCESS_STATE_LOCKED, /**< CUDA API locks are taken so further CUDA API calls will block */ + CU_PROCESS_STATE_CHECKPOINTED, /**< Application memory contents have been checkpointed and underlying allocations and device handles have been released */ + CU_PROCESS_STATE_FAILED, /**< Application entered an uncorrectable error during the checkpoint/restore process */ +} CUprocessState; + +/** + * CUDA checkpoint optional lock arguments + */ +typedef struct CUcheckpointLockArgs_st { + unsigned int timeoutMs; /**< Timeout in milliseconds to attempt to lock the process, 0 indicates no timeout */ + unsigned int reserved0; /**< Reserved for future use, must be zero */ + cuuint64_t reserved1[7]; /**< Reserved for future use, must be zeroed */ +} CUcheckpointLockArgs; + +/** + * CUDA checkpoint optional checkpoint arguments + */ +typedef struct CUcheckpointCheckpointArgs_st { + cuuint64_t reserved[8]; /**< Reserved for future use, must be zeroed */ +} CUcheckpointCheckpointArgs; + +/** + * CUDA checkpoint GPU UUID pairs for device remapping during restore + */ +typedef struct CUcheckpointGpuPair_st { + CUuuid oldUuid; /**< UUID of the GPU that was checkpointed */ + CUuuid newUuid; /**< UUID of the GPU to restore onto */ +} CUcheckpointGpuPair; + +/** + * CUDA checkpoint optional restore arguments + */ +typedef struct CUcheckpointRestoreArgs_st { + CUcheckpointGpuPair *gpuPairs; /**< Pointer to array of gpu pairs that indicate how to remap GPUs during restore */ + unsigned int gpuPairsCount; /**< Number of gpu pairs to remap */ + char reserved[52 - sizeof(CUcheckpointGpuPair *)]; /**< Reserved for future use, must be zeroed */ + cuuint64_t reserved1; /**< Reserved for future use, must be zeroed */ +} CUcheckpointRestoreArgs; + +/** + * CUDA checkpoint optional unlock arguments + */ +typedef struct CUcheckpointUnlockArgs_st { + cuuint64_t reserved[8]; /**< Reserved for future use, must be zeroed */ +} CUcheckpointUnlockArgs; + /** @} */ /* END CUDA_TYPES */ #if defined(__GNUC__) @@ -4092,6 +5190,10 @@ CUresult CUDAAPI cuGetErrorName(CUresult error, const char **pStr); * has not been called, any function from the driver API will return * ::CUDA_ERROR_NOT_INITIALIZED. * + * Note: cuInit preloads various libraries needed for JIT compilation. To opt-out of this behavior, + * set the environment variable CUDA_FORCE_PRELOAD_LIBRARIES=0. CUDA will lazily load JIT libraries as needed. + * To disable JIT entirely, set the environment variable CUDA_DISABLE_JIT=1. + * * \param Flags - Initialization flag for CUDA. * * \return @@ -4123,7 +5225,7 @@ CUresult CUDAAPI cuInit(unsigned int Flags); * * Returns in \p *driverVersion the version of CUDA supported by * the driver. The version is returned as - * (1000 × major + 10 × minor). For example, CUDA 9.2 + * (1000 * major + 10 * minor). For example, CUDA 9.2 * would be represented by 9020. * * This function automatically returns ::CUDA_ERROR_INVALID_VALUE if @@ -4219,7 +5321,7 @@ CUresult CUDAAPI cuDeviceGetCount(int *count); * * Returns an ASCII string identifying the device \p dev in the NULL-terminated * string pointed to by \p name. \p len specifies the maximum length of the - * string that may be returned. + * string that may be returned. \p name is shortened to the specified \p len, if \p len is less than the device name * * \param name - Returned identifier string for the device * \param len - Maximum length of string to store in \p name @@ -4249,39 +5351,6 @@ CUresult CUDAAPI cuDeviceGetName(char *name, int len, CUdevice dev); /** * \brief Return an UUID for the device * - * Note there is a later version of this API, ::cuDeviceGetUuid_v2. It will - * supplant this version in 12.0, which is retained for minor version compatibility. - * - * Returns 16-octets identifying the device \p dev in the structure - * pointed by the \p uuid. - * - * \param uuid - Returned UUID - * \param dev - Device to get identifier string for - * - * \return - * ::CUDA_SUCCESS, - * ::CUDA_ERROR_DEINITIALIZED, - * ::CUDA_ERROR_NOT_INITIALIZED, - * ::CUDA_ERROR_INVALID_VALUE, - * ::CUDA_ERROR_INVALID_DEVICE - * \notefnerr - * - * \sa - * ::cuDeviceGetUuid_v2 - * ::cuDeviceGetAttribute, - * ::cuDeviceGetCount, - * ::cuDeviceGetName, - * ::cuDeviceGetLuid, - * ::cuDeviceGet, - * ::cuDeviceTotalMem, - * ::cuDeviceGetExecAffinitySupport, - * ::cudaGetDeviceProperties - */ -CUresult CUDAAPI cuDeviceGetUuid(CUuuid *uuid, CUdevice dev); - -/** - * \brief Return an UUID for the device (11.4+) - * * Returns 16-octets identifying the device \p dev in the structure * pointed by the \p uuid. If the device is in MIG mode, returns its * MIG UUID which uniquely identifies the subscribed MIG compute instance. @@ -4306,7 +5375,7 @@ CUresult CUDAAPI cuDeviceGetUuid(CUuuid *uuid, CUdevice dev); * ::cuDeviceTotalMem, * ::cudaGetDeviceProperties */ -CUresult CUDAAPI cuDeviceGetUuid_v2(CUuuid *uuid, CUdevice dev); +CUresult CUDAAPI cuDeviceGetUuid(CUuuid *uuid, CUdevice dev); /** * \brief Return an LUID and device node mask for the device @@ -4401,199 +5470,7 @@ CUresult CUDAAPI cuDeviceGetTexture1DLinearMaxWidth(size_t *maxWidthInElements, * \brief Returns information about the device * * Returns in \p *pi the integer value of the attribute \p attrib on device - * \p dev. The supported attributes are: - * - ::CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK: Maximum number of threads per - * block; - * - ::CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X: Maximum x-dimension of a block - * - ::CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Y: Maximum y-dimension of a block - * - ::CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Z: Maximum z-dimension of a block - * - ::CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X: Maximum x-dimension of a grid - * - ::CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Y: Maximum y-dimension of a grid - * - ::CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Z: Maximum z-dimension of a grid - * - ::CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK: Maximum amount of - * shared memory available to a thread block in bytes - * - ::CU_DEVICE_ATTRIBUTE_TOTAL_CONSTANT_MEMORY: Memory available on device for - * __constant__ variables in a CUDA C kernel in bytes - * - ::CU_DEVICE_ATTRIBUTE_WARP_SIZE: Warp size in threads - * - ::CU_DEVICE_ATTRIBUTE_MAX_PITCH: Maximum pitch in bytes allowed by the - * memory copy functions that involve memory regions allocated through - * ::cuMemAllocPitch() - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE1D_WIDTH: Maximum 1D - * texture width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE1D_LINEAR_WIDTH: Maximum width - * for a 1D texture bound to linear memory - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE1D_MIPMAPPED_WIDTH: Maximum - * mipmapped 1D texture width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE2D_WIDTH: Maximum 2D - * texture width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE2D_HEIGHT: Maximum 2D - * texture height - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE2D_LINEAR_WIDTH: Maximum width - * for a 2D texture bound to linear memory - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE2D_LINEAR_HEIGHT: Maximum height - * for a 2D texture bound to linear memory - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE2D_LINEAR_PITCH: Maximum pitch - * in bytes for a 2D texture bound to linear memory - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE2D_MIPMAPPED_WIDTH: Maximum - * mipmapped 2D texture width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE2D_MIPMAPPED_HEIGHT: Maximum - * mipmapped 2D texture height - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE3D_WIDTH: Maximum 3D - * texture width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE3D_HEIGHT: Maximum 3D - * texture height - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE3D_DEPTH: Maximum 3D - * texture depth - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE3D_WIDTH_ALTERNATE: - * Alternate maximum 3D texture width, 0 if no alternate - * maximum 3D texture size is supported - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE3D_HEIGHT_ALTERNATE: - * Alternate maximum 3D texture height, 0 if no alternate - * maximum 3D texture size is supported - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE3D_DEPTH_ALTERNATE: - * Alternate maximum 3D texture depth, 0 if no alternate - * maximum 3D texture size is supported - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURECUBEMAP_WIDTH: - * Maximum cubemap texture width or height - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE1D_LAYERED_WIDTH: - * Maximum 1D layered texture width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE1D_LAYERED_LAYERS: - * Maximum layers in a 1D layered texture - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE2D_LAYERED_WIDTH: - * Maximum 2D layered texture width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE2D_LAYERED_HEIGHT: - * Maximum 2D layered texture height - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURE2D_LAYERED_LAYERS: - * Maximum layers in a 2D layered texture - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURECUBEMAP_LAYERED_WIDTH: - * Maximum cubemap layered texture width or height - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_TEXTURECUBEMAP_LAYERED_LAYERS: - * Maximum layers in a cubemap layered texture - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE1D_WIDTH: - * Maximum 1D surface width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE2D_WIDTH: - * Maximum 2D surface width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE2D_HEIGHT: - * Maximum 2D surface height - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE3D_WIDTH: - * Maximum 3D surface width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE3D_HEIGHT: - * Maximum 3D surface height - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE3D_DEPTH: - * Maximum 3D surface depth - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE1D_LAYERED_WIDTH: - * Maximum 1D layered surface width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE1D_LAYERED_LAYERS: - * Maximum layers in a 1D layered surface - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE2D_LAYERED_WIDTH: - * Maximum 2D layered surface width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE2D_LAYERED_HEIGHT: - * Maximum 2D layered surface height - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACE2D_LAYERED_LAYERS: - * Maximum layers in a 2D layered surface - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACECUBEMAP_WIDTH: - * Maximum cubemap surface width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACECUBEMAP_LAYERED_WIDTH: - * Maximum cubemap layered surface width - * - ::CU_DEVICE_ATTRIBUTE_MAXIMUM_SURFACECUBEMAP_LAYERED_LAYERS: - * Maximum layers in a cubemap layered surface - * - ::CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_BLOCK: Maximum number of 32-bit - * registers available to a thread block - * - ::CU_DEVICE_ATTRIBUTE_CLOCK_RATE: The typical clock frequency in kilohertz - * - ::CU_DEVICE_ATTRIBUTE_TEXTURE_ALIGNMENT: Alignment requirement; texture - * base addresses aligned to ::textureAlign bytes do not need an offset - * applied to texture fetches - * - ::CU_DEVICE_ATTRIBUTE_TEXTURE_PITCH_ALIGNMENT: Pitch alignment requirement - * for 2D texture references bound to pitched memory - * - ::CU_DEVICE_ATTRIBUTE_GPU_OVERLAP: 1 if the device can concurrently copy - * memory between host and device while executing a kernel, or 0 if not - * - ::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT: Number of multiprocessors on - * the device - * - ::CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT: 1 if there is a run time limit - * for kernels executed on the device, or 0 if not - * - ::CU_DEVICE_ATTRIBUTE_INTEGRATED: 1 if the device is integrated with the - * memory subsystem, or 0 if not - * - ::CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY: 1 if the device can map host - * memory into the CUDA address space, or 0 if not - * - ::CU_DEVICE_ATTRIBUTE_COMPUTE_MODE: Compute mode that device is currently - * in. Available modes are as follows: - * - ::CU_COMPUTEMODE_DEFAULT: Default mode - Device is not restricted and - * can have multiple CUDA contexts present at a single time. - * - ::CU_COMPUTEMODE_PROHIBITED: Compute-prohibited mode - Device is - * prohibited from creating new CUDA contexts. - * - ::CU_COMPUTEMODE_EXCLUSIVE_PROCESS: Compute-exclusive-process mode - Device - * can have only one context used by a single process at a time. - * - ::CU_DEVICE_ATTRIBUTE_CONCURRENT_KERNELS: 1 if the device supports - * executing multiple kernels within the same context simultaneously, or 0 if - * not. It is not guaranteed that multiple kernels will be resident - * on the device concurrently so this feature should not be relied upon for - * correctness. - * - ::CU_DEVICE_ATTRIBUTE_ECC_ENABLED: 1 if error correction is enabled on the - * device, 0 if error correction is disabled or not supported by the device - * - ::CU_DEVICE_ATTRIBUTE_PCI_BUS_ID: PCI bus identifier of the device - * - ::CU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID: PCI device (also known as slot) identifier - * of the device - * - ::CU_DEVICE_ATTRIBUTE_PCI_DOMAIN_ID: PCI domain identifier of the device - * - ::CU_DEVICE_ATTRIBUTE_TCC_DRIVER: 1 if the device is using a TCC driver. TCC - * is only available on Tesla hardware running Windows Vista or later - * - ::CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE: Peak memory clock frequency in kilohertz - * - ::CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH: Global memory bus width in bits - * - ::CU_DEVICE_ATTRIBUTE_L2_CACHE_SIZE: Size of L2 cache in bytes. 0 if the device doesn't have L2 cache - * - ::CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR: Maximum resident threads per multiprocessor - * - ::CU_DEVICE_ATTRIBUTE_UNIFIED_ADDRESSING: 1 if the device shares a unified address space with - * the host, or 0 if not - * - ::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR: Major compute capability version number - * - ::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR: Minor compute capability version number - * - ::CU_DEVICE_ATTRIBUTE_GLOBAL_L1_CACHE_SUPPORTED: 1 if device supports caching globals - * in L1 cache, 0 if caching globals in L1 cache is not supported by the device - * - ::CU_DEVICE_ATTRIBUTE_LOCAL_L1_CACHE_SUPPORTED: 1 if device supports caching locals - * in L1 cache, 0 if caching locals in L1 cache is not supported by the device - * - ::CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR: Maximum amount of - * shared memory available to a multiprocessor in bytes; this amount is shared - * by all thread blocks simultaneously resident on a multiprocessor - * - ::CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_MULTIPROCESSOR: Maximum number of 32-bit - * registers available to a multiprocessor; this number is shared by all thread - * blocks simultaneously resident on a multiprocessor - * - ::CU_DEVICE_ATTRIBUTE_MANAGED_MEMORY: 1 if device supports allocating managed memory - * on this system, 0 if allocating managed memory is not supported by the device on this system. - * - ::CU_DEVICE_ATTRIBUTE_MULTI_GPU_BOARD: 1 if device is on a multi-GPU board, 0 if not. - * - ::CU_DEVICE_ATTRIBUTE_MULTI_GPU_BOARD_GROUP_ID: Unique identifier for a group of devices - * associated with the same board. Devices on the same multi-GPU board will share the same identifier. - * - ::CU_DEVICE_ATTRIBUTE_HOST_NATIVE_ATOMIC_SUPPORTED: 1 if Link between the device and the host - * supports native atomic operations. - * - ::CU_DEVICE_ATTRIBUTE_SINGLE_TO_DOUBLE_PRECISION_PERF_RATIO: Ratio of single precision performance - * (in floating-point operations per second) to double precision performance. - * - ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS: Device supports coherently accessing - * pageable memory without calling cudaHostRegister on it. - * - ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS: Device can coherently access managed memory - * concurrently with the CPU. - * - ::CU_DEVICE_ATTRIBUTE_COMPUTE_PREEMPTION_SUPPORTED: Device supports Compute Preemption. - * - ::CU_DEVICE_ATTRIBUTE_CAN_USE_HOST_POINTER_FOR_REGISTERED_MEM: Device can access host registered - * memory at the same virtual address as the CPU. - * - ::CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN: The maximum per block shared memory size - * supported on this device. This is the maximum value that can be opted into when using the cuFuncSetAttribute() or cuKernelSetAttribute() call. - * For more details see ::CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES - * - ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS_USES_HOST_PAGE_TABLES: Device accesses pageable memory via the host's - * page tables. - * - ::CU_DEVICE_ATTRIBUTE_DIRECT_MANAGED_MEM_ACCESS_FROM_HOST: The host can directly access managed memory on the device without migration. - * - ::CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED: Device supports virtual memory management APIs like ::cuMemAddressReserve, ::cuMemCreate, ::cuMemMap and related APIs - * - ::CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR_SUPPORTED: Device supports exporting memory to a posix file descriptor with ::cuMemExportToShareableHandle, if requested via ::cuMemCreate - * - ::CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_WIN32_HANDLE_SUPPORTED: Device supports exporting memory to a Win32 NT handle with ::cuMemExportToShareableHandle, if requested via ::cuMemCreate - * - ::CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_WIN32_KMT_HANDLE_SUPPORTED: Device supports exporting memory to a Win32 KMT handle with ::cuMemExportToShareableHandle, if requested via ::cuMemCreate - * - ::CU_DEVICE_ATTRIBUTE_MAX_BLOCKS_PER_MULTIPROCESSOR: Maximum number of thread blocks that can reside on a multiprocessor - * - ::CU_DEVICE_ATTRIBUTE_GENERIC_COMPRESSION_SUPPORTED: Device supports compressible memory allocation via ::cuMemCreate - * - ::CU_DEVICE_ATTRIBUTE_MAX_PERSISTING_L2_CACHE_SIZE: Maximum L2 persisting lines capacity setting in bytes - * - ::CU_DEVICE_ATTRIBUTE_MAX_ACCESS_POLICY_WINDOW_SIZE: Maximum value of CUaccessPolicyWindow::num_bytes - * - ::CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WITH_CUDA_VMM_SUPPORTED: Device supports specifying the GPUDirect RDMA flag with ::cuMemCreate. - * - ::CU_DEVICE_ATTRIBUTE_RESERVED_SHARED_MEMORY_PER_BLOCK: Amount of shared memory per block reserved by CUDA driver in bytes - * - ::CU_DEVICE_ATTRIBUTE_SPARSE_CUDA_ARRAY_SUPPORTED: Device supports sparse CUDA arrays and sparse CUDA mipmapped arrays. - * - ::CU_DEVICE_ATTRIBUTE_READ_ONLY_HOST_REGISTER_SUPPORTED: Device supports using the ::cuMemHostRegister flag ::CU_MEMHOSTERGISTER_READ_ONLY to register memory that must be mapped as read-only to the GPU - * - ::CU_DEVICE_ATTRIBUTE_MEMORY_POOLS_SUPPORTED: Device supports using the ::cuMemAllocAsync and ::cuMemPool family of APIs - * - ::CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_SUPPORTED: Device supports GPUDirect RDMA APIs, like nvidia_p2p_get_pages (see https://docs.nvidia.com/cuda/gpudirect-rdma for more information) - * - ::CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_FLUSH_WRITES_OPTIONS: The returned attribute shall be interpreted as a bitmask, where the individual bits are described by the ::CUflushGPUDirectRDMAWritesOptions enum - * - ::CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WRITES_ORDERING: GPUDirect RDMA writes to the device do not need to be flushed for consumers within the scope indicated by the returned attribute. See ::CUGPUDirectRDMAWritesOrdering for the numerical values returned here. - * - ::CU_DEVICE_ATTRIBUTE_MEMPOOL_SUPPORTED_HANDLE_TYPES: Bitmask of handle types supported with mempool based IPC - * - ::CU_DEVICE_ATTRIBUTE_DEFERRED_MAPPING_CUDA_ARRAY_SUPPORTED: Device supports deferred mapping CUDA arrays and CUDA mipmapped arrays. + * \p dev. * * \param pi - Returned device attribute value * \param attrib - Device attribute to query @@ -4620,6 +5497,39 @@ CUresult CUDAAPI cuDeviceGetTexture1DLinearMaxWidth(size_t *maxWidthInElements, */ CUresult CUDAAPI cuDeviceGetAttribute(int *pi, CUdevice_attribute attrib, CUdevice dev); +/** + * \brief Queries details about atomic operations supported between the device and host. + * + * Returns in \p *capabilities the details about requested atomic \p *operations over the + * the link between \p dev and the host. The allocated size of \p *operations and + * \p *capabilities must be \p count. + * + * For each ::CUatomicOperation in \p *operations, the corresponding result in \p *capabilities + * will be a bitmask indicating which of ::CUatomicOperationCapability the link supports natively. + * + * Returns ::CUDA_ERROR_INVALID_DEVICE if \p dev is not valid. + * + * Returns ::CUDA_ERROR_INVALID_VALUE if \p *capabilities or \p *operations is NULL, if \p count is 0, + * or if any of \p *operations is not valid. + * + * \param capabilities - Returned capability details of each requested operation + * \param operations - Requested operations + * \param count - Count of requested operations and size of capabilities + * \param dev - Device handle + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_DEVICE, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * + * \sa + * ::cuDeviceGetAttribute, + * ::cuDeviceGetP2PAtomicCapabilities, + * ::cudaDeviceGeHostAtomicCapabilities + */ +CUresult CUDAAPI cuDeviceGetHostAtomicCapabilities(unsigned int* capabilities, const CUatomicOperation* operations, unsigned int count, CUdevice dev); + /** * \brief Return NvSciSync attributes that this device can support. * @@ -4780,6 +5690,15 @@ CUresult CUDAAPI cuDeviceGetExecAffinitySupport(int *pi, CUexecAffinityType type * determined by comparing the numerical values between the two enums, with * smaller scopes having smaller values. * + * On platforms that support GPUDirect RDMA writes via more than one path in + * hardware (see ::CU_MEM_RANGE_FLAG_DMA_BUF_MAPPING_TYPE_PCIE), the user should + * consider those paths as belonging to separate ordering domains. Note that in + * such cases CUDA driver will report both RDMA writes ordering and RDMA write + * scope as ALL_DEVICES and a call to cuFlushGPUDirectRDMA will be a no-op, + * but when these multiple paths are used simultaneously, it is the user's + * responsibility to ensure ordering by using mechanisms outside the scope of + * CUDA. + * * Users may query support for this API via * ::CU_DEVICE_ATTRIBUTE_FLUSH_FLUSH_GPU_DIRECT_RDMA_OPTIONS. * @@ -5088,6 +6007,11 @@ CUresult CUDAAPI cuDevicePrimaryCtxRelease(CUdevice dev); * modified by calling ::cuCoredumpSetAttribute from the created context after * it becomes current. * + * - ::CU_CTX_SYNC_MEMOPS: Ensures that synchronous memory operations initiated + * on this context will always synchronize. See further documentation in the + * section titled "API Synchronization behavior" to learn more about cases when + * synchronous memory operations can exhibit asynchronous behavior. + * * \param dev - Device for which the primary context flags are set * \param flags - New flags for the device * @@ -5195,15 +6119,35 @@ CUresult CUDAAPI cuDevicePrimaryCtxReset(CUdevice dev); /** * \brief Create a CUDA context * - * \note In most cases it is recommended to use ::cuDevicePrimaryCtxRetain. - * * Creates a new CUDA context and associates it with the calling thread. The * \p flags parameter is described below. The context is created with a usage * count of 1 and the caller of ::cuCtxCreate() must call ::cuCtxDestroy() * when done using the context. If a context is already current to the thread, * it is supplanted by the newly created context and may be restored by a subsequent * call to ::cuCtxPopCurrent(). - * + * + * CUDA context can be created with execution affinity. The type and the amount of + execution resource the context can use is limited by \p paramsArray and \p numExecAffinityParams + in \p execAffinity. The \p paramsArray is an array of \p CUexecAffinityParam and the \p numExecAffinityParams + * describes the size of the paramsArray. If two \p CUexecAffinityParam in the array have the same type, + * the latter execution affinity parameter overrides the former execution affinity parameter. + * The supported execution affinity types are: + * - ::CU_EXEC_AFFINITY_TYPE_SM_COUNT limits the portion of SMs that the context can use. The portion + * of SMs is specified as the number of SMs via \p CUexecAffinitySmCount. This limit will be internally + * rounded up to the next hardware-supported amount. Hence, it is imperative to query the actual execution + * affinity of the context via \p cuCtxGetExecAffinity after context creation. Currently, this attribute + * is only supported under Volta+ MPS. + * + * CUDA context can be created in CIG(CUDA in Graphics) mode by setting \p cigParams. + * Data from graphics client is shared with CUDA via the \p sharedData in \p cigParams. + * Support for D3D12 graphics client can be determined using ::cuDeviceGetAttribute() with + * ::CU_DEVICE_ATTRIBUTE_D3D12_CIG_SUPPORTED. \p sharedData is a ID3D12CommandQueue handle. + * Support for Vulkan graphics client can be determined using ::cuDeviceGetAttribute() with + * ::CU_DEVICE_ATTRIBUTE_VULKAN_CIG_SUPPORTED. \p sharedData is a Nvidia specific data blob + * populated by calling vkGetExternalComputeQueueDataNV(). + * Either \p execAffinityParams or \p cigParams can be set to a non-null value. Setting both to a + * non-null value will result in an undefined behavior. + * * The three LSBs of the \p flags parameter can be used to control how the OS * thread, which owns the CUDA context at the time of an API call, interacts * with the OS scheduler when waiting for results from the GPU. Only one of @@ -5256,12 +6200,13 @@ CUresult CUDAAPI cuDevicePrimaryCtxReset(CUdevice dev); * are described in the CUDA-GDB user guide under the "GPU core dump support" * section. * The initial attributes will be taken from the global attributes at the time of - * context creation. The other attributes that control coredump output can be + * context creation. The other attributes that control coredump output can be * modified by calling ::cuCoredumpSetAttribute from the created context after - * it becomes current. + * it becomes current. This flag is not supported when CUDA context is created in + * CIG(CUDA in Graphics) mode. * * - ::CU_CTX_USER_COREDUMP_ENABLE: If user-triggered GPU coredumps have not - * been enabled globally with ::cuCoredumpSetAttributeGlobal or environment + * been enabled globally with ::cuCoredumpSetAttributeGlobal or environment * variables, this flag can be set during context creation to instruct CUDA to * create a coredump if data is written to a certain pipe that is present in the * OS space. These environment variables are described in the CUDA-GDB user @@ -5270,11 +6215,18 @@ CUresult CUDAAPI cuDevicePrimaryCtxReset(CUdevice dev); * ::cuCoredumpSetAttributeGlobal before creating the context if this flag is * used. Setting this flag implies that ::CU_CTX_COREDUMP_ENABLE is set. * The initial attributes will be taken from the global attributes at the time of - * context creation. The other attributes that control coredump output can be - * modified by calling ::cuCoredumpSetAttribute from the created context after + * context creation. The other attributes that control coredump output can be + * modified by calling ::cuCoredumpSetAttribute from the created context after * it becomes current. - * Setting this flag on any context creation is equivalent to setting the + * Setting this flag on any context creation is equivalent to setting the * ::CU_COREDUMP_ENABLE_USER_TRIGGER attribute to \p true globally. + * This flag is not supported when CUDA context is created in + * CIG(CUDA in Graphics) mode. + * + * - ::CU_CTX_SYNC_MEMOPS: Ensures that synchronous memory operations initiated + * on this context will always synchronize. See further documentation in the + * section titled "API Synchronization behavior" to learn more about cases when + * synchronous memory operations can exhibit asynchronous behavior. * * Context creation will fail with ::CUDA_ERROR_UNKNOWN if the compute mode of * the device is ::CU_COMPUTEMODE_PROHIBITED. The function ::cuDeviceGetAttribute() @@ -5284,9 +6236,15 @@ CUresult CUDAAPI cuDevicePrimaryCtxReset(CUdevice dev); * Documentation for nvidia-smi can be obtained by passing a * -h option to it. * - * \param pctx - Returned context handle of the new context - * \param flags - Context creation flags - * \param dev - Device to create context on + * Context creation will fail with :: CUDA_ERROR_INVALID_VALUE if invalid parameter was + * passed by client to create the CUDA context. + * + * Context creation in CIG mode will fail with ::CUDA_ERROR_NOT_SUPPORTED if CIG is not supported + * by the device or the driver. + * \param pctx - Returned context handle of the new context + * \param ctxCreateParams - Context creation parameters + * \param flags - Context creation flags + * \param dev - Device to create context on * * \return * ::CUDA_SUCCESS, @@ -5295,6 +6253,7 @@ CUresult CUDAAPI cuDevicePrimaryCtxReset(CUdevice dev); * ::CUDA_ERROR_INVALID_CONTEXT, * ::CUDA_ERROR_INVALID_DEVICE, * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_NOT_SUPPORTED, * ::CUDA_ERROR_OUT_OF_MEMORY, * ::CUDA_ERROR_UNKNOWN * \notefnerr @@ -5313,174 +6272,43 @@ CUresult CUDAAPI cuDevicePrimaryCtxReset(CUdevice dev); * ::cuCoredumpSetAttribute, * ::cuCtxSynchronize */ -CUresult CUDAAPI cuCtxCreate(CUcontext *pctx, unsigned int flags, CUdevice dev); - +CUresult CUDAAPI cuCtxCreate(CUcontext *pctx, CUctxCreateParams *ctxCreateParams, unsigned int flags, CUdevice dev); /** - * \brief Create a CUDA context with execution affinity - * - * Creates a new CUDA context with execution affinity and associates it with - * the calling thread. The \p paramsArray and \p flags parameter are described below. - * The context is created with a usage count of 1 and the caller of ::cuCtxCreate() must - * call ::cuCtxDestroy() when done using the context. If a context is already - * current to the thread, it is supplanted by the newly created context and may - * be restored by a subsequent call to ::cuCtxPopCurrent(). + * \brief Destroy a CUDA context * - * The type and the amount of execution resource the context can use is limited by \p paramsArray - * and \p numParams. The \p paramsArray is an array of \p CUexecAffinityParam and the \p numParams - * describes the size of the array. If two \p CUexecAffinityParam in the array have the same type, - * the latter execution affinity parameter overrides the former execution affinity parameter. - * The supported execution affinity types are: - * - ::CU_EXEC_AFFINITY_TYPE_SM_COUNT limits the portion of SMs that the context can use. The portion - * of SMs is specified as the number of SMs via \p CUexecAffinitySmCount. This limit will be internally - * rounded up to the next hardware-supported amount. Hence, it is imperative to query the actual execution - * affinity of the context via \p cuCtxGetExecAffinity after context creation. Currently, this attribute - * is only supported under Volta+ MPS. + * Destroys the CUDA context specified by \p ctx. The context \p ctx will be + * destroyed regardless of how many threads it is current to. + * It is the responsibility of the calling function to ensure that no API + * call issues using \p ctx while ::cuCtxDestroy() is executing. * - * The three LSBs of the \p flags parameter can be used to control how the OS - * thread, which owns the CUDA context at the time of an API call, interacts - * with the OS scheduler when waiting for results from the GPU. Only one of - * the scheduling flags can be set when creating a context. + * Destroys and cleans up all resources associated with the context. + * It is the caller's responsibility to ensure that the context or its resources + * are not accessed or passed in subsequent API calls and doing so will result in undefined behavior. + * These resources include CUDA types ::CUmodule, ::CUfunction, ::CUstream, ::CUevent, + * ::CUarray, ::CUmipmappedArray, ::CUtexObject, ::CUsurfObject, ::CUtexref, ::CUsurfref, + * ::CUgraphicsResource, ::CUlinkState, ::CUexternalMemory and ::CUexternalSemaphore. + * These resources also include memory allocations by ::cuMemAlloc(), ::cuMemAllocHost(), + * ::cuMemAllocManaged() and ::cuMemAllocPitch(). * - * - ::CU_CTX_SCHED_SPIN: Instruct CUDA to actively spin when waiting for - * results from the GPU. This can decrease latency when waiting for the GPU, - * but may lower the performance of CPU threads if they are performing work in - * parallel with the CUDA thread. + * If \p ctx is current to the calling thread then \p ctx will also be + * popped from the current thread's context stack (as though ::cuCtxPopCurrent() + * were called). If \p ctx is current to other threads, then \p ctx will + * remain current to those threads, and attempting to access \p ctx from + * those threads will result in the error ::CUDA_ERROR_CONTEXT_IS_DESTROYED. * - * - ::CU_CTX_SCHED_YIELD: Instruct CUDA to yield its thread when waiting for - * results from the GPU. This can increase latency when waiting for the GPU, - * but can increase the performance of CPU threads performing work in parallel - * with the GPU. + * \note ::cuCtxDestroy() will not destroy memory allocations by ::cuMemCreate(), ::cuMemAllocAsync() and + * ::cuMemAllocFromPoolAsync(). These memory allocations are not associated with any CUDA context and need to + * be destroyed explicitly. * - * - ::CU_CTX_SCHED_BLOCKING_SYNC: Instruct CUDA to block the CPU thread on a - * synchronization primitive when waiting for the GPU to finish work. + * \param ctx - Context to destroy * - * - ::CU_CTX_BLOCKING_SYNC: Instruct CUDA to block the CPU thread on a - * synchronization primitive when waiting for the GPU to finish work.
- * Deprecated: This flag was deprecated as of CUDA 4.0 and was - * replaced with ::CU_CTX_SCHED_BLOCKING_SYNC. - * - * - ::CU_CTX_SCHED_AUTO: The default value if the \p flags parameter is zero, - * uses a heuristic based on the number of active CUDA contexts in the - * process \e C and the number of logical processors in the system \e P. If - * \e C > \e P, then CUDA will yield to other OS threads when waiting for - * the GPU (::CU_CTX_SCHED_YIELD), otherwise CUDA will not yield while - * waiting for results and actively spin on the processor (::CU_CTX_SCHED_SPIN). - * Additionally, on Tegra devices, ::CU_CTX_SCHED_AUTO uses a heuristic based on - * the power profile of the platform and may choose ::CU_CTX_SCHED_BLOCKING_SYNC - * for low-powered devices. - * - * - ::CU_CTX_MAP_HOST: Instruct CUDA to support mapped pinned allocations. - * This flag must be set in order to allocate pinned host memory that is - * accessible to the GPU. - * - * - ::CU_CTX_LMEM_RESIZE_TO_MAX: Instruct CUDA to not reduce local memory - * after resizing local memory for a kernel. This can prevent thrashing by - * local memory allocations when launching many kernels with high local - * memory usage at the cost of potentially increased memory usage.
- * Deprecated: This flag is deprecated and the behavior enabled - * by this flag is now the default and cannot be disabled. - * Instead, the per-thread stack size can be controlled with ::cuCtxSetLimit(). - * - * - ::CU_CTX_COREDUMP_ENABLE: If GPU coredumps have not been enabled globally - * with ::cuCoredumpSetAttributeGlobal or environment variables, this flag can - * be set during context creation to instruct CUDA to create a coredump if - * this context raises an exception during execution. These environment variables - * are described in the CUDA-GDB user guide under the "GPU core dump support" - * section. - * The initial attributes will be taken from the global attributes at the time of - * context creation. The other attributes that control coredump output can be - * modified by calling ::cuCoredumpSetAttribute from the created context after - * it becomes current. - * - * - ::CU_CTX_USER_COREDUMP_ENABLE: If user-triggered GPU coredumps have not - * been enabled globally with ::cuCoredumpSetAttributeGlobal or environment - * variables, this flag can be set during context creation to instruct CUDA to - * create a coredump if data is written to a certain pipe that is present in the - * OS space. These environment variables are described in the CUDA-GDB user - * guide under the "GPU core dump support" section. - * It is important to note that the pipe name *must* be set with - * ::cuCoredumpSetAttributeGlobal before creating the context if this flag is - * used. Setting this flag implies that ::CU_CTX_COREDUMP_ENABLE is set. - * The initial attributes will be taken from the global attributes at the time of - * context creation. The other attributes that control coredump output can be - * modified by calling ::cuCoredumpSetAttribute from the created context after - * it becomes current. - * Setting this flag on any context creation is equivalent to setting the - * ::CU_COREDUMP_ENABLE_USER_TRIGGER attribute to \p true globally. - * - * Context creation will fail with ::CUDA_ERROR_UNKNOWN if the compute mode of - * the device is ::CU_COMPUTEMODE_PROHIBITED. The function ::cuDeviceGetAttribute() - * can be used with ::CU_DEVICE_ATTRIBUTE_COMPUTE_MODE to determine the - * compute mode of the device. The nvidia-smi tool can be used to set - * the compute mode for * devices. - * Documentation for nvidia-smi can be obtained by passing a - * -h option to it. - * - * \param pctx - Returned context handle of the new context - * \param paramsArray - Execution affinity parameters - * \param numParams - Number of execution affinity parameters - * \param flags - Context creation flags - * \param dev - Device to create context on - * - * \return - * ::CUDA_SUCCESS, - * ::CUDA_ERROR_DEINITIALIZED, - * ::CUDA_ERROR_NOT_INITIALIZED, - * ::CUDA_ERROR_INVALID_CONTEXT, - * ::CUDA_ERROR_INVALID_DEVICE, - * ::CUDA_ERROR_INVALID_VALUE, - * ::CUDA_ERROR_OUT_OF_MEMORY, - * ::CUDA_ERROR_UNSUPPORTED_EXEC_AFFINITY, - * ::CUDA_ERROR_UNKNOWN - * \notefnerr - * - * \sa ::cuCtxDestroy, - * ::cuCtxGetApiVersion, - * ::cuCtxGetCacheConfig, - * ::cuCtxGetDevice, - * ::cuCtxGetFlags, - * ::cuCtxGetLimit, - * ::cuCtxPopCurrent, - * ::cuCtxPushCurrent, - * ::cuCtxSetCacheConfig, - * ::cuCtxSetLimit, - * ::cuCtxSynchronize, - * ::cuCoredumpSetAttributeGlobal, - * ::cuCoredumpSetAttribute, - * ::CUexecAffinityParam - */ -CUresult CUDAAPI cuCtxCreate_v3(CUcontext *pctx, CUexecAffinityParam *paramsArray, int numParams, unsigned int flags, CUdevice dev); - -/** - * \brief Destroy a CUDA context - * - * Destroys the CUDA context specified by \p ctx. The context \p ctx will be - * destroyed regardless of how many threads it is current to. - * It is the responsibility of the calling function to ensure that no API - * call issues using \p ctx while ::cuCtxDestroy() is executing. - * - * Destroys and cleans up all resources associated with the context. - * It is the caller's responsibility to ensure that the context or its resources - * are not accessed or passed in subsequent API calls and doing so will result in undefined behavior. - * These resources include CUDA types such as ::CUmodule, ::CUfunction, ::CUstream, ::CUevent, - * ::CUarray, ::CUmipmappedArray, ::CUtexObject, ::CUsurfObject, ::CUtexref, ::CUsurfref, - * ::CUgraphicsResource, ::CUlinkState, ::CUexternalMemory and ::CUexternalSemaphore. - * - * If \p ctx is current to the calling thread then \p ctx will also be - * popped from the current thread's context stack (as though ::cuCtxPopCurrent() - * were called). If \p ctx is current to other threads, then \p ctx will - * remain current to those threads, and attempting to access \p ctx from - * those threads will result in the error ::CUDA_ERROR_CONTEXT_IS_DESTROYED. - * - * \param ctx - Context to destroy - * - * \return - * ::CUDA_SUCCESS, - * ::CUDA_ERROR_DEINITIALIZED, - * ::CUDA_ERROR_NOT_INITIALIZED, - * ::CUDA_ERROR_INVALID_CONTEXT, - * ::CUDA_ERROR_INVALID_VALUE - * \notefnerr + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr * * \sa ::cuCtxCreate, * ::cuCtxGetApiVersion, @@ -5618,11 +6446,11 @@ CUresult CUDAAPI cuCtxSetCurrent(CUcontext ctx); CUresult CUDAAPI cuCtxGetCurrent(CUcontext *pctx); /** - * \brief Returns the device ID for the current context + * \brief Returns the device handle for the current context * - * Returns in \p *device the ordinal of the current context's device. + * Returns in \p *device the handle of the current context's device. * - * \param device - Returned device ID for the current context + * \param device - Returned device handle for the current context * * \return * ::CUDA_SUCCESS, @@ -5647,6 +6475,29 @@ CUresult CUDAAPI cuCtxGetCurrent(CUcontext *pctx); */ CUresult CUDAAPI cuCtxGetDevice(CUdevice *device); +/** + * \brief Returns the device handle for the specified context + * + * Returns in \p *device the handle of the specified context's device. If the specified context is NULL, the API will + * return the current context's device. + * + * \param device - Returned device handle for the specified context + * \param ctx - Context for which to obtain the device + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * + * \sa ::cuCtxGetCurrent, + * ::cuCtxPopCurrent, + * ::cuCtxPushCurrent + */ +CUresult CUDAAPI cuCtxGetDevice_v2(CUdevice *device, CUcontext ctx); + /** * \brief Returns the flags for the current context * @@ -5679,6 +6530,9 @@ CUresult CUDAAPI cuCtxGetFlags(unsigned int *flags); /** * \brief Sets the flags for the current context * + * Sets the flags for the current context overwriting previously set ones. See + * ::cuDevicePrimaryCtxSetFlags for flag values. + * * \param flags - Flags to set on the current context * * \return @@ -5735,9 +6589,11 @@ CUresult CUDAAPI cuCtxSetFlags(unsigned int flags); CUresult CUDAAPI cuCtxGetId(CUcontext ctx, unsigned long long *ctxId); /** - * \brief Block for a context's tasks to complete + * \brief Block for the current context's tasks to complete * - * Blocks until the device has completed all preceding requested tasks. + * Blocks until the current context has completed all preceding requested tasks. + * If the current context is the primary context, green contexts that have been + * created will also be synchronized. * ::cuCtxSynchronize() returns an error if one of the preceding tasks failed. * If the context was created with the ::CU_CTX_SCHED_BLOCKING_SYNC flag, the * CPU thread will block until the GPU context has finished its work. @@ -5764,6 +6620,38 @@ CUresult CUDAAPI cuCtxGetId(CUcontext ctx, unsigned long long *ctxId); */ CUresult CUDAAPI cuCtxSynchronize(void); +/** + * \brief Block for the specified context's tasks to complete + * + * Blocks until the specified context has completed all preceding requested tasks. + * If the specified context is the primary context, green contexts that have been + * created will also be synchronized. + * The API returns an error if one of the preceding tasks failed. + * + * If the context was created with the ::CU_CTX_SCHED_BLOCKING_SYNC flag, the + * CPU thread will block until the GPU context has finished its work. + * + * If the specified context is NULL, the API will operate on the current context. + * + * \param ctx - Context to synchronize + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * + * \sa ::cuCtxGetCurrent, + * ::cuCtxPopCurrent, + * ::cuCtxPushCurrent, + * ::cuGreenCtxCreate, + * ::cuCtxFromGreenCtx, + * ::cudaDeviceSynchronize + */ +CUresult CUDAAPI cuCtxSynchronize_v2(CUcontext ctx); + /** * \brief Set resource limits * @@ -6001,101 +6889,6 @@ CUresult CUDAAPI cuCtxGetCacheConfig(CUfunc_cache *pconfig); */ CUresult CUDAAPI cuCtxSetCacheConfig(CUfunc_cache config); -/** - * \brief Returns the current shared memory configuration for the current context. - * - * This function will return in \p pConfig the current size of shared memory banks - * in the current context. On devices with configurable shared memory banks, - * ::cuCtxSetSharedMemConfig can be used to change this setting, so that all - * subsequent kernel launches will by default use the new bank size. When - * ::cuCtxGetSharedMemConfig is called on devices without configurable shared - * memory, it will return the fixed bank size of the hardware. - * - * The returned bank configurations can be either: - * - ::CU_SHARED_MEM_CONFIG_FOUR_BYTE_BANK_SIZE: shared memory bank width is - * four bytes. - * - ::CU_SHARED_MEM_CONFIG_EIGHT_BYTE_BANK_SIZE: shared memory bank width will - * eight bytes. - * - * \param pConfig - returned shared memory configuration - * \return - * ::CUDA_SUCCESS, - * ::CUDA_ERROR_DEINITIALIZED, - * ::CUDA_ERROR_NOT_INITIALIZED, - * ::CUDA_ERROR_INVALID_CONTEXT, - * ::CUDA_ERROR_INVALID_VALUE - * \notefnerr - * - * \sa ::cuCtxCreate, - * ::cuCtxDestroy, - * ::cuCtxGetApiVersion, - * ::cuCtxGetCacheConfig, - * ::cuCtxGetDevice, - * ::cuCtxGetFlags, - * ::cuCtxGetLimit, - * ::cuCtxPopCurrent, - * ::cuCtxPushCurrent, - * ::cuCtxSetLimit, - * ::cuCtxSynchronize, - * ::cuCtxGetSharedMemConfig, - * ::cuFuncSetCacheConfig, - * ::cudaDeviceGetSharedMemConfig - */ -CUresult CUDAAPI cuCtxGetSharedMemConfig(CUsharedconfig *pConfig); - -/** - * \brief Sets the shared memory configuration for the current context. - * - * On devices with configurable shared memory banks, this function will set - * the context's shared memory bank size which is used for subsequent kernel - * launches. - * - * Changed the shared memory configuration between launches may insert a device - * side synchronization point between those launches. - * - * Changing the shared memory bank size will not increase shared memory usage - * or affect occupancy of kernels, but may have major effects on performance. - * Larger bank sizes will allow for greater potential bandwidth to shared memory, - * but will change what kinds of accesses to shared memory will result in bank - * conflicts. - * - * This function will do nothing on devices with fixed shared memory bank size. - * - * The supported bank configurations are: - * - ::CU_SHARED_MEM_CONFIG_DEFAULT_BANK_SIZE: set bank width to the default initial - * setting (currently, four bytes). - * - ::CU_SHARED_MEM_CONFIG_FOUR_BYTE_BANK_SIZE: set shared memory bank width to - * be natively four bytes. - * - ::CU_SHARED_MEM_CONFIG_EIGHT_BYTE_BANK_SIZE: set shared memory bank width to - * be natively eight bytes. - * - * \param config - requested shared memory configuration - * - * \return - * ::CUDA_SUCCESS, - * ::CUDA_ERROR_DEINITIALIZED, - * ::CUDA_ERROR_NOT_INITIALIZED, - * ::CUDA_ERROR_INVALID_CONTEXT, - * ::CUDA_ERROR_INVALID_VALUE - * \notefnerr - * - * \sa ::cuCtxCreate, - * ::cuCtxDestroy, - * ::cuCtxGetApiVersion, - * ::cuCtxGetCacheConfig, - * ::cuCtxGetDevice, - * ::cuCtxGetFlags, - * ::cuCtxGetLimit, - * ::cuCtxPopCurrent, - * ::cuCtxPushCurrent, - * ::cuCtxSetLimit, - * ::cuCtxSynchronize, - * ::cuCtxGetSharedMemConfig, - * ::cuFuncSetCacheConfig, - * ::cudaDeviceSetSharedMemConfig - */ -CUresult CUDAAPI cuCtxSetSharedMemConfig(CUsharedconfig config); - /** * \brief Gets the context's API version. * @@ -6214,25 +7007,98 @@ CUresult CUDAAPI cuCtxResetPersistingL2Cache(void); */ CUresult CUDAAPI cuCtxGetExecAffinity(CUexecAffinityParam *pExecAffinity, CUexecAffinityType type); - -/** @} */ /* END CUDA_CTX */ - /** - * \defgroup CUDA_CTX_DEPRECATED Context Management [DEPRECATED] - * - * ___MANBRIEF___ deprecated context management functions of the low-level CUDA - * driver API (___CURRENT_FILE___) ___ENDMANBRIEF___ - * - * This section describes the deprecated context management functions of the low-level - * CUDA driver application programming interface. + * \brief Records an event. * - * @{ - */ - -/** - * \brief Increment a context's usage-count + * Captures in \p hEvent all the activities of the context \p hCtx + * at the time of this call. \p hEvent and \p hCtx must be from the same + * CUDA context, otherwise ::CUDA_ERROR_INVALID_HANDLE will be returned. + * Calls such as ::cuEventQuery() or ::cuCtxWaitEvent() will then examine + * or wait for completion of the work that was captured. + * Uses of \p hCtx after this call do not modify \p hEvent. + * If the context passed to \p hCtx is the primary context, \p hEvent will + * capture all the activities of the primary context and its green contexts. + * If the context passed to \p hCtx is a context converted from green context + * via ::cuCtxFromGreenCtx(), \p hEvent will capture only the activities of the green context. * - * \deprecated + * \note The API will return ::CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED if the + * specified context \p hCtx has a stream in the capture mode. In such a case, + * the call will invalidate all the conflicting captures. + * + * \param hCtx - Context to record event for + * \param hEvent - Event to record + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED + * + * \sa + * ::cuCtxWaitEvent, + * ::cuGreenCtxRecordEvent, + * ::cuGreenCtxWaitEvent, + * ::cuEventRecord + */ +CUresult CUDAAPI cuCtxRecordEvent(CUcontext hCtx, CUevent hEvent); + +/** + * \brief Make a context wait on an event + * + * Makes all future work submitted to context \p hCtx wait for all work + * captured in \p hEvent. The synchronization will be performed on the device + * and will not block the calling CPU thread. See ::cuCtxRecordEvent() + * for details on what is captured by an event. + * If the context passed to \p hCtx is the primary context, the primary context + * and its green contexts will wait for \p hEvent. + * If the context passed to \p hCtx is a context converted from green context + * via ::cuCtxFromGreenCtx(), the green context will wait for \p hEvent. + * + * \note \p hEvent may be from a different context or device than \p hCtx. + * + * \note The API will return ::CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED and + * invalidate the capture if the specified event \p hEvent is part of an ongoing + * capture sequence or if the specified context \p hCtx has a stream in the capture mode. + * + * \param hCtx - Context to wait + * \param hEvent - Event to wait on + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED + * + * \sa + * ::cuCtxRecordEvent, + * ::cuGreenCtxRecordEvent, + * ::cuGreenCtxWaitEvent, + * ::cuStreamWaitEvent + */ +CUresult CUDAAPI cuCtxWaitEvent(CUcontext hCtx, CUevent hEvent); + +/** @} */ /* END CUDA_CTX */ + +/** + * \defgroup CUDA_CTX_DEPRECATED Context Management [DEPRECATED] + * + * ___MANBRIEF___ deprecated context management functions of the low-level CUDA + * driver API (___CURRENT_FILE___) ___ENDMANBRIEF___ + * + * This section describes the deprecated context management functions of the low-level + * CUDA driver application programming interface. + * + * @{ + */ + +/** + * \brief Increment a context's usage-count + * + * \deprecated * * Note that this function is deprecated and should not be used. * @@ -6306,6 +7172,106 @@ __CUDA_DEPRECATED CUresult CUDAAPI cuCtxAttach(CUcontext *pctx, unsigned int fla */ __CUDA_DEPRECATED CUresult CUDAAPI cuCtxDetach(CUcontext ctx); + +/** + * \brief Returns the current shared memory configuration for the current context. + * + * \deprecated + * + * This function will return in \p pConfig the current size of shared memory banks + * in the current context. On devices with configurable shared memory banks, + * ::cuCtxSetSharedMemConfig can be used to change this setting, so that all + * subsequent kernel launches will by default use the new bank size. When + * ::cuCtxGetSharedMemConfig is called on devices without configurable shared + * memory, it will return the fixed bank size of the hardware. + * + * The returned bank configurations can be either: + * - ::CU_SHARED_MEM_CONFIG_FOUR_BYTE_BANK_SIZE: shared memory bank width is + * four bytes. + * - ::CU_SHARED_MEM_CONFIG_EIGHT_BYTE_BANK_SIZE: shared memory bank width will + * eight bytes. + * + * \param pConfig - returned shared memory configuration + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * + * \sa ::cuCtxCreate, + * ::cuCtxDestroy, + * ::cuCtxGetApiVersion, + * ::cuCtxGetCacheConfig, + * ::cuCtxGetDevice, + * ::cuCtxGetFlags, + * ::cuCtxGetLimit, + * ::cuCtxPopCurrent, + * ::cuCtxPushCurrent, + * ::cuCtxSetLimit, + * ::cuCtxSynchronize, + * ::cuCtxGetSharedMemConfig, + * ::cuFuncSetCacheConfig, + * ::cudaDeviceGetSharedMemConfig + */ +__CUDA_DEPRECATED CUresult CUDAAPI cuCtxGetSharedMemConfig(CUsharedconfig *pConfig); + +/** + * \brief Sets the shared memory configuration for the current context. + * + * \deprecated + * + * On devices with configurable shared memory banks, this function will set + * the context's shared memory bank size which is used for subsequent kernel + * launches. + * + * Changed the shared memory configuration between launches may insert a device + * side synchronization point between those launches. + * + * Changing the shared memory bank size will not increase shared memory usage + * or affect occupancy of kernels, but may have major effects on performance. + * Larger bank sizes will allow for greater potential bandwidth to shared memory, + * but will change what kinds of accesses to shared memory will result in bank + * conflicts. + * + * This function will do nothing on devices with fixed shared memory bank size. + * + * The supported bank configurations are: + * - ::CU_SHARED_MEM_CONFIG_DEFAULT_BANK_SIZE: set bank width to the default initial + * setting (currently, four bytes). + * - ::CU_SHARED_MEM_CONFIG_FOUR_BYTE_BANK_SIZE: set shared memory bank width to + * be natively four bytes. + * - ::CU_SHARED_MEM_CONFIG_EIGHT_BYTE_BANK_SIZE: set shared memory bank width to + * be natively eight bytes. + * + * \param config - requested shared memory configuration + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * + * \sa ::cuCtxCreate, + * ::cuCtxDestroy, + * ::cuCtxGetApiVersion, + * ::cuCtxGetCacheConfig, + * ::cuCtxGetDevice, + * ::cuCtxGetFlags, + * ::cuCtxGetLimit, + * ::cuCtxPopCurrent, + * ::cuCtxPushCurrent, + * ::cuCtxSetLimit, + * ::cuCtxSynchronize, + * ::cuCtxGetSharedMemConfig, + * ::cuFuncSetCacheConfig, + * ::cudaDeviceSetSharedMemConfig + */ +__CUDA_DEPRECATED CUresult CUDAAPI cuCtxSetSharedMemConfig(CUsharedconfig config); + /** @} */ /* END CUDA_CTX_DEPRECATED */ @@ -6366,11 +7332,9 @@ CUresult CUDAAPI cuModuleLoad(CUmodule *module, const char *fname); * \brief Load a module's data * * Takes a pointer \p image and loads the corresponding module \p module into - * the current context. The pointer may be obtained by mapping a \e cubin or - * \e PTX or \e fatbin file, passing a \e cubin or \e PTX or \e fatbin file - * as a NULL-terminated text string, or incorporating a \e cubin or \e fatbin - * object into the executable resources and using operating system calls such - * as Windows \c FindResource() to obtain the pointer. + * the current context. The \p image may be a \e cubin or \e fatbin + * as output by \b nvcc, or a NULL-terminated \e PTX, either as output by \b nvcc + * or hand-written. * * \param module - Returned module * \param image - Module data to load @@ -6404,14 +7368,9 @@ CUresult CUDAAPI cuModuleLoadData(CUmodule *module, const void *image); * \brief Load a module's data with options * * Takes a pointer \p image and loads the corresponding module \p module into - * the current context. The pointer may be obtained by mapping a \e cubin or - * \e PTX or \e fatbin file, passing a \e cubin or \e PTX or \e fatbin file - * as a NULL-terminated text string, or incorporating a \e cubin or \e fatbin - * object into the executable resources and using operating system calls such - * as Windows \c FindResource() to obtain the pointer. Options are passed as - * an array via \p options and any corresponding parameters are passed in - * \p optionValues. The number of total options is supplied via \p numOptions. - * Any outputs will be returned via \p optionValues. + * the current context. The \p image may be a \e cubin or \e fatbin + * as output by \b nvcc, or a NULL-terminated \e PTX, either as output by \b nvcc + * or hand-written. * * \param module - Returned module * \param image - Module data to load @@ -6490,7 +7449,9 @@ CUresult CUDAAPI cuModuleLoadFatBinary(CUmodule *module, const void *fatCubin); /** * \brief Unloads a module * - * Unloads a module \p hmod from the current context. + * Unloads a module \p hmod from the current context. Attempting to unload + * a module which was obtained from the Library Management API such as + * ::cuLibraryGetModule will return ::CUDA_ERROR_NOT_PERMITTED. * * \param hmod - Module to unload * @@ -6499,7 +7460,8 @@ CUresult CUDAAPI cuModuleLoadFatBinary(CUmodule *module, const void *fatCubin); * ::CUDA_ERROR_DEINITIALIZED, * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_INVALID_CONTEXT, - * ::CUDA_ERROR_INVALID_VALUE + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_NOT_PERMITTED * \notefnerr * \note_destroy_ub * @@ -6569,6 +7531,48 @@ CUresult CUDAAPI cuModuleGetLoadingMode(CUmoduleLoadingMode *mode); */ CUresult CUDAAPI cuModuleGetFunction(CUfunction *hfunc, CUmodule hmod, const char *name); +/** + * \brief Returns the number of functions within a module + * + * Returns in \p count the number of functions in \p mod. + * + * \param count - Number of functions found within the module + * \param mod - Module to query + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_INVALID_VALUE + */ +CUresult CUDAAPI cuModuleGetFunctionCount(unsigned int *count, CUmodule mod); + +/** + * \brief Returns the function handles within a module. + * + * Returns in \p functions a maximum number of \p numFunctions function handles within \p mod. When + * function loading mode is set to LAZY the function retrieved may be partially loaded. The loading + * state of a function can be queried using ::cuFunctionIsLoaded. CUDA APIs may load the function + * automatically when called with partially loaded function handle which may incur additional + * latency. Alternatively, ::cuFunctionLoad can be used to explicitly load a function. The returned + * function handles become invalid when the module is unloaded. + * + * \param functions - Buffer where the function handles are returned to + * \param numFunctions - Maximum number of function handles may be returned to the buffer + * \param mod - Module to query from + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_INVALID_VALUE + * + * \sa ::cuModuleGetFunction, + * ::cuModuleGetFunctionCount, + * ::cuFuncIsLoaded, + * ::cuFuncLoad + */ +CUresult CUDAAPI cuModuleEnumerateFunctions(CUfunction *functions, unsigned int numFunctions, CUmodule mod); + /** * \brief Returns a global pointer from a module * @@ -6617,6 +7621,11 @@ CUresult CUDAAPI cuModuleGetGlobal(CUdeviceptr *dptr, size_t *bytes, CUmodule hm * ::CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES, and ::CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES * will accumulate data until the CUlinkState is destroyed. * + * The data passed in via ::cuLinkAddData and ::cuLinkAddFile will be treated + * as relocatable (-rdc=true to nvcc) when linking the final cubin during + * ::cuLinkComplete and will have similar consequences as offline relocatable + * device code linking. + * * \p optionValues must remain valid for the life of the CUlinkState if output * options are used. No other references to inputs are maintained after this * call returns. @@ -6871,14 +7880,21 @@ __CUDA_DEPRECATED CUresult CUDAAPI cuModuleGetSurfRef(CUsurfref *pSurfRef, CUmod /** * \brief Load a library with specified code and options * - * Takes a pointer \p code and loads the corresponding library \p library into - * all contexts existent at the time of the call and future contexts at the time - * of creation until the library is unloaded with ::cuLibraryUnload(). + * Takes a pointer \p code and loads the corresponding library \p library based on + * the application defined library loading mode: + * - If module loading is set to EAGER, via the environment variables described in "Module loading", + * \p library is loaded eagerly into all contexts at the time of the call and future contexts + * at the time of creation until the library is unloaded with ::cuLibraryUnload(). + * - If the environment variables are set to LAZY, \p library + * is not immediately loaded onto all existent contexts and will only be + * loaded when a function is needed for that context, such as a kernel launch. * - * The pointer may be obtained by mapping a \e cubin or \e PTX or \e fatbin file, - * passing a \e cubin or \e PTX or \e fatbin file as a NULL-terminated text string, or - * incorporating a \e cubin or \e fatbin object into the executable resources and - * using operating system calls such as Windows \c FindResource() to obtain the pointer. + * These environment variables are described in the CUDA programming guide under the + * "CUDA environment variables" section. + * + * The \p code may be a \e cubin or \e fatbin as output by \b nvcc, + * or a NULL-terminated \e PTX, either as output by \b nvcc or hand-written. + * A fatbin should also contain relocatable code when doing separate compilation. * * Options are passed as an array via \p jitOptions and any corresponding parameters are passed in * \p jitOptionsValues. The number of total JIT options is supplied via \p numJitOptions. @@ -6887,6 +7903,9 @@ __CUDA_DEPRECATED CUresult CUDAAPI cuModuleGetSurfRef(CUsurfref *pSurfRef, CUmod * Library load options are passed as an array via \p libraryOptions and any corresponding parameters are passed in * \p libraryOptionValues. The number of total library load options is supplied via \p numLibraryOptions. * + * \note If the library contains managed variables and no device in the system + * supports managed variables this call is expected to return ::CUDA_ERROR_NOT_SUPPORTED + * * \param library - Returned library * \param code - Code to load * \param jitOptions - Options for JIT @@ -6907,7 +7926,8 @@ __CUDA_DEPRECATED CUresult CUDAAPI cuModuleGetSurfRef(CUsurfref *pSurfRef, CUmod * ::CUDA_ERROR_NO_BINARY_FOR_GPU, * ::CUDA_ERROR_SHARED_OBJECT_SYMBOL_NOT_FOUND, * ::CUDA_ERROR_SHARED_OBJECT_INIT_FAILED, - * ::CUDA_ERROR_JIT_COMPILER_NOT_FOUND + * ::CUDA_ERROR_JIT_COMPILER_NOT_FOUND, + * ::CUDA_ERROR_NOT_SUPPORTED * * \sa ::cuLibraryLoadFromFile, * ::cuLibraryUnload, @@ -6922,13 +7942,21 @@ CUresult CUDAAPI cuLibraryLoadData(CUlibrary *library, const void *code, /** * \brief Load a library with specified file and options * - * Takes a filename \p fileName and loads the corresponding library \p library into - * all contexts existent at the time of the call and future contexts at the time of - * creation until the library is unloaded with ::cuLibraryUnload(). + * Takes a pointer \p code and loads the corresponding library \p library based on + * the application defined library loading mode: + * - If module loading is set to EAGER, via the environment variables described in "Module loading", + * \p library is loaded eagerly into all contexts at the time of the call and future contexts + * at the time of creation until the library is unloaded with ::cuLibraryUnload(). + * - If the environment variables are set to LAZY, \p library + * is not immediately loaded onto all existent contexts and will only be + * loaded when a function is needed for that context, such as a kernel launch. + * + * These environment variables are described in the CUDA programming guide under the + * "CUDA environment variables" section. * * The file should be a \e cubin file as output by \b nvcc, or a \e PTX file either - * as output by \b nvcc or handwritten, or a \e fatbin file as output by \b nvcc - * from toolchain 4.0 or later. + * as output by \b nvcc or handwritten, or a \e fatbin file as output by \b nvcc. + * A fatbin should also contain relocatable code when doing separate compilation. * * Options are passed as an array via \p jitOptions and any corresponding parameters are * passed in \p jitOptionsValues. The number of total options is supplied via \p numJitOptions. @@ -6937,6 +7965,9 @@ CUresult CUDAAPI cuLibraryLoadData(CUlibrary *library, const void *code, * Library load options are passed as an array via \p libraryOptions and any corresponding parameters are passed in * \p libraryOptionValues. The number of total library load options is supplied via \p numLibraryOptions. * + * \note If the library contains managed variables and no device in the system + * supports managed variables this call is expected to return ::CUDA_ERROR_NOT_SUPPORTED + * * \param library - Returned library * \param fileName - File to load from * \param jitOptions - Options for JIT @@ -6957,7 +7988,8 @@ CUresult CUDAAPI cuLibraryLoadData(CUlibrary *library, const void *code, * ::CUDA_ERROR_NO_BINARY_FOR_GPU, * ::CUDA_ERROR_SHARED_OBJECT_SYMBOL_NOT_FOUND, * ::CUDA_ERROR_SHARED_OBJECT_INIT_FAILED, - * ::CUDA_ERROR_JIT_COMPILER_NOT_FOUND + * ::CUDA_ERROR_JIT_COMPILER_NOT_FOUND, + * ::CUDA_ERROR_NOT_SUPPORTED * * \sa ::cuLibraryLoadData, * ::cuLibraryUnload, @@ -7004,7 +8036,7 @@ CUresult CUDAAPI cuLibraryUnload(CUlibrary library); * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_INVALID_VALUE, * ::CUDA_ERROR_INVALID_HANDLE, - * ::CUDA_ERROR_NOT_FOUND, + * ::CUDA_ERROR_NOT_FOUND * * \sa ::cuLibraryLoadData, * ::cuLibraryLoadFromFile, @@ -7015,6 +8047,40 @@ CUresult CUDAAPI cuLibraryUnload(CUlibrary library); */ CUresult CUDAAPI cuLibraryGetKernel(CUkernel *pKernel, CUlibrary library, const char *name); +/** + * \brief Returns the number of kernels within a library + * + * Returns in \p count the number of kernels in \p lib. + * + * \param count - Number of kernels found within the library + * \param lib - Library to query + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_INVALID_VALUE + */ +CUresult CUDAAPI cuLibraryGetKernelCount(unsigned int *count, CUlibrary lib); + +/** + * \brief Retrieve the kernel handles within a library. + * + * Returns in \p kernels a maximum number of \p numKernels kernel handles within \p lib. + * The returned kernel handle becomes invalid when the library is unloaded. + * + * \param kernels - Buffer where the kernel handles are returned to + * \param numKernels - Maximum number of kernel handles may be returned to the buffer + * \param lib - Library to query from + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_INVALID_VALUE + * + * \sa ::cuLibraryGetKernelCount + */ +CUresult CUDAAPI cuLibraryEnumerateKernels(CUkernel *kernels, unsigned int numKernels, CUlibrary lib); + /** * \brief Returns a module handle * @@ -7069,6 +8135,29 @@ CUresult CUDAAPI cuLibraryGetModule(CUmodule *pMod, CUlibrary library); */ CUresult CUDAAPI cuKernelGetFunction(CUfunction *pFunc, CUkernel kernel); +/** + * \brief Returns a library handle + * + * Returns in \p pLib the handle of the library for the requested kernel \p kernel + * + * \param pLib - Returned library handle + * \param kernel - Kernel to retrieve library handle + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_NOT_FOUND + * + * \sa ::cuLibraryLoadData, + * ::cuLibraryLoadFromFile, + * ::cuLibraryUnload, + * ::cuLibraryGetKernel + */ +CUresult CUDAAPI cuKernelGetLibrary(CUlibrary *pLib, CUkernel kernel); + /** * \brief Returns a global device pointer * @@ -7111,9 +8200,6 @@ CUresult CUDAAPI cuLibraryGetGlobal(CUdeviceptr *dptr, size_t *bytes, CUlibrary * Note that managed memory for library \p library is shared across devices and is registered * when the library is loaded into atleast one context. * - * \note The API requires a CUDA context to be present and initialized on at least one device. - * If no context is present, the call returns ::CUDA_ERROR_NOT_FOUND. - * * \param dptr - Returned pointer to the managed memory * \param bytes - Returned memory size in bytes * \param library - Library to retrieve managed memory from @@ -7125,11 +8211,11 @@ CUresult CUDAAPI cuLibraryGetGlobal(CUdeviceptr *dptr, size_t *bytes, CUlibrary * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_INVALID_VALUE, * ::CUDA_ERROR_INVALID_HANDLE, - * ::CUDA_ERROR_NOT_FOUND, + * ::CUDA_ERROR_NOT_FOUND * * \sa ::cuLibraryLoadData, * ::cuLibraryLoadFromFile, - * ::cuLibraryUnload, + * ::cuLibraryUnload */ CUresult CUDAAPI cuLibraryGetManaged(CUdeviceptr *dptr, size_t *bytes, CUlibrary library, const char *name); @@ -7151,11 +8237,11 @@ CUresult CUDAAPI cuLibraryGetManaged(CUdeviceptr *dptr, size_t *bytes, CUlibrary * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_INVALID_VALUE, * ::CUDA_ERROR_INVALID_HANDLE, - * ::CUDA_ERROR_NOT_FOUND, + * ::CUDA_ERROR_NOT_FOUND * * \sa ::cuLibraryLoadData, * ::cuLibraryLoadFromFile, - * ::cuLibraryUnload, + * ::cuLibraryUnload */ CUresult CUDAAPI cuLibraryGetUnifiedFunction(void **fptr, CUlibrary library, const char *symbol); @@ -7290,6 +8376,9 @@ CUresult CUDAAPI cuKernelGetAttribute(int *pi, CUfunction_attribute attrib, CUke * positive. The validity of the cluster dimensions is checked at launch time. * If the value is set during compile time, it cannot be set at runtime. * Setting it at runtime will return CUDA_ERROR_NOT_PERMITTED. + * - ::CU_FUNC_ATTRIBUTE_NON_PORTABLE_CLUSTER_SIZE_ALLOWED: Indicates whether + * the function can be launched with non-portable cluster size. 1 is allowed, + * 0 is disallowed. * - ::CU_FUNC_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE: The block * scheduling policy of a function. The value type is CUclusterSchedulingPolicy. * @@ -7384,6 +8473,50 @@ CUresult CUDAAPI cuKernelSetAttribute(CUfunction_attribute attrib, int val, CUke */ CUresult CUDAAPI cuKernelSetCacheConfig(CUkernel kernel, CUfunc_cache config, CUdevice dev); +/** + * \brief Returns the function name for a ::CUkernel handle + * + * Returns in \p **name the function name associated with the kernel handle \p hfunc . + * The function name is returned as a null-terminated string. The returned name is only + * valid when the kernel handle is valid. If the library is unloaded or reloaded, one + * must call the API again to get the updated name. This API may return a mangled name if + * the function is not declared as having C linkage. If either \p **name or \p hfunc + * is NULL, ::CUDA_ERROR_INVALID_VALUE is returned. + * + * \param name - The returned name of the function + * \param hfunc - The function handle to retrieve the name for + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * + */ +CUresult CUDAAPI cuKernelGetName(const char **name, CUkernel hfunc); + +/** + * \brief Returns the offset and size of a kernel parameter in the device-side parameter layout + * + * Queries the kernel parameter at \p paramIndex into \p kernel's list of parameters, and returns + * in \p paramOffset and \p paramSize the offset and size, respectively, where the parameter + * will reside in the device-side parameter layout. This information can be used to update kernel + * node parameters from the device via ::cudaGraphKernelNodeSetParam() and + * ::cudaGraphKernelNodeUpdatesApply(). \p paramIndex must be less than the number of parameters + * that \p kernel takes. \p paramSize can be set to NULL if only the parameter offset is desired. + * + * \param kernel - The kernel to query + * \param paramIndex - The parameter index to query + * \param paramOffset - Returns the offset into the device-side parameter layout at which the parameter resides + * \param paramSize - Optionally returns the size of the parameter in the device-side parameter layout + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE, + * \notefnerr + * +* \sa ::cuFuncGetParamInfo + */ +CUresult CUDAAPI cuKernelGetParamInfo(CUkernel kernel, size_t paramIndex, size_t *paramOffset, size_t *paramSize); /** @} */ /* END CUDA_LIBRARY */ /** @@ -7545,9 +8678,10 @@ CUresult CUDAAPI cuMemAllocPitch(CUdeviceptr *dptr, size_t *pPitch, size_t Width * ::cuMemAllocPitch(), ::cuMemAllocManaged(), ::cuMemAllocAsync(), ::cuMemAllocFromPoolAsync() * * Note - This API will not perform any implict synchronization when the pointer was allocated with - * ::cuMemAllocAsync or ::cuMemAllocFromPoolAsync. Callers must ensure that all accesses to the + * ::cuMemAllocAsync or ::cuMemAllocFromPoolAsync. Callers must ensure that all accesses to these * pointer have completed before invoking ::cuMemFree. For best performance and memory reuse, users * should use ::cuMemFreeAsync to free memory allocated via the stream ordered memory allocator. + * For all other pointers, this API may perform implicit synchronization. * * \param dptr - Pointer to memory to free * @@ -7615,20 +8749,24 @@ CUresult CUDAAPI cuMemGetAddressRange(CUdeviceptr *pbase, size_t *psize, CUdevic * allocated with this function and automatically accelerates calls to * functions such as ::cuMemcpy(). Since the memory can be accessed directly by * the device, it can be read or written with much higher bandwidth than - * pageable memory obtained with functions such as ::malloc(). Allocating - * excessive amounts of memory with ::cuMemAllocHost() may degrade system + * pageable memory obtained with functions such as ::malloc(). + * + * On systems where ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS_USES_HOST_PAGE_TABLES + * is true, ::cuMemAllocHost may not page-lock the allocated memory. + * + * Page-locking excessive amounts of memory with ::cuMemAllocHost() may degrade system * performance, since it reduces the amount of memory available to the system * for paging. As a result, this function is best used sparingly to allocate * staging areas for data exchange between host and device. * - * Note all host memory allocated using ::cuMemHostAlloc() will automatically + * Note all host memory allocated using ::cuMemAllocHost() will automatically * be immediately accessible to all contexts on all devices which support unified * addressing (as may be queried using ::CU_DEVICE_ATTRIBUTE_UNIFIED_ADDRESSING). * The device pointer that may be used to access this host memory from those * contexts is always equal to the returned host pointer \p *pp. * See \ref CUDA_UNIFIED for additional details. * - * \param pp - Returned host pointer to page-locked memory + * \param pp - Returned pointer to host memory * \param bytesize - Requested allocation size in bytes * * \return @@ -7692,11 +8830,15 @@ CUresult CUDAAPI cuMemFreeHost(void *p); * this function and automatically accelerates calls to functions such as * ::cuMemcpyHtoD(). Since the memory can be accessed directly by the device, * it can be read or written with much higher bandwidth than pageable memory - * obtained with functions such as ::malloc(). Allocating excessive amounts of - * pinned memory may degrade system performance, since it reduces the amount - * of memory available to the system for paging. As a result, this function is - * best used sparingly to allocate staging areas for data exchange between - * host and device. + * obtained with functions such as ::malloc(). + * + * On systems where ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS_USES_HOST_PAGE_TABLES + * is true, ::cuMemHostAlloc may not page-lock the allocated memory. + * + * Page-locking excessive amounts of memory may degrade system performance, + * since it reduces the amount of memory available to the system for paging. + * As a result, this function is best used sparingly to allocate staging areas + * for data exchange between host and device. * * The \p Flags parameter enables different options to be specified that * affect the allocation, as follows. @@ -7736,7 +8878,7 @@ CUresult CUDAAPI cuMemFreeHost(void *p); * to query the device pointer, even if the context supports unified addressing. * See \ref CUDA_UNIFIED for additional details. * - * \param pp - Returned host pointer to page-locked memory + * \param pp - Returned pointer to host memory * \param bytesize - Requested allocation size in bytes * \param Flags - Flags for allocation request * @@ -7878,7 +9020,7 @@ CUresult CUDAAPI cuMemHostGetFlags(unsigned int *pFlags, void *p); * such GPUs may be evicted from device memory to host memory at any time by the Unified * Memory driver in order to make room for other allocations. * - * In a multi-GPU system where all GPUs have a non-zero value for the device attribute + * In a system where all GPUs have a non-zero value for the device attribute * ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS, managed memory may not be populated when this * API returns and instead may be populated on access. In such systems, managed memory can * migrate to any processor's memory at any time. The Unified Memory driver will employ heuristics to @@ -7953,6 +9095,66 @@ CUresult CUDAAPI cuMemHostGetFlags(unsigned int *pFlags, void *p); */ CUresult CUDAAPI cuMemAllocManaged(CUdeviceptr *dptr, size_t bytesize, unsigned int flags); +/** +* \brief Registers a callback function to receive async notifications +* +* Registers \p callbackFunc to receive async notifications. +* +* The \p userData parameter is passed to the callback function at async notification time. +* Likewise, \p callback is also passed to the callback function to distinguish between +* multiple registered callbacks. +* +* The callback function being registered should be designed to return quickly (~10ms). +* Any long running tasks should be queued for execution on an application thread. +* +* Callbacks may not call cuDeviceRegisterAsyncNotification or cuDeviceUnregisterAsyncNotification. +* Doing so will result in ::CUDA_ERROR_NOT_PERMITTED. Async notification callbacks execute +* in an undefined order and may be serialized. +* +* Returns in \p *callback a handle representing the registered callback instance. +* +* \param device - The device on which to register the callback +* \param callbackFunc - The function to register as a callback +* \param userData - A generic pointer to user data. This is passed into the callback function. +* \param callback - A handle representing the registered callback instance +* +* \return +* ::CUDA_SUCCESS, +* ::CUDA_ERROR_NOT_SUPPORTED, +* ::CUDA_ERROR_INVALID_DEVICE, +* ::CUDA_ERROR_INVALID_VALUE, +* ::CUDA_ERROR_NOT_PERMITTED, +* ::CUDA_ERROR_UNKNOWN +* \notefnerr +* +* \sa +* ::cuDeviceUnregisterAsyncNotification +*/ +CUresult CUDAAPI cuDeviceRegisterAsyncNotification(CUdevice device, CUasyncCallback callbackFunc, void *userData, CUasyncCallbackHandle *callback); + +/** +* \brief Unregisters an async notification callback +* +* Unregisters \p callback so that the corresponding callback function will stop receiving +* async notifications. +* +* \param device - The device from which to remove \p callback. +* \param callback - The callback instance to unregister from receiving async notifications. +* +* \return +* ::CUDA_SUCCESS, +* ::CUDA_ERROR_NOT_SUPPORTED, +* ::CUDA_ERROR_INVALID_DEVICE, +* ::CUDA_ERROR_INVALID_VALUE, +* ::CUDA_ERROR_NOT_PERMITTED, +* ::CUDA_ERROR_UNKNOWN +* \notefnerr +* +* \sa +* ::cuDeviceRegisterAsyncNotification +*/ +CUresult CUDAAPI cuDeviceUnregisterAsyncNotification(CUdevice device, CUasyncCallbackHandle callback); + /** * \brief Returns a handle to a compute device * @@ -8031,9 +9233,10 @@ CUresult CUDAAPI cuDeviceGetPCIBusId(char *pciBusId, int len, CUdevice dev); * * IPC functionality is restricted to devices with support for unified * addressing on Linux and Windows operating systems. - * IPC functionality on Windows is restricted to GPUs in TCC mode + * IPC functionality on Windows is supported for compatibility purposes + * but not recommended as it comes with performance cost. * Users can test their device for IPC functionality by calling - * ::cuapiDeviceGetAttribute with ::CU_DEVICE_ATTRIBUTE_IPC_EVENT_SUPPORTED + * ::cuDeviceGetAttribute with ::CU_DEVICE_ATTRIBUTE_IPC_EVENT_SUPPORTED * * \param pHandle - Pointer to a user allocated CUipcEventHandle * in which to return the opaque event handle @@ -8074,7 +9277,8 @@ CUresult CUDAAPI cuIpcGetEventHandle(CUipcEventHandle *pHandle, CUevent event); * * IPC functionality is restricted to devices with support for unified * addressing on Linux and Windows operating systems. - * IPC functionality on Windows is restricted to GPUs in TCC mode + * IPC functionality on Windows is supported for compatibility purposes + * but not recommended as it comes with performance cost. * Users can test their device for IPC functionality by calling * ::cuapiDeviceGetAttribute with ::CU_DEVICE_ATTRIBUTE_IPC_EVENT_SUPPORTED * @@ -8119,7 +9323,8 @@ CUresult CUDAAPI cuIpcOpenEventHandle(CUevent *phEvent, CUipcEventHandle handle) * * IPC functionality is restricted to devices with support for unified * addressing on Linux and Windows operating systems. - * IPC functionality on Windows is restricted to GPUs in TCC mode + * IPC functionality on Windows is supported for compatibility purposes + * but not recommended as it comes with performance cost. * Users can test their device for IPC functionality by calling * ::cuapiDeviceGetAttribute with ::CU_DEVICE_ATTRIBUTE_IPC_EVENT_SUPPORTED * @@ -8174,7 +9379,8 @@ CUresult CUDAAPI cuIpcGetMemHandle(CUipcMemHandle *pHandle, CUdeviceptr dptr); * * IPC functionality is restricted to devices with support for unified * addressing on Linux and Windows operating systems. - * IPC functionality on Windows is restricted to GPUs in TCC mode + * IPC functionality on Windows is supported for compatibility purposes + * but not recommended as it comes with performance cost. * Users can test their device for IPC functionality by calling * ::cuapiDeviceGetAttribute with ::CU_DEVICE_ATTRIBUTE_IPC_EVENT_SUPPORTED * @@ -8219,7 +9425,8 @@ CUresult CUDAAPI cuIpcOpenMemHandle(CUdeviceptr *pdptr, CUipcMemHandle handle, u * * IPC functionality is restricted to devices with support for unified * addressing on Linux and Windows operating systems. - * IPC functionality on Windows is restricted to GPUs in TCC mode + * IPC functionality on Windows is supported for compatibility purposes + * but not recommended as it comes with performance cost. * Users can test their device for IPC functionality by calling * ::cuapiDeviceGetAttribute with ::CU_DEVICE_ATTRIBUTE_IPC_EVENT_SUPPORTED * @@ -9898,6 +11105,143 @@ CUresult CUDAAPI cuMemcpy3DAsync(const CUDA_MEMCPY3D *pCopy, CUstream hStream); */ CUresult CUDAAPI cuMemcpy3DPeerAsync(const CUDA_MEMCPY3D_PEER *pCopy, CUstream hStream); +/** + * \brief Performs a batch of memory copies asynchronously. + * + * Performs a batch of memory copies. The batch as a whole executes in stream order but copies within a + * batch are not guaranteed to execute in any specific order. This API only supports pointer-to-pointer copies. + * For copies involving CUDA arrays, please see ::cuMemcpy3DBatchAsync. + * + * Performs memory copies from source buffers specified in \p srcs to destination buffers specified in \p dsts. + * The size of each copy is specified in \p sizes. All three arrays must be of the same length as specified + * by \p count. Since there are no ordering guarantees for copies within a batch, specifying any dependent copies + * within a batch will result in undefined behavior. + * + * Every copy in the batch has to be associated with a set of attributes specified in the \p attrs array. + * Each entry in this array can apply to more than one copy. This can be done by specifying in the \p attrsIdxs array, + * the index of the first copy that the corresponding entry in the \p attrs array applies to. Both \p attrs and + * \p attrsIdxs must be of the same length as specified by \p numAttrs. For example, if a batch has 10 copies listed + * in dst/src/sizes, the first 6 of which have one set of attributes and the remaining 4 another, then \p numAttrs + * will be 2, \p attrsIdxs will be {0, 6} and \p attrs will contains the two sets of attributes. Note that the first entry + * in \p attrsIdxs must always be 0. Also, each entry must be greater than the previous entry and the last entry should be + * less than \p count. Furthermore, \p numAttrs must be lesser than or equal to \p count. + * + * The ::CUmemcpyAttributes::srcAccessOrder indicates the source access ordering to be observed for copies associated + * with the attribute. If the source access order is set to ::CU_MEMCPY_SRC_ACCESS_ORDER_STREAM, then the source will + * be accessed in stream order. If the source access order is set to ::CU_MEMCPY_SRC_ACCESS_ORDER_DURING_API_CALL then + * it indicates that access to the source pointer can be out of stream order and all accesses must be complete before + * the API call returns. This flag is suited for ephemeral sources (ex., stack variables) when it's known that no prior + * operations in the stream can be accessing the memory and also that the lifetime of the memory is limited to the scope + * that the source variable was declared in. Specifying this flag allows the driver to optimize the copy and removes the + * need for the user to synchronize the stream after the API call. If the source access order is set to + * ::CU_MEMCPY_SRC_ACCESS_ORDER_ANY then it indicates that access to the source pointer can be out of stream order and the + * accesses can happen even after the API call returns. This flag is suited for host pointers allocated + * outside CUDA (ex., via malloc) when it's known that no prior operations in the stream can be accessing the memory. + * Specifying this flag allows the driver to optimize the copy on certain platforms. Each memcpy operation in the batch must + * have a valid ::CUmemcpyAttributes corresponding to it including the appropriate srcAccessOrder setting, otherwise the API + * will return ::CUDA_ERROR_INVALID_VALUE. + * + * The ::CUmemcpyAttributes::srcLocHint and ::CUmemcpyAttributes::dstLocHint allows applications to specify hint locations + * for operands of a copy when the operand doesn't have a fixed location. That is, these hints are + * only applicable for managed memory pointers on devices where ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS is true or + * system-allocated pageable memory on devices where ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS is true. + * For other cases, these hints are ignored. + * + * The ::CUmemcpyAttributes::flags field can be used to specify certain flags for copies. Setting the + * ::CU_MEMCPY_FLAG_PREFER_OVERLAP_WITH_COMPUTE flag indicates that the associated copies should preferably overlap with + * any compute work. Note that this flag is a hint and can be ignored depending on the platform and other parameters of the copy. + * + * + * \param dsts - Array of destination pointers. + * \param srcs - Array of memcpy source pointers. + * \param sizes - Array of sizes for memcpy operations. + * \param count - Size of \p dsts, \p srcs and \p sizes arrays + * \param attrs - Array of memcpy attributes. + * \param attrsIdxs - Array of indices to specify which copies each entry in the \p attrs array applies to. + The attributes specified in attrs[k] will be applied to copies starting from attrsIdxs[k] + through attrsIdxs[k+1] - 1. Also attrs[numAttrs-1] will apply to copies starting from + attrsIdxs[numAttrs-1] through count - 1. + * \param numAttrs - Size of \p attrs and \p attrsIdxs arrays. + * \param hStream - The stream to enqueue the operations in. Must not be legacy NULL stream. + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * \note_async + * \note_memcpy + */ +CUresult CUDAAPI cuMemcpyBatchAsync(CUdeviceptr *dsts, CUdeviceptr *srcs, size_t *sizes, size_t count, + CUmemcpyAttributes *attrs, size_t *attrsIdxs, size_t numAttrs, CUstream hStream); + +/** + * \brief Performs a batch of 3D memory copies asynchronously. + * + * Performs a batch of memory copies. The batch as a whole executes in stream order but copies within a + * batch are not guaranteed to execute in any specific order. Note that this means specifying any dependent + * copies within a batch will result in undefined behavior. + * + * Performs memory copies as specified in the \p opList array. The length of this array is specified in \p numOps. + * Each entry in this array describes a copy operation. This includes among other things, the source and destination + * operands for the copy as specified in ::CUDA_MEMCPY3D_BATCH_OP::src and ::CUDA_MEMCPY3D_BATCH_OP::dst respectively. + * The source and destination operands of a copy can either be a pointer or a CUDA array. The width, height and depth + * of a copy is specified in ::CUDA_MEMCPY3D_BATCH_OP::extent. The width, height and depth of a copy are specified in + * elements and must not be zero. For pointer-to-pointer copies, the element size is considered to be 1. For pointer + * to CUDA array or vice versa copies, the element size is determined by the CUDA array. For CUDA array to CUDA array copies, + * the element size of the two CUDA arrays must match. + * + * For a given operand, if ::CUmemcpy3DOperand::type is specified as ::CU_MEMCPY_OPERAND_TYPE_POINTER, then + * ::CUmemcpy3DOperand::op::ptr will be used. The ::CUmemcpy3DOperand::op::ptr::ptr field must contain the pointer where + * the copy should begin. The ::CUmemcpy3DOperand::op::ptr::rowLength field specifies the length of each row in elements and + * must either be zero or be greater than or equal to the width of the copy specified in ::CUDA_MEMCPY3D_BATCH_OP::extent::width. + * The ::CUmemcpy3DOperand::op::ptr::layerHeight field specifies the height of each layer and must either be zero or be greater than + * or equal to the height of the copy specified in ::CUDA_MEMCPY3D_BATCH_OP::extent::height. When either of these values is zero, + * that aspect of the operand is considered to be tightly packed according to the copy extent. For managed memory pointers on devices where + * ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS is true or system-allocated pageable memory on devices where + * ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS is true, the ::CUmemcpy3DOperand::op::ptr::locHint field can be used to hint + * the location of the operand. + * + * If an operand's type is specified as ::CU_MEMCPY_OPERAND_TYPE_ARRAY, then ::CUmemcpy3DOperand::op::array will be used. + * The ::CUmemcpy3DOperand::op::array::array field specifies the CUDA array and ::CUmemcpy3DOperand::op::array::offset specifies + * the 3D offset into that array where the copy begins. + * + * The ::CUmemcpyAttributes::srcAccessOrder indicates the source access ordering to be observed for copies associated + * with the attribute. If the source access order is set to ::CU_MEMCPY_SRC_ACCESS_ORDER_STREAM, then the source will + * be accessed in stream order. If the source access order is set to ::CU_MEMCPY_SRC_ACCESS_ORDER_DURING_API_CALL then + * it indicates that access to the source pointer can be out of stream order and all accesses must be complete before + * the API call returns. This flag is suited for ephemeral sources (ex., stack variables) when it's known that no prior + * operations in the stream can be accessing the memory and also that the lifetime of the memory is limited to the scope + * that the source variable was declared in. Specifying this flag allows the driver to optimize the copy and removes the + * need for the user to synchronize the stream after the API call. If the source access order is set to + * ::CU_MEMCPY_SRC_ACCESS_ORDER_ANY then it indicates that access to the source pointer can be out of stream order and the + * accesses can happen even after the API call returns. This flag is suited for host pointers allocated + * outside CUDA (ex., via malloc) when it's known that no prior operations in the stream can be accessing the memory. + * Specifying this flag allows the driver to optimize the copy on certain platforms. Each memcopy operation in \p opList must + * have a valid srcAccessOrder setting, otherwise this API will return ::CUDA_ERROR_INVALID_VALUE. + * + * The ::CUmemcpyAttributes::flags field can be used to specify certain flags for copies. Setting the + * ::CU_MEMCPY_FLAG_PREFER_OVERLAP_WITH_COMPUTE flag indicates that the associated copies should preferably overlap with + * any compute work. Note that this flag is a hint and can be ignored depending on the platform and other parameters of the copy. + * + * + * \param numOps - Total number of memcpy operations. + * \param opList - Array of size \p numOps containing the actual memcpy operations. + * \param flags - Flags for future use, must be zero now. + * \param hStream - The stream to enqueue the operations in. Must not be default NULL stream. + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * \note_async + * \note_memcpy + */ +CUresult CUDAAPI cuMemcpy3DBatchAsync(size_t numOps, CUDA_MEMCPY3D_BATCH_OP *opList, unsigned long long flags, CUstream hStream); + /** * \brief Initializes device memory * @@ -10394,8 +11738,51 @@ CUresult CUDAAPI cuMemsetD2D32Async(CUdeviceptr dstDevice, size_t dstPitch, unsi CU_AD_FORMAT_SIGNED_INT16 = 0x09, CU_AD_FORMAT_SIGNED_INT32 = 0x0a, CU_AD_FORMAT_HALF = 0x10, - CU_AD_FORMAT_FLOAT = 0x20 - } CUarray_format; + CU_AD_FORMAT_FLOAT = 0x20, + CU_AD_FORMAT_NV12 = 0xb0, + CU_AD_FORMAT_UNORM_INT8X1 = 0xc0, + CU_AD_FORMAT_UNORM_INT8X2 = 0xc1, + CU_AD_FORMAT_UNORM_INT8X4 = 0xc2, + CU_AD_FORMAT_UNORM_INT16X1 = 0xc3, + CU_AD_FORMAT_UNORM_INT16X2 = 0xc4, + CU_AD_FORMAT_UNORM_INT16X4 = 0xc5, + CU_AD_FORMAT_SNORM_INT8X1 = 0xc6, + CU_AD_FORMAT_SNORM_INT8X2 = 0xc7, + CU_AD_FORMAT_SNORM_INT8X4 = 0xc8, + CU_AD_FORMAT_SNORM_INT16X1 = 0xc9, + CU_AD_FORMAT_SNORM_INT16X2 = 0xca, + CU_AD_FORMAT_SNORM_INT16X4 = 0xcb, + CU_AD_FORMAT_BC1_UNORM = 0x91, + CU_AD_FORMAT_BC1_UNORM_SRGB = 0x92, + CU_AD_FORMAT_BC2_UNORM = 0x93, + CU_AD_FORMAT_BC2_UNORM_SRGB = 0x94, + CU_AD_FORMAT_BC3_UNORM = 0x95, + CU_AD_FORMAT_BC3_UNORM_SRGB = 0x96, + CU_AD_FORMAT_BC4_UNORM = 0x97, + CU_AD_FORMAT_BC4_SNORM = 0x98, + CU_AD_FORMAT_BC5_UNORM = 0x99, + CU_AD_FORMAT_BC5_SNORM = 0x9a, + CU_AD_FORMAT_BC6H_UF16 = 0x9b, + CU_AD_FORMAT_BC6H_SF16 = 0x9c, + CU_AD_FORMAT_BC7_UNORM = 0x9d, + CU_AD_FORMAT_BC7_UNORM_SRGB = 0x9e, + CU_AD_FORMAT_P010 = 0x9f, + CU_AD_FORMAT_P016 = 0xa1, + CU_AD_FORMAT_NV16 = 0xa2, + CU_AD_FORMAT_P210 = 0xa3, + CU_AD_FORMAT_P216 = 0xa4, + CU_AD_FORMAT_YUY2 = 0xa5, + CU_AD_FORMAT_Y210 = 0xa6, + CU_AD_FORMAT_Y216 = 0xa7, + CU_AD_FORMAT_AYUV = 0xa8, + CU_AD_FORMAT_Y410 = 0xa9, + CU_AD_FORMAT_Y416 = 0xb1, + CU_AD_FORMAT_Y444_PLANAR8 = 0xb2, + CU_AD_FORMAT_Y444_PLANAR10 = 0xb3, + CU_AD_FORMAT_YUV444_8bit_SemiPlanar = 0xb4, + CU_AD_FORMAT_YUV444_16bit_SemiPlanar = 0xb5, + CU_AD_FORMAT_UNORM_INT_101010_2 = 0x50, + } CUarray_format; * \endcode * - \p NumChannels specifies the number of packed components per CUDA array * element; it may be 1, 2, or 4; @@ -10714,7 +12101,50 @@ CUresult CUDAAPI cuArrayDestroy(CUarray hArray); CU_AD_FORMAT_SIGNED_INT16 = 0x09, CU_AD_FORMAT_SIGNED_INT32 = 0x0a, CU_AD_FORMAT_HALF = 0x10, - CU_AD_FORMAT_FLOAT = 0x20 + CU_AD_FORMAT_FLOAT = 0x20, + CU_AD_FORMAT_NV12 = 0xb0, + CU_AD_FORMAT_UNORM_INT8X1 = 0xc0, + CU_AD_FORMAT_UNORM_INT8X2 = 0xc1, + CU_AD_FORMAT_UNORM_INT8X4 = 0xc2, + CU_AD_FORMAT_UNORM_INT16X1 = 0xc3, + CU_AD_FORMAT_UNORM_INT16X2 = 0xc4, + CU_AD_FORMAT_UNORM_INT16X4 = 0xc5, + CU_AD_FORMAT_SNORM_INT8X1 = 0xc6, + CU_AD_FORMAT_SNORM_INT8X2 = 0xc7, + CU_AD_FORMAT_SNORM_INT8X4 = 0xc8, + CU_AD_FORMAT_SNORM_INT16X1 = 0xc9, + CU_AD_FORMAT_SNORM_INT16X2 = 0xca, + CU_AD_FORMAT_SNORM_INT16X4 = 0xcb, + CU_AD_FORMAT_BC1_UNORM = 0x91, + CU_AD_FORMAT_BC1_UNORM_SRGB = 0x92, + CU_AD_FORMAT_BC2_UNORM = 0x93, + CU_AD_FORMAT_BC2_UNORM_SRGB = 0x94, + CU_AD_FORMAT_BC3_UNORM = 0x95, + CU_AD_FORMAT_BC3_UNORM_SRGB = 0x96, + CU_AD_FORMAT_BC4_UNORM = 0x97, + CU_AD_FORMAT_BC4_SNORM = 0x98, + CU_AD_FORMAT_BC5_UNORM = 0x99, + CU_AD_FORMAT_BC5_SNORM = 0x9a, + CU_AD_FORMAT_BC6H_UF16 = 0x9b, + CU_AD_FORMAT_BC6H_SF16 = 0x9c, + CU_AD_FORMAT_BC7_UNORM = 0x9d, + CU_AD_FORMAT_BC7_UNORM_SRGB = 0x9e, + CU_AD_FORMAT_P010 = 0x9f, + CU_AD_FORMAT_P016 = 0xa1, + CU_AD_FORMAT_NV16 = 0xa2, + CU_AD_FORMAT_P210 = 0xa3, + CU_AD_FORMAT_P216 = 0xa4, + CU_AD_FORMAT_YUY2 = 0xa5, + CU_AD_FORMAT_Y210 = 0xa6, + CU_AD_FORMAT_Y216 = 0xa7, + CU_AD_FORMAT_AYUV = 0xa8, + CU_AD_FORMAT_Y410 = 0xa9, + CU_AD_FORMAT_Y416 = 0xb1, + CU_AD_FORMAT_Y444_PLANAR8 = 0xb2, + CU_AD_FORMAT_Y444_PLANAR10 = 0xb3, + CU_AD_FORMAT_YUV444_8bit_SemiPlanar = 0xb4, + CU_AD_FORMAT_YUV444_16bit_SemiPlanar = 0xb5, + CU_AD_FORMAT_UNORM_INT_101010_2 = 0x50, } CUarray_format; * \endcode * @@ -10935,7 +12365,50 @@ CUresult CUDAAPI cuArray3DGetDescriptor(CUDA_ARRAY3D_DESCRIPTOR *pArrayDescripto CU_AD_FORMAT_SIGNED_INT16 = 0x09, CU_AD_FORMAT_SIGNED_INT32 = 0x0a, CU_AD_FORMAT_HALF = 0x10, - CU_AD_FORMAT_FLOAT = 0x20 + CU_AD_FORMAT_FLOAT = 0x20, + CU_AD_FORMAT_NV12 = 0xb0, + CU_AD_FORMAT_UNORM_INT8X1 = 0xc0, + CU_AD_FORMAT_UNORM_INT8X2 = 0xc1, + CU_AD_FORMAT_UNORM_INT8X4 = 0xc2, + CU_AD_FORMAT_UNORM_INT16X1 = 0xc3, + CU_AD_FORMAT_UNORM_INT16X2 = 0xc4, + CU_AD_FORMAT_UNORM_INT16X4 = 0xc5, + CU_AD_FORMAT_SNORM_INT8X1 = 0xc6, + CU_AD_FORMAT_SNORM_INT8X2 = 0xc7, + CU_AD_FORMAT_SNORM_INT8X4 = 0xc8, + CU_AD_FORMAT_SNORM_INT16X1 = 0xc9, + CU_AD_FORMAT_SNORM_INT16X2 = 0xca, + CU_AD_FORMAT_SNORM_INT16X4 = 0xcb, + CU_AD_FORMAT_BC1_UNORM = 0x91, + CU_AD_FORMAT_BC1_UNORM_SRGB = 0x92, + CU_AD_FORMAT_BC2_UNORM = 0x93, + CU_AD_FORMAT_BC2_UNORM_SRGB = 0x94, + CU_AD_FORMAT_BC3_UNORM = 0x95, + CU_AD_FORMAT_BC3_UNORM_SRGB = 0x96, + CU_AD_FORMAT_BC4_UNORM = 0x97, + CU_AD_FORMAT_BC4_SNORM = 0x98, + CU_AD_FORMAT_BC5_UNORM = 0x99, + CU_AD_FORMAT_BC5_SNORM = 0x9a, + CU_AD_FORMAT_BC6H_UF16 = 0x9b, + CU_AD_FORMAT_BC6H_SF16 = 0x9c, + CU_AD_FORMAT_BC7_UNORM = 0x9d, + CU_AD_FORMAT_BC7_UNORM_SRGB = 0x9e, + CU_AD_FORMAT_P010 = 0x9f, + CU_AD_FORMAT_P016 = 0xa1, + CU_AD_FORMAT_NV16 = 0xa2, + CU_AD_FORMAT_P210 = 0xa3, + CU_AD_FORMAT_P216 = 0xa4, + CU_AD_FORMAT_YUY2 = 0xa5, + CU_AD_FORMAT_Y210 = 0xa6, + CU_AD_FORMAT_Y216 = 0xa7, + CU_AD_FORMAT_AYUV = 0xa8, + CU_AD_FORMAT_Y410 = 0xa9, + CU_AD_FORMAT_Y416 = 0xb1, + CU_AD_FORMAT_Y444_PLANAR8 = 0xb2, + CU_AD_FORMAT_Y444_PLANAR10 = 0xb3, + CU_AD_FORMAT_YUV444_8bit_SemiPlanar = 0xb4, + CU_AD_FORMAT_YUV444_16bit_SemiPlanar = 0xb5, + CU_AD_FORMAT_UNORM_INT_101010_2 = 0x50, } CUarray_format; * \endcode * @@ -11076,39 +12549,160 @@ CUresult CUDAAPI cuMipmappedArrayGetLevel(CUarray *pLevelArray, CUmipmappedArray */ CUresult CUDAAPI cuMipmappedArrayDestroy(CUmipmappedArray hMipmappedArray); -/** -* \brief Retrieve handle for an address range -* -* Get a handle of the specified type to an address range. The address range -* must have been obtained by a prior call to either ::cuMemAlloc or ::cuMemAddressReserve. -* If the address range was obtained via ::cuMemAddressReserve, it must also be fully mapped via ::cuMemMap. -* -* Users must ensure the \p dptr and \p size are aligned to the host page size. -* +/** +* \brief Retrieve handle for an address range +* +* Get a handle of the specified type to an address range. * When requesting CUmemRangeHandleType::CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, -* users are expected to query for dma_buf support for the platform -* by using ::CU_DEVICE_ATTRIBUTE_DMA_BUF_SUPPORTED device attribute before calling -* this API. The \p handle will be interpreted as a pointer to an integer to store the dma_buf file descriptor. +* address range obtained by a prior call to either ::cuMemAlloc or +* ::cuMemAddressReserve is supported if the ::CU_DEVICE_ATTRIBUTE_DMA_BUF_SUPPORTED +* device attribute returns true. +* If the address range was obtained via ::cuMemAddressReserve, it +* must also be fully mapped via ::cuMemMap. +* Address range obtained by a prior call to either ::cuMemAllocHost or ::cuMemHostAlloc +* is supported if the ::CU_DEVICE_ATTRIBUTE_HOST_ALLOC_DMA_BUF_SUPPORTED device +* attribute returns true. +* +* As of CUDA 13.0, querying support for address range obtained by calling +* ::cuMemAllocHost or ::cuMemHostAlloc using the +* ::CU_DEVICE_ATTRIBUTE_DMA_BUF_SUPPORTED device attribute is deprecated. +* +* Users must ensure the \p dptr and \p size are aligned to the host page size. +* +* The \p handle will be interpreted as a pointer to an integer to store the dma_buf file descriptor. * Users must ensure the entire address range is backed and mapped when * the address range is allocated by ::cuMemAddressReserve. All the physical * allocations backing the address range must be resident on the same device and * have identical allocation properties. Users are also expected to retrieve a * new handle every time the underlying physical allocation(s) corresponding * to a previously queried VA range are changed. -* -* \param[out] handle - Pointer to the location where the returned handle will be stored. +* +* For CUmemRangeHandleType::CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, users may set +* flags to ::CU_MEM_RANGE_FLAG_DMA_BUF_MAPPING_TYPE_PCIE. Which when set on a +* supported platform, will give a DMA_BUF handle mapped via PCIE BAR1 or will +* return an error otherwise. +* +* \param[out] handle - Pointer to the location where the returned handle will be stored. * \param[in] dptr - Pointer to a valid CUDA device allocation. Must be aligned to host page size. * \param[in] size - Length of the address range. Must be aligned to host page size. * \param[in] handleType - Type of handle requested (defines type and size of the \p handle output parameter) -* \param[in] flags - Reserved, must be zero +* \param[in] flags - When requesting CUmemRangeHandleType::CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD the value could be +* ::CU_MEM_RANGE_FLAG_DMA_BUF_MAPPING_TYPE_PCIE, otherwise 0. * * \return -* CUDA_SUCCESS -* CUDA_ERROR_INVALID_VALUE -* CUDA_ERROR_NOT_SUPPORTED +* ::CUDA_SUCCESS, +* ::CUDA_ERROR_INVALID_VALUE, +* ::CUDA_ERROR_NOT_SUPPORTED */ CUresult CUDAAPI cuMemGetHandleForAddressRange(void *handle, CUdeviceptr dptr, size_t size, CUmemRangeHandleType handleType, unsigned long long flags); +/** + * \brief Bitmasks for CU_DEVICE_ATTRIBUTE_MEM_DECOMPRESS_ALGORITHM_MASK. + */ +typedef enum CUmemDecompressAlgorithm_enum { + CU_MEM_DECOMPRESS_UNSUPPORTED = 0, /**< Decompression is unsupported. */ + CU_MEM_DECOMPRESS_ALGORITHM_DEFLATE = 1<<0, /**< Deflate is supported. */ + CU_MEM_DECOMPRESS_ALGORITHM_SNAPPY = 1<<1, /**< Snappy is supported. */ + CU_MEM_DECOMPRESS_ALGORITHM_LZ4 = 1<<2 /**< LZ4 is supported. */ +} CUmemDecompressAlgorithm; + +/** + * \brief Structure describing the parameters that compose a single + * decompression operation. + */ +typedef struct CUmemDecompressParams_st { + /** The number of bytes to be read and decompressed from + * ::CUmemDecompressParams_st.src. */ + size_t srcNumBytes; + /** The number of bytes that the decompression operation will be expected to + * write to ::CUmemDecompressParams_st.dst. This value is optional; if + * present, it may be used by the CUDA driver as a heuristic for scheduling + * the individual decompression operations. */ + size_t dstNumBytes; + /** After the decompression operation has completed, the actual number of + * bytes written to ::CUmemDecompressParams.dst will be recorded as a 32-bit + * unsigned integer in the memory at this address. */ + cuuint32_t *dstActBytes; + /** Pointer to a buffer of at least ::CUmemDecompressParams_st.srcNumBytes + * compressed bytes. */ + const void *src; + /** Pointer to a buffer where the decompressed data will be written. The + * number of bytes written to this location will be recorded in the memory + * pointed to by ::CUmemDecompressParams_st.dstActBytes */ + void *dst; + /** The decompression algorithm to use. */ + CUmemDecompressAlgorithm algo; + /* These bytes are unused and must be zeroed. This ensures compatibility if + * additional fields are added in the future. */ + unsigned char padding[20]; +} CUmemDecompressParams; + +/** + * \brief Submit a batch of \p count independent decompression operations. + * + * \details Each of the \p count decompression operations is described by a + * single entry in the \p paramsArray array. Once the batch has been + * submitted, the function will return, and decompression will happen + * asynchronously w.r.t. the CPU. To the work completion tracking + * mechanisms in the CUDA driver, the batch will be considered a single + * unit of work and processed according to stream semantics, i.e., it + * is not possible to query the completion of individual decompression + * operations within a batch. + * + * The memory pointed to by each of ::CUmemDecompressParams.src, + * ::CUmemDecompressParams.dst, and ::CUmemDecompressParams.dstActBytes, + * must be capable of usage with the hardware decompress feature. That + * is, for each of said pointers, the pointer attribute + * ::CU_POINTER_ATTRIBUTE_IS_HW_DECOMPRESS_CAPABLE should give a + * non-zero value. To ensure this, the memory backing the pointers + * should have been allocated using one of the following CUDA memory + * allocators: + * * ::cuMemAlloc() + * * ::cuMemCreate() with the usage flag ::CU_MEM_CREATE_USAGE_HW_DECOMPRESS + * * ::cuMemAllocFromPoolAsync() from a pool that was created with + * the usage flag ::CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS + * Additionally, ::CUmemDecompressParams.src, ::CUmemDecompressParams.dst, + * and ::CUmemDecompressParams.dstActBytes, must all be accessible from + * the device associated with the context where \p stream was created. + * For information on how to ensure this, see the documentation for the + * allocator of interest. + * + * \param[in] paramsArray The array of structures describing the independent + * decompression operations. + * \param[in] count The number of entries in \p paramsArray array. + * \param[in] flags Must be 0. + * \param[out] errorIndex The index into \p paramsArray of the decompression + * operation for which the error returned by this + * function pertains to. If \p index is SIZE_MAX and + * the value returned is not ::CUDA_SUCCESS, then the + * error returned by this function should be considered + * a general error that does not pertain to a + * particular decompression operation. May be \p NULL, + * in which case, no index will be recorded in the + * event of error. + * \param[in] stream The stream where the work will be enqueued. + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_INVALID_HANDLE + * \notefnerr + * \note_async + * \note_null_stream + * + * \sa ::cuMemAlloc, ::cuMemPoolCreate, ::cuMemAllocFromPoolAsync + */ +CUresult CUDAAPI cuMemBatchDecompressAsync( + CUmemDecompressParams *paramsArray, + size_t count, + unsigned int flags, + size_t *errorIndex, + CUstream stream +); + /** @} */ /* END CUDA_MEM */ /** @@ -11130,12 +12724,14 @@ CUresult CUDAAPI cuMemGetHandleForAddressRange(void *handle, CUdeviceptr dptr, s * the starting address of the range in \p ptr. This API requires a system that * supports UVA. The size and address parameters must be a multiple of the * host page size and the alignment must be a power of two or zero for default -* alignment. +* alignment. If \p addr is 0, then the driver chooses the address at which to +* place the start of the reservation whereas when it is non-zero then the driver +* treats it as a hint about where to place the reservation. * * \param[out] ptr - Resulting pointer to start of virtual address range allocated * \param[in] size - Size of the reserved virtual address range requested * \param[in] alignment - Alignment of the reserved virtual address range requested -* \param[in] addr - Fixed starting address range requested +* \param[in] addr - Hint address for the start of the address range * \param[in] flags - Currently unused, must be zero * \return * ::CUDA_SUCCESS, @@ -11186,6 +12782,33 @@ CUresult CUDAAPI cuMemAddressFree(CUdeviceptr ptr, size_t size); * allocation must be a multiple of the the value given via * ::cuMemGetAllocationGranularity with the ::CU_MEM_ALLOC_GRANULARITY_MINIMUM * flag. +* To create a CPU allocation that doesn't target any specific NUMA nodes, applications must +* set ::CUmemAllocationProp::CUmemLocation::type to ::CU_MEM_LOCATION_TYPE_HOST. +* ::CUmemAllocationProp::CUmemLocation::id is ignored for HOST allocations. +* HOST allocations are not IPC capable and ::CUmemAllocationProp::requestedHandleTypes must be 0, +* any other value will result in ::CUDA_ERROR_INVALID_VALUE. +* To create a CPU allocation targeting a specific host NUMA node, applications must +* set ::CUmemAllocationProp::CUmemLocation::type to ::CU_MEM_LOCATION_TYPE_HOST_NUMA and +* ::CUmemAllocationProp::CUmemLocation::id must specify the NUMA ID of the CPU. +* On systems where NUMA is not available ::CUmemAllocationProp::CUmemLocation::id must be set to 0. +* Specifying ::CU_MEM_LOCATION_TYPE_HOST_NUMA_CURRENT as the +* ::CUmemLocation::type will result in ::CUDA_ERROR_INVALID_VALUE. +* +* Applications that intend to use ::CU_MEM_HANDLE_TYPE_FABRIC based memory sharing must ensure: +* (1) `nvidia-caps-imex-channels` character device is created by the driver and is listed under /proc/devices +* (2) have at least one IMEX channel file accessible by the user launching the application. +* +* When exporter and importer CUDA processes have been granted access to the same IMEX channel, they can securely +* share memory. +* +* The IMEX channel security model works on a per user basis. Which means all processes under a user can share +* memory if the user has access to a valid IMEX channel. When multi-user isolation is desired, a separate IMEX +* channel is required for each user. +* +* These channel files exist in /dev/nvidia-caps-imex-channels/channel* and can be created using standard OS +* native calls like mknod on Linux. For example: To create channel0 with the major number from /proc/devices +* users can execute the following command: `mknod /dev/nvidia-caps-imex-channels/channel0 c 0` +* * If ::CUmemAllocationProp::allocFlags::usage contains ::CU_MEM_CREATE_USAGE_TILE_POOL flag then * the memory allocation is intended only to be used as backing tile pool for sparse CUDA arrays * and sparse CUDA mipmapped arrays. @@ -11252,6 +12875,12 @@ CUresult CUDAAPI cuMemRelease(CUmemGenericAllocationHandle handle); * recommended that \p ptr, \p size and \p offset be aligned to the value * returned by ::cuMulticastGetGranularity with the flag * ::CU_MULTICAST_RECOMMENDED_GRANULARITY. +* +* When \p handle represents a multicast object, this call may return +* CUDA_ERROR_ILLEGAL_STATE if the system configuration is in an illegal state. +* In such cases, to continue using multicast, verify that the system +* configuration is in a valid state and all required driver daemons are +* running properly. * * Please note calling ::cuMemMap does not make the address accessible, * the caller needs to update accessibility of a contiguous mapped VA @@ -11280,7 +12909,8 @@ CUresult CUDAAPI cuMemRelease(CUmemGenericAllocationHandle handle); * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_DEINITIALIZED, * ::CUDA_ERROR_NOT_PERMITTED, -* ::CUDA_ERROR_NOT_SUPPORTED +* ::CUDA_ERROR_NOT_SUPPORTED, +* ::CUDA_ERROR_ILLEGAL_STATE * \notefnerr * * \sa ::cuMemUnmap, ::cuMemSetAccess, ::cuMemCreate, ::cuMemAddressReserve, ::cuMemImportFromShareableHandle @@ -11458,6 +13088,9 @@ CUresult CUDAAPI cuMemUnmap(CUdeviceptr ptr, size_t size); * in the array given by \p desc and \p count, set the access flags for the * target locations. The range must be a fully mapped address range * containing all allocations created by ::cuMemMap / ::cuMemCreate. +* Users cannot specify ::CU_MEM_LOCATION_TYPE_HOST_NUMA accessibility for allocations created on with other location types. +* Note: When ::CUmemAccessDesc::CUmemLocation::type is ::CU_MEM_LOCATION_TYPE_HOST_NUMA, ::CUmemAccessDesc::CUmemLocation::id +* is ignored. * When setting the access flags for a virtual address range mapping a multicast * object, \p ptr and \p size must be aligned to the value returned by * ::cuMulticastGetGranularity with the flag ::CU_MULTICAST_MINIMUM_GRANULARITY. @@ -11538,7 +13171,11 @@ CUresult CUDAAPI cuMemExportToShareableHandle(void *shareableHandle, CUmemGeneri * \brief Imports an allocation from a requested shareable handle type. * * If the current process cannot support the memory described by this shareable -* handle, this API will error as CUDA_ERROR_NOT_SUPPORTED. +* handle, this API will error as ::CUDA_ERROR_NOT_SUPPORTED. +* +* If \p shHandleType is ::CU_MEM_HANDLE_TYPE_FABRIC and the importer process has not been +* granted access to the same IMEX channel as the exporter process, this API will error +* as ::CUDA_ERROR_NOT_PERMITTED. * * \note Importing shareable handles exported from some graphics APIs(VUlkan, OpenGL, etc) * created on devices under an SLI group may not be supported, and thus this API will @@ -11865,9 +13502,40 @@ CUresult CUDAAPI cuMemPoolGetAccess(CUmemAccess_flags *flags, CUmemoryPool memPo * \brief Creates a memory pool * * Creates a CUDA memory pool and returns the handle in \p pool. The \p poolProps determines - * the properties of the pool such as the backing device and IPC capabilities. - * - * By default, the pool's memory will be accessible from the device it is allocated on. + * the properties of the pool such as the backing device and IPC capabilities. + * + * To create a memory pool for HOST memory not targeting a specific NUMA node, applications must set + * set ::CUmemPoolProps::CUmemLocation::type to ::CU_MEM_LOCATION_TYPE_HOST. + * ::CUmemPoolProps::CUmemLocation::id is ignored for such pools. + * Pools created with the type ::CU_MEM_LOCATION_TYPE_HOST are not IPC capable + * and ::CUmemPoolProps::handleTypes must be 0, any other values will result in + * ::CUDA_ERROR_INVALID_VALUE. + * To create a memory pool targeting a specific host NUMA node, applications must + * set ::CUmemPoolProps::CUmemLocation::type to ::CU_MEM_LOCATION_TYPE_HOST_NUMA and + * ::CUmemPoolProps::CUmemLocation::id must specify the NUMA ID of the host memory node. + * Specifying ::CU_MEM_LOCATION_TYPE_HOST_NUMA_CURRENT as the + * ::CUmemPoolProps::CUmemLocation::type will result in ::CUDA_ERROR_INVALID_VALUE. +* By default, the pool's memory will be accessible from the device it is allocated on. + * In the case of pools created with ::CU_MEM_LOCATION_TYPE_HOST_NUMA or + * ::CU_MEM_LOCATION_TYPE_HOST, their default accessibility will be from the + * host CPU. + * Applications can control the maximum size of the pool by specifying a non-zero value for ::CUmemPoolProps::maxSize. + * If set to 0, the maximum size of the pool will default to a system dependent value. + * + * Applications that intend to use ::CU_MEM_HANDLE_TYPE_FABRIC based memory sharing must ensure: + * (1) `nvidia-caps-imex-channels` character device is created by the driver and is listed under /proc/devices + * (2) have at least one IMEX channel file accessible by the user launching the application. + * + * When exporter and importer CUDA processes have been granted access to the same IMEX channel, they can securely + * share memory. + * + * The IMEX channel security model works on a per user basis. Which means all processes under a user can share + * memory if the user has access to a valid IMEX channel. When multi-user isolation is desired, a separate IMEX + * channel is required for each user. + * + * These channel files exist in /dev/nvidia-caps-imex-channels/channel* and can be created using standard OS + * native calls like mknod on Linux. For example: To create channel0 with the major number from /proc/devices + * users can execute the following command: `mknod /dev/nvidia-caps-imex-channels/channel0 c 0` * * \note Specifying CU_MEM_HANDLE_TYPE_NONE creates a memory pool that will not support IPC. * @@ -11876,6 +13544,7 @@ CUresult CUDAAPI cuMemPoolGetAccess(CUmemAccess_flags *flags, CUmemoryPool memPo * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_INVALID_VALUE, * ::CUDA_ERROR_OUT_OF_MEMORY, + * ::CUDA_ERROR_NOT_PERMITTED, * ::CUDA_ERROR_NOT_SUPPORTED * * \sa ::cuDeviceSetMemPool, ::cuDeviceGetMemPool, ::cuDeviceGetDefaultMemPool, @@ -11906,6 +13575,77 @@ CUresult CUDAAPI cuMemPoolCreate(CUmemoryPool *pool, const CUmemPoolProps *poolP */ CUresult CUDAAPI cuMemPoolDestroy(CUmemoryPool pool); +/** + * \brief Returns the default memory pool for a given location and allocation type + * + * The memory location can be of one of ::CU_MEM_LOCATION_TYPE_DEVICE, ::CU_MEM_LOCATION_TYPE_HOST or + * ::CU_MEM_LOCATION_TYPE_HOST_NUMA. The allocation type can be one of ::CU_MEM_ALLOCATION_TYPE_PINNED or + * ::CU_MEM_ALLOCATION_TYPE_MANAGED. When the allocation type is ::CU_MEM_ALLOCATION_TYPE_MANAGED, + * the location type can also be ::CU_MEM_LOCATION_TYPE_NONE to indicate no preferred location + * for the managed memory pool. In all other cases, the call returns ::CUDA_ERROR_INVALID_VALUE. + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_INVALID_DEVICE, + * ::CUDA_ERROR_NOT_SUPPORTED + * \notefnerr + * + * \sa ::cuMemAllocAsync, ::cuMemPoolTrimTo, ::cuMemPoolGetAttribute, ::cuMemPoolSetAttribute, cuMemPoolSetAccess, ::cuMemGetMemPool, ::cuMemPoolCreate + */ +CUresult CUDAAPI cuMemGetDefaultMemPool(CUmemoryPool *pool_out, CUmemLocation *location, CUmemAllocationType type); + +/** + * \brief Gets the current memory pool for a memory location and of a particular allocation type + * + * The memory location can be of one of ::CU_MEM_LOCATION_TYPE_DEVICE, ::CU_MEM_LOCATION_TYPE_HOST or + * ::CU_MEM_LOCATION_TYPE_HOST_NUMA. The allocation type can be one of ::CU_MEM_ALLOCATION_TYPE_PINNED or + * ::CU_MEM_ALLOCATION_TYPE_MANAGED. When the allocation type is ::CU_MEM_ALLOCATION_TYPE_MANAGED, + * the location type can also be ::CU_MEM_LOCATION_TYPE_NONE to indicate no preferred location + * for the managed memory pool. In all other cases, the call returns ::CUDA_ERROR_INVALID_VALUE + * + * Returns the last pool provided to ::cuMemSetMemPool or ::cuDeviceSetMemPool for this location and allocation type + * or the location's default memory pool if ::cuMemSetMemPool or ::cuDeviceSetMemPool for that allocType and location + * has never been called. By default the current mempool of a location is the default mempool for a device. + * Otherwise the returned pool must have been set with ::cuDeviceSetMemPool. + * + * \returns + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE + * + * \sa ::cuDeviceGetDefaultMemPool, ::cuMemPoolCreate, ::cuDeviceSetMemPool, ::cuMemSetMemPool + */ +CUresult CUDAAPI cuMemGetMemPool(CUmemoryPool *pool, CUmemLocation *location, CUmemAllocationType type); + +/** + * \brief Sets the current memory pool for a memory location and allocation type + * + * The memory location can be of one of ::CU_MEM_LOCATION_TYPE_DEVICE, ::CU_MEM_LOCATION_TYPE_HOST or + * ::CU_MEM_LOCATION_TYPE_HOST_NUMA. The allocation type can be one of ::CU_MEM_ALLOCATION_TYPE_PINNED or + * ::CU_MEM_ALLOCATION_TYPE_MANAGED. When the allocation type is ::CU_MEM_ALLOCATION_TYPE_MANAGED, + * the location type can also be ::CU_MEM_LOCATION_TYPE_NONE to indicate no preferred location + * for the managed memory pool. In all other cases, the call returns ::CUDA_ERROR_INVALID_VALUE. + * + * When a memory pool is set as the current memory pool, the location parameter should be the same as the location of the pool. + * The location and allocation type specified must match those of the pool otherwise ::CUDA_ERROR_INVALID_VALUE is returned. + * By default, a memory location's current memory pool is its default memory pool that can be obtained via ::cuMemGetDefaultMemPool. + * If the location type is ::CU_MEM_LOCATION_TYPE_DEVICE and the allocation type is ::CU_MEM_ALLOCATION_TYPE_PINNED, then + * this API is the equivalent of calling ::cuDeviceSetMemPool with the location id as the device. + * For further details on the implications, please refer to the documentation for ::cuDeviceSetMemPool. + * + * \note Use ::cuMemAllocFromPoolAsync to specify asynchronous allocations from a device different + * than the one the stream runs on. + * + * \returns + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE + * + * \sa ::cuDeviceGetDefaultMemPool, ::cuDeviceGetMemPool, ::cuMemGetMemPool, ::cuMemPoolCreate, ::cuMemPoolDestroy, ::cuMemAllocFromPoolAsync + */ +CUresult CUDAAPI cuMemSetMemPool(CUmemLocation *location, CUmemAllocationType type, CUmemoryPool pool); + /** * \brief Allocates memory from a specified pool with stream ordered semantics. * @@ -11978,6 +13718,11 @@ CUresult CUDAAPI cuMemPoolExportToShareableHandle(void *handle_out, CUmemoryPool * * Specific allocations can be imported from the imported pool with cuMemPoolImportPointer. * + * If \p handleType is ::CU_MEM_HANDLE_TYPE_FABRIC and the importer process has not been + * granted access to the same IMEX channel as the exporter process, this API will error + * as ::CUDA_ERROR_NOT_PERMITTED. + * + * * \note Imported memory pools do not support creating new allocations. * As such imported memory pools may not be used in cuDeviceSetMemPool * or ::cuMemAllocFromPoolAsync calls. @@ -12066,7 +13811,7 @@ CUresult CUDAAPI cuMemPoolImportPointer(CUdeviceptr *ptr_out, CUmemoryPool pool, * \section CUDA_MULTICAST_overview overview * * A multicast object created via ::cuMulticastCreate enables certain memory - * operations to be broadcasted to a team of devices. Devices can be added to a + * operations to be broadcast to a team of devices. Devices can be added to a * multicast object via ::cuMulticastAddDevice. Memory can be bound on each * participating device via either ::cuMulticastBindMem or ::cuMulticastBindAddr. * Multicast objects can be mapped into a device's virtual address space using @@ -12168,8 +13913,8 @@ CUresult CUDAAPI cuMulticastAddDevice(CUmemGenericAllocationHandle mcHandle, CUd * returned by ::cuMulticastGetGranularity with the flag * ::CU_MULTICAST_GRANULARITY_RECOMMENDED. * - * The \p size + \p memOffset must be smaller than the size of the allocated - * memory. Similarly the \p size + \p mcOffset must be smaller than the size + * The \p size + \p memOffset cannot be larger than the size of the allocated + * memory. Similarly the \p size + \p mcOffset cannot be larger than the size * of the multicast object. * The memory allocation must have beeen created on one of the devices * that was added to the multicast team via ::cuMulticastAddDevice. @@ -12180,6 +13925,11 @@ CUresult CUDAAPI cuMulticastAddDevice(CUmemGenericAllocationHandle mcHandle, CUd * return CUDA_ERROR_SYSTEM_NOT_READY if the necessary system software is not * initialized or running. * + * This call may return CUDA_ERROR_ILLEGAL_STATE if the system configuration + * is in an illegal state. In such cases, to continue using multicast, verify + * that the system configuration is in a valid state and all required driver + * daemons are running properly. + * * \param[in] mcHandle Handle representing a multicast object. * \param[in] mcOffset Offset into the multicast object for attachment. * \param[in] memHandle Handle representing a memory allocation. @@ -12197,7 +13947,8 @@ CUresult CUDAAPI cuMulticastAddDevice(CUmemGenericAllocationHandle mcHandle, CUd * ::CUDA_ERROR_NOT_PERMITTED, * ::CUDA_ERROR_NOT_SUPPORTED, * ::CUDA_ERROR_OUT_OF_MEMORY, - * ::CUDA_ERROR_SYSTEM_NOT_READY + * ::CUDA_ERROR_SYSTEM_NOT_READY, + * ::CUDA_ERROR_ILLEGAL_STATE * * \sa ::cuMulticastCreate, ::cuMulticastAddDevice, ::cuMemCreate */ @@ -12216,8 +13967,8 @@ CUresult CUDAAPI cuMulticastBindMem(CUmemGenericAllocationHandle mcHandle, size_ * aligned to the value returned by ::cuMulticastGetGranularity with the flag * ::CU_MULTICAST_GRANULARITY_RECOMMENDED. * - * The \p size must be smaller than the size of the allocated memory. - * Similarly the \p size + \p mcOffset must be smaller than the total size + * The \p size cannot be larger than the size of the allocated memory. + * Similarly the \p size + \p mcOffset cannot be larger than the total size * of the multicast object. * The memory allocation must have beeen created on one of the devices * that was added to the multicast team via ::cuMulticastAddDevice. @@ -12228,6 +13979,11 @@ CUresult CUDAAPI cuMulticastBindMem(CUmemGenericAllocationHandle mcHandle, size_ * return CUDA_ERROR_SYSTEM_NOT_READY if the necessary system software is not * initialized or running. * + * This call may return CUDA_ERROR_ILLEGAL_STATE if the system configuration + * is in an illegal state. In such cases, to continue using multicast, verify + * that the system configuration is in a valid state and all required driver + * daemons are running properly. + * * \param[in] mcHandle Handle representing a multicast object. * \param[in] mcOffset Offset into multicast va range for attachment. * \param[in] memptr Virtual address of the memory allocation. @@ -12244,7 +14000,8 @@ CUresult CUDAAPI cuMulticastBindMem(CUmemGenericAllocationHandle mcHandle, size_ * ::CUDA_ERROR_NOT_PERMITTED, * ::CUDA_ERROR_NOT_SUPPORTED, * ::CUDA_ERROR_OUT_OF_MEMORY, - * ::CUDA_ERROR_SYSTEM_NOT_READY + * ::CUDA_ERROR_SYSTEM_NOT_READY, + * ::CUDA_ERROR_ILLEGAL_STATE * * \sa ::cuMulticastCreate, ::cuMulticastAddDevice, ::cuMemCreate */ @@ -12258,7 +14015,7 @@ CUresult CUDAAPI cuMulticastBindAddr(CUmemGenericAllocationHandle mcHandle, size * The intended \p size of the unbind and the offset in the multicast range * ( \p mcOffset ) must be a multiple of the value returned by * ::cuMulticastGetGranularity flag ::CU_MULTICAST_GRANULARITY_MINIMUM. - * The \p size + \p mcOffset must be smaller than the total size of the + * The \p size + \p mcOffset cannot be larger than the total size of the * multicast object. * * \note @@ -12549,6 +14306,12 @@ CUresult CUDAAPI cuMulticastGetGranularity(size_t *granularity, const CUmulticas * * Returns in \p *data the handle to the mempool that the allocation was obtained from. * + * - ::CU_POINTER_ATTRIBUTE_IS_HW_DECOMPRESS_CAPABLE: + * + * Returns in \p *data a boolean that indicates whether the pointer points + * to memory that is capable to be used for hardware accelerated + * decompression. + * * \par * * Note that for most allocations in the unified virtual address space @@ -12594,18 +14357,24 @@ CUresult CUDAAPI cuMulticastGetGranularity(size_t *granularity, const CUmulticas CUresult CUDAAPI cuPointerGetAttribute(void *data, CUpointer_attribute attribute, CUdeviceptr ptr); /** - * \brief Prefetches memory to the specified destination device + * \brief Prefetches memory to the specified destination location * - * Prefetches memory to the specified destination device. \p devPtr is the - * base device pointer of the memory to be prefetched and \p dstDevice is the - * destination device. \p count specifies the number of bytes to copy. \p hStream + * Prefetches memory to the specified destination location. \p devPtr is the + * base device pointer of the memory to be prefetched and \p location specifies the + * destination location. \p count specifies the number of bytes to copy. \p hStream * is the stream in which the operation is enqueued. The memory range must refer * to managed memory allocated via ::cuMemAllocManaged or declared via __managed__ variables. * - * Passing in CU_DEVICE_CPU for \p dstDevice will prefetch the data to host memory. If - * \p dstDevice is a GPU, then the device attribute ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS - * must be non-zero. Additionally, \p hStream must be associated with a device that has a - * non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS. + * Specifying ::CU_MEM_LOCATION_TYPE_DEVICE for ::CUmemLocation::type will prefetch memory to GPU + * specified by device ordinal ::CUmemLocation::id which must have non-zero value for the device attribute + * ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS. Additionally, \p hStream must be associated with a device + * that has a non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS. + * Specifying ::CU_MEM_LOCATION_TYPE_HOST as ::CUmemLocation::type will prefetch data to host memory. + * Applications can request prefetching memory to a specific host NUMA node by specifying + * ::CU_MEM_LOCATION_TYPE_HOST_NUMA for ::CUmemLocation::type and a valid host NUMA node id in ::CUmemLocation::id + * Users can also request prefetching memory to the host NUMA node closest to the current thread's CPU by specifying + * ::CU_MEM_LOCATION_TYPE_HOST_NUMA_CURRENT for ::CUmemLocation::type. Note when ::CUmemLocation::type is etiher + * ::CU_MEM_LOCATION_TYPE_HOST OR ::CU_MEM_LOCATION_TYPE_HOST_NUMA_CURRENT, ::CUmemLocation::id will be ignored. * * The start address and end address of the memory range will be rounded down and rounded up * respectively to be aligned to CPU page size before the prefetch operation is enqueued @@ -12618,15 +14387,17 @@ CUresult CUDAAPI cuPointerGetAttribute(void *data, CUpointer_attribute attribute * allocated using ::cuMemAlloc or ::cuArrayCreate will not be evicted. * * By default, any mappings to the previous location of the migrated pages are removed and - * mappings for the new location are only setup on \p dstDevice. The exact behavior however + * mappings for the new location are only setup on the destination location. The exact behavior however * also depends on the settings applied to this memory range via ::cuMemAdvise as described * below: * * If ::CU_MEM_ADVISE_SET_READ_MOSTLY was set on any subset of this memory range, - * then that subset will create a read-only copy of the pages on \p dstDevice. + * then that subset will create a read-only copy of the pages on destination location. + * If however the destination location is a host NUMA node, then any pages of that subset + * that are already in another host NUMA node will be transferred to the destination. * * If ::CU_MEM_ADVISE_SET_PREFERRED_LOCATION was called on any subset of this memory - * range, then the pages will be migrated to \p dstDevice even if \p dstDevice is not the + * range, then the pages will be migrated to \p location even if \p location is not the * preferred location of any pages in the memory range. * * If ::CU_MEM_ADVISE_SET_ACCESSED_BY was called on any subset of this memory range, @@ -12644,8 +14415,9 @@ CUresult CUDAAPI cuPointerGetAttribute(void *data, CUpointer_attribute attribute * * \param devPtr - Pointer to be prefetched * \param count - Size in bytes - * \param dstDevice - Destination device to prefetch to - * \param hStream - Stream to enqueue prefetch operation + * \param location - Location to prefetch to + * \param flags - flags for future use, must be zero now. + * \param hStream - Stream to enqueue prefetch operation * * \return * ::CUDA_SUCCESS, @@ -12659,7 +14431,7 @@ CUresult CUDAAPI cuPointerGetAttribute(void *data, CUpointer_attribute attribute * ::cuMemcpy3DPeerAsync, ::cuMemAdvise, * ::cudaMemPrefetchAsync */ -CUresult CUDAAPI cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice dstDevice, CUstream hStream); +CUresult CUDAAPI cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUmemLocation location, unsigned int flags, CUstream hStream); /** * \brief Advise about the usage of a given memory range @@ -12678,8 +14450,11 @@ CUresult CUDAAPI cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice d * from and only occasionally written to. Any read accesses from any processor to this region will create a * read-only copy of at least the accessed pages in that processor's memory. Additionally, if ::cuMemPrefetchAsync * is called on this region, it will create a read-only copy of the data on the destination processor. + * If the target location for ::cuMemPrefetchAsync is a host NUMA node and a read-only copy already exists on + * another host NUMA node, that copy will be migrated to the targeted host NUMA node. * If any processor writes to this region, all copies of the corresponding page will be invalidated - * except for the one where the write occurred. The \p device argument is ignored for this advice. + * except for the one where the write occurred. If the writing processor is the CPU and the preferred location of + * the page is a host NUMA node, then the page will also be migrated to that host NUMA node. The \p location argument is ignored for this advice. * Note that for a page to be read-duplicated, the accessing processor must either be the CPU or a GPU * that has a non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS. * Also, if a context is created on a device that does not have the device attribute @@ -12696,12 +14471,17 @@ CUresult CUDAAPI cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice d * copies of the data will be collapsed into a single copy. The location for the collapsed * copy will be the preferred location if the page has a preferred location and one of the read-duplicated * copies was resident at that location. Otherwise, the location chosen is arbitrary. + * Note: The \p location argument is ignored for this advice. * * - ::CU_MEM_ADVISE_SET_PREFERRED_LOCATION: This advice sets the preferred location for the - * data to be the memory belonging to \p device. Passing in CU_DEVICE_CPU for \p device sets the - * preferred location as host memory. If \p device is a GPU, then it must have a non-zero value for the - * device attribute ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS. Setting the preferred location - * does not cause data to migrate to that location immediately. Instead, it guides the migration policy + * data to be the memory belonging to \p location. When ::CUmemLocation::type is ::CU_MEM_LOCATION_TYPE_HOST, + * ::CUmemLocation::id is ignored and the preferred location is set to be host memory. To set the preferred location + * to a specific host NUMA node, applications must set ::CUmemLocation::type to ::CU_MEM_LOCATION_TYPE_HOST_NUMA and + * ::CUmemLocation::id must specify the NUMA ID of the host NUMA node. If ::CUmemLocation::type is set to ::CU_MEM_LOCATION_TYPE_HOST_NUMA_CURRENT, + * ::CUmemLocation::id will be ignored and the the host NUMA node closest to the calling thread's CPU will be used as the preferred location. + * If ::CUmemLocation::type is a ::CU_MEM_LOCATION_TYPE_DEVICE, then ::CUmemLocation::id must be a valid device ordinal + * and the device must have a non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS. + * Setting the preferred location does not cause data to migrate to that location immediately. Instead, it guides the migration policy * when a fault occurs on that memory region. If the data is already in its preferred location and the * faulting processor can establish a mapping without requiring the data to be migrated, then * data migration will be avoided. On the other hand, if the data is not in its preferred location @@ -12714,19 +14494,18 @@ CUresult CUDAAPI cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice d * if the preferred location is set as device memory, then the page will continue to thrash indefinitely. * If ::CU_MEM_ADVISE_SET_READ_MOSTLY is also set on this memory region or any subset of it, then the * policies associated with that advice will override the policies of this advice, unless read accesses from - * \p device will not result in a read-only copy being created on that device as outlined in description for + * \p location will not result in a read-only copy being created on that procesor as outlined in description for * the advice ::CU_MEM_ADVISE_SET_READ_MOSTLY. - * If the memory region refers to valid system-allocated pageable memory, then \p device must have a non-zero - * value for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS. Additionally, if \p device has - * a non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS_USES_HOST_PAGE_TABLES, - * then this call has no effect. Note however that this behavior may change in the future. + * If the memory region refers to valid system-allocated pageable memory, and ::CUmemLocation::type is CU_MEM_LOCATION_TYPE_DEVICE + * then ::CUmemLocation::id must be a valid device that has a non-zero alue for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS. * * - ::CU_MEM_ADVISE_UNSET_PREFERRED_LOCATION: Undoes the effect of ::CU_MEM_ADVISE_SET_PREFERRED_LOCATION - * and changes the preferred location to none. + * and changes the preferred location to none. The \p location argument is ignored for this advice. * - * - ::CU_MEM_ADVISE_SET_ACCESSED_BY: This advice implies that the data will be accessed by \p device. - * Passing in ::CU_DEVICE_CPU for \p device will set the advice for the CPU. If \p device is a GPU, then - * the device attribute ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS must be non-zero. + * - ::CU_MEM_ADVISE_SET_ACCESSED_BY: This advice implies that the data will be accessed by processor \p location. + * The ::CUmemLocation::type must be either ::CU_MEM_LOCATION_TYPE_DEVICE with ::CUmemLocation::id representing a valid device + * ordinal or ::CU_MEM_LOCATION_TYPE_HOST and ::CUmemLocation::id will be ignored. All other location types are invalid. + * If ::CUmemLocation::id is a GPU, then the device attribute ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS must be non-zero. * This advice does not cause data migration and has no impact on the location of the data per se. Instead, * it causes the data to always be mapped in the specified processor's page tables, as long as the * location of the data permits a mapping to be established. If the data gets migrated for any reason, @@ -12742,24 +14521,24 @@ CUresult CUDAAPI cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice d * page in host memory. * If ::CU_MEM_ADVISE_SET_READ_MOSTLY is also set on this memory region or any subset of it, then the * policies associated with that advice will override the policies of this advice. Additionally, if the - * preferred location of this memory region or any subset of it is also \p device, then the policies + * preferred location of this memory region or any subset of it is also \p location, then the policies * associated with ::CU_MEM_ADVISE_SET_PREFERRED_LOCATION will override the policies of this advice. - * If the memory region refers to valid system-allocated pageable memory, then \p device must have a non-zero - * value for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS. Additionally, if \p device has - * a non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS_USES_HOST_PAGE_TABLES, + * If the memory region refers to valid system-allocated pageable memory, and ::CUmemLocation::type is ::CU_MEM_LOCATION_TYPE_DEVICE + * then device in ::CUmemLocation::id must have a non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS. + * Additionally, if ::CUmemLocation::id has a non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS_USES_HOST_PAGE_TABLES, * then this call has no effect. * * - ::CU_MEM_ADVISE_UNSET_ACCESSED_BY: Undoes the effect of ::CU_MEM_ADVISE_SET_ACCESSED_BY. Any mappings to - * the data from \p device may be removed at any time causing accesses to result in non-fatal page faults. - * If the memory region refers to valid system-allocated pageable memory, then \p device must have a non-zero - * value for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS. Additionally, if \p device has - * a non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS_USES_HOST_PAGE_TABLES, + * the data from \p location may be removed at any time causing accesses to result in non-fatal page faults. + * If the memory region refers to valid system-allocated pageable memory, and ::CUmemLocation::type is ::CU_MEM_LOCATION_TYPE_DEVICE + * then device in ::CUmemLocation::id must have a non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS. + * Additionally, if ::CUmemLocation::id has a non-zero value for the device attribute ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS_USES_HOST_PAGE_TABLES, * then this call has no effect. * - * \param devPtr - Pointer to memory to set the advice for - * \param count - Size in bytes of the memory range - * \param advice - Advice to be applied for the specified memory range - * \param device - Device to apply the advice for + * \param devPtr - Pointer to memory to set the advice for + * \param count - Size in bytes of the memory range + * \param advice - Advice to be applied for the specified memory range + * \param location - location to apply the advice for * * \return * ::CUDA_SUCCESS, @@ -12773,7 +14552,131 @@ CUresult CUDAAPI cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice d * ::cuMemcpy3DPeerAsync, ::cuMemPrefetchAsync, * ::cudaMemAdvise */ -CUresult CUDAAPI cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUdevice device); +CUresult CUDAAPI cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUmemLocation location); + +/** + * \brief Performs a batch of memory prefetches asynchronously + * + * Performs a batch of memory prefetches. The batch as a whole executes in stream order + * but operations within a batch are not guaranteed to execute in any specific order. + * All devices in the system must have a non-zero value for the device attribute + * ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS otherwise the API will return an error. + * + * The semantics of the individual prefetch operations are as described in ::cuMemPrefetchAsync. + * + * Performs memory prefetch on address ranges specified in \p dptrs and \p sizes. + * Both arrays must be of the same length as specified by \p count. Each memory range specified + * must refer to managed memory allocated via ::cuMemAllocManaged or declared via + * __managed__ variables or it may also refer to system-allocated memory when all devices have a non-zero + * value for ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS. The prefetch location for every operation + * in the batch is specified in the \p prefetchLocs array. Each entry in this array can apply to + * more than one operation. This can be done by specifying in the \p prefetchLocIdxs array, the + * index of the first prefetch operation that the corresponding entry in the \p prefetchLocs array + * applies to. Both \p prefetchLocs and \p prefetchLocIdxs must be of the same length as specified + * by \p numPrefetchLocs. For example, if a batch has 10 prefetches listed in dptrs/sizes, the + * first 4 of which are to be prefetched to one location and the remaining 6 are to be prefetched + * to another, then \p numPrefetchLocs will be 2, \p prefetchLocIdxs will be {0, 4} and \p prefetchLocs + * will contain the two locations. Note the first entry in \p prefetchLocIdxs must always be 0. + * Also, each entry must be greater than the previous entry and the last entry should be less than \p count. + * Furthermore, \p numPrefetchLocs must be lesser than or equal to \p count. + * + * \param dptrs - Array of pointers to be prefetched + * \param sizes - Array of sizes for memory prefetch operations. + * \param count - Size of \p dptrs and \p sizes arrays. + * \param prefetchLocs - Array of locations to prefetch to. + * \param prefetchLocIdxs - Array of indices to specify which operands each entry in the \p prefetchLocs array applies to. + * The locations specified in prefetchLocs[k] will be applied to copies starting from prefetchLocIdxs[k] + * through prefetchLocIdxs[k+1] - 1. Also prefetchLocs[numPrefetchLocs - 1] will apply to prefetches starting from + * prefetchLocIdxs[numPrefetchLocs - 1] through count - 1. + * \param numPrefetchLocs - Size of \p prefetchLocs and \p prefetchLocIdxs arrays. + * \param flags - Flags reserved for future use. Must be zero. + * \param hStream - The stream to enqueue the operations in. Must not be legacy NULL stream. + * + */ +CUresult CUDAAPI cuMemPrefetchBatchAsync(CUdeviceptr *dptrs, size_t *sizes, size_t count, + CUmemLocation *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, + unsigned long long flags, CUstream hStream); + +/** + * \brief Performs a batch of memory discards asynchronously + * + * Performs a batch of memory discards. The batch as a whole executes in stream order + * but operations within a batch are not guaranteed to execute in any specific order. + * All devices in the system must have a non-zero value for the device attribute + * ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS otherwise the API will return an error. + * + * Discarding a memory range informs the driver that the contents of that range are no longer useful. + * Discarding memory ranges allows the driver to optimize certain data migrations and can also help + * reduce memory pressure. This operation can be undone on any part of the range by either writing to it + * or prefetching it via ::cuMemPrefetchAsync or ::cuMemPrefetchBatchAsync. Reading from a discarded range, + * without a subsequent write or prefetch to that part of the range, will return an indeterminate value. + * Note that any reads, writes or prefetches to any part of the memory range that occur simultaneously with + * the discard operation result in undefined behavior. + * + * Performs memory discard on address ranges specified in \p dptrs and \p sizes. + * Both arrays must be of the same length as specified by \p count. Each memory range + * specified must refer to managed memory allocated via ::cuMemAllocManaged or declared + * via __managed__ variables or it may also refer to system-allocated memory when all devices + * have a non-zero value for ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS. + * + * \param dptrs - Array of pointers to be discarded + * \param sizes - Array of sizes for memory discard operations. + * \param count - Size of \p dptrs and \p sizes arrays. + * \param flags - Flags reserved for future use. Must be zero. + * \param hStream - The stream to enqueue the operations in. Must not be legacy NULL stream. + * + */ +CUresult CUDAAPI cuMemDiscardBatchAsync(CUdeviceptr *dptrs, size_t *sizes, size_t count, unsigned long long flags, CUstream hStream); + +/** + * \brief Performs a batch of memory discards and prefetches asynchronously + * + * Performs a batch of memory discards followed by prefetches. The batch as a whole executes + * in stream order but operations within a batch are not guaranteed to execute in any specific order. + * All devices in the system must have a non-zero value for the device attribute + * ::CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS otherwise the API will return an error. + * + * Calling ::cuMemDiscardAndPrefetchBatchAsync is semantically equivalent to calling + * ::cuMemDiscardBatchAsync followed by ::cuMemPrefetchBatchAsync, but is more optimal. + * For more details on what discarding and prefetching imply, please refer to ::cuMemDiscardBatchAsync and + * ::cuMemPrefetchBatchAsync respectively. Note that any reads, writes or prefetches to any part + * of the memory range that occur simultaneously with this combined discard+prefetch operation + * result in undefined behavior. + * + * Performs memory discard and prefetch on address ranges specified in \p dptrs and \p sizes. + * Both arrays must be of the same length as specified by \p count. Each memory range specified + * must refer to managed memory allocated via ::cuMemAllocManaged or declared via + * __managed__ variables or it may also refer to system-allocated memory when all devices + * have a non-zero value for ::CU_DEVICE_ATTRIBUTE_PAGEABLE_MEMORY_ACCESS. Every operation in the batch + * has to be associated with a valid location to prefetch the address range to and specified in + * the \p prefetchLocs array. Each entry in this array can apply to more than one operation. + * This can be done by specifying in the \p prefetchLocIdxs array, the index of the first + * operation that the corresponding entry in the \p prefetchLocs array applies to. + * Both \p prefetchLocs and \p prefetchLocIdxs must be of the same length as specified by + * \p numPrefetchLocs. For example, if a batch has 10 operations listed in dptrs/sizes, + * the first 6 of which are to be prefetched to one location and the remaining 4 are to be + * prefetched to another, then \p numPrefetchLocs will be 2, \p prefetchLocIdxs will be {0, 6} + * and \p prefetchLocs will contain the two set of locations. Note the first entry in + * \p prefetchLocIdxs must always be 0. Also, each entry must be greater than the previous + * entry and the last entry should be less than \p count. Furthermore, \p numPrefetchLocs + * must be lesser than or equal to \p count. + * + * \param dptrs - Array of pointers to be discarded + * \param sizes - Array of sizes for memory discard operations. + * \param count - Size of \p dptrs and \p sizes arrays. + * \param prefetchLocs - Array of locations to prefetch to. + * \param prefetchLocIdxs - Array of indices to specify which operands each entry in the \p prefetchLocs array applies to. + * The locations specified in prefetchLocs[k] will be applied to operations starting from prefetchLocIdxs[k] + * through prefetchLocIdxs[k+1] - 1. Also prefetchLocs[numPrefetchLocs - 1] will apply to copies starting from + * prefetchLocIdxs[numPrefetchLocs - 1] through count - 1. + * \param numPrefetchLocs - Size of \p prefetchLocs and \p prefetchLocIdxs arrays. + * \param flags - Flags reserved for future use. Must be zero. + * \param hStream - The stream to enqueue the operations in. Must not be legacy NULL stream. + * + */ +CUresult CUDAAPI cuMemDiscardAndPrefetchBatchAsync(CUdeviceptr *dptrs, size_t *sizes, size_t count, + CUmemLocation *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, + unsigned long long flags, CUstream hStream); /** * \brief Query an attribute of a given memory range @@ -12811,6 +14714,29 @@ CUresult CUDAAPI cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advi * prefetched to the same location, CU_DEVICE_INVALID will be returned. Note that this simply returns the * last location that the application requested to prefetch the memory range to. It gives no indication as to * whether the prefetch operation to that location has completed or even begun. + * - ::CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION_TYPE: If this attribute is specified, \p data will be + * interpreted as a ::CUmemLocationType, and \p dataSize must be sizeof(CUmemLocationType). The ::CUmemLocationType returned will be + * ::CU_MEM_LOCATION_TYPE_DEVICE if all pages in the memory range have the same GPU as their preferred location, or ::CUmemLocationType + * will be ::CU_MEM_LOCATION_TYPE_HOST if all pages in the memory range have the CPU as their preferred location, or it will be ::CU_MEM_LOCATION_TYPE_HOST_NUMA + * if all the pages in the memory range have the same host NUMA node ID as their preferred location or it will be ::CU_MEM_LOCATION_TYPE_INVALID + * if either all the pages don't have the same preferred location or some of the pages don't have a preferred location at all. + * Note that the actual location type of the pages in the memory range at the time of the query may be different from the preferred location type. + * - ::CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION_ID: If this attribute is specified, \p data will be + * interpreted as a 32-bit integer, and \p dataSize must be 4. If the ::CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION_TYPE query for the same address range + * returns ::CU_MEM_LOCATION_TYPE_DEVICE, it will be a valid device ordinal or if it returns ::CU_MEM_LOCATION_TYPE_HOST_NUMA, it will be a valid host NUMA node ID + * or if it returns any other location type, the id should be ignored. + * - ::CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION_TYPE: If this attribute is specified, \p data will be + * interpreted as a ::CUmemLocationType, and \p dataSize must be sizeof(CUmemLocationType). The result returned will be the last location + * to which all pages in the memory range were prefetched explicitly via ::cuMemPrefetchAsync. The ::CUmemLocationType returned + * will be ::CU_MEM_LOCATION_TYPE_DEVICE if the last prefetch location was a GPU or ::CU_MEM_LOCATION_TYPE_HOST if it was the CPU or ::CU_MEM_LOCATION_TYPE_HOST_NUMA if + * the last prefetch location was a specific host NUMA node. If any page in the memory range was never explicitly prefetched or if all pages were not + * prefetched to the same location, ::CUmemLocationType will be ::CU_MEM_LOCATION_TYPE_INVALID. + * Note that this simply returns the last location type that the application requested to prefetch the memory range to. It gives no indication as to + * whether the prefetch operation to that location has completed or even begun. + * - ::CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION_ID: If this attribute is specified, \p data will be + * interpreted as a 32-bit integer, and \p dataSize must be 4. If the ::CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION_TYPE query for the same address range + * returns ::CU_MEM_LOCATION_TYPE_DEVICE, it will be a valid device ordinal or if it returns ::CU_MEM_LOCATION_TYPE_HOST_NUMA, it will be a valid host NUMA node ID + * or if it returns any other location type, the id should be ignored. * * \param data - A pointers to a memory location where the result * of each attribute query will be written to. @@ -12849,6 +14775,10 @@ CUresult CUDAAPI cuMemRangeGetAttribute(void *data, size_t dataSize, CUmem_range * - ::CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION * - ::CU_MEM_RANGE_ATTRIBUTE_ACCESSED_BY * - ::CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION + * - ::CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION_TYPE + * - ::CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION_ID + * - ::CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION_TYPE + * - ::CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION_ID * * \param data - A two-dimensional array containing pointers to memory * locations where the result of each attribute query will be written to. @@ -12934,6 +14864,7 @@ CUresult CUDAAPI cuPointerSetAttribute(const void *value, CUpointer_attribute at * - ::CU_POINTER_ATTRIBUTE_IS_LEGACY_CUDA_IPC_CAPABLE * - ::CU_POINTER_ATTRIBUTE_ALLOWED_HANDLE_TYPES * - ::CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE + * - ::CU_POINTER_ATTRIBUTE_IS_HW_DECOMPRESS_CAPABLE * * \param numAttributes - Number of attributes to query * \param attributes - An array of attributes to query @@ -13004,8 +14935,10 @@ CUresult CUDAAPI cuPointerGetAttributes(unsigned int numAttributes, CUpointer_at * * \sa ::cuStreamDestroy, * ::cuStreamCreateWithPriority, + * ::cuGreenCtxStreamCreate, * ::cuStreamGetPriority, * ::cuStreamGetFlags, + * ::cuStreamGetDevice * ::cuStreamWaitEvent, * ::cuStreamQuery, * ::cuStreamSynchronize, @@ -13019,8 +14952,10 @@ CUresult CUDAAPI cuStreamCreate(CUstream *phStream, unsigned int Flags); * \brief Create a stream with the given priority * * Creates a stream with the specified priority and returns a handle in \p phStream. - * This API alters the scheduler priority of work in the stream. Work in a higher - * priority stream may preempt work already executing in a low priority stream. + * This affects the scheduling priority of work in the stream. Priorities provide a + * hint to preferentially run work with higher priority when possible, but do + * not preempt already-running work or provide any other functional guarantee on + * execution order. * * \p priority follows a convention where lower numbers represent higher priorities. * '0' represents default priority. The range of meaningful numerical priorities can @@ -13053,9 +14988,11 @@ CUresult CUDAAPI cuStreamCreate(CUstream *phStream, unsigned int Flags); * * \sa ::cuStreamDestroy, * ::cuStreamCreate, + * ::cuGreenCtxStreamCreate, * ::cuStreamGetPriority, * ::cuCtxGetStreamPriorityRange, * ::cuStreamGetFlags, + * ::cuStreamGetDevice, * ::cuStreamWaitEvent, * ::cuStreamQuery, * ::cuStreamSynchronize, @@ -13064,11 +15001,10 @@ CUresult CUDAAPI cuStreamCreate(CUstream *phStream, unsigned int Flags); */ CUresult CUDAAPI cuStreamCreateWithPriority(CUstream *phStream, unsigned int flags, int priority); - /** * \brief Query the priority of a given stream * - * Query the priority of a stream created using ::cuStreamCreate or ::cuStreamCreateWithPriority + * Query the priority of a stream created using ::cuStreamCreate, ::cuStreamCreateWithPriority or ::cuGreenCtxStreamCreate * and return the priority in \p priority. Note that if the stream was created with a * priority outside the numerical range returned by ::cuCtxGetStreamPriorityRange, * this function returns the clamped priority. @@ -13089,16 +15025,44 @@ CUresult CUDAAPI cuStreamCreateWithPriority(CUstream *phStream, unsigned int fla * \sa ::cuStreamDestroy, * ::cuStreamCreate, * ::cuStreamCreateWithPriority, + * ::cuGreenCtxStreamCreate, * ::cuCtxGetStreamPriorityRange, * ::cuStreamGetFlags, + * ::cuStreamGetDevice, * ::cudaStreamGetPriority */ CUresult CUDAAPI cuStreamGetPriority(CUstream hStream, int *priority); +/** + * \brief Returns the device handle of the stream + * + * Returns in \p *device the device handle of the stream + * + * \param hStream - Handle to the stream to be queried + * \param device - Returns the device to which a stream belongs + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_OUT_OF_MEMORY + * \notefnerr + * + * \sa + * ::cuStreamDestroy, + * ::cuStreamCreate, + * ::cuGreenCtxStreamCreate, + * ::cuStreamGetFlags + */ +CUresult CUDAAPI cuStreamGetDevice(CUstream hStream, CUdevice *device); + /** * \brief Query the flags of a given stream * - * Query the flags of a stream created using ::cuStreamCreate or ::cuStreamCreateWithPriority + * Query the flags of a stream created using ::cuStreamCreate, ::cuStreamCreateWithPriority or ::cuGreenCtxStreamCreate * and return the flags in \p flags. * * \param hStream - Handle to the stream to be queried @@ -13118,8 +15082,10 @@ CUresult CUDAAPI cuStreamGetPriority(CUstream hStream, int *priority); * * \sa ::cuStreamDestroy, * ::cuStreamCreate, + * ::cuGreenCtxStreamCreate, * ::cuStreamGetPriority, - * ::cudaStreamGetFlags + * ::cudaStreamGetFlags, + * ::cuStreamGetDevice */ CUresult CUDAAPI cuStreamGetFlags(CUstream hStream, unsigned int *flags); @@ -13161,6 +15127,10 @@ CUresult CUDAAPI cuStreamGetId(CUstream hStream, unsigned long long *streamId); * * Returns the CUDA context that the stream is associated with. * + * If the stream was created via the API ::cuGreenCtxStreamCreate, the returned context + * is equivalent to the one returned by ::cuCtxFromGreenCtx() on the green context + * associated with the stream at creation time. + * * The stream handle \p hStream can refer to any of the following: *
    *
  • a stream created via any of the CUDA driver APIs such as ::cuStreamCreate @@ -13185,12 +15155,14 @@ CUresult CUDAAPI cuStreamGetId(CUstream hStream, unsigned long long *streamId); * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_INVALID_CONTEXT, * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_NOT_SUPPORTED * \notefnerr * * \sa ::cuStreamDestroy, * ::cuStreamCreateWithPriority, * ::cuStreamGetPriority, * ::cuStreamGetFlags, + * ::cuStreamGetDevice * ::cuStreamWaitEvent, * ::cuStreamQuery, * ::cuStreamSynchronize, @@ -13200,6 +15172,64 @@ CUresult CUDAAPI cuStreamGetId(CUstream hStream, unsigned long long *streamId); */ CUresult CUDAAPI cuStreamGetCtx(CUstream hStream, CUcontext *pctx); +/** + * \brief Query the contexts associated with a stream + * + * Returns the contexts that the stream is associated with. + * + * If the stream is associated with a green context, the API returns the green context in \p pGreenCtx + * and the primary context of the associated device in \p pCtx. + * + * If the stream is associated with a regular context, the API returns the regular context in \p pCtx + * and NULL in \p pGreenCtx. + * + * The stream handle \p hStream can refer to any of the following: + *
      + *
    • a stream created via any of the CUDA driver APIs such as ::cuStreamCreate, + * ::cuStreamCreateWithPriority and ::cuGreenCtxStreamCreate, or their runtime API equivalents such as + * ::cudaStreamCreate, ::cudaStreamCreateWithFlags and ::cudaStreamCreateWithPriority. + * Passing an invalid handle will result in undefined behavior.
      • + *
    • any of the special streams such as the NULL stream, ::CU_STREAM_LEGACY and + * ::CU_STREAM_PER_THREAD. The runtime API equivalents of these are also accepted, + * which are NULL, ::cudaStreamLegacy and ::cudaStreamPerThread respectively. + * If any of the special handles are specified, the API will operate on the context current to the + * calling thread. If a green context (that was converted via ::cuCtxFromGreenCtx() before setting it current) + * is current to the calling thread, the API will return the green context in \p pGreenCtx + * and the primary context of the associated device in \p pCtx. If a regular context is current, + * the API returns the regular context in \p pCtx and NULL in \p pGreenCtx. + * Note that specifying ::CU_STREAM_PER_THREAD or ::cudaStreamPerThread will return ::CUDA_ERROR_INVALID_HANDLE + * if a green context is current to the calling thread. + * If no context is current to the calling thread, ::CUDA_ERROR_INVALID_CONTEXT is returned.
      • + *
    + * + * \param hStream - Handle to the stream to be queried + * \param pCtx - Returned regular context associated with the stream + * \param pGreenCtx - Returned green context if the stream is associated with a green context or NULL if not + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_HANDLE + * \notefnerr + * + * \sa ::cuStreamDestroy, + * ::cuStreamCreate + * ::cuStreamCreateWithPriority, + * ::cuGreenCtxStreamCreate, + * ::cuStreamGetPriority, + * ::cuStreamGetFlags, + * ::cuStreamGetDevice, + * ::cuStreamWaitEvent, + * ::cuStreamQuery, + * ::cuStreamSynchronize, + * ::cuStreamAddCallback, + * ::cudaStreamCreate, + * ::cudaStreamCreateWithFlags, + */ +CUresult CUDAAPI cuStreamGetCtx_v2(CUstream hStream, CUcontext *pCtx, CUgreenCtx *pGreenCtx); + /** * \brief Make a compute stream wait on an event * @@ -13350,6 +15380,52 @@ CUresult CUDAAPI cuStreamAddCallback(CUstream hStream, CUstreamCallback callback */ CUresult CUDAAPI cuStreamBeginCapture(CUstream hStream, CUstreamCaptureMode mode); +/** + * \brief Begins graph capture on a stream to an existing graph + * + * Begin graph capture on \p hStream, placing new nodes into an existing graph. When a stream is + * in capture mode, all operations pushed into the stream will not be executed, but will instead + * be captured into \p hGraph. The graph will not be instantiable until the user calls + * ::cuStreamEndCapture. + * + * Capture may not be initiated if \p stream is CU_STREAM_LEGACY. Capture must be ended on the + * same stream in which it was initiated, and it may only be initiated if the stream is not + * already in capture mode. The capture mode may be queried via ::cuStreamIsCapturing. A unique id + * representing the capture sequence may be queried via ::cuStreamGetCaptureInfo. + * + * If \p mode is not ::CU_STREAM_CAPTURE_MODE_RELAXED, ::cuStreamEndCapture must be + * called on this stream from the same thread. + * + * \param hStream - Stream in which to initiate capture. + * \param hGraph - Graph to capture into. + * \param dependencies - Dependencies of the first node captured in the stream. Can be NULL if numDependencies is 0. + * \param dependencyData - Optional array of data associated with each dependency. + * \param numDependencies - Number of dependencies. + * \param mode - Controls the interaction of this capture sequence with other API + * calls that are potentially unsafe. For more details see + * ::cuThreadExchangeStreamCaptureMode. + * + * \note Kernels captured using this API must not use texture and surface references. + * Reading or writing through any texture or surface reference is undefined + * behavior. This restriction does not apply to texture and surface objects. + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * + * \sa + * ::cuStreamBeginCapture, + * ::cuStreamCreate, + * ::cuStreamIsCapturing, + * ::cuStreamEndCapture, + * ::cuThreadExchangeStreamCaptureMode, + * ::cuGraphAddNode + */ +CUresult CUDAAPI cuStreamBeginCaptureToGraph(CUstream hStream, CUgraph hGraph, const CUgraphNode *dependencies, const CUgraphEdgeData *dependencyData, size_t numDependencies, CUstreamCaptureMode mode); + /** * \brief Swaps the stream capture interaction mode for a thread * @@ -13429,7 +15505,8 @@ CUresult CUDAAPI cuThreadExchangeStreamCaptureMode(CUstreamCaptureMode *mode); * \sa * ::cuStreamCreate, * ::cuStreamBeginCapture, - * ::cuStreamIsCapturing + * ::cuStreamIsCapturing, + * ::cuGraphDestroy */ CUresult CUDAAPI cuStreamEndCapture(CUstream hStream, CUgraph *phGraph); @@ -13485,6 +15562,11 @@ CUresult CUDAAPI cuStreamIsCapturing(CUstream hStream, CUstreamCaptureStatus *ca * - the call returns CUDA_SUCCESS * - the returned capture status is ::CU_STREAM_CAPTURE_STATUS_ACTIVE * + * If \p edgeData_out is non-NULL then \p dependencies_out must be as well. If + * \p dependencies_out is non-NULL and \p edgeData_out is NULL, but there is non-zero edge + * data for one or more of the current stream dependencies, the call will return + * ::CUDA_ERROR_LOSSY_QUERY. + * * \param hStream - The stream to query * \param captureStatus_out - Location to return the capture status of the stream; required * \param id_out - Optional location to return an id for the capture sequence, which is @@ -13500,17 +15582,23 @@ CUresult CUDAAPI cuStreamIsCapturing(CUstream hStream, CUstreamCaptureStatus *ca * \param dependencies_out - Optional location to store a pointer to an array of nodes. * The next node to be captured in the stream will depend on this set of nodes, * absent operations such as event wait which modify this set. The array pointer - * is valid until the next API call which operates on the stream or until end of - * capture. The node handles may be copied out and are valid until they or the - * graph is destroyed. The driver-owned array may also be passed directly to - * APIs that operate on the graph (not the stream) without copying. + * is valid until the next API call which operates on the stream or until the + * capture is terminated. The node handles may be copied out and are valid until + * they or the graph is destroyed. The driver-owned array may also be passed + * directly to APIs that operate on the graph (not the stream) without copying. + * \param edgeData_out - Optional location to store a pointer to an array of graph edge + * data. This array parallels \c dependencies_out; the next node to be added + * has an edge to \c dependencies_out[i] with annotation \c edgeData_out[i] for + * each \c i. The array pointer is valid until the next API call which operates + * on the stream or until the capture is terminated. * \param numDependencies_out - Optional location to store the size of the array * returned in dependencies_out. * * \return * ::CUDA_SUCCESS, * ::CUDA_ERROR_INVALID_VALUE, - * ::CUDA_ERROR_STREAM_CAPTURE_IMPLICIT + * ::CUDA_ERROR_STREAM_CAPTURE_IMPLICIT, + * ::CUDA_ERROR_LOSSY_QUERY * \note_graph_thread_safety * \notefnerr * @@ -13520,13 +15608,15 @@ CUresult CUDAAPI cuStreamIsCapturing(CUstream hStream, CUstreamCaptureStatus *ca * ::cuStreamUpdateCaptureDependencies */ CUresult CUDAAPI cuStreamGetCaptureInfo(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, - cuuint64_t *id_out, CUgraph *graph_out, const CUgraphNode **dependencies_out, size_t *numDependencies_out); + cuuint64_t *id_out, CUgraph *graph_out, const CUgraphNode **dependencies_out, + const CUgraphEdgeData **edgeData_out, size_t *numDependencies_out); /** - * \brief Update the set of dependencies in a capturing stream (11.3+) + * \brief Update the set of dependencies in a capturing stream * * Modifies the dependency set of a capturing stream. The dependency set is the set - * of nodes that the next captured node in the stream will depend on. + * of nodes that the next captured node in the stream will depend on along with the + * edge data for those dependencies. * * Valid flags are ::CU_STREAM_ADD_CAPTURE_DEPENDENCIES and * ::CU_STREAM_SET_CAPTURE_DEPENDENCIES. These control whether the set passed to @@ -13539,8 +15629,11 @@ CUresult CUDAAPI cuStreamGetCaptureInfo(CUstream hStream, CUstreamCaptureStatus * * Returns ::CUDA_ERROR_ILLEGAL_STATE if the stream is not capturing. * - * This API is new in CUDA 11.3. Developers requiring compatibility across minor - * versions to CUDA 11.0 should not use this API or provide a fallback. + * \param hStream - The stream to update + * \param dependencies - The set of dependencies to add + * \param dependencyData - Optional array of data associated with each dependency. + * \param numDependencies - The size of the dependencies array + * \param flags - See above * * \return * ::CUDA_SUCCESS, @@ -13549,9 +15642,10 @@ CUresult CUDAAPI cuStreamGetCaptureInfo(CUstream hStream, CUstreamCaptureStatus * * \sa * ::cuStreamBeginCapture, - * ::cuStreamGetCaptureInfo, + * ::cuStreamGetCaptureInfo */ -CUresult CUDAAPI cuStreamUpdateCaptureDependencies(CUstream hStream, CUgraphNode *dependencies, size_t numDependencies, unsigned int flags); +CUresult CUDAAPI cuStreamUpdateCaptureDependencies(CUstream hStream, CUgraphNode *dependencies, + const CUgraphEdgeData *dependencyData, size_t numDependencies, unsigned int flags); /** * \brief Attach memory to a stream asynchronously @@ -13855,7 +15949,8 @@ CUresult CUDAAPI cuEventCreate(CUevent *phEvent, unsigned int Flags); * \brief Records an event * * Captures in \p hEvent the contents of \p hStream at the time of this call. - * \p hEvent and \p hStream must be from the same context. + * \p hEvent and \p hStream must be from the same context otherwise + * ::CUDA_ERROR_INVALID_HANDLE is returned. * Calls such as ::cuEventQuery() or ::cuStreamWaitEvent() will then * examine or wait for completion of the work that was captured. Uses of * \p hStream after this call do not modify \p hEvent. See note on default @@ -13897,7 +15992,8 @@ CUresult CUDAAPI cuEventRecord(CUevent hEvent, CUstream hStream); * \brief Records an event * * Captures in \p hEvent the contents of \p hStream at the time of this call. - * \p hEvent and \p hStream must be from the same context. + * \p hEvent and \p hStream must be from the same context otherwise + * ::CUDA_ERROR_INVALID_HANDLE is returned. * Calls such as ::cuEventQuery() or ::cuStreamWaitEvent() will then * examine or wait for completion of the work that was captured. Uses of * \p hStream after this call do not modify \p hEvent. See note on default @@ -14038,7 +16134,10 @@ CUresult CUDAAPI cuEventDestroy(CUevent hEvent); * \brief Computes the elapsed time between two events * * Computes the elapsed time between two events (in milliseconds with a - * resolution of around 0.5 microseconds). + * resolution of around 0.5 microseconds). Note this API is not guaranteed + * to return the latest errors for pending work. As such this API is intended to + * serve as an elapsed time calculation only and any polling for completion on the + * events to be compared should be done with ::cuEventQuery instead. * * If either event was last recorded in a non-NULL stream, the resulting time * may be greater than expected (even if both used the same stream handle). This @@ -14132,7 +16231,8 @@ CUresult CUDAAPI cuEventElapsedTime(float *pMilliseconds, CUevent hStart, CUeven CU_EXTERNAL_MEMORY_HANDLE_TYPE_D3D12_RESOURCE = 5, CU_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_RESOURCE = 6, CU_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_RESOURCE_KMT = 7, - CU_EXTERNAL_MEMORY_HANDLE_TYPE_NVSCIBUF = 8 + CU_EXTERNAL_MEMORY_HANDLE_TYPE_NVSCIBUF = 8, + CU_EXTERNAL_MEMORY_HANDLE_TYPE_DMABUF_FD = 9 } CUexternalMemoryHandleType; * \endcode * @@ -14221,6 +16321,15 @@ CUresult CUDAAPI cuEventElapsedTime(float *pMilliseconds, CUevent hStart, CUeven * See ::CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC and ::CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC * for memory synchronization. * + * If ::CUDA_EXTERNAL_MEMORY_HANDLE_DESC::type is + * ::CU_EXTERNAL_MEMORY_HANDLE_TYPE_DMABUF_FD, then + * ::CUDA_EXTERNAL_MEMORY_HANDLE_DESC::handle::fd must be a valid + * file descriptor referencing a dma_buf object and + * ::CUDA_EXTERNAL_MEMORY_HANDLE_DESC::flags must be zero. Importing + * a dma_buf object is supported only on Tegra Jetson platform starting + * with Thor series. Mapping an imported dma_buf object as CUDA + * mipmapped array using ::cuExternalMemoryGetMappedMipmappedArray + * is not supported. * * The size of the memory object must be specified in * ::CUDA_EXTERNAL_MEMORY_HANDLE_DESC::size. @@ -14346,6 +16455,9 @@ CUresult CUDAAPI cuExternalMemoryGetMappedBuffer(CUdeviceptr *devPtr, CUexternal * If \p extMem was imported from a handle of type ::CU_EXTERNAL_MEMORY_HANDLE_TYPE_NVSCIBUF, then * ::CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC::numLevels must be equal to 1. * + * Mapping \p extMem imported from a handle of type + * ::CU_EXTERNAL_MEMORY_HANDLE_TYPE_DMABUF_FD, is not supported. + * * The returned CUDA mipmapped array must be freed using ::cuMipmappedArrayDestroy. * * \param mipmap - Returned CUDA mipmapped array @@ -14602,6 +16714,19 @@ CUresult CUDAAPI cuImportExternalSemaphore(CUexternalSemaphore *extSem_out, cons * same semaphore object with deterministic fence support enabled in different streams * or by adding explicit dependency amongst such streams so that the semaphore is * signaled in order. + * NvSciSyncFence associated with semaphore object of the type + * ::CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_NVSCISYNC can be timestamp enabled. For this + * the NvSciSyncAttrList used to create the object must have the value of + * NvSciSyncAttrKey_WaiterRequireTimestamps key set to true. Timestamps are emitted + * asynchronously by the GPU and CUDA saves the GPU timestamp in the corresponding + * NvSciSyncFence at the time of signal on GPU. Users are expected to convert + * GPU clocks to CPU clocks using appropriate scaling functions. Users are expected + * to wait for the completion of the fence before extracting timestamp using appropriate + * NvSciSync APIs. Users are expected to ensure that there is only one outstanding + * timestamp enabled fence per Cuda-NvSciSync object at any point of time, failing + * which leads to undefined behavior. Extracting the timestamp before the + * corresponding fence is signalled could lead to undefined behaviour. Timestamp + * extracted via appropriate NvSciSync API would be in microseconds. * * If the semaphore object is any one of the following types: * ::CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_D3D11_KEYED_MUTEX, @@ -14933,9 +17058,7 @@ CUresult CUDAAPI cuStreamWriteValue64(CUstream stream, CUdeviceptr addr, cuuint6 * ordering established through this API is not visible to CUDA. CUDA tasks * that are (even indirectly) ordered by this API should also have that order * expressed with CUDA-visible dependencies such as events. This ensures that - * the scheduler does not serialize them in an improper order. For more - * information, see the Stream Memory Operations section in the programming - * guide(https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html). + * the scheduler does not serialize them in an improper order. * * \param stream The stream to enqueue the operations in. * \param count The number of operations in the array. Must be less than 256. @@ -15029,6 +17152,17 @@ CUresult CUDAAPI cuStreamBatchMemOp(CUstream stream, unsigned int count, CUstrea * - ::CU_FUNC_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE: The block * scheduling policy of a function. The value type is CUclusterSchedulingPolicy. * + * With a few execeptions, function attributes may also be queried on unloaded + * function handles returned from ::cuModuleEnumerateFunctions. + * ::CUDA_ERROR_FUNCTION_NOT_LOADED is returned if the attribute requires a fully + * loaded function but the function is not loaded. The loading state of a function + * may be queried using ::cuFuncIsloaded. ::cuFuncLoad may be called to explicitly + * load a function before querying the following attributes that require the function + * to be loaded: + * - ::CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK + * - ::CU_FUNC_ATTRIBUTE_CONST_SIZE_BYTES + * - ::CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES + * * \param pi - Returned attribute value * \param attrib - Attribute requested * \param hfunc - Function to query attribute of @@ -15039,7 +17173,8 @@ CUresult CUDAAPI cuStreamBatchMemOp(CUstream stream, unsigned int count, CUstrea * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_INVALID_CONTEXT, * ::CUDA_ERROR_INVALID_HANDLE, - * ::CUDA_ERROR_INVALID_VALUE + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_FUNCTION_NOT_LOADED * \notefnerr * * \sa ::cuCtxGetCacheConfig, @@ -15048,6 +17183,8 @@ CUresult CUDAAPI cuStreamBatchMemOp(CUstream stream, unsigned int count, CUstrea * ::cuLaunchKernel, * ::cudaFuncGetAttributes, * ::cudaFuncSetAttribute, + * ::cuFuncIsLoaded, + * ::cuFuncLoad, * ::cuKernelGetAttribute */ CUresult CUDAAPI cuFuncGetAttribute(int *pi, CUfunction_attribute attrib, CUfunction hfunc); @@ -15090,6 +17227,9 @@ CUresult CUDAAPI cuFuncGetAttribute(int *pi, CUfunction_attribute attrib, CUfunc * positive. The validity of the cluster dimensions is checked at launch time. * If the value is set during compile time, it cannot be set at runtime. * Setting it at runtime will return CUDA_ERROR_NOT_PERMITTED. + * - ::CU_FUNC_ATTRIBUTE_NON_PORTABLE_CLUSTER_SIZE_ALLOWED: Indicates whether + * the function can be launched with non-portable cluster size. 1 is allowed, + * 0 is disallowed. * - ::CU_FUNC_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE: The block * scheduling policy of a function. The value type is CUclusterSchedulingPolicy. * @@ -15161,57 +17301,6 @@ CUresult CUDAAPI cuFuncSetAttribute(CUfunction hfunc, CUfunction_attribute attri */ CUresult CUDAAPI cuFuncSetCacheConfig(CUfunction hfunc, CUfunc_cache config); -/** - * \brief Sets the shared memory configuration for a device function. - * - * On devices with configurable shared memory banks, this function will - * force all subsequent launches of the specified device function to have - * the given shared memory bank size configuration. On any given launch of the - * function, the shared memory configuration of the device will be temporarily - * changed if needed to suit the function's preferred configuration. Changes in - * shared memory configuration between subsequent launches of functions, - * may introduce a device side synchronization point. - * - * Any per-function setting of shared memory bank size set via - * ::cuFuncSetSharedMemConfig will override the context wide setting set with - * ::cuCtxSetSharedMemConfig. - * - * Changing the shared memory bank size will not increase shared memory usage - * or affect occupancy of kernels, but may have major effects on performance. - * Larger bank sizes will allow for greater potential bandwidth to shared memory, - * but will change what kinds of accesses to shared memory will result in bank - * conflicts. - * - * This function will do nothing on devices with fixed shared memory bank size. - * - * The supported bank configurations are: - * - ::CU_SHARED_MEM_CONFIG_DEFAULT_BANK_SIZE: use the context's shared memory - * configuration when launching this function. - * - ::CU_SHARED_MEM_CONFIG_FOUR_BYTE_BANK_SIZE: set shared memory bank width to - * be natively four bytes when launching this function. - * - ::CU_SHARED_MEM_CONFIG_EIGHT_BYTE_BANK_SIZE: set shared memory bank width to - * be natively eight bytes when launching this function. - * - * \param hfunc - kernel to be given a shared memory config - * \param config - requested shared memory configuration - * - * \return - * ::CUDA_SUCCESS, - * ::CUDA_ERROR_INVALID_VALUE, - * ::CUDA_ERROR_DEINITIALIZED, - * ::CUDA_ERROR_NOT_INITIALIZED, - * ::CUDA_ERROR_INVALID_CONTEXT - * \notefnerr - * - * \sa ::cuCtxGetCacheConfig, - * ::cuCtxSetCacheConfig, - * ::cuCtxGetSharedMemConfig, - * ::cuCtxSetSharedMemConfig, - * ::cuFuncGetAttribute, - * ::cuLaunchKernel, - * ::cudaFuncSetSharedMemConfig - */ -CUresult CUDAAPI cuFuncSetSharedMemConfig(CUfunction hfunc, CUsharedconfig config); /** * \brief Returns a module handle @@ -15239,6 +17328,93 @@ CUresult CUDAAPI cuFuncSetSharedMemConfig(CUfunction hfunc, CUsharedconfig confi */ CUresult CUDAAPI cuFuncGetModule(CUmodule *hmod, CUfunction hfunc); +/** + * \brief Returns the function name for a ::CUfunction handle + * + * Returns in \p **name the function name associated with the function handle \p hfunc . + * The function name is returned as a null-terminated string. The returned name is only + * valid when the function handle is valid. If the module is unloaded or reloaded, one + * must call the API again to get the updated name. This API may return a mangled name if + * the function is not declared as having C linkage. If either \p **name or \p hfunc + * is NULL, ::CUDA_ERROR_INVALID_VALUE is returned. + * + * \param name - The returned name of the function + * \param hfunc - The function handle to retrieve the name for + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE, + * \notefnerr + * + */ +CUresult CUDAAPI cuFuncGetName(const char **name, CUfunction hfunc); + +/** + * \brief Returns the offset and size of a kernel parameter in the device-side parameter layout + * + * Queries the kernel parameter at \p paramIndex into \p func's list of parameters, and returns + * in \p paramOffset and \p paramSize the offset and size, respectively, where the parameter + * will reside in the device-side parameter layout. This information can be used to update kernel + * node parameters from the device via ::cudaGraphKernelNodeSetParam() and + * ::cudaGraphKernelNodeUpdatesApply(). \p paramIndex must be less than the number of parameters + * that \p func takes. \p paramSize can be set to NULL if only the parameter offset is desired. + * + * \param func - The function to query + * \param paramIndex - The parameter index to query + * \param paramOffset - Returns the offset into the device-side parameter layout at which the parameter resides + * \param paramSize - Optionally returns the size of the parameter in the device-side parameter layout + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE, + * \notefnerr + * +* \sa ::cuKernelGetParamInfo + */ +CUresult CUDAAPI cuFuncGetParamInfo(CUfunction func, size_t paramIndex, size_t *paramOffset, size_t *paramSize); + +typedef enum CUfunctionLoadingState_enum { + CU_FUNCTION_LOADING_STATE_UNLOADED = 0, + CU_FUNCTION_LOADING_STATE_LOADED = 1, + CU_FUNCTION_LOADING_STATE_MAX +} CUfunctionLoadingState; + +/** + * \brief Returns if the function is loaded + * + * Returns in \p state the loading state of \p function. + * + * \param state - returned loading state + * \param function - the function to check + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_INVALID_VALUE + * + * \sa ::cuFuncLoad, + * ::cuModuleEnumerateFunctions + */ +CUresult CUDAAPI cuFuncIsLoaded(CUfunctionLoadingState *state, CUfunction function); + +/** + * \brief Loads a function + * + * Finalizes function loading for \p function. Calling this API with a + * fully loaded function has no effect. + * + * \param function - the function to load + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_INVALID_VALUE + * + * \sa ::cuModuleEnumerateFunctions, + * ::cuFuncIsLoaded + */ +CUresult CUDAAPI cuFuncLoad(CUfunction function); + /** * \brief Launches a CUDA function ::CUfunction or a CUDA kernel ::CUkernel * @@ -15377,20 +17553,22 @@ CUresult CUDAAPI cuLaunchKernel(CUfunction f, * \p config. * * The ::CUlaunchConfig structure is defined as: + * * \code - typedef struct CUlaunchConfig_st { - unsigned int gridDimX; - unsigned int gridDimY; - unsigned int gridDimZ; - unsigned int blockDimX; - unsigned int blockDimY; - unsigned int blockDimZ; - unsigned int sharedMemBytes; - CUstream hStream; - CUlaunchAttribute *attrs; - unsigned int numAttrs; - } CUlaunchConfig; + * typedef struct CUlaunchConfig_st { + * unsigned int gridDimX; + * unsigned int gridDimY; + * unsigned int gridDimZ; + * unsigned int blockDimX; + * unsigned int blockDimY; + * unsigned int blockDimZ; + * unsigned int sharedMemBytes; + * CUstream hStream; + * CUlaunchAttribute *attrs; + * unsigned int numAttrs; + * } CUlaunchConfig; * \endcode + * * where: * - ::CUlaunchConfig::gridDimX is the width of the grid in blocks. * - ::CUlaunchConfig::gridDimY is the height of the grid in blocks. @@ -15407,7 +17585,7 @@ CUresult CUDAAPI cuLaunchKernel(CUfunction f, * continguous ::CUlaunchAttribute elements. The value of this pointer is not * considered if ::CUlaunchConfig::numAttrs is zero. However, in that case, it * is recommended to set the pointer to NULL. - * - ::CUlaunchConfig::numAttrs is the numbers of attributes populating the + * - ::CUlaunchConfig::numAttrs is the number of attributes populating the * first ::CUlaunchConfig::numAttrs positions of the ::CUlaunchConfig::attrs * array. * @@ -15417,10 +17595,10 @@ CUresult CUDAAPI cuLaunchKernel(CUfunction f, * * The ::CUlaunchAttribute structure is defined as: * \code - typedef struct CUlaunchAttribute_st { - CUlaunchAttributeID id; - CUlaunchAttributeValue value; - } CUlaunchAttribute; + * typedef struct CUlaunchAttribute_st { + * CUlaunchAttributeID id; + * CUlaunchAttributeValue value; + * } CUlaunchAttribute; * \endcode * where: * - ::CUlaunchAttribute::id is a unique enum identifying the attribute. @@ -15428,49 +17606,70 @@ CUresult CUDAAPI cuLaunchKernel(CUfunction f, * * An example of using the \p config parameter: * \code - CUlaunchAttribute coopAttr = {.id = CU_LAUNCH_ATTRIBUTE_COOPERATIVE, - .value = 1}; - CUlaunchConfig config = {... // set block and grid dimensions - .attrs = &coopAttr, - .numAttrs = 1}; - - cuLaunchKernelEx(&config, kernel, NULL, NULL); + * CUlaunchAttribute coopAttr = {.id = CU_LAUNCH_ATTRIBUTE_COOPERATIVE, + * .value = 1}; + * CUlaunchConfig config = {... // set block and grid dimensions + * .attrs = &coopAttr, + * .numAttrs = 1}; + * + * cuLaunchKernelEx(&config, kernel, NULL, NULL); * \endcode * * The ::CUlaunchAttributeID enum is defined as: * \code - typedef enum CUlaunchAttributeID_enum { - CU_LAUNCH_ATTRIBUTE_IGNORE = 0, - CU_LAUNCH_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1, - CU_LAUNCH_ATTRIBUTE_COOPERATIVE = 2, - CU_LAUNCH_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3, - CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION = 4, - CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5, - CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION = 6, - CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_EVENT = 7, - } CUlaunchAttributeID; + * typedef enum CUlaunchAttributeID_enum { + * CU_LAUNCH_ATTRIBUTE_IGNORE = 0, + * CU_LAUNCH_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1, + * CU_LAUNCH_ATTRIBUTE_COOPERATIVE = 2, + * CU_LAUNCH_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3, + * CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION = 4, + * CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5, + * CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION = 6, + * CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_EVENT = 7, + * CU_LAUNCH_ATTRIBUTE_PRIORITY = 8, + * CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9, + * CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN = 10, + * CU_LAUNCH_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11, + * CU_LAUNCH_ATTRIBUTE_LAUNCH_COMPLETION_EVENT = 12, + * CU_LAUNCH_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13, + * } CUlaunchAttributeID; * \endcode * * and the corresponding ::CUlaunchAttributeValue union as : * \code - typedef union CUlaunchAttributeValue_union { - cuuint64_t pad[8]; - CUaccessPolicyWindow accessPolicyWindow; - int cooperative; - CUsynchronizationPolicy syncPolicy; - struct { - unsigned int x; - unsigned int y; - unsigned int z; - } clusterDim; - CUclusterSchedulingPolicy clusterSchedulingPolicyPreference; - int programmaticStreamSerializationAllowed; - struct { - CUevent event; - int flags; - int triggerAtBlockStart; - } programmaticEvent; - } CUlaunchAttributeValue; + * typedef union CUlaunchAttributeValue_union { + * CUaccessPolicyWindow accessPolicyWindow; + * int cooperative; + * CUsynchronizationPolicy syncPolicy; + * struct { + * unsigned int x; + * unsigned int y; + * unsigned int z; + * } clusterDim; + * CUclusterSchedulingPolicy clusterSchedulingPolicyPreference; + * int programmaticStreamSerializationAllowed; + * struct { + * CUevent event; + * int flags; + * int triggerAtBlockStart; + * } programmaticEvent; + * int priority; + * CUlaunchMemSyncDomainMap memSyncDomainMap; + * CUlaunchMemSyncDomain memSyncDomain; + * struct { + * unsigned int x; + * unsigned int y; + * unsigned int z; + * } preferredClusterDim; + * struct { + * CUevent event; + * int flags; + * } launchCompletionEvent; + * struct { + * int deviceUpdatable; + * CUgraphDeviceNode devNode; + * } deviceUpdatableKernelNode; + * } CUlaunchAttributeValue; * \endcode * * Setting ::CU_LAUNCH_ATTRIBUTE_COOPERATIVE to a non-zero value causes the @@ -15498,6 +17697,72 @@ CUresult CUDAAPI cuLaunchKernel(CUfunction f, * event. The event must disable timing (i.e. created with * ::CU_EVENT_DISABLE_TIMING flag set). * + * ::CU_LAUNCH_ATTRIBUTE_LAUNCH_COMPLETION_EVENT records an event along with + * the kernel launch. Nominally, the event is triggered once all blocks of the + * kernel have begun execution. Currently this is a best effort. If a kernel B + * has a launch completion dependency on a kernel A, B may wait until A is + * complete. Alternatively, blocks of B may begin before all blocks of A have + * begun, for example: + * + * - If B can claim execution resources unavailable to A, for example if they + * run on different GPUs. + * - If B is a higher priority than A. + * + * Exercise caution if such an ordering inversion could lead to deadlock. The + * event supplied must not be an interprocess or interop event. The event must + * disable timing (i.e. must be created with the ::CU_EVENT_DISABLE_TIMING flag + * set). + * + * Setting ::CU_LAUNCH_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE to 1 + * on a captured launch causes the resulting kernel node to be device-updatable. + * This attribute is specific to graphs, and passing it to a launch in a + * non-capturing stream results in an error. Passing a value other than 0 or 1 is + * not allowed. + * + * On success, a handle will be returned via + * ::CUlaunchAttributeValue::deviceUpdatableKernelNode::devNode which can be passed + * to the various device-side update functions to update the node's kernel parameters + * from within another kernel. For more information on the types of device updates + * that can be made, as well as the relevant limitations thereof, see + * ::cudaGraphKernelNodeUpdatesApply. + * + * Kernel nodes which are device-updatable have additional restrictions compared to regular + * kernel nodes. Firstly, device-updatable nodes cannot be removed from their graph via + * ::cuGraphDestroyNode. Additionally, once opted-in to this functionality, a node cannot + * opt out, and any attempt to set the attribute to 0 will result in an error. Graphs + * containing one or more device-updatable node also do not allow multiple instantiation. + * + * ::CU_LAUNCH_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION allows the kernel launch to + * specify a preferred substitute cluster dimension. Blocks may be grouped + * according to either the dimensions specified with this attribute (grouped + * into a "preferred substitute cluster"), or the one specified with + * ::CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION attribute (grouped into a "regular + * cluster"). The cluster dimensions of a "preferred substitute cluster" shall + * be an integer multiple greater than zero of the regular cluster dimensions. + * The device will attempt - on a best-effort basis - to group thread blocks + * into preferred clusters over grouping them into regular clusters. When it + * deems necessary (primarily when the device temporarily runs out of physical + * resources to launch the larger preferred clusters), the device may switch to + * launch the regular clusters instead to attempt to utilize as much of the + * physical device resources as possible. + * + * Each type of cluster will have its enumeration / coordinate setup as if the + * grid consists solely of its type of cluster. For example, if the preferred + * substitute cluster dimensions double the regular cluster dimensions, there + * might be simultaneously a regular cluster indexed at (1,0,0), and a preferred + * cluster indexed at (1,0,0). In this example, the preferred substitute cluster + * (1,0,0) replaces regular clusters (2,0,0) and (3,0,0) and groups their + * blocks. + * + * This attribute will only take effect when a regular cluster dimension has + * been specified. The preferred substitute The preferred substitute cluster + * dimension must be an integer multiple greater than zero of the regular + * cluster dimension and must divide the grid. It must also be no more than + * `maxBlocksPerCluster`, if it is set in the kernel's `__launch_bounds__`. + * Otherwise it must be less than the maximum value the driver can support. + * Otherwise, setting this attribute to a value physically unable to fit on any + * particular device is permitted. + * * The effect of other attributes is consistent with their effect when set via * persistent APIs. * @@ -15505,7 +17770,7 @@ CUresult CUDAAPI cuLaunchKernel(CUfunction f, * - ::CU_LAUNCH_ATTRIBUTE_ACCESS_POLICY_WINDOW * - ::CU_LAUNCH_ATTRIBUTE_SYNCHRONIZATION_POLICY * - * See ::cuFunctionSetAttribute for + * See ::cuFuncSetAttribute for * - ::CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION * - ::CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE * @@ -15565,12 +17830,6 @@ CUresult CUDAAPI cuLaunchKernelEx(const CUlaunchConfig *config, * grid of blocks. Each block contains \p blockDimX x \p blockDimY x * \p blockDimZ threads. * - * Note that the API can also be used to launch context-less kernel ::CUkernel - * by querying the handle using ::cuLibraryGetKernel() and then passing it - * to the API by casting to ::CUfunction. Here, the context to launch - * the kernel on will either be taken from the specified stream \p hStream - * or the current context in case of NULL stream. - * * \p sharedMemBytes sets the amount of dynamic shared memory that will be * available to each thread block. * @@ -16276,6 +18535,61 @@ __CUDA_DEPRECATED CUresult CUDAAPI cuLaunchGridAsync(CUfunction f, int grid_widt * \notefnerr */ __CUDA_DEPRECATED CUresult CUDAAPI cuParamSetTexRef(CUfunction hfunc, int texunit, CUtexref hTexRef); + +/** + * \brief Sets the shared memory configuration for a device function. + * + * \deprecated + * + * On devices with configurable shared memory banks, this function will + * force all subsequent launches of the specified device function to have + * the given shared memory bank size configuration. On any given launch of the + * function, the shared memory configuration of the device will be temporarily + * changed if needed to suit the function's preferred configuration. Changes in + * shared memory configuration between subsequent launches of functions, + * may introduce a device side synchronization point. + * + * Any per-function setting of shared memory bank size set via + * ::cuFuncSetSharedMemConfig will override the context wide setting set with + * ::cuCtxSetSharedMemConfig. + * + * Changing the shared memory bank size will not increase shared memory usage + * or affect occupancy of kernels, but may have major effects on performance. + * Larger bank sizes will allow for greater potential bandwidth to shared memory, + * but will change what kinds of accesses to shared memory will result in bank + * conflicts. + * + * This function will do nothing on devices with fixed shared memory bank size. + * + * The supported bank configurations are: + * - ::CU_SHARED_MEM_CONFIG_DEFAULT_BANK_SIZE: use the context's shared memory + * configuration when launching this function. + * - ::CU_SHARED_MEM_CONFIG_FOUR_BYTE_BANK_SIZE: set shared memory bank width to + * be natively four bytes when launching this function. + * - ::CU_SHARED_MEM_CONFIG_EIGHT_BYTE_BANK_SIZE: set shared memory bank width to + * be natively eight bytes when launching this function. + * + * \param hfunc - kernel to be given a shared memory config + * \param config - requested shared memory configuration + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT + * \notefnerr + * + * \sa ::cuCtxGetCacheConfig, + * ::cuCtxSetCacheConfig, + * ::cuCtxGetSharedMemConfig, + * ::cuCtxSetSharedMemConfig, + * ::cuFuncGetAttribute, + * ::cuLaunchKernel, + * ::cudaFuncSetSharedMemConfig + */ +__CUDA_DEPRECATED CUresult CUDAAPI cuFuncSetSharedMemConfig(CUfunction hfunc, CUsharedconfig config); + /** @} */ /* END CUDA_EXEC_DEPRECATED */ /** @@ -16346,6 +18660,8 @@ CUresult CUDAAPI cuGraphCreate(CUgraph *phGraph, unsigned int flags); * unsigned int sharedMemBytes; * void **kernelParams; * void **extra; + * CUkernel kern; + * CUcontext ctx; * } CUDA_KERNEL_NODE_PARAMS; * \endcode * @@ -16409,6 +18725,7 @@ CUresult CUDAAPI cuGraphCreate(CUgraph *phGraph, unsigned int flags); * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuLaunchKernel, * ::cuLaunchCooperativeKernel, * ::cuGraphKernelNodeGetParams, @@ -16472,6 +18789,7 @@ CUresult CUDAAPI cuGraphKernelNodeGetParams(CUgraphNode hNode, CUDA_KERNEL_NODE_ * \notefnerr * * \sa + * ::cuGraphNodeSetParams, * ::cuLaunchKernel, * ::cuGraphAddKernelNode, * ::cuGraphKernelNodeGetParams @@ -16513,6 +18831,7 @@ CUresult CUDAAPI cuGraphKernelNodeSetParams(CUgraphNode hNode, const CUDA_KERNEL * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuMemcpy3D, * ::cuGraphMemcpyNodeGetParams, * ::cuGraphMemcpyNodeSetParams, @@ -16566,6 +18885,7 @@ CUresult CUDAAPI cuGraphMemcpyNodeGetParams(CUgraphNode hNode, CUDA_MEMCPY3D *no * \notefnerr * * \sa + * ::cuGraphNodeSetParams, * ::cuMemcpy3D, * ::cuGraphAddMemcpyNode, * ::cuGraphMemcpyNodeGetParams @@ -16601,6 +18921,7 @@ CUresult CUDAAPI cuGraphMemcpyNodeSetParams(CUgraphNode hNode, const CUDA_MEMCPY * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuMemsetD2D32, * ::cuGraphMemsetNodeGetParams, * ::cuGraphMemsetNodeSetParams, @@ -16654,6 +18975,7 @@ CUresult CUDAAPI cuGraphMemsetNodeGetParams(CUgraphNode hNode, CUDA_MEMSET_NODE_ * \notefnerr * * \sa + * ::cuGraphNodeSetParams, * ::cuMemsetD2D32, * ::cuGraphAddMemsetNode, * ::cuGraphMemsetNodeGetParams @@ -16688,6 +19010,7 @@ CUresult CUDAAPI cuGraphMemsetNodeSetParams(CUgraphNode hNode, const CUDA_MEMSET * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuLaunchHostFunc, * ::cuGraphHostNodeGetParams, * ::cuGraphHostNodeSetParams, @@ -16741,6 +19064,7 @@ CUresult CUDAAPI cuGraphHostNodeGetParams(CUgraphNode hNode, CUDA_HOST_NODE_PARA * \notefnerr * * \sa + * ::cuGraphNodeSetParams, * ::cuLaunchHostFunc, * ::cuGraphAddHostNode, * ::cuGraphHostNodeGetParams @@ -16756,7 +19080,8 @@ CUresult CUDAAPI cuGraphHostNodeSetParams(CUgraphNode hNode, const CUDA_HOST_NOD * at the root of the graph. \p dependencies may not have any duplicate entries. * A handle to the new node will be returned in \p phGraphNode. * - * If \p hGraph contains allocation or free nodes, this call will return an error. + * If \p childGraph contains allocation nodes, free nodes, or conditional nodes, this call will + * return an error. * * The node executes an embedded child graph. The child graph is cloned in this call. * @@ -16775,6 +19100,7 @@ CUresult CUDAAPI cuGraphHostNodeSetParams(CUgraphNode hNode, const CUDA_HOST_NOD * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuGraphChildGraphNodeGetGraph, * ::cuGraphCreate, * ::cuGraphDestroyNode, @@ -16842,6 +19168,7 @@ CUresult CUDAAPI cuGraphChildGraphNodeGetGraph(CUgraphNode hNode, CUgraph *phGra * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuGraphCreate, * ::cuGraphDestroyNode, * ::cuGraphAddChildGraphNode, @@ -16880,6 +19207,7 @@ CUresult CUDAAPI cuGraphAddEmptyNode(CUgraphNode *phGraphNode, CUgraph hGraph, c * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuGraphAddEventWaitNode, * ::cuEventRecordWithFlags, * ::cuStreamWaitEvent, @@ -16892,7 +19220,7 @@ CUresult CUDAAPI cuGraphAddEmptyNode(CUgraphNode *phGraphNode, CUgraph hGraph, c * ::cuGraphAddMemsetNode */ CUresult CUDAAPI cuGraphAddEventRecordNode(CUgraphNode *phGraphNode, CUgraph hGraph, const CUgraphNode *dependencies, size_t numDependencies, CUevent event); - + /** * \brief Returns the event associated with an event record node * @@ -16935,6 +19263,7 @@ CUresult CUDAAPI cuGraphEventRecordNodeGetEvent(CUgraphNode hNode, CUevent *even * \notefnerr * * \sa + * ::cuGraphNodeSetParams, * ::cuGraphAddEventRecordNode, * ::cuGraphEventRecordNodeGetEvent, * ::cuGraphEventWaitNodeSetEvent, @@ -16972,6 +19301,7 @@ CUresult CUDAAPI cuGraphEventRecordNodeSetEvent(CUgraphNode hNode, CUevent event * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuGraphAddEventRecordNode, * ::cuEventRecordWithFlags, * ::cuStreamWaitEvent, @@ -17027,6 +19357,7 @@ CUresult CUDAAPI cuGraphEventWaitNodeGetEvent(CUgraphNode hNode, CUevent *event_ * \notefnerr * * \sa + * ::cuGraphNodeSetParams, * ::cuGraphAddEventWaitNode, * ::cuGraphEventWaitNodeGetEvent, * ::cuGraphEventRecordNodeSetEvent, @@ -17064,6 +19395,7 @@ CUresult CUDAAPI cuGraphEventWaitNodeSetEvent(CUgraphNode hNode, CUevent event); * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuGraphExternalSemaphoresSignalNodeGetParams, * ::cuGraphExternalSemaphoresSignalNodeSetParams, * ::cuGraphExecExternalSemaphoresSignalNodeSetParams, @@ -17131,6 +19463,7 @@ CUresult CUDAAPI cuGraphExternalSemaphoresSignalNodeGetParams(CUgraphNode hNode, * \notefnerr * * \sa + * ::cuGraphNodeSetParams, * ::cuGraphAddExternalSemaphoresSignalNode, * ::cuGraphExternalSemaphoresSignalNodeSetParams, * ::cuGraphAddExternalSemaphoresWaitNode, @@ -17168,6 +19501,7 @@ CUresult CUDAAPI cuGraphExternalSemaphoresSignalNodeSetParams(CUgraphNode hNode, * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuGraphExternalSemaphoresWaitNodeGetParams, * ::cuGraphExternalSemaphoresWaitNodeSetParams, * ::cuGraphExecExternalSemaphoresWaitNodeSetParams, @@ -17235,6 +19569,7 @@ CUresult CUDAAPI cuGraphExternalSemaphoresWaitNodeGetParams(CUgraphNode hNode, C * \notefnerr * * \sa + * ::cuGraphNodeSetParams, * ::cuGraphAddExternalSemaphoresWaitNode, * ::cuGraphExternalSemaphoresWaitNodeSetParams, * ::cuGraphAddExternalSemaphoresWaitNode, @@ -17261,9 +19596,7 @@ CUresult CUDAAPI cuGraphExternalSemaphoresWaitNodeSetParams(CUgraphNode hNode, c * ordering established through this API is not visible to CUDA. CUDA tasks * that are (even indirectly) ordered by this API should also have that order * expressed with CUDA-visible dependencies such as events. This ensures that - * the scheduler does not serialize them in an improper order. For more - * information, see the Stream Memory Operations section in the programming - * guide(https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html). + * the scheduler does not serialize them in an improper order. * * \param phGraphNode - Returns newly created node * \param hGraph - Graph to which to add the node @@ -17281,6 +19614,7 @@ CUresult CUDAAPI cuGraphExternalSemaphoresWaitNodeSetParams(CUgraphNode hNode, c * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuStreamBatchMemOp, * ::cuStreamWaitValue32, * ::cuStreamWriteValue32, @@ -17346,6 +19680,7 @@ CUresult CUDAAPI cuGraphBatchMemOpNodeGetParams(CUgraphNode hNode, CUDA_BATCH_ME * \notefnerr * * \sa + * ::cuGraphNodeSetParams, * ::cuStreamBatchMemOp, * ::cuGraphAddBatchMemOpNode, * ::cuGraphBatchMemOpNodeGetParams @@ -17390,6 +19725,7 @@ CUresult CUDAAPI cuGraphBatchMemOpNodeSetParams(CUgraphNode hNode, const CUDA_BA * \notefnerr * * \sa + * ::cuGraphExecNodeSetParams, * ::cuStreamBatchMemOp, * ::cuGraphAddBatchMemOpNode, * ::cuGraphBatchMemOpNodeGetParams, @@ -17437,7 +19773,7 @@ CUresult CUDAAPI cuGraphExecBatchMemOpNodeSetParams(CUgraphExec hGraphExec, CUgr * * The following restrictions apply to graphs which contain allocation and/or memory free nodes: * - Nodes and edges of the graph cannot be deleted. - * - The graph cannot be used in a child node. + * - The graph can only be used in a child node if the ownership is moved to the parent. * - Only one instantiation of the graph may exist at any point in time. * - The graph cannot be cloned. * @@ -17451,6 +19787,7 @@ CUresult CUDAAPI cuGraphExecBatchMemOpNodeSetParams(CUgraphExec hGraphExec, CUgr * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuGraphAddMemFreeNode, * ::cuGraphMemAllocNodeGetParams, * ::cuDeviceGraphMemTrim, @@ -17519,7 +19856,7 @@ CUresult CUDAAPI cuGraphMemAllocNodeGetParams(CUgraphNode hNode, CUDA_MEM_ALLOC_ * * The following restrictions apply to graphs which contain allocation and/or memory free nodes: * - Nodes and edges of the graph cannot be deleted. - * - The graph cannot be used in a child node. + * - The graph can only be used in a child node if the ownership is moved to the parent. * - Only one instantiation of the graph may exist at any point in time. * - The graph cannot be cloned. * @@ -17533,6 +19870,7 @@ CUresult CUDAAPI cuGraphMemAllocNodeGetParams(CUgraphNode hNode, CUDA_MEM_ALLOC_ * \notefnerr * * \sa + * ::cuGraphAddNode, * ::cuGraphAddMemAllocNode, * ::cuGraphMemFreeNodeGetParams, * ::cuDeviceGraphMemTrim, @@ -17658,6 +19996,9 @@ CUresult CUDAAPI cuDeviceSetGraphMemAttribute(CUdevice device, CUgraphMem_attrib * * Child graph nodes in the original graph are recursively copied into the clone. * + * \note: Cloning is not supported for graphs which contain memory allocation nodes, + * memory free nodes, or conditional nodes. + * * \param phGraphClone - Returns newly created cloned graph * \param originalGraph - Graph to clone * @@ -17797,20 +20138,26 @@ CUresult CUDAAPI cuGraphGetRootNodes(CUgraph hGraph, CUgraphNode *rootNodes, siz * \brief Returns a graph's dependency edges * * Returns a list of \p hGraph's dependency edges. Edges are returned via corresponding - * indices in \p from and \p to; that is, the node in \p to[i] has a dependency on the - * node in \p from[i]. \p from and \p to may both be NULL, in which - * case this function only returns the number of edges in \p numEdges. Otherwise, - * \p numEdges entries will be filled in. If \p numEdges is higher than the actual - * number of edges, the remaining entries in \p from and \p to will be set to NULL, and - * the number of edges actually returned will be written to \p numEdges. + * indices in \p from, \p to and \p edgeData; that is, the node in \p to[i] has a + * dependency on the node in \p from[i] with data \p edgeData[i]. \p from and \p to may + * both be NULL, in which case this function only returns the number of edges in + * \p numEdges. Otherwise, \p numEdges entries will be filled in. If \p numEdges is higher + * than the actual number of edges, the remaining entries in \p from and \p to will be + * set to NULL, and the number of edges actually returned will be written to \p numEdges. + * \p edgeData may alone be NULL, in which case the edges must all have default (zeroed) + * edge data. Attempting a lossy query via NULL \p edgeData will result in + * ::CUDA_ERROR_LOSSY_QUERY. If \p edgeData is non-NULL then \p from and \p to must be + * as well. * * \param hGraph - Graph to get the edges from * \param from - Location to return edge endpoints * \param to - Location to return edge endpoints + * \param edgeData - Optional location to return edge data * \param numEdges - See description * * \return * ::CUDA_SUCCESS, + * ::CUDA_ERROR_LOSSY_QUERY, * ::CUDA_ERROR_DEINITIALIZED, * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_INVALID_VALUE @@ -17825,7 +20172,7 @@ CUresult CUDAAPI cuGraphGetRootNodes(CUgraph hGraph, CUgraphNode *rootNodes, siz * ::cuGraphNodeGetDependencies, * ::cuGraphNodeGetDependentNodes */ -CUresult CUDAAPI cuGraphGetEdges(CUgraph hGraph, CUgraphNode *from, CUgraphNode *to, size_t *numEdges); +CUresult CUDAAPI cuGraphGetEdges(CUgraph hGraph, CUgraphNode *from, CUgraphNode *to, CUgraphEdgeData *edgeData, size_t *numEdges); /** * \brief Returns a node's dependencies @@ -17836,12 +20183,18 @@ CUresult CUDAAPI cuGraphGetEdges(CUgraph hGraph, CUgraphNode *from, CUgraphNode * number of dependencies, the remaining entries in \p dependencies will be set to NULL, and the * number of nodes actually obtained will be returned in \p numDependencies. * + * Note that if an edge has non-zero (non-default) edge data and \p edgeData is NULL, + * this API will return ::CUDA_ERROR_LOSSY_QUERY. If \p edgeData is non-NULL, then + * \p dependencies must be as well. + * * \param hNode - Node to query * \param dependencies - Pointer to return the dependencies + * \param edgeData - Optional array to return edge data for each dependency * \param numDependencies - See description * * \return * ::CUDA_SUCCESS, + * ::CUDA_ERROR_LOSSY_QUERY, * ::CUDA_ERROR_DEINITIALIZED, * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_INVALID_VALUE @@ -17856,7 +20209,7 @@ CUresult CUDAAPI cuGraphGetEdges(CUgraph hGraph, CUgraphNode *from, CUgraphNode * ::cuGraphAddDependencies, * ::cuGraphRemoveDependencies */ -CUresult CUDAAPI cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode *dependencies, size_t *numDependencies); +CUresult CUDAAPI cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode *dependencies, CUgraphEdgeData *edgeData, size_t *numDependencies); /** * \brief Returns a node's dependent nodes @@ -17868,12 +20221,18 @@ CUresult CUDAAPI cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode *depe * \p dependentNodes will be set to NULL, and the number of nodes actually obtained will * be returned in \p numDependentNodes. * + * Note that if an edge has non-zero (non-default) edge data and \p edgeData is NULL, + * this API will return ::CUDA_ERROR_LOSSY_QUERY. If \p edgeData is non-NULL, then + * \p dependentNodes must be as well. + * * \param hNode - Node to query * \param dependentNodes - Pointer to return the dependent nodes + * \param edgeData - Optional pointer to return edge data for dependent nodes * \param numDependentNodes - See description * * \return * ::CUDA_SUCCESS, + * ::CUDA_ERROR_LOSSY_QUERY, * ::CUDA_ERROR_DEINITIALIZED, * ::CUDA_ERROR_NOT_INITIALIZED, * ::CUDA_ERROR_INVALID_VALUE @@ -17888,7 +20247,7 @@ CUresult CUDAAPI cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode *depe * ::cuGraphAddDependencies, * ::cuGraphRemoveDependencies */ -CUresult CUDAAPI cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode *dependentNodes, size_t *numDependentNodes); +CUresult CUDAAPI cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode *dependentNodes, CUgraphEdgeData *edgeData, size_t *numDependentNodes); /** * \brief Adds dependency edges to a graph @@ -17903,6 +20262,7 @@ CUresult CUDAAPI cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode *de * \param hGraph - Graph to which dependencies are added * \param from - Array of nodes that provide the dependencies * \param to - Array of dependent nodes + * \param edgeData - Optional array of edge data. If NULL, default (zeroed) edge data is assumed. * \param numDependencies - Number of dependencies to be added * * \return @@ -17917,7 +20277,7 @@ CUresult CUDAAPI cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode *de * ::cuGraphNodeGetDependencies, * ::cuGraphNodeGetDependentNodes */ -CUresult CUDAAPI cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode *from, const CUgraphNode *to, size_t numDependencies); +CUresult CUDAAPI cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode *from, const CUgraphNode *to, const CUgraphEdgeData *edgeData, size_t numDependencies); /** * \brief Removes dependency edges from a graph @@ -17927,7 +20287,9 @@ CUresult CUDAAPI cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode *from, * Each node in \p from and \p to must belong to \p hGraph. * * If \p numDependencies is 0, elements in \p from and \p to will be ignored. - * Specifying a non-existing dependency will return an error. + * Specifying an edge that does not exist in the graph, with data matching + * \p edgeData, results in an error. \p edgeData is nullable, which is equivalent + * to passing default (zeroed) data for each edge. * * Dependencies cannot be removed from graphs which contain allocation or free nodes. * Any attempt to do so will return an error. @@ -17935,6 +20297,8 @@ CUresult CUDAAPI cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode *from, * \param hGraph - Graph from which to remove dependencies * \param from - Array of nodes that provide the dependencies * \param to - Array of dependent nodes + * \param edgeData - Optional array of edge data. If NULL, edge data is assumed to + * be default (zeroed). * \param numDependencies - Number of dependencies to be removed * * \return @@ -17949,7 +20313,7 @@ CUresult CUDAAPI cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode *from, * ::cuGraphNodeGetDependencies, * ::cuGraphNodeGetDependentNodes */ -CUresult CUDAAPI cuGraphRemoveDependencies(CUgraph hGraph, const CUgraphNode *from, const CUgraphNode *to, size_t numDependencies); +CUresult CUDAAPI cuGraphRemoveDependencies(CUgraph hGraph, const CUgraphNode *from, const CUgraphNode *to, const CUgraphEdgeData *edgeData, size_t numDependencies); /** * \brief Remove a node from the graph @@ -18008,6 +20372,7 @@ CUresult CUDAAPI cuGraphDestroyNode(CUgraphNode hNode); * executable graph in existence for that graph at a time. An attempt to instantiate * a second executable graph before destroying the first with ::cuGraphExecDestroy * will result in an error. + * The same also applies if \p hGraph contains any device-updatable kernel nodes. * * If \p hGraph contains kernels which call device-side cudaGraphLaunch() from multiple * contexts, this will result in an error. @@ -18017,6 +20382,7 @@ CUresult CUDAAPI cuGraphDestroyNode(CUgraphNode hNode); * * - The graph's nodes must reside on a single context. * - The graph can only contain kernel nodes, memcpy nodes, memset nodes, and child graph nodes. + * - The graph cannot be empty and must contain at least one kernel, memcpy, or memset node. * Operation-specific restrictions are outlined below. * - Kernel nodes: * - Use of CUDA Dynamic Parallelism is not permitted. @@ -18094,6 +20460,7 @@ CUresult CUDAAPI cuGraphInstantiate(CUgraphExec *phGraphExec, CUgraph hGraph, un * executable graph in existence for that graph at a time. An attempt to instantiate a * second executable graph before destroying the first with ::cuGraphExecDestroy will * result in an error. + * The same also applies if \p hGraph contains any device-updatable kernel nodes. * * If \p hGraph contains kernels which call device-side cudaGraphLaunch() from multiple * contexts, this will result in an error. @@ -18103,6 +20470,7 @@ CUresult CUDAAPI cuGraphInstantiate(CUgraphExec *phGraphExec, CUgraph hGraph, un * * - The graph's nodes must reside on a single context. * - The graph can only contain kernel nodes, memcpy nodes, memset nodes, and child graph nodes. + * - The graph cannot be empty and must contain at least one kernel, memcpy, or memset node. * Operation-specific restrictions are outlined below. * - Kernel nodes: * - Use of CUDA Dynamic Parallelism is not permitted. @@ -18184,6 +20552,8 @@ CUresult CUDAAPI cuGraphExecGetFlags(CUgraphExec hGraphExec, cuuint64_t *flags); * - The owning context of the function cannot change. * - A node whose function originally did not use CUDA dynamic parallelism cannot be updated * to a function which uses CDP + * - A node whose function originally did not make device-side update calls cannot be updated + * to a function which makes device-side update calls. * - If \p hGraphExec was not instantiated for device launch, a node whose function originally * did not use device-side cudaGraphLaunch() cannot be updated to a function which uses * device-side cudaGraphLaunch() unless the node resides on the same context as nodes which @@ -18193,6 +20563,11 @@ CUresult CUDAAPI cuGraphExecGetFlags(CUgraphExec hGraphExec, cuuint64_t *flags); * The modifications only affect future launches of \p hGraphExec. Already * enqueued or running launches of \p hGraphExec are not affected by this call. * \p hNode is also not modified by this call. + * + * If \p hNode is a device-updatable kernel node, the next upload/launch of \p hGraphExec + * will overwrite any previous device-side updates. Additionally, applying host updates to a + * device-updatable kernel node while it is being updated from the device will result in + * undefined behavior. * * \param hGraphExec - The executable graph in which to set the specified node * \param hNode - kernel node from the graph from which graphExec was instantiated @@ -18205,6 +20580,7 @@ CUresult CUDAAPI cuGraphExecGetFlags(CUgraphExec hGraphExec, cuuint64_t *flags); * \notefnerr * * \sa + * ::cuGraphExecNodeSetParams, * ::cuGraphAddKernelNode, * ::cuGraphKernelNodeSetParams, * ::cuGraphExecMemcpyNodeSetParams, @@ -18251,6 +20627,7 @@ CUresult CUDAAPI cuGraphExecKernelNodeSetParams(CUgraphExec hGraphExec, CUgraphN * \notefnerr * * \sa + * ::cuGraphExecNodeSetParams, * ::cuGraphAddMemcpyNode, * ::cuGraphMemcpyNodeSetParams, * ::cuGraphExecKernelNodeSetParams, @@ -18273,18 +20650,22 @@ CUresult CUDAAPI cuGraphExecMemcpyNodeSetParams(CUgraphExec hGraphExec, CUgraphN * contained \p memsetParams at instantiation. hNode must remain in the graph which was * used to instantiate \p hGraphExec. Changed edges to and from hNode are ignored. * - * The destination memory in \p memsetParams must be allocated from the same - * contexts as the original destination memory. Both the instantiation-time - * memory operand and the memory operand in \p memsetParams must be 1-dimensional. - * Zero-length operations are not supported. + * Zero sized operations are not supported. + * + * The new destination pointer in memsetParams must be to the same kind of allocation + * as the original destination pointer and have the same context association and device mapping + * as the original destination pointer. + * + * Both the value and pointer address may be updated. + * Changing other aspects of the memset (width, height, element size or pitch) may cause the update to be rejected. + * Specifically, for 2d memsets, all dimension changes are rejected. + * For 1d memsets, changes in height are explicitly rejected and other changes are opportunistically allowed + * if the resulting work maps onto the work resources already allocated for the node. * * The modifications only affect future launches of \p hGraphExec. Already enqueued * or running launches of \p hGraphExec are not affected by this call. hNode is also * not modified by this call. * - * Returns CUDA_ERROR_INVALID_VALUE if the memory operand's mappings changed or - * either the original or new memory operand are multidimensional. - * * \param hGraphExec - The executable graph in which to set the specified node * \param hNode - Memset node from the graph which was used to instantiate graphExec * \param memsetParams - The updated parameters to set @@ -18297,6 +20678,7 @@ CUresult CUDAAPI cuGraphExecMemcpyNodeSetParams(CUgraphExec hGraphExec, CUgraphN * \notefnerr * * \sa + * ::cuGraphExecNodeSetParams, * ::cuGraphAddMemsetNode, * ::cuGraphMemsetNodeSetParams, * ::cuGraphExecKernelNodeSetParams, @@ -18334,6 +20716,7 @@ CUresult CUDAAPI cuGraphExecMemsetNodeSetParams(CUgraphExec hGraphExec, CUgraphN * \notefnerr * * \sa + * ::cuGraphExecNodeSetParams, * ::cuGraphAddHostNode, * ::cuGraphHostNodeSetParams, * ::cuGraphExecKernelNodeSetParams, @@ -18377,6 +20760,7 @@ CUresult CUDAAPI cuGraphExecHostNodeSetParams(CUgraphExec hGraphExec, CUgraphNod * \notefnerr * * \sa + * ::cuGraphExecNodeSetParams, * ::cuGraphAddChildGraphNode, * ::cuGraphChildGraphNodeGetGraph, * ::cuGraphExecKernelNodeSetParams, @@ -18414,6 +20798,7 @@ CUresult CUDAAPI cuGraphExecChildGraphNodeSetParams(CUgraphExec hGraphExec, CUgr * \notefnerr * * \sa + * ::cuGraphExecNodeSetParams, * ::cuGraphAddEventRecordNode, * ::cuGraphEventRecordNodeGetEvent, * ::cuGraphEventWaitNodeSetEvent, @@ -18454,6 +20839,7 @@ CUresult CUDAAPI cuGraphExecEventRecordNodeSetEvent(CUgraphExec hGraphExec, CUgr * \notefnerr * * \sa + * ::cuGraphExecNodeSetParams, * ::cuGraphAddEventWaitNode, * ::cuGraphEventWaitNodeGetEvent, * ::cuGraphEventRecordNodeSetEvent, @@ -18498,6 +20884,7 @@ CUresult CUDAAPI cuGraphExecEventWaitNodeSetEvent(CUgraphExec hGraphExec, CUgrap * \notefnerr * * \sa + * ::cuGraphExecNodeSetParams, * ::cuGraphAddExternalSemaphoresSignalNode, * ::cuImportExternalSemaphore, * ::cuSignalExternalSemaphoresAsync, @@ -18541,6 +20928,7 @@ CUresult CUDAAPI cuGraphExecExternalSemaphoresSignalNodeSetParams(CUgraphExec hG * \notefnerr * * \sa + * ::cuGraphExecNodeSetParams, * ::cuGraphAddExternalSemaphoresWaitNode, * ::cuImportExternalSemaphore, * ::cuSignalExternalSemaphoresAsync, @@ -18574,6 +20962,11 @@ CUresult CUDAAPI cuGraphExecExternalSemaphoresWaitNodeSetParams(CUgraphExec hGra * enqueued or running launches of \p hGraphExec are not affected by this call. * \p hNode is also not modified by this call. * + * If \p hNode is a device-updatable kernel node, the next upload/launch of \p hGraphExec + * will overwrite any previous device-side updates. Additionally, applying host updates to a + * device-updatable kernel node while it is being updated from the device will result in + * undefined behavior. + * * \note Currently only kernel, memset and memcpy nodes are supported. * * \param hGraphExec - The executable graph in which to set the specified node @@ -18605,6 +20998,7 @@ CUresult CUDAAPI cuGraphNodeSetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode * \p hNode must not have been removed from the original graph. * * \note Currently only kernel, memset and memcpy nodes are supported. + * \note This function will not reflect device-side updates for device-updatable kernel nodes. * * \param hGraphExec - The executable graph in which to set the specified node * \param hNode - Node from the graph from which graphExec was instantiated @@ -18737,6 +21131,8 @@ CUresult CUDAAPI cuGraphDestroy(CUgraph hGraph); * - The owning context of the function cannot change. * - A node whose function originally did not use CUDA dynamic parallelism cannot be updated * to a function which uses CDP. + * - A node whose function originally did not make device-side update calls cannot be updated + * to a function which makes device-side update calls. * - A cooperative node cannot be updated to a non-cooperative node, and vice-versa. * - If the graph was instantiated with CUDA_GRAPH_INSTANTIATE_FLAG_USE_NODE_PRIORITY, the * priority attribute cannot change. Equality is checked on the originally requested @@ -18746,16 +21142,23 @@ CUresult CUDAAPI cuGraphDestroy(CUgraph hGraph); * device-side cudaGraphLaunch() unless the node resides on the same context as nodes which * contained such calls at instantiate-time. If no such calls were present at instantiation, * these updates cannot be performed at all. + * - Neither \p hGraph nor \p hGraphExec may contain device-updatable kernel nodes. * - Memset and memcpy nodes: * - The CUDA device(s) to which the operand(s) was allocated/mapped cannot change. * - The source/destination memory must be allocated from the same contexts as the original * source/destination memory. - * - Only 1D memsets can be changed. + * - For 2d memsets, only address and assigned value may be updated. + * - For 1d memsets, updating dimensions is also allowed, but may fail if the resulting operation doesn't + * map onto the work resources already allocated for the node. * - Additional memcpy node restrictions: * - Changing either the source or destination memory type(i.e. CU_MEMORYTYPE_DEVICE, * CU_MEMORYTYPE_ARRAY, etc.) is not supported. * - External semaphore wait nodes and record nodes: * - Changing the number of semaphores is not supported. + * - Conditional nodes: + * - Changing node parameters is not supported. + * - Changing parameters of nodes within the conditional body graph is subject to the rules above. + * - Conditional handle flags and default values are updated as part of the graph update. * * Note: The API may add further restrictions in future releases. The return code should always be checked. * @@ -19033,81 +21436,236 @@ CUresult CUDAAPI cuGraphRetainUserObject(CUgraph graph, CUuserObject object, uns */ CUresult CUDAAPI cuGraphReleaseUserObject(CUgraph graph, CUuserObject object, unsigned int count); -/** @} */ /* END CUDA_GRAPH */ - /** - * \defgroup CUDA_OCCUPANCY Occupancy + * \brief Adds a node of arbitrary type to a graph * - * ___MANBRIEF___ occupancy calculation functions of the low-level CUDA driver - * API (___CURRENT_FILE___) ___ENDMANBRIEF___ + * Creates a new node in \p hGraph described by \p nodeParams with \p numDependencies + * dependencies specified via \p dependencies. \p numDependencies may be 0. + * \p dependencies may be null if \p numDependencies is 0. \p dependencies may not have + * any duplicate entries. * - * This section describes the occupancy calculation functions of the low-level CUDA - * driver application programming interface. + * \p nodeParams is a tagged union. The node type should be specified in the \p type field, + * and type-specific parameters in the corresponding union member. All unused bytes - that + * is, \p reserved0 and all bytes past the utilized union member - must be set to zero. + * It is recommended to use brace initialization or memset to ensure all bytes are + * initialized. * - * @{ - */ - -/** - * \brief Returns occupancy of a function + * Note that for some node types, \p nodeParams may contain "out parameters" which are + * modified during the call, such as \p nodeParams->alloc.dptr. * - * Returns in \p *numBlocks the number of the maximum active blocks per - * streaming multiprocessor. + * A handle to the new node will be returned in \p phGraphNode. * - * \param numBlocks - Returned occupancy - * \param func - Kernel for which occupancy is calculated - * \param blockSize - Block size the kernel is intended to be launched with - * \param dynamicSMemSize - Per-block dynamic shared memory usage intended, in bytes + * \param phGraphNode - Returns newly created node + * \param hGraph - Graph to which to add the node + * \param dependencies - Dependencies of the node + * \param dependencyData - Optional edge data for the dependencies. If NULL, the data is + * assumed to be default (zeroed) for all dependencies. + * \param numDependencies - Number of dependencies + * \param nodeParams - Specification of the node * * \return * ::CUDA_SUCCESS, - * ::CUDA_ERROR_DEINITIALIZED, - * ::CUDA_ERROR_NOT_INITIALIZED, - * ::CUDA_ERROR_INVALID_CONTEXT, * ::CUDA_ERROR_INVALID_VALUE, - * ::CUDA_ERROR_UNKNOWN + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_NOT_SUPPORTED + * \note_graph_thread_safety * \notefnerr * * \sa - * ::cudaOccupancyMaxActiveBlocksPerMultiprocessor + * ::cuGraphCreate, + * ::cuGraphNodeSetParams, + * ::cuGraphExecNodeSetParams */ -CUresult CUDAAPI cuOccupancyMaxActiveBlocksPerMultiprocessor(int *numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize); +CUresult CUDAAPI cuGraphAddNode(CUgraphNode *phGraphNode, CUgraph hGraph, const CUgraphNode *dependencies, const CUgraphEdgeData *dependencyData, size_t numDependencies, CUgraphNodeParams *nodeParams); /** - * \brief Returns occupancy of a function - * - * Returns in \p *numBlocks the number of the maximum active blocks per - * streaming multiprocessor. - * - * The \p Flags parameter controls how special cases are handled. The - * valid flags are: + * \brief Update's a graph node's parameters * - * - ::CU_OCCUPANCY_DEFAULT, which maintains the default behavior as - * ::cuOccupancyMaxActiveBlocksPerMultiprocessor; + * Sets the parameters of graph node \p hNode to \p nodeParams. The node type specified by + * \p nodeParams->type must match the type of \p hNode. \p nodeParams must be fully + * initialized and all unused bytes (reserved, padding) zeroed. * - * - ::CU_OCCUPANCY_DISABLE_CACHING_OVERRIDE, which suppresses the - * default behavior on platform where global caching affects - * occupancy. On such platforms, if caching is enabled, but - * per-block SM resource usage would result in zero occupancy, the - * occupancy calculator will calculate the occupancy as if caching - * is disabled. Setting ::CU_OCCUPANCY_DISABLE_CACHING_OVERRIDE makes - * the occupancy calculator to return 0 in such cases. More information - * can be found about this feature in the "Unified L1/Texture Cache" - * section of the Maxwell tuning guide. + * Modifying parameters is not supported for node types CU_GRAPH_NODE_TYPE_MEM_ALLOC and + * CU_GRAPH_NODE_TYPE_MEM_FREE. * - * \param numBlocks - Returned occupancy - * \param func - Kernel for which occupancy is calculated - * \param blockSize - Block size the kernel is intended to be launched with - * \param dynamicSMemSize - Per-block dynamic shared memory usage intended, in bytes - * \param flags - Requested behavior for the occupancy calculator + * \param hNode - Node to set the parameters for + * \param nodeParams - Parameters to copy * * \return * ::CUDA_SUCCESS, - * ::CUDA_ERROR_DEINITIALIZED, - * ::CUDA_ERROR_NOT_INITIALIZED, - * ::CUDA_ERROR_INVALID_CONTEXT, * ::CUDA_ERROR_INVALID_VALUE, - * ::CUDA_ERROR_UNKNOWN - * \notefnerr + * ::CUDA_ERROR_NOT_SUPPORTED + * \note_graph_thread_safety + * \notefnerr + * + * \sa + * ::cuGraphAddNode, + * ::cuGraphExecNodeSetParams + */ +CUresult CUDAAPI cuGraphNodeSetParams(CUgraphNode hNode, CUgraphNodeParams *nodeParams); + +/** + * \brief Update's a graph node's parameters in an instantiated graph + * + * Sets the parameters of a node in an executable graph \p hGraphExec. The node is identified + * by the corresponding node \p hNode in the non-executable graph from which the executable + * graph was instantiated. \p hNode must not have been removed from the original graph. + * + * The modifications only affect future launches of \p hGraphExec. Already + * enqueued or running launches of \p hGraphExec are not affected by this call. + * \p hNode is also not modified by this call. + * + * Allowed changes to parameters on executable graphs are as follows: + * + *
    Node typeAllowed changes + *
    kernelSee ::cuGraphExecKernelNodeSetParams + *
    memcpyAddresses for 1-dimensional copies if allocated in same context; see ::cuGraphExecMemcpyNodeSetParams + *
    memsetAddresses for 1-dimensional memsets if allocated in same context; see ::cuGraphExecMemsetNodeSetParams + *
    hostUnrestricted + *
    child graphTopology must match and restrictions apply recursively; see ::cuGraphExecUpdate + *
    event waitUnrestricted + *
    event recordUnrestricted + *
    external semaphore signalNumber of semaphore operations cannot change + *
    external semaphore waitNumber of semaphore operations cannot change + *
    memory allocationAPI unsupported + *
    memory freeAPI unsupported + *
    batch memopsAddresses, values, and operation type for wait operations; see ::cuGraphExecBatchMemOpNodeSetParams + *
    + * + * \param hGraphExec - The executable graph in which to update the specified node + * \param hNode - Corresponding node from the graph from which graphExec was instantiated + * \param nodeParams - Updated Parameters to set + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_NOT_SUPPORTED + * \note_graph_thread_safety + * \notefnerr + * + * \sa + * ::cuGraphAddNode, + * ::cuGraphNodeSetParams + * ::cuGraphExecUpdate, + * ::cuGraphInstantiate + */ +CUresult CUDAAPI cuGraphExecNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraphNodeParams *nodeParams); + +/** + * \brief Create a conditional handle + * + * Creates a conditional handle associated with \p hGraph. + * + * The conditional handle must be associated with a conditional node in this graph or one of its children. + * + * Handles not associated with a conditional node may cause graph instantiation to fail. + * + * Handles can only be set from the context with which they are associated. + * + * \param pHandle_out - Pointer used to return the handle to the caller. + * \param hGraph - Graph which will contain the conditional node using this handle. + * \param ctx - Context for the handle and associated conditional node. + * \param defaultLaunchValue - Optional initial value for the conditional variable. + * Applied at the beginning of each graph execution if CU_GRAPH_COND_ASSIGN_DEFAULT is set in \p flags. + * \param flags - Currently must be CU_GRAPH_COND_ASSIGN_DEFAULT or 0. + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_NOT_SUPPORTED + * \note_graph_thread_safety + * \notefnerr + * + * \sa + * ::cuGraphAddNode + */ +CUresult CUDAAPI cuGraphConditionalHandleCreate(CUgraphConditionalHandle *pHandle_out, CUgraph hGraph, CUcontext ctx, unsigned int defaultLaunchValue, unsigned int flags); + +/** @} */ /* END CUDA_GRAPH */ + +/** + * \defgroup CUDA_OCCUPANCY Occupancy + * + * ___MANBRIEF___ occupancy calculation functions of the low-level CUDA driver + * API (___CURRENT_FILE___) ___ENDMANBRIEF___ + * + * This section describes the occupancy calculation functions of the low-level CUDA + * driver application programming interface. + * + * @{ + */ + +/** + * \brief Returns occupancy of a function + * + * Returns in \p *numBlocks the number of the maximum active blocks per + * streaming multiprocessor. + * + * Note that the API can also be used with context-less kernel ::CUkernel + * by querying the handle using ::cuLibraryGetKernel() and then passing it + * to the API by casting to ::CUfunction. Here, the context to use for calculations + * will be the current context. + * + * \param numBlocks - Returned occupancy + * \param func - Kernel for which occupancy is calculated + * \param blockSize - Block size the kernel is intended to be launched with + * \param dynamicSMemSize - Per-block dynamic shared memory usage intended, in bytes + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_UNKNOWN + * \notefnerr + * + * \sa + * ::cudaOccupancyMaxActiveBlocksPerMultiprocessor + */ +CUresult CUDAAPI cuOccupancyMaxActiveBlocksPerMultiprocessor(int *numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize); + +/** + * \brief Returns occupancy of a function + * + * Returns in \p *numBlocks the number of the maximum active blocks per + * streaming multiprocessor. + * + * The \p Flags parameter controls how special cases are handled. The + * valid flags are: + * + * - ::CU_OCCUPANCY_DEFAULT, which maintains the default behavior as + * ::cuOccupancyMaxActiveBlocksPerMultiprocessor; + * + * - ::CU_OCCUPANCY_DISABLE_CACHING_OVERRIDE, which suppresses the + * default behavior on platform where global caching affects + * occupancy. On such platforms, if caching is enabled, but + * per-block SM resource usage would result in zero occupancy, the + * occupancy calculator will calculate the occupancy as if caching + * is disabled. Setting ::CU_OCCUPANCY_DISABLE_CACHING_OVERRIDE makes + * the occupancy calculator to return 0 in such cases. More information + * can be found about this feature in the "Unified L1/Texture Cache" + * section of the Maxwell tuning guide. + * + * Note that the API can also be with launch context-less kernel ::CUkernel + * by querying the handle using ::cuLibraryGetKernel() and then passing it + * to the API by casting to ::CUfunction. Here, the context to use for calculations + * will be the current context. + * + * \param numBlocks - Returned occupancy + * \param func - Kernel for which occupancy is calculated + * \param blockSize - Block size the kernel is intended to be launched with + * \param dynamicSMemSize - Per-block dynamic shared memory usage intended, in bytes + * \param flags - Requested behavior for the occupancy calculator + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_UNKNOWN + * \notefnerr * * \sa * ::cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags @@ -19145,6 +21703,11 @@ CUresult CUDAAPI cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int *numBl * size_t blockToSmem(int blockSize); * \endcode * + * Note that the API can also be used with context-less kernel ::CUkernel + * by querying the handle using ::cuLibraryGetKernel() and then passing it + * to the API by casting to ::CUfunction. Here, the context to use for calculations + * will be the current context. + * * \param minGridSize - Returned minimum grid size needed to achieve the maximum occupancy * \param blockSize - Returned maximum block size that can achieve the maximum occupancy * \param func - Kernel for which launch configuration is calculated @@ -19190,6 +21753,11 @@ CUresult CUDAAPI cuOccupancyMaxPotentialBlockSize(int *minGridSize, int *blockSi * can be found about this feature in the "Unified L1/Texture Cache" * section of the Maxwell tuning guide. * + * Note that the API can also be used with context-less kernel ::CUkernel + * by querying the handle using ::cuLibraryGetKernel() and then passing it + * to the API by casting to ::CUfunction. Here, the context to use for calculations + * will be the current context. + * * \param minGridSize - Returned minimum grid size needed to achieve the maximum occupancy * \param blockSize - Returned maximum block size that can achieve the maximum occupancy * \param func - Kernel for which launch configuration is calculated @@ -19217,6 +21785,11 @@ CUresult CUDAAPI cuOccupancyMaxPotentialBlockSizeWithFlags(int *minGridSize, int * * Returns in \p *dynamicSmemSize the maximum size of dynamic shared memory to allow \p numBlocks blocks per SM. * + * Note that the API can also be used with context-less kernel ::CUkernel + * by querying the handle using ::cuLibraryGetKernel() and then passing it + * to the API by casting to ::CUfunction. Here, the context to use for calculations + * will be the current context. + * * \param dynamicSmemSize - Returned maximum dynamic shared memory * \param func - Kernel function for which occupancy is calculated * \param numBlocks - Number of blocks to fit on SM @@ -19247,6 +21820,12 @@ CUresult CUDAAPI cuOccupancyAvailableDynamicSMemPerBlock(size_t *dynamicSmemSize * * This function will respect the compile time launch bounds. * + * Note that the API can also be used with context-less kernel ::CUkernel + * by querying the handle using ::cuLibraryGetKernel() and then passing it + * to the API by casting to ::CUfunction. Here, the context to use for calculations + * will either be taken from the specified stream \p config->hStream + * or the current context in case of NULL stream. + * * \param clusterSize - Returned maximum cluster size that can be launched * for the given kernel function and launch configuration * \param func - Kernel function for which maximum cluster @@ -19283,6 +21862,12 @@ CUresult CUDAAPI cuOccupancyMaxPotentialClusterSize(int *clusterSize, CUfunction * calculation. Runtime environment may affect how the hardware schedules * the clusters, so the calculated occupancy is not guaranteed to be achievable. * + * Note that the API can also be used with context-less kernel ::CUkernel + * by querying the handle using ::cuLibraryGetKernel() and then passing it + * to the API by casting to ::CUfunction. Here, the context to use for calculations + * will either be taken from the specified stream \p config->hStream + * or the current context in case of NULL stream. + * * \param numClusters - Returned maximum number of clusters that * could co-exist on the target device * \param func - Kernel function for which maximum number @@ -20247,7 +22832,8 @@ __CUDA_DEPRECATED CUresult CUDAAPI cuSurfRefGetArray(CUarray *phArray, CUsurfref * \p pResViewDesc is an optional argument that specifies an alternate format for * the data described by \p pResDesc, and also describes the subresource region * to restrict access to when texturing. \p pResViewDesc can only be specified if - * the type of resource is a CUDA array or a CUDA mipmapped array. + * the type of resource is a CUDA array or a CUDA mipmapped array not in a block + * compressed format. * * Texture objects are only supported on devices of compute capability 3.0 or higher. * Additionally, a texture object is an opaque value, and, as such, should only be @@ -20655,7 +23241,7 @@ CUresult CUDAAPI cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC *pResDesc, CUsur * * Tensor map objects are only supported on devices of compute capability 9.0 or higher. * Additionally, a tensor map object is an opaque value, and, as such, should only be - * accessed through CUDA API calls. + * accessed through CUDA APIs and PTX. * * The parameters passed are bound to the following requirements: * @@ -20676,38 +23262,50 @@ CUresult CUDAAPI cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC *pResDesc, CUsur CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, // 2 bytes CU_TENSOR_MAP_DATA_TYPE_FLOAT32_FTZ, // 4 bytes CU_TENSOR_MAP_DATA_TYPE_TFLOAT32, // 4 bytes - CU_TENSOR_MAP_DATA_TYPE_TFLOAT32_FTZ // 4 bytes + CU_TENSOR_MAP_DATA_TYPE_TFLOAT32_FTZ, // 4 bytes + CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, // 4 bits + CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, // 4 bits + CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B // 6 bits } CUtensorMapDataType; * \endcode + * ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B copies '16 x U4' packed values to memory aligned as 8 bytes. There are no gaps between packed values. + * ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B copies '16 x U4' packed values to memory aligned as 16 bytes. There are 8 byte gaps between every 8 byte chunk of packed values. + * ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B copies '16 x U6' packed values to memory aligned as 16 bytes. There are 4 byte gaps between every 12 byte chunk of packed values. * * - \p tensorRank must be non-zero and less than or equal to the maximum supported dimensionality of 5. If \p interleave is not * ::CU_TENSOR_MAP_INTERLEAVE_NONE, then \p tensorRank must additionally be greater than or equal to 3. * - * - \p globalAddress, which specifies the starting address of the memory region described, must be 32 byte aligned when \p interleave is - * ::CU_TENSOR_MAP_INTERLEAVE_32B and 16 byte aligned otherwise. + * - \p globalAddress, which specifies the starting address of the memory region described, must be 16 byte aligned. The following requirements need to also be met: + * - When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B, \p globalAddress must be 32 byte aligned. + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, \p globalAddress must be 32 byte aligned. * - * - \p globalDim array, which specifies tensor size of each of the \p tensorRank dimensions, must be non-zero and less than or - * equal to 2^32. + * - \p globalDim array, which specifies tensor size of each of the \p tensorRank dimensions, must be non-zero and less than or + * equal to 2^32. Additionally, the following requirements need to be met for the packed data types: + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, globalDim[0] must be a multiple of 128. + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, \p globalDim[0] must be a multiple of 2. + * - Dimension for the packed data types must reflect the number of individual U# values. * * - \p globalStrides array, which specifies tensor stride of each of the lower \p tensorRank - 1 dimensions in bytes, must be a - * multiple of 16 and less than 2^40. Additionally, the stride must be a multiple of 32 when \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B. + * multiple of 16 and less than 2^40. Additionally, the following requirements need to be met: + * - When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B, the strides must be a multiple of 32. + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, the strides must be a multiple of 32. * Each following dimension specified includes previous dimension stride: * \code globalStrides[0] = globalDim[0] * elementSizeInBytes(tensorDataType) + padding[0]; for (i = 1; i < tensorRank - 1; i++) - globalStrides[i] = globalStrides[i – 1] * globalStrides[i] + padding[i]; - assert(globalStrides[i] >= globalDim[i]); + globalStrides[i] = globalStrides[i – 1] * (globalDim[i] + padding[i]); + assert(globalStrides[i] >= globalDim[i]); * \endcode * - * - \p boxDim array, which specifies number of elements to be traversed along each of the \p tensorRank dimensions, must be less - * than or equal to 8. - * When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_NONE, { \p boxDim[0] * elementSizeInBytes( \p tensorDataType ) } must be a multiple - * of 16 bytes. + * - \p boxDim array, which specifies number of elements to be traversed along each of the \p tensorRank dimensions, must be non-zero + * and less than or equal to 256. Additionally, the following requirements need to be met: + * - When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_NONE, { \p boxDim[0] * elementSizeInBytes( \p tensorDataType ) } must be a multiple of 16 bytes. + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, boxDim[0] must be 128. * * - \p elementStrides array, which specifies the iteration step along each of the \p tensorRank dimensions, must be non-zero and less * than or equal to 8. Note that when \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_NONE, the first element of this array is ignored since * TMA doesn’t support the stride for dimension zero. - * When all elemets of \p elementStrides array is one, \p boxDim specifies the number of elements to load. However, if the \p elementStrides[i] + * When all elements of \p elementStrides array is one, \p boxDim specifies the number of elements to load. However, if the \p elementStrides[i] * is not equal to one, then TMA loads ceil( \p boxDim[i] / \p elementStrides[i]) number of elements along i-th dimension. To load N elements along * i-th dimension, \p boxDim[i] must be set to N * \p elementStrides[i]. * @@ -20723,24 +23321,37 @@ CUresult CUDAAPI cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC *pResDesc, CUsur * uses 32 bytes. * When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_NONE and \p swizzle is not ::CU_TENSOR_MAP_SWIZZLE_NONE, the bounding box inner dimension * (computed as \p boxDim[0] multiplied by element size derived from \p tensorDataType) must be less than or equal to the swizzle size. - * - CU_TENSOR_MAP_SWIZZLE_32B implies the bounding box inner dimension will be <= 32. - * - CU_TENSOR_MAP_SWIZZLE_64B implies the bounding box inner dimension will be <= 64. - * - CU_TENSOR_MAP_SWIZZLE_128B implies the bounding box inner dimension will be <= 128. + * - CU_TENSOR_MAP_SWIZZLE_32B requires the bounding box inner dimension to be <= 32. + * - CU_TENSOR_MAP_SWIZZLE_64B requires the bounding box inner dimension to be <= 64. + * - CU_TENSOR_MAP_SWIZZLE_128B* require the bounding box inner dimension to be <= 128. + * Additionally, \p tensorDataType of ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B requires \p interleave to be ::CU_TENSOR_MAP_INTERLEAVE_NONE. * * - \p swizzle, which specifies the shared memory bank swizzling pattern, has to be of type ::CUtensorMapSwizzle which is defined as: * \code typedef enum CUtensorMapSwizzle_enum { CU_TENSOR_MAP_SWIZZLE_NONE = 0, - CU_TENSOR_MAP_SWIZZLE_32B, - CU_TENSOR_MAP_SWIZZLE_64B, - CU_TENSOR_MAP_SWIZZLE_128B + CU_TENSOR_MAP_SWIZZLE_32B, // Swizzle 16B chunks within 32B span + CU_TENSOR_MAP_SWIZZLE_64B, // Swizzle 16B chunks within 64B span + CU_TENSOR_MAP_SWIZZLE_128B, // Swizzle 16B chunks within 128B span + CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B, // Swizzle 32B chunks within 128B span + CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B_FLIP_8B, // Swizzle 32B chunks within 128B span, additionally swap lower 8B with upper 8B within each 16B for every alternate row + CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B // Swizzle 64B chunks within 128B span } CUtensorMapSwizzle; * \endcode - * Data is organized in specific order in global memory; however, it may not match the order in which data are accessed by application in - * the shared memory. This difference in data organization may cause bank conflicts when shared memory is accessed. In order to avoid this - * problem, data can be loaded to shard memory with shuffling across shared memory banks. - * Note that it’s expected that when \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B, \p swizzle should be ::CU_TENSOR_MAP_SWIZZLE_32B mode. - * Other interleave modes can have any swizzling patterns. + * Data are organized in a specific order in global memory; however, this may not match the order in which the application accesses data + * in shared memory. This difference in data organization may cause bank conflicts when shared memory is accessed. In order to avoid this + * problem, data can be loaded to shared memory with shuffling across shared memory banks. + * When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B, \p swizzle must be ::CU_TENSOR_MAP_SWIZZLE_32B. + * Other interleave modes can have any swizzling pattern. + * When the \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B, only the following swizzle modes are supported: + * - CU_TENSOR_MAP_SWIZZLE_NONE (Load & Store) + * - CU_TENSOR_MAP_SWIZZLE_128B (Load & Store) + * - CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B (Load & Store) + * - CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B (Store only) + * When the \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, only the following swizzle modes are supported: + * - CU_TENSOR_MAP_SWIZZLE_NONE (Load only) + * - CU_TENSOR_MAP_SWIZZLE_128B (Load only) + * - CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B (Load only) * * - \p l2Promotion specifies L2 fetch size which indicates the byte granurality at which L2 requests is filled from DRAM. It must be of * type ::CUtensorMapL2promotion, which is defined as: @@ -20761,7 +23372,8 @@ CUresult CUDAAPI cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC *pResDesc, CUsur CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA } CUtensorMapFloatOOBfill; * \endcode - * Note that ::CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA can only be used when \p tensorDataType represents a floating data type. + * Note that ::CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA can only be used when \p tensorDataType represents a floating-point data type, + * and when \p tensorDataType is not ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, and ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B. * * \param tensorMap - Tensor map object to create * \param tensorDataType - Tensor data type @@ -20785,11 +23397,11 @@ CUresult CUDAAPI cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC *pResDesc, CUsur * * \sa * ::cuTensorMapEncodeIm2col, + * ::cuTensorMapEncodeIm2colWide, * ::cuTensorMapReplaceAddress */ CUresult CUDAAPI cuTensorMapEncodeTiled(CUtensorMap *tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void *globalAddress, const cuuint64_t *globalDim, const cuuint64_t *globalStrides, const cuuint32_t *boxDim, const cuuint32_t *elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill); - /** * \brief Create a tensor map descriptor object representing im2col memory region * @@ -20798,7 +23410,7 @@ CUresult CUDAAPI cuTensorMapEncodeTiled(CUtensorMap *tensorMap, CUtensorMapDataT * * Tensor map objects are only supported on devices of compute capability 9.0 or higher. * Additionally, a tensor map object is an opaque value, and, as such, should only be - * accessed through CUDA API calls. + * accessed through CUDA APIs and PTX. * * The parameters passed are bound to the following requirements: * @@ -20820,41 +23432,54 @@ CUresult CUDAAPI cuTensorMapEncodeTiled(CUtensorMap *tensorMap, CUtensorMapDataT CU_TENSOR_MAP_DATA_TYPE_FLOAT32_FTZ, // 4 bytes CU_TENSOR_MAP_DATA_TYPE_TFLOAT32, // 4 bytes CU_TENSOR_MAP_DATA_TYPE_TFLOAT32_FTZ // 4 bytes + CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, // 4 bits + CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, // 4 bits + CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B // 6 bits } CUtensorMapDataType; * \endcode + * ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B copies '16 x U4' packed values to memory aligned as 8 bytes. There are no gaps between packed values. + * ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B copies '16 x U4' packed values to memory aligned as 16 bytes. There are 8 byte gaps between every 8 byte chunk of packed values. + * ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B copies '16 x U6' packed values to memory aligned as 16 bytes. There are 4 byte gaps between every 12 byte chunk of packed values. * - * - \p tensorRank must be one of dimensions 3, 4, or 5. + * - \p tensorRank, which specifies the number of tensor dimensions, must be 3, 4, or 5. * - * - \p globalAddress, which specifies the starting address of the memory region described, must be 32 byte aligned when \p interleave is - * ::CU_TENSOR_MAP_INTERLEAVE_32B and 16 byte aligned otherwise. + * - \p globalAddress, which specifies the starting address of the memory region described, must be 16 byte aligned. The following requirements need to also be met: + * - When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B, \p globalAddress must be 32 byte aligned. + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, \p globalAddress must be 32 byte aligned. * * - \p globalDim array, which specifies tensor size of each of the \p tensorRank dimensions, must be non-zero and less than or - * equal to 2^32. + * equal to 2^32. Additionally, the following requirements need to be met for the packed data types: + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, globalDim[0] must be a multiple of 128. + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, \p globalDim[0] must be a multiple of 2. + * - Dimension for the packed data types must reflect the number of individual U# values. * * - \p globalStrides array, which specifies tensor stride of each of the lower \p tensorRank - 1 dimensions in bytes, must be a - * multiple of 16 and less than 2^40. Additionally, the stride must be a multiple of 32 when \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B. + * multiple of 16 and less than 2^40. Additionally, the following requirements need to be met: + * - When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B, the strides must be a multiple of 32. + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, the strides must be a multiple of 32. * Each following dimension specified includes previous dimension stride: * \code globalStrides[0] = globalDim[0] * elementSizeInBytes(tensorDataType) + padding[0]; for (i = 1; i < tensorRank - 1; i++) - globalStrides[i] = globalStrides[i – 1] * globalStrides[i] + padding[i]; - assert(globalStrides[i] >= globalDim[i]); + globalStrides[i] = globalStrides[i – 1] * (globalDim[i] + padding[i]); + assert(globalStrides[i] >= globalDim[i]); * \endcode * * - \p pixelBoxLowerCorner array specifies the coordinate offsets {D, H, W} of the bounding box from top/left/front corner. The number of - * offsets and their precision depends on the tensor dimensionality: + * offsets and their precision depend on the tensor dimensionality: * - When \p tensorRank is 3, one signed offset within range [-32768, 32767] is supported. * - When \p tensorRank is 4, two signed offsets each within range [-128, 127] are supported. * - When \p tensorRank is 5, three offsets each within range [-16, 15] are supported. * * - \p pixelBoxUpperCorner array specifies the coordinate offsets {D, H, W} of the bounding box from bottom/right/back corner. The number of - * offsets and their precision depends on the tensor dimensionality: + * offsets and their precision depend on the tensor dimensionality: * - When \p tensorRank is 3, one signed offset within range [-32768, 32767] is supported. * - When \p tensorRank is 4, two signed offsets each within range [-128, 127] are supported. * - When \p tensorRank is 5, three offsets each within range [-16, 15] are supported. * The bounding box specified by \p pixelBoxLowerCorner and \p pixelBoxUpperCorner must have non-zero area. * * - \p channelsPerPixel, which specifies the number of elements which must be accessed along C dimension, must be less than or equal to 256. + * Additionally, when \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, \p channelsPerPixel must be 128. * * - \p pixelsPerColumn, which specifies the number of elements that must be accessed along the {N, D, H, W} dimensions, must be less than or * equal to 1024. @@ -20862,9 +23487,9 @@ CUresult CUDAAPI cuTensorMapEncodeTiled(CUtensorMap *tensorMap, CUtensorMapDataT * - \p elementStrides array, which specifies the iteration step along each of the \p tensorRank dimensions, must be non-zero and less * than or equal to 8. Note that when \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_NONE, the first element of this array is ignored since * TMA doesn’t support the stride for dimension zero. - * When all elemets of \p elementStrides array is one, \p boxDim specifies the number of elements to load. However, if the \p elementStrides[i] - * is not equal to one, then TMA loads ceil( \p boxDim[i] / \p elementStrides[i]) number of elements along i-th dimension. To load N elements along - * i-th dimension, \p boxDim[i] must be set to N * \p elementStrides[i]. + * When all elements of the \p elementStrides array are one, \p boxDim specifies the number of elements to load. However, if \p elementStrides[i] + * is not equal to one for some \p i, then TMA loads ceil( \p boxDim[i] / \p elementStrides[i]) number of elements along i-th dimension. + * To load N elements along i-th dimension, \p boxDim[i] must be set to N * \p elementStrides[i]. * * - \p interleave specifies the interleaved layout of type ::CUtensorMapInterleave, which is defined as: * \code @@ -20877,27 +23502,40 @@ CUresult CUDAAPI cuTensorMapEncodeTiled(CUtensorMap *tensorMap, CUtensorMapDataT * TMA supports interleaved layouts like NC/8HWC8 where C8 utilizes 16 bytes in memory assuming 2 byte per channel or NC/16HWC16 where C16 * uses 32 bytes. * When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_NONE and \p swizzle is not ::CU_TENSOR_MAP_SWIZZLE_NONE, the bounding box inner dimension - * (computed as \p boxDim[0] multiplied by element size derived from \p tensorDataType) must be less than or equal to the swizzle size. - * - CU_TENSOR_MAP_SWIZZLE_32B implies the bounding box inner dimension will be <= 32. - * - CU_TENSOR_MAP_SWIZZLE_64B implies the bounding box inner dimension will be <= 64. - * - CU_TENSOR_MAP_SWIZZLE_128B implies the bounding box inner dimension will be <= 128. + * (computed as \p channelsPerPixel multiplied by element size in bytes derived from \p tensorDataType) must be less than or equal to the swizzle size. + * - CU_TENSOR_MAP_SWIZZLE_32B requires the bounding box inner dimension to be <= 32. + * - CU_TENSOR_MAP_SWIZZLE_64B requires the bounding box inner dimension to be <= 64. + * - CU_TENSOR_MAP_SWIZZLE_128B* require the bounding box inner dimension to be <= 128. + * Additionally, \p tensorDataType of ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B requires \p interleave to be ::CU_TENSOR_MAP_INTERLEAVE_NONE. * * - \p swizzle, which specifies the shared memory bank swizzling pattern, has to be of type ::CUtensorMapSwizzle which is defined as: * \code typedef enum CUtensorMapSwizzle_enum { CU_TENSOR_MAP_SWIZZLE_NONE = 0, - CU_TENSOR_MAP_SWIZZLE_32B, - CU_TENSOR_MAP_SWIZZLE_64B, - CU_TENSOR_MAP_SWIZZLE_128B + CU_TENSOR_MAP_SWIZZLE_32B, // Swizzle 16B chunks within 32B span + CU_TENSOR_MAP_SWIZZLE_64B, // Swizzle 16B chunks within 64B span + CU_TENSOR_MAP_SWIZZLE_128B, // Swizzle 16B chunks within 128B span + CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B, // Swizzle 32B chunks within 128B span + CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B_FLIP_8B, // Swizzle 32B chunks within 128B span, additionally swap lower 8B with upper 8B within each 16B for every alternate row + CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B // Swizzle 64B chunks within 128B span } CUtensorMapSwizzle; * \endcode - * Data is organized in specific order in global memory; however, it may not match the order in which data are accessed by application in - * the shared memory. This difference in data organization may cause bank conflicts when shared memory is accessed. In order to avoid this - * problem, data can be loaded to shard memory with shuffling across shared memory banks. - * Note that it’s expected that when \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B, \p swizzle should be ::CU_TENSOR_MAP_SWIZZLE_32B mode. - * Other interleave modes can have any swizzling patterns. - * - * - \p l2Promotion specifies L2 fetch size which indicates the byte granurality at which L2 requests is filled from DRAM. It must be of + * Data are organized in a specific order in global memory; however, this may not match the order in which the application accesses data + * in shared memory. This difference in data organization may cause bank conflicts when shared memory is accessed. In order to avoid this + * problem, data can be loaded to shared memory with shuffling across shared memory banks. + * When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B, \p swizzle must be ::CU_TENSOR_MAP_SWIZZLE_32B. + * Other interleave modes can have any swizzling pattern. + * When the \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B, only the following swizzle modes are supported: + * - CU_TENSOR_MAP_SWIZZLE_NONE (Load & Store) + * - CU_TENSOR_MAP_SWIZZLE_128B (Load & Store) + * - CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B (Load & Store) + * - CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B (Store only) + * When the \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, only the following swizzle modes are supported: + * - CU_TENSOR_MAP_SWIZZLE_NONE (Load only) + * - CU_TENSOR_MAP_SWIZZLE_128B (Load only) + * - CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B (Load only) + * + * - \p l2Promotion specifies L2 fetch size which indicates the byte granularity at which L2 requests are filled from DRAM. It must be of * type ::CUtensorMapL2promotion, which is defined as: * \code typedef enum CUtensorMapL2promotion_enum { @@ -20916,23 +23554,24 @@ CUresult CUDAAPI cuTensorMapEncodeTiled(CUtensorMap *tensorMap, CUtensorMapDataT CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA } CUtensorMapFloatOOBfill; * \endcode - * Note that ::CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA can only be used when \p tensorDataType represents a floating data type. + * Note that ::CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA can only be used when \p tensorDataType represents a floating-point data type, + * and when \p tensorDataType is not ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, and ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B. * * \param tensorMap - Tensor map object to create * \param tensorDataType - Tensor data type - * \param tensorRank - Dimensionality of tensor, needs to be at least of dimension 3 + * \param tensorRank - Dimensionality of tensor; must be at least 3 * \param globalAddress - Starting address of memory region described by tensor * \param globalDim - Array containing tensor size (number of elements) along each of the \p tensorRank dimensions * \param globalStrides - Array containing stride size (in bytes) along each of the \p tensorRank - 1 dimensions - * \param pixelBoxLowerCorner - Array containing DHW dimentions of lower box corner - * \param pixelBoxUpperCorner - Array containing DHW dimentions of upper box corner + * \param pixelBoxLowerCorner - Array containing DHW dimensions of lower box corner + * \param pixelBoxUpperCorner - Array containing DHW dimensions of upper box corner * \param channelsPerPixel - Number of channels per pixel * \param pixelsPerColumn - Number of pixels per column * \param elementStrides - Array containing traversal stride in each of the \p tensorRank dimensions * \param interleave - Type of interleaved layout the tensor addresses * \param swizzle - Bank swizzling pattern inside shared memory * \param l2Promotion - L2 promotion size - * \param oobFill - Indicate whether zero or special NaN constant must be used to fill out-of-bound elements + * \param oobFill - Indicate whether zero or special NaN constant will be used to fill out-of-bound elements * * \return * ::CUDA_SUCCESS, @@ -20943,10 +23582,195 @@ CUresult CUDAAPI cuTensorMapEncodeTiled(CUtensorMap *tensorMap, CUtensorMapDataT * * \sa * ::cuTensorMapEncodeTiled, + * ::cuTensorMapEncodeIm2colWide, * ::cuTensorMapReplaceAddress */ CUresult CUDAAPI cuTensorMapEncodeIm2col(CUtensorMap *tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void *globalAddress, const cuuint64_t *globalDim, const cuuint64_t *globalStrides, const int *pixelBoxLowerCorner, const int *pixelBoxUpperCorner, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t *elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill); +/** + * \brief Create a tensor map descriptor object representing im2col memory region, but where + * the elements are exclusively loaded along the W dimension. + * + * Creates a descriptor for Tensor Memory Access (TMA) object specified by the parameters + * describing a im2col memory layout and where the row is always loaded along the W dimensuin + * and returns it in \p tensorMap. This assumes the tensor layout in memory is either NDHWC, + * NHWC, or NWC. + * + * This API is only supported on devices of compute capability 10.0 or higher. + * Additionally, a tensor map object is an opaque value, and, as such, should only be + * accessed through CUDA APIs and PTX. + * + * The parameters passed are bound to the following requirements: + * + * - \p tensorMap address must be aligned to 64 bytes. + * + * - \p tensorDataType has to be an enum from ::CUtensorMapDataType which is defined as: + * \code + typedef enum CUtensorMapDataType_enum { + CU_TENSOR_MAP_DATA_TYPE_UINT8 = 0, // 1 byte + CU_TENSOR_MAP_DATA_TYPE_UINT16, // 2 bytes + CU_TENSOR_MAP_DATA_TYPE_UINT32, // 4 bytes + CU_TENSOR_MAP_DATA_TYPE_INT32, // 4 bytes + CU_TENSOR_MAP_DATA_TYPE_UINT64, // 8 bytes + CU_TENSOR_MAP_DATA_TYPE_INT64, // 8 bytes + CU_TENSOR_MAP_DATA_TYPE_FLOAT16, // 2 bytes + CU_TENSOR_MAP_DATA_TYPE_FLOAT32, // 4 bytes + CU_TENSOR_MAP_DATA_TYPE_FLOAT64, // 8 bytes + CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, // 2 bytes + CU_TENSOR_MAP_DATA_TYPE_FLOAT32_FTZ, // 4 bytes + CU_TENSOR_MAP_DATA_TYPE_TFLOAT32, // 4 bytes + CU_TENSOR_MAP_DATA_TYPE_TFLOAT32_FTZ // 4 bytes + CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, // 4 bits + CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, // 4 bits + CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B // 6 bits + } CUtensorMapDataType; + * \endcode + * ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B copies '16 x U4' packed values to memory aligned as 8 bytes. There are no gaps between packed values. + * ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B copies '16 x U4' packed values to memory aligned as 16 bytes. There are 8 byte gaps between every 8 byte chunk of packed values. + * ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B copies '16 x U6' packed values to memory aligned as 16 bytes. There are 4 byte gaps between every 12 byte chunk of packed values. + * + * - \p tensorRank, which specifies the number of tensor dimensions, must be 3, 4, or 5. + * + * - \p globalAddress, which specifies the starting address of the memory region described, must be 16 byte aligned. The following requirements need to also be met: + * - When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B, \p globalAddress must be 32 byte aligned. + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, \p globalAddress must be 32 byte aligned. + * + * - \p globalDim array, which specifies tensor size of each of the \p tensorRank dimensions, must be non-zero and less than or + * equal to 2^32. Additionally, the following requirements need to be met for the packed data types: + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, globalDim[0] must be a multiple of 128. + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, \p globalDim[0] must be a multiple of 2. + * - Dimension for the packed data types must reflect the number of individual U# values. + * + * - \p globalStrides array, which specifies tensor stride of each of the lower \p tensorRank - 1 dimensions in bytes, must be a + * multiple of 16 and less than 2^40. Additionally, the following requirements need to be met: + * - When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_32B, the strides must be a multiple of 32. + * - When \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, the strides must be a multiple of 32. + * Each following dimension specified includes previous dimension stride: + * \code + globalStrides[0] = globalDim[0] * elementSizeInBytes(tensorDataType) + padding[0]; + for (i = 1; i < tensorRank - 1; i++) + globalStrides[i] = globalStrides[i – 1] * (globalDim[i] + padding[i]); + assert(globalStrides[i] >= globalDim[i]); + * \endcode + * + * - \p pixelBoxLowerCornerWidth specifies the coordinate offset W of the bounding box from left corner. The offset must be + * within range [-32768, 32767]. + * + * - \p pixelBoxUpperCornerWidth specifies the coordinate offset W of the bounding box from right corner. The offset must be + * within range [-32768, 32767]. + * + * The bounding box specified by \p pixelBoxLowerCornerWidth and \p pixelBoxUpperCornerWidth must have non-zero area. Note + * that the size of the box along D and H dimensions is always equal to one. + * + * - \p channelsPerPixel, which specifies the number of elements which must be accessed along C dimension, must be less than or equal to 256. + * Additionally, when \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B or ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, \p channelsPerPixel must be 128. + * + * - \p pixelsPerColumn, which specifies the number of elements that must be accessed along the W dimension, must be less than or + * equal to 1024. This field is ignored when \p mode is ::CU_TENSOR_MAP_IM2COL_WIDE_MODE_W128. + * + * - \p elementStrides array, which specifies the iteration step along each of the \p tensorRank dimensions, must be non-zero and less + * than or equal to 8. Note that when \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_NONE, the first element of this array is ignored since + * TMA doesn’t support the stride for dimension zero. + * When all elements of the \p elementStrides array are one, \p boxDim specifies the number of elements to load. However, if \p elementStrides[i] + * is not equal to one for some \p i, then TMA loads ceil( \p boxDim[i] / \p elementStrides[i]) number of elements along i-th dimension. + * To load N elements along i-th dimension, \p boxDim[i] must be set to N * \p elementStrides[i]. + * + * - \p interleave specifies the interleaved layout of type ::CUtensorMapInterleave, which is defined as: + * \code + typedef enum CUtensorMapInterleave_enum { + CU_TENSOR_MAP_INTERLEAVE_NONE = 0, + CU_TENSOR_MAP_INTERLEAVE_16B, + CU_TENSOR_MAP_INTERLEAVE_32B + } CUtensorMapInterleave; + * \endcode + * TMA supports interleaved layouts like NC/8HWC8 where C8 utilizes 16 bytes in memory assuming 2 byte per channel or NC/16HWC16 where C16 + * uses 32 bytes. + * When \p interleave is ::CU_TENSOR_MAP_INTERLEAVE_NONE, the bounding box inner dimension (computed as \p channelsPerPixel multiplied by + * element size in bytes derived from \p tensorDataType) must be less than or equal to the swizzle size. + * - CU_TENSOR_MAP_SWIZZLE_64B requires the bounding box inner dimension to be <= 64. + * - CU_TENSOR_MAP_SWIZZLE_128B* require the bounding box inner dimension to be <= 128. + * Additionally, \p tensorDataType of ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B requires \p interleave to be ::CU_TENSOR_MAP_INTERLEAVE_NONE. + * + * - \p mode, which describes loading of elements loaded along the W dimension, has to be one of the following ::CUtensorMapIm2ColWideMode types: + * \code + * CU_TENSOR_MAP_IM2COL_WIDE_MODE_W, + * CU_TENSOR_MAP_IM2COL_WIDE_MODE_W128 + * \endcode + * ::CU_TENSOR_MAP_IM2COL_WIDE_MODE_W allows the number of elements loaded along the W dimension to be specified + * via the \p pixelsPerColumn field. + * + * - \p swizzle, which specifies the shared memory bank swizzling pattern, must be one of the following + * ::CUtensorMapSwizzle modes (other swizzle modes are not supported): + * \code + typedef enum CUtensorMapSwizzle_enum { + CU_TENSOR_MAP_SWIZZLE_64B, // Swizzle 16B chunks within 64B span + CU_TENSOR_MAP_SWIZZLE_128B, // Swizzle 16B chunks within 128B span + CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B, // Swizzle 32B chunks within 128B span + } CUtensorMapSwizzle; + * \endcode + * Data are organized in a specific order in global memory; however, this may not match the order in which the application accesses data + * in shared memory. This difference in data organization may cause bank conflicts when shared memory is accessed. In order to avoid this + * problem, data can be loaded to shared memory with shuffling across shared memory banks. + * When the \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B, only the following swizzle modes are supported: + * - CU_TENSOR_MAP_SWIZZLE_128B (Load & Store) + * - CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B (Load & Store) + * When the \p tensorDataType is ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, only the following swizzle modes are supported: + * - CU_TENSOR_MAP_SWIZZLE_128B (Load only) + * - CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B (Load only) + * + * - \p l2Promotion specifies L2 fetch size which indicates the byte granularity at which L2 requests are filled from DRAM. It must be of + * type ::CUtensorMapL2promotion, which is defined as: + * \code + typedef enum CUtensorMapL2promotion_enum { + CU_TENSOR_MAP_L2_PROMOTION_NONE = 0, + CU_TENSOR_MAP_L2_PROMOTION_L2_64B, + CU_TENSOR_MAP_L2_PROMOTION_L2_128B, + CU_TENSOR_MAP_L2_PROMOTION_L2_256B + } CUtensorMapL2promotion; + * \endcode + * + * - \p oobFill, which indicates whether zero or a special NaN constant should be used to fill out-of-bound elements, must be of type + * ::CUtensorMapFloatOOBfill which is defined as: + * \code + typedef enum CUtensorMapFloatOOBfill_enum { + CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE = 0, + CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA + } CUtensorMapFloatOOBfill; + * \endcode + * Note that ::CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA can only be used when \p tensorDataType represents a floating-point data type, + * and when \p tensorDataType is not ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, ::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B, and ::CU_TENSOR_MAP_DATA_TYPE_16U6_ALIGN16B. + * + * \param tensorMap - Tensor map object to create + * \param tensorDataType - Tensor data type + * \param tensorRank - Dimensionality of tensor; must be at least 3 + * \param globalAddress - Starting address of memory region described by tensor + * \param globalDim - Array containing tensor size (number of elements) along each of the \p tensorRank dimensions + * \param globalStrides - Array containing stride size (in bytes) along each of the \p tensorRank - 1 dimensions + * \param pixelBoxLowerCornerWidth - Width offset of left box corner + * \param pixelBoxUpperCornerWidth - Width offset of right box corner + * \param channelsPerPixel - Number of channels per pixel + * \param pixelsPerColumn - Number of pixels per column + * \param elementStrides - Array containing traversal stride in each of the \p tensorRank dimensions + * \param interleave - Type of interleaved layout the tensor addresses + * \param mode - W or W128 mode + * \param swizzle - Bank swizzling pattern inside shared memory + * \param l2Promotion - L2 promotion size + * \param oobFill - Indicate whether zero or special NaN constant will be used to fill out-of-bound elements + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE + * + * \sa + * ::cuTensorMapEncodeTiled, + * ::cuTensorMapEncodeIm2col, + * ::cuTensorMapReplaceAddress + */ +CUresult CUDAAPI cuTensorMapEncodeIm2colWide(CUtensorMap *tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void *globalAddress, const cuuint64_t *globalDim, const cuuint64_t *globalStrides, int pixelBoxLowerCornerWidth, int pixelBoxUpperCornerWidth, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t *elementStrides, CUtensorMapInterleave interleave, CUtensorMapIm2ColWideMode mode, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill); + /** * \brief Modify an existing tensor map descriptor with an updated global address * @@ -20969,7 +23793,8 @@ CUresult CUDAAPI cuTensorMapEncodeIm2col(CUtensorMap *tensorMap, CUtensorMapData * * \sa * ::cuTensorMapEncodeTiled, - * ::cuTensorMapEncodeIm2col + * ::cuTensorMapEncodeIm2col, + * ::cuTensorMapEncodeIm2colWide */ CUresult CUDAAPI cuTensorMapReplaceAddress(CUtensorMap *tensorMap, void *globalAddress); @@ -21104,10 +23929,13 @@ CUresult CUDAAPI cuCtxDisablePeerAccess(CUcontext peerContext); * - ::CU_DEVICE_P2P_ATTRIBUTE_PERFORMANCE_RANK: A relative value indicating the * performance of the link between two devices. * - ::CU_DEVICE_P2P_ATTRIBUTE_ACCESS_SUPPORTED P2P: 1 if P2P Access is enable. - * - ::CU_DEVICE_P2P_ATTRIBUTE_NATIVE_ATOMIC_SUPPORTED: 1 if Atomic operations over - * the link are supported. + * - ::CU_DEVICE_P2P_ATTRIBUTE_NATIVE_ATOMIC_SUPPORTED: 1 if all CUDA-valid atomic + * operations over the link are supported. * - ::CU_DEVICE_P2P_ATTRIBUTE_CUDA_ARRAY_ACCESS_SUPPORTED: 1 if cudaArray can * be accessed over the link. + * - ::CU_DEVICE_P2P_ATTRIBUTE_ONLY_PARTIAL_NATIVE_ATOMIC_SUPPORTED: 1 if some + * CUDA-valid atomic operations over the link are supported. Information about + * specific operations can be retrieved with ::cuDeviceGetP2PAtomicCapabilities. * * Returns ::CUDA_ERROR_INVALID_DEVICE if \p srcDevice or \p dstDevice are not valid * or if they represent the same device. @@ -21132,22 +23960,59 @@ CUresult CUDAAPI cuCtxDisablePeerAccess(CUcontext peerContext); * ::cuCtxEnablePeerAccess, * ::cuCtxDisablePeerAccess, * ::cuDeviceCanAccessPeer, + * ::cuDeviceGetP2PAtomicCapabilities, * ::cudaDeviceGetP2PAttribute */ CUresult CUDAAPI cuDeviceGetP2PAttribute(int* value, CUdevice_P2PAttribute attrib, CUdevice srcDevice, CUdevice dstDevice); -/** @} */ /* END CUDA_PEER_ACCESS */ /** - * \defgroup CUDA_GRAPHICS Graphics Interoperability + * \brief Queries details about atomic operations supported between two devices * - * ___MANBRIEF___ graphics interoperability functions of the low-level CUDA - * driver API (___CURRENT_FILE___) ___ENDMANBRIEF___ + * Returns in \p *capabilities the details about requested atomic \p *operations over the + * the link between \p srcDevice and \p dstDevice. The allocated size of \p *operations and + * \p *capabilities must be \p count. * - * This section describes the graphics interoperability functions of the - * low-level CUDA driver application programming interface. + * For each ::CUatomicOperation in \p *operations, the corresponding result in \p *capabilities + * will be a bitmask indicating which of ::CUatomicOperationCapability the link supports natively. * - * @{ + * Returns ::CUDA_ERROR_INVALID_DEVICE if \p srcDevice or \p dstDevice are not valid + * or if they represent the same device. + * + * Returns ::CUDA_ERROR_INVALID_VALUE if \p *capabilities or \p *operations is NULL, if \p count is 0, + * or if any of \p *operations is not valid. + * + * \param capabilities - Returned capability details of each requested operation + * \param operations - Requested operations + * \param count - Count of requested operations and size of capabilities + * \param srcDevice - The source device of the target link + * \param dstDevice - The destination device of the target link + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_DEVICE, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * + * \sa + * ::cuDeviceGetP2PAttribute, + * ::cudaDeviceGetP2PAttribute, + * ::cudaDeviceGetP2PAtomicCapabilities + */ +CUresult CUDAAPI cuDeviceGetP2PAtomicCapabilities(unsigned int* capabilities, const CUatomicOperation* operations, unsigned int count, CUdevice srcDevice, CUdevice dstDevice); + +/** @} */ /* END CUDA_PEER_ACCESS */ + +/** + * \defgroup CUDA_GRAPHICS Graphics Interoperability + * + * ___MANBRIEF___ graphics interoperability functions of the low-level CUDA + * driver API (___CURRENT_FILE___) ___ENDMANBRIEF___ + * + * This section describes the graphics interoperability functions of the + * low-level CUDA driver application programming interface. + * + * @{ */ /** @@ -21427,7 +24292,8 @@ CUresult CUDAAPI cuGraphicsUnmapResources(unsigned int count, CUgraphicsResource * should be specified as 11020. For a requested driver symbol, if the specified * CUDA version is greater than or equal to the CUDA version in which the driver symbol * was introduced, this API will return the function pointer to the corresponding - * versioned function. + * versioned function. If the specified CUDA version is greater than the driver + * version, the API will return ::CUDA_ERROR_INVALID_VALUE. * * The pointer returned by the API should be cast to a function pointer matching the * requested driver function's definition in the API header file. The function pointer @@ -21476,13 +24342,12 @@ CUresult CUDAAPI cuGraphicsUnmapResources(unsigned int count, CUgraphicsResource * \note_version_mixing * * \sa - * ::cudaGetDriverEntryPoint + * ::cudaGetDriverEntryPointByVersion */ CUresult CUDAAPI cuGetProcAddress(const char *symbol, void **pfn, int cudaVersion, cuuint64_t flags, CUdriverProcAddressQueryResult *symbolStatus); /** @} */ /* END CUDA_DRIVER_ENTRY_POINT */ - /** * \defgroup CUDA_COREDUMP Coredump Attributes Control API * @@ -21505,9 +24370,29 @@ typedef enum CUcoredumpSettings_enum { CU_COREDUMP_ENABLE_USER_TRIGGER, CU_COREDUMP_FILE, CU_COREDUMP_PIPE, + CU_COREDUMP_GENERATION_FLAGS, CU_COREDUMP_MAX } CUcoredumpSettings; +/** + * Flags for controlling coredump contents + */ +typedef enum CUCoredumpGenerationFlags { + CU_COREDUMP_DEFAULT_FLAGS = 0, + CU_COREDUMP_SKIP_NONRELOCATED_ELF_IMAGES = (1 << 0), + CU_COREDUMP_SKIP_GLOBAL_MEMORY = (1 << 1), + CU_COREDUMP_SKIP_SHARED_MEMORY = (1 << 2), + CU_COREDUMP_SKIP_LOCAL_MEMORY = (1 << 3), + CU_COREDUMP_SKIP_ABORT = (1 << 4), + CU_COREDUMP_SKIP_CONSTBANK_MEMORY = (1 << 5), + + CU_COREDUMP_LIGHTWEIGHT_FLAGS = CU_COREDUMP_SKIP_NONRELOCATED_ELF_IMAGES + | CU_COREDUMP_SKIP_GLOBAL_MEMORY + | CU_COREDUMP_SKIP_SHARED_MEMORY + | CU_COREDUMP_SKIP_LOCAL_MEMORY + | CU_COREDUMP_SKIP_CONSTBANK_MEMORY +} CUCoredumpGenerationFlags; + /** * \brief Allows caller to fetch a coredump attribute value for the current context * @@ -21524,10 +24409,12 @@ typedef enum CUcoredumpSettings_enum { * CU_CTX_USER_COREDUMP_ENABLE flag was set during context creation. * - ::CU_COREDUMP_TRIGGER_HOST: Bool where ::true means that the host CPU will * also create a coredump. The default value is ::true unless set to ::false globally or - * or locally. + * or locally. This value is deprecated as of CUDA 12.5 - raise the ::CU_COREDUMP_SKIP_ABORT + * flag to disable host device abort() if needed. * - ::CU_COREDUMP_LIGHTWEIGHT: Bool where ::true means that any resulting coredumps * will not have a dump of GPU memory or non-reloc ELF images. The default value is - * ::false unless set to ::true globally or locally. + * ::false unless set to ::true globally or locally. This attribute is deprecated as + * of CUDA 12.5, please use ::CU_COREDUMP_GENERATION_FLAGS instead. * - ::CU_COREDUMP_ENABLE_USER_TRIGGER: Bool where ::true means that a coredump can be * created by writing to the system pipe specified by ::CU_COREDUMP_PIPE. The default * value is ::false unless set to ::true globally or locally. @@ -21539,6 +24426,22 @@ typedef enum CUcoredumpSettings_enum { * that will be monitored if user-triggered coredumps are enabled. The default value is * ::corepipe.cuda.HOSTNAME.PID where ::HOSTNAME is the host name of the machine running * the CUDA application and ::PID is the process ID of the CUDA application. + * - ::CU_COREDUMP_GENERATION_FLAGS: An integer with values to allow granular control the data + * contained in a coredump specified as a bitwise OR combination of the following values: + * + ::CU_COREDUMP_DEFAULT_FLAGS - if set by itself, coredump generation returns to its + * default settings of including all memory regions that it is able to access + * + ::CU_COREDUMP_SKIP_NONRELOCATED_ELF_IMAGES - Coredump will not include the data from + * CUDA source modules that are not relocated at runtime. + * + ::CU_COREDUMP_SKIP_GLOBAL_MEMORY - Coredump will not include device-side global data + * that does not belong to any context. + * + ::CU_COREDUMP_SKIP_SHARED_MEMORY - Coredump will not include grid-scale shared memory + * for the warp that the dumped kernel belonged to. + * + ::CU_COREDUMP_SKIP_LOCAL_MEMORY - Coredump will not include local memory from the kernel. + * + ::CU_COREDUMP_LIGHTWEIGHT_FLAGS - Enables all of the above options. Equiavlent to setting + * the ::CU_COREDUMP_LIGHTWEIGHT attribute to ::true. + * + ::CU_COREDUMP_SKIP_ABORT - If set, GPU exceptions will not raise an abort() in the host CPU + * process. Same functional goal as ::CU_COREDUMP_TRIGGER_HOST but better reflects the default + * behavior. * * \param attrib - The enum defining which value to fetch. * \param value - void* containing the requested data. @@ -21574,10 +24477,13 @@ CUresult CUDAAPI cuCoredumpGetAttribute(CUcoredumpSettings attrib, void* value, * this context will create a coredump at the location specified by ::CU_COREDUMP_FILE. * The default value is ::false. * - ::CU_COREDUMP_TRIGGER_HOST: Bool where ::true means that the host CPU will - * also create a coredump. The default value is ::true. + * also create a coredump. The default value is ::true unless set to ::false globally or + * or locally. This value is deprecated as of CUDA 12.5 - raise the ::CU_COREDUMP_SKIP_ABORT + * flag to disable host device abort() if needed. * - ::CU_COREDUMP_LIGHTWEIGHT: Bool where ::true means that any resulting coredumps * will not have a dump of GPU memory or non-reloc ELF images. The default value is - * ::false. + * ::false. This attribute is deprecated as of CUDA 12.5, please use ::CU_COREDUMP_GENERATION_FLAGS + * instead. * - ::CU_COREDUMP_ENABLE_USER_TRIGGER: Bool where ::true means that a coredump can be * created by writing to the system pipe specified by ::CU_COREDUMP_PIPE. The default * value is ::false. @@ -21589,6 +24495,22 @@ CUresult CUDAAPI cuCoredumpGetAttribute(CUcoredumpSettings attrib, void* value, * that will be monitored if user-triggered coredumps are enabled. The default value is * ::corepipe.cuda.HOSTNAME.PID where ::HOSTNAME is the host name of the machine running * the CUDA application and ::PID is the process ID of the CUDA application. + * - ::CU_COREDUMP_GENERATION_FLAGS: An integer with values to allow granular control the data + * contained in a coredump specified as a bitwise OR combination of the following values: + * + ::CU_COREDUMP_DEFAULT_FLAGS - if set by itself, coredump generation returns to its + * default settings of including all memory regions that it is able to access + * + ::CU_COREDUMP_SKIP_NONRELOCATED_ELF_IMAGES - Coredump will not include the data from + * CUDA source modules that are not relocated at runtime. + * + ::CU_COREDUMP_SKIP_GLOBAL_MEMORY - Coredump will not include device-side global data + * that does not belong to any context. + * + ::CU_COREDUMP_SKIP_SHARED_MEMORY - Coredump will not include grid-scale shared memory + * for the warp that the dumped kernel belonged to. + * + ::CU_COREDUMP_SKIP_LOCAL_MEMORY - Coredump will not include local memory from the kernel. + * + ::CU_COREDUMP_LIGHTWEIGHT_FLAGS - Enables all of the above options. Equiavlent to setting + * the ::CU_COREDUMP_LIGHTWEIGHT attribute to ::true. + * + ::CU_COREDUMP_SKIP_ABORT - If set, GPU exceptions will not raise an abort() in the host CPU + * process. Same functional goal as ::CU_COREDUMP_TRIGGER_HOST but better reflects the default + * behavior. * * \param attrib - The enum defining which value to fetch. * \param value - void* containing the requested data. @@ -21613,7 +24535,7 @@ CUresult CUDAAPI cuCoredumpGetAttributeGlobal(CUcoredumpSettings attrib, void *v * * An important design decision to note is that any coredump environment variable values * set before CUDA initializes will take permanent precedence over any values set with this - * this function. This decision was made to ensure no change in behavior for any users that + * function. This decision was made to ensure no change in behavior for any users that * may be currently using these variables to get coredumps. * * \p *value shall contain the requested value specified by \p set. It is up to the caller @@ -21633,14 +24555,33 @@ CUresult CUDAAPI cuCoredumpGetAttributeGlobal(CUcoredumpSettings attrib, void *v * this context will create a coredump at the location specified by ::CU_COREDUMP_FILE. * The default value is ::false. * - ::CU_COREDUMP_TRIGGER_HOST: Bool where ::true means that the host CPU will - * also create a coredump. The default value is ::true. + * also create a coredump. The default value is ::true unless set to ::false globally or + * or locally. This value is deprecated as of CUDA 12.5 - raise the ::CU_COREDUMP_SKIP_ABORT + * flag to disable host device abort() if needed. * - ::CU_COREDUMP_LIGHTWEIGHT: Bool where ::true means that any resulting coredumps * will not have a dump of GPU memory or non-reloc ELF images. The default value is - * ::false. + * ::false. This attribute is deprecated as of CUDA 12.5, please use ::CU_COREDUMP_GENERATION_FLAGS + * instead. * - ::CU_COREDUMP_FILE: String of up to 1023 characters that defines the location where * any coredumps generated by this context will be written. The default value is * ::core.cuda.HOSTNAME.PID where ::HOSTNAME is the host name of the machine running * the CUDA applications and ::PID is the process ID of the CUDA application. + * - ::CU_COREDUMP_GENERATION_FLAGS: An integer with values to allow granular control the data + * contained in a coredump specified as a bitwise OR combination of the following values: + * + ::CU_COREDUMP_DEFAULT_FLAGS - if set by itself, coredump generation returns to its + * default settings of including all memory regions that it is able to access + * + ::CU_COREDUMP_SKIP_NONRELOCATED_ELF_IMAGES - Coredump will not include the data from + * CUDA source modules that are not relocated at runtime. + * + ::CU_COREDUMP_SKIP_GLOBAL_MEMORY - Coredump will not include device-side global data + * that does not belong to any context. + * + ::CU_COREDUMP_SKIP_SHARED_MEMORY - Coredump will not include grid-scale shared memory + * for the warp that the dumped kernel belonged to. + * + ::CU_COREDUMP_SKIP_LOCAL_MEMORY - Coredump will not include local memory from the kernel. + * + ::CU_COREDUMP_LIGHTWEIGHT_FLAGS - Enables all of the above options. Equiavlent to setting + * the ::CU_COREDUMP_LIGHTWEIGHT attribute to ::true. + * + ::CU_COREDUMP_SKIP_ABORT - If set, GPU exceptions will not raise an abort() in the host CPU + * process. Same functional goal as ::CU_COREDUMP_TRIGGER_HOST but better reflects the default + * behavior. * * \param attrib - The enum defining which value to set. * \param value - void* containing the requested data. @@ -21671,7 +24612,7 @@ CUresult CUDAAPI cuCoredumpSetAttribute(CUcoredumpSettings attrib, void* value, * * An important design decision to note is that any coredump environment variable values * set before CUDA initializes will take permanent precedence over any values set with this - * this function. This decision was made to ensure no change in behavior for any users that + * function. This decision was made to ensure no change in behavior for any users that * may be currently using these variables to get coredumps. * * \p *value shall contain the requested value specified by \p set. It is up to the caller @@ -21685,10 +24626,13 @@ CUresult CUDAAPI cuCoredumpSetAttribute(CUcoredumpSettings attrib, void* value, * this context will create a coredump at the location specified by ::CU_COREDUMP_FILE. * The default value is ::false. * - ::CU_COREDUMP_TRIGGER_HOST: Bool where ::true means that the host CPU will - * also create a coredump. The default value is ::true. + * also create a coredump. The default value is ::true unless set to ::false globally or + * or locally. This value is deprecated as of CUDA 12.5 - raise the ::CU_COREDUMP_SKIP_ABORT + * flag to disable host device abort() if needed. * - ::CU_COREDUMP_LIGHTWEIGHT: Bool where ::true means that any resulting coredumps * will not have a dump of GPU memory or non-reloc ELF images. The default value is - * ::false. + * ::false. This attribute is deprecated as of CUDA 12.5, please use ::CU_COREDUMP_GENERATION_FLAGS + * instead. * - ::CU_COREDUMP_ENABLE_USER_TRIGGER: Bool where ::true means that a coredump can be * created by writing to the system pipe specified by ::CU_COREDUMP_PIPE. The default * value is ::false. @@ -21701,6 +24645,22 @@ CUresult CUDAAPI cuCoredumpSetAttribute(CUcoredumpSettings attrib, void* value, * changed after ::CU_COREDUMP_ENABLE_USER_TRIGGER is set to ::true. The default * value is ::corepipe.cuda.HOSTNAME.PID where ::HOSTNAME is the host name of the machine * running the CUDA application and ::PID is the process ID of the CUDA application. + * - ::CU_COREDUMP_GENERATION_FLAGS: An integer with values to allow granular control the data + * contained in a coredump specified as a bitwise OR combination of the following values: + * + ::CU_COREDUMP_DEFAULT_FLAGS - if set by itself, coredump generation returns to its + * default settings of including all memory regions that it is able to access + * + ::CU_COREDUMP_SKIP_NONRELOCATED_ELF_IMAGES - Coredump will not include the data from + * CUDA source modules that are not relocated at runtime. + * + ::CU_COREDUMP_SKIP_GLOBAL_MEMORY - Coredump will not include device-side global data + * that does not belong to any context. + * + ::CU_COREDUMP_SKIP_SHARED_MEMORY - Coredump will not include grid-scale shared memory + * for the warp that the dumped kernel belonged to. + * + ::CU_COREDUMP_SKIP_LOCAL_MEMORY - Coredump will not include local memory from the kernel. + * + ::CU_COREDUMP_LIGHTWEIGHT_FLAGS - Enables all of the above options. Equiavlent to setting + * the ::CU_COREDUMP_LIGHTWEIGHT attribute to ::true. + * + ::CU_COREDUMP_SKIP_ABORT - If set, GPU exceptions will not raise an abort() in the host CPU + * process. Same functional goal as ::CU_COREDUMP_TRIGGER_HOST but better reflects the default + * behavior. * * \param attrib - The enum defining which value to set. * \param value - void* containing the requested data. @@ -21722,6 +24682,725 @@ CUresult CUDAAPI cuCoredumpSetAttributeGlobal(CUcoredumpSettings attrib, void *v CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExportTableId); +/* +** ******************* GREEN CONTEXTS ********************** +*/ + +/** + * \defgroup CUDA_GREEN_CONTEXTS Green Contexts + * + * ___MANBRIEF___ Driver level API for creation and manipulation of green contexts + * (___CURRENT_FILE___) ___ENDMANBRIEF___ + * + * This section describes the APIs for creation and manipulation of green contexts in the CUDA + * driver. Green contexts are a lightweight alternative to traditional contexts, with the ability + * to pass in a set of resources that they should be initialized with. This allows the developer to + * represent distinct spatial partitions of the GPU, provision resources for them, and target them + * via the same programming model that CUDA exposes (streams, kernel launches, etc.). + * + * There are 4 main steps to using these new set of APIs. + * - (1) Start with an initial set of resources, for example via ::cuDeviceGetDevResource. Only SM type is supported today. + * - (2) Partition this set of resources by providing them as input to a partition API, for example: ::cuDevSmResourceSplitByCount. + * - (3) Finalize the specification of resources by creating a descriptor via ::cuDevResourceGenerateDesc. + * - (4) Provision the resources and create a green context via ::cuGreenCtxCreate. + * + * For \p CU_DEV_RESOURCE_TYPE_SM, the partitions created have minimum SM count requirements, often rounding up and aligning the + * minCount provided to ::cuDevSmResourceSplitByCount. These requirements can be queried with ::cuDeviceGetDevResource from step (1) + * above to determine the minimum partition size (\p sm.minSmPartitionSize) and alignment granularity (\p sm.smCoscheduledAlignment). + * + * While it's recommended to use ::cuDeviceGetDevResource for accurate information, here is a guideline for each compute architecture: + * - On Compute Architecture 6.X: The minimum count is 2 SMs and must be a multiple of 2. + * - On Compute Architecture 7.X: The minimum count is 2 SMs and must be a multiple of 2. + * - On Compute Architecture 8.X: The minimum count is 4 SMs and must be a multiple of 2. + * - On Compute Architecture 9.0+: The minimum count is 8 SMs and must be a multiple of 8. + * + * In the future, flags can be provided to tradeoff functional and performance characteristics versus finer grained SM partitions. + * + * Even if the green contexts have disjoint SM partitions, it is not guaranteed that the kernels launched + * in them will run concurrently or have forward progress guarantees. This is due to other resources (like HW connections, + * see ::CUDA_DEVICE_MAX_CONNECTIONS) that could cause a dependency. Additionally, in certain scenarios, + * it is possible for the workload to run on more SMs than was provisioned (but never less). + * The following are two scenarios which can exhibit this behavior: + * - On Volta+ MPS: When \p CUDA_MPS_ACTIVE_THREAD_PERCENTAGE is used, + * the set of SMs that are used for running kernels can be scaled up to the value of SMs used for the MPS client. + * - On Compute Architecture 9.x: When a module with dynamic parallelism (CDP) is loaded, all future + * kernels running under green contexts may use and share an additional set of 2 SMs. + * + * @{ + */ + +/*! + * \typedef struct CUdevResourceDesc_st* CUdevResourceDesc; + * An opaque descriptor handle. The descriptor encapsulates multiple created and configured resources. + * Created via ::cuDevResourceGenerateDesc + */ +typedef struct CUdevResourceDesc_st *CUdevResourceDesc; + +typedef enum { + CU_GREEN_CTX_DEFAULT_STREAM = 0x1, /**< Required. Creates a default stream to use inside the green context */ +} CUgreenCtxCreate_flags; + +typedef enum { + CU_DEV_SM_RESOURCE_SPLIT_IGNORE_SM_COSCHEDULING = 0x1, + CU_DEV_SM_RESOURCE_SPLIT_MAX_POTENTIAL_CLUSTER_SIZE = 0x2, +} CUdevSmResourceSplit_flags; + +#define RESOURCE_ABI_VERSION 1 +#define RESOURCE_ABI_EXTERNAL_BYTES 48 + +#define _CONCAT_INNER(x, y) x ## y +#define _CONCAT_OUTER(x, y) _CONCAT_INNER(x, y) + +/*! + * \typedef enum CUdevResourceType + * Type of resource + */ +typedef enum { + CU_DEV_RESOURCE_TYPE_INVALID = 0, + CU_DEV_RESOURCE_TYPE_SM = 1, /**< Streaming multiprocessors related information */ +#if defined(__CUDA_API_VERSION_INTERNAL) && !defined(__CUDA_API_VERSION_INTERNAL_ODR) + CU_DEV_RESOURCE_TYPE_MAX, +#endif +} CUdevResourceType; + +/*! + * \struct CUdevSmResource + * Data for SM-related resources + */ +typedef struct CUdevSmResource_st { + unsigned int smCount; /**< The amount of streaming multiprocessors available in this resource. This is an output parameter only, do not write to this field. */ + unsigned int minSmPartitionSize; /**< The minimum number of streaming multiprocessors required to partition this resource. + This is an output parameter only, do not write to this field. */ + unsigned int smCoscheduledAlignment; /**< The number of streaming multiprocessors in this resource that are guaranteed to + be co-scheduled on the same GPU processing cluster. smCount is a multiple of this value. + This is an output parameter only, do not write to this field. */ +} CUdevSmResource; + +/*! + * \struct CUdevResource + * A tagged union describing different resources identified by the type field. This structure should not be directly modified outside of the API that created it. + * \code + * struct { + * CUdevResourceType type; + * union { + * CUdevSmResource sm; + * }; + * }; + * \endcode + * - If \p type is \p CU_DEV_RESOURCE_TYPE_INVALID, this resoure is not valid and cannot be further accessed. + * - If \p type is \p CU_DEV_RESOURCE_TYPE_SM, the ::CUdevSmResource structure \p sm is filled in. For example, + * \p sm.smCount will reflect the amount of streaming multiprocessors available in this resource. + */ +typedef struct CUdevResource_st { + CUdevResourceType type; /**< Type of resource, dictates which union field was last set */ + unsigned char _internal_padding[92]; + union { + CUdevSmResource sm; /**< Resource corresponding to CU_DEV_RESOURCE_TYPE_SM \p type. */ + unsigned char _oversize[RESOURCE_ABI_EXTERNAL_BYTES]; + }; +} _CONCAT_OUTER(CUdevResource_v, RESOURCE_ABI_VERSION); +typedef _CONCAT_OUTER(CUdevResource_v, RESOURCE_ABI_VERSION) CUdevResource; + +#undef _CONCAT_INNER +#undef _CONCAT_OUTER + +#undef ABI_PER_RESOURCE_EXTERNAL_BYTES +#undef ABI_RESOURCE_VERSION + +/** + * \brief Creates a green context with a specified set of resources. + * + * This API creates a green context with the resources specified in the descriptor \p desc and + * returns it in the handle represented by \p phCtx. This API will retain the primary context on device \p dev, + * which will is released when the green context is destroyed. It is advised to have the primary context active + * before calling this API to avoid the heavy cost of triggering primary context initialization and + * deinitialization multiple times. + * + * The API does not set the green context current. In order to set it current, you need to explicitly set it current + * by first converting the green context to a CUcontext using ::cuCtxFromGreenCtx and subsequently calling + * ::cuCtxSetCurrent / ::cuCtxPushCurrent. It should be noted that a green context can be current to only one + * thread at a time. There is no internal synchronization to make API calls accessing the same green context + * from multiple threads work. + * + * Note: The API is not supported on 32-bit platforms. + * + * \param phCtx - Pointer for the output handle to the green context + * \param desc - Descriptor generated via ::cuDevResourceGenerateDesc which contains the set of resources to be used + * \param dev - Device on which to create the green context. + * \param flags - One of the supported green context creation flags. \p CU_GREEN_CTX_DEFAULT_STREAM is required. + * + * The supported flags are: + * - \p CU_GREEN_CTX_DEFAULT_STREAM : Creates a default stream to use inside the green context. Required. + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_DEVICE, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_NOT_SUPPORTED, + * ::CUDA_ERROR_OUT_OF_MEMORY + * + * \sa + * ::cuGreenCtxDestroy, + * ::cuCtxFromGreenCtx, + * ::cuCtxSetCurrent, + * ::cuCtxPushCurrent, + * ::cuDevResourceGenerateDesc, + * ::cuDevicePrimaryCtxRetain, + * ::cuCtxCreate + */ +CUresult CUDAAPI cuGreenCtxCreate(CUgreenCtx* phCtx, CUdevResourceDesc desc, CUdevice dev, unsigned int flags); + +/** + * \brief Destroys a green context + * + * Destroys the green context, releasing the primary context of the device that this green context was created for. + * Any resources provisioned for this green context (that were initially available via the resource descriptor) + * are released as well. + * The API does not destroy streams created via ::cuGreenCtxStreamCreate, ::cuStreamCreate, or ::cuStreamCreateWithPriority. + * Once the green context is destroyed, any subsequent API calls involving these streams (including ::cuStreamDestroy) will return + * ::CUDA_ERROR_CONTEXT_IS_DESTROYED. + * Users must explicitly destroy all such streams before invoking ::cuGreenCtxDestroy. Failure to do so will result in a memory leak. + * + * \param hCtx - Green context to be destroyed + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_CONTEXT_IS_DESTROYED + * + * \sa + * ::cuGreenCtxCreate, + * ::cuCtxDestroy + */ +CUresult CUDAAPI cuGreenCtxDestroy(CUgreenCtx hCtx); + +/** + * \brief Converts a green context into the primary context + * + * The API converts a green context into the primary context returned in \p pContext. It is important + * to note that the converted context \p pContext is a normal primary context but with + * the resources of the specified green context \p hCtx. Once converted, it can then + * be used to set the context current with ::cuCtxSetCurrent or with any of the CUDA APIs + * that accept a CUcontext parameter. + * + * Users are expected to call this API before calling any CUDA APIs that accept a + * CUcontext. Failing to do so will result in the APIs returning ::CUDA_ERROR_INVALID_CONTEXT. + * + * \param pContext Returned primary context with green context resources + * \param hCtx Green context to convert + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE + * + * \sa + * ::cuGreenCtxCreate + */ +CUresult CUDAAPI cuCtxFromGreenCtx(CUcontext *pContext, CUgreenCtx hCtx); + +/** + * \brief Get device resources + * + * Get the \p type resources available to the \p device. + * This may often be the starting point for further partitioning or configuring of resources. + * + * Note: The API is not supported on 32-bit platforms. + * + * \param device - Device to get resource for + * \param resource - Output pointer to a CUdevResource structure + * \param type - Type of resource to retrieve + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_RESOURCE_TYPE, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_INVALID_DEVICE + * + * \sa + * ::cuDevResourceGenerateDesc + */ +CUresult CUDAAPI cuDeviceGetDevResource(CUdevice device, CUdevResource* resource, CUdevResourceType type); + +/** + * \brief Get context resources + * + * Get the \p type resources available to the context represented by \p hCtx + * \param hCtx - Context to get resource for + * + * Note: The API is not supported on 32-bit platforms. + * + * \param resource - Output pointer to a CUdevResource structure + * \param type - Type of resource to retrieve + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_RESOURCE_TYPE, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_INVALID_CONTEXT + * + * \sa + * ::cuDevResourceGenerateDesc + */ +CUresult CUDAAPI cuCtxGetDevResource(CUcontext hCtx, CUdevResource* resource, CUdevResourceType type); + +/** + * \brief Get green context resources + * + * Get the \p type resources available to the green context represented by \p hCtx + * \param hCtx - Green context to get resource for + * \param resource - Output pointer to a CUdevResource structure + * \param type - Type of resource to retrieve + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_RESOURCE_TYPE, + * ::CUDA_ERROR_INVALID_VALUE + * + * \sa + * ::cuDevResourceGenerateDesc + */ +CUresult CUDAAPI cuGreenCtxGetDevResource(CUgreenCtx hCtx, CUdevResource* resource, CUdevResourceType type); + +/** + * \brief Splits \p CU_DEV_RESOURCE_TYPE_SM resources. + * + * Splits \p CU_DEV_RESOURCE_TYPE_SM resources into \p nbGroups, adhering to the minimum SM count specified in \p minCount + * and the usage flags in \p useFlags. If \p result is NULL, the API simulates a split and provides the amount of groups that + * would be created in \p nbGroups. Otherwise, \p nbGroups must point to the amount of elements in \p result and on return, + * the API will overwrite \p nbGroups with the amount actually created. The groups are written to the array in \p result. + * \p nbGroups can be less than the total amount if a smaller number of groups is needed. + * + * This API is used to spatially partition the input resource. The input resource needs to come from one of + * ::cuDeviceGetDevResource, ::cuCtxGetDevResource, or ::cuGreenCtxGetDevResource. + * A limitation of the API is that the output results cannot be split again without + * first creating a descriptor and a green context with that descriptor. + * + * When creating the groups, the API will take into account the performance and functional characteristics of the + * input resource, and guarantee a split that will create a disjoint set of symmetrical partitions. This may lead to fewer groups created + * than purely dividing the total SM count by the \p minCount due to cluster requirements or + * alignment and granularity requirements for the minCount. + * These requirements can be queried with ::cuDeviceGetDevResource, ::cuCtxGetDevResource, and ::cuGreenCtxGetDevResource for + * ::CU_DEV_RESOURCE_TYPE_SM, using the \p minSmPartitionSize and \p smCoscheduledAlignment fields to determine minimum + * partition size and alignment granularity, respectively. + * + * The \p remainder set does not have the same functional or performance guarantees as the groups in \p result. + * Its use should be carefully planned and future partitions of the \p remainder set are discouraged. + * + * The following flags are supported: + * - \p CU_DEV_SM_RESOURCE_SPLIT_IGNORE_SM_COSCHEDULING : Lower the minimum SM count and alignment, and treat each SM independent of its hierarchy. + * This allows more fine grained partitions but at the cost of advanced features (such as large clusters on compute capability 9.0+). + * - \p CU_DEV_SM_RESOURCE_SPLIT_MAX_POTENTIAL_CLUSTER_SIZE : Compute Capability 9.0+ only. Attempt to create groups that may allow + * for maximally sized thread clusters. This can be queried post green context creation using ::cuOccupancyMaxPotentialClusterSize. + * + * A successful API call must either have: + * - A valid array of \p result pointers of size passed in \p nbGroups, with \p input of type \p CU_DEV_RESOURCE_TYPE_SM. + * Value of \p minCount must be between 0 and the SM count specified in \p input. \p remaining may be NULL. + * - NULL passed in for \p result, with a valid integer pointer in \p nbGroups and \p input of type \p CU_DEV_RESOURCE_TYPE_SM. + * Value of \p minCount must be between 0 and the SM count specified in \p input. \p remaining may be NULL. + * This queries the number of groups that would be created by the API. + * + * Note: The API is not supported on 32-bit platforms. + * + * \param result - Output array of \p CUdevResource resources. Can be NULL to query the number of groups. + * \param nbGroups - This is a pointer, specifying the number of groups that would be or should be created as described below. + * \param input - Input SM resource to be split. Must be a valid \p CU_DEV_RESOURCE_TYPE_SM resource. + * \param remaining - If the input resource cannot be cleanly split among \p nbGroups, the remaining is placed in here. + * Can be ommitted (NULL) if the user does not need the remaining set. + * \param useFlags - Flags specifying how these partitions are used or which constraints to abide by when splitting the input. Zero is valid for default behavior. + * \param minCount - Minimum number of SMs required + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_DEVICE, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_INVALID_RESOURCE_TYPE, + * ::CUDA_ERROR_INVALID_RESOURCE_CONFIGURATION + * + * \sa + * ::cuGreenCtxGetDevResource, + * ::cuCtxGetDevResource, + * ::cuDeviceGetDevResource + */ +CUresult CUDAAPI cuDevSmResourceSplitByCount( + CUdevResource* result, unsigned int* nbGroups, const CUdevResource* input, CUdevResource* remaining, unsigned int useFlags, unsigned int minCount); + +/** + * \brief Generate a resource descriptor + * + * Generates a single resource descriptor with the set of resources specified in \p resources. + * The generated resource descriptor is necessary for the creation of green contexts via the ::cuGreenCtxCreate API. + * Resources of the same type can be passed in, provided they meet the requirements as noted below. + * + * A successful API call must have: + * - A valid output pointer for the \p phDesc descriptor as well as a valid array of \p resources pointers, + * with the array size passed in \p nbResources. + * If multiple resources are provided in \p resources, the device they came from must be the same, + * otherwise CUDA_ERROR_INVALID_RESOURCE_CONFIGURATION is returned. + * If multiple resources are provided in \p resources and they are of type ::CU_DEV_RESOURCE_TYPE_SM, + * they must be outputs (whether \p result or \p remaining) from the same split API instance, + * otherwise CUDA_ERROR_INVALID_RESOURCE_CONFIGURATION is returned. + * + * Note: The API is not supported on 32-bit platforms. + * + * \param phDesc - Output descriptor + * \param resources - Array of resources to be included in the descriptor + * \param nbResources - Number of resources passed in \p resources + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_INVALID_RESOURCE_TYPE, + * ::CUDA_ERROR_INVALID_RESOURCE_CONFIGURATION + * + * \sa + * ::cuDevSmResourceSplitByCount + */ +CUresult CUDAAPI cuDevResourceGenerateDesc(CUdevResourceDesc *phDesc, CUdevResource *resources, unsigned int nbResources); + +/** + * \brief Records an event. + * + * Captures in \p hEvent all the activities of the green context of \p hCtx + * at the time of this call. \p hEvent and \p hCtx must be from the same + * primary context otherwise ::CUDA_ERROR_INVALID_HANDLE is returned. + * Calls such as ::cuEventQuery() or ::cuGreenCtxWaitEvent() will + * then examine or wait for completion of the work that was captured. Uses of + * \p hCtx after this call do not modify \p hEvent. + * + * \note The API will return ::CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED if the + * specified green context \p hCtx has a stream in the capture mode. In such + * a case, the call will invalidate all the conflicting captures. + * + * \param hCtx - Green context to record event for + * \param hEvent - Event to record + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED + * + * \sa + * ::cuGreenCtxWaitEvent, + * ::cuEventRecord, + * ::cuCtxRecordEvent, + * ::cuCtxWaitEvent + */ +CUresult CUDAAPI cuGreenCtxRecordEvent(CUgreenCtx hCtx, CUevent hEvent); + +/** + * \brief Make a green context wait on an event + * + * Makes all future work submitted to green context \p hCtx wait for all work + * captured in \p hEvent. The synchronization will be performed on the device + * and will not block the calling CPU thread. See ::cuGreenCtxRecordEvent() + * or ::cuEventRecord(), for details on what is captured by an event. + * + * \note \p hEvent may be from a different context or device than \p hCtx. + * + * \note The API will return ::CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED and + * invalidate the capture if the specified event \p hEvent is part of an + * ongoing capture sequence or if the specified green context \p hCtx has + * a stream in the capture mode. + * + * \param hCtx - Green context to wait + * \param hEvent - Event to wait on + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_HANDLE, + * ::CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED + * + * \sa + * ::cuGreenCtxRecordEvent, + * ::cuStreamWaitEvent, + * ::cuCtxRecordEvent, + * ::cuCtxWaitEvent +*/ +CUresult CUDAAPI cuGreenCtxWaitEvent(CUgreenCtx hCtx, CUevent hEvent); + +/** + * \brief Query the green context associated with a stream + * + * Returns the CUDA green context that the stream is associated with, or NULL if the stream + * is not associated with any green context. + * + * The stream handle \p hStream can refer to any of the following: + *
      + *
    • + * a stream created via any of the CUDA driver APIs such as ::cuStreamCreate, ::cuStreamCreateWithPriority + * and ::cuGreenCtxStreamCreate, or their runtime API equivalents such as + * ::cudaStreamCreate, ::cudaStreamCreateWithFlags and ::cudaStreamCreateWithPriority. + * If during stream creation the context that was active in the calling thread was obtained + * with cuCtxFromGreenCtx, that green context is returned in \p phCtx. + * Otherwise, \p *phCtx is set to NULL instead. + *
      • + *
    • + * special stream such as the NULL stream or ::CU_STREAM_LEGACY. + * In that case if context that is active in the calling thread was obtained + * with cuCtxFromGreenCtx, that green context is returned. + * Otherwise, \p *phCtx is set to NULL instead. + *
      • + *
    + * Passing an invalid handle will result in undefined behavior. + * + * \param hStream - Handle to the stream to be queried + * \param phCtx - Returned green context associated with the stream + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_HANDLE, + * \notefnerr + * + * \sa ::cuStreamDestroy, + * ::cuStreamCreate, + * ::cuStreamCreateWithPriority, + * ::cuStreamGetCtx, + * ::cuGreenCtxStreamCreate, + * ::cuStreamGetPriority, + * ::cuStreamGetFlags, + * ::cuStreamGetDevice, + * ::cuStreamWaitEvent, + * ::cuStreamQuery, + * ::cuStreamSynchronize, + * ::cuStreamAddCallback, + * ::cudaStreamCreate, + * ::cudaStreamCreateWithFlags + */ +CUresult CUDAAPI cuStreamGetGreenCtx(CUstream hStream, CUgreenCtx *phCtx); + +/** + * \brief Create a stream for use in the green context + * + * Creates a stream for use in the specified green context \p greenCtx and returns a handle in \p phStream. + * The stream can be destroyed by calling ::cuStreamDestroy(). Note that the API ignores the context that + * is current to the calling thread and creates a stream in the specified green context \p greenCtx. + * + * The supported values for \p flags are: + * - ::CU_STREAM_NON_BLOCKING: This must be specified. It indicates that work running in the created + * stream may run concurrently with work in the default stream, and that + * the created stream should perform no implicit synchronization with the default stream. + * + * Specifying \p priority affects the scheduling priority of work in the stream. Priorities provide a + * hint to preferentially run work with higher priority when possible, but do not preempt + * already-running work or provide any other functional guarantee on execution order. + * \p priority follows a convention where lower numbers represent higher priorities. + * '0' represents default priority. The range of meaningful numerical priorities can + * be queried using ::cuCtxGetStreamPriorityRange. If the specified priority is + * outside the numerical range returned by ::cuCtxGetStreamPriorityRange, + * it will automatically be clamped to the lowest or the highest number in the range. + * + * \param phStream - Returned newly created stream + * \param greenCtx - Green context for which to create the stream for + * \param flags - Flags for stream creation. \p CU_STREAM_NON_BLOCKING must be specified. + * \param priority - Stream priority. Lower numbers represent higher priorities. + * See ::cuCtxGetStreamPriorityRange for more information about + * meaningful stream priorities that can be passed. + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_OUT_OF_MEMORY + * \notefnerr + * + * \note In the current implementation, only compute kernels launched in + * priority streams are affected by the stream's priority. Stream priorities have + * no effect on host-to-device and device-to-host memory operations. + * + * \sa ::cuStreamDestroy, + * ::cuGreenCtxCreate + * ::cuStreamCreate, + * ::cuStreamGetPriority, + * ::cuCtxGetStreamPriorityRange, + * ::cuStreamGetFlags, + * ::cuStreamGetDevice, + * ::cuStreamWaitEvent, + * ::cuStreamQuery, + * ::cuStreamSynchronize, + * ::cuStreamAddCallback, + * ::cudaStreamCreateWithPriority + */ +CUresult CUDAAPI cuGreenCtxStreamCreate(CUstream* phStream, CUgreenCtx greenCtx, unsigned int flags, int priority); + +/** + * \brief Returns the unique Id associated with the green context supplied + * + * Returns in \p greenCtxId the unique Id which is associated with a given green context. + * The Id is unique for the life of the program for this instance of CUDA. + * If green context is supplied as NULL and the current context is set to a green context, + * the Id of the current green context is returned. + * + * \param greenCtx - Green context for which to obtain the Id + * \param greenCtxId - Pointer to store the Id of the green context + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_CONTEXT_IS_DESTROYED, + * ::CUDA_ERROR_DEINITIALIZED, + * ::CUDA_ERROR_NOT_INITIALIZED, + * ::CUDA_ERROR_INVALID_CONTEXT, + * ::CUDA_ERROR_INVALID_VALUE + * \notefnerr + * + * \sa ::cuGreenCtxCreate, + * ::cuGreenCtxDestroy, + * ::cuCtxGetId + */ +CUresult CUDAAPI cuGreenCtxGetId(CUgreenCtx greenCtx, unsigned long long *greenCtxId); + +/** @} */ + +/* +** *************** END CUDA_GREEN_CONTEXTS ***************** +*/ + +/** + * \defgroup CUDA_LOGS Error Log Management Functions + * + * ___MANBRIEF___ error log management functions for the low-level CUDA API + * (___CURRENT_FILE___) ___ENDMANBRIEF___ + * + * This section describes the error log management functions of the low-level CUDA + * driver application programming interface. + * + * @{ + */ +typedef enum CUlogLevel_enum { + CU_LOG_LEVEL_ERROR = 0, + CU_LOG_LEVEL_WARNING = 1 +} CUlogLevel; + +typedef struct CUlogsCallbackEntry_st *CUlogsCallbackHandle; +typedef void (CUDA_CB *CUlogsCallback)(void *data, CUlogLevel logLevel, char *message, size_t length); +typedef unsigned int CUlogIterator; + +/** + * \brief Register a callback function to receive error log messages + * + * \param callbackFunc - The function to register as a callback + * \param userData - A generic pointer to user data. This is passed into the callback function. + * \param callback_out - Optional location to store the callback handle after it is registered + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_INVALID_VALUE + */ +CUresult CUDAAPI cuLogsRegisterCallback(CUlogsCallback callbackFunc, void *userData, CUlogsCallbackHandle *callback_out); + +/** + * \brief Unregister a log message callback + * + * \param callback - The callback instance to unregister from receiving log messages + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_INVALID_VALUE + */ +CUresult CUDAAPI cuLogsUnregisterCallback(CUlogsCallbackHandle callback); + +/** + * \brief Sets log iterator to point to the end of log buffer, where the next message would be written. + * + * \param iterator_out - Location to store an iterator to the current tail of the logs + * \param flags - Reserved for future use, must be 0 + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_INVALID_VALUE + */ +CUresult CUDAAPI cuLogsCurrent(CUlogIterator *iterator_out, unsigned int flags); + +/** + * \brief Dump accumulated driver logs into a file + * + * Logs generated by the driver are stored in an internal buffer and can be copied out using this API. + * This API dumps all driver logs starting from \p iterator into \p pathToFile provided. + * + * \note \p iterator is auto-advancing. Dumping logs will update the value of + * \p iterator to receive the next generated log. + * + * \note The driver reserves limited memory for storing logs. + * The oldest logs may be overwritten and become unrecoverable. An indication will appear in the + * destination outupt if the logs have been truncated. Call dump after each failed API to mitigate this + * risk. + * + * \param iterator - Optional auto-advancing iterator specifying the starting log to read. NULL value dumps all logs. + * \param pathToFile - Path to output file for dumping logs + * \param flags - Reserved for future use, must be 0 + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_INVALID_VALUE + */ +CUresult CUDAAPI cuLogsDumpToFile(CUlogIterator *iterator, const char *pathToFile, unsigned int flags); + +/** + * \brief Dump accumulated driver logs into a buffer + * + * Logs generated by the driver are stored in an internal buffer and can be copied out using this API. + * This API dumps driver logs from \p iterator into \p buffer up to the size specified in \p *size. + * The driver will always null terminate the buffer but there will not be a null character between log + * entries, only a newline \\n. The driver will then return the actual number of bytes written in + * \p *size, excluding the null terminator. If there are no messages to dump, \p *size will be set to 0 + * and the function will return ::CUDA_SUCCESS. + * If the provided \p buffer is not large enough to hold any messages, \p *size will be set to 0 and + * the function will return ::CUDA_ERROR_INVALID_VALUE. + * + * \note \p iterator is auto-advancing. Dumping logs will update the value of + * \p iterator to receive the next generated log. + * + * \note The driver reserves limited memory for storing logs. The maximum size of the buffer is 25600 bytes. + * The oldest logs may be overwritten and become unrecoverable. An indication will appear in the + * destination outupt if the logs have been truncated. Call dump after each failed API to mitigate this + * risk. + * + * \note If the provided value in \p *size is not large enough to hold all buffered messages, a message will + * be added at the head of the buffer indicating this. The driver then computes the number of messages + * it is able to store in \p buffer and writes it out. The final message in \p buffer will always be + * the most recent log message as of when the API is called. + * + * \param iterator - Optional auto-advancing iterator specifying the starting log to read. NULL value dumps all logs. + * \param buffer - Pointer to dump logs + * \param size - See description + * \param flags - Reserved for future use, must be 0 + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE + */ +CUresult CUDAAPI cuLogsDumpToMemory(CUlogIterator *iterator, char *buffer, size_t *size, unsigned int flags); + +/** @} */ /* END CUDA_LOGS */ + /** * CUDA API versioning support */ @@ -21759,6 +25438,8 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp #undef cuMemcpyDtoDAsync #undef cuMemcpy2DAsync #undef cuMemcpy3DAsync + #undef cuMemcpyBatchAsync + #undef cuMemcpy3DBatchAsync #undef cuMemsetD8 #undef cuMemsetD16 #undef cuMemsetD32 @@ -21778,6 +25459,7 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp #undef cuCtxPushCurrent #undef cuStreamDestroy #undef cuEventDestroy + #undef cuEventElapsedTime #undef cuMemcpy #undef cuMemcpyAsync #undef cuMemcpyPeer @@ -21793,6 +25475,7 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp #undef cuStreamGetPriority #undef cuStreamGetId #undef cuStreamGetFlags + #undef cuStreamGetDevice #undef cuStreamGetCtx #undef cuStreamWaitEvent #undef cuStreamAddCallback @@ -21817,14 +25500,18 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp #undef cuStreamWaitValue64_v2 #undef cuStreamBatchMemOp_v2 #undef cuMemPrefetchAsync + #undef cuMemAdvise + #undef cuMemPrefetchBatchAsync + #undef cuMemDiscardBatchAsync + #undef cuMemDiscardAndPrefetchBatchAsync #undef cuLaunchCooperativeKernel #undef cuSignalExternalSemaphoresAsync #undef cuWaitExternalSemaphoresAsync #undef cuStreamBeginCapture + #undef cuStreamBeginCaptureToGraph #undef cuStreamEndCapture #undef cuStreamIsCapturing #undef cuStreamGetCaptureInfo - #undef cuStreamGetCaptureInfo_v2 #undef cuGraphInstantiateWithParams #undef cuGraphExecUpdate #undef cuGraphUpload @@ -21832,6 +25519,7 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp #undef cuDevicePrimaryCtxRelease #undef cuDevicePrimaryCtxReset #undef cuDevicePrimaryCtxSetFlags + #undef cuDeviceGetUuid #undef cuIpcOpenMemHandle #undef cuStreamCopyAttributes #undef cuStreamSetAttribute @@ -21841,12 +25529,20 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp #undef cuGraphKernelNodeGetParams #undef cuGraphKernelNodeSetParams #undef cuGraphExecKernelNodeSetParams + #undef cuGraphGetEdges + #undef cuGraphNodeGetDependencies + #undef cuGraphNodeGetDependentNodes + #undef cuGraphAddDependencies + #undef cuGraphRemoveDependencies + #undef cuGraphAddNode #undef cuMemMapArrayAsync #undef cuMemFreeAsync #undef cuMemAllocAsync #undef cuMemAllocFromPoolAsync #undef cuStreamUpdateCaptureDependencies #undef cuGetProcAddress + #undef cuStreamGetCtx_v2 + #undef cuMemBatchDecompressAsync CUresult CUDAAPI cuMemHostRegister(void *p, size_t bytesize, unsigned int Flags); CUresult CUDAAPI cuGraphicsResourceSetMapFlags(CUgraphicsResource resource, unsigned int flags); @@ -21934,6 +25630,8 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp CUresult CUDAAPI cuDeviceTotalMem(unsigned int *bytes, CUdevice dev); CUresult CUDAAPI cuCtxCreate(CUcontext *pctx, unsigned int flags, CUdevice dev); + CUresult CUDAAPI cuCtxCreate_v2(CUcontext *pctx, unsigned int flags, CUdevice dev); + CUresult CUDAAPI cuCtxCreate_v3(CUcontext *pctx, CUexecAffinityParam *paramsArray, int numParams, unsigned int flags, CUdevice dev); CUresult CUDAAPI cuModuleGetGlobal(CUdeviceptr_v1 *dptr, unsigned int *bytes, CUmodule hmod, const char *name); CUresult CUDAAPI cuMemGetInfo(unsigned int *free, unsigned int *total); CUresult CUDAAPI cuMemAlloc(CUdeviceptr_v1 *dptr, unsigned int bytesize); @@ -21980,10 +25678,11 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp CUresult CUDAAPI cuCtxPushCurrent(CUcontext ctx); CUresult CUDAAPI cuStreamDestroy(CUstream hStream); CUresult CUDAAPI cuEventDestroy(CUevent hEvent); + CUresult CUDAAPI cuEventElapsedTime(float *pMilliseconds, CUevent hStart, CUevent hEnd); CUresult CUDAAPI cuDevicePrimaryCtxRelease(CUdevice dev); CUresult CUDAAPI cuDevicePrimaryCtxReset(CUdevice dev); CUresult CUDAAPI cuDevicePrimaryCtxSetFlags(CUdevice dev, unsigned int flags); - + CUresult CUDAAPI cuDeviceGetUuid(CUuuid *uuid, CUdevice dev); CUresult CUDAAPI cuMemcpyHtoD_v2(CUdeviceptr dstDevice, const void *srcHost, size_t ByteCount); CUresult CUDAAPI cuMemcpyDtoH_v2(void *dstHost, CUdeviceptr srcDevice, size_t ByteCount); CUresult CUDAAPI cuMemcpyDtoD_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount); @@ -22014,7 +25713,17 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp CUresult CUDAAPI cuMemcpyPeerAsync(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount, CUstream hStream); CUresult CUDAAPI cuMemcpy3DPeer(const CUDA_MEMCPY3D_PEER *pCopy); CUresult CUDAAPI cuMemcpy3DPeerAsync(const CUDA_MEMCPY3D_PEER *pCopy, CUstream hStream); - + CUresult CUDAAPI cuMemcpyBatchAsync(CUdeviceptr *dsts, CUdeviceptr *srcs, size_t *sizes, size_t count, + CUmemcpyAttributes *attrs, size_t *attrsIdxs, size_t numAttrs, + size_t *failIdx, CUstream hStream); + CUresult CUDAAPI cuMemcpy3DBatchAsync(size_t numOps, CUDA_MEMCPY3D_BATCH_OP *opList, + size_t *failIdx, unsigned long long flags, CUstream hStream); + CUresult CUDAAPI cuMemcpyBatchAsync_ptsz(CUdeviceptr *dsts, CUdeviceptr *srcs, size_t *sizes, size_t count, + CUmemcpyAttributes *attrs, size_t *attrsIdxs, size_t numAttrs, size_t *failIdx, CUstream hStream); + CUresult CUDAAPI cuMemcpy3DBatchAsync_ptsz(size_t numOps, CUDA_MEMCPY3D_BATCH_OP *opList, size_t *failIdx, unsigned long long flags, CUstream hStream); + CUresult CUDAAPI cuMemcpyBatchAsync_v2(CUdeviceptr *dsts, CUdeviceptr *srcs, size_t *sizes, size_t count, + CUmemcpyAttributes *attrs, size_t *attrsIdxs, size_t numAttrs, CUstream hStream); + CUresult CUDAAPI cuMemcpy3DBatchAsync_v2(size_t numOps, CUDA_MEMCPY3D_BATCH_OP *opList, unsigned long long flags, CUstream hStream); CUresult CUDAAPI cuMemsetD8Async(CUdeviceptr dstDevice, unsigned char uc, size_t N, CUstream hStream); CUresult CUDAAPI cuMemsetD16Async(CUdeviceptr dstDevice, unsigned short us, size_t N, CUstream hStream); CUresult CUDAAPI cuMemsetD32Async(CUdeviceptr dstDevice, unsigned int ui, size_t N, CUstream hStream); @@ -22025,7 +25734,9 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp CUresult CUDAAPI cuStreamGetPriority(CUstream hStream, int *priority); CUresult CUDAAPI cuStreamGetId(CUstream hStream, unsigned long long *streamId); CUresult CUDAAPI cuStreamGetFlags(CUstream hStream, unsigned int *flags); + CUresult CUDAAPI cuStreamGetDevice(CUstream hStream, CUdevice *device); CUresult CUDAAPI cuStreamGetCtx(CUstream hStream, CUcontext *pctx); + CUresult CUDAAPI cuStreamGetCtx_v2(CUstream hStream, CUcontext *pCtx, CUgreenCtx *pGreenCtx); CUresult CUDAAPI cuStreamWaitEvent(CUstream hStream, CUevent hEvent, unsigned int Flags); CUresult CUDAAPI cuStreamAddCallback(CUstream hStream, CUstreamCallback callback, void *userData, unsigned int flags); CUresult CUDAAPI cuStreamAttachMemAsync(CUstream hStream, CUdeviceptr dptr, size_t length, unsigned int flags); @@ -22056,17 +25767,26 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp CUresult CUDAAPI cuStreamWaitValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags); CUresult CUDAAPI cuStreamBatchMemOp_v2(CUstream stream, unsigned int count, CUstreamBatchMemOpParams *paramArray, unsigned int flags); CUresult CUDAAPI cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice dstDevice, CUstream hStream); + CUresult CUDAAPI cuMemPrefetchAsync_ptsz(CUdeviceptr devPtr, size_t count, CUdevice dstDevice, CUstream hStream); + CUresult CUDAAPI cuMemPrefetchAsync_v2(CUdeviceptr devPtr, size_t count, CUmemLocation location, unsigned int flags, CUstream hStream); + CUresult CUDAAPI cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUdevice device); + CUresult CUDAAPI cuMemPrefetchBatchAsync(CUdeviceptr *dptrs, size_t *sizes, size_t count, CUmemLocation *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, unsigned long long flags, CUstream hStream); + CUresult CUDAAPI cuMemDiscardBatchAsync(CUdeviceptr *dptrs, size_t *sizes, size_t count, unsigned long long flags, CUstream hStream); + CUresult CUDAAPI cuMemDiscardAndPrefetchBatchAsync(CUdeviceptr *dptrs, size_t *sizes, size_t count, CUmemLocation *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, unsigned long long flags, CUstream hStream); CUresult CUDAAPI cuLaunchCooperativeKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void **kernelParams); CUresult CUDAAPI cuSignalExternalSemaphoresAsync(const CUexternalSemaphore *extSemArray, const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS *paramsArray, unsigned int numExtSems, CUstream stream); CUresult CUDAAPI cuWaitExternalSemaphoresAsync(const CUexternalSemaphore *extSemArray, const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS *paramsArray, unsigned int numExtSems, CUstream stream); CUresult CUDAAPI cuStreamBeginCapture(CUstream hStream); CUresult CUDAAPI cuStreamBeginCapture_ptsz(CUstream hStream); CUresult CUDAAPI cuStreamBeginCapture_v2(CUstream hStream, CUstreamCaptureMode mode); + CUresult CUDAAPI cuStreamBeginCaptureToGraph(CUstream hStream, CUgraph hGraph, const CUgraphNode *dependencies, const CUgraphEdgeData *dependencyData, size_t numDependencies, CUstreamCaptureMode mode); CUresult CUDAAPI cuStreamEndCapture(CUstream hStream, CUgraph *phGraph); CUresult CUDAAPI cuStreamIsCapturing(CUstream hStream, CUstreamCaptureStatus *captureStatus); CUresult CUDAAPI cuStreamGetCaptureInfo(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out); CUresult CUDAAPI cuStreamGetCaptureInfo_ptsz(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out); CUresult CUDAAPI cuStreamGetCaptureInfo_v2(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out, CUgraph *graph_out, const CUgraphNode **dependencies_out, size_t *numDependencies_out); + CUresult CUDAAPI cuStreamGetCaptureInfo_v2_ptsz(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out, CUgraph *graph_out, const CUgraphNode **dependencies_out, size_t *numDependencies_out); + CUresult CUDAAPI cuStreamGetCaptureInfo_v3(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out, CUgraph *graph_out, const CUgraphNode **dependencies_out, const CUgraphEdgeData **edgeData_out, size_t *numDependencies_out); CUresult CUDAAPI cuGraphAddKernelNode(CUgraphNode *phGraphNode, CUgraph hGraph, const CUgraphNode *dependencies, size_t numDependencies, const CUDA_KERNEL_NODE_PARAMS_v1 *nodeParams); CUresult CUDAAPI cuGraphKernelNodeGetParams(CUgraphNode hNode, CUDA_KERNEL_NODE_PARAMS_v1 *nodeParams); CUresult CUDAAPI cuGraphKernelNodeSetParams(CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS_v1 *nodeParams); @@ -22075,6 +25795,12 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp CUresult CUDAAPI cuGraphExecUpdate(CUgraphExec hGraphExec, CUgraph hGraph, CUgraphNode *hErrorNode_out, CUgraphExecUpdateResult *updateResult_out); CUresult CUDAAPI cuGraphUpload(CUgraphExec hGraph, CUstream hStream); CUresult CUDAAPI cuGraphLaunch(CUgraphExec hGraph, CUstream hStream); + CUresult CUDAAPI cuGraphGetEdges(CUgraph hGraph, CUgraphNode *from, CUgraphNode *to, size_t *numEdges); + CUresult CUDAAPI cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode *dependencies, size_t *numDependencies); + CUresult CUDAAPI cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode *dependentNodes, size_t *numDependentNodes); + CUresult CUDAAPI cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode *from, const CUgraphNode *to, size_t numDependencies); + CUresult CUDAAPI cuGraphRemoveDependencies(CUgraph hGraph, const CUgraphNode *from, const CUgraphNode *to, size_t numDependencies); + CUresult CUDAAPI cuGraphAddNode(CUgraphNode *phGraphNode, CUgraph hGraph, const CUgraphNode *dependencies, size_t numDependencies, CUgraphNodeParams *nodeParams); CUresult CUDAAPI cuStreamCopyAttributes(CUstream dstStream, CUstream srcStream); CUresult CUDAAPI cuStreamGetAttribute(CUstream hStream, CUstreamAttrID attr, CUstreamAttrValue *value); CUresult CUDAAPI cuStreamSetAttribute(CUstream hStream, CUstreamAttrID attr, const CUstreamAttrValue *param); @@ -22090,6 +25816,17 @@ CUresult CUDAAPI cuGetExportTable(const void **ppExportTable, const CUuuid *pExp CUresult CUDAAPI cuMemAllocFromPoolAsync(CUdeviceptr *dptr, size_t bytesize, CUmemoryPool pool, CUstream hStream); CUresult CUDAAPI cuStreamUpdateCaptureDependencies(CUstream hStream, CUgraphNode *dependencies, size_t numDependencies, unsigned int flags); + CUresult CUDAAPI cuStreamUpdateCaptureDependencies_ptsz(CUstream hStream, CUgraphNode *dependencies, size_t numDependencies, unsigned int flags); + CUresult CUDAAPI cuStreamUpdateCaptureDependencies_v2(CUstream hStream, CUgraphNode *dependencies, const CUgraphEdgeData *dependencyData, size_t numDependencies, unsigned int flags); + + CUresult CUDAAPI cuMemBatchDecompressAsync( + CUmemDecompressParams *paramsArray, + size_t count, + unsigned int flags, + size_t *errorIndex, + CUstream stream + ); + CUresult CUDAAPI cuGetProcAddress(const char *symbol, void **pfn, int cudaVersion, cuuint64_t flags); #elif defined(__CUDA_API_PER_THREAD_DEFAULT_STREAM) @@ -22104,6 +25841,157 @@ static inline CUresult cuGetProcAddress_v2_ptsz(const char *symbol, void **funcP #define cuGetProcAddress_v2 cuGetProcAddress_v2_ptsz #endif +/** + * \defgroup CUDA_CHECKPOINT CUDA Checkpointing + * + * ___MANBRIEF___ CUDA checkpoint and restore functionality of the low-level + * CUDA driver API (___CURRENT_FILE___) ___ENDMANBRIEF___ + * + * This sections describes the checkpoint and restore functions of the low-level + * CUDA driver application programming interface. + * + * The CUDA checkpoint and restore API's provide a way to save and restore GPU + * state for full process checkpoints when used with CPU side process + * checkpointing solutions. They can also be used to pause GPU work and suspend + * a CUDA process to allow other applications to make use of GPU resources. + * + * Checkpoint and restore capabilities are currently restricted to Linux. + * + * @{ + */ + +/** + * \brief Returns the restore thread ID for a CUDA process + * + * Returns in \p *tid the thread ID of the CUDA restore thread for the process + * specified by \p pid. + * + * \param pid - The process ID of the CUDA process + * \param tid - Returned restore thread ID + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_INVALID_VALUE + * ::CUDA_ERROR_NOT_INITIALIZED + * ::CUDA_ERROR_NOT_SUPPORTED + */ +CUresult CUDAAPI cuCheckpointProcessGetRestoreThreadId(int pid, int *tid); + +/** + * \brief Returns the process state of a CUDA process + * + * Returns in \p *state the current state of the CUDA process specified by \p pid. + * + * \param pid - The process ID of the CUDA process + * \param state - Returned CUDA process state + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_INVALID_VALUE + * ::CUDA_ERROR_NOT_INITIALIZED + * ::CUDA_ERROR_NOT_SUPPORTED + */ +CUresult CUDAAPI cuCheckpointProcessGetState(int pid, CUprocessState *state); + +/** + * \brief Lock a running CUDA process + * + * Lock the CUDA process specified by \p pid which will block further CUDA API + * calls. Process must be in the RUNNING state in order to lock. + * + * Upon successful return the process will be in the LOCKED state. + * + * If timeoutMs is specified and the timeout is reached the process will be left + * in the RUNNING state upon return. + * + * \param pid - The process ID of the CUDA process + * \param args - Optional lock operation arguments + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_INVALID_VALUE + * ::CUDA_ERROR_NOT_INITIALIZED + * ::CUDA_ERROR_ILLEGAL_STATE + * ::CUDA_ERROR_NOT_SUPPORTED + * ::CUDA_ERROR_NOT_READY + */ +CUresult CUDAAPI cuCheckpointProcessLock(int pid, CUcheckpointLockArgs *args); + +/** + * \brief Checkpoint a CUDA process's GPU memory contents + * + * Checkpoints a CUDA process specified by \p pid that is in the LOCKED + * state. The GPU memory contents will be brought into host memory and all + * underlying references will be released. Process must be in the LOCKED state + * to checkpoint. + * + * Upon successful return the process will be in the CHECKPOINTED state. + * + * \param pid - The process ID of the CUDA process + * \param args - Optional checkpoint operation arguments + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_INVALID_VALUE + * ::CUDA_ERROR_NOT_INITIALIZED + * ::CUDA_ERROR_ILLEGAL_STATE + * ::CUDA_ERROR_NOT_SUPPORTED + */ +CUresult CUDAAPI cuCheckpointProcessCheckpoint(int pid, CUcheckpointCheckpointArgs *args); + +/** + * \brief Restore a CUDA process's GPU memory contents from its last checkpoint + * + * Restores a CUDA process specified by \p pid from its last checkpoint. Process + * must be in the CHECKPOINTED state to restore. + * + * GPU UUID pairs can be specified in \p args to remap the process old GPUs onto + * new GPUs. The GPU to restore onto needs to have enough memory and be of the + * same chip type as the old GPU. If an array of GPU UUID pairs is specified, it + * must contain every checkpointed GPU. + * + * Upon successful return the process will be in the LOCKED state. + * + * CUDA process restore requires persistence mode to be enabled or ::cuInit to + * have been called before execution. + * + * \param pid - The process ID of the CUDA process + * \param args - Optional restore operation arguments + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_INVALID_VALUE + * ::CUDA_ERROR_NOT_INITIALIZED + * ::CUDA_ERROR_ILLEGAL_STATE + * ::CUDA_ERROR_NOT_SUPPORTED + * + * \sa + * ::cuInit + */ +CUresult CUDAAPI cuCheckpointProcessRestore(int pid, CUcheckpointRestoreArgs *args); + +/** + * \brief Unlock a CUDA process to allow CUDA API calls + * + * Unlocks a process specified by \p pid allowing it to resume making CUDA API + * calls. Process must be in the LOCKED state. + * + * Upon successful return the process will be in the RUNNING state. + * + * \param pid - The process ID of the CUDA process + * \param args - Optional unlock operation arguments + * + * \return + * ::CUDA_SUCCESS + * ::CUDA_ERROR_INVALID_VALUE + * ::CUDA_ERROR_NOT_INITIALIZED + * ::CUDA_ERROR_ILLEGAL_STATE + * ::CUDA_ERROR_NOT_SUPPORTED + */ +CUresult CUDAAPI cuCheckpointProcessUnlock(int pid, CUcheckpointUnlockArgs *args); + +/** @} */ /* End CUDA_CHECKPOINT */ + #ifdef __cplusplus } #endif diff --git a/backends/cuda/include/cudaProfilerTypedefs.h b/backends/cuda/include/cudaProfilerTypedefs.h index bea7df457..d20246712 100644 --- a/backends/cuda/include/cudaProfilerTypedefs.h +++ b/backends/cuda/include/cudaProfilerTypedefs.h @@ -56,14 +56,6 @@ extern "C" { #endif // __cplusplus -/* - * Macros for the latest version for each driver function in cudaProfiler.h - */ -#define PFN_cuProfilerInitialize PFN_cuProfilerInitialize_v4000 -#define PFN_cuProfilerStart PFN_cuProfilerStart_v4000 -#define PFN_cuProfilerStop PFN_cuProfilerStop_v4000 - - /** * Type definitions for functions defined in cudaProfiler.h */ diff --git a/backends/cuda/include/cudaTypedefs.h b/backends/cuda/include/cudaTypedefs.h index c3c7117a9..bfb0e8b7c 100644 --- a/backends/cuda/include/cudaTypedefs.h +++ b/backends/cuda/include/cudaTypedefs.h @@ -64,409 +64,6 @@ extern "C" { #endif // __cplusplus -/* - * Macros for the latest version for each driver function in cuda.h - */ -#define PFN_cuGetErrorString PFN_cuGetErrorString_v6000 -#define PFN_cuGetErrorName PFN_cuGetErrorName_v6000 -#define PFN_cuInit PFN_cuInit_v2000 -#define PFN_cuDriverGetVersion PFN_cuDriverGetVersion_v2020 -#define PFN_cuDeviceGet PFN_cuDeviceGet_v2000 -#define PFN_cuDeviceGetCount PFN_cuDeviceGetCount_v2000 -#define PFN_cuDeviceGetName PFN_cuDeviceGetName_v2000 -#define PFN_cuDeviceGetUuid PFN_cuDeviceGetUuid_v11040 -#define PFN_cuDeviceGetLuid PFN_cuDeviceGetLuid_v10000 -#define PFN_cuDeviceTotalMem PFN_cuDeviceTotalMem_v3020 -#define PFN_cuDeviceGetTexture1DLinearMaxWidth PFN_cuDeviceGetTexture1DLinearMaxWidth_v11010 -#define PFN_cuDeviceGetAttribute PFN_cuDeviceGetAttribute_v2000 -#define PFN_cuDeviceGetNvSciSyncAttributes PFN_cuDeviceGetNvSciSyncAttributes_v10020 -#define PFN_cuDeviceSetMemPool PFN_cuDeviceSetMemPool_v11020 -#define PFN_cuDeviceGetMemPool PFN_cuDeviceGetMemPool_v11020 -#define PFN_cuDeviceGetDefaultMemPool PFN_cuDeviceGetDefaultMemPool_v11020 -#define PFN_cuDeviceGetProperties PFN_cuDeviceGetProperties_v2000 -#define PFN_cuDeviceComputeCapability PFN_cuDeviceComputeCapability_v2000 -#define PFN_cuDevicePrimaryCtxRetain PFN_cuDevicePrimaryCtxRetain_v7000 -#define PFN_cuDevicePrimaryCtxRelease PFN_cuDevicePrimaryCtxRelease_v11000 -#define PFN_cuDevicePrimaryCtxSetFlags PFN_cuDevicePrimaryCtxSetFlags_v11000 -#define PFN_cuDevicePrimaryCtxGetState PFN_cuDevicePrimaryCtxGetState_v7000 -#define PFN_cuDevicePrimaryCtxReset PFN_cuDevicePrimaryCtxReset_v11000 -#define PFN_cuDeviceGetExecAffinitySupport PFN_cuDeviceGetExecAffinitySupport_v11040 -#define PFN_cuCtxCreate PFN_cuCtxCreate_v11040 -#define PFN_cuCtxGetId PFN_cuCtxGetId_v12000 -#define PFN_cuCtxDestroy PFN_cuCtxDestroy_v4000 -#define PFN_cuCtxPushCurrent PFN_cuCtxPushCurrent_v4000 -#define PFN_cuCtxPopCurrent PFN_cuCtxPopCurrent_v4000 -#define PFN_cuCtxSetCurrent PFN_cuCtxSetCurrent_v4000 -#define PFN_cuCtxGetCurrent PFN_cuCtxGetCurrent_v4000 -#define PFN_cuCtxGetDevice PFN_cuCtxGetDevice_v2000 -#define PFN_cuCtxGetFlags PFN_cuCtxGetFlags_v7000 -#define PFN_cuCtxSetFlags PFN_cuCtxSetFlags_v12010 -#define PFN_cuCtxSynchronize PFN_cuCtxSynchronize_v2000 -#define PFN_cuCtxSetLimit PFN_cuCtxSetLimit_v3010 -#define PFN_cuCtxGetLimit PFN_cuCtxGetLimit_v3010 -#define PFN_cuCtxGetCacheConfig PFN_cuCtxGetCacheConfig_v3020 -#define PFN_cuCtxSetCacheConfig PFN_cuCtxSetCacheConfig_v3020 -#define PFN_cuCtxGetSharedMemConfig PFN_cuCtxGetSharedMemConfig_v4020 -#define PFN_cuCtxSetSharedMemConfig PFN_cuCtxSetSharedMemConfig_v4020 -#define PFN_cuCtxGetApiVersion PFN_cuCtxGetApiVersion_v3020 -#define PFN_cuCtxGetStreamPriorityRange PFN_cuCtxGetStreamPriorityRange_v5050 -#define PFN_cuCtxResetPersistingL2Cache PFN_cuCtxResetPersistingL2Cache_v11000 -#define PFN_cuCtxAttach PFN_cuCtxAttach_v2000 -#define PFN_cuCtxDetach PFN_cuCtxDetach_v2000 -#define PFN_cuCtxGetExecAffinity PFN_cuCtxGetExecAffinity_v11040 -#define PFN_cuModuleLoad PFN_cuModuleLoad_v2000 -#define PFN_cuModuleLoadData PFN_cuModuleLoadData_v2000 -#define PFN_cuModuleLoadDataEx PFN_cuModuleLoadDataEx_v2010 -#define PFN_cuModuleLoadFatBinary PFN_cuModuleLoadFatBinary_v2000 -#define PFN_cuModuleUnload PFN_cuModuleUnload_v2000 -#define PFN_cuModuleGetFunction PFN_cuModuleGetFunction_v2000 -#define PFN_cuModuleGetGlobal PFN_cuModuleGetGlobal_v3020 -#define PFN_cuModuleGetTexRef PFN_cuModuleGetTexRef_v2000 -#define PFN_cuModuleGetSurfRef PFN_cuModuleGetSurfRef_v3000 -#define PFN_cuLinkCreate PFN_cuLinkCreate_v6050 -#define PFN_cuLinkAddData PFN_cuLinkAddData_v6050 -#define PFN_cuLinkAddFile PFN_cuLinkAddFile_v6050 -#define PFN_cuLinkComplete PFN_cuLinkComplete_v5050 -#define PFN_cuLinkDestroy PFN_cuLinkDestroy_v5050 -#define PFN_cuMemGetInfo PFN_cuMemGetInfo_v3020 -#define PFN_cuMemAlloc PFN_cuMemAlloc_v3020 -#define PFN_cuMemAllocPitch PFN_cuMemAllocPitch_v3020 -#define PFN_cuMemFree PFN_cuMemFree_v3020 -#define PFN_cuMemGetAddressRange PFN_cuMemGetAddressRange_v3020 -#define PFN_cuMemAllocHost PFN_cuMemAllocHost_v3020 -#define PFN_cuMemFreeHost PFN_cuMemFreeHost_v2000 -#define PFN_cuMemHostAlloc PFN_cuMemHostAlloc_v2020 -#define PFN_cuMemHostGetDevicePointer PFN_cuMemHostGetDevicePointer_v3020 -#define PFN_cuMemHostGetFlags PFN_cuMemHostGetFlags_v2030 -#define PFN_cuMemAllocManaged PFN_cuMemAllocManaged_v6000 -#define PFN_cuDeviceGetByPCIBusId PFN_cuDeviceGetByPCIBusId_v4010 -#define PFN_cuDeviceGetPCIBusId PFN_cuDeviceGetPCIBusId_v4010 -#define PFN_cuIpcGetEventHandle PFN_cuIpcGetEventHandle_v4010 -#define PFN_cuIpcOpenEventHandle PFN_cuIpcOpenEventHandle_v4010 -#define PFN_cuIpcGetMemHandle PFN_cuIpcGetMemHandle_v4010 -#define PFN_cuIpcOpenMemHandle PFN_cuIpcOpenMemHandle_v11000 -#define PFN_cuIpcCloseMemHandle PFN_cuIpcCloseMemHandle_v4010 -#define PFN_cuMemHostRegister PFN_cuMemHostRegister_v6050 -#define PFN_cuMemHostUnregister PFN_cuMemHostUnregister_v4000 -#define PFN_cuMemcpy __API_TYPEDEF_PTDS(PFN_cuMemcpy, 4000, 7000) -#define PFN_cuMemcpyPeer __API_TYPEDEF_PTDS(PFN_cuMemcpyPeer, 4000, 7000) -#define PFN_cuMemcpyHtoD __API_TYPEDEF_PTDS(PFN_cuMemcpyHtoD, 3020, 7000) -#define PFN_cuMemcpyDtoH __API_TYPEDEF_PTDS(PFN_cuMemcpyDtoH, 3020, 7000) -#define PFN_cuMemcpyDtoD __API_TYPEDEF_PTDS(PFN_cuMemcpyDtoD, 3020, 7000) -#define PFN_cuMemcpyDtoA __API_TYPEDEF_PTDS(PFN_cuMemcpyDtoA, 3020, 7000) -#define PFN_cuMemcpyAtoD __API_TYPEDEF_PTDS(PFN_cuMemcpyAtoD, 3020, 7000) -#define PFN_cuMemcpyHtoA __API_TYPEDEF_PTDS(PFN_cuMemcpyHtoA, 3020, 7000) -#define PFN_cuMemcpyAtoH __API_TYPEDEF_PTDS(PFN_cuMemcpyAtoH, 3020, 7000) -#define PFN_cuMemcpyAtoA __API_TYPEDEF_PTDS(PFN_cuMemcpyAtoA, 3020, 7000) -#define PFN_cuMemcpy2D __API_TYPEDEF_PTDS(PFN_cuMemcpy2D, 3020, 7000) -#define PFN_cuMemcpy2DUnaligned __API_TYPEDEF_PTDS(PFN_cuMemcpy2DUnaligned, 3020, 7000) -#define PFN_cuMemcpy3D __API_TYPEDEF_PTDS(PFN_cuMemcpy3D, 3020, 7000) -#define PFN_cuMemcpy3DPeer __API_TYPEDEF_PTDS(PFN_cuMemcpy3DPeer, 4000, 7000) -#define PFN_cuMemcpyAsync __API_TYPEDEF_PTSZ(PFN_cuMemcpyAsync, 4000, 7000) -#define PFN_cuMemcpyPeerAsync __API_TYPEDEF_PTSZ(PFN_cuMemcpyPeerAsync, 4000, 7000) -#define PFN_cuMemcpyHtoDAsync __API_TYPEDEF_PTSZ(PFN_cuMemcpyHtoDAsync, 3020, 7000) -#define PFN_cuMemcpyDtoHAsync __API_TYPEDEF_PTSZ(PFN_cuMemcpyDtoHAsync, 3020, 7000) -#define PFN_cuMemcpyDtoDAsync __API_TYPEDEF_PTSZ(PFN_cuMemcpyDtoDAsync, 3020, 7000) -#define PFN_cuMemcpyHtoAAsync __API_TYPEDEF_PTSZ(PFN_cuMemcpyHtoAAsync, 3020, 7000) -#define PFN_cuMemcpyAtoHAsync __API_TYPEDEF_PTSZ(PFN_cuMemcpyAtoHAsync, 3020, 7000) -#define PFN_cuMemcpy2DAsync __API_TYPEDEF_PTSZ(PFN_cuMemcpy2DAsync, 3020, 7000) -#define PFN_cuMemcpy3DAsync __API_TYPEDEF_PTSZ(PFN_cuMemcpy3DAsync, 3020, 7000) -#define PFN_cuMemcpy3DPeerAsync __API_TYPEDEF_PTSZ(PFN_cuMemcpy3DPeerAsync, 4000, 7000) -#define PFN_cuMemsetD8 __API_TYPEDEF_PTDS(PFN_cuMemsetD8, 3020, 7000) -#define PFN_cuMemsetD16 __API_TYPEDEF_PTDS(PFN_cuMemsetD16, 3020, 7000) -#define PFN_cuMemsetD32 __API_TYPEDEF_PTDS(PFN_cuMemsetD32, 3020, 7000) -#define PFN_cuMemsetD2D8 __API_TYPEDEF_PTDS(PFN_cuMemsetD2D8, 3020, 7000) -#define PFN_cuMemsetD2D16 __API_TYPEDEF_PTDS(PFN_cuMemsetD2D16, 3020, 7000) -#define PFN_cuMemsetD2D32 __API_TYPEDEF_PTDS(PFN_cuMemsetD2D32, 3020, 7000) -#define PFN_cuMemsetD8Async __API_TYPEDEF_PTSZ(PFN_cuMemsetD8Async, 3020, 7000) -#define PFN_cuMemsetD16Async __API_TYPEDEF_PTSZ(PFN_cuMemsetD16Async, 3020, 7000) -#define PFN_cuMemsetD32Async __API_TYPEDEF_PTSZ(PFN_cuMemsetD32Async, 3020, 7000) -#define PFN_cuMemsetD2D8Async __API_TYPEDEF_PTSZ(PFN_cuMemsetD2D8Async, 3020, 7000) -#define PFN_cuMemsetD2D16Async __API_TYPEDEF_PTSZ(PFN_cuMemsetD2D16Async, 3020, 7000) -#define PFN_cuMemsetD2D32Async __API_TYPEDEF_PTSZ(PFN_cuMemsetD2D32Async, 3020, 7000) -#define PFN_cuArrayCreate PFN_cuArrayCreate_v3020 -#define PFN_cuArrayGetDescriptor PFN_cuArrayGetDescriptor_v3020 -#define PFN_cuArrayGetSparseProperties PFN_cuArrayGetSparseProperties_v11010 -#define PFN_cuMipmappedArrayGetSparseProperties PFN_cuMipmappedArrayGetSparseProperties_v11010 -#define PFN_cuArrayGetMemoryRequirements PFN_cuArrayGetMemoryRequirements_v11060 -#define PFN_cuMipmappedArrayGetMemoryRequirements PFN_cuMipmappedArrayGetMemoryRequirements_v11060 -#define PFN_cuArrayGetPlane PFN_cuArrayGetPlane_v11020 -#define PFN_cuArrayDestroy PFN_cuArrayDestroy_v2000 -#define PFN_cuArray3DCreate PFN_cuArray3DCreate_v3020 -#define PFN_cuArray3DGetDescriptor PFN_cuArray3DGetDescriptor_v3020 -#define PFN_cuMipmappedArrayCreate PFN_cuMipmappedArrayCreate_v5000 -#define PFN_cuMipmappedArrayGetLevel PFN_cuMipmappedArrayGetLevel_v5000 -#define PFN_cuMipmappedArrayDestroy PFN_cuMipmappedArrayDestroy_v5000 -#define PFN_cuMemAddressReserve PFN_cuMemAddressReserve_v10020 -#define PFN_cuMemAddressFree PFN_cuMemAddressFree_v10020 -#define PFN_cuMemCreate PFN_cuMemCreate_v10020 -#define PFN_cuMemRelease PFN_cuMemRelease_v10020 -#define PFN_cuMemMap PFN_cuMemMap_v10020 -#define PFN_cuMemMapArrayAsync __API_TYPEDEF_PTSZ(PFN_cuMemMapArrayAsync, 11010, 11010) -#define PFN_cuMemUnmap PFN_cuMemUnmap_v10020 -#define PFN_cuMemSetAccess PFN_cuMemSetAccess_v10020 -#define PFN_cuMemGetAccess PFN_cuMemGetAccess_v10020 -#define PFN_cuMemExportToShareableHandle PFN_cuMemExportToShareableHandle_v10020 -#define PFN_cuMemImportFromShareableHandle PFN_cuMemImportFromShareableHandle_v10020 -#define PFN_cuMemGetAllocationGranularity PFN_cuMemGetAllocationGranularity_v10020 -#define PFN_cuMemGetAllocationPropertiesFromHandle PFN_cuMemGetAllocationPropertiesFromHandle_v10020 -#define PFN_cuMemRetainAllocationHandle PFN_cuMemRetainAllocationHandle_v11000 -#define PFN_cuMemFreeAsync __API_TYPEDEF_PTSZ(PFN_cuMemFreeAsync, 11020, 11020) -#define PFN_cuMemAllocAsync __API_TYPEDEF_PTSZ(PFN_cuMemAllocAsync, 11020, 11020) -#define PFN_cuMemPoolTrimTo PFN_cuMemPoolTrimTo_v11020 -#define PFN_cuMemPoolSetAttribute PFN_cuMemPoolSetAttribute_v11020 -#define PFN_cuMemPoolGetAttribute PFN_cuMemPoolGetAttribute_v11020 -#define PFN_cuMemPoolSetAccess PFN_cuMemPoolSetAccess_v11020 -#define PFN_cuMemPoolGetAccess PFN_cuMemPoolGetAccess_v11020 -#define PFN_cuMemPoolCreate PFN_cuMemPoolCreate_v11020 -#define PFN_cuMemPoolDestroy PFN_cuMemPoolDestroy_v11020 -#define PFN_cuMemAllocFromPoolAsync __API_TYPEDEF_PTSZ(PFN_cuMemAllocFromPoolAsync, 11020, 11020) -#define PFN_cuMemPoolExportToShareableHandle PFN_cuMemPoolExportToShareableHandle_v11020 -#define PFN_cuMemPoolImportFromShareableHandle PFN_cuMemPoolImportFromShareableHandle_v11020 -#define PFN_cuMemPoolExportPointer PFN_cuMemPoolExportPointer_v11020 -#define PFN_cuMemPoolImportPointer PFN_cuMemPoolImportPointer_v11020 -#define PFN_cuPointerGetAttribute PFN_cuPointerGetAttribute_v4000 -#define PFN_cuMemPrefetchAsync __API_TYPEDEF_PTSZ(PFN_cuMemPrefetchAsync, 8000, 8000) -#define PFN_cuMemAdvise PFN_cuMemAdvise_v8000 -#define PFN_cuMemRangeGetAttribute PFN_cuMemRangeGetAttribute_v8000 -#define PFN_cuMemRangeGetAttributes PFN_cuMemRangeGetAttributes_v8000 -#define PFN_cuMulticastCreate PFN_cuMulticastCreate_v12010 -#define PFN_cuMulticastAddDevice PFN_cuMulticastAddDevice_v12010 -#define PFN_cuMulticastBindMem PFN_cuMulticastBindMem_v12010 -#define PFN_cuMulticastBindAddr PFN_cuMulticastBindAddr_v12010 -#define PFN_cuMulticastUnbind PFN_cuMulticastUnbind_v12010 -#define PFN_cuMulticastGetGranularity PFN_cuMulticastGetGranularity_v12010 -#define PFN_cuPointerSetAttribute PFN_cuPointerSetAttribute_v6000 -#define PFN_cuPointerGetAttributes PFN_cuPointerGetAttributes_v7000 -#define PFN_cuStreamCreate PFN_cuStreamCreate_v2000 -#define PFN_cuStreamCreateWithPriority PFN_cuStreamCreateWithPriority_v5050 -#define PFN_cuStreamGetId __API_TYPEDEF_PTSZ(PFN_cuStreamGetId_v12000, 12000, 12000) -#define PFN_cuStreamGetPriority __API_TYPEDEF_PTSZ(PFN_cuStreamGetPriority, 5050, 7000) -#define PFN_cuStreamGetFlags __API_TYPEDEF_PTSZ(PFN_cuStreamGetFlags, 5050, 7000) -#define PFN_cuStreamGetCtx __API_TYPEDEF_PTSZ(PFN_cuStreamGetCtx, 9020, 9020) -#define PFN_cuStreamWaitEvent __API_TYPEDEF_PTSZ(PFN_cuStreamWaitEvent, 3020, 7000) -#define PFN_cuStreamAddCallback __API_TYPEDEF_PTSZ(PFN_cuStreamAddCallback, 5000, 7000) -#define PFN_cuStreamBeginCapture __API_TYPEDEF_PTSZ(PFN_cuStreamBeginCapture, 10010, 10010) -#define PFN_cuThreadExchangeStreamCaptureMode PFN_cuThreadExchangeStreamCaptureMode_v10010 -#define PFN_cuStreamEndCapture __API_TYPEDEF_PTSZ(PFN_cuStreamEndCapture, 10000, 10000) -#define PFN_cuStreamIsCapturing __API_TYPEDEF_PTSZ(PFN_cuStreamIsCapturing, 10000, 10000) -#define PFN_cuStreamGetCaptureInfo __API_TYPEDEF_PTSZ(PFN_cuStreamGetCaptureInfo, 10010, 10010) -#define PFN_cuStreamGetCaptureInfo_v2 __API_TYPEDEF_PTSZ(PFN_cuStreamGetCaptureInfo, 11030, 11030) -#define PFN_cuStreamUpdateCaptureDependencies __API_TYPEDEF_PTSZ(PFN_cuStreamUpdateCaptureDependencies, 11030, 11030) -#define PFN_cuStreamAttachMemAsync __API_TYPEDEF_PTSZ(PFN_cuStreamAttachMemAsync, 6000, 7000) -#define PFN_cuStreamQuery __API_TYPEDEF_PTSZ(PFN_cuStreamQuery, 2000, 7000) -#define PFN_cuStreamSynchronize __API_TYPEDEF_PTSZ(PFN_cuStreamSynchronize, 2000, 7000) -#define PFN_cuStreamDestroy PFN_cuStreamDestroy_v4000 -#define PFN_cuStreamCopyAttributes __API_TYPEDEF_PTSZ(PFN_cuStreamCopyAttributes, 11000, 11000) -#define PFN_cuStreamGetAttribute __API_TYPEDEF_PTSZ(PFN_cuStreamGetAttribute, 11000, 11000) -#define PFN_cuStreamSetAttribute __API_TYPEDEF_PTSZ(PFN_cuStreamSetAttribute, 11000, 11000) -#define PFN_cuEventCreate PFN_cuEventCreate_v2000 -#define PFN_cuEventRecord __API_TYPEDEF_PTSZ(PFN_cuEventRecord, 2000, 7000) -#define PFN_cuEventRecordWithFlags __API_TYPEDEF_PTSZ(PFN_cuEventRecordWithFlags, 11010, 11010) -#define PFN_cuEventQuery PFN_cuEventQuery_v2000 -#define PFN_cuEventSynchronize PFN_cuEventSynchronize_v2000 -#define PFN_cuEventDestroy PFN_cuEventDestroy_v4000 -#define PFN_cuEventElapsedTime PFN_cuEventElapsedTime_v2000 -#define PFN_cuImportExternalMemory PFN_cuImportExternalMemory_v10000 -#define PFN_cuExternalMemoryGetMappedBuffer PFN_cuExternalMemoryGetMappedBuffer_v10000 -#define PFN_cuExternalMemoryGetMappedMipmappedArray PFN_cuExternalMemoryGetMappedMipmappedArray_v10000 -#define PFN_cuDestroyExternalMemory PFN_cuDestroyExternalMemory_v10000 -#define PFN_cuImportExternalSemaphore PFN_cuImportExternalSemaphore_v10000 -#define PFN_cuSignalExternalSemaphoresAsync __API_TYPEDEF_PTSZ(PFN_cuSignalExternalSemaphoresAsync, 10000, 10000) -#define PFN_cuWaitExternalSemaphoresAsync __API_TYPEDEF_PTSZ(PFN_cuWaitExternalSemaphoresAsync, 10000, 10000) -#define PFN_cuDestroyExternalSemaphore PFN_cuDestroyExternalSemaphore_v10000 -#define PFN_cuStreamWaitValue32 __API_TYPEDEF_PTSZ(PFN_cuStreamWaitValue32, 8000, 8000) -#define PFN_cuStreamWaitValue64 __API_TYPEDEF_PTSZ(PFN_cuStreamWaitValue64, 9000, 9000) -#define PFN_cuStreamWriteValue32 __API_TYPEDEF_PTSZ(PFN_cuStreamWriteValue32, 8000, 8000) -#define PFN_cuStreamWriteValue64 __API_TYPEDEF_PTSZ(PFN_cuStreamWriteValue64, 9000, 9000) -#define PFN_cuStreamBatchMemOp __API_TYPEDEF_PTSZ(PFN_cuStreamBatchMemOp, 8000, 8000) -#define PFN_cuStreamWaitValue32_v2 __API_TYPEDEF_PTSZ(PFN_cuStreamWaitValue32, 11070, 11070) -#define PFN_cuStreamWaitValue64_v2 __API_TYPEDEF_PTSZ(PFN_cuStreamWaitValue64, 11070, 11070) -#define PFN_cuStreamWriteValue32_v2 __API_TYPEDEF_PTSZ(PFN_cuStreamWriteValue32, 11070, 11070) -#define PFN_cuStreamWriteValue64_v2 __API_TYPEDEF_PTSZ(PFN_cuStreamWriteValue64, 11070, 11070) -#define PFN_cuStreamBatchMemOp_v2 __API_TYPEDEF_PTSZ(PFN_cuStreamBatchMemOp, 11070, 11070) -#define PFN_cuFuncGetAttribute PFN_cuFuncGetAttribute_v2020 -#define PFN_cuFuncSetAttribute PFN_cuFuncSetAttribute_v9000 -#define PFN_cuFuncSetCacheConfig PFN_cuFuncSetCacheConfig_v3000 -#define PFN_cuFuncSetSharedMemConfig PFN_cuFuncSetSharedMemConfig_v4020 -#define PFN_cuLaunchKernel __API_TYPEDEF_PTSZ(PFN_cuLaunchKernel, 4000, 7000) -#define PFN_cuLaunchKernelEx __API_TYPEDEF_PTSZ(PFN_cuLaunchKernelEx, 11060, 11060) -#define PFN_cuLaunchCooperativeKernel __API_TYPEDEF_PTSZ(PFN_cuLaunchCooperativeKernel, 9000, 9000) -#define PFN_cuLaunchCooperativeKernelMultiDevice PFN_cuLaunchCooperativeKernelMultiDevice_v9000 -#define PFN_cuLaunchHostFunc __API_TYPEDEF_PTSZ(PFN_cuLaunchHostFunc, 10000, 10000) -#define PFN_cuFuncSetBlockShape PFN_cuFuncSetBlockShape_v2000 -#define PFN_cuFuncSetSharedSize PFN_cuFuncSetSharedSize_v2000 -#define PFN_cuParamSetSize PFN_cuParamSetSize_v2000 -#define PFN_cuParamSeti PFN_cuParamSeti_v2000 -#define PFN_cuParamSetf PFN_cuParamSetf_v2000 -#define PFN_cuParamSetv PFN_cuParamSetv_v2000 -#define PFN_cuLaunch PFN_cuLaunch_v2000 -#define PFN_cuLaunchGrid PFN_cuLaunchGrid_v2000 -#define PFN_cuLaunchGridAsync PFN_cuLaunchGridAsync_v2000 -#define PFN_cuParamSetTexRef PFN_cuParamSetTexRef_v2000 -#define PFN_cuGraphCreate PFN_cuGraphCreate_v10000 -#define PFN_cuGraphAddKernelNode PFN_cuGraphAddKernelNode_v12000 -#define PFN_cuGraphKernelNodeGetParams PFN_cuGraphKernelNodeGetParams_v12000 -#define PFN_cuGraphKernelNodeSetParams PFN_cuGraphKernelNodeSetParams_v12000 -#define PFN_cuGraphAddMemcpyNode PFN_cuGraphAddMemcpyNode_v10000 -#define PFN_cuGraphMemcpyNodeGetParams PFN_cuGraphMemcpyNodeGetParams_v10000 -#define PFN_cuGraphMemcpyNodeSetParams PFN_cuGraphMemcpyNodeSetParams_v10000 -#define PFN_cuGraphAddMemsetNode PFN_cuGraphAddMemsetNode_v10000 -#define PFN_cuGraphMemsetNodeGetParams PFN_cuGraphMemsetNodeGetParams_v10000 -#define PFN_cuGraphMemsetNodeSetParams PFN_cuGraphMemsetNodeSetParams_v10000 -#define PFN_cuGraphAddHostNode PFN_cuGraphAddHostNode_v10000 -#define PFN_cuGraphHostNodeGetParams PFN_cuGraphHostNodeGetParams_v10000 -#define PFN_cuGraphHostNodeSetParams PFN_cuGraphHostNodeSetParams_v10000 -#define PFN_cuGraphAddChildGraphNode PFN_cuGraphAddChildGraphNode_v10000 -#define PFN_cuGraphChildGraphNodeGetGraph PFN_cuGraphChildGraphNodeGetGraph_v10000 -#define PFN_cuGraphAddEmptyNode PFN_cuGraphAddEmptyNode_v10000 -#define PFN_cuGraphAddEventRecordNode PFN_cuGraphAddEventRecordNode_v11010 -#define PFN_cuGraphEventRecordNodeGetEvent PFN_cuGraphEventRecordNodeGetEvent_v11010 -#define PFN_cuGraphEventRecordNodeSetEvent PFN_cuGraphEventRecordNodeSetEvent_v11010 -#define PFN_cuGraphAddEventWaitNode PFN_cuGraphAddEventWaitNode_v11010 -#define PFN_cuGraphEventWaitNodeGetEvent PFN_cuGraphEventWaitNodeGetEvent_v11010 -#define PFN_cuGraphEventWaitNodeSetEvent PFN_cuGraphEventWaitNodeSetEvent_v11010 -#define PFN_cuGraphAddExternalSemaphoresSignalNode PFN_cuGraphAddExternalSemaphoresSignalNode_v11020 -#define PFN_cuGraphExternalSemaphoresSignalNodeGetParams PFN_cuGraphExternalSemaphoresSignalNodeGetParams_v11020 -#define PFN_cuGraphExternalSemaphoresSignalNodeSetParams PFN_cuGraphExternalSemaphoresSignalNodeSetParams_v11020 -#define PFN_cuGraphAddExternalSemaphoresWaitNode PFN_cuGraphAddExternalSemaphoresWaitNode_v11020 -#define PFN_cuGraphExternalSemaphoresWaitNodeGetParams PFN_cuGraphExternalSemaphoresWaitNodeGetParams_v11020 -#define PFN_cuGraphExternalSemaphoresWaitNodeSetParams PFN_cuGraphExternalSemaphoresWaitNodeSetParams_v11020 -#define PFN_cuGraphAddBatchMemOpNode PFN_cuGraphAddBatchMemOpNode_v11070 -#define PFN_cuGraphBatchMemOpNodeGetParams PFN_cuGraphBatchMemOpNodeGetParams_v11070 -#define PFN_cuGraphBatchMemOpNodeSetParams PFN_cuGraphBatchMemOpNodeSetParams _v11070 -#define PFN_cuGraphExecBatchMemOpNodeSetParams PFN_cuGraphExecBatchMemOpNodeSetParams_v11070 -#define PFN_cuGraphClone PFN_cuGraphClone_v10000 -#define PFN_cuGraphNodeFindInClone PFN_cuGraphNodeFindInClone_v10000 -#define PFN_cuGraphNodeGetType PFN_cuGraphNodeGetType_v10000 -#define PFN_cuGraphGetNodes PFN_cuGraphGetNodes_v10000 -#define PFN_cuGraphGetRootNodes PFN_cuGraphGetRootNodes_v10000 -#define PFN_cuGraphGetEdges PFN_cuGraphGetEdges_v10000 -#define PFN_cuGraphNodeGetDependencies PFN_cuGraphNodeGetDependencies_v10000 -#define PFN_cuGraphNodeGetDependentNodes PFN_cuGraphNodeGetDependentNodes_v10000 -#define PFN_cuGraphAddDependencies PFN_cuGraphAddDependencies_v10000 -#define PFN_cuGraphRemoveDependencies PFN_cuGraphRemoveDependencies_v10000 -#define PFN_cuGraphDestroyNode PFN_cuGraphDestroyNode_v10000 - -#define PFN_cuGraphInstantiate PFN_cuGraphInstantiateWithFlags_v11040 - -#define PFN_cuGraphInstantiateWithFlags PFN_cuGraphInstantiateWithFlags_v11040 -#define PFN_cuGraphInstantiateWithParams __API_TYPEDEF_PTSZ(PFN_cuGraphInstantiateWithParams, 12000, 12000) -#define PFN_cuGraphExecGetFlags PFN_cuGraphExecGetFlags_v12000 -#define PFN_cuGraphExecKernelNodeSetParams PFN_cuGraphExecKernelNodeSetParams_v12000 -#define PFN_cuGraphExecMemcpyNodeSetParams PFN_cuGraphExecMemcpyNodeSetParams_v10020 -#define PFN_cuGraphExecMemsetNodeSetParams PFN_cuGraphExecMemsetNodeSetParams_v10020 -#define PFN_cuGraphExecHostNodeSetParams PFN_cuGraphExecHostNodeSetParams_v10020 -#define PFN_cuGraphExecChildGraphNodeSetParams PFN_cuGraphExecChildGraphNodeSetParams_v11010 -#define PFN_cuGraphExecEventRecordNodeSetEvent PFN_cuGraphExecEventRecordNodeSetEvent_v11010 -#define PFN_cuGraphExecEventWaitNodeSetEvent PFN_cuGraphExecEventWaitNodeSetEvent_v11010 -#define PFN_cuGraphExecExternalSemaphoresSignalNodeSetParams PFN_cuGraphExecExternalSemaphoresSignalNodeSetParams_v11020 -#define PFN_cuGraphExecExternalSemaphoresWaitNodeSetParams PFN_cuGraphExecExternalSemaphoresWaitNodeSetParams_v11020 -#define PFN_cuGraphUpload __API_TYPEDEF_PTSZ(PFN_cuGraphUpload, 11010, 11010) -#define PFN_cuGraphLaunch __API_TYPEDEF_PTSZ(PFN_cuGraphLaunch, 10000, 10000) -#define PFN_cuGraphExecDestroy PFN_cuGraphExecDestroy_v10000 -#define PFN_cuGraphDestroy PFN_cuGraphDestroy_v10000 -#define PFN_cuGraphExecUpdate PFN_cuGraphExecUpdate_v12000 -#define PFN_cuGraphKernelNodeCopyAttributes PFN_cuGraphKernelNodeCopyAttributes_v11000 -#define PFN_cuGraphKernelNodeGetAttribute PFN_cuGraphKernelNodeGetAttribute_v11000 -#define PFN_cuGraphKernelNodeSetAttribute PFN_cuGraphKernelNodeSetAttribute_v11000 -#define PFN_cuGraphDebugDotPrint PFN_cuGraphDebugDotPrint_v11030 -#define PFN_cuGraphAddMemAllocNode PFN_cuGraphAddMemAllocNode_v11040 -#define PFN_cuGraphMemAllocNodeGetParams PFN_cuGraphMemAllocNodeGetParams_v11040 -#define PFN_cuGraphAddMemFreeNode PFN_cuGraphAddMemFreeNode_v11040 -#define PFN_cuGraphMemFreeNodeGetParams PFN_cuGraphMemFreeNodeGetParams_v11040 -#define PFN_cuGraphNodeSetEnabled PFN_cuGraphNodeSetEnabled_v11060 -#define PFN_cuGraphNodeGetEnabled PFN_cuGraphNodeGetEnabled_v11060 -#define PFN_cuDeviceGraphMemTrim PFN_cuDeviceGraphMemTrim_v11040 -#define PFN_cuDeviceGetGraphMemAttribute PFN_cuDeviceGetGraphMemAttribute_v11040 -#define PFN_cuDeviceSetGraphMemAttribute PFN_cuDeviceSetGraphMemAttribute_v11040 -#define PFN_cuOccupancyMaxActiveBlocksPerMultiprocessor PFN_cuOccupancyMaxActiveBlocksPerMultiprocessor_v6050 -#define PFN_cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags PFN_cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags_v7000 -#define PFN_cuOccupancyMaxPotentialBlockSize PFN_cuOccupancyMaxPotentialBlockSize_v6050 -#define PFN_cuOccupancyMaxPotentialBlockSizeWithFlags PFN_cuOccupancyMaxPotentialBlockSizeWithFlags_v7000 -#define PFN_cuOccupancyAvailableDynamicSMemPerBlock PFN_cuOccupancyAvailableDynamicSMemPerBlock_v10020 -#define PFN_cuOccupancyMaxPotentialClusterSize PFN_cuOccupancyMaxPotentialClusterSize_v11070 -#define PFN_cuOccupancyMaxActiveClusters PFN_cuOccupancyMaxActiveClusters_v11070 -#define PFN_cuTexRefSetArray PFN_cuTexRefSetArray_v2000 -#define PFN_cuTexRefSetMipmappedArray PFN_cuTexRefSetMipmappedArray_v5000 -#define PFN_cuTexRefSetAddress PFN_cuTexRefSetAddress_v3020 -#define PFN_cuTexRefSetAddress2D PFN_cuTexRefSetAddress2D_v4010 -#define PFN_cuTexRefSetFormat PFN_cuTexRefSetFormat_v2000 -#define PFN_cuTexRefSetAddressMode PFN_cuTexRefSetAddressMode_v2000 -#define PFN_cuTexRefSetFilterMode PFN_cuTexRefSetFilterMode_v2000 -#define PFN_cuTexRefSetMipmapFilterMode PFN_cuTexRefSetMipmapFilterMode_v5000 -#define PFN_cuTexRefSetMipmapLevelBias PFN_cuTexRefSetMipmapLevelBias_v5000 -#define PFN_cuTexRefSetMipmapLevelClamp PFN_cuTexRefSetMipmapLevelClamp_v5000 -#define PFN_cuTexRefSetMaxAnisotropy PFN_cuTexRefSetMaxAnisotropy_v5000 -#define PFN_cuTexRefSetBorderColor PFN_cuTexRefSetBorderColor_v8000 -#define PFN_cuTexRefSetFlags PFN_cuTexRefSetFlags_v2000 -#define PFN_cuTexRefGetAddress PFN_cuTexRefGetAddress_v3020 -#define PFN_cuTexRefGetArray PFN_cuTexRefGetArray_v2000 -#define PFN_cuTexRefGetMipmappedArray PFN_cuTexRefGetMipmappedArray_v5000 -#define PFN_cuTexRefGetAddressMode PFN_cuTexRefGetAddressMode_v2000 -#define PFN_cuTexRefGetFilterMode PFN_cuTexRefGetFilterMode_v2000 -#define PFN_cuTexRefGetFormat PFN_cuTexRefGetFormat_v2000 -#define PFN_cuTexRefGetMipmapFilterMode PFN_cuTexRefGetMipmapFilterMode_v5000 -#define PFN_cuTexRefGetMipmapLevelBias PFN_cuTexRefGetMipmapLevelBias_v5000 -#define PFN_cuTexRefGetMipmapLevelClamp PFN_cuTexRefGetMipmapLevelClamp_v5000 -#define PFN_cuTexRefGetMaxAnisotropy PFN_cuTexRefGetMaxAnisotropy_v5000 -#define PFN_cuTexRefGetBorderColor PFN_cuTexRefGetBorderColor_v8000 -#define PFN_cuTexRefGetFlags PFN_cuTexRefGetFlags_v2000 -#define PFN_cuTexRefCreate PFN_cuTexRefCreate_v2000 -#define PFN_cuTexRefDestroy PFN_cuTexRefDestroy_v2000 -#define PFN_cuSurfRefSetArray PFN_cuSurfRefSetArray_v3000 -#define PFN_cuSurfRefGetArray PFN_cuSurfRefGetArray_v3000 -#define PFN_cuTexObjectCreate PFN_cuTexObjectCreate_v5000 -#define PFN_cuTexObjectDestroy PFN_cuTexObjectDestroy_v5000 -#define PFN_cuTexObjectGetResourceDesc PFN_cuTexObjectGetResourceDesc_v5000 -#define PFN_cuTexObjectGetTextureDesc PFN_cuTexObjectGetTextureDesc_v5000 -#define PFN_cuTexObjectGetResourceViewDesc PFN_cuTexObjectGetResourceViewDesc_v5000 -#define PFN_cuSurfObjectCreate PFN_cuSurfObjectCreate_v5000 -#define PFN_cuSurfObjectDestroy PFN_cuSurfObjectDestroy_v5000 -#define PFN_cuSurfObjectGetResourceDesc PFN_cuSurfObjectGetResourceDesc_v5000 -#define PFN_cuTensorMapEncodeTiled PFN_cuTensorMapEncodeTiled_v12000 -#define PFN_cuTensorMapEncodeIm2col PFN_cuTensorMapEncodeIm2col_v12000 -#define PFN_cuTensorMapReplaceAddress PFN_cuTensorMapReplaceAddress_v12000 -#define PFN_cuDeviceCanAccessPeer PFN_cuDeviceCanAccessPeer_v4000 -#define PFN_cuCtxEnablePeerAccess PFN_cuCtxEnablePeerAccess_v4000 -#define PFN_cuCtxDisablePeerAccess PFN_cuCtxDisablePeerAccess_v4000 -#define PFN_cuDeviceGetP2PAttribute PFN_cuDeviceGetP2PAttribute_v8000 -#define PFN_cuGraphicsUnregisterResource PFN_cuGraphicsUnregisterResource_v3000 -#define PFN_cuGraphicsSubResourceGetMappedArray PFN_cuGraphicsSubResourceGetMappedArray_v3000 -#define PFN_cuGraphicsResourceGetMappedMipmappedArray PFN_cuGraphicsResourceGetMappedMipmappedArray_v5000 -#define PFN_cuGraphicsResourceGetMappedPointer PFN_cuGraphicsResourceGetMappedPointer_v3020 -#define PFN_cuGraphicsResourceSetMapFlags PFN_cuGraphicsResourceSetMapFlags_v6050 -#define PFN_cuGraphicsMapResources __API_TYPEDEF_PTSZ(PFN_cuGraphicsMapResources, 3000, 7000) -#define PFN_cuGraphicsUnmapResources __API_TYPEDEF_PTSZ(PFN_cuGraphicsUnmapResources, 3000, 7000) -#define PFN_cuGetExportTable PFN_cuGetExportTable_v3000 -#define PFN_cuFuncGetModule PFN_cuFuncGetModule_v11000 -#define PFN_cuFlushGPUDirectRDMAWrites PFN_cuFlushGPUDirectRDMAWrites_v11030 -#define PFN_cuGetProcAddress PFN_cuGetProcAddress_v12000 -#define PFN_cuUserObjectCreate PFN_cuUserObjectCreate_v11030 -#define PFN_cuUserObjectRetain PFN_cuUserObjectRetain_v11030 -#define PFN_cuUserObjectRelease PFN_cuUserObjectRelease_v11030 -#define PFN_cuGraphRetainUserObject PFN_cuGraphRetainUserObject_v11030 -#define PFN_cuGraphReleaseUserObject PFN_cuGraphReleaseUserObject_v11030 -#define PFN_cuModuleGetLoadingMode PFN_cuModuleGetLoadingMode_v11070 -#define PFN_cuMemGetHandleForAddressRange PFN_cuMemGetHandleForAddressRange_v11070 -#define PFN_cuLibraryLoadData PFN_cuLibraryLoadData_v12000 -#define PFN_cuLibraryLoadFromFile PFN_cuLibraryLoadFromFile_v12000 -#define PFN_cuLibraryUnload PFN_cuLibraryUnload_v12000 -#define PFN_cuLibraryGetKernel PFN_cuLibraryGetKernel_v12000 -#define PFN_cuLibraryGetModule PFN_cuLibraryGetModule_v12000 -#define PFN_cuKernelGetFunction PFN_cuKernelGetFunction_v12000 -#define PFN_cuLibraryGetGlobal PFN_cuLibraryGetGlobal_v12000 -#define PFN_cuLibraryGetManaged PFN_cuLibraryGetManaged_v12000 -#define PFN_cuKernelGetAttribute PFN_cuKernelGetAttribute_v12000 -#define PFN_cuKernelSetAttribute PFN_cuKernelSetAttribute_v12000 -#define PFN_cuKernelSetCacheConfig PFN_cuKernelSetCacheConfig_v12000 -#define PFN_cuLibraryGetUnifiedFunction PFN_cuLibraryGetUnifiedFunction_v12000 -#define PFN_cuCoredumpGetAttribute PFN_cuCoredumpGetAttribute_v12010 -#define PFN_cuCoredumpGetAttributeGlobal PFN_cuCoredumpGetAttributeGlobal_v12010 -#define PFN_cuCoredumpSetAttribute PFN_cuCoredumpSetAttribute_v12010 -#define PFN_cuCoredumpSetAttributeGlobal PFN_cuCoredumpSetAttributeGlobal_v12010 - - /* * Type definitions for functions defined in cuda.h */ @@ -483,6 +80,7 @@ typedef CUresult (CUDAAPI *PFN_cuDeviceGetLuid_v10000)(char *luid, unsigned int typedef CUresult (CUDAAPI *PFN_cuDeviceTotalMem_v3020)(size_t *bytes, CUdevice_v1 dev); typedef CUresult (CUDAAPI *PFN_cuDeviceGetTexture1DLinearMaxWidth_v11010)(size_t *maxWidthInElements, CUarray_format format, unsigned numChannels, CUdevice_v1 dev); typedef CUresult (CUDAAPI *PFN_cuDeviceGetAttribute_v2000)(int *pi, CUdevice_attribute attrib, CUdevice_v1 dev); +typedef CUresult (CUDAAPI *PFN_cuDeviceGetHostAtomicCapabilities_v13000)(unsigned int* capabilities, const CUatomicOperation* operations, unsigned int count, CUdevice dev); typedef CUresult (CUDAAPI *PFN_cuDeviceGetNvSciSyncAttributes_v10020)(void *nvSciSyncAttrList, CUdevice_v1 dev, int flags); typedef CUresult (CUDAAPI *PFN_cuDeviceSetMemPool_v11020)(CUdevice_v1 dev, CUmemoryPool pool); typedef CUresult (CUDAAPI *PFN_cuDeviceGetMemPool_v11020)(CUmemoryPool *pool, CUdevice_v1 dev); @@ -497,6 +95,7 @@ typedef CUresult (CUDAAPI *PFN_cuDevicePrimaryCtxReset_v11000)(CUdevice_v1 dev); typedef CUresult (CUDAAPI *PFN_cuDeviceGetExecAffinitySupport_v11040)(int *pi, CUexecAffinityType type, CUdevice dev); typedef CUresult (CUDAAPI *PFN_cuCtxCreate_v3020)(CUcontext *pctx, unsigned int flags, CUdevice_v1 dev); typedef CUresult (CUDAAPI *PFN_cuCtxCreate_v11040)(CUcontext *pctx, CUexecAffinityParam *paramsArray, int numParams, unsigned int flags, CUdevice_v1 dev); +typedef CUresult (CUDAAPI *PFN_cuCtxCreate_v12050)(CUcontext *pctx, CUctxCreateParams *ctxCreateParams, unsigned int flags, CUdevice dev); typedef CUresult (CUDAAPI *PFN_cuCtxGetId_v12000)(CUcontext ctx, unsigned long long *ctxId); typedef CUresult (CUDAAPI *PFN_cuCtxDestroy_v4000)(CUcontext ctx); typedef CUresult (CUDAAPI *PFN_cuCtxPushCurrent_v4000)(CUcontext ctx); @@ -504,9 +103,11 @@ typedef CUresult (CUDAAPI *PFN_cuCtxPopCurrent_v4000)(CUcontext *pctx); typedef CUresult (CUDAAPI *PFN_cuCtxSetCurrent_v4000)(CUcontext ctx); typedef CUresult (CUDAAPI *PFN_cuCtxGetCurrent_v4000)(CUcontext *pctx); typedef CUresult (CUDAAPI *PFN_cuCtxGetDevice_v2000)(CUdevice_v1 *device); +typedef CUresult (CUDAAPI *PFN_cuCtxGetDevice_v13000)(CUdevice *device, CUcontext ctx); typedef CUresult (CUDAAPI *PFN_cuCtxGetFlags_v7000)(unsigned int *flags); typedef CUresult (CUDAAPI *PFN_cuCtxSetFlags_v12010)(unsigned int flags); typedef CUresult (CUDAAPI *PFN_cuCtxSynchronize_v2000)(void); +typedef CUresult (CUDAAPI *PFN_cuCtxSynchronize_v13000)(CUcontext ctx); typedef CUresult (CUDAAPI *PFN_cuCtxSetLimit_v3010)(CUlimit limit, size_t value); typedef CUresult (CUDAAPI *PFN_cuCtxGetLimit_v3010)(size_t *pvalue, CUlimit limit); typedef CUresult (CUDAAPI *PFN_cuCtxGetCacheConfig_v3020)(CUfunc_cache *pconfig); @@ -528,6 +129,8 @@ typedef CUresult (CUDAAPI *PFN_cuModuleGetFunction_v2000)(CUfunction *hfunc, CUm typedef CUresult (CUDAAPI *PFN_cuModuleGetGlobal_v3020)(CUdeviceptr_v2 *dptr, size_t *bytes, CUmodule hmod, const char *name); typedef CUresult (CUDAAPI *PFN_cuModuleGetTexRef_v2000)(CUtexref *pTexRef, CUmodule hmod, const char *name); typedef CUresult (CUDAAPI *PFN_cuModuleGetSurfRef_v3000)(CUsurfref *pSurfRef, CUmodule hmod, const char *name); +typedef CUresult (CUDAAPI *PFN_cuModuleGetFunctionCount_v12040)(unsigned int *count, CUmodule hmod); +typedef CUresult (CUDAAPI *PFN_cuModuleEnumerateFunctions_v12040)(CUfunction *functions, unsigned int numFunctions, CUmodule mod); typedef CUresult (CUDAAPI *PFN_cuLinkCreate_v6050)(unsigned int numOptions, CUjit_option *options, void **optionValues, CUlinkState *stateOut); typedef CUresult (CUDAAPI *PFN_cuLinkAddData_v6050)(CUlinkState state, CUjitInputType type, void *data, size_t size, const char *name, unsigned int numOptions, CUjit_option *options, void **optionValues); typedef CUresult (CUDAAPI *PFN_cuLinkAddFile_v6050)(CUlinkState state, CUjitInputType type, const char *path, unsigned int numOptions, CUjit_option *options, void **optionValues); @@ -577,6 +180,10 @@ typedef CUresult (CUDAAPI *PFN_cuMemcpyAtoHAsync_v7000_ptsz)(void *dstHost, CUar typedef CUresult (CUDAAPI *PFN_cuMemcpy2DAsync_v7000_ptsz)(const CUDA_MEMCPY2D_v2 *pCopy, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuMemcpy3DAsync_v7000_ptsz)(const CUDA_MEMCPY3D_v2 *pCopy, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuMemcpy3DPeerAsync_v7000_ptsz)(const CUDA_MEMCPY3D_PEER_v1 *pCopy, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemcpyBatchAsync_v12080_ptsz)(CUdeviceptr_v2 *dsts, CUdeviceptr_v2 *srcs, size_t *sizes, size_t count, CUmemcpyAttributes_v1 *attrs, size_t *attrIdxs, size_t numAttrs, size_t *failIdx, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemcpy3DBatchAsync_v12080_ptsz)(size_t numParams, CUDA_MEMCPY3D_BATCH_OP_v1 *opList, size_t *failIdx, unsigned long long flags, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemcpyBatchAsync_v13000_ptsz)(CUdeviceptr_v2 *dsts, CUdeviceptr_v2 *srcs, size_t *sizes, size_t count, CUmemcpyAttributes_v1 *attrs, size_t *attrIdxs, size_t numAttrs, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemcpy3DBatchAsync_v13000_ptsz)(size_t numParams, CUDA_MEMCPY3D_BATCH_OP_v1 *opList, unsigned long long flags, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuMemsetD8_v7000_ptds)(CUdeviceptr_v2 dstDevice, unsigned char uc, size_t N); typedef CUresult (CUDAAPI *PFN_cuMemsetD16_v7000_ptds)(CUdeviceptr_v2 dstDevice, unsigned short us, size_t N); typedef CUresult (CUDAAPI *PFN_cuMemsetD32_v7000_ptds)(CUdeviceptr_v2 dstDevice, unsigned int ui, size_t N); @@ -625,6 +232,9 @@ typedef CUresult (CUDAAPI *PFN_cuMemPoolSetAccess_v11020)(CUmemoryPool pool, con typedef CUresult (CUDAAPI *PFN_cuMemPoolGetAccess_v11020)(CUmemAccess_flags *flags, CUmemoryPool memPool, CUmemLocation_v1 *location); typedef CUresult (CUDAAPI *PFN_cuMemPoolCreate_v11020)(CUmemoryPool *pool, const CUmemPoolProps_v1 *poolProps); typedef CUresult (CUDAAPI *PFN_cuMemPoolDestroy_v11020)(CUmemoryPool pool); +typedef CUresult (CUDAAPI *PFN_cuMemGetDefaultMemPool_v13000)(CUmemoryPool *pool_out, CUmemLocation *location, CUmemAllocationType type); +typedef CUresult (CUDAAPI *PFN_cuMemGetMemPool_v13000)(CUmemoryPool *pool_out, CUmemLocation *location, CUmemAllocationType type); +typedef CUresult (CUDAAPI *PFN_cuMemSetMemPool_v13000)(CUmemLocation *location, CUmemAllocationType type, CUmemoryPool pool); typedef CUresult (CUDAAPI *PFN_cuMemAllocFromPoolAsync_v11020_ptsz)(CUdeviceptr_v2 *dptr, size_t bytesize, CUmemoryPool pool, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuMemPoolExportToShareableHandle_v11020)(void *handle_out, CUmemoryPool pool, CUmemAllocationHandleType handleType, unsigned long long flags); typedef CUresult (CUDAAPI *PFN_cuMemPoolImportFromShareableHandle_v11020)(CUmemoryPool *pool_out, void *handle, CUmemAllocationHandleType handleType, unsigned long long flags); @@ -633,6 +243,11 @@ typedef CUresult (CUDAAPI *PFN_cuMemPoolImportPointer_v11020)(CUdeviceptr_v2 *pt typedef CUresult (CUDAAPI *PFN_cuPointerGetAttribute_v4000)(void *data, CUpointer_attribute attribute, CUdeviceptr_v2 ptr); typedef CUresult (CUDAAPI *PFN_cuMemPrefetchAsync_v8000_ptsz)(CUdeviceptr_v2 devPtr, size_t count, CUdevice_v1 dstDevice, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuMemAdvise_v8000)(CUdeviceptr_v2 devPtr, size_t count, CUmem_advise advice, CUdevice_v1 device); +typedef CUresult (CUDAAPI *PFN_cuMemAdvise_v12020)(CUdeviceptr_v2 devPtr, size_t count, CUmem_advise advice, CUmemLocation_v1 location); +typedef CUresult (CUDAAPI *PFN_cuMemPrefetchAsync_v12020_ptsz)(CUdeviceptr_v2 devPtr, size_t count, CUmemLocation_v1 location, unsigned int flags, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemPrefetchBatchAsync_v13000_ptsz)(CUdeviceptr_v2 *dptrs, size_t *sizes, size_t count, CUmemLocation_v1 *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, unsigned long long flags, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemDiscardBatchAsync_v13000_ptsz)(CUdeviceptr_v2 *dptrs, size_t *sizes, size_t count, unsigned long long flags, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemDiscardAndPrefetchBatchAsync_v13000_ptsz)(CUdeviceptr_v2 *dptrs, size_t *sizes, size_t count, CUmemLocation_v1 *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, unsigned long long flags, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuMemRangeGetAttribute_v8000)(void *data, size_t dataSize, CUmem_range_attribute attribute, CUdeviceptr_v2 devPtr, size_t count); typedef CUresult (CUDAAPI *PFN_cuMemRangeGetAttributes_v8000)(void **data, size_t *dataSizes, CUmem_range_attribute *attributes, size_t numAttributes, CUdeviceptr_v2 devPtr, size_t count); typedef CUresult (CUDAAPI *PFN_cuMulticastCreate_v12010)(CUmemGenericAllocationHandle *mcHandle, const CUmulticastObjectProp *prop); @@ -653,12 +268,15 @@ typedef CUresult (CUDAAPI *PFN_cuStreamGetCtx_v9020_ptsz)(CUstream hStream, CUco typedef CUresult (CUDAAPI *PFN_cuStreamWaitEvent_v7000_ptsz)(CUstream hStream, CUevent hEvent, unsigned int Flags); typedef CUresult (CUDAAPI *PFN_cuStreamAddCallback_v7000_ptsz)(CUstream hStream, CUstreamCallback callback, void *userData, unsigned int flags); typedef CUresult (CUDAAPI *PFN_cuStreamBeginCapture_v10010_ptsz)(CUstream hStream, CUstreamCaptureMode mode); +typedef CUresult (CUDAAPI *PFN_cuStreamBeginCaptureToGraph_v12030_ptsz)(CUstream hStream, CUgraph hGraph, const CUgraphNode *dependencies, const CUgraphEdgeData *dependencyData, size_t numDependencies, CUstreamCaptureMode mode); typedef CUresult (CUDAAPI *PFN_cuThreadExchangeStreamCaptureMode_v10010)(CUstreamCaptureMode *mode); typedef CUresult (CUDAAPI *PFN_cuStreamEndCapture_v10000_ptsz)(CUstream hStream, CUgraph *phGraph); typedef CUresult (CUDAAPI *PFN_cuStreamIsCapturing_v10000_ptsz)(CUstream hStream, CUstreamCaptureStatus *captureStatus); typedef CUresult (CUDAAPI *PFN_cuStreamGetCaptureInfo_v10010_ptsz)(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out); typedef CUresult (CUDAAPI *PFN_cuStreamGetCaptureInfo_v11030_ptsz)(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out, CUgraph *graph_out, const CUgraphNode **dependencies_out, size_t *numDependencies_out); +typedef CUresult (CUDAAPI *PFN_cuStreamGetCaptureInfo_v12030_ptsz)(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out, CUgraph *graph_out, const CUgraphNode **dependencies_out, const CUgraphEdgeData **edgeData_out, size_t *numDependencies_out); typedef CUresult (CUDAAPI *PFN_cuStreamUpdateCaptureDependencies_v11030_ptsz)(CUstream hStream, CUgraphNode *dependencies, size_t numDependencies, unsigned int flags); +typedef CUresult (CUDAAPI *PFN_cuStreamUpdateCaptureDependencies_v12030_ptsz)(CUstream hStream, CUgraphNode *dependencies, const CUgraphEdgeData *dependencyData, size_t numDependencies, unsigned int flags); typedef CUresult (CUDAAPI *PFN_cuStreamAttachMemAsync_v7000_ptsz)(CUstream hStream, CUdeviceptr_v2 dptr, size_t length, unsigned int flags); typedef CUresult (CUDAAPI *PFN_cuStreamQuery_v7000_ptsz)(CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuStreamSynchronize_v7000_ptsz)(CUstream hStream); @@ -673,6 +291,7 @@ typedef CUresult (CUDAAPI *PFN_cuEventQuery_v2000)(CUevent hEvent); typedef CUresult (CUDAAPI *PFN_cuEventSynchronize_v2000)(CUevent hEvent); typedef CUresult (CUDAAPI *PFN_cuEventDestroy_v4000)(CUevent hEvent); typedef CUresult (CUDAAPI *PFN_cuEventElapsedTime_v2000)(float *pMilliseconds, CUevent hStart, CUevent hEnd); +typedef CUresult (CUDAAPI *PFN_cuEventElapsedTime_v12080)(float *pMilliseconds, CUevent hStart, CUevent hEnd); typedef CUresult (CUDAAPI *PFN_cuImportExternalMemory_v10000)(CUexternalMemory *extMem_out, const CUDA_EXTERNAL_MEMORY_HANDLE_DESC_v1 *memHandleDesc); typedef CUresult (CUDAAPI *PFN_cuExternalMemoryGetMappedBuffer_v10000)(CUdeviceptr_v2 *devPtr, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_BUFFER_DESC_v1 *bufferDesc); typedef CUresult (CUDAAPI *PFN_cuExternalMemoryGetMappedMipmappedArray_v10000)(CUmipmappedArray *mipmap, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC_v1 *mipmapDesc); @@ -695,6 +314,10 @@ typedef CUresult (CUDAAPI *PFN_cuFuncGetAttribute_v2020)(int *pi, CUfunction_att typedef CUresult (CUDAAPI *PFN_cuFuncSetAttribute_v9000)(CUfunction hfunc, CUfunction_attribute attrib, int value); typedef CUresult (CUDAAPI *PFN_cuFuncSetCacheConfig_v3000)(CUfunction hfunc, CUfunc_cache config); typedef CUresult (CUDAAPI *PFN_cuFuncSetSharedMemConfig_v4020)(CUfunction hfunc, CUsharedconfig config); +typedef CUresult (CUDAAPI *PFN_cuFuncGetName_v12030)(const char **name, CUfunction hfunc); +typedef CUresult (CUDAAPI *PFN_cuFuncGetParamInfo_v12040)(CUfunction func, size_t paramIndex, size_t *paramOffset, size_t *paramSize); +typedef CUresult (CUDAAPI *PFN_cuFuncIsLoaded_v12040)(CUfunctionLoadingState *state, CUfunction hfunc); +typedef CUresult (CUDAAPI *PFN_cuFuncLoad_v12040)(CUfunction hfunc); typedef CUresult (CUDAAPI *PFN_cuLaunchKernel_v7000_ptsz)(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void **kernelParams, void **extra); typedef CUresult (CUDAAPI *PFN_cuLaunchKernelEx_v11060_ptsz)(const CUlaunchConfig *config, CUfunction f, void **kernelParams, void **extra); typedef CUresult (CUDAAPI *PFN_cuLaunchCooperativeKernel_v9000_ptsz)(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void **kernelParams); @@ -751,10 +374,15 @@ typedef CUresult (CUDAAPI *PFN_cuGraphNodeGetType_v10000)(CUgraphNode hNode, CUg typedef CUresult (CUDAAPI *PFN_cuGraphGetNodes_v10000)(CUgraph hGraph, CUgraphNode *nodes, size_t *numNodes); typedef CUresult (CUDAAPI *PFN_cuGraphGetRootNodes_v10000)(CUgraph hGraph, CUgraphNode *rootNodes, size_t *numRootNodes); typedef CUresult (CUDAAPI *PFN_cuGraphGetEdges_v10000)(CUgraph hGraph, CUgraphNode *from, CUgraphNode *to, size_t *numEdges); +typedef CUresult (CUDAAPI *PFN_cuGraphGetEdges_v12030)(CUgraph hGraph, CUgraphNode *from, CUgraphNode *to, CUgraphEdgeData *edgeData, size_t *numEdges); typedef CUresult (CUDAAPI *PFN_cuGraphNodeGetDependencies_v10000)(CUgraphNode hNode, CUgraphNode *dependencies, size_t *numDependencies); +typedef CUresult (CUDAAPI *PFN_cuGraphNodeGetDependencies_v12030)(CUgraphNode hNode, CUgraphNode *dependencies, CUgraphEdgeData *edgeData, size_t *numDependencies); typedef CUresult (CUDAAPI *PFN_cuGraphNodeGetDependentNodes_v10000)(CUgraphNode hNode, CUgraphNode *dependentNodes, size_t *numDependentNodes); +typedef CUresult (CUDAAPI *PFN_cuGraphNodeGetDependentNodes_v12030)(CUgraphNode hNode, CUgraphNode *dependentNodes, CUgraphEdgeData *edgeData, size_t *numDependentNodes); typedef CUresult (CUDAAPI *PFN_cuGraphAddDependencies_v10000)(CUgraph hGraph, const CUgraphNode *from, const CUgraphNode *to, size_t numDependencies); +typedef CUresult (CUDAAPI *PFN_cuGraphAddDependencies_v12030)(CUgraph hGraph, const CUgraphNode *from, const CUgraphNode *to, const CUgraphEdgeData *edgeData, size_t numDependencies); typedef CUresult (CUDAAPI *PFN_cuGraphRemoveDependencies_v10000)(CUgraph hGraph, const CUgraphNode *from, const CUgraphNode *to, size_t numDependencies); +typedef CUresult (CUDAAPI *PFN_cuGraphRemoveDependencies_v12030)(CUgraph hGraph, const CUgraphNode *from, const CUgraphNode *to, const CUgraphEdgeData *edgeData, size_t numDependencies); typedef CUresult (CUDAAPI *PFN_cuGraphDestroyNode_v10000)(CUgraphNode hNode); typedef CUresult (CUDAAPI *PFN_cuGraphInstantiateWithFlags_v11040)(CUgraphExec *phGraphExec, CUgraph hGraph, unsigned long long flags); typedef CUresult (CUDAAPI *PFN_cuGraphInstantiateWithParams_v12000_ptsz)(CUgraphExec *phGraphExec, CUgraph hGraph, CUDA_GRAPH_INSTANTIATE_PARAMS *instantiateParams); @@ -785,6 +413,11 @@ typedef CUresult (CUDAAPI *PFN_cuGraphAddMemFreeNode_v11040)(CUgraphNode *phGrap typedef CUresult (CUDAAPI *PFN_cuGraphMemFreeNodeGetParams_v11040)(CUgraphNode hNode, CUdeviceptr *dptr_out); typedef CUresult (CUDAAPI *PFN_cuGraphNodeSetEnabled_v11060)(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int isEnabled); typedef CUresult (CUDAAPI *PFN_cuGraphNodeGetEnabled_v11060)(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int *isEnabled); +typedef CUresult (CUDAAPI *PFN_cuGraphAddNode_v12020)(CUgraphNode *phGraphNode, CUgraph hGraph, const CUgraphNode *dependencies, size_t numDependencies, CUgraphNodeParams *nodeParams); +typedef CUresult (CUDAAPI *PFN_cuGraphAddNode_v12030)(CUgraphNode *phGraphNode, CUgraph hGraph, const CUgraphNode *dependencies, const CUgraphEdgeData *dependencyData, size_t numDependencies, CUgraphNodeParams *nodeParams); +typedef CUresult (CUDAAPI *PFN_cuGraphNodeSetParams_v12020)(CUgraphNode hNode, CUgraphNodeParams *nodeParams); +typedef CUresult (CUDAAPI *PFN_cuGraphExecNodeSetParams_v12020)(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraphNodeParams *nodeParams); +typedef CUresult (CUDAAPI *PFN_cuGraphConditionalHandleCreate_v12030)(CUgraphConditionalHandle *pHandle_out, CUgraph hGraph, CUcontext ctx, unsigned int defaultLaunchValue, unsigned int flags); typedef CUresult (CUDAAPI *PFN_cuDeviceGraphMemTrim_v11040)(CUdevice device); typedef CUresult (CUDAAPI *PFN_cuDeviceGetGraphMemAttribute_v11040)(CUdevice device, CUgraphMem_attribute attr, void* value); typedef CUresult (CUDAAPI *PFN_cuDeviceSetGraphMemAttribute_v11040)(CUdevice device, CUgraphMem_attribute attr, void* value); @@ -835,10 +468,12 @@ typedef CUresult (CUDAAPI *PFN_cuSurfObjectGetResourceDesc_v5000)(CUDA_RESOURCE_ typedef CUresult (CUDAAPI *PFN_cuTensorMapEncodeTiled_v12000)(CUtensorMap *tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void *globalAddress, const cuuint64_t *globalDim, const cuuint64_t *globalStrides, const cuuint32_t *boxDim, const cuuint32_t *elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill); typedef CUresult (CUDAAPI *PFN_cuTensorMapEncodeIm2col_v12000)(CUtensorMap *tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void *globalAddress, const cuuint64_t *globalDim, const cuuint64_t *globalStrides, const int *pixelBoxLowerCorner, const int *pixelBoxUpperCorner, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t *elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill); typedef CUresult (CUDAAPI *PFN_cuTensorMapReplaceAddress_v12000)(CUtensorMap *tensorMap, void *globalAddress); +typedef CUresult (CUDAAPI *PFN_cuTensorMapEncodeIm2colWide_v12080)(CUtensorMap *tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void *globalAddress, const cuuint64_t *globalDim, const cuuint64_t *globalStrides, int pixelBoxLowerCornerWidth, int pixelBoxUpperCornerWidth, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t *elementStrides, CUtensorMapInterleave interleave, CUtensorMapIm2ColWideMode mode, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill); typedef CUresult (CUDAAPI *PFN_cuDeviceCanAccessPeer_v4000)(int *canAccessPeer, CUdevice_v1 dev, CUdevice_v1 peerDev); typedef CUresult (CUDAAPI *PFN_cuCtxEnablePeerAccess_v4000)(CUcontext peerContext, unsigned int Flags); typedef CUresult (CUDAAPI *PFN_cuCtxDisablePeerAccess_v4000)(CUcontext peerContext); typedef CUresult (CUDAAPI *PFN_cuDeviceGetP2PAttribute_v8000)(int *value, CUdevice_P2PAttribute attrib, CUdevice_v1 srcDevice, CUdevice_v1 dstDevice); +typedef CUresult (CUDAAPI *PFN_cuDeviceGetP2PAtomicCapabilities_v13000)(unsigned int* capabilities, const CUatomicOperation* operations, unsigned int count, CUdevice srcDevice, CUdevice dstDevice); typedef CUresult (CUDAAPI *PFN_cuGraphicsUnregisterResource_v3000)(CUgraphicsResource resource); typedef CUresult (CUDAAPI *PFN_cuGraphicsSubResourceGetMappedArray_v3000)(CUarray *pArray, CUgraphicsResource resource, unsigned int arrayIndex, unsigned int mipLevel); typedef CUresult (CUDAAPI *PFN_cuGraphicsResourceGetMappedMipmappedArray_v5000)(CUmipmappedArray *pMipmappedArray, CUgraphicsResource resource); @@ -880,6 +515,10 @@ typedef CUresult (CUDAAPI *PFN_cuMemcpyPeer_v4000)(CUdeviceptr_v2 dstDevice, CUc typedef CUresult (CUDAAPI *PFN_cuMemcpyPeerAsync_v4000)(CUdeviceptr_v2 dstDevice, CUcontext dstContext, CUdeviceptr_v2 srcDevice, CUcontext srcContext, size_t ByteCount, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuMemcpy3DPeer_v4000)(const CUDA_MEMCPY3D_PEER_v1 *pCopy); typedef CUresult (CUDAAPI *PFN_cuMemcpy3DPeerAsync_v4000)(const CUDA_MEMCPY3D_PEER_v1 *pCopy, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemcpyBatchAsync_v12080)(CUdeviceptr_v2 *dsts, CUdeviceptr_v2 *srcs, size_t *sizes, size_t count, CUmemcpyAttributes_v1 *attrs, size_t *attrIdxs, size_t numAttrs, size_t *failIdx, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemcpy3DBatchAsync_v12080)(size_t numParams, CUDA_MEMCPY3D_BATCH_OP_v1 *opList, size_t *failIdx, unsigned long long flags, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemcpyBatchAsync_v13000)(CUdeviceptr_v2 *dsts, CUdeviceptr_v2 *srcs, size_t *sizes, size_t count, CUmemcpyAttributes_v1 *attrs, size_t *attrIdxs, size_t numAttrs, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemcpy3DBatchAsync_v13000)(size_t numParams, CUDA_MEMCPY3D_BATCH_OP_v1 *opList, unsigned long long flags, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuMemsetD8Async_v3020)(CUdeviceptr_v2 dstDevice, unsigned char uc, size_t N, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuMemsetD16Async_v3020)(CUdeviceptr_v2 dstDevice, unsigned short us, size_t N, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuMemsetD32Async_v3020)(CUdeviceptr_v2 dstDevice, unsigned int ui, size_t N, CUstream hStream); @@ -889,6 +528,8 @@ typedef CUresult (CUDAAPI *PFN_cuMemsetD2D32Async_v3020)(CUdeviceptr_v2 dstDevic typedef CUresult (CUDAAPI *PFN_cuStreamGetPriority_v5050)(CUstream hStream, int *priority); typedef CUresult (CUDAAPI *PFN_cuStreamGetFlags_v5050)(CUstream hStream, unsigned int *flags); typedef CUresult (CUDAAPI *PFN_cuStreamGetCtx_v9020)(CUstream hStream, CUcontext *pctx); +typedef CUresult (CUDAAPI *PFN_cuStreamGetDevice_v12080)(CUstream hStream, CUdevice *device); +typedef CUresult (CUDAAPI *PFN_cuStreamGetDevice_v12080_ptsz)(CUstream hStream, CUdevice *device); typedef CUresult (CUDAAPI *PFN_cuStreamWaitEvent_v3020)(CUstream hStream, CUevent hEvent, unsigned int Flags); typedef CUresult (CUDAAPI *PFN_cuStreamAddCallback_v5000)(CUstream hStream, CUstreamCallback callback, void *userData, unsigned int flags); typedef CUresult (CUDAAPI *PFN_cuStreamAttachMemAsync_v6000)(CUstream hStream, CUdeviceptr_v2 dptr, size_t length, unsigned int flags); @@ -912,15 +553,22 @@ typedef CUresult (CUDAAPI *PFN_cuStreamWriteValue64_v11070)(CUstream stream, CUd typedef CUresult (CUDAAPI *PFN_cuStreamWaitValue64_v11070)(CUstream stream, CUdeviceptr_v2 addr, cuuint64_t value, unsigned int flags); typedef CUresult (CUDAAPI *PFN_cuStreamBatchMemOp_v11070)(CUstream stream, unsigned int count, CUstreamBatchMemOpParams *paramArray, unsigned int flags); typedef CUresult (CUDAAPI *PFN_cuMemPrefetchAsync_v8000)(CUdeviceptr_v2 devPtr, size_t count, CUdevice_v1 dstDevice, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemPrefetchAsync_v12020)(CUdeviceptr_v2 devPtr, size_t count, CUmemLocation_v1 location, unsigned int flags, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemPrefetchBatchAsync_v13000)(CUdeviceptr_v2 *dptrs, size_t *sizes, size_t count, CUmemLocation_v1 *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, unsigned long long flags, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemDiscardBatchAsync_v13000)(CUdeviceptr_v2 *dptrs, size_t *sizes, size_t count, unsigned long long flags, CUstream hStream); +typedef CUresult (CUDAAPI *PFN_cuMemDiscardAndPrefetchBatchAsync_v13000)(CUdeviceptr_v2 *dptrs, size_t *sizes, size_t count, CUmemLocation_v1 *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, unsigned long long flags, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuLaunchCooperativeKernel_v9000)(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void **kernelParams); typedef CUresult (CUDAAPI *PFN_cuSignalExternalSemaphoresAsync_v10000)(const CUexternalSemaphore *extSemArray, const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS_v1 *paramsArray, unsigned int numExtSems, CUstream stream); typedef CUresult (CUDAAPI *PFN_cuWaitExternalSemaphoresAsync_v10000)(const CUexternalSemaphore *extSemArray, const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS_v1 *paramsArray, unsigned int numExtSems, CUstream stream); typedef CUresult (CUDAAPI *PFN_cuStreamBeginCapture_v10010)(CUstream hStream, CUstreamCaptureMode mode); +typedef CUresult (CUDAAPI *PFN_cuStreamBeginCaptureToGraph_v12030)(CUstream hStream, CUgraph hGraph, const CUgraphNode *dependencies, const CUgraphEdgeData *dependencyData, size_t numDependencies, CUstreamCaptureMode mode); typedef CUresult (CUDAAPI *PFN_cuStreamEndCapture_v10000)(CUstream hStream, CUgraph *phGraph); typedef CUresult (CUDAAPI *PFN_cuStreamIsCapturing_v10000)(CUstream hStream, CUstreamCaptureStatus *captureStatus); typedef CUresult (CUDAAPI *PFN_cuStreamGetCaptureInfo_v10010)(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out); typedef CUresult (CUDAAPI *PFN_cuStreamGetCaptureInfo_v11030)(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out, CUgraph *graph_out, const CUgraphNode **dependencies_out, size_t *numDependencies_out); +typedef CUresult (CUDAAPI *PFN_cuStreamGetCaptureInfo_v12030)(CUstream hStream, CUstreamCaptureStatus *captureStatus_out, cuuint64_t *id_out, CUgraph *graph_out, const CUgraphNode **dependencies_out, const CUgraphEdgeData **edgeData_out, size_t *numDependencies_out); typedef CUresult (CUDAAPI *PFN_cuStreamUpdateCaptureDependencies_v11030)(CUstream hStream, CUgraphNode *dependencies, size_t numDependencies, unsigned int flags); +typedef CUresult (CUDAAPI *PFN_cuStreamUpdateCaptureDependencies_v12030)(CUstream hStream, CUgraphNode *dependencies, const CUgraphEdgeData *dependencyData, size_t numDependencies, unsigned int flags); typedef CUresult (CUDAAPI *PFN_cuGraphInstantiateWithParams_v12000)(CUgraphExec *phGraphExec, CUgraph hGraph, CUDA_GRAPH_INSTANTIATE_PARAMS *instantiateParams); typedef CUresult (CUDAAPI *PFN_cuGraphUpload_v11010)(CUgraphExec hGraph, CUstream hStream); typedef CUresult (CUDAAPI *PFN_cuGraphLaunch_v10000)(CUgraphExec hGraph, CUstream hStream); @@ -944,17 +592,48 @@ typedef CUresult (CUDAAPI *PFN_cuLibraryLoadFromFile_v12000)(CUlibrary *library, typedef CUresult (CUDAAPI *PFN_cuLibraryUnload_v12000)(CUlibrary library); typedef CUresult (CUDAAPI *PFN_cuLibraryGetKernel_v12000)(CUkernel *pKernel, CUlibrary library, const char *name); typedef CUresult (CUDAAPI *PFN_cuLibraryGetModule_v12000)(CUmodule *pMod, CUlibrary library); +typedef CUresult (CUDAAPI *PFN_cuLibraryGetKernelCount_v12040)(unsigned int *count, CUlibrary lib); +typedef CUresult (CUDAAPI *PFN_cuLibraryEnumerateKernels_v12040)(CUkernel *kernels, unsigned int numKernels, CUlibrary lib); typedef CUresult (CUDAAPI *PFN_cuKernelGetFunction_v12000)(CUfunction *pFunc, CUkernel kernel); +typedef CUresult (CUDAAPI *PFN_cuKernelGetLibrary_v12050)(CUlibrary *pLib, CUkernel kernel); typedef CUresult (CUDAAPI *PFN_cuLibraryGetGlobal_v12000)(CUdeviceptr *dptr, size_t *bytes, CUlibrary library, const char *name); typedef CUresult (CUDAAPI *PFN_cuLibraryGetManaged_v12000)(CUdeviceptr *dptr, size_t *bytes, CUlibrary library, const char *name); typedef CUresult (CUDAAPI *PFN_cuKernelGetAttribute_v12000)(int *pi, CUfunction_attribute attrib, CUkernel kernel, CUdevice dev); typedef CUresult (CUDAAPI *PFN_cuKernelSetAttribute_v12000)(CUfunction_attribute attrib, int val, CUkernel kernel, CUdevice dev); typedef CUresult (CUDAAPI *PFN_cuKernelSetCacheConfig_v12000)(CUkernel kernel, CUfunc_cache config, CUdevice dev); +typedef CUresult (CUDAAPI *PFN_cuKernelGetName_v12030)(const char **name, CUkernel hfunc); +typedef CUresult (CUDAAPI *PFN_cuKernelGetParamInfo_v12040)(CUkernel kernel, size_t paramIndex, size_t *paramOffset, size_t *paramSize); typedef CUresult (CUDAAPI *PFN_cuLibraryGetUnifiedFunction_v12000)(void **fptr, CUlibrary library, const char *symbol); typedef CUresult(CUDAAPI *PFN_cuCoredumpGetAttribute_v12010)(CUcoredumpSettings get, void *value, size_t *size); typedef CUresult(CUDAAPI *PFN_cuCoredumpGetAttributeGlobal_v12010)(CUcoredumpSettings get, void *value, size_t *size); typedef CUresult(CUDAAPI *PFN_cuCoredumpSetAttribute_v12010)(CUcoredumpSettings set, void *value, size_t *size); typedef CUresult(CUDAAPI *PFN_cuCoredumpSetAttributeGlobal_v12010)(CUcoredumpSettings set, void *value, size_t *size); +typedef CUresult(CUDAAPI *PFN_cuDeviceRegisterAsyncNotification_v12040)(CUdevice device, CUasyncCallback callbackFunc, void *userData, CUasyncCallbackHandle *callback); +typedef CUresult(CUDAAPI *PFN_cuDeviceUnregisterAsyncNotification_v12040)(CUdevice device, CUasyncCallbackHandle callback); +typedef CUresult(CUDAAPI *PFN_cuGreenCtxCreate_v12040)(CUgreenCtx* phCtx, CUdevResourceDesc desc, CUdevice dev, unsigned int flags); +typedef CUresult(CUDAAPI *PFN_cuGreenCtxDestroy_v12040)(CUgreenCtx hCtx); +typedef CUresult(CUDAAPI *PFN_cuDeviceGetDevResource_v12040)(CUdevice dev, CUdevResource* result, CUdevResourceType type); +typedef CUresult(CUDAAPI *PFN_cuCtxGetDevResource_v12040)(CUcontext hCtx, CUdevResource* result, CUdevResourceType type); +typedef CUresult(CUDAAPI *PFN_cuGreenCtxGetDevResource_v12040)(CUgreenCtx hCtx, CUdevResource* result, CUdevResourceType type); +typedef CUresult(CUDAAPI *PFN_cuGreenCtxRecordEvent_v12040)(CUgreenCtx hCtx, CUevent hEvent); +typedef CUresult(CUDAAPI *PFN_cuGreenCtxWaitEvent_v12040)(CUgreenCtx hCtx, CUevent hEvent); +typedef CUresult(CUDAAPI *PFN_cuDevResourceGenerateDesc_v12040)(CUdevResourceDesc* phDesc, CUdevResource* resources, unsigned int nbResources); +typedef CUresult(CUDAAPI *PFN_cuDevSmResourceSplitByCount_v12040)(CUdevResource* result, unsigned int* nbGroups, const CUdevResource* input, CUdevResource* remaining, unsigned int useFlags, unsigned int minCount); +typedef CUresult(CUDAAPI *PFN_cuStreamGetGreenCtx_v12040)(CUstream hStream, CUgreenCtx *phCtx); +typedef CUresult(CUDAAPI *PFN_cuCtxFromGreenCtx_v12040)(CUcontext *pContext, CUgreenCtx hCtx); +typedef CUresult(CUDAAPI *PFN_cuGreenCtxGetId_v12090)(CUgreenCtx greenCtx, unsigned long long *greenCtxId); +typedef CUresult(CUDAAPI *PFN_cuCtxRecordEvent_v12050)(CUcontext hCtx, CUevent hEvent); +typedef CUresult(CUDAAPI *PFN_cuCtxWaitEvent_v12050)(CUcontext hCtx, CUevent hEvent); +typedef CUresult (CUDAAPI *PFN_cuGreenCtxStreamCreate_v12050)(CUstream* phStream, CUgreenCtx greenCtx, unsigned int flags, int priority); +typedef CUresult (CUDAAPI *PFN_cuStreamGetCtx_v12050)(CUstream hStream, CUcontext *pctx, CUgreenCtx *pGreenCtx); +typedef CUresult (CUDAAPI *PFN_cuStreamGetCtx_v12050_ptsz)(CUstream hStream, CUcontext *pctx, CUgreenCtx *pGreenCtx); + typedef CUresult (CUDAAPI *PFN_cuMemBatchDecompressAsync_v12060)(CUmemDecompressParams *paramsArray, size_t count, unsigned int flags, size_t *errorIndex, CUstream stream); + typedef CUresult (CUDAAPI *PFN_cuMemBatchDecompressAsync_v12060_ptsz)(CUmemDecompressParams *paramsArray, size_t count, unsigned int flags, size_t *errorIndex, CUstream stream); +typedef CUresult (CUDAAPI *PFN_cuLogsRegisterCallback_v12090)(CUlogsCallback callbackFunc, void *userData, CUlogsCallbackHandle *callback); +typedef CUresult (CUDAAPI *PFN_cuLogsUnregisterCallback_v12090)(CUlogsCallbackHandle callback); +typedef CUresult (CUDAAPI *PFN_cuLogsCurrent_v12090)(CUlogIterator *iterator, unsigned int flags); +typedef CUresult (CUDAAPI *PFN_cuLogsDumpToFile_v12090)(CUlogIterator *iterator, const char *pathToFile, unsigned int flags); +typedef CUresult (CUDAAPI *PFN_cuLogsDumpToMemory_v12090)(CUlogIterator *iterator, char *buffer, size_t *size, unsigned int flags); /* * Type definitions for older versioned functions in cuda.h */ @@ -1022,6 +701,13 @@ typedef CUresult(CUDAAPI *PFN_cuCoredumpSetAttributeGlobal_v12010)(CUcoredumpSet typedef CUresult (CUDAAPI *PFN_cuGraphInstantiate_v11000)(CUgraphExec *phGraphExec, CUgraph hGraph, CUgraphNode *phErrorNode, char *logBuffer, size_t bufferSize); #endif + typedef CUresult (CUDAAPI *PFN_cuCheckpointProcessGetRestoreThreadId_v12080)(int pid, int *tid); + typedef CUresult (CUDAAPI *PFN_cuCheckpointProcessGetState_v12080)(int pid, CUprocessState *state); + typedef CUresult (CUDAAPI *PFN_cuCheckpointProcessLock_v12080)(int pid, CUcheckpointLockArgs *args); + typedef CUresult (CUDAAPI *PFN_cuCheckpointProcessCheckpoint_v12080)(int pid, CUcheckpointCheckpointArgs *args); + typedef CUresult (CUDAAPI *PFN_cuCheckpointProcessRestore_v12080)(int pid, CUcheckpointRestoreArgs *args); + typedef CUresult (CUDAAPI *PFN_cuCheckpointProcessUnlock_v12080)(int pid, CUcheckpointUnlockArgs *args); + #ifdef __cplusplus } #endif // __cplusplus diff --git a/backends/cuda/include/cudaVDPAUTypedefs.h b/backends/cuda/include/cudaVDPAUTypedefs.h index 2bfd14863..2b2c90dbe 100644 --- a/backends/cuda/include/cudaVDPAUTypedefs.h +++ b/backends/cuda/include/cudaVDPAUTypedefs.h @@ -59,15 +59,6 @@ extern "C" { #endif // __cplusplus -/* - * Macros for the latest version for each driver function in cudaVDPAU.h - */ -#define PFN_cuVDPAUGetDevice PFN_cuVDPAUGetDevice_v3010 -#define PFN_cuVDPAUCtxCreate PFN_cuVDPAUCtxCreate_v3020 -#define PFN_cuGraphicsVDPAURegisterVideoSurface PFN_cuGraphicsVDPAURegisterVideoSurface_v3010 -#define PFN_cuGraphicsVDPAURegisterOutputSurface PFN_cuGraphicsVDPAURegisterOutputSurface_v3010 - - /** * Type definitions for functions defined in cudaVDPAU.h */ diff --git a/backends/cuda/include/cuda_device_runtime_api.h b/backends/cuda/include/cuda_device_runtime_api.h index 2730d51bf..060a44c02 100644 --- a/backends/cuda/include/cuda_device_runtime_api.h +++ b/backends/cuda/include/cuda_device_runtime_api.h @@ -74,6 +74,9 @@ extern "C" { struct cudaFuncAttributes; +// Bug 4398304 +// WAR for doxgyen processing duplicate entries causing warnings to be listed in the documentation +/** \cond impl_private */ #ifndef __CUDA_INTERNAL_USE_CDP2 inline __device__ cudaError_t CUDARTAPI cudaMalloc(void **p, size_t s) @@ -137,6 +140,7 @@ inline __device__ cudaError_t CUDARTAPI __cudaCDP2OccupancyMaxActiveBlocksPerMu } #endif // __CUDA_INTERNAL_USE_CDP2 +/** \endcond */ #if defined(__cplusplus) } @@ -170,10 +174,11 @@ inline __device__ cudaError_t CUDARTAPI __cudaCDP2OccupancyMaxActiveBlocksPerMu #define cudaStreamGraphTailLaunch (cudaStream_t)0x0100000000000000 #define cudaStreamGraphFireAndForget (cudaStream_t)0x0200000000000000 +#define cudaStreamGraphFireAndForgetAsSibling (cudaStream_t)0x0300000000000000 #ifdef __CUDA_INTERNAL_USE_CDP2 -#define cudaStreamTailLaunch ((cudaStream_t)0x3) /**< Per-grid stream with a fire-and-forget synchronization behavior. Only applicable when used with CUDA Dynamic Parallelism. */ -#define cudaStreamFireAndForget ((cudaStream_t)0x4) /**< Per-grid stream with a tail launch semantics. Only applicable when used with CUDA Dynamic Parallelism. */ +#define cudaStreamTailLaunch ((cudaStream_t)0x3) /**< Per-grid stream with a tail launch semantics. Only applicable when used with CUDA Dynamic Parallelism. */ +#define cudaStreamFireAndForget ((cudaStream_t)0x4) /**< Per-grid stream with a fire-and-forget synchronization behavior. Only applicable when used with CUDA Dynamic Parallelism. */ #endif extern "C" @@ -185,12 +190,16 @@ extern "C" extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI __cudaDeviceSynchronizeDeprecationAvoidance(void); +// Bug 4398304 +// WAR for doxgyen processing duplicate entries causing warnings to be listed in the documentation +/** \cond impl_private */ + #ifndef __CUDA_INTERNAL_USE_CDP2 //// CDP1 endpoints extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetAttribute(int *value, enum cudaDeviceAttr attr, int device); extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetLimit(size_t *pValue, enum cudaLimit limit); extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetCacheConfig(enum cudaFuncCache *pCacheConfig); -extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetSharedMemConfig(enum cudaSharedMemConfig *pConfig); +extern __DEPRECATED__("cudaDeviceGetSharedMemConfig deprecated") __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetSharedMemConfig(enum cudaSharedMemConfig *pConfig); #if (__CUDA_ARCH__ < 900) && (defined(CUDA_FORCE_CDP1_IF_SUPPORTED) || (defined(_WIN32) && !defined(_WIN64))) // cudaDeviceSynchronize is removed on sm_90+ extern __device__ __cudart_builtin__ __CDPRT_DEPRECATED(cudaDeviceSynchronize) cudaError_t CUDARTAPI cudaDeviceSynchronize(void); @@ -289,13 +298,15 @@ static inline __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGraphLaun } #endif +/** \endcond */ + /** * \ingroup CUDART_GRAPH * \brief Get the currently running device graph id. * * Get the currently running device graph id. * \return Returns the current device graph id, 0 if the call is outside of a device graph. - * \sa cudaLaunchDevice + * \sa cudaGraphLaunch */ static inline __device__ __cudart_builtin__ cudaGraphExec_t CUDARTAPI cudaGetCurrentGraphExec(void) { @@ -304,6 +315,120 @@ static inline __device__ __cudart_builtin__ cudaGraphExec_t CUDARTAPI cudaGetCur return (cudaGraphExec_t)current_graph_exec; } +/** + * \ingroup CUDART_GRAPH + * \brief Updates the kernel parameters of the given kernel node + * + * Updates \p size bytes in the kernel parameters of \p node at \p offset to + * the contents of \p value. \p node must be device-updatable, and must reside upon the same + * device as the calling kernel. + * + * If this function is called for the node's immediate dependent and that dependent is configured + * for programmatic dependent launch, then a memory fence must be invoked via __threadfence() before + * kickoff of the dependent is triggered via ::cudaTriggerProgrammaticLaunchCompletion() to ensure + * that the update is visible to that dependent node before it is launched. + * + * \param node - The node to update + * \param offset - The offset into the params at which to make the update + * \param value - Buffer containing the params to write + * \param size - Size in bytes to update + * + * \return + * cudaSuccess, + * cudaErrorInvalidValue + * \notefnerr + * + * \sa + * ::cudaGraphKernelNodeSetEnabled, + * ::cudaGraphKernelNodeSetGridDim, + * ::cudaGraphKernelNodeUpdatesApply + */ +extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGraphKernelNodeSetParam(cudaGraphDeviceNode_t node, size_t offset, const void *value , size_t size); + +/** + * \ingroup CUDART_GRAPH + * \brief Enables or disables the given kernel node + * + * Enables or disables \p node based upon \p enable. If \p enable is true, the node will be enabled; + * if it is false, the node will be disabled. Disabled nodes will act as a NOP during execution. + * \p node must be device-updatable, and must reside upon the same device as the calling kernel. + * + * If this function is called for the node's immediate dependent and that dependent is configured + * for programmatic dependent launch, then a memory fence must be invoked via __threadfence() before + * kickoff of the dependent is triggered via ::cudaTriggerProgrammaticLaunchCompletion() to ensure + * that the update is visible to that dependent node before it is launched. + * + * \param node - The node to update + * \param enable - Whether to enable or disable the node + * + * \return + * cudaSuccess, + * cudaErrorInvalidValue + * \notefnerr + * + * \sa + * ::cudaGraphKernelNodeSetParam, + * ::cudaGraphKernelNodeSetGridDim, + * ::cudaGraphKernelNodeUpdatesApply + */ +extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGraphKernelNodeSetEnabled(cudaGraphDeviceNode_t node, bool enable); + +/** + * \ingroup CUDART_GRAPH + * \brief Updates the grid dimensions of the given kernel node + * + * Sets the grid dimensions of \p node to \p gridDim. \p node must be device-updatable, + * and must reside upon the same device as thecalling kernel. + * + * If this function is called for the node's immediate dependent and that dependent is configured + * for programmatic dependent launch, then a memory fence must be invoked via __threadfence() before + * kickoff of the dependent is triggered via ::cudaTriggerProgrammaticLaunchCompletion() to ensure + * that the update is visible to that dependent node before it is launched. + * + * \param node - The node to update + * \param gridDim - The grid dimensions to set + * + * \return + * cudaSuccess, + * cudaErrorInvalidValue + * \notefnerr + * + * \sa + * ::cudaGraphKernelNodeSetParam, + * ::cudaGraphKernelNodeSetEnabled, + * ::cudaGraphKernelNodeUpdatesApply + */ +extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGraphKernelNodeSetGridDim(cudaGraphDeviceNode_t node, dim3 gridDim); + +/** + * \ingroup CUDART_GRAPH + * \brief Batch applies multiple kernel node updates + * + * Batch applies one or more kernel node updates based on the information provided in \p updates. + * \p updateCount specifies the number of updates to apply. Each entry in \p updates must specify + * a node to update, the type of update to apply, and the parameters for that type of update. See + * the documentation for ::cudaGraphKernelNodeUpdate for more detail. + * + * If this function is called for the node's immediate dependent and that dependent is configured + * for programmatic dependent launch, then a memory fence must be invoked via __threadfence() before + * kickoff of the dependent is triggered via ::cudaTriggerProgrammaticLaunchCompletion() to ensure + * that the update is visible to that dependent node before it is launched. + * + * \param updates - The updates to apply + * \param updateCount - The number of updates to apply + * + * \return + * cudaSuccess, + * cudaErrorInvalidValue + * \notefnerr + * + * \sa + * ::cudaGraphKernelNodeSetParam, + * ::cudaGraphKernelNodeSetEnabled, + * ::cudaGraphKernelNodeSetGridDim + */ +extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGraphKernelNodeUpdatesApply(const cudaGraphKernelNodeUpdate *updates, size_t updateCount); + /** * \ingroup CUDART_EXECUTION * \brief Programmatic dependency trigger @@ -341,6 +466,18 @@ static inline __device__ __cudart_builtin__ void CUDARTAPI cudaGridDependencySyn asm volatile("griddepcontrol.wait;":::"memory"); } +/** + * \ingroup CUDART_GRAPH + * \brief Sets the condition value associated with a conditional node. + * + * Sets the condition value associated with a conditional node. + * + * Note: \p handle must be associated with the same context as the kernel calling this function. + * Note: It is undefined behavior to have racing / possibly concurrent calls to ::cudaGraphSetConditional. + * + * \sa cudaGraphConditionalHandleCreate + */ +extern __device__ __cudart_builtin__ void CUDARTAPI cudaGraphSetConditional(cudaGraphConditionalHandle handle, unsigned int value); //// CG API extern __device__ __cudart_builtin__ unsigned long long CUDARTAPI cudaCGGetIntrinsicHandle(enum cudaCGScope scope); @@ -354,6 +491,10 @@ extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaCGGetRank(unsigne #ifdef __CUDA_ARCH__ +// Bug 4398304 +// WAR for doxgyen processing duplicate entries causing warnings to be listed in the documentation +/** \cond impl_private */ + #ifdef __CUDA_INTERNAL_USE_CDP2 static __inline__ __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetAttribute(int *value, enum cudaDeviceAttr attr, int device) { @@ -546,6 +687,8 @@ static __inline__ __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaOccupa } #endif // __CUDA_INTERNAL_USE_CDP2 +/** \endcond */ + #endif // __CUDA_ARCH__ @@ -578,32 +721,6 @@ extern __device__ __cudart_builtin__ void * CUDARTAPI cudaGetParameterBuffer(siz #endif -/** - * \ingroup CUDART_EXECUTION - * \brief Launches a specified kernel - * - * Launches a specified kernel with the specified parameter buffer. A parameter buffer can be obtained - * by calling ::cudaGetParameterBuffer(). - * - * This is a low level API and can only be accessed from Parallel Thread Execution (PTX). - * CUDA user code should use <<< >>> to launch the kernels. - * - * \param func - Pointer to the kernel to be launched - * \param parameterBuffer - Holds the parameters to the launched kernel. parameterBuffer can be NULL. (Optional) - * \param gridDimension - Specifies grid dimensions - * \param blockDimension - Specifies block dimensions - * \param sharedMemSize - Specifies size of shared memory - * \param stream - Specifies the stream to be used - * - * \return - * ::cudaSuccess, ::cudaErrorInvalidDevice, ::cudaErrorLaunchMaxDepthExceeded, ::cudaErrorInvalidConfiguration, - * ::cudaErrorStartupFailure, ::cudaErrorLaunchPendingCountExceeded, ::cudaErrorLaunchOutOfResources - * \notefnerr - * \n Please refer to Execution Configuration and Parameter Buffer Layout from the CUDA Programming - * Guide for the detailed descriptions of launch configuration and parameter layout respectively. - * - * \sa cudaGetParameterBuffer - */ #ifdef __CUDA_INTERNAL_USE_CDP2 static __inline__ __device__ __cudart_builtin__ void * CUDARTAPI cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 blockDimension, unsigned int sharedMemSize) { @@ -630,6 +747,32 @@ extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaLaunchDeviceV2_pt #endif +/** + * \ingroup CUDART_EXECUTION + * \brief Launches a specified kernel + * + * Launches a specified kernel with the specified parameter buffer. A parameter buffer can be obtained + * by calling ::cudaGetParameterBuffer(). + * + * This is a low level API and can only be accessed from Parallel Thread Execution (PTX). + * CUDA user code should use <<< >>> to launch the kernels. + * + * \param func - Pointer to the kernel to be launched + * \param parameterBuffer - Holds the parameters to the launched kernel. parameterBuffer can be NULL. (Optional) + * \param gridDimension - Specifies grid dimensions + * \param blockDimension - Specifies block dimensions + * \param sharedMemSize - Specifies size of shared memory + * \param stream - Specifies the stream to be used + * + * \return + * ::cudaSuccess, ::cudaErrorInvalidDevice, ::cudaErrorLaunchMaxDepthExceeded, ::cudaErrorInvalidConfiguration, + * ::cudaErrorStartupFailure, ::cudaErrorLaunchPendingCountExceeded, ::cudaErrorLaunchOutOfResources + * \notefnerr + * \n Please refer to Execution Configuration and Parameter Buffer Layout from the CUDA Programming + * Guide for the detailed descriptions of launch configuration and parameter layout respectively. + * + * \sa cudaGetParameterBuffer + */ #if defined(CUDA_API_PER_THREAD_DEFAULT_STREAM) && defined(__CUDA_ARCH__) // When compiling for the device and per thread default stream is enabled, add // a static inline redirect to the per thread stream entry points. @@ -719,11 +862,48 @@ extern __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaLaunchDeviceV2(vo } +// Bug 4398304 +// WAR for doxgyen processing duplicate entries causing warnings to be listed in the documentation +/** \cond impl_private */ + template static __inline__ __device__ __cudart_builtin__ cudaError_t cudaMalloc(T **devPtr, size_t size); template static __inline__ __device__ __cudart_builtin__ cudaError_t cudaFuncGetAttributes(struct cudaFuncAttributes *attr, T *entry); template static __inline__ __device__ __cudart_builtin__ cudaError_t cudaOccupancyMaxActiveBlocksPerMultiprocessor(int *numBlocks, T func, int blockSize, size_t dynamicSmemSize); template static __inline__ __device__ __cudart_builtin__ cudaError_t cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int *numBlocks, T func, int blockSize, size_t dynamicSmemSize, unsigned int flags); +/** \endcond */ + +/** + * \ingroup CUDART_GRAPH + * \brief Updates the kernel parameters of the given kernel node + * + * Updates the kernel parameters of \p node at \p offset to \p value. \p node must be + * device-updatable, and must reside upon the same device as the calling kernel. + * + * If this function is called for the node's immediate dependent and that dependent is configured + * for programmatic dependent launch, then a memory fence must be invoked via __threadfence() before + * kickoff of the dependent is triggered via ::cudaTriggerProgrammaticLaunchCompletion() to ensure + * that the update is visible to that dependent node before it is launched. + * + * \param node - The node to update + * \param offset - The offset into the params at which to make the update + * \param value - Parameter value to write + * + * \return + * cudaSuccess, + * cudaErrorInvalidValue + * \notefnerr + * + * \sa + * ::cudaGraphKernelNodeSetEnabled, + * ::cudaGraphKernelNodeSetGridDim, + * ::cudaGraphKernelNodeUpdatesApply + */ +template +static __inline__ __device__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGraphKernelNodeSetParam(cudaGraphDeviceNode_t node, size_t offset, const T &value) +{ + return cudaGraphKernelNodeSetParam(node, offset, &value, sizeof(T)); +} #endif // !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 350) #endif /* defined(__cplusplus) && defined(__CUDACC__) */ diff --git a/backends/cuda/include/cuda_runtime_api.h b/backends/cuda/include/cuda_runtime_api.h index fa7883487..d703d66fb 100644 --- a/backends/cuda/include/cuda_runtime_api.h +++ b/backends/cuda/include/cuda_runtime_api.h @@ -1,5 +1,5 @@ /* - * Copyright 1993-2018 NVIDIA Corporation. All rights reserved. + * Copyright 1993-2024 NVIDIA Corporation. All rights reserved. * * NOTICE TO LICENSEE: * @@ -93,15 +93,16 @@ * \subsection MemcpyAsynchronousBehavior Asynchronous * *
      - *
    1. For transfers from device memory to pageable host memory, the function - * will return only once the copy has completed. + *
    2. For transfers between device memory and pageable host memory, the function might + * be synchronous with respect to host. * *
    3. For transfers from any host memory to any host memory, the function is fully * synchronous with respect to the host. * - *
    4. For all other transfers, the function is fully asynchronous. If pageable - * memory must first be staged to pinned memory, this will be handled - * asynchronously with a worker thread. + *
    5. If pageable memory must first be staged to pinned memory, the driver may + * synchronize with the stream and stage the copy into pinned memory. + * + *
    6. For all other transfers, the function should be fully asynchronous. *
    * * \section memset_sync_async_behavior Memset @@ -135,7 +136,7 @@ */ /** CUDA Runtime API Version */ -#define CUDART_VERSION 12010 +#define CUDART_VERSION 13000 #if defined(__CUDA_API_VER_MAJOR__) && defined(__CUDA_API_VER_MINOR__) # define __CUDART_API_VERSION ((__CUDA_API_VER_MAJOR__ * 1000) + (__CUDA_API_VER_MINOR__ * 10)) @@ -148,8 +149,12 @@ #endif #include "builtin_types.h" +#if !defined(__CUDACC_RTC_MINIMAL__) && ((defined(__CUDACC_RDC__) || defined(__CUDACC_EWP__) || !defined(__CUDACC_RTC__))) #include "cuda_device_runtime_api.h" +#endif /* !defined(__CUDACC_RTC_MINIMAL__) && (defined(__CUDACC_RDC__) || defined(__CUDACC_EWP__) || !defined(__CUDACC_RTC__)) */ + +#ifndef __CUDACC_RTC_MINIMAL__ #if defined(CUDA_API_PER_THREAD_DEFAULT_STREAM) || defined(__CUDA_API_VERSION_INTERNAL) #define __CUDART_API_PER_THREAD_DEFAULT_STREAM #define __CUDART_API_PTDS(api) api ## _ptds @@ -159,13 +164,6 @@ #define __CUDART_API_PTSZ(api) api #endif -#define cudaSignalExternalSemaphoresAsync __CUDART_API_PTSZ(cudaSignalExternalSemaphoresAsync_v2) -#define cudaWaitExternalSemaphoresAsync __CUDART_API_PTSZ(cudaWaitExternalSemaphoresAsync_v2) - - #define cudaStreamGetCaptureInfo __CUDART_API_PTSZ(cudaStreamGetCaptureInfo_v2) - -#define cudaGetDeviceProperties cudaGetDeviceProperties_v2 - #if defined(__CUDART_API_PER_THREAD_DEFAULT_STREAM) #define cudaMemcpy __CUDART_API_PTDS(cudaMemcpy) #define cudaMemcpyToSymbol __CUDART_API_PTDS(cudaMemcpyToSymbol) @@ -186,7 +184,10 @@ #define cudaGraphUpload __CUDART_API_PTSZ(cudaGraphUpload) #define cudaGraphLaunch __CUDART_API_PTSZ(cudaGraphLaunch) #define cudaStreamBeginCapture __CUDART_API_PTSZ(cudaStreamBeginCapture) + #define cudaStreamBeginCaptureToGraph __CUDART_API_PTSZ(cudaStreamBeginCaptureToGraph) #define cudaStreamEndCapture __CUDART_API_PTSZ(cudaStreamEndCapture) + #define cudaStreamGetCaptureInfo __CUDART_API_PTSZ(cudaStreamGetCaptureInfo) + #define cudaStreamUpdateCaptureDependencies __CUDART_API_PTSZ(cudaStreamUpdateCaptureDependencies) #define cudaStreamIsCapturing __CUDART_API_PTSZ(cudaStreamIsCapturing) #define cudaMemcpyAsync __CUDART_API_PTSZ(cudaMemcpyAsync) #define cudaMemcpyToSymbolAsync __CUDART_API_PTSZ(cudaMemcpyToSymbolAsync) @@ -198,10 +199,13 @@ #define cudaMemcpy2DFromArrayAsync __CUDART_API_PTSZ(cudaMemcpy2DFromArrayAsync) #define cudaMemcpy3DAsync __CUDART_API_PTSZ(cudaMemcpy3DAsync) #define cudaMemcpy3DPeerAsync __CUDART_API_PTSZ(cudaMemcpy3DPeerAsync) + #define cudaMemcpyBatchAsync __CUDART_API_PTSZ(cudaMemcpyBatchAsync) + #define cudaMemcpy3DBatchAsync __CUDART_API_PTSZ(cudaMemcpy3DBatchAsync) #define cudaMemsetAsync __CUDART_API_PTSZ(cudaMemsetAsync) #define cudaMemset2DAsync __CUDART_API_PTSZ(cudaMemset2DAsync) #define cudaMemset3DAsync __CUDART_API_PTSZ(cudaMemset3DAsync) #define cudaStreamQuery __CUDART_API_PTSZ(cudaStreamQuery) + #define cudaStreamGetDevice __CUDART_API_PTSZ(cudaStreamGetDevice) #define cudaStreamGetFlags __CUDART_API_PTSZ(cudaStreamGetFlags) #define cudaStreamGetId __CUDART_API_PTSZ(cudaStreamGetId) #define cudaStreamGetPriority __CUDART_API_PTSZ(cudaStreamGetPriority) @@ -211,10 +215,15 @@ #define cudaStreamAddCallback __CUDART_API_PTSZ(cudaStreamAddCallback) #define cudaStreamAttachMemAsync __CUDART_API_PTSZ(cudaStreamAttachMemAsync) #define cudaStreamSynchronize __CUDART_API_PTSZ(cudaStreamSynchronize) +#define cudaSignalExternalSemaphoresAsync __CUDART_API_PTSZ(cudaSignalExternalSemaphoresAsync) +#define cudaWaitExternalSemaphoresAsync __CUDART_API_PTSZ(cudaWaitExternalSemaphoresAsync) #define cudaLaunchKernel __CUDART_API_PTSZ(cudaLaunchKernel) #define cudaLaunchKernelExC __CUDART_API_PTSZ(cudaLaunchKernelExC) #define cudaLaunchHostFunc __CUDART_API_PTSZ(cudaLaunchHostFunc) #define cudaMemPrefetchAsync __CUDART_API_PTSZ(cudaMemPrefetchAsync) + #define cudaMemPrefetchBatchAsync __CUDART_API_PTSZ(cudaMemPrefetchBatchAsync) + #define cudaMemDiscardBatchAsync __CUDART_API_PTSZ(cudaMemDiscardBatchAsync) + #define cudaMemDiscardAndPrefetchBatchAsync __CUDART_API_PTSZ(cudaMemDiscardAndPrefetchBatchAsync) #define cudaLaunchCooperativeKernel __CUDART_API_PTSZ(cudaLaunchCooperativeKernel) #define cudaStreamCopyAttributes __CUDART_API_PTSZ(cudaStreamCopyAttributes) #define cudaStreamGetAttribute __CUDART_API_PTSZ(cudaStreamGetAttribute) @@ -223,8 +232,11 @@ #define cudaFreeAsync __CUDART_API_PTSZ(cudaFreeAsync) #define cudaMallocFromPoolAsync __CUDART_API_PTSZ(cudaMallocFromPoolAsync) #define cudaGetDriverEntryPoint __CUDART_API_PTSZ(cudaGetDriverEntryPoint) + #define cudaGetDriverEntryPointByVersion __CUDART_API_PTSZ(cudaGetDriverEntryPointByVersion) #endif +#endif /* __CUDACC_RTC_MINIMAL__ */ + /** \cond impl_private */ #if !defined(__dv) @@ -291,15 +303,22 @@ extern "C" { * device in the current process. It is the caller's responsibility to ensure * that the resources are not accessed or passed in subsequent API calls and * doing so will result in undefined behavior. These resources include CUDA types - * such as ::cudaStream_t, ::cudaEvent_t, ::cudaArray_t, ::cudaMipmappedArray_t, + * ::cudaStream_t, ::cudaEvent_t, ::cudaArray_t, ::cudaMipmappedArray_t, ::cudaPitchedPtr, * ::cudaTextureObject_t, ::cudaSurfaceObject_t, ::textureReference, ::surfaceReference, * ::cudaExternalMemory_t, ::cudaExternalSemaphore_t and ::cudaGraphicsResource_t. + * These resources also include memory allocations by ::cudaMalloc, ::cudaMallocHost, + * ::cudaMallocManaged and ::cudaMallocPitch. * Any subsequent API call to this device will reinitialize the device. * * Note that this function will reset the device immediately. It is the caller's * responsibility to ensure that the device is not being accessed by any * other host threads from the process when this function is called. * + * \note ::cudaDeviceReset() will not destroy memory allocations by ::cudaMallocAsync() and + * ::cudaMallocFromPoolAsync(). These memory allocations need to be destroyed explicitly. + * \note If a non-primary ::CUcontext is current to the thread, ::cudaDeviceReset() + * will destroy only the internal CUDA RT state for that ::CUcontext. + * * \return * ::cudaSuccess * \notefnerr @@ -592,81 +611,6 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetStreamPrio */ extern __host__ cudaError_t CUDARTAPI cudaDeviceSetCacheConfig(enum cudaFuncCache cacheConfig); -/** - * \brief Returns the shared memory configuration for the current device. - * - * This function will return in \p pConfig the current size of shared memory banks - * on the current device. On devices with configurable shared memory banks, - * ::cudaDeviceSetSharedMemConfig can be used to change this setting, so that all - * subsequent kernel launches will by default use the new bank size. When - * ::cudaDeviceGetSharedMemConfig is called on devices without configurable shared - * memory, it will return the fixed bank size of the hardware. - * - * The returned bank configurations can be either: - * - ::cudaSharedMemBankSizeFourByte - shared memory bank width is four bytes. - * - ::cudaSharedMemBankSizeEightByte - shared memory bank width is eight bytes. - * - * \param pConfig - Returned cache configuration - * - * \return - * ::cudaSuccess, - * ::cudaErrorInvalidValue - * \notefnerr - * \note_init_rt - * \note_callback - * - * \sa ::cudaDeviceSetCacheConfig, - * ::cudaDeviceGetCacheConfig, - * ::cudaDeviceSetSharedMemConfig, - * ::cudaFuncSetCacheConfig, - * ::cuCtxGetSharedMemConfig - */ -extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetSharedMemConfig(enum cudaSharedMemConfig *pConfig); - -/** - * \brief Sets the shared memory configuration for the current device. - * - * On devices with configurable shared memory banks, this function will set - * the shared memory bank size which is used for all subsequent kernel launches. - * Any per-function setting of shared memory set via ::cudaFuncSetSharedMemConfig - * will override the device wide setting. - * - * Changing the shared memory configuration between launches may introduce - * a device side synchronization point. - * - * Changing the shared memory bank size will not increase shared memory usage - * or affect occupancy of kernels, but may have major effects on performance. - * Larger bank sizes will allow for greater potential bandwidth to shared memory, - * but will change what kinds of accesses to shared memory will result in bank - * conflicts. - * - * This function will do nothing on devices with fixed shared memory bank size. - * - * The supported bank configurations are: - * - ::cudaSharedMemBankSizeDefault: set bank width the device default (currently, - * four bytes) - * - ::cudaSharedMemBankSizeFourByte: set shared memory bank width to be four bytes - * natively. - * - ::cudaSharedMemBankSizeEightByte: set shared memory bank width to be eight - * bytes natively. - * - * \param config - Requested cache configuration - * - * \return - * ::cudaSuccess, - * ::cudaErrorInvalidValue - * \notefnerr - * \note_init_rt - * \note_callback - * - * \sa ::cudaDeviceSetCacheConfig, - * ::cudaDeviceGetCacheConfig, - * ::cudaDeviceGetSharedMemConfig, - * ::cudaFuncSetCacheConfig, - * ::cuCtxSetSharedMemConfig - */ -extern __host__ cudaError_t CUDARTAPI cudaDeviceSetSharedMemConfig(enum cudaSharedMemConfig config); - /** * \brief Returns a handle to a compute device * @@ -741,7 +685,8 @@ extern __host__ cudaError_t CUDARTAPI cudaDeviceGetPCIBusId(char *pciBusId, int * * IPC functionality is restricted to devices with support for unified * addressing on Linux and Windows operating systems. - * IPC functionality on Windows is restricted to GPUs in TCC mode. + * IPC functionality on Windows is supported for compatibility purposes + * but not recommended as it comes with performance cost. * Users can test their device for IPC functionality by calling * ::cudaDeviceGetAttribute with ::cudaDevAttrIpcEventSupport * @@ -787,7 +732,8 @@ extern __host__ cudaError_t CUDARTAPI cudaIpcGetEventHandle(cudaIpcEventHandle_t * * IPC functionality is restricted to devices with support for unified * addressing on Linux and Windows operating systems. - * IPC functionality on Windows is restricted to GPUs in TCC mode. + * IPC functionality on Windows is supported for compatibility purposes + * but not recommended as it comes with performance cost. * Users can test their device for IPC functionality by calling * ::cudaDeviceGetAttribute with ::cudaDevAttrIpcEventSupport * @@ -833,7 +779,8 @@ extern __host__ cudaError_t CUDARTAPI cudaIpcOpenEventHandle(cudaEvent_t *event, * * IPC functionality is restricted to devices with support for unified * addressing on Linux and Windows operating systems. - * IPC functionality on Windows is restricted to GPUs in TCC mode. + * IPC functionality on Windows is supported for compatibility purposes + * but not recommended as it comes with performance cost. * Users can test their device for IPC functionality by calling * ::cudaDeviceGetAttribute with ::cudaDevAttrIpcEventSupport * @@ -892,7 +839,8 @@ extern __host__ cudaError_t CUDARTAPI cudaIpcGetMemHandle(cudaIpcMemHandle_t *ha * * IPC functionality is restricted to devices with support for unified * addressing on Linux and Windows operating systems. - * IPC functionality on Windows is restricted to GPUs in TCC mode. + * IPC functionality on Windows is supported for compatibility purposes + * but not recommended as it comes with performance cost. * Users can test their device for IPC functionality by calling * ::cudaDeviceGetAttribute with ::cudaDevAttrIpcEventSupport * @@ -940,7 +888,8 @@ extern __host__ cudaError_t CUDARTAPI cudaIpcOpenMemHandle(void **devPtr, cudaIp * * IPC functionality is restricted to devices with support for unified * addressing on Linux and Windows operating systems. - * IPC functionality on Windows is restricted to GPUs in TCC mode. + * IPC functionality on Windows is supported for compatibility purposes + * but not recommended as it comes with performance cost. * Users can test their device for IPC functionality by calling * ::cudaDeviceGetAttribute with ::cudaDevAttrIpcEventSupport * @@ -998,306 +947,225 @@ extern __host__ cudaError_t CUDARTAPI cudaIpcCloseMemHandle(void *devPtr); extern __host__ cudaError_t CUDARTAPI cudaDeviceFlushGPUDirectRDMAWrites(enum cudaFlushGPUDirectRDMAWritesTarget target, enum cudaFlushGPUDirectRDMAWritesScope scope); #endif +/** +* \brief Registers a callback function to receive async notifications +* +* Registers \p callbackFunc to receive async notifications. +* +* The \p userData parameter is passed to the callback function at async notification time. +* Likewise, \p callback is also passed to the callback function to distinguish between +* multiple registered callbacks. +* +* The callback function being registered should be designed to return quickly (~10ms). +* Any long running tasks should be queued for execution on an application thread. +* +* Callbacks may not call cudaDeviceRegisterAsyncNotification or cudaDeviceUnregisterAsyncNotification. +* Doing so will result in ::cudaErrorNotPermitted. Async notification callbacks execute +* in an undefined order and may be serialized. +* +* Returns in \p *callback a handle representing the registered callback instance. +* +* \param device - The device on which to register the callback +* \param callbackFunc - The function to register as a callback +* \param userData - A generic pointer to user data. This is passed into the callback function. +* \param callback - A handle representing the registered callback instance +* +* \return +* ::cudaSuccess +* ::cudaErrorNotSupported +* ::cudaErrorInvalidDevice +* ::cudaErrorInvalidValue +* ::cudaErrorNotPermitted +* ::cudaErrorUnknown +* \notefnerr +* +* \sa +* ::cudaDeviceUnregisterAsyncNotification +*/ +extern __host__ cudaError_t CUDARTAPI cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback); + +/** +* \brief Unregisters an async notification callback +* +* Unregisters \p callback so that the corresponding callback function will stop receiving +* async notifications. +* +* \param device - The device from which to remove \p callback. +* \param callback - The callback instance to unregister from receiving async notifications. +* +* \return +* ::cudaSuccess +* ::cudaErrorNotSupported +* ::cudaErrorInvalidDevice +* ::cudaErrorInvalidValue +* ::cudaErrorNotPermitted +* ::cudaErrorUnknown +* \notefnerr +* +* \sa +* ::cudaDeviceRegisterAsyncNotification +*/ +extern __host__ cudaError_t CUDARTAPI cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback); + /** @} */ /* END CUDART_DEVICE */ /** - * \defgroup CUDART_THREAD_DEPRECATED Thread Management [DEPRECATED] + * \defgroup CUDART_DEVICE_DEPRECATED Device Management [DEPRECATED] * - * ___MANBRIEF___ deprecated thread management functions of the CUDA runtime - * API (___CURRENT_FILE___) ___ENDMANBRIEF___ + * ___MANBRIEF___ deprecated device management functions of the CUDA runtime API + * (___CURRENT_FILE___) ___ENDMANBRIEF___ * - * This section describes deprecated thread management functions of the CUDA runtime + * This section describes the deprecated device management functions of the CUDA runtime * application programming interface. * * @{ */ /** - * \brief Exit and clean up from CUDA launches + * \brief Returns the shared memory configuration for the current device. * * \deprecated * - * Note that this function is deprecated because its name does not - * reflect its behavior. Its functionality is identical to the - * non-deprecated function ::cudaDeviceReset(), which should be used - * instead. - * - * Explicitly destroys all cleans up all resources associated with the current - * device in the current process. Any subsequent API call to this device will - * reinitialize the device. - * - * Note that this function will reset the device immediately. It is the caller's - * responsibility to ensure that the device is not being accessed by any - * other host threads from the process when this function is called. - * - * \return - * ::cudaSuccess - * \notefnerr - * \note_init_rt - * \note_callback - * - * \sa ::cudaDeviceReset - */ -extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaThreadExit(void); - -/** - * \brief Wait for compute device to finish - * - * \deprecated + * This function will return in \p pConfig the current size of shared memory banks + * on the current device. On devices with configurable shared memory banks, + * ::cudaDeviceSetSharedMemConfig can be used to change this setting, so that all + * subsequent kernel launches will by default use the new bank size. When + * ::cudaDeviceGetSharedMemConfig is called on devices without configurable shared + * memory, it will return the fixed bank size of the hardware. * - * Note that this function is deprecated because its name does not - * reflect its behavior. Its functionality is similar to the - * non-deprecated function ::cudaDeviceSynchronize(), which should be used - * instead. + * The returned bank configurations can be either: + * - ::cudaSharedMemBankSizeFourByte - shared memory bank width is four bytes. + * - ::cudaSharedMemBankSizeEightByte - shared memory bank width is eight bytes. * - * Blocks until the device has completed all preceding requested tasks. - * ::cudaThreadSynchronize() returns an error if one of the preceding tasks - * has failed. If the ::cudaDeviceScheduleBlockingSync flag was set for - * this device, the host thread will block until the device has finished - * its work. + * \param pConfig - Returned cache configuration * * \return - * ::cudaSuccess + * ::cudaSuccess, + * ::cudaErrorInvalidValue * \notefnerr * \note_init_rt * \note_callback * - * \sa ::cudaDeviceSynchronize + * \sa ::cudaDeviceSetCacheConfig, + * ::cudaDeviceGetCacheConfig, + * ::cudaDeviceSetSharedMemConfig, + * ::cudaFuncSetCacheConfig, + * ::cuCtxGetSharedMemConfig */ -extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaThreadSynchronize(void); +extern __CUDA_DEPRECATED __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetSharedMemConfig(enum cudaSharedMemConfig *pConfig); /** - * \brief Set resource limits + * \brief Sets the shared memory configuration for the current device. * * \deprecated * - * Note that this function is deprecated because its name does not - * reflect its behavior. Its functionality is identical to the - * non-deprecated function ::cudaDeviceSetLimit(), which should be used - * instead. - * - * Setting \p limit to \p value is a request by the application to update - * the current limit maintained by the device. The driver is free to - * modify the requested value to meet h/w requirements (this could be - * clamping to minimum or maximum values, rounding up to nearest element - * size, etc). The application can use ::cudaThreadGetLimit() to find out - * exactly what the limit has been set to. + * On devices with configurable shared memory banks, this function will set + * the shared memory bank size which is used for all subsequent kernel launches. + * Any per-function setting of shared memory set via ::cudaFuncSetSharedMemConfig + * will override the device wide setting. * - * Setting each ::cudaLimit has its own specific restrictions, so each is - * discussed here. + * Changing the shared memory configuration between launches may introduce + * a device side synchronization point. * - * - ::cudaLimitStackSize controls the stack size of each GPU thread. + * Changing the shared memory bank size will not increase shared memory usage + * or affect occupancy of kernels, but may have major effects on performance. + * Larger bank sizes will allow for greater potential bandwidth to shared memory, + * but will change what kinds of accesses to shared memory will result in bank + * conflicts. * - * - ::cudaLimitPrintfFifoSize controls the size of the shared FIFO - * used by the ::printf() device system call. - * Setting ::cudaLimitPrintfFifoSize must be performed before - * launching any kernel that uses the ::printf() device - * system call, otherwise ::cudaErrorInvalidValue will be returned. + * This function will do nothing on devices with fixed shared memory bank size. * - * - ::cudaLimitMallocHeapSize controls the size of the heap used - * by the ::malloc() and ::free() device system calls. Setting - * ::cudaLimitMallocHeapSize must be performed before launching - * any kernel that uses the ::malloc() or ::free() device system calls, - * otherwise ::cudaErrorInvalidValue will be returned. + * The supported bank configurations are: + * - ::cudaSharedMemBankSizeDefault: set bank width the device default (currently, + * four bytes) + * - ::cudaSharedMemBankSizeFourByte: set shared memory bank width to be four bytes + * natively. + * - ::cudaSharedMemBankSizeEightByte: set shared memory bank width to be eight + * bytes natively. * - * \param limit - Limit to set - * \param value - Size in bytes of limit + * \param config - Requested cache configuration * * \return * ::cudaSuccess, - * ::cudaErrorUnsupportedLimit, * ::cudaErrorInvalidValue * \notefnerr * \note_init_rt * \note_callback * - * \sa ::cudaDeviceSetLimit + * \sa ::cudaDeviceSetCacheConfig, + * ::cudaDeviceGetCacheConfig, + * ::cudaDeviceGetSharedMemConfig, + * ::cudaFuncSetCacheConfig, + * ::cuCtxSetSharedMemConfig */ -extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaThreadSetLimit(enum cudaLimit limit, size_t value); +extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaDeviceSetSharedMemConfig(enum cudaSharedMemConfig config); +/** @} */ /* END CUDART_DEVICE_DEPRECATED */ /** - * \brief Returns resource limits - * - * \deprecated - * - * Note that this function is deprecated because its name does not - * reflect its behavior. Its functionality is identical to the - * non-deprecated function ::cudaDeviceGetLimit(), which should be used - * instead. - * - * Returns in \p *pValue the current size of \p limit. The supported - * ::cudaLimit values are: - * - ::cudaLimitStackSize: stack size of each GPU thread; - * - ::cudaLimitPrintfFifoSize: size of the shared FIFO used by the - * ::printf() device system call. - * - ::cudaLimitMallocHeapSize: size of the heap used by the - * ::malloc() and ::free() device system calls; + * \defgroup CUDART_ERROR Error Handling * - * \param limit - Limit to query - * \param pValue - Returned size in bytes of limit + * ___MANBRIEF___ error handling functions of the CUDA runtime API + * (___CURRENT_FILE___) ___ENDMANBRIEF___ * - * \return - * ::cudaSuccess, - * ::cudaErrorUnsupportedLimit, - * ::cudaErrorInvalidValue - * \notefnerr - * \note_init_rt - * \note_callback + * This section describes the error handling functions of the CUDA runtime + * application programming interface. * - * \sa ::cudaDeviceGetLimit + * @{ */ -extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaThreadGetLimit(size_t *pValue, enum cudaLimit limit); /** - * \brief Returns the preferred cache configuration for the current device. - * - * \deprecated - * - * Note that this function is deprecated because its name does not - * reflect its behavior. Its functionality is identical to the - * non-deprecated function ::cudaDeviceGetCacheConfig(), which should be - * used instead. - * - * On devices where the L1 cache and shared memory use the same hardware - * resources, this returns through \p pCacheConfig the preferred cache - * configuration for the current device. This is only a preference. The - * runtime will use the requested configuration if possible, but it is free to - * choose a different configuration if required to execute functions. - * - * This will return a \p pCacheConfig of ::cudaFuncCachePreferNone on devices - * where the size of the L1 cache and shared memory are fixed. + * \brief Returns the last error from a runtime call * - * The supported cache configurations are: - * - ::cudaFuncCachePreferNone: no preference for shared memory or L1 (default) - * - ::cudaFuncCachePreferShared: prefer larger shared memory and smaller L1 cache - * - ::cudaFuncCachePreferL1: prefer larger L1 cache and smaller shared memory + * Returns the last error that has been produced by any of the runtime calls + * in the same instance of the CUDA Runtime library in the host thread and + * resets it to ::cudaSuccess. * - * \param pCacheConfig - Returned cache configuration + * Note: Multiple instances of the CUDA Runtime library can be present in an + * application when using a library that statically links the CUDA Runtime. * * \return - * ::cudaSuccess + * ::cudaSuccess, + * ::cudaErrorMissingConfiguration, + * ::cudaErrorMemoryAllocation, + * ::cudaErrorInitializationError, + * ::cudaErrorLaunchFailure, + * ::cudaErrorLaunchTimeout, + * ::cudaErrorLaunchOutOfResources, + * ::cudaErrorInvalidDeviceFunction, + * ::cudaErrorInvalidConfiguration, + * ::cudaErrorInvalidDevice, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidPitchValue, + * ::cudaErrorInvalidSymbol, + * ::cudaErrorUnmapBufferObjectFailed, + * ::cudaErrorInvalidDevicePointer, + * ::cudaErrorInvalidTexture, + * ::cudaErrorInvalidTextureBinding, + * ::cudaErrorInvalidChannelDescriptor, + * ::cudaErrorInvalidMemcpyDirection, + * ::cudaErrorInvalidFilterSetting, + * ::cudaErrorInvalidNormSetting, + * ::cudaErrorUnknown, + * ::cudaErrorInvalidResourceHandle, + * ::cudaErrorInsufficientDriver, + * ::cudaErrorNoDevice, + * ::cudaErrorSetOnActiveProcess, + * ::cudaErrorStartupFailure, + * ::cudaErrorInvalidPtx, + * ::cudaErrorUnsupportedPtxVersion, + * ::cudaErrorNoKernelImageForDevice, + * ::cudaErrorJitCompilerNotFound, + * ::cudaErrorJitCompilationDisabled * \notefnerr * \note_init_rt * \note_callback * - * \sa ::cudaDeviceGetCacheConfig + * \sa ::cudaPeekAtLastError, ::cudaGetErrorName, ::cudaGetErrorString, ::cudaError */ -extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaThreadGetCacheConfig(enum cudaFuncCache *pCacheConfig); +extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGetLastError(void); /** - * \brief Sets the preferred cache configuration for the current device. - * - * \deprecated - * - * Note that this function is deprecated because its name does not - * reflect its behavior. Its functionality is identical to the - * non-deprecated function ::cudaDeviceSetCacheConfig(), which should be - * used instead. - * - * On devices where the L1 cache and shared memory use the same hardware - * resources, this sets through \p cacheConfig the preferred cache - * configuration for the current device. This is only a preference. The - * runtime will use the requested configuration if possible, but it is free to - * choose a different configuration if required to execute the function. Any - * function preference set via - * \ref ::cudaFuncSetCacheConfig(const void*, enum cudaFuncCache) "cudaFuncSetCacheConfig (C API)" - * or - * \ref ::cudaFuncSetCacheConfig(T*, enum cudaFuncCache) "cudaFuncSetCacheConfig (C++ API)" - * will be preferred over this device-wide setting. Setting the device-wide - * cache configuration to ::cudaFuncCachePreferNone will cause subsequent - * kernel launches to prefer to not change the cache configuration unless - * required to launch the kernel. - * - * This setting does nothing on devices where the size of the L1 cache and - * shared memory are fixed. - * - * Launching a kernel with a different preference than the most recent - * preference setting may insert a device-side synchronization point. - * - * The supported cache configurations are: - * - ::cudaFuncCachePreferNone: no preference for shared memory or L1 (default) - * - ::cudaFuncCachePreferShared: prefer larger shared memory and smaller L1 cache - * - ::cudaFuncCachePreferL1: prefer larger L1 cache and smaller shared memory - * - * \param cacheConfig - Requested cache configuration - * - * \return - * ::cudaSuccess - * \notefnerr - * \note_init_rt - * \note_callback - * - * \sa ::cudaDeviceSetCacheConfig - */ -extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaThreadSetCacheConfig(enum cudaFuncCache cacheConfig); - -/** @} */ /* END CUDART_THREAD_DEPRECATED */ - -/** - * \defgroup CUDART_ERROR Error Handling - * - * ___MANBRIEF___ error handling functions of the CUDA runtime API - * (___CURRENT_FILE___) ___ENDMANBRIEF___ - * - * This section describes the error handling functions of the CUDA runtime - * application programming interface. - * - * @{ - */ - -/** - * \brief Returns the last error from a runtime call - * - * Returns the last error that has been produced by any of the runtime calls - * in the same instance of the CUDA Runtime library in the host thread and - * resets it to ::cudaSuccess. - * - * Note: Multiple instances of the CUDA Runtime library can be present in an - * application when using a library that statically links the CUDA Runtime. - * - * \return - * ::cudaSuccess, - * ::cudaErrorMissingConfiguration, - * ::cudaErrorMemoryAllocation, - * ::cudaErrorInitializationError, - * ::cudaErrorLaunchFailure, - * ::cudaErrorLaunchTimeout, - * ::cudaErrorLaunchOutOfResources, - * ::cudaErrorInvalidDeviceFunction, - * ::cudaErrorInvalidConfiguration, - * ::cudaErrorInvalidDevice, - * ::cudaErrorInvalidValue, - * ::cudaErrorInvalidPitchValue, - * ::cudaErrorInvalidSymbol, - * ::cudaErrorUnmapBufferObjectFailed, - * ::cudaErrorInvalidDevicePointer, - * ::cudaErrorInvalidTexture, - * ::cudaErrorInvalidTextureBinding, - * ::cudaErrorInvalidChannelDescriptor, - * ::cudaErrorInvalidMemcpyDirection, - * ::cudaErrorInvalidFilterSetting, - * ::cudaErrorInvalidNormSetting, - * ::cudaErrorUnknown, - * ::cudaErrorInvalidResourceHandle, - * ::cudaErrorInsufficientDriver, - * ::cudaErrorNoDevice, - * ::cudaErrorSetOnActiveProcess, - * ::cudaErrorStartupFailure, - * ::cudaErrorInvalidPtx, - * ::cudaErrorUnsupportedPtxVersion, - * ::cudaErrorNoKernelImageForDevice, - * ::cudaErrorJitCompilerNotFound, - * ::cudaErrorJitCompilationDisabled - * \notefnerr - * \note_init_rt - * \note_callback - * - * \sa ::cudaPeekAtLastError, ::cudaGetErrorName, ::cudaGetErrorString, ::cudaError - */ -extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGetLastError(void); - -/** - * \brief Returns the last error from a runtime call + * \brief Returns the last error from a runtime call * * Returns the last error that has been produced by any of the runtime calls * in the same instance of the CUDA Runtime library in the host thread. This @@ -1411,290 +1279,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGetDeviceCount(int /** * \brief Returns information about the compute-device * - * Returns in \p *prop the properties of device \p dev. The ::cudaDeviceProp - * structure is defined as: - * \code - struct cudaDeviceProp { - char name[256]; - cudaUUID_t uuid; - size_t totalGlobalMem; - size_t sharedMemPerBlock; - int regsPerBlock; - int warpSize; - size_t memPitch; - int maxThreadsPerBlock; - int maxThreadsDim[3]; - int maxGridSize[3]; - int clockRate; - size_t totalConstMem; - int major; - int minor; - size_t textureAlignment; - size_t texturePitchAlignment; - int deviceOverlap; - int multiProcessorCount; - int kernelExecTimeoutEnabled; - int integrated; - int canMapHostMemory; - int computeMode; - int maxTexture1D; - int maxTexture1DMipmap; - int maxTexture1DLinear; - int maxTexture2D[2]; - int maxTexture2DMipmap[2]; - int maxTexture2DLinear[3]; - int maxTexture2DGather[2]; - int maxTexture3D[3]; - int maxTexture3DAlt[3]; - int maxTextureCubemap; - int maxTexture1DLayered[2]; - int maxTexture2DLayered[3]; - int maxTextureCubemapLayered[2]; - int maxSurface1D; - int maxSurface2D[2]; - int maxSurface3D[3]; - int maxSurface1DLayered[2]; - int maxSurface2DLayered[3]; - int maxSurfaceCubemap; - int maxSurfaceCubemapLayered[2]; - size_t surfaceAlignment; - int concurrentKernels; - int ECCEnabled; - int pciBusID; - int pciDeviceID; - int pciDomainID; - int tccDriver; - int asyncEngineCount; - int unifiedAddressing; - int memoryClockRate; - int memoryBusWidth; - int l2CacheSize; - int persistingL2CacheMaxSize; - int maxThreadsPerMultiProcessor; - int streamPrioritiesSupported; - int globalL1CacheSupported; - int localL1CacheSupported; - size_t sharedMemPerMultiprocessor; - int regsPerMultiprocessor; - int managedMemory; - int isMultiGpuBoard; - int multiGpuBoardGroupID; - int singleToDoublePrecisionPerfRatio; - int pageableMemoryAccess; - int concurrentManagedAccess; - int computePreemptionSupported; - int canUseHostPointerForRegisteredMem; - int cooperativeLaunch; - int cooperativeMultiDeviceLaunch; - int pageableMemoryAccessUsesHostPageTables; - int directManagedMemAccessFromHost; - int accessPolicyMaxWindowSize; - } - \endcode - * where: - * - \ref ::cudaDeviceProp::name "name[256]" is an ASCII string identifying - * the device. - * - \ref ::cudaDeviceProp::uuid "uuid" is a 16-byte unique identifier. - * - \ref ::cudaDeviceProp::totalGlobalMem "totalGlobalMem" is the total - * amount of global memory available on the device in bytes. - * - \ref ::cudaDeviceProp::sharedMemPerBlock "sharedMemPerBlock" is the - * maximum amount of shared memory available to a thread block in bytes. - * - \ref ::cudaDeviceProp::regsPerBlock "regsPerBlock" is the maximum number - * of 32-bit registers available to a thread block. - * - \ref ::cudaDeviceProp::warpSize "warpSize" is the warp size in threads. - * - \ref ::cudaDeviceProp::memPitch "memPitch" is the maximum pitch in - * bytes allowed by the memory copy functions that involve memory regions - * allocated through ::cudaMallocPitch(). - * - \ref ::cudaDeviceProp::maxThreadsPerBlock "maxThreadsPerBlock" is the - * maximum number of threads per block. - * - \ref ::cudaDeviceProp::maxThreadsDim "maxThreadsDim[3]" contains the - * maximum size of each dimension of a block. - * - \ref ::cudaDeviceProp::maxGridSize "maxGridSize[3]" contains the - * maximum size of each dimension of a grid. - * - \ref ::cudaDeviceProp::clockRate "clockRate" is the clock frequency in - * kilohertz. - * - \ref ::cudaDeviceProp::totalConstMem "totalConstMem" is the total amount - * of constant memory available on the device in bytes. - * - \ref ::cudaDeviceProp::major "major", - * \ref ::cudaDeviceProp::minor "minor" are the major and minor revision - * numbers defining the device's compute capability. - * - \ref ::cudaDeviceProp::textureAlignment "textureAlignment" is the - * alignment requirement; texture base addresses that are aligned to - * \ref ::cudaDeviceProp::textureAlignment "textureAlignment" bytes do not - * need an offset applied to texture fetches. - * - \ref ::cudaDeviceProp::texturePitchAlignment "texturePitchAlignment" is the - * pitch alignment requirement for 2D texture references that are bound to - * pitched memory. - * - \ref ::cudaDeviceProp::deviceOverlap "deviceOverlap" is 1 if the device - * can concurrently copy memory between host and device while executing a - * kernel, or 0 if not. Deprecated, use instead asyncEngineCount. - * - \ref ::cudaDeviceProp::multiProcessorCount "multiProcessorCount" is the - * number of multiprocessors on the device. - * - \ref ::cudaDeviceProp::kernelExecTimeoutEnabled "kernelExecTimeoutEnabled" - * is 1 if there is a run time limit for kernels executed on the device, or - * 0 if not. - * - \ref ::cudaDeviceProp::integrated "integrated" is 1 if the device is an - * integrated (motherboard) GPU and 0 if it is a discrete (card) component. - * - \ref ::cudaDeviceProp::canMapHostMemory "canMapHostMemory" is 1 if the - * device can map host memory into the CUDA address space for use with - * ::cudaHostAlloc()/::cudaHostGetDevicePointer(), or 0 if not. - * - \ref ::cudaDeviceProp::computeMode "computeMode" is the compute mode - * that the device is currently in. Available modes are as follows: - * - cudaComputeModeDefault: Default mode - Device is not restricted and - * multiple threads can use ::cudaSetDevice() with this device. - * - cudaComputeModeProhibited: Compute-prohibited mode - No threads can use - * ::cudaSetDevice() with this device. - * - cudaComputeModeExclusiveProcess: Compute-exclusive-process mode - Many - * threads in one process will be able to use ::cudaSetDevice() with this device. - *
    When an occupied exclusive mode device is chosen with ::cudaSetDevice, - * all subsequent non-device management runtime functions will return - * ::cudaErrorDevicesUnavailable. - * - \ref ::cudaDeviceProp::maxTexture1D "maxTexture1D" is the maximum 1D - * texture size. - * - \ref ::cudaDeviceProp::maxTexture1DMipmap "maxTexture1DMipmap" is the maximum - * 1D mipmapped texture texture size. - * - \ref ::cudaDeviceProp::maxTexture1DLinear "maxTexture1DLinear" is the maximum - * 1D texture size for textures bound to linear memory. - * - \ref ::cudaDeviceProp::maxTexture2D "maxTexture2D[2]" contains the maximum - * 2D texture dimensions. - * - \ref ::cudaDeviceProp::maxTexture2DMipmap "maxTexture2DMipmap[2]" contains the - * maximum 2D mipmapped texture dimensions. - * - \ref ::cudaDeviceProp::maxTexture2DLinear "maxTexture2DLinear[3]" contains the - * maximum 2D texture dimensions for 2D textures bound to pitch linear memory. - * - \ref ::cudaDeviceProp::maxTexture2DGather "maxTexture2DGather[2]" contains the - * maximum 2D texture dimensions if texture gather operations have to be performed. - * - \ref ::cudaDeviceProp::maxTexture3D "maxTexture3D[3]" contains the maximum - * 3D texture dimensions. - * - \ref ::cudaDeviceProp::maxTexture3DAlt "maxTexture3DAlt[3]" - * contains the maximum alternate 3D texture dimensions. - * - \ref ::cudaDeviceProp::maxTextureCubemap "maxTextureCubemap" is the - * maximum cubemap texture width or height. - * - \ref ::cudaDeviceProp::maxTexture1DLayered "maxTexture1DLayered[2]" contains - * the maximum 1D layered texture dimensions. - * - \ref ::cudaDeviceProp::maxTexture2DLayered "maxTexture2DLayered[3]" contains - * the maximum 2D layered texture dimensions. - * - \ref ::cudaDeviceProp::maxTextureCubemapLayered "maxTextureCubemapLayered[2]" - * contains the maximum cubemap layered texture dimensions. - * - \ref ::cudaDeviceProp::maxSurface1D "maxSurface1D" is the maximum 1D - * surface size. - * - \ref ::cudaDeviceProp::maxSurface2D "maxSurface2D[2]" contains the maximum - * 2D surface dimensions. - * - \ref ::cudaDeviceProp::maxSurface3D "maxSurface3D[3]" contains the maximum - * 3D surface dimensions. - * - \ref ::cudaDeviceProp::maxSurface1DLayered "maxSurface1DLayered[2]" contains - * the maximum 1D layered surface dimensions. - * - \ref ::cudaDeviceProp::maxSurface2DLayered "maxSurface2DLayered[3]" contains - * the maximum 2D layered surface dimensions. - * - \ref ::cudaDeviceProp::maxSurfaceCubemap "maxSurfaceCubemap" is the maximum - * cubemap surface width or height. - * - \ref ::cudaDeviceProp::maxSurfaceCubemapLayered "maxSurfaceCubemapLayered[2]" - * contains the maximum cubemap layered surface dimensions. - * - \ref ::cudaDeviceProp::surfaceAlignment "surfaceAlignment" specifies the - * alignment requirements for surfaces. - * - \ref ::cudaDeviceProp::concurrentKernels "concurrentKernels" is 1 if the - * device supports executing multiple kernels within the same context - * simultaneously, or 0 if not. It is not guaranteed that multiple kernels - * will be resident on the device concurrently so this feature should not be - * relied upon for correctness. - * - \ref ::cudaDeviceProp::ECCEnabled "ECCEnabled" is 1 if the device has ECC - * support turned on, or 0 if not. - * - \ref ::cudaDeviceProp::pciBusID "pciBusID" is the PCI bus identifier of - * the device. - * - \ref ::cudaDeviceProp::pciDeviceID "pciDeviceID" is the PCI device - * (sometimes called slot) identifier of the device. - * - \ref ::cudaDeviceProp::pciDomainID "pciDomainID" is the PCI domain identifier - * of the device. - * - \ref ::cudaDeviceProp::tccDriver "tccDriver" is 1 if the device is using a - * TCC driver or 0 if not. - * - \ref ::cudaDeviceProp::asyncEngineCount "asyncEngineCount" is 1 when the - * device can concurrently copy memory between host and device while executing - * a kernel. It is 2 when the device can concurrently copy memory between host - * and device in both directions and execute a kernel at the same time. It is - * 0 if neither of these is supported. - * - \ref ::cudaDeviceProp::unifiedAddressing "unifiedAddressing" is 1 if the device - * shares a unified address space with the host and 0 otherwise. - * - \ref ::cudaDeviceProp::memoryClockRate "memoryClockRate" is the peak memory - * clock frequency in kilohertz. - * - \ref ::cudaDeviceProp::memoryBusWidth "memoryBusWidth" is the memory bus width - * in bits. - * - \ref ::cudaDeviceProp::l2CacheSize "l2CacheSize" is L2 cache size in bytes. - * - \ref ::cudaDeviceProp::persistingL2CacheMaxSize "persistingL2CacheMaxSize" is L2 cache's maximum persisting lines size in bytes. - * - \ref ::cudaDeviceProp::maxThreadsPerMultiProcessor "maxThreadsPerMultiProcessor" - * is the number of maximum resident threads per multiprocessor. - * - \ref ::cudaDeviceProp::streamPrioritiesSupported "streamPrioritiesSupported" - * is 1 if the device supports stream priorities, or 0 if it is not supported. - * - \ref ::cudaDeviceProp::globalL1CacheSupported "globalL1CacheSupported" - * is 1 if the device supports caching of globals in L1 cache, or 0 if it is not supported. - * - \ref ::cudaDeviceProp::localL1CacheSupported "localL1CacheSupported" - * is 1 if the device supports caching of locals in L1 cache, or 0 if it is not supported. - * - \ref ::cudaDeviceProp::sharedMemPerMultiprocessor "sharedMemPerMultiprocessor" is the - * maximum amount of shared memory available to a multiprocessor in bytes; this amount is - * shared by all thread blocks simultaneously resident on a multiprocessor. - * - \ref ::cudaDeviceProp::regsPerMultiprocessor "regsPerMultiprocessor" is the maximum number - * of 32-bit registers available to a multiprocessor; this number is shared - * by all thread blocks simultaneously resident on a multiprocessor. - * - \ref ::cudaDeviceProp::managedMemory "managedMemory" - * is 1 if the device supports allocating managed memory on this system, or 0 if it is not supported. - * - \ref ::cudaDeviceProp::isMultiGpuBoard "isMultiGpuBoard" - * is 1 if the device is on a multi-GPU board (e.g. Gemini cards), and 0 if not; - * - \ref ::cudaDeviceProp::multiGpuBoardGroupID "multiGpuBoardGroupID" is a unique identifier - * for a group of devices associated with the same board. - * Devices on the same multi-GPU board will share the same identifier. - * - \ref ::cudaDeviceProp::hostNativeAtomicSupported "hostNativeAtomicSupported" - * is 1 if the link between the device and the host supports native atomic operations, or 0 if it is not supported. - * - \ref ::cudaDeviceProp::singleToDoublePrecisionPerfRatio "singleToDoublePrecisionPerfRatio" - * is the ratio of single precision performance (in floating-point operations per second) - * to double precision performance. - * - \ref ::cudaDeviceProp::pageableMemoryAccess "pageableMemoryAccess" is 1 if the device supports - * coherently accessing pageable memory without calling cudaHostRegister on it, and 0 otherwise. - * - \ref ::cudaDeviceProp::concurrentManagedAccess "concurrentManagedAccess" is 1 if the device can - * coherently access managed memory concurrently with the CPU, and 0 otherwise. - * - \ref ::cudaDeviceProp::computePreemptionSupported "computePreemptionSupported" is 1 if the device - * supports Compute Preemption, and 0 otherwise. - * - \ref ::cudaDeviceProp::canUseHostPointerForRegisteredMem "canUseHostPointerForRegisteredMem" is 1 if - * the device can access host registered memory at the same virtual address as the CPU, and 0 otherwise. - * - \ref ::cudaDeviceProp::cooperativeLaunch "cooperativeLaunch" is 1 if the device supports launching - * cooperative kernels via ::cudaLaunchCooperativeKernel, and 0 otherwise. - * - \ref ::cudaDeviceProp::cooperativeMultiDeviceLaunch "cooperativeMultiDeviceLaunch" is 1 if the device - * supports launching cooperative kernels via ::cudaLaunchCooperativeKernelMultiDevice, and 0 otherwise. - * - \ref ::cudaDeviceProp::sharedMemPerBlockOptin "sharedMemPerBlockOptin" - * is the per device maximum shared memory per block usable by special opt in - * - \ref ::cudaDeviceProp::pageableMemoryAccessUsesHostPageTables "pageableMemoryAccessUsesHostPageTables" is 1 if the device accesses - * pageable memory via the host's page tables, and 0 otherwise. - * - \ref ::cudaDeviceProp::directManagedMemAccessFromHost "directManagedMemAccessFromHost" is 1 if the host can directly access managed - * memory on the device without migration, and 0 otherwise. - * - \ref ::cudaDeviceProp::maxBlocksPerMultiProcessor "maxBlocksPerMultiProcessor" is the maximum number of thread blocks - * that can reside on a multiprocessor. - * - \ref ::cudaDeviceProp::accessPolicyMaxWindowSize "accessPolicyMaxWindowSize" is - * the maximum value of ::cudaAccessPolicyWindow::num_bytes. - * - \ref ::cudaDeviceProp::reservedSharedMemPerBlock "reservedSharedMemPerBlock" - * is the shared memory reserved by CUDA driver per block in bytes - * - \ref ::cudaDeviceProp::hostRegisterSupported "hostRegisterSupported" - * is 1 if the device supports host memory registration via ::cudaHostRegister, and 0 otherwise. - * - \ref ::cudaDeviceProp::sparseCudaArraySupported "sparseCudaArraySupported" - * is 1 if the device supports sparse CUDA arrays and sparse CUDA mipmapped arrays, 0 otherwise - * - \ref ::cudaDeviceProp::hostRegisterReadOnlySupported "hostRegisterReadOnlySupported" - * is 1 if the device supports using the ::cudaHostRegister flag cudaHostRegisterReadOnly to register memory that must be mapped as - * read-only to the GPU - * - \ref ::cudaDeviceProp::timelineSemaphoreInteropSupported "timelineSemaphoreInteropSupported" - * is 1 if external timeline semaphore interop is supported on the device, 0 otherwise - * - \ref ::cudaDeviceProp::memoryPoolsSupported "memoryPoolsSupported" - * is 1 if the device supports using the cudaMallocAsync and cudaMemPool family of APIs, 0 otherwise - * - \ref ::cudaDeviceProp::gpuDirectRDMASupported "gpuDirectRDMASupported" - * is 1 if the device supports GPUDirect RDMA APIs, 0 otherwise - * - \ref ::cudaDeviceProp::gpuDirectRDMAFlushWritesOptions "gpuDirectRDMAFlushWritesOptions" - * is a bitmask to be interpreted according to the ::cudaFlushGPUDirectRDMAWritesOptions enum - * - \ref ::cudaDeviceProp::gpuDirectRDMAWritesOrdering "gpuDirectRDMAWritesOrdering" - * See the ::cudaGPUDirectRDMAWritesOrdering enum for numerical values - * - \ref ::cudaDeviceProp::memoryPoolSupportedHandleTypes "memoryPoolSupportedHandleTypes" - * is a bitmask of handle types supported with mempool-based IPC - * - \ref ::cudaDeviceProp::deferredMappingCudaArraySupported "deferredMappingCudaArraySupported" - * is 1 if the device supports deferred mapping CUDA arrays and CUDA mipmapped arrays - * - \ref ::cudaDeviceProp::ipcEventSupported "ipcEventSupported" - * is 1 if the device supports IPC Events, and 0 otherwise - * - \ref ::cudaDeviceProp::unifiedFunctionPointers "unifiedFunctionPointers" - * is 1 if the device support unified pointers, and 0 otherwise - * + * Returns in \p *prop the properties of device \p dev. * \param prop - Properties for the specified device * \param device - Device number to get properties for * @@ -1717,188 +1302,11 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGetDeviceProperties * \brief Returns information about the device * * Returns in \p *value the integer value of the attribute \p attr on device - * \p device. The supported attributes are: - * - ::cudaDevAttrMaxThreadsPerBlock: Maximum number of threads per block - * - ::cudaDevAttrMaxBlockDimX: Maximum x-dimension of a block - * - ::cudaDevAttrMaxBlockDimY: Maximum y-dimension of a block - * - ::cudaDevAttrMaxBlockDimZ: Maximum z-dimension of a block - * - ::cudaDevAttrMaxGridDimX: Maximum x-dimension of a grid - * - ::cudaDevAttrMaxGridDimY: Maximum y-dimension of a grid - * - ::cudaDevAttrMaxGridDimZ: Maximum z-dimension of a grid - * - ::cudaDevAttrMaxSharedMemoryPerBlock: Maximum amount of shared memory - * available to a thread block in bytes - * - ::cudaDevAttrTotalConstantMemory: Memory available on device for - * __constant__ variables in a CUDA C kernel in bytes - * - ::cudaDevAttrWarpSize: Warp size in threads - * - ::cudaDevAttrMaxPitch: Maximum pitch in bytes allowed by the memory copy - * functions that involve memory regions allocated through ::cudaMallocPitch() - * - ::cudaDevAttrMaxTexture1DWidth: Maximum 1D texture width - * - ::cudaDevAttrMaxTexture1DLinearWidth: Maximum width for a 1D texture bound - * to linear memory - * - ::cudaDevAttrMaxTexture1DMipmappedWidth: Maximum mipmapped 1D texture width - * - ::cudaDevAttrMaxTexture2DWidth: Maximum 2D texture width - * - ::cudaDevAttrMaxTexture2DHeight: Maximum 2D texture height - * - ::cudaDevAttrMaxTexture2DLinearWidth: Maximum width for a 2D texture - * bound to linear memory - * - ::cudaDevAttrMaxTexture2DLinearHeight: Maximum height for a 2D texture - * bound to linear memory - * - ::cudaDevAttrMaxTexture2DLinearPitch: Maximum pitch in bytes for a 2D - * texture bound to linear memory - * - ::cudaDevAttrMaxTexture2DMipmappedWidth: Maximum mipmapped 2D texture - * width - * - ::cudaDevAttrMaxTexture2DMipmappedHeight: Maximum mipmapped 2D texture - * height - * - ::cudaDevAttrMaxTexture3DWidth: Maximum 3D texture width - * - ::cudaDevAttrMaxTexture3DHeight: Maximum 3D texture height - * - ::cudaDevAttrMaxTexture3DDepth: Maximum 3D texture depth - * - ::cudaDevAttrMaxTexture3DWidthAlt: Alternate maximum 3D texture width, - * 0 if no alternate maximum 3D texture size is supported - * - ::cudaDevAttrMaxTexture3DHeightAlt: Alternate maximum 3D texture height, - * 0 if no alternate maximum 3D texture size is supported - * - ::cudaDevAttrMaxTexture3DDepthAlt: Alternate maximum 3D texture depth, - * 0 if no alternate maximum 3D texture size is supported - * - ::cudaDevAttrMaxTextureCubemapWidth: Maximum cubemap texture width or - * height - * - ::cudaDevAttrMaxTexture1DLayeredWidth: Maximum 1D layered texture width - * - ::cudaDevAttrMaxTexture1DLayeredLayers: Maximum layers in a 1D layered - * texture - * - ::cudaDevAttrMaxTexture2DLayeredWidth: Maximum 2D layered texture width - * - ::cudaDevAttrMaxTexture2DLayeredHeight: Maximum 2D layered texture height - * - ::cudaDevAttrMaxTexture2DLayeredLayers: Maximum layers in a 2D layered - * texture - * - ::cudaDevAttrMaxTextureCubemapLayeredWidth: Maximum cubemap layered - * texture width or height - * - ::cudaDevAttrMaxTextureCubemapLayeredLayers: Maximum layers in a cubemap - * layered texture - * - ::cudaDevAttrMaxSurface1DWidth: Maximum 1D surface width - * - ::cudaDevAttrMaxSurface2DWidth: Maximum 2D surface width - * - ::cudaDevAttrMaxSurface2DHeight: Maximum 2D surface height - * - ::cudaDevAttrMaxSurface3DWidth: Maximum 3D surface width - * - ::cudaDevAttrMaxSurface3DHeight: Maximum 3D surface height - * - ::cudaDevAttrMaxSurface3DDepth: Maximum 3D surface depth - * - ::cudaDevAttrMaxSurface1DLayeredWidth: Maximum 1D layered surface width - * - ::cudaDevAttrMaxSurface1DLayeredLayers: Maximum layers in a 1D layered - * surface - * - ::cudaDevAttrMaxSurface2DLayeredWidth: Maximum 2D layered surface width - * - ::cudaDevAttrMaxSurface2DLayeredHeight: Maximum 2D layered surface height - * - ::cudaDevAttrMaxSurface2DLayeredLayers: Maximum layers in a 2D layered - * surface - * - ::cudaDevAttrMaxSurfaceCubemapWidth: Maximum cubemap surface width - * - ::cudaDevAttrMaxSurfaceCubemapLayeredWidth: Maximum cubemap layered - * surface width - * - ::cudaDevAttrMaxSurfaceCubemapLayeredLayers: Maximum layers in a cubemap - * layered surface - * - ::cudaDevAttrMaxRegistersPerBlock: Maximum number of 32-bit registers - * available to a thread block - * - ::cudaDevAttrClockRate: Peak clock frequency in kilohertz - * - ::cudaDevAttrTextureAlignment: Alignment requirement; texture base - * addresses aligned to ::textureAlign bytes do not need an offset applied - * to texture fetches - * - ::cudaDevAttrTexturePitchAlignment: Pitch alignment requirement for 2D - * texture references bound to pitched memory - * - ::cudaDevAttrGpuOverlap: 1 if the device can concurrently copy memory - * between host and device while executing a kernel, or 0 if not - * - ::cudaDevAttrMultiProcessorCount: Number of multiprocessors on the device - * - ::cudaDevAttrKernelExecTimeout: 1 if there is a run time limit for kernels - * executed on the device, or 0 if not - * - ::cudaDevAttrIntegrated: 1 if the device is integrated with the memory - * subsystem, or 0 if not - * - ::cudaDevAttrCanMapHostMemory: 1 if the device can map host memory into - * the CUDA address space, or 0 if not - * - ::cudaDevAttrComputeMode: Compute mode is the compute mode that the device - * is currently in. Available modes are as follows: - * - ::cudaComputeModeDefault: Default mode - Device is not restricted and - * multiple threads can use ::cudaSetDevice() with this device. - * - ::cudaComputeModeProhibited: Compute-prohibited mode - No threads can use - * ::cudaSetDevice() with this device. - * - ::cudaComputeModeExclusiveProcess: Compute-exclusive-process mode - Many - * threads in one process will be able to use ::cudaSetDevice() with this - * device. - * - ::cudaDevAttrConcurrentKernels: 1 if the device supports executing - * multiple kernels within the same context simultaneously, or 0 if - * not. It is not guaranteed that multiple kernels will be resident on the - * device concurrently so this feature should not be relied upon for - * correctness. - * - ::cudaDevAttrEccEnabled: 1 if error correction is enabled on the device, - * 0 if error correction is disabled or not supported by the device - * - ::cudaDevAttrPciBusId: PCI bus identifier of the device - * - ::cudaDevAttrPciDeviceId: PCI device (also known as slot) identifier of - * the device - * - ::cudaDevAttrTccDriver: 1 if the device is using a TCC driver. TCC is only - * available on Tesla hardware running Windows Vista or later. - * - ::cudaDevAttrMemoryClockRate: Peak memory clock frequency in kilohertz - * - ::cudaDevAttrGlobalMemoryBusWidth: Global memory bus width in bits - * - ::cudaDevAttrL2CacheSize: Size of L2 cache in bytes. 0 if the device - * doesn't have L2 cache. - * - ::cudaDevAttrMaxThreadsPerMultiProcessor: Maximum resident threads per - * multiprocessor - * - ::cudaDevAttrUnifiedAddressing: 1 if the device shares a unified address - * space with the host, or 0 if not - * - ::cudaDevAttrComputeCapabilityMajor: Major compute capability version - * number - * - ::cudaDevAttrComputeCapabilityMinor: Minor compute capability version - * number - * - ::cudaDevAttrStreamPrioritiesSupported: 1 if the device supports stream - * priorities, or 0 if not - * - ::cudaDevAttrGlobalL1CacheSupported: 1 if device supports caching globals - * in L1 cache, 0 if not - * - ::cudaDevAttrLocalL1CacheSupported: 1 if device supports caching locals - * in L1 cache, 0 if not - * - ::cudaDevAttrMaxSharedMemoryPerMultiprocessor: Maximum amount of shared memory - * available to a multiprocessor in bytes; this amount is shared by all - * thread blocks simultaneously resident on a multiprocessor - * - ::cudaDevAttrMaxRegistersPerMultiprocessor: Maximum number of 32-bit registers - * available to a multiprocessor; this number is shared by all thread blocks - * simultaneously resident on a multiprocessor - * - ::cudaDevAttrManagedMemory: 1 if device supports allocating - * managed memory, 0 if not - * - ::cudaDevAttrIsMultiGpuBoard: 1 if device is on a multi-GPU board, 0 if not - * - ::cudaDevAttrMultiGpuBoardGroupID: Unique identifier for a group of devices on the - * same multi-GPU board - * - ::cudaDevAttrHostNativeAtomicSupported: 1 if the link between the device and the - * host supports native atomic operations - * - ::cudaDevAttrSingleToDoublePrecisionPerfRatio: Ratio of single precision performance - * (in floating-point operations per second) to double precision performance - * - ::cudaDevAttrPageableMemoryAccess: 1 if the device supports coherently accessing - * pageable memory without calling cudaHostRegister on it, and 0 otherwise - * - ::cudaDevAttrConcurrentManagedAccess: 1 if the device can coherently access managed - * memory concurrently with the CPU, and 0 otherwise - * - ::cudaDevAttrComputePreemptionSupported: 1 if the device supports - * Compute Preemption, 0 if not - * - ::cudaDevAttrCanUseHostPointerForRegisteredMem: 1 if the device can access host - * registered memory at the same virtual address as the CPU, and 0 otherwise - * - ::cudaDevAttrCooperativeLaunch: 1 if the device supports launching cooperative kernels - * via ::cudaLaunchCooperativeKernel, and 0 otherwise - * - ::cudaDevAttrCooperativeMultiDeviceLaunch: 1 if the device supports launching cooperative - * kernels via ::cudaLaunchCooperativeKernelMultiDevice, and 0 otherwise - * - ::cudaDevAttrCanFlushRemoteWrites: 1 if the device supports flushing of outstanding - * remote writes, and 0 otherwise - * - ::cudaDevAttrHostRegisterSupported: 1 if the device supports host memory registration - * via ::cudaHostRegister, and 0 otherwise - * - ::cudaDevAttrPageableMemoryAccessUsesHostPageTables: 1 if the device accesses pageable memory via the - * host's page tables, and 0 otherwise - * - ::cudaDevAttrDirectManagedMemAccessFromHost: 1 if the host can directly access managed memory on the device - * without migration, and 0 otherwise - * - ::cudaDevAttrMaxSharedMemoryPerBlockOptin: Maximum per block shared memory size on the device. This value can - * be opted into when using ::cudaFuncSetAttribute - * - ::cudaDevAttrMaxBlocksPerMultiprocessor: Maximum number of thread blocks that can reside on a multiprocessor - * - ::cudaDevAttrMaxPersistingL2CacheSize: Maximum L2 persisting lines capacity setting in bytes - * - ::cudaDevAttrMaxAccessPolicyWindowSize: Maximum value of cudaAccessPolicyWindow::num_bytes - * - ::cudaDevAttrReservedSharedMemoryPerBlock: Shared memory reserved by CUDA driver per block in bytes - * - ::cudaDevAttrSparseCudaArraySupported: 1 if the device supports sparse CUDA arrays and sparse CUDA mipmapped arrays. - * - ::cudaDevAttrHostRegisterReadOnlySupported: Device supports using the ::cudaHostRegister flag cudaHostRegisterReadOnly - * to register memory that must be mapped as read-only to the GPU - * - ::cudaDevAttrMemoryPoolsSupported: 1 if the device supports using the cudaMallocAsync and cudaMemPool family of APIs, and 0 otherwise - * - ::cudaDevAttrGPUDirectRDMASupported: 1 if the device supports GPUDirect RDMA APIs, and 0 otherwise - * - ::cudaDevAttrGPUDirectRDMAFlushWritesOptions: bitmask to be interpreted according to the ::cudaFlushGPUDirectRDMAWritesOptions enum - * - ::cudaDevAttrGPUDirectRDMAWritesOrdering: see the ::cudaGPUDirectRDMAWritesOrdering enum for numerical values - * - ::cudaDevAttrMemoryPoolSupportedHandleTypes: Bitmask of handle types supported with mempool based IPC - * - ::cudaDevAttrDeferredMappingCudaArraySupported : 1 if the device supports deferred mapping CUDA arrays and CUDA mipmapped arrays. - * - ::cudaDevAttrIpcEventSupport: 1 if the device supports IPC Events. + * \p device. * * \param value - Returned device attribute value * \param attr - Device attribute to query - * \param device - Device number to query + * \param device - Device number to query * * \return * ::cudaSuccess, @@ -1909,12 +1317,45 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGetDeviceProperties * \note_callback * * \sa ::cudaGetDeviceCount, ::cudaGetDevice, ::cudaSetDevice, ::cudaChooseDevice, - * ::cudaGetDeviceProperties, + * ::cudaGetDeviceProperties, * ::cudaInitDevice, * ::cuDeviceGetAttribute */ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetAttribute(int *value, enum cudaDeviceAttr attr, int device); +/** + * \brief Queries details about atomic operations supported between the device and host. + * + * Returns in \p *capabilities the details about requested atomic \p *operations over the + * the link between \p dev and the host. The allocated size of \p *operations and + * \p *capabilities must be \p count. + * + * For each ::cudaAtomicOperation in \p *operations, the corresponding result in \p *capabilities + * will be a bitmask indicating which of ::cudaAtomicOperationCapability the link supports natively. + * + * Returns ::cudaErrorInvalidDevice if \p dev is not valid. + * + * Returns ::cudaErrorInvalidValue if \p *capabilities or \p *operations is NULL, if \p count is 0, + * or if any of \p *operations is not valid. + * + * \param capabilities - Returned capability details of each requested operation + * \param operations - Requested operations + * \param count - Count of requested operations and size of capabilities + * \param dev - Device handle + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidDevice, + * ::cudaErrorInvalidValue + * \notefnerr + * + * \sa + * ::cudaDeviceGetAttribute, + * ::cudaDeviceGetP2PAtomicCapabilities, + * ::cuDeviceGeHostAtomicCapabilities + */ +extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetHostAtomicCapabilities(unsigned int *capabilities, const enum cudaAtomicOperation* operations, unsigned int count, int device); + /** * \brief Returns the default mempool of a device * @@ -2048,10 +1489,13 @@ extern __host__ cudaError_t CUDARTAPI cudaDeviceGetNvSciSyncAttributes(void *nvS * performance of the link between two devices. Lower value means better * performance (0 being the value used for most performant link). * - ::cudaDevP2PAttrAccessSupported: 1 if peer access is enabled. - * - ::cudaDevP2PAttrNativeAtomicSupported: 1 if native atomic operations over - * the link are supported. + * - ::cudaDevP2PAttrNativeAtomicSupported: 1 if all native atomic operations + * over the link are supported. * - ::cudaDevP2PAttrCudaArrayAccessSupported: 1 if accessing CUDA arrays over * the link is supported. + * - ::cudaDevP2PAttrOnlyPartialNativeAtomicSupported: 1 if some + * CUDA-valid atomic operations over the link are supported. Information about + * specific operations can be retrieved with ::cudaDeviceGetP2PAtomicCapabilities. * * Returns ::cudaErrorInvalidDevice if \p srcDevice or \p dstDevice are not valid * or if they represent the same device. @@ -2076,9 +1520,47 @@ extern __host__ cudaError_t CUDARTAPI cudaDeviceGetNvSciSyncAttributes(void *nvS * ::cudaDeviceDisablePeerAccess, * ::cudaDeviceCanAccessPeer, * ::cuDeviceGetP2PAttribute + * ::cudaDeviceGetP2PAtomicCapabilities */ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetP2PAttribute(int *value, enum cudaDeviceP2PAttr attr, int srcDevice, int dstDevice); + +/** + * \brief Queries details about atomic operations supported between two devices + * + * Returns in \p *capabilities the details about requested atomic \p *operations over the + * the link between \p srcDevice and \p dstDevice. The allocated size of \p *operations and + * \p *capabilities must be \p count. + * + * For each ::cudaAtomicOperation in \p *operations, the corresponding result in \p *capabilities + * will be a bitmask indicating which of ::cudaAtomicOperationCapability the link supports natively. + * + * Returns ::cudaErrorInvalidDevice if \p srcDevice or \p dstDevice are not valid + * or if they represent the same device. + * + * Returns ::cudaErrorInvalidValue if \p *capabilities or \p *operations is NULL, if \p count is 0, + * or if any of \p *operations is not valid. + * + * \param capabilities - Returned capability details of each requested operation + * \param operations - Requested operations + * \param count - Count of requested operations and size of capabilities + * \param srcDevice - The source device of the target link + * \param dstDevice - The destination device of the target link + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidDevice, + * ::cudaErrorInvalidValue + * \notefnerr + * + * \sa + * ::cudaDeviceGetP2PAttribute, + * ::cuDeviceGetP2PAttribute, + * ::cuDeviceGetP2PAtomicCapabilities + */ +extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaDeviceGetP2PAtomicCapabilities(unsigned int *capabilities, const enum cudaAtomicOperation *operations, unsigned int count, int srcDevice, int dstDevice); + + /** * \brief Select compute-device which best matches criteria * @@ -2239,7 +1721,7 @@ extern __host__ cudaError_t CUDARTAPI cudaSetValidDevices(int *device_arr, int l * the calling thread, a default device is selected and initialized with the * provided flags. * - * The two LSBs of the \p flags parameter can be used to control how the CPU + * The three LSBs of the \p flags parameter can be used to control how the CPU * thread interacts with the OS scheduler when waiting for results from the * device. * @@ -2275,8 +1757,12 @@ extern __host__ cudaError_t CUDARTAPI cudaSetValidDevices(int *device_arr, int l * after resizing local memory for a kernel. This can prevent thrashing by * local memory allocations when launching many kernels with high local * memory usage at the cost of potentially increased memory usage.
    - * \ref deprecated "Deprecated:" This flag is deprecated and the behavior enabled + * \ref deprecated "Deprecated:" This flag is deprecated and the behavior enabled * by this flag is now the default and cannot be disabled. + * - ::cudaDeviceSyncMemops: Ensures that synchronous memory operations initiated + * on this context will always synchronize. See further documentation in the + * section titled "API Synchronization behavior" to learn more about cases when + * synchronous memory operations can exhibit asynchronous behavior. * * \param flags - Parameters for device operation * @@ -2356,7 +1842,9 @@ extern __host__ cudaError_t CUDARTAPI cudaGetDeviceFlags( unsigned int *flags ); /** * \brief Create an asynchronous stream * - * Creates a new asynchronous stream. + * Creates a new asynchronous stream on the context that is current to the calling host thread. + * If no context is current to the calling host thread, then the primary context for a device + * is selected, made current to the calling thread, and initialized before creating a stream on it. * * \param pStream - Pointer to new stream identifier * @@ -2371,10 +1859,12 @@ extern __host__ cudaError_t CUDARTAPI cudaGetDeviceFlags( unsigned int *flags ); * ::cudaStreamCreateWithFlags, * ::cudaStreamGetPriority, * ::cudaStreamGetFlags, + * ::cudaStreamGetDevice, * ::cudaStreamQuery, * ::cudaStreamSynchronize, * ::cudaStreamWaitEvent, * ::cudaStreamAddCallback, + * ::cudaSetDevice, * ::cudaStreamDestroy, * ::cuStreamCreate */ @@ -2383,8 +1873,10 @@ extern __host__ cudaError_t CUDARTAPI cudaStreamCreate(cudaStream_t *pStream); /** * \brief Create an asynchronous stream * - * Creates a new asynchronous stream. The \p flags argument determines the - * behaviors of the stream. Valid values for \p flags are + * Creates a new asynchronous stream on the context that is current to the calling host thread. + * If no context is current to the calling host thread, then the primary context for a device + * is selected, made current to the calling thread, and initialized before creating a stream on it. + * The \p flags argument determines the behaviors of the stream. Valid values for \p flags are * - ::cudaStreamDefault: Default stream creation flag. * - ::cudaStreamNonBlocking: Specifies that work running in the created * stream may run concurrently with work in stream 0 (the NULL stream), and that @@ -2403,10 +1895,12 @@ extern __host__ cudaError_t CUDARTAPI cudaStreamCreate(cudaStream_t *pStream); * \sa ::cudaStreamCreate, * ::cudaStreamCreateWithPriority, * ::cudaStreamGetFlags, + * ::cudaStreamGetDevice, * ::cudaStreamQuery, * ::cudaStreamSynchronize, * ::cudaStreamWaitEvent, * ::cudaStreamAddCallback, + * ::cudaSetDevice, * ::cudaStreamDestroy, * ::cuStreamCreate */ @@ -2416,8 +1910,13 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamCreateWithFla * \brief Create an asynchronous stream with the specified priority * * Creates a stream with the specified priority and returns a handle in \p pStream. - * This API alters the scheduler priority of work in the stream. Work in a higher - * priority stream may preempt work already executing in a low priority stream. + * The stream is created on the context that is current to the calling host thread. + * If no context is current to the calling host thread, then the primary context for a device + * is selected, made current to the calling thread, and initialized before creating a stream on it. + * This affects the scheduling priority of work in the stream. Priorities provide a + * hint to preferentially run work with higher priority when possible, but do + * not preempt already-running work or provide any other functional guarantee on + * execution order. * * \p priority follows a convention where lower numbers represent higher priorities. * '0' represents default priority. The range of meaningful numerical priorities can @@ -2453,6 +1952,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamCreateWithFla * ::cudaStreamWaitEvent, * ::cudaStreamAddCallback, * ::cudaStreamSynchronize, + * ::cudaSetDevice, * ::cudaStreamDestroy, * ::cuStreamCreateWithPriority */ @@ -2481,6 +1981,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamCreateWithPri * \sa ::cudaStreamCreateWithPriority, * ::cudaDeviceGetStreamPriorityRange, * ::cudaStreamGetFlags, + * ::cudaStreamGetDevice, * ::cuStreamGetPriority */ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamGetPriority(cudaStream_t hStream, int *priority); @@ -2506,6 +2007,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamGetPriority(c * \sa ::cudaStreamCreateWithPriority, * ::cudaStreamCreateWithFlags, * ::cudaStreamGetPriority, + * ::cudaStreamGetDevice, * ::cuStreamGetFlags */ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamGetFlags(cudaStream_t hStream, unsigned int *flags); @@ -2547,6 +2049,32 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamGetFlags(cuda */ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamGetId(cudaStream_t hStream, unsigned long long *streamId); +/** + * \brief Query the device of a stream + * + * Returns in \p *device the device of the stream. + * + * \param hStream - Handle to the stream to be queried + * \param device - Returns the device to which the stream belongs + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + * ::cudaErrorDeviceUnavailable, + * \note_null_stream + * \notefnerr + * \note_init_rt + * \note_callback + * + * \sa ::cudaSetDevice, + * ::cudaGetDevice, + * ::cudaStreamCreate, + * ::cudaStreamGetPriority, + * ::cudaStreamGetFlags, + * ::cuStreamGetId + */ +extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamGetDevice(cudaStream_t hStream, int *device); + /** * \brief Resets all persisting lines in cache to normal status. * @@ -2939,6 +2467,47 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamAttachMemAsyn */ extern __host__ cudaError_t CUDARTAPI cudaStreamBeginCapture(cudaStream_t stream, enum cudaStreamCaptureMode mode); +/** + * \brief Begins graph capture on a stream to an existing graph + * + * Begin graph capture on \p stream. When a stream is in capture mode, all operations + * pushed into the stream will not be executed, but will instead be captured into + * \p graph, which will be returned via ::cudaStreamEndCapture. + * + * Capture may not be initiated if \p stream is ::cudaStreamLegacy. Capture must be ended on the + * same stream in which it was initiated, and it may only be initiated if the stream is not + * already in capture mode. The capture mode may be queried via ::cudaStreamIsCapturing. A unique id + * representing the capture sequence may be queried via ::cudaStreamGetCaptureInfo. + * + * If \p mode is not ::cudaStreamCaptureModeRelaxed, ::cudaStreamEndCapture must be + * called on this stream from the same thread. + * + * \note Kernels captured using this API must not use texture and surface references. + * Reading or writing through any texture or surface reference is undefined + * behavior. This restriction does not apply to texture and surface objects. + * + * \param stream - Stream in which to initiate capture. + * \param graph - Graph to capture into. + * \param dependencies - Dependencies of the first node captured in the stream. Can be NULL if numDependencies is 0. + * \param dependencyData - Optional array of data associated with each dependency. + * \param numDependencies - Number of dependencies. + * \param mode - Controls the interaction of this capture sequence with other API + * calls that are potentially unsafe. For more details see + * ::cudaThreadExchangeStreamCaptureMode. + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue + * \notefnerr + * + * \sa + * ::cudaStreamCreate, + * ::cudaStreamIsCapturing, + * ::cudaStreamEndCapture, + * ::cudaThreadExchangeStreamCaptureMode + */ +extern __host__ cudaError_t CUDARTAPI cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t *dependencies, const cudaGraphEdgeData *dependencyData, size_t numDependencies, enum cudaStreamCaptureMode mode); + /** * \brief Swaps the stream capture interaction mode for a thread * @@ -3014,7 +2583,8 @@ extern __host__ cudaError_t CUDARTAPI cudaThreadExchangeStreamCaptureMode(enum c * \sa * ::cudaStreamCreate, * ::cudaStreamBeginCapture, - * ::cudaStreamIsCapturing + * ::cudaStreamIsCapturing, + * ::cudaGraphDestroy */ extern __host__ cudaError_t CUDARTAPI cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t *pGraph); @@ -3068,6 +2638,11 @@ extern __host__ cudaError_t CUDARTAPI cudaStreamIsCapturing(cudaStream_t stream, * - the call returns cudaSuccess * - the returned capture status is ::cudaStreamCaptureStatusActive * + * If \p edgeData_out is non-NULL then \p dependencies_out must be as well. If + * \p dependencies_out is non-NULL and \p edgeData_out is NULL, but there is non-zero edge + * data for one or more of the current stream dependencies, the call will return + * ::cudaErrorLossyQuery. + * * \param stream - The stream to query * \param captureStatus_out - Location to return the capture status of the stream; required * \param id_out - Optional location to return an id for the capture sequence, which is @@ -3083,17 +2658,23 @@ extern __host__ cudaError_t CUDARTAPI cudaStreamIsCapturing(cudaStream_t stream, * \param dependencies_out - Optional location to store a pointer to an array of nodes. * The next node to be captured in the stream will depend on this set of nodes, * absent operations such as event wait which modify this set. The array pointer - * is valid until the next API call which operates on the stream or until end of - * capture. The node handles may be copied out and are valid until they or the - * graph is destroyed. The driver-owned array may also be passed directly to - * APIs that operate on the graph (not the stream) without copying. + * is valid until the next API call which operates on the stream or until the + * capture is terminated. The node handles may be copied out and are valid until + * they or the graph is destroyed. The driver-owned array may also be passed + * directly to APIs that operate on the graph (not the stream) without copying. + * \param edgeData_out - Optional location to store a pointer to an array of graph edge + * data. This array parallels \c dependencies_out; the next node to be added + * has an edge to \c dependencies_out[i] with annotation \c edgeData_out[i] for + * each \c i. The array pointer is valid until the next API call which operates + * on the stream or until the capture is terminated. * \param numDependencies_out - Optional location to store the size of the array * returned in dependencies_out. * * \return * ::cudaSuccess, * ::cudaErrorInvalidValue, - * ::cudaErrorStreamCaptureImplicit + * ::cudaErrorStreamCaptureImplicit, + * ::cudaErrorLossyQuery * \note_graph_thread_safety * \notefnerr * @@ -3102,10 +2683,13 @@ extern __host__ cudaError_t CUDARTAPI cudaStreamIsCapturing(cudaStream_t stream, * ::cudaStreamIsCapturing, * ::cudaStreamUpdateCaptureDependencies */ -extern __host__ cudaError_t CUDARTAPI cudaStreamGetCaptureInfo(cudaStream_t stream, enum cudaStreamCaptureStatus *captureStatus_out, unsigned long long *id_out __dv(0), cudaGraph_t *graph_out __dv(0), const cudaGraphNode_t **dependencies_out __dv(0), size_t *numDependencies_out __dv(0)); +extern __host__ cudaError_t CUDARTAPI cudaStreamGetCaptureInfo(cudaStream_t stream, + enum cudaStreamCaptureStatus *captureStatus_out, unsigned long long *id_out __dv(0), + cudaGraph_t *graph_out __dv(0), const cudaGraphNode_t **dependencies_out __dv(0), + const cudaGraphEdgeData **edgeData_out __dv(0), size_t *numDependencies_out __dv(0)); /** - * \brief Update the set of dependencies in a capturing stream (11.3+) + * \brief Update the set of dependencies in a capturing stream * * Modifies the dependency set of a capturing stream. The dependency set is the set * of nodes that the next captured node in the stream will depend on. @@ -3121,8 +2705,11 @@ extern __host__ cudaError_t CUDARTAPI cudaStreamGetCaptureInfo(cudaStream_t stre * * Returns ::cudaErrorIllegalState if the stream is not capturing. * - * This API is new in CUDA 11.3. Developers requiring compatibility across minor - * versions of the CUDA driver to 11.0 should not use this API or provide a fallback. + * \param stream - The stream to update + * \param dependencies - The set of dependencies to add + * \param dependencyData - Optional array of data associated with each dependency. + * \param numDependencies - The size of the dependencies array + * \param flags - See above * * \return * ::cudaSuccess, @@ -3134,7 +2721,7 @@ extern __host__ cudaError_t CUDARTAPI cudaStreamGetCaptureInfo(cudaStream_t stre * ::cudaStreamBeginCapture, * ::cudaStreamGetCaptureInfo, */ -extern __host__ cudaError_t CUDARTAPI cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t *dependencies, size_t numDependencies, unsigned int flags __dv(0)); +extern __host__ cudaError_t CUDARTAPI cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t *dependencies, const cudaGraphEdgeData *dependencyData, size_t numDependencies, unsigned int flags __dv(0)); /** @} */ /* END CUDART_STREAM */ /** @@ -3240,6 +2827,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaEventCreateWithFlag * \notefnerr * \note_init_rt * \note_callback + * \note_null_event * * \sa \ref ::cudaEventCreate(cudaEvent_t*) "cudaEventCreate (C API)", * ::cudaEventCreateWithFlags, ::cudaEventQuery, @@ -3286,6 +2874,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaEventRecord(cudaEve * \notefnerr * \note_init_rt * \note_callback + * \note_null_event * * \sa \ref ::cudaEventCreate(cudaEvent_t*) "cudaEventCreate (C API)", * ::cudaEventCreateWithFlags, ::cudaEventQuery, @@ -3321,6 +2910,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaEventRecord(cudaEve * \notefnerr * \note_init_rt * \note_callback + * \note_null_event * * \sa \ref ::cudaEventCreate(cudaEvent_t*) "cudaEventCreate (C API)", * ::cudaEventCreateWithFlags, ::cudaEventRecord, @@ -3351,6 +2941,7 @@ extern __host__ cudaError_t CUDARTAPI cudaEventQuery(cudaEvent_t event); * \notefnerr * \note_init_rt * \note_callback + * \note_null_event * * \sa \ref ::cudaEventCreate(cudaEvent_t*) "cudaEventCreate (C API)", * ::cudaEventCreateWithFlags, ::cudaEventRecord, @@ -3380,6 +2971,7 @@ extern __host__ cudaError_t CUDARTAPI cudaEventSynchronize(cudaEvent_t event); * \note_init_rt * \note_callback * \note_destroy_ub + * \note_null_event * * \sa \ref ::cudaEventCreate(cudaEvent_t*) "cudaEventCreate (C API)", * ::cudaEventCreateWithFlags, ::cudaEventQuery, @@ -3392,7 +2984,10 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaEventDestroy(cudaEv * \brief Computes the elapsed time between events * * Computes the elapsed time between two events (in milliseconds with a - * resolution of around 0.5 microseconds). + * resolution of around 0.5 microseconds). Note this API is not guaranteed + * to return the latest errors for pending work. As such this API is intended to + * serve as a elapsed time calculation only and polling for completion on the + * events to be compared should be done with ::cudaEventQuery instead. * * If either event was last recorded in a non-NULL stream, the resulting time * may be greater than expected (even if both used the same stream handle). This @@ -3424,6 +3019,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaEventDestroy(cudaEv * \notefnerr * \note_init_rt * \note_callback + * \note_null_event * * \sa \ref ::cudaEventCreate(cudaEvent_t*) "cudaEventCreate (C API)", * ::cudaEventCreateWithFlags, ::cudaEventQuery, @@ -3964,6 +3560,19 @@ extern __host__ cudaError_t CUDARTAPI cudaImportExternalSemaphore(cudaExternalSe * same semaphore object with deterministic fence support enabled in different streams * or by adding explicit dependency amongst such streams so that the semaphore is * signaled in order. + * ::cudaExternalSemaphoreSignalParams::params::nvSciSync::fence associated with + * semaphore object of the type ::cudaExternalSemaphoreHandleTypeNvSciSync can be + * timestamp enabled. For this the NvSciSyncAttrList used to create the object must + * have the value of NvSciSyncAttrKey_WaiterRequireTimestamps key set to true. Timestamps + * are emitted asynchronously by the GPU and CUDA saves the GPU timestamp in the + * corresponding NvSciSyncFence at the time of signal on GPU. Users are expected to + * convert GPU clocks to CPU clocks using appropriate scaling functions. Users are + * expected to wait for the completion of the fence before extracting timestamp using + * appropriate NvSciSync APIs. Users are expected to ensure that there is only one + * outstanding timestamp enabled fence per Cuda-NvSciSync object at any point of time, + * failing which leads to undefined behavior. Extracting the timestamp before the + * corresponding fence is signalled could lead to undefined behaviour. Timestamp + * extracted via appropriate NvSciSync API would be in microseconds. * * If the semaphore object is any one of the following types: * ::cudaExternalSemaphoreHandleTypeKeyedMutex, @@ -4148,6 +3757,7 @@ extern __host__ cudaError_t CUDARTAPI cudaDestroyExternalSemaphore(cudaExternalS * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa * \ref ::cudaLaunchKernel(const T *func, dim3 gridDim, dim3 blockDim, void **args, size_t sharedMem, cudaStream_t stream) "cudaLaunchKernel (C++ API)", @@ -4210,6 +3820,7 @@ extern __host__ cudaError_t CUDARTAPI cudaLaunchKernel(const void *func, dim3 gr * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa * \ref ::cudaLaunchKernelEx(const cudaLaunchConfig_t *config, void (*kernel)(ExpTypes...), ActTypes &&... args) "cudaLaunchKernelEx (C++ API)", @@ -4266,115 +3877,14 @@ extern __host__ cudaError_t CUDARTAPI cudaLaunchKernelExC(const cudaLaunchConfig * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa * \ref ::cudaLaunchCooperativeKernel(const T *func, dim3 gridDim, dim3 blockDim, void **args, size_t sharedMem, cudaStream_t stream) "cudaLaunchCooperativeKernel (C++ API)", - * ::cudaLaunchCooperativeKernelMultiDevice, * ::cuLaunchCooperativeKernel */ extern __host__ cudaError_t CUDARTAPI cudaLaunchCooperativeKernel(const void *func, dim3 gridDim, dim3 blockDim, void **args, size_t sharedMem, cudaStream_t stream); -/** - * \brief Launches device functions on multiple devices where thread blocks can cooperate and synchronize as they execute - * - * \deprecated This function is deprecated as of CUDA 11.3. - * - * Invokes kernels as specified in the \p launchParamsList array where each element - * of the array specifies all the parameters required to perform a single kernel launch. - * These kernels can cooperate and synchronize as they execute. The size of the array is - * specified by \p numDevices. - * - * No two kernels can be launched on the same device. All the devices targeted by this - * multi-device launch must be identical. All devices must have a non-zero value for the - * device attribute ::cudaDevAttrCooperativeMultiDeviceLaunch. - * - * The same kernel must be launched on all devices. Note that any __device__ or __constant__ - * variables are independently instantiated on every device. It is the application's - * responsiblity to ensure these variables are initialized and used appropriately. - * - * The size of the grids as specified in blocks, the size of the blocks themselves and the - * amount of shared memory used by each thread block must also match across all launched kernels. - * - * The streams used to launch these kernels must have been created via either ::cudaStreamCreate - * or ::cudaStreamCreateWithPriority or ::cudaStreamCreateWithPriority. The NULL stream or - * ::cudaStreamLegacy or ::cudaStreamPerThread cannot be used. - * - * The total number of blocks launched per kernel cannot exceed the maximum number of blocks - * per multiprocessor as returned by ::cudaOccupancyMaxActiveBlocksPerMultiprocessor (or - * ::cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags) times the number of multiprocessors - * as specified by the device attribute ::cudaDevAttrMultiProcessorCount. Since the - * total number of blocks launched per device has to match across all devices, the maximum - * number of blocks that can be launched per device will be limited by the device with the - * least number of multiprocessors. - * - * The kernel cannot make use of CUDA dynamic parallelism. - * - * The ::cudaLaunchParams structure is defined as: - * \code - struct cudaLaunchParams - { - void *func; - dim3 gridDim; - dim3 blockDim; - void **args; - size_t sharedMem; - cudaStream_t stream; - }; - * \endcode - * where: - * - ::cudaLaunchParams::func specifies the kernel to be launched. This same functions must - * be launched on all devices. For templated functions, pass the function symbol as follows: - * func_name - * - ::cudaLaunchParams::gridDim specifies the width, height and depth of the grid in blocks. - * This must match across all kernels launched. - * - ::cudaLaunchParams::blockDim is the width, height and depth of each thread block. This - * must match across all kernels launched. - * - ::cudaLaunchParams::args specifies the arguments to the kernel. If the kernel has - * N parameters then ::cudaLaunchParams::args should point to array of N pointers. Each - * pointer, from ::cudaLaunchParams::args[0] to ::cudaLaunchParams::args[N - 1], - * point to the region of memory from which the actual parameter will be copied. - * - ::cudaLaunchParams::sharedMem is the dynamic shared-memory size per thread block in bytes. - * This must match across all kernels launched. - * - ::cudaLaunchParams::stream is the handle to the stream to perform the launch in. This cannot - * be the NULL stream or ::cudaStreamLegacy or ::cudaStreamPerThread. - * - * By default, the kernel won't begin execution on any GPU until all prior work in all the specified - * streams has completed. This behavior can be overridden by specifying the flag - * ::cudaCooperativeLaunchMultiDeviceNoPreSync. When this flag is specified, each kernel - * will only wait for prior work in the stream corresponding to that GPU to complete before it begins - * execution. - * - * Similarly, by default, any subsequent work pushed in any of the specified streams will not begin - * execution until the kernels on all GPUs have completed. This behavior can be overridden by specifying - * the flag ::cudaCooperativeLaunchMultiDeviceNoPostSync. When this flag is specified, - * any subsequent work pushed in any of the specified streams will only wait for the kernel launched - * on the GPU corresponding to that stream to complete before it begins execution. - * - * \param launchParamsList - List of launch parameters, one per device - * \param numDevices - Size of the \p launchParamsList array - * \param flags - Flags to control launch behavior - * - * \return - * ::cudaSuccess, - * ::cudaErrorInvalidDeviceFunction, - * ::cudaErrorInvalidConfiguration, - * ::cudaErrorLaunchFailure, - * ::cudaErrorLaunchTimeout, - * ::cudaErrorLaunchOutOfResources, - * ::cudaErrorCooperativeLaunchTooLarge, - * ::cudaErrorSharedObjectInitFailed - * \note_null_stream - * \notefnerr - * \note_init_rt - * \note_callback - * - * \sa - * \ref ::cudaLaunchCooperativeKernel(const T *func, dim3 gridDim, dim3 blockDim, void **args, size_t sharedMem, cudaStream_t stream) "cudaLaunchCooperativeKernel (C++ API)", - * ::cudaLaunchCooperativeKernel, - * ::cuLaunchCooperativeKernelMultiDevice - */ -extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaLaunchCooperativeKernelMultiDevice(struct cudaLaunchParams *launchParamsList, unsigned int numDevices, unsigned int flags __dv(0)); - /** * \brief Sets the preferred cache configuration for a device function * @@ -4412,6 +3922,11 @@ extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaLaunchCooperativeKer * \note_init_rt * \note_callback * + * \note This API does not accept a ::cudaKernel_t casted as void*. If cache config modification + * is required for a ::cudaKernel_t (or a __global__ function), it can be replaced with a call to + * ::cudaFuncSetAttributes with the attribute ::cudaFuncAttributePreferredSharedMemoryCarveout + * to specify a more granular L1 cache and shared memory split configuration. + * * \sa * \ref ::cudaFuncSetCacheConfig(T*, enum cudaFuncCache) "cudaFuncSetCacheConfig (C++ API)", * \ref ::cudaFuncGetAttributes(struct cudaFuncAttributes*, const void*) "cudaFuncGetAttributes (C API)", @@ -4421,73 +3936,19 @@ extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaLaunchCooperativeKer extern __host__ cudaError_t CUDARTAPI cudaFuncSetCacheConfig(const void *func, enum cudaFuncCache cacheConfig); /** - * \brief Sets the shared memory configuration for a device function + * \brief Find out attributes for a given function * - * On devices with configurable shared memory banks, this function will - * force all subsequent launches of the specified device function to have - * the given shared memory bank size configuration. On any given launch of the - * function, the shared memory configuration of the device will be temporarily - * changed if needed to suit the function's preferred configuration. Changes in - * shared memory configuration between subsequent launches of functions, - * may introduce a device side synchronization point. + * This function obtains the attributes of a function specified via \p func. + * \p func is a device function symbol and must be declared as a + * \c __global__ function. The fetched attributes are placed in \p attr. + * If the specified function does not exist, then it is assumed to + * be a ::cudaKernel_t and used as is. + * For templated functions, pass the function symbol as follows: + * func_name * - * Any per-function setting of shared memory bank size set via - * ::cudaFuncSetSharedMemConfig will override the device wide setting set by - * ::cudaDeviceSetSharedMemConfig. - * - * Changing the shared memory bank size will not increase shared memory usage - * or affect occupancy of kernels, but may have major effects on performance. - * Larger bank sizes will allow for greater potential bandwidth to shared memory, - * but will change what kinds of accesses to shared memory will result in bank - * conflicts. - * - * This function will do nothing on devices with fixed shared memory bank size. - * - * For templated functions, pass the function symbol as follows: - * func_name - * - * The supported bank configurations are: - * - ::cudaSharedMemBankSizeDefault: use the device's shared memory configuration - * when launching this function. - * - ::cudaSharedMemBankSizeFourByte: set shared memory bank width to be - * four bytes natively when launching this function. - * - ::cudaSharedMemBankSizeEightByte: set shared memory bank width to be eight - * bytes natively when launching this function. - * - * \param func - Device function symbol - * \param config - Requested shared memory configuration - * - * \return - * ::cudaSuccess, - * ::cudaErrorInvalidDeviceFunction, - * ::cudaErrorInvalidValue, - * \notefnerr - * \note_string_api_deprecation2 - * \note_init_rt - * \note_callback - * - * \sa ::cudaDeviceSetSharedMemConfig, - * ::cudaDeviceGetSharedMemConfig, - * ::cudaDeviceSetCacheConfig, - * ::cudaDeviceGetCacheConfig, - * ::cudaFuncSetCacheConfig, - * ::cuFuncSetSharedMemConfig - */ -extern __host__ cudaError_t CUDARTAPI cudaFuncSetSharedMemConfig(const void *func, enum cudaSharedMemConfig config); - -/** - * \brief Find out attributes for a given function - * - * This function obtains the attributes of a function specified via \p func. - * \p func is a device function symbol and must be declared as a - * \c __global__ function. The fetched attributes are placed in \p attr. - * If the specified function does not exist, then - * ::cudaErrorInvalidDeviceFunction is returned. For templated functions, pass - * the function symbol as follows: func_name - * - * Note that some function attributes such as - * \ref ::cudaFuncAttributes::maxThreadsPerBlock "maxThreadsPerBlock" - * may vary based on the device that is currently being used. + * Note that some function attributes such as + * \ref ::cudaFuncAttributes::maxThreadsPerBlock "maxThreadsPerBlock" + * may vary based on the device that is currently being used. * * \param attr - Return pointer to function's attributes * \param func - Device function symbol @@ -4499,6 +3960,7 @@ extern __host__ cudaError_t CUDARTAPI cudaFuncSetSharedMemConfig(const void *fun * \note_string_api_deprecation2 * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa * \ref ::cudaFuncSetCacheConfig(const void*, enum cudaFuncCache) "cudaFuncSetCacheConfig (C API)", @@ -4516,7 +3978,8 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaFuncGetAttributes(s * The parameter \p func must be a pointer to a function that executes * on the device. The parameter specified by \p func must be declared as a \p __global__ * function. The enumeration defined by \p attr is set to the value defined by \p value. - * If the specified function does not exist, then ::cudaErrorInvalidDeviceFunction is returned. + * If the specified function does not exist, then it is assumed to + * be a ::cudaKernel_t and used as is. * If the specified attribute cannot be written, or if the value is incorrect, * then ::cudaErrorInvalidValue is returned. * @@ -4526,6 +3989,26 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaFuncGetAttributes(s * - ::cudaFuncAttributePreferredSharedMemoryCarveout - On devices where the L1 cache and shared memory use the same hardware resources, * this sets the shared memory carveout preference, in percent of the total shared memory. See ::cudaDevAttrMaxSharedMemoryPerMultiprocessor. * This is only a hint, and the driver can choose a different ratio if required to execute the function. + * - ::cudaFuncAttributeRequiredClusterWidth: The required cluster width in + * blocks. The width, height, and depth values must either all be 0 or all be + * positive. The validity of the cluster dimensions is checked at launch time. + * If the value is set during compile time, it cannot be set at runtime. + * Setting it at runtime will return cudaErrorNotPermitted. + * - ::cudaFuncAttributeRequiredClusterHeight: The required cluster height in + * blocks. The width, height, and depth values must either all be 0 or all be + * positive. The validity of the cluster dimensions is checked at launch time. + * If the value is set during compile time, it cannot be set at runtime. + * Setting it at runtime will return cudaErrorNotPermitted. + * - ::cudaFuncAttributeRequiredClusterDepth: The required cluster depth in + * blocks. The width, height, and depth values must either all be 0 or all be + * positive. The validity of the cluster dimensions is checked at launch time. + * If the value is set during compile time, it cannot be set at runtime. + * Setting it at runtime will return cudaErrorNotPermitted. + * - ::cudaFuncAttributeNonPortableClusterSizeAllowed: Indicates whether the + * function can be launched with non-portable cluster size. 1 is allowed, 0 is + * disallowed. + * - ::cudaFuncAttributeClusterSchedulingPolicyPreference: The block + * scheduling policy of a function. The value type is cudaClusterSchedulingPolicy. * * \param func - Function to get attributes of * \param attr - Attribute to set @@ -4538,6 +4021,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaFuncGetAttributes(s * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \ref ::cudaLaunchKernel(const T *func, dim3 gridDim, dim3 blockDim, void **args, size_t sharedMem, cudaStream_t stream) "cudaLaunchKernel (C++ API)", * \ref ::cudaFuncSetCacheConfig(T*, enum cudaFuncCache) "cudaFuncSetCacheConfig (C++ API)", @@ -4546,52 +4030,53 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaFuncGetAttributes(s extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaFuncSetAttribute(const void *func, enum cudaFuncAttribute attr, int value); /** - * \brief Converts a double argument to be executed on a device - * - * \param d - Double to convert + * \brief Returns the function name for a device entry function pointer. * - * \deprecated This function is deprecated as of CUDA 7.5 + * Returns in \p **name the function name associated with the symbol \p func . + * The function name is returned as a null-terminated string. This API may + * return a mangled name if the function is not declared as having C linkage. + * If \p **name is NULL, ::cudaErrorInvalidValue is returned. + * If \p func is not a device entry function, then it is assumed to + * be a ::cudaKernel_t and used as is. * - * Converts the double value of \p d to an internal float representation if - * the device does not support double arithmetic. If the device does natively - * support doubles, then this function does nothing. + * \param name - The returned name of the function + * \param func - The function pointer to retrieve name for * * \return - * ::cudaSuccess + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidDeviceFunction * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * - * \sa - * \ref ::cudaFuncSetCacheConfig(const void*, enum cudaFuncCache) "cudaFuncSetCacheConfig (C API)", - * \ref ::cudaFuncGetAttributes(struct cudaFuncAttributes*, const void*) "cudaFuncGetAttributes (C API)", - * ::cudaSetDoubleForHost + * \ref ::cudaFuncGetName(const char **name, const T *func) "cudaFuncGetName (C++ API)" */ -extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaSetDoubleForDevice(double *d); +extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaFuncGetName(const char **name, const void *func); /** - * \brief Converts a double argument after execution on a device - * - * \deprecated This function is deprecated as of CUDA 7.5 + * \brief Returns the offset and size of a kernel parameter in the device-side parameter layout. * - * Converts the double value of \p d from a potentially internal float - * representation if the device does not support double arithmetic. If the - * device does natively support doubles, then this function does nothing. + * Queries the kernel parameter at \p paramIndex in \p func's list of parameters and returns + * parameter information via \p paramOffset and \p paramSize. \p paramOffset returns the + * offset of the parameter in the device-side parameter layout. \p paramSize returns the size + * in bytes of the parameter. This information can be used to update kernel node parameters + * from the device via ::cudaGraphKernelNodeSetParam() and ::cudaGraphKernelNodeUpdatesApply(). + * \p paramIndex must be less than the number of parameters that \p func takes. * - * \param d - Double to convert + * \param func - The function to query + * \param paramIndex - The parameter index to query + * \param paramOffset - The offset into the device-side parameter layout at which the parameter resides + * \param paramSize - The size of the parameter in the device-side parameter layout * * \return - * ::cudaSuccess + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE, * \notefnerr - * \note_init_rt - * \note_callback - * - * \sa - * \ref ::cudaFuncSetCacheConfig(const void*, enum cudaFuncCache) "cudaFuncSetCacheConfig (C API)", - * \ref ::cudaFuncGetAttributes(struct cudaFuncAttributes*, const void*) "cudaFuncGetAttributes (C API)", - * ::cudaSetDoubleForDevice + * \note_cudaKernel_t */ -extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaSetDoubleForHost(double *d); +extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaFuncGetParamInfo(const void *func, size_t paramIndex, size_t *paramOffset, size_t *paramSize); /** * \brief Enqueues a host function call in a stream @@ -4661,6 +4146,79 @@ extern __host__ cudaError_t CUDARTAPI cudaLaunchHostFunc(cudaStream_t stream, cu /** @} */ /* END CUDART_EXECUTION */ +/** + * \defgroup CUDART_EXECUTION_DEPRECATED Execution Control [DEPRECATED] + * + * ___MANBRIEF___ deprecated execution control functions of the CUDA runtime API + * (___CURRENT_FILE___) ___ENDMANBRIEF___ + * + * This section describes the deprecated execution control functions of the CUDA runtime + * application programming interface. + * + * Some functions have overloaded C++ API template versions documented separately in the + * \ref CUDART_HIGHLEVEL "C++ API Routines" module. + * + * @{ + */ + +/** + * \brief Sets the shared memory configuration for a device function + * + * \deprecated + * + * On devices with configurable shared memory banks, this function will + * force all subsequent launches of the specified device function to have + * the given shared memory bank size configuration. On any given launch of the + * function, the shared memory configuration of the device will be temporarily + * changed if needed to suit the function's preferred configuration. Changes in + * shared memory configuration between subsequent launches of functions, + * may introduce a device side synchronization point. + * + * Any per-function setting of shared memory bank size set via + * ::cudaFuncSetSharedMemConfig will override the device wide setting set by + * ::cudaDeviceSetSharedMemConfig. + * + * Changing the shared memory bank size will not increase shared memory usage + * or affect occupancy of kernels, but may have major effects on performance. + * Larger bank sizes will allow for greater potential bandwidth to shared memory, + * but will change what kinds of accesses to shared memory will result in bank + * conflicts. + * + * This function will do nothing on devices with fixed shared memory bank size. + * + * For templated functions, pass the function symbol as follows: + * func_name + * + * The supported bank configurations are: + * - ::cudaSharedMemBankSizeDefault: use the device's shared memory configuration + * when launching this function. + * - ::cudaSharedMemBankSizeFourByte: set shared memory bank width to be + * four bytes natively when launching this function. + * - ::cudaSharedMemBankSizeEightByte: set shared memory bank width to be eight + * bytes natively when launching this function. + * + * \param func - Device function symbol + * \param config - Requested shared memory configuration + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidDeviceFunction, + * ::cudaErrorInvalidValue, + * \notefnerr + * \note_string_api_deprecation2 + * \note_init_rt + * \note_callback + * + * \sa ::cudaDeviceSetSharedMemConfig, + * ::cudaDeviceGetSharedMemConfig, + * ::cudaDeviceSetCacheConfig, + * ::cudaDeviceGetCacheConfig, + * ::cudaFuncSetCacheConfig, + * ::cuFuncSetSharedMemConfig + */ +extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaFuncSetSharedMemConfig(const void *func, enum cudaSharedMemConfig config); +/** @} */ /* END CUDART_EXECUTION_DEPRECATED */ + /** * \defgroup CUDART_OCCUPANCY Occupancy * @@ -4705,6 +4263,7 @@ extern __host__ cudaError_t CUDARTAPI cudaLaunchHostFunc(cudaStream_t stream, cu * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa ::cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, * \ref ::cudaOccupancyMaxPotentialBlockSize(int*, int*, T, size_t, int) "cudaOccupancyMaxPotentialBlockSize (C++ API)", @@ -4735,6 +4294,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaOccupancyMaxActiveB * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa ::cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, * \ref ::cudaOccupancyMaxPotentialBlockSize(int*, int*, T, size_t, int) "cudaOccupancyMaxPotentialBlockSize (C++ API)", @@ -4779,6 +4339,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaOccupancyAvailableD * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa ::cudaOccupancyMaxActiveBlocksPerMultiprocessor, * \ref ::cudaOccupancyMaxPotentialBlockSize(int*, int*, T, size_t, int) "cudaOccupancyMaxPotentialBlockSize (C++ API)", @@ -4818,6 +4379,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaOccupancyMaxActiveB * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa ::cudaFuncGetAttributes * \ref ::cudaOccupancyMaxPotentialClusterSize(int*, T, const cudaLaunchConfig_t*) "cudaOccupancyMaxPotentialClusterSize (C++ API)", @@ -4856,6 +4418,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaOccupancyMaxPotenti * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa * ::cudaFuncGetAttributes @@ -4916,7 +4479,7 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaOccupancyMaxActiveC * such GPUs may be evicted from device memory to host memory at any time by the Unified * Memory driver in order to make room for other allocations. * - * In a multi-GPU system where all GPUs have a non-zero value for the device attribute + * In a system where all GPUs have a non-zero value for the device attribute * ::cudaDevAttrConcurrentManagedAccess, managed memory may not be populated when this * API returns and instead may be populated on access. In such systems, managed memory can * migrate to any processor's memory at any time. The Unified Memory driver will employ heuristics to @@ -5025,11 +4588,15 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaMalloc(void **devPt * this function and automatically accelerates calls to functions such as * ::cudaMemcpy*(). Since the memory can be accessed directly by the device, * it can be read or written with much higher bandwidth than pageable memory - * obtained with functions such as ::malloc(). Allocating excessive amounts of - * memory with ::cudaMallocHost() may degrade system performance, since it - * reduces the amount of memory available to the system for paging. As a - * result, this function is best used sparingly to allocate staging areas for - * data exchange between host and device. + * obtained with functions such as ::malloc(). + + * On systems where ::pageableMemoryAccessUsesHostPageTables + * is true, ::cudaMallocHost may not page-lock the allocated memory. + + * Page-locking excessive amounts of memory with ::cudaMallocHost() may degrade + * system performance, since it reduces the amount of memory available to the + * system for paging. As a result, this function is best used sparingly to allocate + * staging areas for data exchange between host and device. * * \param ptr - Pointer to allocated host memory * \param size - Requested allocation size in bytes @@ -5155,9 +4722,10 @@ extern __host__ cudaError_t CUDARTAPI cudaMallocArray(cudaArray_t *array, const * * Note - This API will not perform any implicit synchronization when the pointer was * allocated with ::cudaMallocAsync or ::cudaMallocFromPoolAsync. Callers must ensure - * that all accesses to the pointer have completed before invoking ::cudaFree. For + * that all accesses to these pointer have completed before invoking ::cudaFree. For * best performance and memory reuse, users should use ::cudaFreeAsync to free memory * allocated via the stream ordered memory allocator. + * For all other pointers, this API may perform implicit synchronization. * * If ::cudaFree(\p devPtr) has already been called before, * an error is returned. If \p devPtr is 0, no operation is performed. @@ -5364,7 +4932,7 @@ extern __host__ cudaError_t CUDARTAPI cudaHostAlloc(void **pHost, size_t size, u * platforms without ::cudaDevAttrPageableMemoryAccessUsesHostPageTables, this * flag is required in order to register memory mapped to the CPU as * read-only. Support for the use of this flag can be queried from the device - * attribute cudaDeviceAttrReadOnlyHostRegisterSupported. Using this flag with + * attribute ::cudaDevAttrHostRegisterReadOnlySupported. Using this flag with * a current context associated with a device that does not have this attribute * set will cause ::cudaHostRegister to error with cudaErrorNotSupported. * @@ -6662,6 +6230,7 @@ extern __host__ cudaError_t CUDARTAPI cudaMemcpy2DArrayToArray(cudaArray_t dst, */ extern __host__ cudaError_t CUDARTAPI cudaMemcpyToSymbol(const void *symbol, const void *src, size_t count, size_t offset __dv(0), enum cudaMemcpyKind kind __dv(cudaMemcpyHostToDevice)); + /** * \brief Copies data from the given symbol on the device * @@ -6797,6 +6366,137 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaMemcpyAsync(void *d */ extern __host__ cudaError_t CUDARTAPI cudaMemcpyPeerAsync(void *dst, int dstDevice, const void *src, int srcDevice, size_t count, cudaStream_t stream __dv(0)); +/** + * \brief Performs a batch of memory copies asynchronously. + * + * Performs a batch of memory copies. The batch as a whole executes in stream order but copies within a + * batch are not guaranteed to execute in any specific order. This API only supports pointer-to-pointer copies. + * For copies involving CUDA arrays, please see ::cudaMemcpy3DBatchAsync. + * + * Performs memory copies from source buffers specified in \p srcs to destination buffers specified in \p dsts. + * The size of each copy is specified in \p sizes. All three arrays must be of the same length as specified + * by \p count. Since there are no ordering guarantees for copies within a batch, specifying any dependent copies + * within a batch will result in undefined behavior. + * + * Every copy in the batch has to be associated with a set of attributes specified in the \p attrs array. + * Each entry in this array can apply to more than one copy. This can be done by specifying in the \p attrsIdxs array, + * the index of the first copy that the corresponding entry in the \p attrs array applies to. Both \p attrs and + * \p attrsIdxs must be of the same length as specified by \p numAttrs. For example, if a batch has 10 copies listed + * in dst/src/sizes, the first 6 of which have one set of attributes and the remaining 4 another, then \p numAttrs + * will be 2, \p attrsIdxs will be {0, 6} and \p attrs will contains the two sets of attributes. Note that the first entry + * in \p attrsIdxs must always be 0. Also, each entry must be greater than the previous entry and the last entry should be + * less than \p count. Furthermore, \p numAttrs must be lesser than or equal to \p count. + * + * The ::cudaMemcpyAttributes::srcAccessOrder indicates the source access ordering to be observed for copies associated + * with the attribute. If the source access order is set to ::cudaMemcpySrcAccessOrderStream, then the source will + * be accessed in stream order. If the source access order is set to ::cudaMemcpySrcAccessOrderDuringApiCall then + * it indicates that access to the source pointer can be out of stream order and all accesses must be complete before the + * API call returns. This flag is suited for ephemeral sources (ex., stack variables) when it's known that no prior + * operations in the stream can be accessing the memory and also that the lifetime of the memory is limited to the scope + * that the source variable was declared in. Specifying this flag allows the driver to optimize the copy and removes the + * need for the user to synchronize the stream after the API call. If the source access order is set to + * ::cudaMemcpySrcAccessOrderAny then it indicates that access to the source pointer can be out of stream order and the + * accesses can happen even after the API call returns. This flag is suited for host pointers allocated + * outside CUDA (ex., via malloc) when it's known that no prior operations in the stream can be accessing the memory. + * Specifying this flag allows the driver to optimize the copy on certain platforms. Each memcpy operation in the batch + * must have a valid ::cudaMemcpyAttributes corresponding to it including the appropriate srcAccessOrder setting, + * otherwise the API will return ::cudaErrorInvalidValue. + * + * The ::cudaMemcpyAttributes::srcLocHint and ::cudaMemcpyAttributes::dstLocHint allows applications to specify hint locations + * for operands of a copy when the operand doesn't have a fixed location. That is, these hints are + * only applicable for managed memory pointers on devices where ::cudaDevAttrConcurrentManagedAccess is true or + * system-allocated pageable memory on devices where ::cudaDevAttrPageableMemoryAccess is true. + * For other cases, these hints are ignored. + * + * The ::cudaMemcpyAttributes::flags field can be used to specify certain flags for copies. Setting the + * ::cudaMemcpyFlagPreferOverlapWithCompute flag indicates that the associated copies should preferably overlap with + * any compute work. Note that this flag is a hint and can be ignored depending on the platform and other parameters of the copy. + * + * + * \param dsts - Array of destination pointers. + * \param srcs - Array of memcpy source pointers. + * \param sizes - Array of sizes for memcpy operations. + * \param count - Size of \p dsts, \p srcs and \p sizes arrays + * \param attrs - Array of memcpy attributes. + * \param attrsIdxs - Array of indices to specify which copies each entry in the \p attrs array applies to. + * The attributes specified in attrs[k] will be applied to copies starting from attrsIdxs[k] + * through attrsIdxs[k+1] - 1. Also attrs[numAttrs-1] will apply to copies starting from + * attrsIdxs[numAttrs-1] through count - 1. + * \param numAttrs - Size of \p attrs and \p attrsIdxs arrays. + * \param hStream - The stream to enqueue the operations in. Must not be legacy NULL stream. + * + * \return + * ::cudaSuccess + * ::cudaErrorInvalidValue + * \notefnerr + * \note_async + * \note_memcpy + */ +extern __host__ cudaError_t CUDARTAPI cudaMemcpyBatchAsync(void *const *dsts, const void *const *srcs, const size_t *sizes, size_t count, struct cudaMemcpyAttributes *attrs, size_t *attrsIdxs, size_t numAttrs, cudaStream_t stream); + +/** + * \brief Performs a batch of 3D memory copies asynchronously. + * + * Performs a batch of memory copies. The batch as a whole executes in stream order but copies within a + * batch are not guaranteed to execute in any specific order. Note that this means specifying any dependent + * copies within a batch will result in undefined behavior. + * + * Performs memory copies as specified in the \p opList array. The length of this array is specified in \p numOps. + * Each entry in this array describes a copy operation. This includes among other things, the source and destination + * operands for the copy as specified in ::cudaMemcpy3DBatchOp::src and ::cudaMemcpy3DBatchOp::dst respectively. + * The source and destination operands of a copy can either be a pointer or a CUDA array. The width, height and depth + * of a copy is specified in ::cudaMemcpy3DBatchOp::extent. The width, height and depth of a copy are specified in + * elements and must not be zero. For pointer-to-pointer copies, the element size is considered to be 1. For pointer + * to CUDA array or vice versa copies, the element size is determined by the CUDA array. For CUDA array to CUDA array copies, + * the element size of the two CUDA arrays must match. + * + * For a given operand, if ::cudaMemcpy3DOperand::type is specified as ::cudaMemcpyOperandTypePointer, then + * ::cudaMemcpy3DOperand::op::ptr will be used. The ::cudaMemcpy3DOperand::op::ptr::ptr field must contain the pointer where + * the copy should begin. The ::cudaMemcpy3DOperand::op::ptr::rowLength field specifies the length of each row in elements and + * must either be zero or be greater than or equal to the width of the copy specified in ::cudaMemcpy3DBatchOp::extent::width. + * The ::cudaMemcpy3DOperand::op::ptr::layerHeight field specifies the height of each layer and must either be + * zero or be greater than or equal to the height of the copy specified in ::cudaMemcpy3DBatchOp::extent::height. + * When either of these values is zero, that aspect of the operand is considered to be tightly packed according to the copy extent. + * For managed memory pointers on devices where ::cudaDevAttrConcurrentManagedAccess is true or system-allocated pageable memory + * on devices where ::cudaDevAttrPageableMemoryAccess is true, the ::cudaMemcpy3DOperand::op::ptr::locHint field can be used to hint + * the location of the operand. + * + * If an operand's type is specified as ::cudaMemcpyOperandTypeArray, then ::cudaMemcpy3DOperand::op::array will be used. + * The ::cudaMemcpy3DOperand::op::array::array field specifies the CUDA array and ::cudaMemcpy3DOperand::op::array::offset specifies + * the 3D offset into that array where the copy begins. + * + * The ::cudaMemcpyAttributes::srcAccessOrder indicates the source access ordering to be observed for copies associated + * with the attribute. If the source access order is set to ::cudaMemcpySrcAccessOrderStream, then the source will + * be accessed in stream order. If the source access order is set to ::cudaMemcpySrcAccessOrderDuringApiCall then + * it indicates that access to the source pointer can be out of stream order and all accesses must be complete before the + * API call returns. This flag is suited for ephemeral sources (ex., stack variables) when it's known that no prior + * operations in the stream can be accessing the memory and also that the lifetime of the memory is limited to the scope + * that the source variable was declared in. Specifying this flag allows the driver to optimize the copy and removes the + * need for the user to synchronize the stream after the API call. If the source access order is set to + * ::cudaMemcpySrcAccessOrderAny then it indicates that access to the source pointer can be out of stream order and the + * accesses can happen even after the API call returns. This flag is suited for host pointers allocated + * outside CUDA (ex., via malloc) when it's known that no prior operations in the stream can be accessing the memory. + * Specifying this flag allows the driver to optimize the copy on certain platforms. Each memcopy operation in \p opList + * must have a valid srcAccessOrder setting, otherwise this API will return ::cudaErrorInvalidValue. + * + * The ::cudaMemcpyAttributes::flags field can be used to specify certain flags for copies. Setting the + * ::cudaMemcpyFlagPreferOverlapWithCompute flag indicates that the associated copies should preferably overlap with + * any compute work. Note that this flag is a hint and can be ignored depending on the platform and other parameters of the copy. + * + * + * \param numOps - Total number of memcpy operations. + * \param opList - Array of size \p numOps containing the actual memcpy operations. + * \param flags - Flags for future use, must be zero now. + * \param hStream - The stream to enqueue the operations in. Must not be default NULL stream. + * + * \return + * ::cudaSuccess + * ::cudaErrorInvalidValue + * \notefnerr + * \note_async + * \note_memcpy + */ +extern __host__ cudaError_t CUDARTAPI cudaMemcpy3DBatchAsync(size_t numOps, struct cudaMemcpy3DBatchOp *opList, unsigned long long flags, cudaStream_t stream); /** * \brief Copies data between host and device * @@ -7372,18 +7072,26 @@ extern __host__ cudaError_t CUDARTAPI cudaGetSymbolAddress(void **devPtr, const extern __host__ cudaError_t CUDARTAPI cudaGetSymbolSize(size_t *size, const void *symbol); /** - * \brief Prefetches memory to the specified destination device + * \brief Prefetches memory to the specified destination location * - * Prefetches memory to the specified destination device. \p devPtr is the - * base device pointer of the memory to be prefetched and \p dstDevice is the - * destination device. \p count specifies the number of bytes to copy. \p stream + * Prefetches memory to the specified destination location. \p devPtr is the + * base device pointer of the memory to be prefetched and \p location specifies the + * destination location. \p count specifies the number of bytes to copy. \p stream * is the stream in which the operation is enqueued. The memory range must refer - * to managed memory allocated via ::cudaMallocManaged or declared via __managed__ variables. - * - * Passing in cudaCpuDeviceId for \p dstDevice will prefetch the data to host memory. If - * \p dstDevice is a GPU, then the device attribute ::cudaDevAttrConcurrentManagedAccess - * must be non-zero. Additionally, \p stream must be associated with a device that has a - * non-zero value for the device attribute ::cudaDevAttrConcurrentManagedAccess. + * to managed memory allocated via ::cudaMallocManaged or declared via __managed__ variables, + * or it may also refer to system-allocated memory on systems with non-zero + * cudaDevAttrPageableMemoryAccess. + * + * Specifying ::cudaMemLocationTypeDevice for ::cudaMemLocation::type will prefetch memory to GPU + * specified by device ordinal ::cudaMemLocation::id which must have non-zero value for the device attribute + * ::concurrentManagedAccess. Additionally, \p stream must be associated with a device + * that has a non-zero value for the device attribute ::concurrentManagedAccess. + * Specifying ::cudaMemLocationTypeHost as ::cudaMemLocation::type will prefetch data to host memory. + * Applications can request prefetching memory to a specific host NUMA node by specifying + * ::cudaMemLocationTypeHostNuma for ::cudaMemLocation::type and a valid host NUMA node id in ::cudaMemLocation::id + * Users can also request prefetching memory to the host NUMA node closest to the current thread's CPU by specifying + * ::cudaMemLocationTypeHostNumaCurrent for ::cudaMemLocation::type. Note when ::cudaMemLocation::type is etiher + * ::cudaMemLocationTypeHost OR ::cudaMemLocationTypeHostNumaCurrent, ::cudaMemLocation::id will be ignored. * * The start address and end address of the memory range will be rounded down and rounded up * respectively to be aligned to CPU page size before the prefetch operation is enqueued @@ -7396,15 +7104,17 @@ extern __host__ cudaError_t CUDARTAPI cudaGetSymbolSize(size_t *size, const void * allocated using ::cudaMalloc or ::cudaMallocArray will not be evicted. * * By default, any mappings to the previous location of the migrated pages are removed and - * mappings for the new location are only setup on \p dstDevice. The exact behavior however - * also depends on the settings applied to this memory range via ::cudaMemAdvise as described + * mappings for the new location are only setup on the destination location. The exact behavior however + * also depends on the settings applied to this memory range via ::cuMemAdvise as described * below: * * If ::cudaMemAdviseSetReadMostly was set on any subset of this memory range, - * then that subset will create a read-only copy of the pages on \p dstDevice. + * then that subset will create a read-only copy of the pages on destination location. + * If however the destination location is a host NUMA node, then any pages of that subset + * that are already in another host NUMA node will be transferred to the destination. * * If ::cudaMemAdviseSetPreferredLocation was called on any subset of this memory - * range, then the pages will be migrated to \p dstDevice even if \p dstDevice is not the + * range, then the pages will be migrated to \p location even if \p location is not the * preferred location of any pages in the memory range. * * If ::cudaMemAdviseSetAccessedBy was called on any subset of this memory range, @@ -7422,7 +7132,8 @@ extern __host__ cudaError_t CUDARTAPI cudaGetSymbolSize(size_t *size, const void * * \param devPtr - Pointer to be prefetched * \param count - Size in bytes - * \param dstDevice - Destination device to prefetch to + * \param location - location to prefetch to + * \param flags - flags for future use, must be zero now. * \param stream - Stream to enqueue prefetch operation * * \return @@ -7436,10 +7147,133 @@ extern __host__ cudaError_t CUDARTAPI cudaGetSymbolSize(size_t *size, const void * \note_callback * * \sa ::cudaMemcpy, ::cudaMemcpyPeer, ::cudaMemcpyAsync, - * ::cudaMemcpy3DPeerAsync, ::cudaMemAdvise, - * ::cuMemPrefetchAsync + * ::cudaMemcpy3DPeerAsync, ::cudaMemAdvise, ::cuMemPrefetchAsync + */ +extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchAsync(const void *devPtr, size_t count, struct cudaMemLocation location, unsigned int flags, cudaStream_t stream __dv(0)); + +/** + * \brief Performs a batch of memory prefetches asynchronously + * + * Performs a batch of memory prefetches. The batch as a whole executes in stream order + * but operations within a batch are not guaranteed to execute in any specific order. + * All devices in the system must have a non-zero value for the device attribute + * ::cudaDevAttrConcurrentManagedAccess otherwise the API will return an error. + * + * The semantics of the individual prefetch operations are as described in ::cudaMemPrefetchAsync. + * + * Performs memory prefetch on address ranges specified in \p dptrs and \p sizes. + * Both arrays must be of the same length as specified by \p count. Each memory range specified + * must refer to managed memory allocated via ::cudaMallocManaged or declared via + * __managed__ variables or it may also refer to system-allocated memory when all devices + * have a non-zero value for ::cudaDevAttrPageableMemoryAccess. The prefetch location for + * every operation in the batch is specified in the \p prefetchLocs array. Each entry in + * this array can apply to more than one operation. This can be done by specifying in the + * \p prefetchLocIdxs array, the index of the first prefetch operation that the corresponding entry + * in the \p prefetchLocs array applies to. Both \p prefetchLocs and \p prefetchLocIdxs must be of + * the same length as specified by \p numPrefetchLocs. For example, if a batch has 10 prefetches listed + * in dptrs/sizes, the first 4 of which are to be prefetched to one location and the remaining 6 are to be prefetched + * to another, then \p numPrefetchLocs will be 2, \p prefetchLocIdxs will be {0, 4} and \p prefetchLocs + * will contain the two locations. Note the first entry in \p prefetchLocIdxs must always be 0. + * Also, each entry must be greater than the previous entry and the last entry should be less than \p count. + * Furthermore, \p numPrefetchLocs must be lesser than or equal to \p count. + * + * \param dptrs - Array of pointers to be prefetched + * \param sizes - Array of sizes for memory prefetch operations. + * \param count - Size of \p dptrs and \p sizes arrays. + * \param prefetchLocs - Array of locations to prefetch to. + * \param prefetchLocIdxs - Array of indices to specify which operands each entry in the \p prefetchLocs array applies to. + * The locations specified in prefetchLocs[k] will be applied to copies starting from prefetchLocIdxs[k] + * through prefetchLocIdxs[k+1] - 1. Also prefetchLocs[numPrefetchLocs - 1] will apply to prefetches starting from + * prefetchLocIdxs[numPrefetchLocs - 1] through count - 1. + * \param numPrefetchLocs - Size of \p prefetchLocs and \p prefetchLocIdxs arrays. + * \param flags - Flags reserved for future use. Must be zero. + * \param hStream - The stream to enqueue the operations in. Must not be legacy NULL stream. + * + */ +extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchBatchAsync(void **dptrs, size_t *sizes, size_t count, + struct cudaMemLocation *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, + unsigned long long flags, cudaStream_t stream); + +/** + * \brief Performs a batch of memory discards asynchronously + * + * Performs a batch of memory discards. The batch as a whole executes in stream order + * but operations within a batch are not guaranteed to execute in any specific order. + * All devices in the system must have a non-zero value for the device attribute + * ::cudaDevAttrConcurrentManagedAccess otherwise the API will return an error. + * + * Discarding a memory range informs the driver that the contents of that range are no longer useful. + * Discarding memory ranges allows the driver to optimize certain data migrations and can also help + * reduce memory pressure. This operation can be undone on any part of the range by either writing to it + * or prefetching it via ::cudaMemPrefetchAsync or ::cudaMemPrefetchBatchAsync. Reading from a discarded range, + * without a subsequent write or prefetch to that part of the range, will return an indeterminate value. + * Note that any reads, writes or prefetches to any part of the memory range that occur simultaneously with + * the discard operation result in undefined behavior. + * + * Performs memory discard on address ranges specified in \p dptrs and \p sizes. + * Both arrays must be of the same length as specified by \p count. Each memory range + * specified must refer to managed memory allocated via ::cudaMallocManaged or declared + * via __managed__ variables or it may also refer to system-allocated memory when all devices + * have a non-zero value for ::cudaDevAttrPageableMemoryAccess. + * + * \param dptrs - Array of pointers to be discarded + * \param sizes - Array of sizes for memory discard operations. + * \param count - Size of \p dptrs and \p sizes arrays. + * \param flags - Flags reserved for future use. Must be zero. + * \param hStream - The stream to enqueue the operations in. Must not be legacy NULL stream. + * */ -extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchAsync(const void *devPtr, size_t count, int dstDevice, cudaStream_t stream __dv(0)); +extern __host__ cudaError_t CUDARTAPI cudaMemDiscardBatchAsync(void **dptrs, size_t *sizes, size_t count, unsigned long long flags, cudaStream_t stream); + +/** + * \brief Performs a batch of memory discards and prefetches asynchronously + * + * Performs a batch of memory discards followed by prefetches. The batch as a whole executes + * in stream order but operations within a batch are not guaranteed to execute in any specific order. + * All devices in the system must have a non-zero value for the device attribute + * ::cudaDevAttrConcurrentManagedAccess otherwise the API will return an error. + * + * Calling ::cudaMemDiscardAndPrefetchBatchAsync is semantically equivalent to calling + * ::cudaMemDiscardBatchAsync followed by ::cudaMemPrefetchBatchAsync, but is more optimal. + * For more details on what discarding and prefetching imply, please refer to ::cudaMemDiscardBatchAsync + * and ::cudaMemPrefetchBatchAsync respectively. Note that any reads, writes or prefetches to any part + * of the memory range that occur simultaneously with this combined discard+prefetch operation + * result in undefined behavior. + * + * Performs memory discard and prefetch on address ranges specified in \p dptrs and \p sizes. + * Both arrays must be of the same length as specified by \p count. Each memory range specified + * must refer to managed memory allocated via ::cudaMallocManaged or declared via + * __managed__ variables or it may also refer to system-allocated memory when all devices + * have a non-zero value for ::cudaDevAttrPageableMemoryAccess. Every operation in the batch + * has to be associated with a valid location to prefetch the address range to and specified in + * the \p prefetchLocs array. Each entry in this array can apply to more than one operation. + * This can be done by specifying in the \p prefetchLocIdxs array, the index of the first + * operation that the corresponding entry in the \p prefetchLocs array applies to. + * Both \p prefetchLocs and \p prefetchLocIdxs must be of the same length as specified by + * \p numPrefetchLocs. For example, if a batch has 10 operations listed in dptrs/sizes, + * the first 6 of which are to be prefetched to one location and the remaining 4 are to be + * prefetched to another, then \p numPrefetchLocs will be 2, \p prefetchLocIdxs will be {0, 6} + * and \p prefetchLocs will contain the two set of locations. Note the first entry in + * \p prefetchLocIdxs must always be 0. Also, each entry must be greater than the previous + * entry and the last entry should be less than \p count. Furthermore, \p numPrefetchLocs + * must be lesser than or equal to \p count. + * + * \param dptrs - Array of pointers to be discarded + * \param sizes - Array of sizes for memory discard operations. + * \param count - Size of \p dptrs and \p sizes arrays. + * \param prefetchLocs - Array of locations to prefetch to. + * \param prefetchLocIdxs - Array of indices to specify which operands each entry in the \p prefetchLocs array applies to. + * The locations specified in prefetchLocs[k] will be applied to operations starting from prefetchLocIdxs[k] + * through prefetchLocIdxs[k+1] - 1. Also prefetchLocs[numPrefetchLocs - 1] will apply to copies starting from + * prefetchLocIdxs[numPrefetchLocs - 1] through count - 1. + * \param numPrefetchLocs - Size of \p prefetchLocs and \p prefetchLocIdxs arrays. + * \param flags - Flags reserved for future use. Must be zero. + * \param hStream - The stream to enqueue the operations in. Must not be legacy NULL stream. + * + */ +extern __host__ cudaError_t CUDARTAPI cudaMemDiscardAndPrefetchBatchAsync(void **dptrs, size_t *sizes, size_t count, + struct cudaMemLocation *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, + unsigned long long flags, cudaStream_t stream); /** * \brief Advise about the usage of a given memory range @@ -7457,9 +7291,12 @@ extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchAsync(const void *devPtr, s * - ::cudaMemAdviseSetReadMostly: This implies that the data is mostly going to be read * from and only occasionally written to. Any read accesses from any processor to this region will create a * read-only copy of at least the accessed pages in that processor's memory. Additionally, if ::cudaMemPrefetchAsync - * is called on this region, it will create a read-only copy of the data on the destination processor. + * or ::cudaMemPrefetchAsync is called on this region, it will create a read-only copy of the data on the destination processor. + * If the target location for ::cudaMemPrefetchAsync is a host NUMA node and a read-only copy already exists on + * another host NUMA node, that copy will be migrated to the targeted host NUMA node. * If any processor writes to this region, all copies of the corresponding page will be invalidated - * except for the one where the write occurred. The \p device argument is ignored for this advice. + * except for the one where the write occurred. If the writing processor is the CPU and the preferred location of + * the page is a host NUMA node, then the page will also be migrated to that host NUMA node. The \p location argument is ignored for this advice. * Note that for a page to be read-duplicated, the accessing processor must either be the CPU or a GPU * that has a non-zero value for the device attribute ::cudaDevAttrConcurrentManagedAccess. * Also, if a context is created on a device that does not have the device attribute @@ -7471,17 +7308,22 @@ extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchAsync(const void *devPtr, s * device attribute ::cudaDevAttrPageableMemoryAccessUsesHostPageTables, then setting this advice * will not create a read-only copy when that device accesses this memory region. * - * - ::cudaMemAdviceUnsetReadMostly: Undoes the effect of ::cudaMemAdviceReadMostly and also prevents the + * - ::cudaMemAdviceUnsetReadMostly: Undoes the effect of ::cudaMemAdviseSetReadMostly and also prevents the * Unified Memory driver from attempting heuristic read-duplication on the memory range. Any read-duplicated * copies of the data will be collapsed into a single copy. The location for the collapsed * copy will be the preferred location if the page has a preferred location and one of the read-duplicated * copies was resident at that location. Otherwise, the location chosen is arbitrary. + * Note: The \p location argument is ignored for this advice. * * - ::cudaMemAdviseSetPreferredLocation: This advice sets the preferred location for the - * data to be the memory belonging to \p device. Passing in cudaCpuDeviceId for \p device sets the - * preferred location as host memory. If \p device is a GPU, then it must have a non-zero value for the - * device attribute ::cudaDevAttrConcurrentManagedAccess. Setting the preferred location - * does not cause data to migrate to that location immediately. Instead, it guides the migration policy + * data to be the memory belonging to \p location. When ::cudaMemLocation::type is ::cudaMemLocationTypeHost, + * ::cudaMemLocation::id is ignored and the preferred location is set to be host memory. To set the preferred location + * to a specific host NUMA node, applications must set ::cudaMemLocation::type to ::cudaMemLocationTypeHostNuma and + * ::cudaMemLocation::id must specify the NUMA ID of the host NUMA node. If ::cudaMemLocation::type is set to ::cudaMemLocationTypeHostNumaCurrent, + * ::cudaMemLocation::id will be ignored and the host NUMA node closest to the calling thread's CPU will be used as the preferred location. + * If ::cudaMemLocation::type is a ::cudaMemLocationTypeDevice, then ::cudaMemLocation::id must be a valid device ordinal + * and the device must have a non-zero value for the device attribute ::cudaDevAttrConcurrentManagedAccess. + * Setting the preferred location does not cause data to migrate to that location immediately. Instead, it guides the migration policy * when a fault occurs on that memory region. If the data is already in its preferred location and the * faulting processor can establish a mapping without requiring the data to be migrated, then * data migration will be avoided. On the other hand, if the data is not in its preferred location @@ -7494,19 +7336,18 @@ extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchAsync(const void *devPtr, s * if the preferred location is set as device memory, then the page will continue to thrash indefinitely. * If ::cudaMemAdviseSetReadMostly is also set on this memory region or any subset of it, then the * policies associated with that advice will override the policies of this advice, unless read accesses from - * \p device will not result in a read-only copy being created on that device as outlined in description for + * \p location will not result in a read-only copy being created on that procesor as outlined in description for * the advice ::cudaMemAdviseSetReadMostly. - * If the memory region refers to valid system-allocated pageable memory, then \p device must have a non-zero - * value for the device attribute ::cudaDevAttrPageableMemoryAccess. Additionally, if \p device has - * a non-zero value for the device attribute ::cudaDevAttrPageableMemoryAccessUsesHostPageTables, - * then this call has no effect. Note however that this behavior may change in the future. + * If the memory region refers to valid system-allocated pageable memory, and ::cudaMemLocation::type is ::cudaMemLocationTypeDevice + * then ::cudaMemLocation::id must be a valid device that has a non-zero alue for the device attribute ::cudaDevAttrPageableMemoryAccess. * * - ::cudaMemAdviseUnsetPreferredLocation: Undoes the effect of ::cudaMemAdviseSetPreferredLocation - * and changes the preferred location to none. + * and changes the preferred location to none. The \p location argument is ignored for this advice. * - * - ::cudaMemAdviseSetAccessedBy: This advice implies that the data will be accessed by \p device. - * Passing in ::cudaCpuDeviceId for \p device will set the advice for the CPU. If \p device is a GPU, then - * the device attribute ::cudaDevAttrConcurrentManagedAccess must be non-zero. + * - ::cudaMemAdviseSetAccessedBy: This advice implies that the data will be accessed by processor \p location. + * The ::cudaMemLocation::type must be either ::cudaMemLocationTypeDevice with ::cudaMemLocation::id representing a valid device + * ordinal or ::cudaMemLocationTypeHost and ::cudaMemLocation::id will be ignored. All other location types are invalid. + * If ::cudaMemLocation::id is a GPU, then the device attribute ::cudaDevAttrConcurrentManagedAccess must be non-zero. * This advice does not cause data migration and has no impact on the location of the data per se. Instead, * it causes the data to always be mapped in the specified processor's page tables, as long as the * location of the data permits a mapping to be established. If the data gets migrated for any reason, @@ -7518,28 +7359,28 @@ extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchAsync(const void *devPtr, s * migration may be too high. But preventing faults can still help improve performance, and so having * a mapping set up in advance is useful. Note that on CPU access of this data, the data may be migrated * to host memory because the CPU typically cannot access device memory directly. Any GPU that had the - * ::cudaMemAdviceSetAccessedBy flag set for this data will now have its mapping updated to point to the + * ::cudaMemAdviseSetAccessedBy flag set for this data will now have its mapping updated to point to the * page in host memory. * If ::cudaMemAdviseSetReadMostly is also set on this memory region or any subset of it, then the * policies associated with that advice will override the policies of this advice. Additionally, if the - * preferred location of this memory region or any subset of it is also \p device, then the policies - * associated with ::cudaMemAdviseSetPreferredLocation will override the policies of this advice. - * If the memory region refers to valid system-allocated pageable memory, then \p device must have a non-zero - * value for the device attribute ::cudaDevAttrPageableMemoryAccess. Additionally, if \p device has - * a non-zero value for the device attribute ::cudaDevAttrPageableMemoryAccessUsesHostPageTables, + * preferred location of this memory region or any subset of it is also \p location, then the policies + * associated with ::CU_MEM_ADVISE_SET_PREFERRED_LOCATION will override the policies of this advice. + * If the memory region refers to valid system-allocated pageable memory, and ::cudaMemLocation::type is ::cudaMemLocationTypeDevice + * then device in ::cudaMemLocation::id must have a non-zero value for the device attribute ::cudaDevAttrPageableMemoryAccess. + * Additionally, if ::cudaMemLocation::id has a non-zero value for the device attribute ::cudaDevAttrPageableMemoryAccessUsesHostPageTables, * then this call has no effect. * - * - ::cudaMemAdviseUnsetAccessedBy: Undoes the effect of ::cudaMemAdviseSetAccessedBy. Any mappings to - * the data from \p device may be removed at any time causing accesses to result in non-fatal page faults. - * If the memory region refers to valid system-allocated pageable memory, then \p device must have a non-zero - * value for the device attribute ::cudaDevAttrPageableMemoryAccess. Additionally, if \p device has - * a non-zero value for the device attribute ::cudaDevAttrPageableMemoryAccessUsesHostPageTables, + * - ::CU_MEM_ADVISE_UNSET_ACCESSED_BY: Undoes the effect of ::cudaMemAdviseSetAccessedBy. Any mappings to + * the data from \p location may be removed at any time causing accesses to result in non-fatal page faults. + * If the memory region refers to valid system-allocated pageable memory, and ::cudaMemLocation::type is ::cudaMemLocationTypeDevice + * then device in ::cudaMemLocation::id must have a non-zero value for the device attribute ::cudaDevAttrPageableMemoryAccess. + * Additionally, if ::cudaMemLocation::id has a non-zero value for the device attribute ::cudaDevAttrPageableMemoryAccessUsesHostPageTables, * then this call has no effect. * - * \param devPtr - Pointer to memory to set the advice for - * \param count - Size in bytes of the memory range - * \param advice - Advice to be applied for the specified memory range - * \param device - Device to apply the advice for + * \param devPtr - Pointer to memory to set the advice for + * \param count - Size in bytes of the memory range + * \param advice - Advice to be applied for the specified memory range + * \param location - location to apply the advice for * * \return * ::cudaSuccess, @@ -7555,7 +7396,7 @@ extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchAsync(const void *devPtr, s * ::cudaMemcpy3DPeerAsync, ::cudaMemPrefetchAsync, * ::cuMemAdvise */ -extern __host__ cudaError_t CUDARTAPI cudaMemAdvise(const void *devPtr, size_t count, enum cudaMemoryAdvise advice, int device); +extern __host__ cudaError_t CUDARTAPI cudaMemAdvise(const void *devPtr, size_t count, enum cudaMemoryAdvise advice, struct cudaMemLocation location); /** * \brief Query an attribute of a given memory range @@ -7593,6 +7434,29 @@ extern __host__ cudaError_t CUDARTAPI cudaMemAdvise(const void *devPtr, size_t c * prefetched to the same location, cudaInvalidDeviceId will be returned. Note that this simply returns the * last location that the applicaton requested to prefetch the memory range to. It gives no indication as to * whether the prefetch operation to that location has completed or even begun. +* - ::cudaMemRangeAttributePreferredLocationType: If this attribute is specified, \p data will be +* interpreted as a ::cudaMemLocationType, and \p dataSize must be sizeof(cudaMemLocationType). The ::cudaMemLocationType returned will be +* ::cudaMemLocationTypeDevice if all pages in the memory range have the same GPU as their preferred location, or ::cudaMemLocationType +* will be ::cudaMemLocationTypeHost if all pages in the memory range have the CPU as their preferred location, or or it will be ::cudaMemLocationTypeHostNuma +* if all the pages in the memory range have the same host NUMA node ID as their preferred location or it will be ::cudaMemLocationTypeInvalid +* if either all the pages don't have the same preferred location or some of the pages don't have a preferred location at all. +* Note that the actual location type of the pages in the memory range at the time of the query may be different from the preferred location type. +* - ::cudaMemRangeAttributePreferredLocationId: If this attribute is specified, \p data will be +* interpreted as a 32-bit integer, and \p dataSize must be 4. If the ::cudaMemRangeAttributePreferredLocationType query for the same address range +* returns ::cudaMemLocationTypeDevice, it will be a valid device ordinal or if it returns ::cudaMemLocationTypeHostNuma, it will be a valid host NUMA node ID +* or if it returns any other location type, the id should be ignored. +* - ::cudaMemRangeAttributeLastPrefetchLocationType: If this attribute is specified, \p data will be +* interpreted as a ::cudaMemLocationType, and \p dataSize must be sizeof(cudaMemLocationType). The result returned will be the last location type +* to which all pages in the memory range were prefetched explicitly via ::cuMemPrefetchAsync. The ::cudaMemLocationType returned +* will be ::cudaMemLocationTypeDevice if the last prefetch location was the GPU or ::cudaMemLocationTypeHost if it was the CPU or ::cudaMemLocationTypeHostNuma if +* the last prefetch location was a specific host NUMA node. If any page in the memory range was never explicitly prefetched or if all pages were not +* prefetched to the same location, ::CUmemLocationType will be ::cudaMemLocationTypeInvalid. +* Note that this simply returns the last location type that the application requested to prefetch the memory range to. It gives no indication as to +* whether the prefetch operation to that location has completed or even begun. +* - ::cudaMemRangeAttributeLastPrefetchLocationId: If this attribute is specified, \p data will be +* interpreted as a 32-bit integer, and \p dataSize must be 4. If the ::cudaMemRangeAttributeLastPrefetchLocationType query for the same address range +* returns ::cudaMemLocationTypeDevice, it will be a valid device ordinal or if it returns ::cudaMemLocationTypeHostNuma, it will be a valid host NUMA node ID +* or if it returns any other location type, the id should be ignored. * * \param data - A pointers to a memory location where the result * of each attribute query will be written to. @@ -7632,6 +7496,10 @@ extern __host__ cudaError_t CUDARTAPI cudaMemRangeGetAttribute(void *data, size_ * - ::cudaMemRangeAttributePreferredLocation * - ::cudaMemRangeAttributeAccessedBy * - ::cudaMemRangeAttributeLastPrefetchLocation + * - :: cudaMemRangeAttributePreferredLocationType + * - :: cudaMemRangeAttributePreferredLocationId + * - :: cudaMemRangeAttributeLastPrefetchLocationType + * - :: cudaMemRangeAttributeLastPrefetchLocationId * * \param data - A two-dimensional array containing pointers to memory * locations where the result of each attribute query will be written to. @@ -8147,7 +8015,38 @@ extern __host__ cudaError_t CUDARTAPI cudaMemPoolGetAccess(enum cudaMemAccessFla * Creates a CUDA memory pool and returns the handle in \p pool. The \p poolProps determines * the properties of the pool such as the backing device and IPC capabilities. * - * By default, the pool's memory will be accessible from the device it is allocated on. + * To create a memory pool for host memory not targeting a specific NUMA node, applications must set + * set ::cudaMemPoolProps::cudaMemLocation::type to ::cudaMemLocationTypeHost. + * ::cudaMemPoolProps::cudaMemLocation::id is ignored for such pools. + * Pools created with the type ::cudaMemLocationTypeHost are not IPC capable and + * ::cudaMemPoolProps::handleTypes must be 0, any other values will result in + * ::cudaErrorInvalidValue. + * To create a memory pool targeting a specific host NUMA node, applications must + * set ::cudaMemPoolProps::cudaMemLocation::type to ::cudaMemLocationTypeHostNuma and + * ::cudaMemPoolProps::cudaMemLocation::id must specify the NUMA ID of the host memory node. + * Specifying ::cudaMemLocationTypeHostNumaCurrent as the + * ::cudaMemPoolProps::cudaMemLocation::type will result in ::cudaErrorInvalidValue. +* By default, the pool's memory will be accessible from the device it is allocated on. + * In the case of pools created with ::cudaMemLocationTypeHostNuma or + * ::cudaMemLocationTypeHost, their default accessibility will be from the host + * CPU. + * Applications can control the maximum size of the pool by specifying a non-zero value for ::cudaMemPoolProps::maxSize. + * If set to 0, the maximum size of the pool will default to a system dependent value. + * + * Applications that intend to use ::CU_MEM_HANDLE_TYPE_FABRIC based memory sharing must ensure: + * (1) `nvidia-caps-imex-channels` character device is created by the driver and is listed under /proc/devices + * (2) have at least one IMEX channel file accessible by the user launching the application. + * + * When exporter and importer CUDA processes have been granted access to the same IMEX channel, they can securely + * share memory. + * + * The IMEX channel security model works on a per user basis. Which means all processes under a user can share + * memory if the user has access to a valid IMEX channel. When multi-user isolation is desired, a separate IMEX + * channel is required for each user. + * + * These channel files exist in /dev/nvidia-caps-imex-channels/channel* and can be created using standard OS + * native calls like mknod on Linux. For example: To create channel0 with the major number from /proc/devices + * users can execute the following command: `mknod /dev/nvidia-caps-imex-channels/channel0 c 0` * * \note Specifying cudaMemHandleTypeNone creates a memory pool that will not support IPC. * @@ -8183,6 +8082,76 @@ extern __host__ cudaError_t CUDARTAPI cudaMemPoolCreate(cudaMemPool_t *memPool, */ extern __host__ cudaError_t CUDARTAPI cudaMemPoolDestroy(cudaMemPool_t memPool); +/** + * \brief Returns the default memory pool for a given location and allocation type + * + * The memory location can be of one of ::cudaMemLocationTypeDevice, ::cudaMemLocationTypeHost or + * ::cudaMemLocationTypeHostNuma. The allocation type can be one of ::cudaMemAllocationTypePinned or + * ::cudaMemAllocationTypeManaged. When the allocation type is ::cudaMemAllocationTypeManaged, + * the location type can also be ::cudaMemLocationTypeNone to indicate no preferred location + * for the managed memory pool. In all other cases, the call return ::cudaErrorInvalidValue + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + * ::cudaErrorNotSupported, + * \notefnerr + * + * \sa ::cuMemAllocAsync, ::cuMemPoolTrimTo, ::cuMemPoolGetAttribute, ::cuMemPoolSetAttribute, cuMemPoolSetAccess, ::cuMemGetMemPool, ::cuMemPoolCreate + */ +extern __host__ cudaError_t CUDARTAPI cudaMemGetDefaultMemPool(cudaMemPool_t *memPool, struct cudaMemLocation *location, enum cudaMemAllocationType type); + +/** + * \brief Gets the current memory pool for a given memory location and allocation type + * + * The memory location can be of one of ::cudaMemLocationTypeDevice, ::cudaMemLocationTypeHost or + * ::cudaMemLocationTypeHostNuma. The allocation type can be one of ::cudaMemAllocationTypePinned or + * ::cudaMemAllocationTypeManaged. When the allocation type is ::cudaMemAllocationTypeManaged, + * the location type can also be ::cudaMemLocationTypeNone to indicate no preferred location + * for the managed memory pool. In all other cases, the call return ::cudaErrorInvalidValue + * + * Returns the last pool provided to ::cudaMemSetMemPool or ::cudaDeviceSetMemPool for this location and allocation type + * or the location's default memory pool if ::cudaMemSetMemPool or ::cudaDeviceSetMemPool for that allocType and location + * has never been called. + * By default the current mempool of a location is the default mempool for a device that can be obtained via cudaMemGetDefaultMemPool + * Otherwise the returned pool must have been set with ::cudaDeviceSetMemPool. + * + * \returns + * ::cudaSuccess, + * ::cudaErrorInvalidValue + * + * \sa ::cuDeviceGetDefaultMemPool, ::cuMemPoolCreate, ::cuDeviceSetMemPool, ::cuMemSetMemPool + */ +extern __host__ cudaError_t CUDARTAPI cudaMemGetMemPool(cudaMemPool_t *memPool, struct cudaMemLocation *location, enum cudaMemAllocationType type); + +/** + * \brief Sets the current memory pool for a memory location and allocation type + * + * The memory location can be of one of ::cudaMemLocationTypeDevice, ::cudaMemLocationTypeHost or + * ::cudaMemLocationTypeHostNuma. The allocation type can be one of ::cudaMemAllocationTypePinned or + * ::cudaMemAllocationTypeManaged. When the allocation type is ::cudaMemAllocationTypeManaged, + * the location type can also be ::cudaMemLocationTypeNone to indicate no preferred location + * for the managed memory pool. In all other cases, the call return ::cudaErrorInvalidValue + * + * When a memory pool is set as the current memory pool, the location parameter should be the same as the location of the pool. + * If the location type or index don't match, the call returns ::cudaErrorInvalidValue. + * The type of memory pool should also match the parameter allocType. Else the call returns ::cudaErrorInvalidValue. + * By default, a memory location's current memory pool is its default memory pool. + * If the location type is ::cudaMemLocationTypeDevice and the allocation type is ::cudaMemAllocationTypePinned, then + * this API is the equivalent of calling ::cudaDeviceSetMemPool with the location id as the device. + * For further details on the implications, please refer to the documentation for ::cudaDeviceSetMemPool. + * + * \note Use ::cudaMallocFromPoolAsync to specify asynchronous allocations from a device different + * than the one the stream runs on. + * + * \returns + * ::cudaSuccess, + * ::cudaErrorInvalidValue + * + * \sa ::cuDeviceGetDefaultMemPool, ::cuDeviceGetMemPool, ::cuMemGetMemPool, ::cuMemPoolCreate, ::cuMemPoolDestroy, ::cuMemAllocFromPoolAsync + */ +extern __host__ cudaError_t CUDARTAPI cudaMemSetMemPool(struct cudaMemLocation *location, enum cudaMemAllocationType type, cudaMemPool_t memPool); + /** * \brief Allocates memory from a specified pool with stream ordered semantics. * @@ -8926,7 +8895,8 @@ extern __host__ struct cudaChannelFormatDesc CUDARTAPI cudaCreateChannelDesc(int * \p pResViewDesc is an optional argument that specifies an alternate format for * the data described by \p pResDesc, and also describes the subresource region * to restrict access to when texturing. \p pResViewDesc can only be specified if - * the type of resource is a CUDA array or a CUDA mipmapped array. + * the type of resource is a CUDA array or a CUDA mipmapped array not in a block + * compressed format. * * Texture objects are only supported on devices of compute capability 3.0 or higher. * Additionally, a texture object is an opaque value, and, as such, should only be @@ -8984,7 +8954,7 @@ extern __host__ struct cudaChannelFormatDesc CUDARTAPI cudaCreateChannelDesc(int * must be set to a valid device pointer, that is aligned to ::cudaDeviceProp::textureAlignment. * ::cudaResourceDesc::res::linear::desc describes the format and the number of components per array element. ::cudaResourceDesc::res::linear::sizeInBytes * specifies the size of the array in bytes. The total number of elements in the linear address range cannot exceed - * ::cudaDeviceProp::maxTexture1DLinear. The number of elements is computed as (sizeInBytes / sizeof(desc)). + * ::cudaDeviceGetTexture1DLinearMaxWidth(). The number of elements is computed as (sizeInBytes / sizeof(desc)). * * \par * If ::cudaResourceDesc::resType is set to ::cudaResourceTypePitch2D, ::cudaResourceDesc::res::pitch2D::devPtr @@ -9372,6 +9342,126 @@ extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaRuntimeGetVersion(i /** @} */ /* END CUDART__VERSION */ +/** + * \defgroup CUDART_LOGS Error Log Management Functions + * + * ___MANBRIEF___ error log management interface for the CUDA Runtime and Driver + * (___CURRENT_FILE___) ___ENDMANBRIEF___ + * + * This section describes the error log management functions of the CUDA runtime + * application programming interface. The Error Log Management interface will operate + * on both the CUDA Driver and CUDA Runtime. + * + * @{ + */ + +/** + * Type of public error reporting callback functions. + * \param data User parameter provided at registration + * \param logLevel Severity level of the log message + * \param message Error log message being reported + * \param length Length of the message in bytes + */ +typedef void (CUDART_CB *cudaLogsCallback_t)(void *data, cudaLogLevel logLevel, char *message, size_t length); + +/** + * \brief Register a callback function to receive error log messages + * + * \param callbackFunc - The function to register as a callback + * \param userData - A generic pointer to user data. This is passed into the callback function. + * \param callback_out - Optional location to store the callback handle after it is registered + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + */ +extern __host__ cudaError_t CUDARTAPI cudaLogsRegisterCallback(cudaLogsCallback_t callbackFunc, void *userData, cudaLogsCallbackHandle *callback_out); + +/** + * \brief Unregister a log message callback + * + * \param callback - The callback instance to unregister from receiving log messages + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + */ +extern __host__ cudaError_t CUDARTAPI cudaLogsUnregisterCallback(cudaLogsCallbackHandle callback); + +/** + * \brief Sets log iterator to point to the end of log buffer, where the next message would be written. + * + * \param iterator_out - Location to store an iterator to the current tail of the logs + * \param flags - Reserved for future use, must be 0 + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + */ +extern __host__ cudaError_t CUDARTAPI cudaLogsCurrent(cudaLogIterator *iterator_out, unsigned int flags); + +/** + * \brief Dump accumulated driver logs into a file + * + * Logs generated by the driver are stored in an internal buffer and can be copied out using this API. + * This API dumps all driver logs starting from \p iterator into \p pathToFile provided. + * + * \note \p iterator is auto-advancing. Dumping logs will update the value of + * \p iterator to receive the next generated log. + * + * \note The driver reserves limited memory for storing logs. + * The oldest logs may be overwritten and become unrecoverable. An indication will appear in the + * destination outupt if the logs have been truncated. Call dump after each failed API to mitigate this + * risk. + * + * \param iterator - Optional auto-advancing iterator specifying the starting log to read. NULL value dumps all logs. + * \param pathToFile - Path to output file for dumping logs + * \param flags - Reserved for future use, must be 0 + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + */ +extern __host__ cudaError_t CUDARTAPI cudaLogsDumpToFile(cudaLogIterator *iterator, const char *pathToFile, unsigned int flags); + +/** + * \brief Dump accumulated driver logs into a buffer + * + * Logs generated by the driver are stored in an internal buffer and can be copied out using this API. + * This API dumps driver logs from \p iterator into \p buffer up to the size specified in \p *size. + * The driver will always null terminate the buffer but there will not be a null character between log + * entries, only a newline \\n. The driver will then return the actual number of bytes written in + * \p *size, excluding the null terminator. If there are no messages to dump, \p *size will be set to 0 + * and the function will return ::CUDA_SUCCESS. + * If the provided \p buffer is not large enough to hold any messages, \p *size will be set to 0 and + * the function will return ::CUDA_ERROR_INVALID_VALUE. + * + * \note \p iterator is auto-advancing. Dumping logs will update the value of + * \p iterator to receive the next generated log. + * + * \note The driver reserves limited memory for storing logs. The maximum size of the buffer is 25600 bytes. + * The oldest logs may be overwritten and become unrecoverable. An indication will appear in the + * destination outupt if the logs have been truncated. Call dump after each failed API to mitigate this + * risk. + * + * \note If the provided value in \p *size is not large enough to hold all buffered messages, a message will + * be added at the head of the buffer indicating this. The driver then computes the number of messages + * it is able to store in \p buffer and writes it out. The final message in \p buffer will always be + * the most recent log message as of when the API is called. + * + * \param iterator - Optional auto-advancing iterator specifying the starting log to read. NULL value dumps all logs. + * \param buffer - Pointer to dump logs + * \param size - See description + * \param flags - Reserved for future use, must be 0 + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + */ +extern __host__ cudaError_t CUDARTAPI cudaLogsDumpToMemory(cudaLogIterator *iterator, char *buffer, size_t *size, unsigned int flags); + + /** @} */ /* END CUDART_LOGS */ + /** * \defgroup CUDART_GRAPH Graph Management * @@ -9499,8 +9589,10 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphCreate(cudaGraph_t *pGraph, unsig * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa + * ::cudaGraphAddNode, * ::cudaLaunchKernel, * ::cudaGraphKernelNodeGetParams, * ::cudaGraphKernelNodeSetParams, @@ -9564,8 +9656,10 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphKernelNodeGetParams(cudaGraphNode * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa + * ::cudaGraphNodeSetParams, * ::cudaLaunchKernel, * ::cudaGraphAddKernelNode, * ::cudaGraphKernelNodeGetParams @@ -9575,11 +9669,11 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphKernelNodeSetParams(cudaGraphNode /** * \brief Copies attributes from source node to destination node. * - * Copies attributes from source node \p src to destination node \p dst. + * Copies attributes from source node \p hSrc to destination node \p hDst. * Both node must have the same context. * - * \param[out] dst Destination node - * \param[in] src Source node + * \param[out] hDst Destination node + * \param[in] hSrc Source node * For list of attributes see ::cudaKernelNodeAttrID * * \return @@ -9591,8 +9685,8 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphKernelNodeSetParams(cudaGraphNode * ::cudaAccessPolicyWindow */ extern __host__ cudaError_t CUDARTAPI cudaGraphKernelNodeCopyAttributes( - cudaGraphNode_t hSrc, - cudaGraphNode_t hDst); + cudaGraphNode_t hDst, + cudaGraphNode_t hSrc); /** * \brief Queries node attribute. @@ -9673,6 +9767,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphKernelNodeSetAttribute( * \note_callback * * \sa + * ::cudaGraphAddNode, * ::cudaMemcpy3D, * ::cudaGraphAddMemcpyNodeToSymbol, * ::cudaGraphAddMemcpyNodeFromSymbol, @@ -9934,6 +10029,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphMemcpyNodeGetParams(cudaGraphNode * \note_callback * * \sa + * ::cudaGraphNodeSetParams, * ::cudaMemcpy3D, * ::cudaGraphMemcpyNodeSetParamsToSymbol, * ::cudaGraphMemcpyNodeSetParamsFromSymbol, @@ -10108,6 +10204,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphMemcpyNodeSetParams(cudaGraphNode * \note_callback * * \sa + * ::cudaGraphAddNode, * ::cudaMemset2D, * ::cudaGraphMemsetNodeGetParams, * ::cudaGraphMemsetNodeSetParams, @@ -10161,6 +10258,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphMemsetNodeGetParams(cudaGraphNode * \note_callback * * \sa + * ::cudaGraphNodeSetParams, * ::cudaMemset2D, * ::cudaGraphAddMemsetNode, * ::cudaGraphMemsetNodeGetParams @@ -10195,6 +10293,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphMemsetNodeSetParams(cudaGraphNode * \note_callback * * \sa + * ::cudaGraphAddNode, * ::cudaLaunchHostFunc, * ::cudaGraphHostNodeGetParams, * ::cudaGraphHostNodeSetParams, @@ -10248,6 +10347,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphHostNodeGetParams(cudaGraphNode_t * \note_callback * * \sa + * ::cudaGraphNodeSetParams, * ::cudaLaunchHostFunc, * ::cudaGraphAddHostNode, * ::cudaGraphHostNodeGetParams @@ -10263,7 +10363,8 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphHostNodeSetParams(cudaGraphNode_t * at the root of the graph. \p pDependencies may not have any duplicate entries. * A handle to the new node will be returned in \p pGraphNode. * - * If \p hGraph contains allocation or free nodes, this call will return an error. + * If \p childGraph contains allocation nodes, free nodes, or conditional nodes, this call will + * return an error. * * The node executes an embedded child graph. The child graph is cloned in this call. * @@ -10282,6 +10383,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphHostNodeSetParams(cudaGraphNode_t * \note_callback * * \sa + * ::cudaGraphAddNode, * ::cudaGraphChildGraphNodeGetGraph, * ::cudaGraphCreate, * ::cudaGraphDestroyNode, @@ -10348,6 +10450,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphChildGraphNodeGetGraph(cudaGraphN * \note_callback * * \sa + * ::cudaGraphAddNode, * ::cudaGraphCreate, * ::cudaGraphDestroyNode, * ::cudaGraphAddChildGraphNode, @@ -10387,6 +10490,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphAddEmptyNode(cudaGraphNode_t *pGr * \note_callback * * \sa + * ::cudaGraphAddNode, * ::cudaGraphAddEventWaitNode, * ::cudaEventRecordWithFlags, * ::cudaStreamWaitEvent, @@ -10446,6 +10550,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphAddEmptyNode(cudaGraphNode_t *pGr * \note_callback * * \sa + * ::cudaGraphNodeSetParams, * ::cudaGraphAddEventRecordNode, * ::cudaGraphEventRecordNodeGetEvent, * ::cudaGraphEventWaitNodeSetEvent, @@ -10487,6 +10592,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphAddEmptyNode(cudaGraphNode_t *pGr * \note_callback * * \sa + * ::cudaGraphAddNode, * ::cudaGraphAddEventRecordNode, * ::cudaEventRecordWithFlags, * ::cudaStreamWaitEvent, @@ -10546,6 +10652,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphAddEmptyNode(cudaGraphNode_t *pGr * \note_callback * * \sa + * ::cudaGraphNodeSetParams, * ::cudaGraphAddEventWaitNode, * ::cudaGraphEventWaitNodeGetEvent, * ::cudaGraphEventRecordNodeSetEvent, @@ -10584,6 +10691,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphAddEmptyNode(cudaGraphNode_t *pGr * \note_callback * * \sa + * ::cudaGraphAddNode, * ::cudaGraphExternalSemaphoresSignalNodeGetParams, * ::cudaGraphExternalSemaphoresSignalNodeSetParams, * ::cudaGraphExecExternalSemaphoresSignalNodeSetParams, @@ -10655,6 +10763,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExternalSemaphoresSignalNodeGetPa * \note_callback * * \sa + * ::cudaGraphNodeSetParams, * ::cudaGraphAddExternalSemaphoresSignalNode, * ::cudaGraphExternalSemaphoresSignalNodeSetParams, * ::cudaGraphAddExternalSemaphoresWaitNode, @@ -10693,6 +10802,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExternalSemaphoresSignalNodeSetPa * \note_callback * * \sa + * ::cudaGraphAddNode, * ::cudaGraphExternalSemaphoresWaitNodeGetParams, * ::cudaGraphExternalSemaphoresWaitNodeSetParams, * ::cudaGraphExecExternalSemaphoresWaitNodeSetParams, @@ -10764,6 +10874,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExternalSemaphoresWaitNodeGetPara * \note_callback * * \sa + * ::cudaGraphNodeSetParams, * ::cudaGraphAddExternalSemaphoresWaitNode, * ::cudaGraphExternalSemaphoresWaitNodeSetParams, * ::cudaGraphAddExternalSemaphoresWaitNode, @@ -10813,7 +10924,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExternalSemaphoresWaitNodeSetPara * * The following restrictions apply to graphs which contain allocation and/or memory free nodes: * - Nodes and edges of the graph cannot be deleted. - * - The graph cannot be used in a child node. + * - The graph can only be used in a child node if the ownership is moved to the parent. * - Only one instantiation of the graph may exist at any point in time. * - The graph cannot be cloned. * @@ -10828,6 +10939,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExternalSemaphoresWaitNodeSetPara * \notefnerr * * \sa + * ::cudaGraphAddNode, * ::cudaGraphAddMemFreeNode, * ::cudaGraphMemAllocNodeGetParams, * ::cudaDeviceGraphMemTrim, @@ -10900,7 +11012,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphMemAllocNodeGetParams(cudaGraphNo * * The following restrictions apply to graphs which contain allocation and/or memory free nodes: * - Nodes and edges of the graph cannot be deleted. - * - The graph cannot be used in a child node. + * - The graph can only be used in a child node if the ownership is moved to the parent. * - Only one instantiation of the graph may exist at any point in time. * - The graph cannot be cloned. * @@ -10915,6 +11027,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphMemAllocNodeGetParams(cudaGraphNo * \notefnerr * * \sa + * ::cudaGraphAddNode, * ::cudaGraphAddMemAllocNode, * ::cudaGraphMemFreeNodeGetParams, * ::cudaDeviceGraphMemTrim, @@ -11070,6 +11183,9 @@ extern __host__ cudaError_t CUDARTAPI cudaDeviceSetGraphMemAttribute(int device, * * Child graph nodes in the original graph are recursively copied into the clone. * + * \note: Cloning is not supported for graphs which contain memory allocation nodes, + * memory free nodes, or conditional nodes. + * * \param pGraphClone - Returns newly created cloned graph * \param originalGraph - Graph to clone * @@ -11213,20 +11329,26 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphGetRootNodes(cudaGraph_t graph, c * \brief Returns a graph's dependency edges * * Returns a list of \p graph's dependency edges. Edges are returned via corresponding - * indices in \p from and \p to; that is, the node in \p to[i] has a dependency on the - * node in \p from[i]. \p from and \p to may both be NULL, in which - * case this function only returns the number of edges in \p numEdges. Otherwise, - * \p numEdges entries will be filled in. If \p numEdges is higher than the actual - * number of edges, the remaining entries in \p from and \p to will be set to NULL, and - * the number of edges actually returned will be written to \p numEdges. + * indices in \p from, \p to and \p edgeData; that is, the node in \p to[i] has a + * dependency on the node in \p from[i] with data \p edgeData[i]. \p from and \p to may + * both be NULL, in which case this function only returns the number of edges in + * \p numEdges. Otherwise, \p numEdges entries will be filled in. If \p numEdges is higher + * than the actual number of edges, the remaining entries in \p from and \p to will be + * set to NULL, and the number of edges actually returned will be written to \p numEdges. + * \p edgeData may alone be NULL, in which case the edges must all have default (zeroed) + * edge data. Attempting a losst query via NULL \p edgeData will result in + * ::cudaErrorLossyQuery. If \p edgeData is non-NULL then \p from and \p to must be as + * well. * * \param graph - Graph to get the edges from * \param from - Location to return edge endpoints * \param to - Location to return edge endpoints + * \param edgeData - Optional location to return edge data * \param numEdges - See description * * \return * ::cudaSuccess, + * ::cudaErrorLossyQuery, * ::cudaErrorInvalidValue * \note_graph_thread_safety * \notefnerr @@ -11241,7 +11363,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphGetRootNodes(cudaGraph_t graph, c * ::cudaGraphNodeGetDependencies, * ::cudaGraphNodeGetDependentNodes */ -extern __host__ cudaError_t CUDARTAPI cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t *from, cudaGraphNode_t *to, size_t *numEdges); +extern __host__ cudaError_t CUDARTAPI cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t *from, cudaGraphNode_t *to, cudaGraphEdgeData *edgeData, size_t *numEdges); /** * \brief Returns a node's dependencies @@ -11252,12 +11374,18 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphGetEdges(cudaGraph_t graph, cudaG * number of dependencies, the remaining entries in \p pDependencies will be set to NULL, and the * number of nodes actually obtained will be returned in \p pNumDependencies. * - * \param node - Node to query + * Note that if an edge has non-zero (non-default) edge data and \p edgeData is NULL, + * this API will return ::cudaErrorLossyQuery. If \p edgeData is non-NULL, then + * \p pDependencies must be as well. + * + * \param node - Node to query * \param pDependencies - Pointer to return the dependencies + * \param edgeData - Optional array to return edge data for each dependency * \param pNumDependencies - See description * * \return * ::cudaSuccess, + * ::cudaErrorLossyQuery, * ::cudaErrorInvalidValue * \note_graph_thread_safety * \notefnerr @@ -11272,7 +11400,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphGetEdges(cudaGraph_t graph, cudaG * ::cudaGraphAddDependencies, * ::cudaGraphRemoveDependencies */ -extern __host__ cudaError_t CUDARTAPI cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t *pDependencies, size_t *pNumDependencies); +extern __host__ cudaError_t CUDARTAPI cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t *pDependencies, cudaGraphEdgeData *edgeData, size_t *pNumDependencies); /** * \brief Returns a node's dependent nodes @@ -11284,12 +11412,18 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphNodeGetDependencies(cudaGraphNode * \p pDependentNodes will be set to NULL, and the number of nodes actually obtained will * be returned in \p pNumDependentNodes. * - * \param node - Node to query + * Note that if an edge has non-zero (non-default) edge data and \p edgeData is NULL, + * this API will return ::cudaErrorLossyQuery. If \p edgeData is non-NULL, then + * \p pDependentNodes must be as well. + * + * \param node - Node to query * \param pDependentNodes - Pointer to return the dependent nodes + * \param edgeData - Optional pointer to return edge data for dependent nodes * \param pNumDependentNodes - See description * * \return * ::cudaSuccess, + * ::cudaErrorLossyQuery, * ::cudaErrorInvalidValue * \note_graph_thread_safety * \notefnerr @@ -11304,7 +11438,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphNodeGetDependencies(cudaGraphNode * ::cudaGraphAddDependencies, * ::cudaGraphRemoveDependencies */ -extern __host__ cudaError_t CUDARTAPI cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t *pDependentNodes, size_t *pNumDependentNodes); +extern __host__ cudaError_t CUDARTAPI cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t *pDependentNodes, cudaGraphEdgeData *edgeData, size_t *pNumDependentNodes); /** * \brief Adds dependency edges to a graph. @@ -11319,6 +11453,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphNodeGetDependentNodes(cudaGraphNo * \param graph - Graph to which dependencies are added * \param from - Array of nodes that provide the dependencies * \param to - Array of dependent nodes + * \param edgeData - Optional array of edge data. If NULL, default (zeroed) edge data is assumed. * \param numDependencies - Number of dependencies to be added * * \return @@ -11335,7 +11470,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphNodeGetDependentNodes(cudaGraphNo * ::cudaGraphNodeGetDependencies, * ::cudaGraphNodeGetDependentNodes */ -extern __host__ cudaError_t CUDARTAPI cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t *from, const cudaGraphNode_t *to, size_t numDependencies); +extern __host__ cudaError_t CUDARTAPI cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t *from, const cudaGraphNode_t *to, const cudaGraphEdgeData *edgeData, size_t numDependencies); /** * \brief Removes dependency edges from a graph. @@ -11345,11 +11480,15 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphAddDependencies(cudaGraph_t graph * Each node in \p pFrom and \p pTo must belong to \p graph. * * If \p numDependencies is 0, elements in \p pFrom and \p pTo will be ignored. - * Specifying a non-existing dependency will return an error. + * Specifying an edge that does not exist in the graph, with data matching + * \p edgeData, results in an error. \p edgeData is nullable, which is equivalent + * to passing default (zeroed) data for each edge. * * \param graph - Graph from which to remove dependencies * \param from - Array of nodes that provide the dependencies * \param to - Array of dependent nodes + * \param edgeData - Optional array of edge data. If NULL, edge data is assumed to + * be default (zeroed). * \param numDependencies - Number of dependencies to be removed * * \return @@ -11366,7 +11505,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphAddDependencies(cudaGraph_t graph * ::cudaGraphNodeGetDependencies, * ::cudaGraphNodeGetDependentNodes */ -extern __host__ cudaError_t CUDARTAPI cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t *from, const cudaGraphNode_t *to, size_t numDependencies); +extern __host__ cudaError_t CUDARTAPI cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t *from, const cudaGraphNode_t *to, const cudaGraphEdgeData *edgeData, size_t numDependencies); /** * \brief Remove a node from the graph @@ -11427,14 +11566,23 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphDestroyNode(cudaGraphNode_t node) * executable graph in existence for that graph at a time. An attempt to * instantiate a second executable graph before destroying the first with * ::cudaGraphExecDestroy will result in an error. + * The same also applies if \p graph contains any device-updatable kernel nodes. * * Graphs instantiated for launch on the device have additional restrictions which do not * apply to host graphs: * * - The graph's nodes must reside on a single device. - * - * - The graph can only contain kernel nodes. Furthermore, use of CUDA Dynamic Parallelism - * is not permitted. Cooperative launches are permitted as long as MPS is not in use. + * - The graph can only contain kernel nodes, memcpy nodes, memset nodes, and child graph nodes. + * - The graph cannot be empty and must contain at least one kernel, memcpy, or memset node. + * Operation-specific restrictions are outlined below. + * - Kernel nodes: + * - Use of CUDA Dynamic Parallelism is not permitted. + * - Cooperative launches are permitted as long as MPS is not in use. + * - Memcpy nodes: + * - Only copies involving device memory and/or pinned device-mapped host memory are permitted. + * - Copies involving CUDA arrays are not permitted. + * - Both operands must be accessible from the current device, and the current device must + * match the device of other nodes in the graph. * * If \p graph is not instantiated for launch on the device but contains kernels which * call device-side cudaGraphLaunch() from multiple devices, this will result in an error. @@ -11490,6 +11638,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphInstantiate(cudaGraphExec_t *pGra * executable graph in existence for that graph at a time. An attempt to * instantiate a second executable graph before destroying the first with * ::cudaGraphExecDestroy will result in an error. + * The same also applies if \p graph contains any device-updatable kernel nodes. * * If \p graph contains kernels which call device-side cudaGraphLaunch() from multiple * devices, this will result in an error. @@ -11499,6 +11648,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphInstantiate(cudaGraphExec_t *pGra * * - The graph's nodes must reside on a single device. * - The graph can only contain kernel nodes, memcpy nodes, memset nodes, and child graph nodes. + * - The graph cannot be empty and must contain at least one kernel, memcpy, or memset node. * Operation-specific restrictions are outlined below. * - Kernel nodes: * - Use of CUDA Dynamic Parallelism is not permitted. @@ -11578,6 +11728,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphInstantiateWithFlags(cudaGraphExe * executable graph in existence for that graph at a time. An attempt to instantiate a * second executable graph before destroying the first with ::cudaGraphExecDestroy will * result in an error. + * The same also applies if \p graph contains any device-updatable kernel nodes. * * If \p graph contains kernels which call device-side cudaGraphLaunch() from multiple * devices, this will result in an error. @@ -11587,6 +11738,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphInstantiateWithFlags(cudaGraphExe * * - The graph's nodes must reside on a single device. * - The graph can only contain kernel nodes, memcpy nodes, memset nodes, and child graph nodes. + * - The graph cannot be empty and must contain at least one kernel, memcpy, or memset node. * Operation-specific restrictions are outlined below. * - Kernel nodes: * - Use of CUDA Dynamic Parallelism is not permitted. @@ -11668,12 +11820,14 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecGetFlags(cudaGraphExec_t grap * The node is identified by the corresponding node \p node in the * non-executable graph, from which the executable graph was instantiated. * - * \p hNode must not have been removed from the original graph. All \p nodeParams + * \p node must not have been removed from the original graph. All \p nodeParams * fields may change, but the following restrictions apply to \p func updates: * * - The owning device of the function cannot change. * - A node whose function originally did not use CUDA dynamic parallelism cannot be updated * to a function which uses CDP + * - A node whose function originally did not make device-side update calls cannot be updated + * to a function which makes device-side update calls. * - If \p hGraphExec was not instantiated for device launch, a node whose function originally * did not use device-side cudaGraphLaunch() cannot be updated to a function which uses * device-side cudaGraphLaunch() unless the node resides on the same device as nodes which @@ -11684,6 +11838,11 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecGetFlags(cudaGraphExec_t grap * enqueued or running launches of \p hGraphExec are not affected by this call. * \p node is also not modified by this call. * + * If \p node is a device-updatable kernel node, the next upload/launch of \p hGraphExec + * will overwrite any previous device-side updates. Additionally, applying host updates to a + * device-updatable kernel node while it is being updated from the device will result in + * undefined behavior. + * * \param hGraphExec - The executable graph in which to set the specified node * \param node - kernel node from the graph from which graphExec was instantiated * \param pNodeParams - Updated Parameters to set @@ -11695,8 +11854,10 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecGetFlags(cudaGraphExec_t grap * \notefnerr * \note_init_rt * \note_callback + * \note_cudaKernel_t * * \sa + * ::cudaGraphExecNodeSetParams, * ::cudaGraphAddKernelNode, * ::cudaGraphKernelNodeSetParams, * ::cudaGraphExecMemcpyNodeSetParams, @@ -11744,6 +11905,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecKernelNodeSetParams(cudaGraph * \note_callback * * \sa + * ::cudaGraphExecNodeSetParams, * ::cudaGraphAddMemcpyNode, * ::cudaGraphMemcpyNodeSetParams, * ::cudaGraphExecMemcpyNodeSetParamsToSymbol, @@ -11955,18 +12117,22 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecMemcpyNodeSetParams(cudaGraph * contained \p pNodeParams at instantiation. \p node must remain in the graph which was * used to instantiate \p hGraphExec. Changed edges to and from \p node are ignored. * - * The destination memory in \p pNodeParams must be allocated from the same - * context as the original destination memory. Both the instantiation-time - * memory operand and the memory operand in \p pNodeParams must be 1-dimensional. - * Zero-length operations are not supported. + * Zero sized operations are not supported. * + * The new destination pointer in \p pNodeParams must be to the same kind of allocation + * as the original destination pointer and have the same context association and device mapping + * as the original destination pointer. + * + * Both the value and pointer address may be updated. + * Changing other aspects of the memset (width, height, element size or pitch) may cause the update to be rejected. + * Specifically, for 2d memsets, all dimension changes are rejected. + * For 1d memsets, changes in height are explicitly rejected and other changes are opportunistically allowed + * if the resulting work maps onto the work resources already allocated for the node. + * The modifications only affect future launches of \p hGraphExec. Already enqueued * or running launches of \p hGraphExec are not affected by this call. \p node is also * not modified by this call. * - * Returns cudaErrorInvalidValue if the memory operand's mappings changed or - * either the original or new memory operand are multidimensional. - * * \param hGraphExec - The executable graph in which to set the specified node * \param node - Memset node from the graph which was used to instantiate graphExec * \param pNodeParams - Updated Parameters to set @@ -11980,6 +12146,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecMemcpyNodeSetParams(cudaGraph * \note_callback * * \sa + * ::cudaGraphExecNodeSetParams, * ::cudaGraphAddMemsetNode, * ::cudaGraphMemsetNodeSetParams, * ::cudaGraphExecKernelNodeSetParams, @@ -12019,6 +12186,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecMemsetNodeSetParams(cudaGraph * \note_callback * * \sa + * ::cudaGraphExecNodeSetParams, * ::cudaGraphAddHostNode, * ::cudaGraphHostNodeSetParams, * ::cudaGraphExecKernelNodeSetParams, @@ -12064,6 +12232,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecHostNodeSetParams(cudaGraphEx * \note_callback * * \sa + * ::cudaGraphExecNodeSetParams, * ::cudaGraphAddChildGraphNode, * ::cudaGraphChildGraphNodeGetGraph, * ::cudaGraphExecKernelNodeSetParams, @@ -12105,6 +12274,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecHostNodeSetParams(cudaGraphEx * \note_callback * * \sa + * ::cudaGraphExecNodeSetParams, * ::cudaGraphAddEventRecordNode, * ::cudaGraphEventRecordNodeGetEvent, * ::cudaGraphEventWaitNodeSetEvent, @@ -12149,6 +12319,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecHostNodeSetParams(cudaGraphEx * \note_callback * * \sa + * ::cudaGraphExecNodeSetParams, * ::cudaGraphAddEventWaitNode, * ::cudaGraphEventWaitNodeGetEvent, * ::cudaGraphEventRecordNodeSetEvent, @@ -12197,6 +12368,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecHostNodeSetParams(cudaGraphEx * \note_callback * * \sa + * ::cudaGraphExecNodeSetParams, * ::cudaGraphAddExternalSemaphoresSignalNode, * ::cudaImportExternalSemaphore, * ::cudaSignalExternalSemaphoresAsync, @@ -12244,6 +12416,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphExecExternalSemaphoresSignalNodeS * \note_callback * * \sa + * ::cudaGraphExecNodeSetParams, * ::cudaGraphAddExternalSemaphoresWaitNode, * ::cudaImportExternalSemaphore, * ::cudaSignalExternalSemaphoresAsync, @@ -12349,6 +12522,8 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphNodeGetEnabled(cudaGraphExec_t hG * - The owning context of the function cannot change. * - A node whose function originally did not use CUDA dynamic parallelism cannot be updated * to a function which uses CDP. + * - A node whose function originally did not make device-side update calls cannot be updated + * to a function which makes device-side update calls. * - A cooperative node cannot be updated to a non-cooperative node, and vice-versa. * - If the graph was instantiated with cudaGraphInstantiateFlagUseNodePriority, the * priority attribute cannot change. Equality is checked on the originally requested @@ -12358,14 +12533,21 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphNodeGetEnabled(cudaGraphExec_t hG * device-side cudaGraphLaunch() unless the node resides on the same device as nodes which * contained such calls at instantiate-time. If no such calls were present at instantiation, * these updates cannot be performed at all. + * - Neither \p hGraph nor \p hGraphExec may contain device-updatable kernel nodes. * - Memset and memcpy nodes: * - The CUDA device(s) to which the operand(s) was allocated/mapped cannot change. * - The source/destination memory must be allocated from the same contexts as the original * source/destination memory. - * - Only 1D memsets can be changed. + * - For 2d memsets, only address and assigned value may be updated. + * - For 1d memsets, updating dimensions is also allowed, but may fail if the resulting operation doesn't + * map onto the work resources already allocated for the node. * - Additional memcpy node restrictions: * - Changing either the source or destination memory type(i.e. CU_MEMORYTYPE_DEVICE, * CU_MEMORYTYPE_ARRAY, etc.) is not supported. + * - Conditional nodes: + * - Changing node parameters is not supported. + * - Changing parameters of nodes within the conditional body graph is subject to the rules above. + * - Conditional handle flags and default values are updated as part of the graph update. * * Note: The API may add further restrictions in future releases. The return code should always be checked. * @@ -12681,10 +12863,159 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphRetainUserObject(cudaGraph_t grap */ extern __host__ cudaError_t CUDARTAPI cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count __dv(1)); -/** @} */ /* END CUDART_GRAPH */ - /** - * \defgroup CUDART_DRIVER_ENTRY_POINT Driver Entry Point Access + * \brief Adds a node of arbitrary type to a graph + * + * Creates a new node in \p graph described by \p nodeParams with \p numDependencies + * dependencies specified via \p pDependencies. \p numDependencies may be 0. + * \p pDependencies may be null if \p numDependencies is 0. \p pDependencies may not have + * any duplicate entries. + * + * \p nodeParams is a tagged union. The node type should be specified in the \p type field, + * and type-specific parameters in the corresponding union member. All unused bytes - that + * is, \p reserved0 and all bytes past the utilized union member - must be set to zero. + * It is recommended to use brace initialization or memset to ensure all bytes are + * initialized. + * + * Note that for some node types, \p nodeParams may contain "out parameters" which are + * modified during the call, such as \p nodeParams->alloc.dptr. + * + * A handle to the new node will be returned in \p phGraphNode. + * + * \param pGraphNode - Returns newly created node + * \param graph - Graph to which to add the node + * \param pDependencies - Dependencies of the node + * \param dependencyData - Optional edge data for the dependencies. If NULL, the data is + * assumed to be default (zeroed) for all dependencies. + * \param numDependencies - Number of dependencies + * \param nodeParams - Specification of the node + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidDeviceFunction, + * ::cudaErrorNotSupported + * \note_graph_thread_safety + * \notefnerr + * \note_init_rt + * \note_callback + * + * \sa + * ::cudaGraphCreate, + * ::cudaGraphNodeSetParams, + * ::cudaGraphExecNodeSetParams + */ +extern __host__ cudaError_t CUDARTAPI cudaGraphAddNode(cudaGraphNode_t *pGraphNode, cudaGraph_t graph, const cudaGraphNode_t *pDependencies, const cudaGraphEdgeData *dependencyData, size_t numDependencies, struct cudaGraphNodeParams *nodeParams); + +/** + * \brief Update's a graph node's parameters + * + * Sets the parameters of graph node \p node to \p nodeParams. The node type specified by + * \p nodeParams->type must match the type of \p node. \p nodeParams must be fully + * initialized and all unused bytes (reserved, padding) zeroed. + * + * Modifying parameters is not supported for node types cudaGraphNodeTypeMemAlloc and + * cudaGraphNodeTypeMemFree. + * + * \param node - Node to set the parameters for + * \param nodeParams - Parameters to copy + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidDeviceFunction, + * ::cudaErrorNotSupported + * \note_graph_thread_safety + * \notefnerr + * \note_init_rt + * \note_callback + * + * \sa + * ::cudaGraphAddNode, + * ::cudaGraphExecNodeSetParams + */ +extern __host__ cudaError_t CUDARTAPI cudaGraphNodeSetParams(cudaGraphNode_t node, struct cudaGraphNodeParams *nodeParams); + +/** + * \brief Update's a graph node's parameters in an instantiated graph + * + * Sets the parameters of a node in an executable graph \p graphExec. The node is identified + * by the corresponding node \p node in the non-executable graph from which the executable + * graph was instantiated. \p node must not have been removed from the original graph. + * + * The modifications only affect future launches of \p graphExec. Already + * enqueued or running launches of \p graphExec are not affected by this call. + * \p node is also not modified by this call. + * + * Allowed changes to parameters on executable graphs are as follows: + * + *
    Node typeAllowed changes + *
    kernelSee ::cudaGraphExecKernelNodeSetParams + *
    memcpyAddresses for 1-dimensional copies if allocated in same context; see ::cudaGraphExecMemcpyNodeSetParams + *
    memsetAddresses for 1-dimensional memsets if allocated in same context; see ::cudaGraphExecMemsetNodeSetParams + *
    hostUnrestricted + *
    child graphTopology must match and restrictions apply recursively; see ::cudaGraphExecUpdate + *
    event waitUnrestricted + *
    event recordUnrestricted + *
    external semaphore signalNumber of semaphore operations cannot change + *
    external semaphore waitNumber of semaphore operations cannot change + *
    memory allocationAPI unsupported + *
    memory freeAPI unsupported + *
    + * + * \param graphExec - The executable graph in which to update the specified node + * \param node - Corresponding node from the graph from which graphExec was instantiated + * \param nodeParams - Updated Parameters to set + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidDeviceFunction, + * ::cudaErrorNotSupported + * \note_graph_thread_safety + * \notefnerr + * \note_init_rt + * \note_callback + * + * \sa + * ::cudaGraphAddNode, + * ::cudaGraphNodeSetParams + * ::cudaGraphExecUpdate, + * ::cudaGraphInstantiate + */ +extern __host__ cudaError_t CUDARTAPI cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, struct cudaGraphNodeParams *nodeParams); + +/** + * \brief Create a conditional handle + * + * Creates a conditional handle associated with \p hGraph. + * + * The conditional handle must be associated with a conditional node in this graph or one of its children. + * + * Handles not associated with a conditional node may cause graph instantiation to fail. + * + * \param pHandle_out - Pointer used to return the handle to the caller. + * \param hGraph - Graph which will contain the conditional node using this handle. + * \param defaultLaunchValue - Optional initial value for the conditional variable. + * Applied at the beginning of each graph execution if cudaGraphCondAssignDefault is set in \p flags. + * \param flags - Currently must be cudaGraphCondAssignDefault or 0. + * + * \return + * ::CUDA_SUCCESS, + * ::CUDA_ERROR_INVALID_VALUE, + * ::CUDA_ERROR_NOT_SUPPORTED + * \note_graph_thread_safety + * \notefnerr + * + * \sa + * ::cuGraphAddNode, + */ +extern __host__ cudaError_t CUDARTAPI cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle *pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue __dv(0), unsigned int flags __dv(0)); + +/** @} */ /* END CUDART_GRAPH */ + +/** + * \defgroup CUDART_DRIVER_ENTRY_POINT Driver Entry Point Access * * ___MANBRIEF___ driver entry point access functions of the CUDA runtime API * (___CURRENT_FILE___) ___ENDMANBRIEF___ @@ -12698,6 +13029,9 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphReleaseUserObject(cudaGraph_t gra /** * \brief Returns the requested driver API function pointer * + * \deprecated This function is deprecated as of CUDA 13.0 + * + * * Returns in \p **funcPtr the address of the CUDA driver function for the requested flags. * * For a requested driver symbol, if the CUDA version in which the driver symbol was @@ -12709,6 +13043,9 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphReleaseUserObject(cudaGraph_t gra * typedef can be picked up from the corresponding typedefs header file. For example, * cudaTypedefs.h consists of function pointer typedefs for driver APIs defined in cuda.h. * + * The API will return ::cudaSuccess and set the returned \p funcPtr if the + * requested driver function is valid and supported on the platform. + * * The API will return ::cudaSuccess and set the returned \p funcPtr to NULL if the * requested driver function is not supported on the platform, no ABI * compatible driver function exists for the CUDA runtime version or if the @@ -12745,6 +13082,10 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphReleaseUserObject(cudaGraph_t gra * the driver. See ::cudaDriverEntryPointQueryResult for * possible values. * + * \note This API is deprecated and ::cudaGetDriverEntryPointByVersion (with a hardcoded + * ::cudaVersion) should be used instead. + * + * * \return * ::cudaSuccess, * ::cudaErrorInvalidValue, @@ -12757,13 +13098,475 @@ extern __host__ cudaError_t CUDARTAPI cudaGraphReleaseUserObject(cudaGraph_t gra * ::cuGetProcAddress */ #if defined(__cplusplus) -extern __host__ cudaError_t CUDARTAPI cudaGetDriverEntryPoint(const char *symbol, void **funcPtr, unsigned long long flags, enum cudaDriverEntryPointQueryResult *driverStatus = NULL); +extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaGetDriverEntryPoint(const char *symbol, void **funcPtr, unsigned long long flags, enum cudaDriverEntryPointQueryResult *driverStatus = NULL); #else -extern __host__ cudaError_t CUDARTAPI cudaGetDriverEntryPoint(const char *symbol, void **funcPtr, unsigned long long flags, enum cudaDriverEntryPointQueryResult *driverStatus); +extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaGetDriverEntryPoint(const char *symbol, void **funcPtr, unsigned long long flags, enum cudaDriverEntryPointQueryResult *driverStatus); +#endif + +/** + * \brief Returns the requested driver API function pointer by CUDA version + * + * Returns in \p **funcPtr the address of the CUDA driver function for the requested flags and CUDA driver version. + * + * The CUDA version is specified as (1000 * major + 10 * minor), so CUDA 11.2 + * should be specified as 11020. For a requested driver symbol, if the specified + * CUDA version is greater than or equal to the CUDA version in which the driver symbol + * was introduced, this API will return the function pointer to the corresponding + * versioned function. If the specified CUDA version is greater than the driver + * version, the API will return ::cudaErrorInvalidValue. + * + * The pointer returned by the API should be cast to a function pointer matching the + * requested driver function's definition in the API header file. The function pointer + * typedef can be picked up from the corresponding typedefs header file. For example, + * cudaTypedefs.h consists of function pointer typedefs for driver APIs defined in cuda.h. + * + * For the case where the CUDA version requested is greater than the CUDA Toolkit + * installed, there may not be an appropriate function pointer typedef in the + * corresponding header file and may need a custom typedef to match the driver + * function signature returned. This can be done by getting the typedefs from a later + * toolkit or creating appropriately matching custom function typedefs. + * + * The API will return ::cudaSuccess and set the returned \p funcPtr if the + * requested driver function is valid and supported on the platform. + * + * The API will return ::cudaSuccess and set the returned \p funcPtr to NULL if the + * requested driver function is not supported on the platform, no ABI + * compatible driver function exists for the requested version or if the + * driver symbol is invalid. + * + * It will also set the optional \p driverStatus to one of the values in + * ::cudaDriverEntryPointQueryResult with the following meanings: + * - ::cudaDriverEntryPointSuccess - The requested symbol was succesfully found based + * on input arguments and \p pfn is valid + * - ::cudaDriverEntryPointSymbolNotFound - The requested symbol was not found + * - ::cudaDriverEntryPointVersionNotSufficent - The requested symbol was found but is + * not supported by the specified version \p cudaVersion + * + * The requested flags can be: + * - ::cudaEnableDefault: This is the default mode. This is equivalent to + * ::cudaEnablePerThreadDefaultStream if the code is compiled with + * --default-stream per-thread compilation flag or the macro CUDA_API_PER_THREAD_DEFAULT_STREAM + * is defined; ::cudaEnableLegacyStream otherwise. + * - ::cudaEnableLegacyStream: This will enable the search for all driver symbols + * that match the requested driver symbol name except the corresponding per-thread versions. + * - ::cudaEnablePerThreadDefaultStream: This will enable the search for all + * driver symbols that match the requested driver symbol name including the per-thread + * versions. If a per-thread version is not found, the API will return the legacy version + * of the driver function. + * + * \param symbol - The base name of the driver API function to look for. As an example, + * for the driver API ::cuMemAlloc_v2, \p symbol would be cuMemAlloc. + * \param funcPtr - Location to return the function pointer to the requested driver function + * \param cudaVersion - The CUDA version to look for the requested driver symbol + * \param flags - Flags to specify search options. + * \param driverStatus - Optional location to store the status of finding the symbol from + * the driver. See ::cudaDriverEntryPointQueryResult for + * possible values. + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + * ::cudaErrorNotSupported + * \note_version_mixing + * \note_init_rt + * \note_callback + * + * \sa + * ::cuGetProcAddress + */ +#if defined(__cplusplus) +extern __host__ cudaError_t CUDARTAPI cudaGetDriverEntryPointByVersion(const char *symbol, void **funcPtr, unsigned int cudaVersion, unsigned long long flags, enum cudaDriverEntryPointQueryResult *driverStatus = NULL); +#else +extern __host__ cudaError_t CUDARTAPI cudaGetDriverEntryPointByVersion(const char *symbol, void **funcPtr, unsigned int cudaVersion, unsigned long long flags, enum cudaDriverEntryPointQueryResult *driverStatus); #endif /** @} */ /* END CUDART_DRIVER_ENTRY_POINT */ +/** + * \defgroup CUDART_LIBRARY Library Management + * + * ___MANBRIEF___ library management functions of the CUDA runtime API + * (___CURRENT_FILE___) ___ENDMANBRIEF___ + * + * This section describes the library management functions of the CUDA runtime + * application programming interface. + * + * @{ + */ + +/** + * \brief Load a library with specified code and options + * + * Takes a pointer \p code and loads the corresponding library \p library based on + * the application defined library loading mode: + * - If module loading is set to EAGER, via the environment variables described in "Module loading", + * \p library is loaded eagerly into all contexts at the time of the call and future contexts + * at the time of creation until the library is unloaded with ::cudaLibraryUnload(). + * - If the environment variables are set to LAZY, \p library + * is not immediately loaded onto all existent contexts and will only be + * loaded when a function is needed for that context, such as a kernel launch. + * + * These environment variables are described in the CUDA programming guide under the + * "CUDA environment variables" section. + * + * The \p code may be a \e cubin or \e fatbin as output by \b nvcc, + * or a NULL-terminated \e PTX, either as output by \b nvcc or hand-written. + * A fatbin should also contain relocatable code when doing separate compilation. + * Please also see the documentation for nvrtc (https://docs.nvidia.com/cuda/nvrtc/index.html), + * nvjitlink (https://docs.nvidia.com/cuda/nvjitlink/index.html), and nvfatbin + * (https://docs.nvidia.com/cuda/nvfatbin/index.html) for more information on generating + * loadable code at runtime. + * + * Options are passed as an array via \p jitOptions and any corresponding parameters are passed in + * \p jitOptionsValues. The number of total JIT options is supplied via \p numJitOptions. + * Any outputs will be returned via \p jitOptionsValues. + * + * Library load options are passed as an array via \p libraryOptions and any corresponding parameters are passed in + * \p libraryOptionValues. The number of total library load options is supplied via \p numLibraryOptions. + * + * \param library - Returned library + * \param code - Code to load + * \param jitOptions - Options for JIT + * \param jitOptionsValues - Option values for JIT + * \param numJitOptions - Number of options + * \param libraryOptions - Options for loading + * \param libraryOptionValues - Option values for loading + * \param numLibraryOptions - Number of options for loading + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + * ::cudaErrorMemoryAllocation, + * ::cudaErrorInitializationError, + * ::cudaErrorCudartUnloading, + * ::cudaErrorInvalidPtx, + * ::cudaErrorUnsupportedPtxVersion, + * ::cudaErrorNoKernelImageForDevice, + * ::cudaErrorSharedObjectSymbolNotFound, + * ::cudaErrorSharedObjectInitFailed, + * ::cudaErrorJitCompilerNotFound + * + * \sa ::cudaLibraryLoadFromFile, + * ::cudaLibraryUnload, + * ::cuLibraryLoadData + */ +extern __host__ cudaError_t CUDARTAPI cudaLibraryLoadData(cudaLibrary_t *library, const void *code, + enum cudaJitOption *jitOptions, void **jitOptionsValues, unsigned int numJitOptions, + enum cudaLibraryOption *libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions); + +/** + * \brief Load a library with specified file and options + * + * Takes a pointer \p code and loads the corresponding library \p library based on + * the application defined library loading mode: + * - If module loading is set to EAGER, via the environment variables described in "Module loading", + * \p library is loaded eagerly into all contexts at the time of the call and future contexts + * at the time of creation until the library is unloaded with ::cudaLibraryUnload(). + * - If the environment variables are set to LAZY, \p library + * is not immediately loaded onto all existent contexts and will only be + * loaded when a function is needed for that context, such as a kernel launch. + * + * These environment variables are described in the CUDA programming guide under the + * "CUDA environment variables" section. + * + * The file should be a \e cubin file as output by \b nvcc, or a \e PTX file either + * as output by \b nvcc or handwritten, or a \e fatbin file as output by \b nvcc. + * A fatbin should also contain relocatable code when doing separate compilation. + * Please also see the documentation for nvrtc (https://docs.nvidia.com/cuda/nvrtc/index.html), + * nvjitlink (https://docs.nvidia.com/cuda/nvjitlink/index.html), and nvfatbin + * (https://docs.nvidia.com/cuda/nvfatbin/index.html) for more information on generating + * loadable code at runtime. + * + * Options are passed as an array via \p jitOptions and any corresponding parameters are + * passed in \p jitOptionsValues. The number of total options is supplied via \p numJitOptions. + * Any outputs will be returned via \p jitOptionsValues. + * + * Library load options are passed as an array via \p libraryOptions and any corresponding parameters are passed in + * \p libraryOptionValues. The number of total library load options is supplied via \p numLibraryOptions. + * + * \param library - Returned library + * \param fileName - File to load from + * \param jitOptions - Options for JIT + * \param jitOptionsValues - Option values for JIT + * \param numJitOptions - Number of options + * \param libraryOptions - Options for loading + * \param libraryOptionValues - Option values for loading + * \param numLibraryOptions - Number of options for loading + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidValue, + * ::cudaErrorMemoryAllocation, + * ::cudaErrorInitializationError, + * ::cudaErrorCudartUnloading, + * ::cudaErrorInvalidPtx, + * ::cudaErrorUnsupportedPtxVersion, + * ::cudaErrorNoKernelImageForDevice, + * ::cudaErrorSharedObjectSymbolNotFound, + * ::cudaErrorSharedObjectInitFailed, + * ::cudaErrorJitCompilerNotFound + * + * \sa ::cudaLibraryLoadData, + * ::cudaLibraryUnload, + * ::cuLibraryLoadFromFile + */ +extern __host__ cudaError_t CUDARTAPI cudaLibraryLoadFromFile(cudaLibrary_t *library, const char *fileName, + enum cudaJitOption *jitOptions, void **jitOptionsValues, unsigned int numJitOptions, + enum cudaLibraryOption *libraryOptions, void **libraryOptionValues, unsigned int numLibraryOptions); + +/** + * \brief Unloads a library + * + * Unloads the library specified with \p library + * + * \param library - Library to unload + * + * \return + * ::cudaSuccess, + * ::cudaErrorCudartUnloading, + * ::cudaErrorInitializationError, + * ::cudaErrorInvalidValue + * + * \sa ::cudaLibraryLoadData, + * ::cudaLibraryLoadFromFile, + * ::cuLibraryUnload + */ +extern __host__ cudaError_t CUDARTAPI cudaLibraryUnload(cudaLibrary_t library); + +/** + * \brief Returns a kernel handle + * + * Returns in \p pKernel the handle of the kernel with name \p name located in library \p library. + * If kernel handle is not found, the call returns ::cudaErrorSymbolNotFound. + * + * \param pKernel - Returned kernel handle + * \param library - Library to retrieve kernel from + * \param name - Name of kernel to retrieve + * + * \return + * ::cudaSuccess, + * ::cudaErrorCudartUnloading, + * ::cudaErrorInitializationError, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidResourceHandle, + * ::cudaErrorSymbolNotFound + * + * \sa ::cudaLibraryLoadData, + * ::cudaLibraryLoadFromFile, + * ::cudaLibraryUnload, + * ::cuLibraryGetKernel + */ +extern __host__ cudaError_t CUDARTAPI cudaLibraryGetKernel(cudaKernel_t *pKernel, cudaLibrary_t library, const char *name); + +/** + * \brief Returns a global device pointer + * + * Returns in \p *dptr and \p *bytes the base pointer and size of the global with + * name \p name for the requested library \p library and the current device. + * If no global for the requested name \p name exists, the call returns ::cudaErrorSymbolNotFound. + * One of the parameters \p dptr or \p bytes (not both) can be NULL in which + * case it is ignored. The returned \p dptr cannot be passed to the Symbol APIs + * such as ::cudaMemcpyToSymbol, ::cudaMemcpyFromSymbol, ::cudaGetSymbolAddress, or + * ::cudaGetSymbolSize. + * + * \param dptr - Returned global device pointer for the requested library + * \param bytes - Returned global size in bytes + * \param library - Library to retrieve global from + * \param name - Name of global to retrieve + * + * \return + * ::cudaSuccess, + * ::cudaErrorCudartUnloading, + * ::cudaErrorInitializationError, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidResourceHandle, + * ::cudaErrorSymbolNotFound + * ::cudaErrorDeviceUninitialized, + * ::cudaErrorContextIsDestroyed + * + * \sa ::cudaLibraryLoadData, + * ::cudaLibraryLoadFromFile, + * ::cudaLibraryUnload, + * ::cudaLibraryGetManaged, + * ::cuLibraryGetGlobal + */ +extern __host__ cudaError_t CUDARTAPI cudaLibraryGetGlobal(void **dptr, size_t *bytes, cudaLibrary_t library, const char *name); + +/** + * \brief Returns a pointer to managed memory + * + * Returns in \p *dptr and \p *bytes the base pointer and size of the managed memory with + * name \p name for the requested library \p library. If no managed memory with the + * requested name \p name exists, the call returns ::cudaErrorSymbolNotFound. One of the parameters + * \p dptr or \p bytes (not both) can be NULL in which case it is ignored. + * Note that managed memory for library \p library is shared across devices and is registered + * when the library is loaded. The returned \p dptr cannot be passed to the Symbol APIs + * such as ::cudaMemcpyToSymbol, ::cudaMemcpyFromSymbol, ::cudaGetSymbolAddress, or + * ::cudaGetSymbolSize. + * + * \param dptr - Returned pointer to the managed memory + * \param bytes - Returned memory size in bytes + * \param library - Library to retrieve managed memory from + * \param name - Name of managed memory to retrieve + * + * \return + * ::cudaSuccess, + * ::cudaErrorCudartUnloading, + * ::cudaErrorInitializationError, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidResourceHandle, + * ::cudaErrorSymbolNotFound + * + * \sa ::cudaLibraryLoadData, + * ::cudaLibraryLoadFromFile, + * ::cudaLibraryUnload, + * ::cudaLibraryGetGlobal, + * ::cuLibraryGetManaged + */ +extern __host__ cudaError_t CUDARTAPI cudaLibraryGetManaged(void **dptr, size_t *bytes, cudaLibrary_t library, const char *name); + +/** + * \brief Returns a pointer to a unified function + * + * Returns in \p *fptr the function pointer to a unified function denoted by \p symbol. + * If no unified function with name \p symbol exists, the call returns ::cudaErrorSymbolNotFound. + * If there is no device with attribute ::cudaDeviceProp::unifiedFunctionPointers present in the system, + * the call may return ::cudaErrorSymbolNotFound. + * + * \param fptr - Returned pointer to a unified function + * \param library - Library to retrieve function pointer memory from + * \param symbol - Name of function pointer to retrieve + * + * \return + * ::cudaSuccess, + * ::cudaErrorCudartUnloading, + * ::cudaErrorInitializationError, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidResourceHandle, + * ::cudaErrorSymbolNotFound + * + * \sa ::cudaLibraryLoadData, + * ::cudaLibraryLoadFromFile, + * ::cudaLibraryUnload, + * ::cuLibraryGetUnifiedFunction + */ +extern __host__ cudaError_t CUDARTAPI cudaLibraryGetUnifiedFunction(void **fptr, cudaLibrary_t library, const char *symbol); + +/** + * \brief Returns the number of kernels within a library + * + * Returns in \p count the number of kernels in \p lib. + * + * \param count - Number of kernels found within the library + * \param lib - Library to query + * + * \return + * ::cudaSuccess, + * ::cudaErrorCudartUnloading, + * ::cudaErrorInitializationError, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidResourceHandle + * + * \sa ::cudaLibraryEnumerateKernels, + * ::cudaLibraryLoadFromFile, + * ::cudaLibraryLoadData, + * ::cuLibraryGetKernelCount + */ +extern __host__ cudaError_t CUDARTAPI cudaLibraryGetKernelCount(unsigned int *count, cudaLibrary_t lib); + +/** + * \brief Retrieve the kernel handles within a library. + * + * Returns in \p kernels a maximum number of \p numKernels kernel handles within \p lib. + * The returned kernel handle becomes invalid when the library is unloaded. + * + * \param kernels - Buffer where the kernel handles are returned to + * \param numKernels - Maximum number of kernel handles may be returned to the buffer + * \param lib - Library to query from + * + * \return + * ::cudaSuccess, + * ::cudaErrorCudartUnloading, + * ::cudaErrorInitializationError, + * ::cudaErrorInvalidValue, + * ::cudaErrorInvalidResourceHandle + * + * \sa ::cudaLibraryGetKernelCount, + * ::cuLibraryEnumerateKernels + */ +extern __host__ cudaError_t CUDARTAPI cudaLibraryEnumerateKernels(cudaKernel_t *kernels, unsigned int numKernels, cudaLibrary_t lib); + +/** + * \brief Sets information about a kernel + * + * This call sets the value of a specified attribute \p attr on the kernel \p kernel + * for the requested device \p device to an integer value specified by \p value. + * This function returns ::cudaSuccess if the new value of the attribute could be + * successfully set. If the set fails, this call will return an error. + * Not all attributes can have values set. Attempting to set a value on a read-only + * attribute will result in an error (::cudaErrorInvalidValue) + * + * Note that attributes set using ::cudaFuncSetAttribute() will override the attribute + * set by this API irrespective of whether the call to ::cudaFuncSetAttribute() is made + * before or after this API call. Because of this and the stricter locking requirements + * mentioned below it is suggested that this call be used during the initialization path + * and not on each thread accessing \p kernel such as on kernel launches or on the + * critical path. + * + * Valid values for \p attr are: + * - ::cudaFuncAttributeMaxDynamicSharedMemorySize - The requested maximum size in bytes of dynamically-allocated shared memory. The sum of this value and the function attribute ::sharedSizeBytes + * cannot exceed the device attribute ::cudaDevAttrMaxSharedMemoryPerBlockOptin. The maximal size of requestable dynamic shared memory may differ by GPU architecture. + * - ::cudaFuncAttributePreferredSharedMemoryCarveout - On devices where the L1 cache and shared memory use the same hardware resources, + * this sets the shared memory carveout preference, in percent of the total shared memory. See ::cudaDevAttrMaxSharedMemoryPerMultiprocessor. + * This is only a hint, and the driver can choose a different ratio if required to execute the function. + * - ::cudaFuncAttributeRequiredClusterWidth: The required cluster width in + * blocks. The width, height, and depth values must either all be 0 or all be + * positive. The validity of the cluster dimensions is checked at launch time. + * If the value is set during compile time, it cannot be set at runtime. + * Setting it at runtime will return cudaErrorNotPermitted. + * - ::cudaFuncAttributeRequiredClusterHeight: The required cluster height in + * blocks. The width, height, and depth values must either all be 0 or all be + * positive. The validity of the cluster dimensions is checked at launch time. + * If the value is set during compile time, it cannot be set at runtime. + * Setting it at runtime will return cudaErrorNotPermitted. + * - ::cudaFuncAttributeRequiredClusterDepth: The required cluster depth in + * blocks. The width, height, and depth values must either all be 0 or all be + * positive. The validity of the cluster dimensions is checked at launch time. + * If the value is set during compile time, it cannot be set at runtime. + * Setting it at runtime will return cudaErrorNotPermitted. + * - ::cudaFuncAttributeNonPortableClusterSizeAllowed: Indicates whether the + * function can be launched with non-portable cluster size. 1 is allowed, 0 is + * disallowed. + * - ::cudaFuncAttributeClusterSchedulingPolicyPreference: The block + * scheduling policy of a function. The value type is cudaClusterSchedulingPolicy. + * + * \note The API has stricter locking requirements in comparison to its legacy counterpart + * ::cudaFuncSetAttribute() due to device-wide semantics. If multiple threads are trying to + * set the same attribute on the same device simultaneously, the attribute setting will depend + * on the interleavings chosen by the OS scheduler and memory consistency. + * + * \param kernel - Kernel to set attribute of + * \param attr - Attribute requested + * \param value - Value to set + * \param device - Device to set attribute of + * + * \return + * ::cudaSuccess, + * ::cudaErrorInvalidDeviceFunction, + * ::cudaErrorInvalidValue + * + * \sa ::cudaLibraryLoadData, + * ::cudaLibraryLoadFromFile, + * ::cudaLibraryUnload, + * ::cudaLibraryGetKernel, + * ::cudaLaunchKernel, + * ::cudaFuncSetAttribute, + * ::cuKernelSetAttribute + */ +extern __host__ cudaError_t CUDARTAPI cudaKernelSetAttributeForDevice(cudaKernel_t kernel, enum cudaFuncAttribute attr, int value, int device); + +/** @} */ /* END CUDART_LIBRARY */ + /** \cond impl_private */ extern __host__ cudaError_t CUDARTAPI cudaGetExportTable(const void **ppExportTable, const cudaUUID_t *pExportTableId); /** \endcond impl_private */ @@ -12825,9 +13628,8 @@ extern __host__ cudaError_t CUDARTAPI cudaGetExportTable(const void **ppExportTa * ::cudaGLSetGLDevice(), and * ::cudaVDPAUSetVDPAUDevice(). * Note that these functions will fail with ::cudaErrorSetOnActiveProcess if they are - * called when the primary context for the specified device has already been initialized. - * (or if the current device has already been initialized, in the case of - * ::cudaSetDeviceFlags()). + * called when the primary context for the specified device has already been initialized, + * except for ::cudaSetDeviceFlags() which will simply overwrite the previous settings. * * Primary contexts will remain active until they are explicitly deinitialized * using ::cudaDeviceReset(). The function ::cudaDeviceReset() will deinitialize the @@ -12952,13 +13754,21 @@ extern __host__ cudaError_t CUDARTAPI cudaGetExportTable(const void **ppExportTa * ::cudaSuccess * */ -extern __host__ cudaError_t cudaGetFuncBySymbol(cudaFunction_t* functionPtr, const void* symbolPtr); +extern __host__ cudaError_t CUDARTAPI cudaGetFuncBySymbol(cudaFunction_t* functionPtr, const void* symbolPtr); /** * \brief Get pointer to device kernel that matches entry function \p entryFuncAddr * * Returns in \p kernelPtr the device kernel corresponding to the entry function \p entryFuncAddr. * + * Note that it is possible that there are multiple symbols belonging to different + * translation units with the same \p entryFuncAddr registered with this CUDA Runtime + * and so the order which the translation units are loaded and registered with the + * CUDA Runtime can lead to differing return pointers in \p kernelPtr . + * Suggested methods of ensuring uniqueness are to limit visibility of __global__ + * device functions by using static or hidden visibility attribute in the + * respective translation units. + * * \param kernelPtr - Returns the device kernel * \param entryFuncAddr - Address of device entry function to search kernel for * @@ -12998,10 +13808,13 @@ extern __host__ cudaError_t CUDARTAPI cudaGetKernel(cudaKernel_t *kernelPtr, con #undef cudaMemcpy2DFromArrayAsync #undef cudaMemcpy3DAsync #undef cudaMemcpy3DPeerAsync + #undef cudaMemcpyBatchAsync + #undef cudaMemcpy3DBatchAsync #undef cudaMemsetAsync #undef cudaMemset2DAsync #undef cudaMemset3DAsync #undef cudaStreamQuery + #undef cudaStreamGetDevice #undef cudaStreamGetFlags #undef cudaStreamGetId #undef cudaStreamGetPriority @@ -13015,6 +13828,9 @@ extern __host__ cudaError_t CUDARTAPI cudaGetKernel(cudaKernel_t *kernelPtr, con #undef cudaLaunchKernelExC #undef cudaLaunchHostFunc #undef cudaMemPrefetchAsync + #undef cudaMemPrefetchBatchAsync + #undef cudaMemDiscardBatchAsync + #undef cudaMemDiscardAndPrefetchBatchAsync #undef cudaLaunchCooperativeKernel #undef cudaSignalExternalSemaphoresAsync #undef cudaWaitExternalSemaphoresAsync @@ -13022,10 +13838,11 @@ extern __host__ cudaError_t CUDARTAPI cudaGetKernel(cudaKernel_t *kernelPtr, con #undef cudaGraphUpload #undef cudaGraphLaunch #undef cudaStreamBeginCapture + #undef cudaStreamBeginCaptureToGraph #undef cudaStreamEndCapture #undef cudaStreamIsCapturing #undef cudaStreamGetCaptureInfo - #undef cudaStreamGetCaptureInfo_v2 + #undef cudaStreamUpdateCaptureDependencies #undef cudaStreamCopyAttributes #undef cudaStreamGetAttribute #undef cudaStreamSetAttribute @@ -13033,8 +13850,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGetKernel(cudaKernel_t *kernelPtr, con #undef cudaFreeAsync #undef cudaMallocFromPoolAsync #undef cudaGetDriverEntryPoint - - #undef cudaGetDeviceProperties + #undef cudaGetDriverEntryPointByVersion extern __host__ cudaError_t CUDARTAPI cudaMemcpy(void *dst, const void *src, size_t count, enum cudaMemcpyKind kind); extern __host__ cudaError_t CUDARTAPI cudaMemcpyToSymbol(const void *symbol, const void *src, size_t count, size_t offset __dv(0), enum cudaMemcpyKind kind __dv(cudaMemcpyHostToDevice)); @@ -13048,6 +13864,8 @@ extern __host__ cudaError_t CUDARTAPI cudaGetKernel(cudaKernel_t *kernelPtr, con extern __host__ cudaError_t CUDARTAPI cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, enum cudaMemcpyKind kind __dv(cudaMemcpyDeviceToDevice)); extern __host__ cudaError_t CUDARTAPI cudaMemcpy3D(const struct cudaMemcpy3DParms *p); extern __host__ cudaError_t CUDARTAPI cudaMemcpy3DPeer(const struct cudaMemcpy3DPeerParms *p); + extern __host__ cudaError_t CUDARTAPI cudaMemcpyBatchAsync(void *const *dsts, const void *const *srcs, const size_t *sizes, size_t count, struct cudaMemcpyAttributes *attrs, size_t *attrsIdxs, size_t numAttrs, cudaStream_t stream); + extern __host__ cudaError_t CUDARTAPI cudaMemcpy3DBatchAsync(size_t numOps, struct cudaMemcpy3DBatchOp *opList, unsigned long long flags, cudaStream_t stream); extern __host__ cudaError_t CUDARTAPI cudaMemset(void *devPtr, int value, size_t count); extern __host__ cudaError_t CUDARTAPI cudaMemset2D(void *devPtr, size_t pitch, int value, size_t width, size_t height); extern __host__ cudaError_t CUDARTAPI cudaMemset3D(struct cudaPitchedPtr pitchedDevPtr, int value, struct cudaExtent extent); @@ -13065,6 +13883,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGetKernel(cudaKernel_t *kernelPtr, con extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaMemset2DAsync(void *devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream __dv(0)); extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaMemset3DAsync(struct cudaPitchedPtr pitchedDevPtr, int value, struct cudaExtent extent, cudaStream_t stream __dv(0)); extern __host__ cudaError_t CUDARTAPI cudaStreamQuery(cudaStream_t stream); + extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamGetDevice(cudaStream_t hStream, int *device); extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamGetFlags(cudaStream_t hStream, unsigned int *flags); extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamGetId(cudaStream_t hStream, unsigned long long *streamId); extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaStreamGetPriority(cudaStream_t hStream, int *priority); @@ -13078,23 +13897,21 @@ extern __host__ cudaError_t CUDARTAPI cudaGetKernel(cudaKernel_t *kernelPtr, con extern __host__ cudaError_t CUDARTAPI cudaLaunchKernelExC(const cudaLaunchConfig_t *config, const void *func, void **args); extern __host__ cudaError_t CUDARTAPI cudaLaunchCooperativeKernel(const void *func, dim3 gridDim, dim3 blockDim, void **args, size_t sharedMem, cudaStream_t stream); extern __host__ cudaError_t CUDARTAPI cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void *userData); - extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchAsync(const void *devPtr, size_t count, int dstDevice, cudaStream_t stream); - extern __host__ cudaError_t CUDARTAPI cudaSignalExternalSemaphoresAsync(const cudaExternalSemaphore_t *extSemArray, const struct cudaExternalSemaphoreSignalParams_v1 *paramsArray, unsigned int numExtSems, cudaStream_t stream __dv(0)); - extern __host__ cudaError_t CUDARTAPI cudaSignalExternalSemaphoresAsync_ptsz(const cudaExternalSemaphore_t *extSemArray, const struct cudaExternalSemaphoreSignalParams_v1 *paramsArray, unsigned int numExtSems, cudaStream_t stream __dv(0)); - extern __host__ cudaError_t CUDARTAPI cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t *extSemArray, const struct cudaExternalSemaphoreSignalParams *paramsArray, unsigned int numExtSems, cudaStream_t stream __dv(0)); - extern __host__ cudaError_t CUDARTAPI cudaWaitExternalSemaphoresAsync(const cudaExternalSemaphore_t *extSemArray, const struct cudaExternalSemaphoreWaitParams_v1 *paramsArray, unsigned int numExtSems, cudaStream_t stream __dv(0)); - extern __host__ cudaError_t CUDARTAPI cudaWaitExternalSemaphoresAsync_ptsz(const cudaExternalSemaphore_t *extSemArray, const struct cudaExternalSemaphoreWaitParams_v1 *paramsArray, unsigned int numExtSems, cudaStream_t stream __dv(0)); - extern __host__ cudaError_t CUDARTAPI cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t *extSemArray, const struct cudaExternalSemaphoreWaitParams *paramsArray, unsigned int numExtSems, cudaStream_t stream __dv(0)); + extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchAsync(const void *devPtr, size_t count, struct cudaMemLocation location, unsigned int flags, cudaStream_t stream); + extern __host__ cudaError_t CUDARTAPI cudaMemPrefetchBatchAsync(void **dptrs, size_t *sizes, size_t count, struct cudaMemLocation *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, unsigned long long flags, cudaStream_t stream); + extern __host__ cudaError_t CUDARTAPI cudaMemDiscardBatchAsync(void **dptrs, size_t *sizes, size_t count, unsigned long long flags, cudaStream_t stream); + extern __host__ cudaError_t CUDARTAPI cudaMemDiscardAndPrefetchBatchAsync(void **dptrs, size_t *sizes, size_t count, struct cudaMemLocation *prefetchLocs, size_t *prefetchLocIdxs, size_t numPrefetchLocs, unsigned long long flags, cudaStream_t stream); + extern __host__ cudaError_t CUDARTAPI cudaSignalExternalSemaphoresAsync(const cudaExternalSemaphore_t *extSemArray, const struct cudaExternalSemaphoreSignalParams *paramsArray, unsigned int numExtSems, cudaStream_t stream __dv(0)); + extern __host__ cudaError_t CUDARTAPI cudaWaitExternalSemaphoresAsync(const cudaExternalSemaphore_t *extSemArray, const struct cudaExternalSemaphoreWaitParams *paramsArray, unsigned int numExtSems, cudaStream_t stream __dv(0)); extern __host__ cudaError_t CUDARTAPI cudaGraphInstantiateWithParams(cudaGraphExec_t *pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams *instantiateParams); extern __host__ cudaError_t CUDARTAPI cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream); extern __host__ cudaError_t CUDARTAPI cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream); extern __host__ cudaError_t CUDARTAPI cudaStreamBeginCapture(cudaStream_t stream, enum cudaStreamCaptureMode mode); + extern __host__ cudaError_t CUDARTAPI cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t *dependencies, const cudaGraphEdgeData *dependencyData, size_t numDependencies, enum cudaStreamCaptureMode mode); extern __host__ cudaError_t CUDARTAPI cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t *pGraph); extern __host__ cudaError_t CUDARTAPI cudaStreamIsCapturing(cudaStream_t stream, enum cudaStreamCaptureStatus *pCaptureStatus); - extern __host__ cudaError_t CUDARTAPI cudaStreamGetCaptureInfo(cudaStream_t stream, enum cudaStreamCaptureStatus *captureStatus_out, unsigned long long *id_out); - extern __host__ cudaError_t CUDARTAPI cudaStreamGetCaptureInfo_ptsz(cudaStream_t stream, enum cudaStreamCaptureStatus *captureStatus_out, unsigned long long *id_out); - extern __host__ cudaError_t CUDARTAPI cudaStreamGetCaptureInfo_v2(cudaStream_t stream, enum cudaStreamCaptureStatus *captureStatus_out, unsigned long long *id_out __dv(0), cudaGraph_t *graph_out __dv(0), const cudaGraphNode_t **dependencies_out __dv(0), size_t *numDependencies_out __dv(0)); - extern __host__ cudaError_t CUDARTAPI cudaStreamUpdateCaptureDependencies_ptsz(cudaStream_t stream, cudaGraphNode_t *dependencies, size_t numDependencies, unsigned int flags __dv(0)); + extern __host__ cudaError_t CUDARTAPI cudaStreamGetCaptureInfo(cudaStream_t stream, enum cudaStreamCaptureStatus *captureStatus_out, unsigned long long *id_out __dv(0), cudaGraph_t *graph_out __dv(0), const cudaGraphNode_t **dependencies_out __dv(0), const cudaGraphEdgeData **edgeData_out __dv(0), size_t *numDependencies_out __dv(0)); + extern __host__ cudaError_t CUDARTAPI cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t *dependencies, const cudaGraphEdgeData *dependencyData, size_t numDependencies, unsigned int flags __dv(0)); extern __host__ cudaError_t CUDARTAPI cudaStreamCopyAttributes(cudaStream_t dstStream, cudaStream_t srcStream); extern __host__ cudaError_t CUDARTAPI cudaStreamGetAttribute(cudaStream_t stream, cudaStreamAttrID attr, cudaStreamAttrValue *value); extern __host__ cudaError_t CUDARTAPI cudaStreamSetAttribute(cudaStream_t stream, cudaStreamAttrID attr, const cudaStreamAttrValue *param); @@ -13103,8 +13920,7 @@ extern __host__ cudaError_t CUDARTAPI cudaGetKernel(cudaKernel_t *kernelPtr, con extern __host__ cudaError_t CUDARTAPI cudaFreeAsync(void *devPtr, cudaStream_t hStream); extern __host__ cudaError_t CUDARTAPI cudaMallocFromPoolAsync(void **ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream); extern __host__ cudaError_t CUDARTAPI cudaGetDriverEntryPoint(const char *symbol, void **funcPtr, unsigned long long flags, enum cudaDriverEntryPointQueryResult *driverStatus); - - extern __host__ __cudart_builtin__ cudaError_t CUDARTAPI cudaGetDeviceProperties(struct cudaDeviceProp *prop, int device); + extern __host__ cudaError_t CUDARTAPI cudaGetDriverEntryPointByVersion(const char *symbol, void **funcPtr, unsigned int cudaVersion, unsigned long long flags, enum cudaDriverEntryPointQueryResult *driverStatus); #elif defined(__CUDART_API_PER_THREAD_DEFAULT_STREAM) // nvcc stubs reference the 'cudaLaunch'/'cudaLaunchKernel' identifier even if it was defined diff --git a/backends/cuda/include/driver_types.h b/backends/cuda/include/driver_types.h index 988702c0e..199b763cb 100644 --- a/backends/cuda/include/driver_types.h +++ b/backends/cuda/include/driver_types.h @@ -1,5 +1,5 @@ /* - * Copyright 1993-2018 NVIDIA Corporation. All rights reserved. + * Copyright 1993-2023 NVIDIA Corporation. All rights reserved. * * NOTICE TO LICENSEE: * @@ -62,6 +62,7 @@ +#ifndef __CUDACC_RTC_MINIMAL__ /** * \defgroup CUDART_TYPES Data types used by CUDA Runtime * \ingroup CUDART @@ -77,6 +78,7 @@ #if !defined(__CUDA_INTERNAL_COMPILATION__) + #if !defined(__CUDACC_RTC__) #include #include @@ -139,7 +141,7 @@ #define cudaDeviceScheduleMask 0x07 /**< Device schedule flags mask */ #define cudaDeviceMapHost 0x08 /**< Device flag - Support mapped pinned allocations */ #define cudaDeviceLmemResizeToMax 0x10 /**< Device flag - Keep local memory allocation after launch */ -#define cudaDeviceSyncMemops 0x80 /**< Device flag - Use synchronous behavior for cudaMemcpy/cudaMemset */ +#define cudaDeviceSyncMemops 0x80 /**< Device flag - Ensure synchronous memory operations on this context will synchronize */ #define cudaDeviceMask 0xff /**< Device flags mask */ #define cudaArrayDefault 0x00 /**< Default CUDA array allocation flag */ @@ -163,19 +165,6 @@ #define cudaCpuDeviceId ((int)-1) /**< Device id that represents the CPU */ #define cudaInvalidDeviceId ((int)-2) /**< Device id that represents an invalid device */ #define cudaInitDeviceFlagsAreValid 0x01 /**< Tell the CUDA runtime that DeviceFlags is being set in cudaInitDevice call */ -/** - * If set, each kernel launched as part of ::cudaLaunchCooperativeKernelMultiDevice only - * waits for prior work in the stream corresponding to that GPU to complete before the - * kernel begins execution. - */ -#define cudaCooperativeLaunchMultiDeviceNoPreSync 0x01 - -/** - * If set, any subsequent work pushed in a stream that participated in a call to - * ::cudaLaunchCooperativeKernelMultiDevice will only wait for the kernel launched on - * the GPU corresponding to that stream to complete before it begins execution. - */ -#define cudaCooperativeLaunchMultiDeviceNoPostSync 0x02 #endif /* !__CUDA_INTERNAL_COMPILATION__ */ @@ -216,8 +205,8 @@ enum __device_builtin__ cudaError cudaErrorInvalidValue = 1, /** - * The API call failed because it was unable to allocate enough memory to - * perform the requested operation. + * The API call failed because it was unable to allocate enough memory or + * other resources to perform the requested operation. */ cudaErrorMemoryAllocation = 2, @@ -262,7 +251,6 @@ enum __device_builtin__ cudaError * to call cudaProfilerStop() when profiling is already disabled. */ cudaErrorProfilerAlreadyStopped = 8, - /** * This indicates that a kernel launch is requesting resources that can * never be satisfied by the current device. Requesting more shared memory @@ -283,7 +271,7 @@ enum __device_builtin__ cudaError * is not a valid name or identifier. */ cudaErrorInvalidSymbol = 13, - + /** * This indicates that at least one host pointer passed to the API call is * not a valid host pointer. @@ -299,7 +287,6 @@ enum __device_builtin__ cudaError * This error return is deprecated as of CUDA 10.1. */ cudaErrorInvalidDevicePointer = 17, - /** * This indicates that the texture passed to the API call is not a valid * texture. @@ -324,7 +311,7 @@ enum __device_builtin__ cudaError * not one of the types specified by ::cudaMemcpyKind. */ cudaErrorInvalidMemcpyDirection = 21, - + /** * This indicated that the user has taken the address of a constant variable, * which was forbidden up until the CUDA 3.1 release. @@ -361,7 +348,6 @@ enum __device_builtin__ cudaError * removed with the CUDA 3.1 release. */ cudaErrorSynchronizationError = 25, - /** * This indicates that a non-float texture was being accessed with linear * filtering. This is not supported by CUDA. @@ -369,11 +355,11 @@ enum __device_builtin__ cudaError cudaErrorInvalidFilterSetting = 26, /** - * This indicates that an attempt was made to read a non-float texture as a + * This indicates that an attempt was made to read an unsupported data type as a * normalized float. This is not supported by CUDA. */ cudaErrorInvalidNormSetting = 27, - + /** * Mixing of device and device emulation code was not allowed. * \deprecated @@ -398,7 +384,6 @@ enum __device_builtin__ cudaError * removed with the CUDA 3.1 release. */ cudaErrorMemoryValueTooLarge = 32, - /** * This indicates that the CUDA driver that the application has loaded is a * stub library. Applications that run with the stub rather than a real @@ -472,7 +457,7 @@ enum __device_builtin__ cudaError * previously configured via the ::cudaConfigureCall() function. */ cudaErrorMissingConfiguration = 52, - + /** * This indicated that a previous kernel launch failed. This was previously * used for device emulation of kernel launches. @@ -481,7 +466,6 @@ enum __device_builtin__ cudaError * removed with the CUDA 3.1 release. */ cudaErrorPriorLaunchFailure = 53, - /** * This error indicates that a device runtime grid launch did not occur * because the depth of the child grid would exceed the maximum supported @@ -710,6 +694,17 @@ enum __device_builtin__ cudaError */ cudaErrorUnsupportedDevSideSync = 225, + /** + * This indicates that an exception occurred on the device that is now + * contained by the GPU's error containment capability. Common causes are - + * a. Certain types of invalid accesses of peer GPU memory over nvlink + * b. Certain classes of hardware errors + * This leaves the process in an inconsistent state and any further CUDA + * work will return the same error. To continue using CUDA, the process must + * be terminated and relaunched. + */ + cudaErrorContained = 226, + /** * This indicates that the device kernel source is invalid. */ @@ -748,6 +743,14 @@ enum __device_builtin__ cudaError */ cudaErrorIllegalState = 401, + /** + * This indicates an attempt was made to introspect an object in a way that + * would discard semantically important information. This is either due to + * the object using funtionality newer than the API version used to + * introspect it or omission of optional return arguments. + */ + cudaErrorLossyQuery = 402, + /** * This indicates that a named symbol was not found. Examples of symbols * are global/constant variable names, driver function names, texture names, @@ -782,8 +785,8 @@ enum __device_builtin__ cudaError /** * This indicates that the device kernel took too long to execute. This can - * only occur if timeouts are enabled - see the device property - * \ref ::cudaDeviceProp::kernelExecTimeoutEnabled "kernelExecTimeoutEnabled" + * only occur if timeouts are enabled - see the device attribute + * \ref ::cudaDeviceAttr::cudaDevAttrKernelExecTimeout "cudaDevAttrKernelExecTimeout" * for more information. * This leaves the process in an inconsistent state and any further CUDA work * will return the same error. To continue using CUDA, the process must be terminated @@ -915,12 +918,20 @@ enum __device_builtin__ cudaError /** * This error indicates that the number of blocks launched per grid for a kernel that was - * launched via either ::cudaLaunchCooperativeKernel or ::cudaLaunchCooperativeKernelMultiDevice + * launched via either ::cudaLaunchCooperativeKernel * exceeds the maximum number of blocks as allowed by ::cudaOccupancyMaxActiveBlocksPerMultiprocessor * or ::cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags times the number of multiprocessors * as specified by the device attribute ::cudaDevAttrMultiProcessorCount. */ cudaErrorCooperativeLaunchTooLarge = 720, + + /** + * An exception occurred on the device while exiting a kernel using tensor memory: the + * tensor memory was not completely deallocated. This leaves the process in an inconsistent + * state and any further CUDA work will return the same error. To continue using CUDA, the + * process must be terminated and relaunched. + */ + cudaErrorTensorMemoryLeak = 721, /** * This error indicates the attempted operation is not permitted. @@ -1080,10 +1091,28 @@ enum __device_builtin__ cudaError */ cudaErrorInvalidClusterSize = 912, + /** + * Indiciates a function handle is not loaded when calling an API that requires + * a loaded function. + */ + cudaErrorFunctionNotLoaded = 913, + + /** + * This error indicates one or more resources passed in are not valid resource + * types for the operation. + */ + cudaErrorInvalidResourceType = 914, + + /** + * This error indicates one or more resources are insufficient or non-applicable for + * the operation. + */ + cudaErrorInvalidResourceConfiguration = 915, + /** * This indicates that an unknown internal error has occurred. */ - cudaErrorUnknown = 999, + cudaErrorUnknown = 999 /** * Any unhandled CUDA driver error is added to this value and returned via @@ -1091,7 +1120,7 @@ enum __device_builtin__ cudaError * \deprecated * This error return is deprecated as of CUDA 4.1. */ - cudaErrorApiFailureBase = 10000 + , cudaErrorApiFailureBase = 10000 }; /** @@ -1129,7 +1158,9 @@ enum __device_builtin__ cudaChannelFormatKind cudaChannelFormatKindUnsignedBlockCompressed6H = 27, /**< 3 channel unsigned half-float block-compressed (BC6H compression) format */ cudaChannelFormatKindSignedBlockCompressed6H = 28, /**< 3 channel signed half-float block-compressed (BC6H compression) format */ cudaChannelFormatKindUnsignedBlockCompressed7 = 29, /**< 4 channel unsigned normalized block-compressed (BC7 compression) format */ - cudaChannelFormatKindUnsignedBlockCompressed7SRGB = 30 /**< 4 channel unsigned normalized block-compressed (BC7 compression) format with sRGB encoding */ + cudaChannelFormatKindUnsignedBlockCompressed7SRGB = 30, /**< 4 channel unsigned normalized block-compressed (BC7 compression) format with sRGB encoding */ + cudaChannelFormatKindUnsignedNormalized1010102 = 31 /**< 4 channel unsigned normalized (10-bit, 10-bit, 10-bit, 2-bit) format */ + }; /** @@ -1274,6 +1305,15 @@ struct __device_builtin__ cudaMemcpy3DParms enum cudaMemcpyKind kind; /**< Type of transfer */ }; +/** + * Memcpy node parameters + */ +struct __device_builtin__ cudaMemcpyNodeParams { + int flags; /**< Must be zero */ + int reserved[3]; /**< Must be zero */ + struct cudaMemcpy3DParms copyParams; /**< Parameters for the memory copy */ +}; + /** * CUDA 3D cross-device memory copying parameters */ @@ -1304,6 +1344,18 @@ struct __device_builtin__ cudaMemsetParams { size_t height; /**< Number of rows */ }; +/** + * CUDA Memset node parameters + */ +struct __device_builtin__ cudaMemsetParamsV2 { + void *dst; /**< Destination device pointer */ + size_t pitch; /**< Pitch of destination device pointer. Unused if height is 1 */ + unsigned int value; /**< Value to be set */ + unsigned int elementSize; /**< Size of each element in bytes. Must be 1, 2, or 4. */ + size_t width; /**< Width of the row in elements */ + size_t height; /**< Number of rows */ +}; + /** * Specifies performance hint with ::cudaAccessPolicyWindow for hitProp and missProp members. */ @@ -1352,6 +1404,14 @@ struct __device_builtin__ cudaHostNodeParams { void* userData; /**< Argument to pass to the function */ }; +/** + * CUDA host node parameters + */ +struct __device_builtin__ cudaHostNodeParamsV2 { + cudaHostFn_t fn; /**< The function to call when the node executes */ + void* userData; /**< Argument to pass to the function */ +}; + /** * Possible stream capture statuses returned by ::cudaStreamIsCapturing */ @@ -1528,7 +1588,12 @@ struct __device_builtin__ cudaResourceDesc { size_t height; /**< Height of the array in elements */ size_t pitchInBytes; /**< Pitch between two rows in bytes */ } pitch2D; + struct { + int reserved[32]; + } reserved; } res; + + unsigned int flags; /**< Flags (must be zero) */ }; /** @@ -1544,6 +1609,7 @@ struct __device_builtin__ cudaResourceViewDesc unsigned int lastMipmapLevel; /**< Last defined mipmap level */ unsigned int firstLayer; /**< First layer index */ unsigned int lastLayer; /**< Last layer index */ + unsigned int reserved[16]; /**< Must be zero */ }; /** @@ -1582,6 +1648,11 @@ struct __device_builtin__ cudaPointerAttributes * may contain invalid pointer if an invalid pointer has been passed to CUDA. */ void *hostPointer; + + /** + * Must be zero + */ + long reserved[8]; }; /** @@ -1737,8 +1808,8 @@ enum __device_builtin__ cudaFuncCache /** * CUDA shared memory configuration + * \deprecated */ - enum __device_builtin__ cudaSharedMemConfig { cudaSharedMemBankSizeDefault = 0, @@ -1798,10 +1869,14 @@ enum __device_builtin__ cudaMemoryAdvise */ enum __device_builtin__ cudaMemRangeAttribute { - cudaMemRangeAttributeReadMostly = 1, /**< Whether the range will mostly be read and only occassionally be written to */ - cudaMemRangeAttributePreferredLocation = 2, /**< The preferred location of the range */ - cudaMemRangeAttributeAccessedBy = 3, /**< Memory range has ::cudaMemAdviseSetAccessedBy set for specified device */ - cudaMemRangeAttributeLastPrefetchLocation = 4 /**< The last location to which the range was prefetched */ + cudaMemRangeAttributeReadMostly = 1, /**< Whether the range will mostly be read and only occassionally be written to */ + cudaMemRangeAttributePreferredLocation = 2, /**< The preferred location of the range */ + cudaMemRangeAttributeAccessedBy = 3, /**< Memory range has ::cudaMemAdviseSetAccessedBy set for specified device */ + cudaMemRangeAttributeLastPrefetchLocation = 4, /**< The last location to which the range was prefetched */ + cudaMemRangeAttributePreferredLocationType = 5, /**< The preferred location type of the range */ + cudaMemRangeAttributePreferredLocationId = 6, /**< The preferred location id of the range */ + cudaMemRangeAttributeLastPrefetchLocationType = 7, /**< The last location type to which the range was prefetched */ + cudaMemRangeAttributeLastPrefetchLocationId = 8 /**< The last location id to which the range was prefetched */ }; /** @@ -1936,7 +2011,7 @@ enum __device_builtin__ cudaDeviceAttr cudaDevAttrReserved93 = 93, cudaDevAttrReserved94 = 94, cudaDevAttrCooperativeLaunch = 95, /**< Device supports launching cooperative kernels via ::cudaLaunchCooperativeKernel*/ - cudaDevAttrCooperativeMultiDeviceLaunch = 96, /**< Deprecated, cudaLaunchCooperativeKernelMultiDevice is deprecated. */ + cudaDevAttrReserved96 = 96, cudaDevAttrMaxSharedMemoryPerBlockOptin = 97, /**< The maximum optin shared memory per block. This value may vary by chip. See ::cudaFuncSetAttribute */ cudaDevAttrCanFlushRemoteWrites = 98, /**< Device supports flushing of outstanding remote writes. */ cudaDevAttrHostRegisterSupported = 99, /**< Device supports host memory registration via ::cudaHostRegister. */ @@ -1949,7 +2024,6 @@ enum __device_builtin__ cudaDeviceAttr cudaDevAttrSparseCudaArraySupported = 112, /**< Device supports sparse CUDA arrays and sparse CUDA mipmapped arrays */ cudaDevAttrHostRegisterReadOnlySupported = 113, /**< Device supports using the ::cudaHostRegister flag cudaHostRegisterReadOnly to register memory that must be mapped as read-only to the GPU */ cudaDevAttrTimelineSemaphoreInteropSupported = 114, /**< External timeline semaphore interop is supported on the device */ - cudaDevAttrMaxTimelineSemaphoreInteropSupported = 114, /**< Deprecated, External timeline semaphore interop is supported on the device */ cudaDevAttrMemoryPoolsSupported = 115, /**< Device supports using the ::cudaMallocAsync and ::cudaMemPool family of APIs */ cudaDevAttrGPUDirectRDMASupported = 116, /**< Device supports GPUDirect RDMA APIs, like nvidia_p2p_get_pages (see https://docs.nvidia.com/cuda/gpudirect-rdma for more information) */ cudaDevAttrGPUDirectRDMAFlushWritesOptions = 117, /**< The returned attribute shall be interpreted as a bitmask, where the individual bits are listed in the ::cudaFlushGPUDirectRDMAWritesOptions enum */ @@ -1965,7 +2039,22 @@ enum __device_builtin__ cudaDeviceAttr cudaDevAttrReserved127 = 127, cudaDevAttrReserved128 = 128, cudaDevAttrReserved129 = 129, + cudaDevAttrNumaConfig = 130, /**< NUMA configuration of a device: value is of type ::cudaDeviceNumaConfig enum */ + cudaDevAttrNumaId = 131, /**< NUMA node ID of the GPU memory */ cudaDevAttrReserved132 = 132, + cudaDevAttrMpsEnabled = 133, /**< Contexts created on this device will be shared via MPS */ + cudaDevAttrHostNumaId = 134, /**< NUMA ID of the host node closest to the device or -1 when system does not support NUMA */ + cudaDevAttrD3D12CigSupported = 135, /**< Device supports CIG with D3D12. */ + cudaDevAttrVulkanCigSupported = 138, /**< Device supports CIG with Vulkan. */ + cudaDevAttrGpuPciDeviceId = 139, /**< The combined 16-bit PCI device ID and 16-bit PCI vendor ID. */ + cudaDevAttrGpuPciSubsystemId = 140, /**< The combined 16-bit PCI subsystem ID and 16-bit PCI subsystem vendor ID. */ + cudaDevAttrReserved141 = 141, + cudaDevAttrHostNumaMemoryPoolsSupported = 142, /**< Device supports HOST_NUMA location with the ::cudaMallocAsync and ::cudaMemPool family of APIs */ + cudaDevAttrHostNumaMultinodeIpcSupported = 143, /**< Device supports HostNuma location IPC between nodes in a multi-node system. */ + cudaDevAttrHostMemoryPoolsSupported = 144, /**< Device suports HOST location with the ::cuMemAllocAsync and ::cuMemPool family of APIs */ + cudaDevAttrReserved145 = 145, + cudaDevAttrOnlyPartialHostNativeAtomicSupported = 147, /**< Link between the device and the host supports only some native atomic operations */ + cudaDevAttrMax }; @@ -2042,13 +2131,18 @@ enum __device_builtin__ cudaMemPoolAttr */ enum __device_builtin__ cudaMemLocationType { cudaMemLocationTypeInvalid = 0, - cudaMemLocationTypeDevice = 1 /**< Location is a device location, thus id is a device ordinal */ + cudaMemLocationTypeNone = 0, /**< Location is unspecified. This is used when creating a managed memory pool to indicate no preferred location for the pool */ + cudaMemLocationTypeDevice = 1, /**< Location is a device location, thus id is a device ordinal */ + cudaMemLocationTypeHost = 2 /**< Location is host, id is ignored */ + , cudaMemLocationTypeHostNuma = 3 /**< Location is a host NUMA node, thus id is a host NUMA node id */ + , cudaMemLocationTypeHostNumaCurrent = 4 /**< Location is the host NUMA node closest to the current thread's CPU, id is ignored */ }; /** * Specifies a memory location. * * To specify a gpu, set type = ::cudaMemLocationTypeDevice and set id = the gpu's device ordinal. + * To specify a cpu NUMA node, set type = ::cudaMemLocationTypeHostNuma and set id = host NUMA node id. */ struct __device_builtin__ cudaMemLocation { enum cudaMemLocationType type; /**< Specifies the location type, which modifies the meaning of id. */ @@ -2081,6 +2175,9 @@ enum __device_builtin__ cudaMemAllocationType { * location while the application is actively using it */ cudaMemAllocationTypePinned = 0x1, + /** This allocation type is managed memory + */ + cudaMemAllocationTypeManaged = 0x2, cudaMemAllocationTypeMax = 0x7FFFFFFF }; @@ -2091,9 +2188,16 @@ enum __device_builtin__ cudaMemAllocationHandleType { cudaMemHandleTypeNone = 0x0, /**< Does not allow any export mechanism. > */ cudaMemHandleTypePosixFileDescriptor = 0x1, /**< Allows a file descriptor to be used for exporting. Permitted only on POSIX systems. (int) */ cudaMemHandleTypeWin32 = 0x2, /**< Allows a Win32 NT handle to be used for exporting. (HANDLE) */ - cudaMemHandleTypeWin32Kmt = 0x4 /**< Allows a Win32 KMT handle to be used for exporting. (D3DKMT_HANDLE) */ + cudaMemHandleTypeWin32Kmt = 0x4, /**< Allows a Win32 KMT handle to be used for exporting. (D3DKMT_HANDLE) */ + cudaMemHandleTypeFabric = 0x8 /**< Allows a fabric handle to be used for exporting. (cudaMemFabricHandle_t) */ }; +/** + * This flag, if set, indicates that the memory will be used as a buffer for + * hardware accelerated decompression. + */ +#define cudaMemPoolCreateUsageHwDecompress 0x2 + /** * Specifies the properties of allocations made from the pool. */ @@ -2108,7 +2212,9 @@ struct __device_builtin__ cudaMemPoolProps { * processes. In all other cases, this field is required to be zero. */ void *win32SecurityAttributes; - unsigned char reserved[64]; /**< reserved for future use, must be 0 */ + size_t maxSize; /**< Maximum pool size. When set to 0, defaults to a system dependent value.*/ + unsigned short usage; /**< Bitmask indicating intended usage for the pool. */ + unsigned char reserved[54]; /**< reserved for future use, must be 0 */ }; /** @@ -2133,6 +2239,28 @@ struct __device_builtin__ cudaMemAllocNodeParams { void *dptr; /**< out: address of the allocation returned by CUDA */ }; +/** + * Memory allocation node parameters + */ +struct __device_builtin__ cudaMemAllocNodeParamsV2 { + /** + * in: location where the allocation should reside (specified in ::location). + * ::handleTypes must be ::cudaMemHandleTypeNone. IPC is not supported. + */ + struct cudaMemPoolProps poolProps; /**< in: array of memory access descriptors. Used to describe peer GPU access */ + const struct cudaMemAccessDesc *accessDescs; /**< in: number of memory access descriptors. Must not exceed the number of GPUs. */ + size_t accessDescCount; /**< in: Number of `accessDescs`s */ + size_t bytesize; /**< in: size in bytes of the requested allocation */ + void *dptr; /**< out: address of the allocation returned by CUDA */ +}; + +/** + * Memory free node parameters + */ +struct __device_builtin__ cudaMemFreeNodeParams { + void *dptr; /**< in: the pointer to free */ +}; + /** * Graph memory attributes */ @@ -2165,6 +2293,114 @@ enum __device_builtin__ cudaGraphMemAttributeType { cudaGraphMemAttrReservedMemHigh = 0x3 }; +/** + * Flags to specify for copies within a batch. For more details see ::cudaMemcpyBatchAsync. + */ +enum __device_builtin__ cudaMemcpyFlags { + cudaMemcpyFlagDefault = 0x0, + + /** + * Hint to the driver to try and overlap the copy with compute work on the SMs. + */ + cudaMemcpyFlagPreferOverlapWithCompute = 0x1 +}; + +enum __device_builtin__ cudaMemcpySrcAccessOrder { + /** + * Default invalid. + */ + cudaMemcpySrcAccessOrderInvalid = 0x0, + + /** + * Indicates that access to the source pointer must be in stream order. + */ + cudaMemcpySrcAccessOrderStream = 0x1, + + /** + * Indicates that access to the source pointer can be out of stream order and all + * accesses must be complete before the API call returns. This flag is suited for + * ephemeral sources (ex., stack variables) when it's known that no prior operations + * in the stream can be accessing the memory and also that the lifetime of the memory + * is limited to the scope that the source variable was declared in. Specifying + * this flag allows the driver to optimize the copy and removes the need for the user + * to synchronize the stream after the API call. + */ + cudaMemcpySrcAccessOrderDuringApiCall = 0x2, + + /** + * Indicates that access to the source pointer can be out of stream order and the accesses + * can happen even after the API call returns. This flag is suited for host pointers + * allocated outside CUDA (ex., via malloc) when it's known that no prior operations + * in the stream can be accessing the memory. Specifying this flag allows the driver + * to optimize the copy on certain platforms. + */ + cudaMemcpySrcAccessOrderAny = 0x3, + + cudaMemcpySrcAccessOrderMax = 0x7FFFFFFF +}; + +/** + * Attributes specific to copies within a batch. For more details on usage see ::cudaMemcpyBatchAsync. + */ +struct __device_builtin__ cudaMemcpyAttributes { + enum cudaMemcpySrcAccessOrder srcAccessOrder; /**< Source access ordering to be observed for copies with this attribute. */ + struct cudaMemLocation srcLocHint; /**< Hint location for the source operand. Ignored when the pointers are not managed memory or memory allocated outside CUDA. */ + struct cudaMemLocation dstLocHint; /**< Hint location for the destination operand. Ignored when the pointers are not managed memory or memory allocated outside CUDA. */ + unsigned int flags; /**< Additional flags for copies with this attribute. See ::cudaMemcpyFlags. */ +}; + +/** + * These flags allow applications to convey the operand type for individual copies specified in ::cudaMemcpy3DBatchAsync. + */ +enum __device_builtin__ cudaMemcpy3DOperandType { + cudaMemcpyOperandTypePointer = 0x1, /**< Memcpy operand is a valid pointer. */ + cudaMemcpyOperandTypeArray = 0x2, /**< Memcpy operand is a CUarray. */ + cudaMemcpyOperandTypeMax = 0x7FFFFFFF +}; + +/** + * Struct representing offset into a ::cudaArray_t in elements + */ +struct __device_builtin__ cudaOffset3D { + size_t x; + size_t y; + size_t z; +}; + +/** + * Struct representing an operand for copy with ::cudaMemcpy3DBatchAsync + */ +struct __device_builtin__ cudaMemcpy3DOperand { + enum cudaMemcpy3DOperandType type; + union { + /** + * Struct representing an operand when ::cudaMemcpy3DOperand::type is ::cudaMemcpyOperandTypePointer + */ + struct { + void *ptr; + size_t rowLength; /**< Length of each row in elements. */ + size_t layerHeight; /**< Height of each layer in elements. */ + struct cudaMemLocation locHint; /**< Hint location for the operand. Ignored when the pointers are not managed memory or memory allocated outside CUDA. */ + } ptr; + + /** + * Struct representing an operand when ::cudaMemcpy3DOperand::type is ::cudaMemcpyOperandTypeArray + */ + struct { + cudaArray_t array; + struct cudaOffset3D offset; + } array; + } op; +}; + +struct __device_builtin__ cudaMemcpy3DBatchOp { + struct cudaMemcpy3DOperand src; /**< Source memcpy operand. */ + struct cudaMemcpy3DOperand dst; /**< Destination memcpy operand. */ + struct cudaExtent extent; /**< Extents of the memcpy between src and dst. The width, height and depth components must not be 0.*/ + enum cudaMemcpySrcAccessOrder srcAccessOrder; /**< Source access ordering to be observed for copy from src to dst. */ + unsigned int flags; /**< Additional flags for copy from src to dst. See ::cudaMemcpyFlags. */ +}; + /** * CUDA device P2P attributes */ @@ -2174,8 +2410,45 @@ enum __device_builtin__ cudaDeviceP2PAttr { cudaDevP2PAttrAccessSupported = 2, /**< Peer access is enabled */ cudaDevP2PAttrNativeAtomicSupported = 3, /**< Native atomic operation over the link supported */ cudaDevP2PAttrCudaArrayAccessSupported = 4 /**< Accessing CUDA arrays over the link supported */ + , + cudaDevP2PAttrOnlyPartialNativeAtomicSupported = 5 /**< Only some CUDA-valid atomic operations over the link are supported. */ + }; +/** + * CUDA-valid Atomic Operations + */ +enum __device_builtin__ cudaAtomicOperation { + cudaAtomicOperationIntegerAdd = 0, + cudaAtomicOperationIntegerMin = 1, + cudaAtomicOperationIntegerMax = 2, + cudaAtomicOperationIntegerIncrement = 3, + cudaAtomicOperationIntegerDecrement = 4, + cudaAtomicOperationAnd = 5, + cudaAtomicOperationOr = 6, + cudaAtomicOperationXOR = 7, + cudaAtomicOperationExchange = 8, + cudaAtomicOperationCAS = 9, + cudaAtomicOperationFloatAdd = 10, + cudaAtomicOperationFloatMin = 11, + cudaAtomicOperationFloatMax = 12, +}; + +/** + * CUDA-valid Atomic Operation capabilities + */ +enum __device_builtin__ cudaAtomicOperationCapability { + cudaAtomicCapabilitySigned = 1u<<0, + cudaAtomicCapabilityUnsigned = 1u<<1, + cudaAtomicCapabilityReduction = 1u<<2, + cudaAtomicCapabilityScalar32 = 1u<<3, + cudaAtomicCapabilityScalar64 = 1u<<4, + cudaAtomicCapabilityScalar128 = 1u<<5, + cudaAtomicCapabilityVector32x4 = 1u<<6, +}; + + + /** * CUDA UUID types */ @@ -2205,21 +2478,16 @@ struct __device_builtin__ cudaDeviceProp int maxThreadsPerBlock; /**< Maximum number of threads per block */ int maxThreadsDim[3]; /**< Maximum size of each dimension of a block */ int maxGridSize[3]; /**< Maximum size of each dimension of a grid */ - int clockRate; /**< Deprecated, Clock frequency in kilohertz */ size_t totalConstMem; /**< Constant memory available on device in bytes */ int major; /**< Major compute capability */ int minor; /**< Minor compute capability */ size_t textureAlignment; /**< Alignment requirement for textures */ size_t texturePitchAlignment; /**< Pitch alignment requirement for texture references bound to pitched memory */ - int deviceOverlap; /**< Device can concurrently copy memory and execute a kernel. Deprecated. Use instead asyncEngineCount. */ int multiProcessorCount; /**< Number of multiprocessors on device */ - int kernelExecTimeoutEnabled; /**< Deprecated, Specified whether there is a run time limit on kernels */ int integrated; /**< Device is integrated as opposed to discrete */ int canMapHostMemory; /**< Device can map host memory with cudaHostAlloc/cudaHostGetDevicePointer */ - int computeMode; /**< Deprecated, Compute mode (See ::cudaComputeMode) */ int maxTexture1D; /**< Maximum 1D texture size */ int maxTexture1DMipmap; /**< Maximum 1D mipmapped texture size */ - int maxTexture1DLinear; /**< Deprecated, do not use. Use cudaDeviceGetTexture1DLinearMaxWidth() or cuDeviceGetTexture1DLinearMaxWidth() instead. */ int maxTexture2D[2]; /**< Maximum 2D texture dimensions */ int maxTexture2DMipmap[2]; /**< Maximum 2D mipmapped texture dimensions */ int maxTexture2DLinear[3]; /**< Maximum dimensions (width, height, pitch) for 2D textures bound to pitched memory */ @@ -2246,7 +2514,6 @@ struct __device_builtin__ cudaDeviceProp int tccDriver; /**< 1 if device is a Tesla device using TCC driver, 0 otherwise */ int asyncEngineCount; /**< Number of asynchronous engines */ int unifiedAddressing; /**< Device shares a unified address space with the host */ - int memoryClockRate; /**< Deprecated, Peak memory clock frequency in kilohertz */ int memoryBusWidth; /**< Global memory bus width in bits */ int l2CacheSize; /**< Size of L2 cache in bytes */ int persistingL2CacheMaxSize; /**< Device's maximum l2 persisting lines capacity setting in bytes */ @@ -2260,13 +2527,11 @@ struct __device_builtin__ cudaDeviceProp int isMultiGpuBoard; /**< Device is on a multi-GPU board */ int multiGpuBoardGroupID; /**< Unique identifier for a group of devices on the same multi-GPU board */ int hostNativeAtomicSupported; /**< Link between the device and the host supports native atomic operations */ - int singleToDoublePrecisionPerfRatio; /**< Deprecated, Ratio of single precision performance (in floating-point operations per second) to double precision performance */ int pageableMemoryAccess; /**< Device supports coherently accessing pageable memory without calling cudaHostRegister on it */ int concurrentManagedAccess; /**< Device can coherently access managed memory concurrently with the CPU */ int computePreemptionSupported; /**< Device supports Compute Preemption */ int canUseHostPointerForRegisteredMem; /**< Device can access host registered memory at the same virtual address as the CPU */ int cooperativeLaunch; /**< Device supports launching cooperative kernels via ::cudaLaunchCooperativeKernel */ - int cooperativeMultiDeviceLaunch; /**< Deprecated, cudaLaunchCooperativeKernelMultiDevice is deprecated. */ size_t sharedMemPerBlockOptin; /**< Per device maximum shared memory per block usable by special opt in */ int pageableMemoryAccessUsesHostPageTables; /**< Device accesses pageable memory via the host's page tables */ int directManagedMemAccessFromHost; /**< Host can directly access managed memory on the device without migration. */ @@ -2286,8 +2551,14 @@ struct __device_builtin__ cudaDeviceProp int ipcEventSupported; /**< Device supports IPC Events. */ int clusterLaunch; /**< Indicates device supports cluster launch */ int unifiedFunctionPointers; /**< Indicates device supports unified pointers */ - int reserved2[2]; - int reserved[61]; /**< Reserved for future use */ + int deviceNumaConfig; /**< NUMA configuration of a device: value is of type ::cudaDeviceNumaConfig enum */ + int deviceNumaId; /**< NUMA node ID of the GPU memory */ + int mpsEnabled; /**< Indicates if contexts created on this device will be shared via MPS */ + int hostNumaId; /**< NUMA ID of the host node closest to the device or -1 when system does not support NUMA */ + unsigned int gpuPciDeviceID; /**< The combined 16-bit PCI device ID and 16-bit PCI vendor ID */ + unsigned int gpuPciSubsystemID; /**< The combined 16-bit PCI subsystem ID and 16-bit PCI subsystem vendor ID */ + int hostNumaMultinodeIpcSupported; /**< 1 if the device supports HostNuma location IPC between nodes in a multi-node system. */ + int reserved[56]; /**< Reserved for future use */ }; /** @@ -2311,6 +2582,14 @@ typedef __device_builtin__ struct __device_builtin__ cudaIpcMemHandle_st char reserved[CUDA_IPC_HANDLE_SIZE]; }cudaIpcMemHandle_t; +/* + * CUDA Mem Fabric Handle + */ +typedef __device_builtin__ struct __device_builtin__ cudaMemFabricHandle_st +{ + char reserved[CUDA_IPC_HANDLE_SIZE]; +}cudaMemFabricHandle_t; + /** * External memory handle types */ @@ -2441,6 +2720,10 @@ struct __device_builtin__ cudaExternalMemoryHandleDesc { * Flags must either be zero or ::cudaExternalMemoryDedicated */ unsigned int flags; + /** + * Must be zero + */ + unsigned int reserved[16]; }; /** @@ -2459,6 +2742,10 @@ struct __device_builtin__ cudaExternalMemoryBufferDesc { * Flags reserved for future use. Must be zero. */ unsigned int flags; + /** + * Must be zero + */ + unsigned int reserved[16]; }; /** @@ -2487,6 +2774,10 @@ struct __device_builtin__ cudaExternalMemoryMipmappedArrayDesc { * Total number of levels in the mipmap chain */ unsigned int numLevels; + /** + * Must be zero + */ + unsigned int reserved[16]; }; /** @@ -2586,100 +2877,10 @@ struct __device_builtin__ cudaExternalSemaphoreHandleDesc { * Flags reserved for the future. Must be zero. */ unsigned int flags; -}; - -/** - * External semaphore signal parameters(deprecated) - */ -struct __device_builtin__ cudaExternalSemaphoreSignalParams_v1 { - struct { - /** - * Parameters for fence objects - */ - struct { - /** - * Value of fence to be signaled - */ - unsigned long long value; - } fence; - union { - /** - * Pointer to NvSciSyncFence. Valid if ::cudaExternalSemaphoreHandleType - * is of type ::cudaExternalSemaphoreHandleTypeNvSciSync. - */ - void *fence; - unsigned long long reserved; - } nvSciSync; - /** - * Parameters for keyed mutex objects - */ - struct { - /* - * Value of key to release the mutex with - */ - unsigned long long key; - } keyedMutex; - } params; - /** - * Only when ::cudaExternalSemaphoreSignalParams is used to - * signal a ::cudaExternalSemaphore_t of type - * ::cudaExternalSemaphoreHandleTypeNvSciSync, the valid flag is - * ::cudaExternalSemaphoreSignalSkipNvSciBufMemSync: which indicates - * that while signaling the ::cudaExternalSemaphore_t, no memory - * synchronization operations should be performed for any external memory - * object imported as ::cudaExternalMemoryHandleTypeNvSciBuf. - * For all other types of ::cudaExternalSemaphore_t, flags must be zero. - */ - unsigned int flags; -}; - -/** -* External semaphore wait parameters(deprecated) -*/ -struct __device_builtin__ cudaExternalSemaphoreWaitParams_v1 { - struct { - /** - * Parameters for fence objects - */ - struct { - /** - * Value of fence to be waited on - */ - unsigned long long value; - } fence; - union { - /** - * Pointer to NvSciSyncFence. Valid if ::cudaExternalSemaphoreHandleType - * is of type ::cudaExternalSemaphoreHandleTypeNvSciSync. - */ - void *fence; - unsigned long long reserved; - } nvSciSync; - /** - * Parameters for keyed mutex objects - */ - struct { - /** - * Value of key to acquire the mutex with - */ - unsigned long long key; - /** - * Timeout in milliseconds to wait to acquire the mutex - */ - unsigned int timeoutMs; - } keyedMutex; - } params; /** - * Only when ::cudaExternalSemaphoreSignalParams is used to - * signal a ::cudaExternalSemaphore_t of type - * ::cudaExternalSemaphoreHandleTypeNvSciSync, the valid flag is - * ::cudaExternalSemaphoreSignalSkipNvSciBufMemSync: which indicates - * that while waiting for the ::cudaExternalSemaphore_t, no memory - * synchronization operations should be performed for any external memory - * object imported as ::cudaExternalMemoryHandleTypeNvSciBuf. - * For all other types of ::cudaExternalSemaphore_t, flags must be zero. + * Must be zero */ - unsigned int flags; + unsigned int reserved[16]; }; /** @@ -2831,6 +3032,11 @@ typedef __device_builtin__ struct CUgraphNode_st *cudaGraphNode_t; */ typedef __device_builtin__ struct CUuserObject_st *cudaUserObject_t; +/** + * CUDA handle for conditional graph nodes + */ +typedef __device_builtin__ unsigned long long cudaGraphConditionalHandle; + /** * CUDA function */ @@ -2841,6 +3047,219 @@ typedef __device_builtin__ struct CUfunc_st *cudaFunction_t; */ typedef __device_builtin__ struct CUkern_st *cudaKernel_t; +/** + * Online compiler and linker options + */ +enum __device_builtin__ cudaJitOption +{ + /** + * Max number of registers that a thread may use.\n + * Option type: unsigned int\n + * Applies to: compiler only + */ + cudaJitMaxRegisters = 0, + + /** + * IN: Specifies minimum number of threads per block to target compilation + * for\n + * OUT: Returns the number of threads the compiler actually targeted. + * This restricts the resource utilization of the compiler (e.g. max + * registers) such that a block with the given number of threads should be + * able to launch based on register limitations. Note, this option does not + * currently take into account any other resource limitations, such as + * shared memory utilization.\n + * Option type: unsigned int\n + * Applies to: compiler only + */ + cudaJitThreadsPerBlock = 1, + + /** + * Overwrites the option value with the total wall clock time, in + * milliseconds, spent in the compiler and linker\n + * Option type: float\n + * Applies to: compiler and linker + */ + cudaJitWallTime = 2, + + /** + * Pointer to a buffer in which to print any log messages + * that are informational in nature (the buffer size is specified via + * option ::cudaJitInfoLogBufferSizeBytes)\n + * Option type: char *\n + * Applies to: compiler and linker + */ + cudaJitInfoLogBuffer = 3, + + /** + * IN: Log buffer size in bytes. Log messages will be capped at this size + * (including null terminator)\n + * OUT: Amount of log buffer filled with messages\n + * Option type: unsigned int\n + * Applies to: compiler and linker + */ + cudaJitInfoLogBufferSizeBytes = 4, + + /** + * Pointer to a buffer in which to print any log messages that + * reflect errors (the buffer size is specified via option + * ::cudaJitErrorLogBufferSizeBytes)\n + * Option type: char *\n + * Applies to: compiler and linker + */ + cudaJitErrorLogBuffer = 5, + + /** + * IN: Log buffer size in bytes. Log messages will be capped at this size + * (including null terminator)\n + * OUT: Amount of log buffer filled with messages\n + * Option type: unsigned int\n + * Applies to: compiler and linker + */ + cudaJitErrorLogBufferSizeBytes = 6, + + /** + * Level of optimizations to apply to generated code (0 - 4), with 4 + * being the default and highest level of optimizations.\n + * Option type: unsigned int\n + * Applies to: compiler only + */ + cudaJitOptimizationLevel = 7, + + /** + * Specifies choice of fallback strategy if matching cubin is not found. + * Choice is based on supplied ::cudaJit_Fallback. + * Option type: unsigned int for enumerated type ::cudaJit_Fallback\n + * Applies to: compiler only + */ + cudaJitFallbackStrategy = 10, + + /** + * Specifies whether to create debug information in output (-g) + * (0: false, default)\n + * Option type: int\n + * Applies to: compiler and linker + */ + cudaJitGenerateDebugInfo = 11, + + /** + * Generate verbose log messages (0: false, default)\n + * Option type: int\n + * Applies to: compiler and linker + */ + cudaJitLogVerbose = 12, + + /** + * Generate line number information (-lineinfo) (0: false, default)\n + * Option type: int\n + * Applies to: compiler only + */ + cudaJitGenerateLineInfo = 13, + + /** + * Specifies whether to enable caching explicitly (-dlcm) \n + * Choice is based on supplied ::cudaJit_CacheMode.\n + * Option type: unsigned int for enumerated type ::cudaJit_CacheMode\n + * Applies to: compiler only + */ + cudaJitCacheMode = 14, + + /** + * Generate position independent code (0: false)\n + * Option type: int\n + * Applies to: compiler only + */ + cudaJitPositionIndependentCode = 30, + + /** + * This option hints to the JIT compiler the minimum number of CTAs from the + * kernel’s grid to be mapped to a SM. This option is ignored when used together + * with ::cudaJitMaxRegisters or ::cudaJitThreadsPerBlock. + * Optimizations based on this option need ::cudaJitMaxThreadsPerBlock to + * be specified as well. For kernels already using PTX directive .minnctapersm, + * this option will be ignored by default. Use ::cudaJitOverrideDirectiveValues + * to let this option take precedence over the PTX directive. + * Option type: unsigned int\n + * Applies to: compiler only + */ + cudaJitMinCtaPerSm = 31, + + /** + * Maximum number threads in a thread block, computed as the product of + * the maximum extent specifed for each dimension of the block. This limit + * is guaranteed not to be exeeded in any invocation of the kernel. Exceeding + * the the maximum number of threads results in runtime error or kernel launch + * failure. For kernels already using PTX directive .maxntid, this option will + * be ignored by default. Use ::cudaJitOverrideDirectiveValues to let this + * option take precedence over the PTX directive. + * Option type: int\n + * Applies to: compiler only + */ + cudaJitMaxThreadsPerBlock = 32, + + /** + * This option lets the values specified using ::cudaJitMaxRegisters, + * ::cudaJitThreadsPerBlock, ::cudaJitMaxThreadsPerBlock and + * ::cudaJitMinCtaPerSm take precedence over any PTX directives. + * (0: Disable, default; 1: Enable) + * Option type: int\n + * Applies to: compiler only + */ + cudaJitOverrideDirectiveValues = 33, +}; + + +/** + * Library options to be specified with ::cudaLibraryLoadData() or ::cudaLibraryLoadFromFile() + */ +enum __device_builtin__ cudaLibraryOption +{ + cudaLibraryHostUniversalFunctionAndDataTable = 0, + + /** + * Specifes that the argument \p code passed to ::cudaLibraryLoadData() will be preserved. + * Specifying this option will let the driver know that \p code can be accessed at any point + * until ::cudaLibraryUnload(). The default behavior is for the driver to allocate and + * maintain its own copy of \p code. Note that this is only a memory usage optimization + * hint and the driver can choose to ignore it if required. + * Specifying this option with ::cudaLibraryLoadFromFile() is invalid and + * will return ::cudaErrorInvalidValue. + */ + cudaLibraryBinaryIsPreserved = 1, +}; + +struct __device_builtin__ cudalibraryHostUniversalFunctionAndDataTable +{ + void *functionTable; + size_t functionWindowSize; + void *dataTable; + size_t dataWindowSize; +}; + +/** + * Caching modes for dlcm + */ +enum __device_builtin__ cudaJit_CacheMode +{ + cudaJitCacheOptionNone = 0, /**< Compile with no -dlcm flag specified */ + cudaJitCacheOptionCG, /**< Compile with L1 cache disabled */ + cudaJitCacheOptionCA /**< Compile with L1 cache enabled */ +}; + +/** + * Cubin matching fallback strategies + */ +enum __device_builtin__ cudaJit_Fallback +{ + cudaPreferPtx = 0, /**< Prefer to compile ptx if exact binary match not found */ + + cudaPreferBinary /**< Prefer to fall back to compatible binary code if exact match not found */ +}; + +/** + * CUDA library + */ +typedef __device_builtin__ struct CUlib_st *cudaLibrary_t; + /** * CUDA memory pool */ @@ -2852,29 +3271,34 @@ typedef __device_builtin__ struct CUmemPoolHandle_st *cudaMemPool_t; enum __device_builtin__ cudaCGScope { cudaCGScopeInvalid = 0, /**< Invalid cooperative group scope */ cudaCGScopeGrid = 1, /**< Scope represented by a grid_group */ - cudaCGScopeMultiGrid = 2 /**< Scope represented by a multi_grid_group */ + cudaCGScopeReserved = 2 /**< Reserved */ }; /** - * CUDA launch parameters + * CUDA GPU kernel node parameters */ -struct __device_builtin__ cudaLaunchParams -{ - void *func; /**< Device function symbol */ - dim3 gridDim; /**< Grid dimentions */ - dim3 blockDim; /**< Block dimentions */ - void **args; /**< Arguments */ - size_t sharedMem; /**< Shared memory */ - cudaStream_t stream; /**< Stream identifier */ +struct __device_builtin__ cudaKernelNodeParams { + void* func; /**< Kernel to launch */ + dim3 gridDim; /**< Grid dimensions */ + dim3 blockDim; /**< Block dimensions */ + unsigned int sharedMemBytes; /**< Dynamic shared-memory size per thread block in bytes */ + void **kernelParams; /**< Array of pointers to individual kernel arguments*/ + void **extra; /**< Pointer to kernel arguments in the "extra" format */ }; /** * CUDA GPU kernel node parameters */ -struct __device_builtin__ cudaKernelNodeParams { +struct __device_builtin__ cudaKernelNodeParamsV2 { void* func; /**< Kernel to launch */ - dim3 gridDim; /**< Grid dimensions */ - dim3 blockDim; /**< Block dimensions */ + #if !defined(__cplusplus) || __cplusplus >= 201103L + dim3 gridDim; /**< Grid dimensions */ + dim3 blockDim; /**< Block dimensions */ + #else + /* Union members cannot have nontrivial constructors until C++11. */ + uint3 gridDim; /**< Grid dimensions */ + uint3 blockDim; /**< Block dimensions */ + #endif unsigned int sharedMemBytes; /**< Dynamic shared-memory size per thread block in bytes */ void **kernelParams; /**< Array of pointers to individual kernel arguments*/ void **extra; /**< Pointer to kernel arguments in the "extra" format */ @@ -2889,6 +3313,15 @@ struct __device_builtin__ cudaExternalSemaphoreSignalNodeParams { unsigned int numExtSems; /**< Number of handles and parameters supplied in extSemArray and paramsArray. */ }; +/** + * External semaphore signal node parameters + */ +struct __device_builtin__ cudaExternalSemaphoreSignalNodeParamsV2 { + cudaExternalSemaphore_t* extSemArray; /**< Array of external semaphore handles. */ + const struct cudaExternalSemaphoreSignalParams* paramsArray; /**< Array of external semaphore signal parameters. */ + unsigned int numExtSems; /**< Number of handles and parameters supplied in extSemArray and paramsArray. */ +}; + /** * External semaphore wait node parameters */ @@ -2898,6 +3331,59 @@ struct __device_builtin__ cudaExternalSemaphoreWaitNodeParams { unsigned int numExtSems; /**< Number of handles and parameters supplied in extSemArray and paramsArray. */ }; +/** + * External semaphore wait node parameters + */ +struct __device_builtin__ cudaExternalSemaphoreWaitNodeParamsV2 { + cudaExternalSemaphore_t* extSemArray; /**< Array of external semaphore handles. */ + const struct cudaExternalSemaphoreWaitParams* paramsArray; /**< Array of external semaphore wait parameters. */ + unsigned int numExtSems; /**< Number of handles and parameters supplied in extSemArray and paramsArray. */ +}; + +enum __device_builtin__ cudaGraphConditionalHandleFlags { + cudaGraphCondAssignDefault = 1 /**< Apply default handle value when graph is launched. */ +}; + +/** + * CUDA conditional node types + */ +enum __device_builtin__ cudaGraphConditionalNodeType { + cudaGraphCondTypeIf = 0, /**< Conditional 'if/else' Node. Body[0] executed if condition is non-zero. If \p size == 2, an optional ELSE graph is created and this is executed if the condition is zero. */ + cudaGraphCondTypeWhile = 1, /**< Conditional 'while' Node. Body executed repeatedly while condition value is non-zero. */ + cudaGraphCondTypeSwitch = 2, /**< Conditional 'switch' Node. Body[n] is executed once, where 'n' is the value of the condition. If the condition does not match a body index, no body is launched. */ +}; + +/** + * CUDA conditional node parameters + */ +struct __device_builtin__ cudaConditionalNodeParams { + cudaGraphConditionalHandle handle; /**< Conditional node handle. + Handles must be created in advance of creating the node + using ::cudaGraphConditionalHandleCreate. */ + enum cudaGraphConditionalNodeType type; /**< Type of conditional node. */ + unsigned int size; /**< Size of graph output array. Allowed values are 1 for cudaGraphCondTypeWhile, 1 or 2 + for cudaGraphCondTypeIf, or any value greater than zero for cudaGraphCondTypeSwitch. */ + cudaGraph_t *phGraph_out; /**< CUDA-owned array populated with conditional node child graphs during creation of the node. + Valid for the lifetime of the conditional node. + The contents of the graph(s) are subject to the following constraints: + + - Allowed node types are kernel nodes, empty nodes, child graphs, memsets, + memcopies, and conditionals. This applies recursively to child graphs and conditional bodies. + - All kernels, including kernels in nested conditionals or child graphs at any level, + must belong to the same CUDA context. + + These graphs may be populated using graph node creation APIs or ::cudaStreamBeginCaptureToGraph. + cudaGraphCondTypeIf: + phGraph_out[0] is executed when the condition is non-zero. If \p size == 2, phGraph_out[1] will + be executed when the condition is zero. + cudaGraphCondTypeWhile: + phGraph_out[0] is executed as long as the condition is non-zero. + cudaGraphCondTypeSwitch: + phGraph_out[n] is executed when the condition is equal to n. If the condition >= \p size, + no body graph is executed. + */ +}; + /** * CUDA Graph node types */ @@ -2914,9 +3400,155 @@ enum __device_builtin__ cudaGraphNodeType { cudaGraphNodeTypeExtSemaphoreWait = 0x09, /**< External semaphore wait node */ cudaGraphNodeTypeMemAlloc = 0x0a, /**< Memory allocation node */ cudaGraphNodeTypeMemFree = 0x0b, /**< Memory free node */ + cudaGraphNodeTypeConditional = 0x0d, /**< Conditional node + + May be used to implement a conditional execution path or loop + inside of a graph. The graph(s) contained within the body of the conditional node + can be selectively executed or iterated upon based on the value of a conditional + variable. + + Handles must be created in advance of creating the node + using ::cudaGraphConditionalHandleCreate. + + The following restrictions apply to graphs which contain conditional nodes: + The graph cannot be used in a child node. + Only one instantiation of the graph may exist at any point in time. + The graph cannot be cloned. + + To set the control value, supply a default value when creating the handle and/or + call ::cudaGraphSetConditional from device code.*/ cudaGraphNodeTypeCount }; +/** + * Child graph node ownership + */ +enum __device_builtin__ cudaGraphChildGraphNodeOwnership { + cudaGraphChildGraphOwnershipClone = 0, /**< Default behavior for a child graph node. Child graph is cloned + into the parent and memory allocation/free nodes can't be present + in the child graph. */ + cudaGraphChildGraphOwnershipMove = 1, /**< The child graph is moved to the parent. The handle to the child graph + is owned by the parent and will be destroyed when the parent is + destroyed. + + The following restrictions apply to child graphs after they have been moved: + Cannot be independently instantiated or destroyed; + Cannot be added as a child graph of a separate parent graph; + Cannot be used as an argument to cudaGraphExecUpdate; + Cannot have additional memory allocation or free nodes added. */ +}; + +/** + * Child graph node parameters + */ +struct __device_builtin__ cudaChildGraphNodeParams { + cudaGraph_t graph; /**< The child graph to clone into the node for node creation, or + * a handle to the graph owned by the node for node query. + * The graph must not contain conditional nodes. Graphs + * containing memory allocation or memory free nodes must + * set the ownership to be moved to the parent. + */ + enum cudaGraphChildGraphNodeOwnership ownership; /**< The ownership relationship of the child graph node. */ +}; + +/** + * Event record node parameters + */ +struct __device_builtin__ cudaEventRecordNodeParams { + cudaEvent_t event; /**< The event to record when the node executes */ +}; + +/** + * Event wait node parameters + */ +struct __device_builtin__ cudaEventWaitNodeParams { + cudaEvent_t event; /**< The event to wait on from the node */ +}; + +/** + * Graph node parameters. See ::cudaGraphAddNode. + */ +struct __device_builtin__ cudaGraphNodeParams { + enum cudaGraphNodeType type; /**< Type of the node */ + int reserved0[3]; /**< Reserved. Must be zero. */ + + union { + long long reserved1[29]; /**< Padding. Unused bytes must be zero. */ + struct cudaKernelNodeParamsV2 kernel; /**< Kernel node parameters. */ + struct cudaMemcpyNodeParams memcpy; /**< Memcpy node parameters. */ + struct cudaMemsetParamsV2 memset; /**< Memset node parameters. */ + struct cudaHostNodeParamsV2 host; /**< Host node parameters. */ + struct cudaChildGraphNodeParams graph; /**< Child graph node parameters. */ + struct cudaEventWaitNodeParams eventWait; /**< Event wait node parameters. */ + struct cudaEventRecordNodeParams eventRecord; /**< Event record node parameters. */ + struct cudaExternalSemaphoreSignalNodeParamsV2 extSemSignal; /**< External semaphore signal node parameters. */ + struct cudaExternalSemaphoreWaitNodeParamsV2 extSemWait; /**< External semaphore wait node parameters. */ + struct cudaMemAllocNodeParamsV2 alloc; /**< Memory allocation node parameters. */ + struct cudaMemFreeNodeParams free; /**< Memory free node parameters. */ + struct cudaConditionalNodeParams conditional; /**< Conditional node parameters. */ + }; + + long long reserved2; /**< Reserved bytes. Must be zero. */ +}; + +/** + * Type annotations that can be applied to graph edges as part of ::cudaGraphEdgeData. + */ +typedef __device_builtin__ enum cudaGraphDependencyType_enum { + cudaGraphDependencyTypeDefault = 0, /**< This is an ordinary dependency. */ + cudaGraphDependencyTypeProgrammatic = 1 /**< This dependency type allows the downstream node to + use \c cudaGridDependencySynchronize(). It may only be used + between kernel nodes, and must be used with either the + ::cudaGraphKernelNodePortProgrammatic or + ::cudaGraphKernelNodePortLaunchCompletion outgoing port. */ +} cudaGraphDependencyType; + +/** + * Optional annotation for edges in a CUDA graph. Note, all edges implicitly have annotations and + * default to a zero-initialized value if not specified. A zero-initialized struct indicates a + * standard full serialization of two nodes with memory visibility. + */ +typedef __device_builtin__ struct cudaGraphEdgeData_st { + unsigned char from_port; /**< This indicates when the dependency is triggered from the upstream + node on the edge. The meaning is specfic to the node type. A value + of 0 in all cases means full completion of the upstream node, with + memory visibility to the downstream node or portion thereof + (indicated by \c to_port). +
    + Only kernel nodes define non-zero ports. A kernel node + can use the following output port types: + ::cudaGraphKernelNodePortDefault, ::cudaGraphKernelNodePortProgrammatic, + or ::cudaGraphKernelNodePortLaunchCompletion. */ + unsigned char to_port; /**< This indicates what portion of the downstream node is dependent on + the upstream node or portion thereof (indicated by \c from_port). The + meaning is specific to the node type. A value of 0 in all cases means + the entirety of the downstream node is dependent on the upstream work. +
    + Currently no node types define non-zero ports. Accordingly, this field + must be set to zero. */ + unsigned char type; /**< This should be populated with a value from ::cudaGraphDependencyType. (It + is typed as char due to compiler-specific layout of bitfields.) See + ::cudaGraphDependencyType. */ + unsigned char reserved[5]; /**< These bytes are unused and must be zeroed. This ensures + compatibility if additional fields are added in the future. */ +} cudaGraphEdgeData; + +/** + * This port activates when the kernel has finished executing. + */ +#define cudaGraphKernelNodePortDefault 0 +/** + * This port activates when all blocks of the kernel have performed cudaTriggerProgrammaticLaunchCompletion() + * or have terminated. It must be used with edge type ::cudaGraphDependencyTypeProgrammatic. See also + * ::cudaLaunchAttributeProgrammaticEvent. + */ +#define cudaGraphKernelNodePortProgrammatic 1 +/** + * This port activates when all blocks of the kernel have begun execution. See also + * ::cudaLaunchAttributeLaunchCompletionEvent. + */ +#define cudaGraphKernelNodePortLaunchCompletion 2 + /** * CUDA executable (launchable) graph */ @@ -2945,7 +3577,8 @@ typedef __device_builtin__ enum cudaGraphInstantiateResult { cudaGraphInstantiateError = 1, /**< Instantiation failed for an unexpected reason which is described in the return value of the function */ cudaGraphInstantiateInvalidStructure = 2, /**< Instantiation failed due to invalid structure, such as cycles */ cudaGraphInstantiateNodeOperationNotSupported = 3, /**< Instantiation for device launch failed because the graph contained an unsupported operation */ - cudaGraphInstantiateMultipleDevicesNotSupported = 4 /**< Instantiation for device launch failed due to the nodes belonging to different contexts */ + cudaGraphInstantiateMultipleDevicesNotSupported = 4, /**< Instantiation for device launch failed due to the nodes belonging to different contexts */ + cudaGraphInstantiateConditionalHandleUnused = 5 /**< One or more conditional handles are not associated with conditional nodes */ } cudaGraphInstantiateResult; /** @@ -2981,6 +3614,44 @@ typedef __device_builtin__ struct cudaGraphExecUpdateResultInfo_st { cudaGraphNode_t errorFromNode; } cudaGraphExecUpdateResultInfo; +/** + * CUDA device node handle for device-side node update + */ +typedef struct CUgraphDeviceUpdatableNode_st* cudaGraphDeviceNode_t; + +/** + * Specifies the field to update when performing multiple node updates from the device + */ +enum __device_builtin__ cudaGraphKernelNodeField +{ + cudaGraphKernelNodeFieldInvalid = 0, /**< Invalid field */ + cudaGraphKernelNodeFieldGridDim, /**< Grid dimension update */ + cudaGraphKernelNodeFieldParam, /**< Kernel parameter update */ + cudaGraphKernelNodeFieldEnabled /**< Node enable/disable */ +}; + +/** + * Struct to specify a single node update to pass as part of a larger array to ::cudaGraphKernelNodeUpdatesApply + */ +struct __device_builtin__ cudaGraphKernelNodeUpdate { + cudaGraphDeviceNode_t node; /**< Node to update */ + enum cudaGraphKernelNodeField field; /**< Which type of update to apply. Determines how updateData is interpreted */ + union { +#if !defined(__cplusplus) || __cplusplus >= 201103L + dim3 gridDim; /**< Grid dimensions */ +#else + /* Union members cannot have nontrivial constructors until C++11. */ + uint3 gridDim; /**< Grid dimensions */ +#endif + struct { + const void *pValue; /**< Kernel parameter data to write in */ + size_t offset; /**< Offset into the parameter buffer at which to apply the update */ + size_t size; /**< Number of bytes to update */ + } param; /**< Kernel parameter data */ + unsigned int isEnabled; /**< Node enable/disable data. Nonzero if the node should be enabled, 0 if it should be disabled */ + } updateData; /**< Update data to apply. Which field is used depends on field's value */ +}; + /** * Flags to specify search options to be used with ::cudaGetDriverEntryPoint * For more details see ::cuGetProcAddress @@ -3013,7 +3684,8 @@ enum __device_builtin__ cudaGraphDebugDotFlags { cudaGraphDebugDotFlagsExtSemasSignalNodeParams = 1<<7, /**< Adds cudaExternalSemaphoreSignalNodeParams values to output */ cudaGraphDebugDotFlagsExtSemasWaitNodeParams = 1<<8, /**< Adds cudaExternalSemaphoreWaitNodeParams to output */ cudaGraphDebugDotFlagsKernelNodeAttributes = 1<<9, /**< Adds cudaKernelNodeAttrID values to output */ - cudaGraphDebugDotFlagsHandles = 1<<10 /**< Adds node handles and every kernel function handle to output */ + cudaGraphDebugDotFlagsHandles = 1<<10, /**< Adds node handles and every kernel function handle to output */ + cudaGraphDebugDotFlagsConditionalNodeParams = 1<<15, /**< Adds cudaConditionalNodeParams to output */ }; /** @@ -3021,20 +3693,51 @@ enum __device_builtin__ cudaGraphDebugDotFlags { */ enum __device_builtin__ cudaGraphInstantiateFlags { cudaGraphInstantiateFlagAutoFreeOnLaunch = 1 /**< Automatically free memory allocated in a graph before relaunching. */ - , cudaGraphInstantiateFlagUpload = 2 /**< Automatically upload the graph after instantiaton. */ - , cudaGraphInstantiateFlagDeviceLaunch = 4 /**< Instantiate the graph to be launchable from the device. */ + , cudaGraphInstantiateFlagUpload = 2 /**< Automatically upload the graph after instantiation. Only supported by + ::cudaGraphInstantiateWithParams. The upload will be performed using the + stream provided in \p instantiateParams. */ + , cudaGraphInstantiateFlagDeviceLaunch = 4 /**< Instantiate the graph to be launchable from the device. This flag can only + be used on platforms which support unified addressing. This flag cannot be + used in conjunction with cudaGraphInstantiateFlagAutoFreeOnLaunch. */ , cudaGraphInstantiateFlagUseNodePriority = 8 /**< Run the graph using the per-node priority attributes rather than the priority of the stream it is launched into. */ }; +/** + * Memory Synchronization Domain + * + * A kernel can be launched in a specified memory synchronization domain that affects all memory operations issued by + * that kernel. A memory barrier issued in one domain will only order memory operations in that domain, thus eliminating + * latency increase from memory barriers ordering unrelated traffic. + * + * By default, kernels are launched in domain 0. Kernel launched with ::cudaLaunchMemSyncDomainRemote will have a + * different domain ID. User may also alter the domain ID with ::cudaLaunchMemSyncDomainMap for a specific stream / + * graph node / kernel launch. See ::cudaLaunchAttributeMemSyncDomain, ::cudaStreamSetAttribute, ::cudaLaunchKernelEx, + * ::cudaGraphKernelNodeSetAttribute. + * + * Memory operations done in kernels launched in different domains are considered system-scope distanced. In other + * words, a GPU scoped memory synchronization is not sufficient for memory order to be observed by kernels in another + * memory synchronization domain even if they are on the same GPU. + */ typedef __device_builtin__ enum cudaLaunchMemSyncDomain { - cudaLaunchMemSyncDomainDefault = 0, - cudaLaunchMemSyncDomainRemote = 1 + cudaLaunchMemSyncDomainDefault = 0, /**< Launch kernels in the default domain */ + cudaLaunchMemSyncDomainRemote = 1 /**< Launch kernels in the remote domain */ } cudaLaunchMemSyncDomain; +/** + * Memory Synchronization Domain map + * + * See ::cudaLaunchMemSyncDomain. + * + * By default, kernels are launched in domain 0. Kernel launched with ::cudaLaunchMemSyncDomainRemote will have a + * different domain ID. User may also alter the domain ID with ::cudaLaunchMemSyncDomainMap for a specific stream / + * graph node / kernel launch. See ::cudaLaunchAttributeMemSyncDomainMap. + * + * Domain ID range is available through ::cudaDevAttrMemSyncDomainCount. + */ typedef __device_builtin__ struct cudaLaunchMemSyncDomainMap_st { - unsigned char default_; - unsigned char remote; + unsigned char default_; /**< The default domain ID to use for designated kernels */ + unsigned char remote; /**< The remote domain ID to use for designated kernels */ } cudaLaunchMemSyncDomainMap; /** @@ -3042,26 +3745,32 @@ typedef __device_builtin__ struct cudaLaunchMemSyncDomainMap_st { */ typedef __device_builtin__ enum cudaLaunchAttributeID { cudaLaunchAttributeIgnore = 0 /**< Ignored entry, for convenient composition */ - , cudaLaunchAttributeAccessPolicyWindow = 1 /**< Valid for streams, graph nodes, launches. */ - , cudaLaunchAttributeCooperative = 2 /**< Valid for graph nodes, launches. */ - , cudaLaunchAttributeSynchronizationPolicy = 3 /**< Valid for streams. */ - , cudaLaunchAttributeClusterDimension = 4 /**< Valid for graph nodes, launches. */ - , cudaLaunchAttributeClusterSchedulingPolicyPreference = 5 /**< Valid for graph nodes, launches. */ + , cudaLaunchAttributeAccessPolicyWindow = 1 /**< Valid for streams, graph nodes, launches. See + ::cudaLaunchAttributeValue::accessPolicyWindow. */ + , cudaLaunchAttributeCooperative = 2 /**< Valid for graph nodes, launches. See + ::cudaLaunchAttributeValue::cooperative. */ + , cudaLaunchAttributeSynchronizationPolicy = 3 /**< Valid for streams. See ::cudaLaunchAttributeValue::syncPolicy. */ + , cudaLaunchAttributeClusterDimension = 4 /**< Valid for graph nodes, launches. See + ::cudaLaunchAttributeValue::clusterDim. */ + , cudaLaunchAttributeClusterSchedulingPolicyPreference = 5 /**< Valid for graph nodes, launches. See + ::cudaLaunchAttributeValue::clusterSchedulingPolicyPreference. */ , cudaLaunchAttributeProgrammaticStreamSerialization = 6 /**< Valid for launches. Setting - programmaticStreamSerializationAllowed to non-0 - signals that the kernel will use programmatic - means to resolve its stream dependency, so that - the CUDA runtime should opportunistically allow - the grid's execution to overlap with the previous - kernel in the stream, if that kernel requests the - overlap. The dependent launches can choose to wait on - the dependency using the programmatic sync + ::cudaLaunchAttributeValue::programmaticStreamSerializationAllowed + to non-0 signals that the kernel will use programmatic + means to resolve its stream dependency, so that the + CUDA runtime should opportunistically allow the grid's + execution to overlap with the previous kernel in the + stream, if that kernel requests the overlap. The + dependent launches can choose to wait on the + dependency using the programmatic sync (cudaGridDependencySynchronize() or equivalent PTX instructions). */ - , cudaLaunchAttributeProgrammaticEvent = 7 /**< Valid for launches. Event recorded through this - launch attribute is guaranteed to only trigger after - all block in the associated kernel trigger the event. - A block can trigger the event programmatically in a + , cudaLaunchAttributeProgrammaticEvent = 7 /**< Valid for launches. Set + ::cudaLaunchAttributeValue::programmaticEvent to + record the event. Event recorded through this launch + attribute is guaranteed to only trigger after all + block in the associated kernel trigger the event. A + block can trigger the event programmatically in a future CUDA release. A trigger can also be inserted at the beginning of each block's execution if triggerAtBlockStart is set to non-0. The dependent @@ -3075,12 +3784,125 @@ typedef __device_builtin__ enum cudaLaunchAttributeID { event trigger long after the associated kernel has completed. This recording type is primarily meant for establishing programmatic dependency between device - tasks. The event supplied must not be an interprocess - or interop event. The event must disable timing (i.e. - created with ::cudaEventDisableTiming flag set). */ - , cudaLaunchAttributePriority = 8 /**< Valid for streams, graph nodes, launches. */ - , cudaLaunchAttributeMemSyncDomainMap = 9 - , cudaLaunchAttributeMemSyncDomain = 10 + tasks. Note also this type of dependency allows, but + does not guarantee, concurrent execution of tasks. +
    + The event supplied must not be an interprocess or + interop event. The event must disable timing (i.e. + must be created with the ::cudaEventDisableTiming flag + set). */ + , cudaLaunchAttributePriority = 8 /**< Valid for streams, graph nodes, launches. See + ::cudaLaunchAttributeValue::priority. */ + , cudaLaunchAttributeMemSyncDomainMap = 9 /**< Valid for streams, graph nodes, launches. See + ::cudaLaunchAttributeValue::memSyncDomainMap. */ + , cudaLaunchAttributeMemSyncDomain = 10 /**< Valid for streams, graph nodes, launches. See + ::cudaLaunchAttributeValue::memSyncDomain. */ + , cudaLaunchAttributePreferredClusterDimension = 11 /**< Valid for graph nodes and launches. Set + ::cudaLaunchAttributeValue::preferredClusterDim + to allow the kernel launch to specify a preferred substitute + cluster dimension. Blocks may be grouped according to either + the dimensions specified with this attribute (grouped into a + "preferred substitute cluster"), or the one specified with + ::cudaLaunchAttributeClusterDimension attribute (grouped + into a "regular cluster"). The cluster dimensions of a + "preferred substitute cluster" shall be an integer multiple + greater than zero of the regular cluster dimensions. The + device will attempt - on a best-effort basis - to group + thread blocks into preferred clusters over grouping them + into regular clusters. When it deems necessary (primarily + when the device temporarily runs out of physical resources + to launch the larger preferred clusters), the device may + switch to launch the regular clusters instead to attempt to + utilize as much of the physical device resources as possible. +
    + Each type of cluster will have its enumeration / coordinate + setup as if the grid consists solely of its type of cluster. + For example, if the preferred substitute cluster dimensions + double the regular cluster dimensions, there might be + simultaneously a regular cluster indexed at (1,0,0), and a + preferred cluster indexed at (1,0,0). In this example, the + preferred substitute cluster (1,0,0) replaces regular + clusters (2,0,0) and (3,0,0) and groups their blocks. +
    + This attribute will only take effect when a regular cluster + dimension has been specified. The preferred substitute cluster + dimension must be an integer multiple greater than zero of the + regular cluster dimension and must divide the grid. It must + also be no more than `maxBlocksPerCluster`, if it is set in + the kernel's `__launch_bounds__`. Otherwise it must be less + than the maximum value the driver can support. Otherwise, + setting this attribute to a value physically unable to fit on + any particular device is permitted. */ + , cudaLaunchAttributeLaunchCompletionEvent = 12 /**< Valid for launches. Set + ::cudaLaunchAttributeValue::launchCompletionEvent to record the + event. +
    + Nominally, the event is triggered once all blocks of the kernel + have begun execution. Currently this is a best effort. If a kernel + B has a launch completion dependency on a kernel A, B may wait + until A is complete. Alternatively, blocks of B may begin before + all blocks of A have begun, for example if B can claim execution + resources unavailable to A (e.g. they run on different GPUs) or + if B is a higher priority than A. + Exercise caution if such an ordering inversion could lead + to deadlock. +
    + A launch completion event is nominally similar to a programmatic + event with \c triggerAtBlockStart set except that it is not + visible to \c cudaGridDependencySynchronize() and can be used with + compute capability less than 9.0. +
    + The event supplied must not be an interprocess or interop event. + The event must disable timing (i.e. must be created with the + ::cudaEventDisableTiming flag set). */ + , cudaLaunchAttributeDeviceUpdatableKernelNode = 13 /**< Valid for graph nodes, launches. This attribute is graphs-only, + and passing it to a launch in a non-capturing stream will result + in an error. +
    + :cudaLaunchAttributeValue::deviceUpdatableKernelNode::deviceUpdatable can + only be set to 0 or 1. Setting the field to 1 indicates that the + corresponding kernel node should be device-updatable. On success, a handle + will be returned via + ::cudaLaunchAttributeValue::deviceUpdatableKernelNode::devNode which can be + passed to the various device-side update functions to update the node's + kernel parameters from within another kernel. For more information on the + types of device updates that can be made, as well as the relevant limitations + thereof, see ::cudaGraphKernelNodeUpdatesApply. +
    + Nodes which are device-updatable have additional restrictions compared to + regular kernel nodes. Firstly, device-updatable nodes cannot be removed + from their graph via ::cudaGraphDestroyNode. Additionally, once opted-in + to this functionality, a node cannot opt out, and any attempt to set the + deviceUpdatable attribute to 0 will result in an error. Device-updatable + kernel nodes also cannot have their attributes copied to/from another kernel + node via ::cudaGraphKernelNodeCopyAttributes. Graphs containing one or more + device-updatable nodes also do not allow multiple instantiation, and neither + the graph nor its instantiated version can be passed to ::cudaGraphExecUpdate. +
    + If a graph contains device-updatable nodes and updates those nodes from the device + from within the graph, the graph must be uploaded with ::cuGraphUpload before it + is launched. For such a graph, if host-side executable graph updates are made to the + device-updatable nodes, the graph must be uploaded before it is launched again. */ + , cudaLaunchAttributePreferredSharedMemoryCarveout = 14 /**< Valid for launches. On devices where the L1 cache and shared memory use the + same hardware resources, setting ::cudaLaunchAttributeValue::sharedMemCarveout + to a percentage between 0-100 signals sets the shared memory carveout + preference in percent of the total shared memory for that kernel launch. + This attribute takes precedence over ::cudaFuncAttributePreferredSharedMemoryCarveout. + This is only a hint, and the driver can choose a different configuration if + required for the launch.*/ + , cudaLaunchAttributeNvlinkUtilCentricScheduling = 16 /**< Valid for streams, graph nodes, launches. This attribute is a hint to the CUDA runtime that the + launch should attempt to make the kernel maximize its NVLINK utilization. +
    + When possible to honor this hint, CUDA will assume each block in the grid launch will carry out an even amount + of NVLINK traffic, and make a best-effort attempt to adjust the kernel launch based on that assumption. +
    + This attribute is a hint only. CUDA makes no functional or performance guarantee. Its applicability can be + affected by many different factors, including driver version (i.e. CUDA doesn't guarantee the performance + characteristics will be maintained between driver versions or a driver update could alter or regress + previously observed perf characteristics.) It also doesn't guarantee a successful result, i.e. applying + the attribute may not improve the performance of either the targeted kernel or the encapsulating application. +
    + Valid values for ::cudaLaunchAttributeValue::nvlinkUtilCentricScheduling are 0 (disabled) and 1 (enabled). */ } cudaLaunchAttributeID; /** @@ -3088,33 +3910,106 @@ typedef __device_builtin__ enum cudaLaunchAttributeID { */ typedef __device_builtin__ union cudaLaunchAttributeValue { char pad[64]; /* Pad to 64 bytes */ - struct cudaAccessPolicyWindow accessPolicyWindow; /**< Attribute ::cudaAccessPolicyWindow. */ - int cooperative; /**< Nonzero indicates a cooperative kernel (see ::cudaLaunchCooperativeKernel). */ - enum cudaSynchronizationPolicy syncPolicy; /**< ::cudaSynchronizationPolicy for work queued up in this stream */ + struct cudaAccessPolicyWindow accessPolicyWindow; /**< Value of launch attribute ::cudaLaunchAttributeAccessPolicyWindow. */ + int cooperative; /**< Value of launch attribute ::cudaLaunchAttributeCooperative. Nonzero indicates a cooperative + kernel (see ::cudaLaunchCooperativeKernel). */ + enum cudaSynchronizationPolicy syncPolicy; /**< Value of launch attribute + ::cudaLaunchAttributeSynchronizationPolicy. ::cudaSynchronizationPolicy + for work queued up in this stream. */ + /** + * Value of launch attribute ::cudaLaunchAttributeClusterDimension that + * represents the desired cluster dimensions for the kernel. Opaque type + * with the following fields: + * - \p x - The X dimension of the cluster, in blocks. Must be a divisor + * of the grid X dimension. + * - \p y - The Y dimension of the cluster, in blocks. Must be a divisor + * of the grid Y dimension. + * - \p z - The Z dimension of the cluster, in blocks. Must be a divisor + * of the grid Z dimension. + */ struct { unsigned int x; unsigned int y; unsigned int z; - } clusterDim; /**< Cluster dimensions for the kernel node. */ - enum cudaClusterSchedulingPolicy clusterSchedulingPolicyPreference; /**< Cluster scheduling policy preference for the kernel node. */ - int programmaticStreamSerializationAllowed; + } clusterDim; + enum cudaClusterSchedulingPolicy clusterSchedulingPolicyPreference; /**< Value of launch attribute + ::cudaLaunchAttributeClusterSchedulingPolicyPreference. Cluster + scheduling policy preference for the kernel. */ + int programmaticStreamSerializationAllowed; /**< Value of launch attribute + ::cudaLaunchAttributeProgrammaticStreamSerialization. */ + + /** + * Value of launch attribute ::cudaLaunchAttributeProgrammaticEvent + * with the following fields: + * - \p cudaEvent_t event - Event to fire when all blocks trigger it. + * - \p int flags; - Event record flags, see ::cudaEventRecordWithFlags. Does not accept + * ::cudaEventRecordExternal. + * - \p int triggerAtBlockStart - If this is set to non-0, each block launch will automatically trigger the event. + */ struct { cudaEvent_t event; int flags; int triggerAtBlockStart; } programmaticEvent; - int priority; /**< Execution priority of the kernel. */ - cudaLaunchMemSyncDomainMap memSyncDomainMap; - cudaLaunchMemSyncDomain memSyncDomain; + int priority; /**< Value of launch attribute ::cudaLaunchAttributePriority. Execution priority of the kernel. */ + cudaLaunchMemSyncDomainMap memSyncDomainMap; /**< Value of launch attribute + ::cudaLaunchAttributeMemSyncDomainMap. See + ::cudaLaunchMemSyncDomainMap. */ + cudaLaunchMemSyncDomain memSyncDomain; /**< Value of launch attribute ::cudaLaunchAttributeMemSyncDomain. See + ::cudaLaunchMemSyncDomain. */ + /** + * Value of launch attribute ::cudaLaunchAttributePreferredClusterDimension + * that represents the desired preferred cluster dimensions for the kernel. + * Opaque type with the following fields: + * - \p x - The X dimension of the preferred cluster, in blocks. Must be + * a divisor of the grid X dimension, and must be a multiple of + * the \p x field of ::cudaLaunchAttributeValue::clusterDim. + * - \p y - The Y dimension of the preferred cluster, in blocks. Must be + * a divisor of the grid Y dimension, and must be a multiple of + * the \p y field of ::cudaLaunchAttributeValue::clusterDim. + * - \p z - The Z dimension of the preferred cluster, in blocks. Must be + * equal to the \p z field of ::cudaLaunchAttributeValue::clusterDim. + */ + struct { + unsigned int x; + unsigned int y; + unsigned int z; + } preferredClusterDim; + + /** + * Value of launch attribute ::cudaLaunchAttributeLaunchCompletionEvent + * with the following fields: + * - \p cudaEvent_t event - Event to fire when the last block launches. + * - \p int flags - Event record flags, see ::cudaEventRecordWithFlags. Does not accept + * ::cudaEventRecordExternal. + */ + struct { + cudaEvent_t event; + int flags; + } launchCompletionEvent; + + /** + * Value of launch attribute ::cudaLaunchAttributeDeviceUpdatableKernelNode + * with the following fields: + * - \p int deviceUpdatable - Whether or not the resulting kernel node should be device-updatable. + * - \p cudaGraphDeviceNode_t devNode - Returns a handle to pass to the various device-side update functions. + */ + struct { + int deviceUpdatable; + cudaGraphDeviceNode_t devNode; + } deviceUpdatableKernelNode; + unsigned int sharedMemCarveout; /**< Value of launch attribute ::cudaLaunchAttributePreferredSharedMemoryCarveout. */ + unsigned int nvlinkUtilCentricScheduling; /**< Value of launch attribute ::cudaLaunchAttributeNvlinkUtilCentricScheduling. */ + } cudaLaunchAttributeValue; /** * Launch attribute */ typedef __device_builtin__ struct cudaLaunchAttribute_st { - cudaLaunchAttributeID id; + cudaLaunchAttributeID id; /**< Attribute to set */ char pad[8 - sizeof(cudaLaunchAttributeID)]; - cudaLaunchAttributeValue val; + cudaLaunchAttributeValue val; /**< Value of the attribute */ } cudaLaunchAttribute; /** @@ -3125,8 +4020,8 @@ typedef __device_builtin__ struct cudaLaunchConfig_st { dim3 blockDim; /**< Block dimensions */ size_t dynamicSmemBytes; /**< Dynamic shared-memory size per thread block in bytes */ cudaStream_t stream; /**< Stream identifier */ - cudaLaunchAttribute *attrs; /**< nullable if numAttrs == 0 */ - unsigned int numAttrs; /**< Number of attributes populated in attrs */ + cudaLaunchAttribute *attrs; /**< List of attributes; nullable if ::cudaLaunchConfig_t::numAttrs == 0 */ + unsigned int numAttrs; /**< Number of attributes populated in ::cudaLaunchConfig_t::attrs */ } cudaLaunchConfig_t; #define cudaStreamAttrID cudaLaunchAttributeID @@ -3146,12 +4041,63 @@ typedef __device_builtin__ struct cudaLaunchConfig_st { #define cudaKernelNodeAttributeClusterSchedulingPolicyPreference cudaLaunchAttributeClusterSchedulingPolicyPreference #define cudaKernelNodeAttributeMemSyncDomainMap cudaLaunchAttributeMemSyncDomainMap #define cudaKernelNodeAttributeMemSyncDomain cudaLaunchAttributeMemSyncDomain +#define cudaKernelNodeAttributePreferredSharedMemoryCarveout cudaLaunchAttributePreferredSharedMemoryCarveout +#define cudaKernelNodeAttributeDeviceUpdatableKernelNode cudaLaunchAttributeDeviceUpdatableKernelNode +#define cudaKernelNodeAttributeNvlinkUtilCentricScheduling cudaLaunchAttributeNvlinkUtilCentricScheduling + #define cudaKernelNodeAttrValue cudaLaunchAttributeValue +/** + * CUDA device NUMA config + */ +enum __device_builtin__ cudaDeviceNumaConfig { + cudaDeviceNumaConfigNone = 0, /**< The GPU is not a NUMA node */ + cudaDeviceNumaConfigNumaNode, /**< The GPU is a NUMA node, cudaDevAttrNumaId contains its NUMA ID */ +}; + +/** + * CUDA async callback handle + */ +typedef struct cudaAsyncCallbackEntry* cudaAsyncCallbackHandle_t; + +struct cudaAsyncCallbackEntry; + +/** +* Types of async notification that can occur +*/ +typedef __device_builtin__ enum cudaAsyncNotificationType_enum { + cudaAsyncNotificationTypeOverBudget = 0x1 /**< Sent when the process has exceeded its device memory budget */ +} cudaAsyncNotificationType; + +/** +* Information describing an async notification event +*/ +typedef __device_builtin__ struct cudaAsyncNotificationInfo +{ + cudaAsyncNotificationType type; /**< The type of notification being sent */ + union { + struct { + unsigned long long bytesOverBudget; /**< The number of bytes that the process has allocated above its device memory budget */ + } overBudget; /**< Information about notifications of type \p cudaAsyncNotificationTypeOverBudget */ + } info; /**< Information about the notification. \p type must be checked in order to interpret this field. */ +} cudaAsyncNotificationInfo_t; + +typedef void (*cudaAsyncCallback)(cudaAsyncNotificationInfo_t*, void*, cudaAsyncCallbackHandle_t); + +typedef __device_builtin__ enum CUDAlogLevel_enum { + cudaLogLevelError = 0, + cudaLogLevelWarning = 1 +} cudaLogLevel; + +typedef __device_builtin__ struct CUlogsCallbackEntry_st *cudaLogsCallbackHandle; +typedef __device_builtin__ unsigned int cudaLogIterator; + /** @} */ /** @} */ /* END CUDART_TYPES */ +#endif /* !__CUDACC_RTC_MINIMAL__ */ + #if defined(__UNDEF_CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS_DRIVER_TYPES_H__) #undef __CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__ #undef __UNDEF_CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS_DRIVER_TYPES_H__ @@ -3159,4 +4105,6 @@ typedef __device_builtin__ struct cudaLaunchConfig_st { #undef __CUDA_DEPRECATED + + #endif /* !__DRIVER_TYPES_H__ */ diff --git a/backends/cuda/include/surface_types.h b/backends/cuda/include/surface_types.h index 739dfe490..e1d0eccde 100644 --- a/backends/cuda/include/surface_types.h +++ b/backends/cuda/include/surface_types.h @@ -1,5 +1,5 @@ /* - * Copyright 1993-2012 NVIDIA Corporation. All rights reserved. + * Copyright 1993-2023 NVIDIA Corporation. All rights reserved. * * NOTICE TO LICENSEE: * @@ -58,6 +58,8 @@ #include "driver_types.h" +#ifndef __CUDACC_RTC_MINIMAL__ + /** * \addtogroup CUDART_TYPES * @@ -105,4 +107,5 @@ typedef __device_builtin__ unsigned long long cudaSurfaceObject_t; /** @} */ /** @} */ /* END CUDART_TYPES */ +#endif /* !__CUDACC_RTC_MINIMAL__ */ #endif /* !__SURFACE_TYPES_H__ */ diff --git a/backends/cuda/include/texture_types.h b/backends/cuda/include/texture_types.h index 5289dc6bb..a3e3e90ef 100644 --- a/backends/cuda/include/texture_types.h +++ b/backends/cuda/include/texture_types.h @@ -1,5 +1,5 @@ /* - * Copyright 1993-2012 NVIDIA Corporation. All rights reserved. + * Copyright 1993-2023 NVIDIA Corporation. All rights reserved. * * NOTICE TO LICENSEE: * @@ -58,6 +58,8 @@ #include "driver_types.h" +#ifndef __CUDACC_RTC_MINIMAL__ + /** * \addtogroup CUDART_TYPES * @@ -174,4 +176,5 @@ typedef __device_builtin__ unsigned long long cudaTextureObject_t; /** @} */ /** @} */ /* END CUDART_TYPES */ +#endif /* !__CUDACC_RTC_MINIMAL__ */ #endif /* !__TEXTURE_TYPES_H__ */ diff --git a/backends/cuda/include/vector_types.h b/backends/cuda/include/vector_types.h index 4cfabcff8..54b9fe40b 100644 --- a/backends/cuda/include/vector_types.h +++ b/backends/cuda/include/vector_types.h @@ -1,5 +1,5 @@ /* - * Copyright 1993-2018 NVIDIA Corporation. All rights reserved. + * Copyright 1993-2025 NVIDIA Corporation. All rights reserved. * * NOTICE TO LICENSEE: * @@ -65,12 +65,85 @@ #include "crt/host_defines.h" #endif +/* NVRTC compiler defines these instead of in the header (to reduce compile time) +*/ +#ifndef __CUDACC_RTC_BUILTIN_VECTOR_TYPES__ + /******************************************************************************* * * * * * * *******************************************************************************/ + +#if defined(__NVCOMPILER) +# define __NV_SILENCE_HOST_DEPRECATION_BEGIN \ + _Pragma("diag_suppress 1216") \ + _Pragma("diag_suppress deprecated_entity_with_custom_message") +# define __NV_SILENCE_HOST_DEPRECATION_END \ + _Pragma("diag_default 1216") \ + _Pragma("diag_default deprecated_entity_with_custom_message") +#elif defined(__GNUC__) || defined(__clang__) +# define __NV_SILENCE_HOST_DEPRECATION_BEGIN \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +# define __NV_SILENCE_HOST_DEPRECATION_END \ + _Pragma("GCC diagnostic pop") +#elif defined(_MSC_VER) +# define __NV_SILENCE_HOST_DEPRECATION_BEGIN \ + __pragma(warning(push)) \ + __pragma(warning(disable:4996)) +# define __NV_SILENCE_HOST_DEPRECATION_END \ + __pragma(warning(pop)) +#elif defined(__EDG__) +# define __NV_SILENCE_HOST_DEPRECATION_BEGIN \ + _Pragma("warning push") \ + _Pragma("warning disable 1444") +# define __NV_SILENCE_HOST_DEPRECATION_END \ + _Pragma("warning pop") +#else + #define __NV_SILENCE_HOST_DEPRECATION_BEGIN + #define __NV_SILENCE_HOST_DEPRECATION_END +#endif + +#if defined(__NV_NO_VECTOR_DEPRECATION_DIAG) +# if defined(__CUDACC__) +# pragma nv_diag_suppress vector_deprecation +# endif +# define __NV_SILENCE_DEPRECATION_BEGIN +# define __NV_SILENCE_DEPRECATION_END +#elif defined(__CUDACC__) && defined(_MSC_VER) +# define __NV_SILENCE_DEPRECATION_BEGIN \ + __NV_SILENCE_HOST_DEPRECATION_BEGIN \ + __pragma(nv_diagnostic push) \ + __pragma(nv_diag_suppress 1444) +# define __NV_SILENCE_DEPRECATION_END \ + __NV_SILENCE_HOST_DEPRECATION_END \ + __pragma(nv_diagnostic pop) +#elif defined(__CUDACC__) +# define __NV_SILENCE_DEPRECATION_BEGIN \ + _Pragma("nv_diagnostic push") \ + _Pragma("nv_diag_suppress 20199") \ + _Pragma("nv_diag_suppress 1444") \ + __NV_SILENCE_HOST_DEPRECATION_BEGIN +# define __NV_SILENCE_DEPRECATION_END \ + __NV_SILENCE_HOST_DEPRECATION_END \ + _Pragma("nv_diagnostic pop") +#else +#define __NV_SILENCE_DEPRECATION_BEGIN __NV_SILENCE_HOST_DEPRECATION_BEGIN +#define __NV_SILENCE_DEPRECATION_END __NV_SILENCE_HOST_DEPRECATION_END +#endif + +#if defined(__NV_NO_VECTOR_DEPRECATION_DIAG) +# define __VECTOR_TYPE_DEPRECATED__(msg) +#elif defined(_WIN32) +# define __VECTOR_TYPE_DEPRECATED__(msg) __declspec(deprecated(msg)) +#elif (defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 5 && !defined(__clang__)))) +# define __VECTOR_TYPE_DEPRECATED__(msg) __attribute__((deprecated)) +#else +# define __VECTOR_TYPE_DEPRECATED__(msg) __attribute__((deprecated(msg))) +#endif + #if !defined(__CUDACC__) && !defined(__CUDACC_RTC__) && \ defined(_WIN32) && !defined(_WIN64) @@ -241,16 +314,45 @@ struct __device_builtin__ ulong3 unsigned long int x, y, z; }; -struct __device_builtin__ __builtin_align__(16) long4 +struct __VECTOR_TYPE_DEPRECATED__("use long4_16a or long4_32a") +__device_builtin__ __builtin_align__(16) long4 +{ + long int x, y, z, w; +}; + +struct __VECTOR_TYPE_DEPRECATED__("use ulong4_16a or ulong4_32a") +__device_builtin__ __builtin_align__(16) ulong4 +{ + unsigned long int x, y, z, w; +}; + +struct __device_builtin__ __builtin_align__(16) long4_16a { long int x, y, z, w; }; -struct __device_builtin__ __builtin_align__(16) ulong4 +struct __device_builtin__ __builtin_align__(16) ulong4_16a { unsigned long int x, y, z, w; }; +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable: 4324) +#endif +struct __device_builtin__ __builtin_align__(32) long4_32a +{ + long int x, y, z, w; +}; + +struct __device_builtin__ __builtin_align__(32) ulong4_32a +{ + unsigned long int x, y, z, w; +}; +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + struct __device_builtin__ float1 { float x; @@ -318,16 +420,39 @@ struct __device_builtin__ ulonglong3 unsigned long long int x, y, z; }; -struct __device_builtin__ __builtin_align__(16) longlong4 +struct __VECTOR_TYPE_DEPRECATED__("use longlong4_16a or longlong4_32a") +__device_builtin__ __builtin_align__(16) longlong4 +{ + long long int x, y, z ,w; +}; + +struct __device_builtin__ __builtin_align__(16) longlong4_16a { long long int x, y, z ,w; }; -struct __device_builtin__ __builtin_align__(16) ulonglong4 +struct __device_builtin__ __builtin_align__(32) longlong4_32a +{ + long long int x, y, z ,w; +}; + +struct __VECTOR_TYPE_DEPRECATED__("use ulonglong4_16a or ulonglong4_32a") +__device_builtin__ __builtin_align__(16) ulonglong4 +{ + unsigned long long int x, y, z, w; +}; + +struct __device_builtin__ __builtin_align__(16) ulonglong4_16a +{ + unsigned long long int x, y, z, w; +}; + +struct __device_builtin__ __builtin_align__(32) ulonglong4_32a { unsigned long long int x, y, z, w; }; + struct __device_builtin__ double1 { double x; @@ -343,7 +468,18 @@ struct __device_builtin__ double3 double x, y, z; }; -struct __device_builtin__ __builtin_align__(16) double4 +struct __VECTOR_TYPE_DEPRECATED__("use double4_16a or double4_32a") +__device_builtin__ __builtin_align__(16) double4 +{ + double x, y, z, w; +}; + +struct __device_builtin__ __builtin_align__(16) double4_16a +{ + double x, y, z, w; +}; + +struct __device_builtin__ __builtin_align__(32) double4_32a { double x, y, z, w; }; @@ -390,8 +526,14 @@ typedef __device_builtin__ struct long2 long2; typedef __device_builtin__ struct ulong2 ulong2; typedef __device_builtin__ struct long3 long3; typedef __device_builtin__ struct ulong3 ulong3; -typedef __device_builtin__ struct long4 long4; -typedef __device_builtin__ struct ulong4 ulong4; +__NV_SILENCE_DEPRECATION_BEGIN +typedef __device_builtin__ struct long4 __VECTOR_TYPE_DEPRECATED__("use long4_16a or long4_32a") long4; +typedef __device_builtin__ struct ulong4 __VECTOR_TYPE_DEPRECATED__("use ulong4_16a or ulong4_32a") ulong4; +__NV_SILENCE_DEPRECATION_END +typedef __device_builtin__ struct long4_16a long4_16a; +typedef __device_builtin__ struct ulong4_16a ulong4_16a; +typedef __device_builtin__ struct long4_32a long4_32a; +typedef __device_builtin__ struct ulong4_32a ulong4_32a; typedef __device_builtin__ struct float1 float1; typedef __device_builtin__ struct float2 float2; typedef __device_builtin__ struct float3 float3; @@ -402,12 +544,26 @@ typedef __device_builtin__ struct longlong2 longlong2; typedef __device_builtin__ struct ulonglong2 ulonglong2; typedef __device_builtin__ struct longlong3 longlong3; typedef __device_builtin__ struct ulonglong3 ulonglong3; -typedef __device_builtin__ struct longlong4 longlong4; -typedef __device_builtin__ struct ulonglong4 ulonglong4; +__NV_SILENCE_DEPRECATION_BEGIN +typedef __device_builtin__ struct longlong4 __VECTOR_TYPE_DEPRECATED__("use longlong4_16a or longlong4_32a") longlong4; +typedef __device_builtin__ struct ulonglong4 __VECTOR_TYPE_DEPRECATED__("use ulonglong4_16a or ulonglong4_32a") ulonglong4; +__NV_SILENCE_DEPRECATION_END +typedef __device_builtin__ struct longlong4_16a longlong4_16a; +typedef __device_builtin__ struct ulonglong4_16a ulonglong4_16a; +typedef __device_builtin__ struct longlong4_32a longlong4_32a; +typedef __device_builtin__ struct ulonglong4_32a ulonglong4_32a; typedef __device_builtin__ struct double1 double1; typedef __device_builtin__ struct double2 double2; typedef __device_builtin__ struct double3 double3; -typedef __device_builtin__ struct double4 double4; +__NV_SILENCE_DEPRECATION_BEGIN +typedef __device_builtin__ struct double4 __VECTOR_TYPE_DEPRECATED__("use double4_16a or double4_32a") double4; +__NV_SILENCE_DEPRECATION_END +typedef __device_builtin__ struct double4_16a double4_16a; +typedef __device_builtin__ struct double4_32a double4_32a; + +#undef __cuda_builtin_vector_align8 + +#endif /* !defined(__CUDACC_RTC_BUILTIN_VECTOR_TYPES__) */ /******************************************************************************* * * @@ -419,7 +575,10 @@ struct __device_builtin__ dim3 { unsigned int x, y, z; #if defined(__cplusplus) -#if __cplusplus >= 201103L +#if __cplusplus >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1900 ) + /* MSVC 2015 introduced support for constexpr constructors. A check in addition to the _cpluscplus macro comparison + that uses the _MSC_VER macro is required because by default, Visual Studio always returns the value 199711L for + the __cplusplus preprocessor macro. */ __host__ __device__ constexpr dim3(unsigned int vx = 1, unsigned int vy = 1, unsigned int vz = 1) : x(vx), y(vy), z(vz) {} __host__ __device__ constexpr dim3(uint3 v) : x(v.x), y(v.y), z(v.z) {} __host__ __device__ constexpr operator uint3(void) const { return uint3{x, y, z}; } @@ -433,7 +592,7 @@ struct __device_builtin__ dim3 typedef __device_builtin__ struct dim3 dim3; -#undef __cuda_builtin_vector_align8 +#undef __VECTOR_TYPE_DEPRECATED__ #if defined(__UNDEF_CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS_VECTOR_TYPES_H__) #undef __CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__ diff --git a/utils/gen_library_base.rb b/utils/gen_library_base.rb index 7c5b26b6b..8bd9e3aaf 100644 --- a/utils/gen_library_base.rb +++ b/utils/gen_library_base.rb @@ -101,10 +101,10 @@ def print_bitfield_with_namespace(namespace, name, enum, check_flags: false) puts "\n" end -def print_enum_with_namespace(namespace, name, enum, filter_members: ->(_m) { true }) +def print_enum_with_namespace(namespace, name, enum, filter_members: ->(_m) { true }, fix_values: ->(v) { v }) members = enum.members.filter(&filter_members).collect do |m| r = [m.name.to_sym] - r.push m.val if m.val + r.push fix_values.call(m.val) if m.val r end print_lambda = lambda { |m|