.TH "cufile.h" 3 "Wed Feb 12 2025" "GPUDirect Storage" \" -*- nroff -*- .ad l .nh .SH NAME cufile.h \- cuFile C APIs .SH SYNOPSIS .br .PP .SS "Data Structures" .in +1c .ti -1c .RI "struct \fBCUfileDescr_t\fP" .br .ti -1c .RI "struct \fBCUfileDrvProps\fP" .br .ti -1c .RI "struct \fBCUfileError\fP" .br .RI "cufileop status string " .ti -1c .RI "struct \fBCUfileFSOps\fP" .br .ti -1c .RI "struct \fBCUfileIOEvents\fP" .br .ti -1c .RI "struct \fBCUfileIOParams\fP" .br .ti -1c .RI "struct \fBcufileRDMAInfo\fP" .br .in -1c .SS "Macros" .in +1c .ti -1c .RI "#define \fBCU_FILE_CUDA_ERR\fP(status) ((status)\&.cu_err)" .br .ti -1c .RI "#define \fBCU_FILE_RDMA_REGISTER\fP 1" .br .ti -1c .RI "#define \fBCU_FILE_RDMA_RELAXED_ORDERING\fP (1<<1)" .br .ti -1c .RI "#define \fBCU_FILE_STREAM_FIXED_BUF_OFFSET\fP 1" .br .ti -1c .RI "#define \fBCU_FILE_STREAM_FIXED_FILE_OFFSET\fP 2" .br .ti -1c .RI "#define \fBCU_FILE_STREAM_FIXED_FILE_SIZE\fP 4" .br .ti -1c .RI "#define \fBCU_FILE_STREAM_PAGE_ALIGNED_INPUTS\fP 8" .br .ti -1c .RI "#define \fBCUFILE_ERRSTR\fP(err) cufileop_status_error((\fBCUfileOpError\fP)abs((err)))" .br .ti -1c .RI "#define \fBcuFileDriverClose\fP \fBcuFileDriverClose_v2\fP" .br .ti -1c .RI "#define \fBIS_CUDA_ERR\fP(status) ((status)\&.err == \fBCU_FILE_CUDA_DRIVER_ERROR\fP)" .br .ti -1c .RI "#define \fBIS_CUFILE_ERR\fP(err) (abs((err)) > CUFILEOP_BASE_ERR)" .br .RI "error macros to inspect error status of type \fBCUfileOpError\fP " .in -1c .SS "Typedefs" .in +1c .ti -1c .RI "typedef void * \fBCUfileBatchHandle_t\fP" .br .ti -1c .RI "typedef enum \fBcufileBatchMode\fP \fBCUfileBatchMode_t\fP" .br .ti -1c .RI "typedef struct \fBCUfileDescr_t\fP \fBCUfileDescr_t\fP" .br .ti -1c .RI "typedef enum \fBCUfileDriverControlFlags\fP \fBCUfileDriverControlFlags_t\fP" .br .ti -1c .RI "typedef enum \fBCUfileDriverStatusFlags\fP \fBCUfileDriverStatusFlags_t\fP" .br .ti -1c .RI "typedef struct \fBCUfileDrvProps\fP \fBCUfileDrvProps_t\fP" .br .ti -1c .RI "typedef struct \fBCUfileError\fP \fBCUfileError_t\fP" .br .RI "cufileop status string " .ti -1c .RI "typedef enum \fBCUfileFeatureFlags\fP \fBCUfileFeatureFlags_t\fP" .br .ti -1c .RI "typedef struct \fBCUfileFSOps\fP \fBCUfileFSOps_t\fP" .br .ti -1c .RI "typedef void * \fBCUfileHandle_t\fP" .br .RI "File handle type\&. " .ti -1c .RI "typedef struct \fBCUfileIOEvents\fP \fBCUfileIOEvents_t\fP" .br .ti -1c .RI "typedef struct \fBCUfileIOParams\fP \fBCUfileIOParams_t\fP" .br .ti -1c .RI "typedef enum \fBCUfileOpcode\fP \fBCUfileOpcode_t\fP" .br .ti -1c .RI "typedef struct \fBcufileRDMAInfo\fP \fBcufileRDMAInfo_t\fP" .br .ti -1c .RI "typedef enum \fBCUFILEStatus_enum\fP \fBCUfileStatus_t\fP" .br .ti -1c .RI "typedef struct sockaddr \fBsockaddr_t\fP" .br .in -1c .SS "Enumerations" .in +1c .ti -1c .RI "enum \fBcufileBatchMode\fP { \fBCUFILE_BATCH\fP = 1 }" .br .ti -1c .RI "enum \fBCUfileDriverControlFlags\fP { \fBCU_FILE_USE_POLL_MODE\fP = 0, \fBCU_FILE_ALLOW_COMPAT_MODE\fP = 1 }" .br .ti -1c .RI "enum \fBCUfileDriverStatusFlags\fP { \fBCU_FILE_LUSTRE_SUPPORTED\fP = 0, \fBCU_FILE_WEKAFS_SUPPORTED\fP = 1, \fBCU_FILE_NFS_SUPPORTED\fP = 2, \fBCU_FILE_GPFS_SUPPORTED\fP = 3, \fBCU_FILE_NVME_SUPPORTED\fP = 4, \fBCU_FILE_NVMEOF_SUPPORTED\fP = 5, \fBCU_FILE_SCSI_SUPPORTED\fP = 6, \fBCU_FILE_SCALEFLUX_CSD_SUPPORTED\fP = 7, \fBCU_FILE_NVMESH_SUPPORTED\fP = 8, \fBCU_FILE_BEEGFS_SUPPORTED\fP = 9, \fBCU_FILE_NVME_P2P_SUPPORTED\fP = 11 }" .br .ti -1c .RI "enum \fBCUfileFeatureFlags\fP { \fBCU_FILE_DYN_ROUTING_SUPPORTED\fP = 0, \fBCU_FILE_BATCH_IO_SUPPORTED\fP = 1, \fBCU_FILE_STREAMS_SUPPORTED\fP = 2, \fBCU_FILE_PARALLEL_IO_SUPPORTED\fP = 3 }" .br .ti -1c .RI "enum \fBCUfileFileHandleType\fP { \fBCU_FILE_HANDLE_TYPE_OPAQUE_FD\fP = 1, \fBCU_FILE_HANDLE_TYPE_OPAQUE_WIN32\fP = 2, \fBCU_FILE_HANDLE_TYPE_USERSPACE_FS\fP = 3 }" .br .ti -1c .RI "enum \fBCUfileOpcode\fP { \fBCUFILE_READ\fP = 0, \fBCUFILE_WRITE\fP }" .br .ti -1c .RI "enum \fBCUFILEStatus_enum\fP { \fBCUFILE_WAITING\fP = 0x000001, \fBCUFILE_PENDING\fP = 0x000002, \fBCUFILE_INVALID\fP = 0x000004, \fBCUFILE_CANCELED\fP = 0x000008, \fBCUFILE_COMPLETE\fP = 0x0000010, \fBCUFILE_TIMEOUT\fP = 0x0000020, \fBCUFILE_FAILED\fP = 0x0000040 }" .br .in -1c .SS "Functions" .in +1c .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileBatchIOCancel\fP (\fBCUfileBatchHandle_t\fP batch_idp)" .br .ti -1c .RI "void \fBcuFileBatchIODestroy\fP (\fBCUfileBatchHandle_t\fP batch_idp)" .br .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileBatchIOGetStatus\fP (\fBCUfileBatchHandle_t\fP batch_idp, unsigned min_nr, unsigned *nr, \fBCUfileIOEvents_t\fP *iocbp, struct timespec *timeout)" .br .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileBatchIOSetUp\fP (\fBCUfileBatchHandle_t\fP *batch_idp, unsigned nr)" .br .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileBatchIOSubmit\fP (\fBCUfileBatchHandle_t\fP batch_idp, unsigned nr, \fBCUfileIOParams_t\fP *iocbp, unsigned int flags)" .br .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileBufDeregister\fP (const void *bufPtr_base)" .br .RI "deregister an already registered device or host memory from cuFile " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileBufRegister\fP (const void *bufPtr_base, size_t length, int flags)" .br .RI "register an existing cudaMalloced memory with cuFile to pin for GPUDirect Storage access or register host allocated memory with cuFile\&. " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileDriverClose\fP (void)" .br .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileDriverClose_v2\fP (void)" .br .RI "reset the cuFile library and release the nvidia-fs driver " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileDriverGetProperties\fP (\fBCUfileDrvProps_t\fP *props)" .br .RI "Gets the Driver session properties\&. " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileDriverOpen\fP (void)" .br .RI "Initialize the cuFile library and open the nvidia-fs driver\&. " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileDriverSetMaxCacheSize\fP (size_t max_cache_size)" .br .RI "Control parameter to set maximum GPU memory reserved per device by the library for internal buffering\&. " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileDriverSetMaxDirectIOSize\fP (size_t max_direct_io_size)" .br .RI "Control parameter to set max IO size(KB) used by the library to talk to nvidia-fs driver\&. " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileDriverSetMaxPinnedMemSize\fP (size_t max_pinned_size)" .br .RI "Sets maximum buffer space that is pinned in KB for use by \fBcuFileBufRegister\fP\&. " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileDriverSetPollMode\fP (bool poll, size_t poll_threshold_size)" .br .RI "Sets whether the Read/Write APIs use polling to do IO operations\&. " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileGetVersion\fP (int *version)" .br .ti -1c .RI "void \fBcuFileHandleDeregister\fP (\fBCUfileHandle_t\fP fh)" .br .RI "releases a registered filehandle from cuFile " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileHandleRegister\fP (\fBCUfileHandle_t\fP *fh, \fBCUfileDescr_t\fP *descr)" .br .RI "cuFileHandleRegister is required, and performs extra checking that is memoized to provide increased performance on later cuFile operations\&. " .ti -1c .RI "ssize_t \fBcuFileRead\fP (\fBCUfileHandle_t\fP fh, void *bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset)" .br .RI "read data from a registered file handle to a specified device or host memory " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileReadAsync\fP (\fBCUfileHandle_t\fP fh, void *bufPtr_base, size_t *size_p, off_t *file_offset_p, off_t *bufPtr_offset_p, ssize_t *bytes_read_p, CUstream stream)" .br .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileStreamDeregister\fP (CUstream stream)" .br .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileStreamRegister\fP (CUstream stream, unsigned flags)" .br .ti -1c .RI "long \fBcuFileUseCount\fP (void)" .br .RI "returns use count of cufile drivers at that moment by the process\&. " .ti -1c .RI "ssize_t \fBcuFileWrite\fP (\fBCUfileHandle_t\fP fh, const void *bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset)" .br .RI "write data from a specified device or host memory to a registered file handle " .ti -1c .RI "\fBCUfileError_t\fP \fBcuFileWriteAsync\fP (\fBCUfileHandle_t\fP fh, void *bufPtr_base, size_t *size_p, off_t *file_offset_p, off_t *bufPtr_offset_p, ssize_t *bytes_written_p, CUstream stream)" .br .in -1c .SS "Variables" .in +1c .ti -1c .RI "\fBCUfileOpError\fP" .br .in -1c .SH "Detailed Description" .PP cuFile C APIs This file contains all the C APIs to perform GPUDirect Storage supported IO operations .SH "Macro Definition Documentation" .PP .SS "#define CU_FILE_CUDA_ERR(status) ((status)\&.cu_err)" .SS "#define CU_FILE_RDMA_REGISTER 1" .SS "#define CU_FILE_RDMA_RELAXED_ORDERING (1<<1)" .SS "#define CU_FILE_STREAM_FIXED_BUF_OFFSET 1" .SS "#define CU_FILE_STREAM_FIXED_FILE_OFFSET 2" .SS "#define CU_FILE_STREAM_FIXED_FILE_SIZE 4" .SS "#define CU_FILE_STREAM_PAGE_ALIGNED_INPUTS 8" .SS "#define CUFILE_ERRSTR(err) cufileop_status_error((\fBCUfileOpError\fP)abs((err)))" .SS "#define cuFileDriverClose \fBcuFileDriverClose_v2\fP" .SS "#define IS_CUDA_ERR(status) ((status)\&.err == \fBCU_FILE_CUDA_DRIVER_ERROR\fP)" .SS "#define IS_CUFILE_ERR(err) (abs((err)) > CUFILEOP_BASE_ERR)" .PP error macros to inspect error status of type \fBCUfileOpError\fP .SH "Typedef Documentation" .PP .SS "typedef void* \fBCUfileBatchHandle_t\fP" .SS "typedef enum \fBcufileBatchMode\fP \fBCUfileBatchMode_t\fP" .SS "typedef struct \fBCUfileDescr_t\fP \fBCUfileDescr_t\fP" .SS "typedef enum \fBCUfileDriverControlFlags\fP \fBCUfileDriverControlFlags_t\fP" .SS "typedef enum \fBCUfileDriverStatusFlags\fP \fBCUfileDriverStatusFlags_t\fP" .SS "typedef struct \fBCUfileDrvProps\fP \fBCUfileDrvProps_t\fP" .SS "typedef struct \fBCUfileError\fP \fBCUfileError_t\fP" .PP cufileop status string .SS "typedef enum \fBCUfileFeatureFlags\fP \fBCUfileFeatureFlags_t\fP" .SS "typedef struct \fBCUfileFSOps\fP \fBCUfileFSOps_t\fP" .SS "typedef void* \fBCUfileHandle_t\fP" .PP File handle type\&. .SS "typedef struct \fBCUfileIOEvents\fP \fBCUfileIOEvents_t\fP" .SS "typedef struct \fBCUfileIOParams\fP \fBCUfileIOParams_t\fP" .SS "typedef enum \fBCUfileOpcode\fP \fBCUfileOpcode_t\fP" .SS "typedef struct \fBcufileRDMAInfo\fP \fBcufileRDMAInfo_t\fP" .SS "typedef enum \fBCUFILEStatus_enum\fP \fBCUfileStatus_t\fP" .SS "typedef struct sockaddr \fBsockaddr_t\fP" .SH "Enumeration Type Documentation" .PP .SS "enum \fBcufileBatchMode\fP" .PP \fBEnumerator\fP .in +1c .TP \fB\fICUFILE_BATCH \fP\fP .SS "enum \fBCUfileDriverControlFlags\fP" .PP \fBEnumerator\fP .in +1c .TP \fB\fICU_FILE_USE_POLL_MODE \fP\fP use POLL mode\&. properties\&.use_poll_mode .TP \fB\fICU_FILE_ALLOW_COMPAT_MODE \fP\fP allow COMPATIBILITY mode\&. properties\&.allow_compat_mode .SS "enum \fBCUfileDriverStatusFlags\fP" .PP \fBEnumerator\fP .in +1c .TP \fB\fICU_FILE_LUSTRE_SUPPORTED \fP\fP Support for DDN LUSTRE .TP \fB\fICU_FILE_WEKAFS_SUPPORTED \fP\fP Support for WEKAFS .TP \fB\fICU_FILE_NFS_SUPPORTED \fP\fP Support for NFS .TP \fB\fICU_FILE_GPFS_SUPPORTED \fP\fP .TP \fB\fICU_FILE_NVME_SUPPORTED \fP\fP < Support for GPFS Support for NVMe .TP \fB\fICU_FILE_NVMEOF_SUPPORTED \fP\fP Support for NVMeOF .TP \fB\fICU_FILE_SCSI_SUPPORTED \fP\fP Support for SCSI .TP \fB\fICU_FILE_SCALEFLUX_CSD_SUPPORTED \fP\fP Support for Scaleflux CSD .TP \fB\fICU_FILE_NVMESH_SUPPORTED \fP\fP Support for NVMesh Block Dev .TP \fB\fICU_FILE_BEEGFS_SUPPORTED \fP\fP Support for BeeGFS .TP \fB\fICU_FILE_NVME_P2P_SUPPORTED \fP\fP Support for NVMe using PCI P2PDMA .SS "enum \fBCUfileFeatureFlags\fP" .PP \fBEnumerator\fP .in +1c .TP \fB\fICU_FILE_DYN_ROUTING_SUPPORTED \fP\fP Support for Dynamic routing to handle devices across the PCIe bridges .TP \fB\fICU_FILE_BATCH_IO_SUPPORTED \fP\fP Unsupported .TP \fB\fICU_FILE_STREAMS_SUPPORTED \fP\fP Unsupported .TP \fB\fICU_FILE_PARALLEL_IO_SUPPORTED \fP\fP Unsupported .SS "enum \fBCUfileFileHandleType\fP" .PP \fBEnumerator\fP .in +1c .TP \fB\fICU_FILE_HANDLE_TYPE_OPAQUE_FD \fP\fP Linux based fd .TP \fB\fICU_FILE_HANDLE_TYPE_OPAQUE_WIN32 \fP\fP Windows based handle (unsupported) .TP \fB\fICU_FILE_HANDLE_TYPE_USERSPACE_FS \fP\fP .SS "enum \fBCUfileOpcode\fP" .PP \fBEnumerator\fP .in +1c .TP \fB\fICUFILE_READ \fP\fP .TP \fB\fICUFILE_WRITE \fP\fP .SS "enum \fBCUFILEStatus_enum\fP" .PP \fBEnumerator\fP .in +1c .TP \fB\fICUFILE_WAITING \fP\fP .TP \fB\fICUFILE_PENDING \fP\fP .TP \fB\fICUFILE_INVALID \fP\fP .TP \fB\fICUFILE_CANCELED \fP\fP .TP \fB\fICUFILE_COMPLETE \fP\fP .TP \fB\fICUFILE_TIMEOUT \fP\fP .TP \fB\fICUFILE_FAILED \fP\fP .SH "Function Documentation" .PP .SS "\fBCUfileError_t\fP cuFileBatchIOCancel (\fBCUfileBatchHandle_t\fP batch_idp)" .SS "void cuFileBatchIODestroy (\fBCUfileBatchHandle_t\fP batch_idp)" .SS "\fBCUfileError_t\fP cuFileBatchIOGetStatus (\fBCUfileBatchHandle_t\fP batch_idp, unsigned min_nr, unsigned * nr, \fBCUfileIOEvents_t\fP * iocbp, struct timespec * timeout)" .SS "\fBCUfileError_t\fP cuFileBatchIOSetUp (\fBCUfileBatchHandle_t\fP * batch_idp, unsigned nr)" .SS "\fBCUfileError_t\fP cuFileBatchIOSubmit (\fBCUfileBatchHandle_t\fP batch_idp, unsigned nr, \fBCUfileIOParams_t\fP * iocbp, unsigned int flags)" .SS "\fBCUfileError_t\fP cuFileBufDeregister (const void * bufPtr_base)" .PP deregister an already registered device or host memory from cuFile .PP \fBParameters:\fP .RS 4 \fIbufPtr_base\fP buffer pointer to deregister .RE .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_INVALID_VALUE on invalid memory pointer or unregistered memory pointer .RE .PP \fBSee also:\fP .RS 4 \fBcuFileBufRegister\fP .PP \fBcuFileRead\fP .PP \fBcuFileWrite\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileBufRegister (const void * bufPtr_base, size_t length, int flags)" .PP register an existing cudaMalloced memory with cuFile to pin for GPUDirect Storage access or register host allocated memory with cuFile\&. .PP \fBParameters:\fP .RS 4 \fIbufPtr_base\fP buffer pointer allocated .br \fIlength\fP size of memory region from the above specified bufPtr .br \fIflags\fP CU_FILE_RDMA_REGISTER .RE .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_NVFS_DRIVER_ERROR .PP CU_FILE_INVALID_VALUE .PP CU_FILE_CUDA_ERROR for unsuported memory type .PP CU_FILE_MEMORY_ALREADY_REGISTERED on error .PP CU_FILE_GPU_MEMORY_PINNING_FAILED if not enough pinned memory is available .RE .PP \fBNote:\fP .RS 4 This memory will be use to perform GPU direct DMA from the supported storage\&. .RE .PP \fBWarning:\fP .RS 4 This API is intended for usecases where the memory is used as streaming buffer that is reused across multiple cuFile IO operations before calling \fBcuFileBufDeregister\fP .RE .PP \fBSee also:\fP .RS 4 \fBcuFileBufDeregister\fP .PP \fBcuFileRead\fP .PP \fBcuFileWrite\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileDriverClose (void)" .SS "\fBCUfileError_t\fP cuFileDriverClose_v2 (void)" .PP reset the cuFile library and release the nvidia-fs driver .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_DRIVER_CLOSING if there are any active IO operations using \fBcuFileRead\fP or \fBcuFileWrite\fP .RE .PP \fBSee also:\fP .RS 4 \fBcuFileDriverOpen\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileDriverGetProperties (\fBCUfileDrvProps_t\fP * props)" .PP Gets the Driver session properties\&. .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .RE .PP \fBSee also:\fP .RS 4 \fBcuFileDriverSetPollMode\fP .PP \fBcuFileDriverSetMaxDirectIOSize\fP .PP \fBcuFileDriverSetMaxCacheSize\fP .PP \fBcuFileDriverSetMaxPinnedMemSize\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileDriverOpen (void)" .PP Initialize the cuFile library and open the nvidia-fs driver\&. .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_DRIVER_NOT_INITIALIZED .PP CU_FILE_DRIVER_VERSION_MISMATCH on driver version mismatch error .RE .PP \fBSee also:\fP .RS 4 \fBcuFileDriverClose\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileDriverSetMaxCacheSize (size_t max_cache_size)" .PP Control parameter to set maximum GPU memory reserved per device by the library for internal buffering\&. .PP \fBParameters:\fP .RS 4 \fImax_cache_size\fP The maximum GPU buffer space per device used for internal use in KB .RE .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized .PP CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error .RE .PP \fBWarning:\fP .RS 4 This is an advanced command and should be tuned based on supported GPU memory .RE .PP \fBSee also:\fP .RS 4 \fBcuFileDriverGetProperties\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileDriverSetMaxDirectIOSize (size_t max_direct_io_size)" .PP Control parameter to set max IO size(KB) used by the library to talk to nvidia-fs driver\&. .PP \fBParameters:\fP .RS 4 \fImax_direct_io_size\fP maximum allowed direct io size in KB .RE .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized .PP CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error .RE .PP \fBWarning:\fP .RS 4 This is an advanced command and should be tuned based on available system memory .RE .PP \fBSee also:\fP .RS 4 \fBcuFileDriverGetProperties\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileDriverSetMaxPinnedMemSize (size_t max_pinned_size)" .PP Sets maximum buffer space that is pinned in KB for use by \fBcuFileBufRegister\fP\&. .PP \fBParameters:\fP .RS 4 \fImax_pinned_size\fP maximum buffer space that is pinned in KB .RE .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized .PP CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error .RE .PP \fBWarning:\fP .RS 4 This is an advanced command and should be tuned based on supported GPU memory .RE .PP \fBSee also:\fP .RS 4 \fBcuFileDriverGetProperties\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileDriverSetPollMode (bool poll, size_t poll_threshold_size)" .PP Sets whether the Read/Write APIs use polling to do IO operations\&. .PP \fBParameters:\fP .RS 4 \fIpoll\fP boolean to indicate whether to use poll mode or not .br \fIpoll_threshold_size\fP max IO size to use for POLLING mode in KB .RE .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized .PP CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error .RE .PP \fBWarning:\fP .RS 4 This is an advanced command and should be tuned based on available system memory .RE .PP \fBSee also:\fP .RS 4 \fBcuFileDriverGetProperties\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileGetVersion (int * version)" .PP \fBReturns:\fP .RS 4 cufile library version\&. .RE .PP version is returned as (1000 major + 10 minor)\&. example, CUFILE 1\&.7\&.0 would be represented by 1070\&. .PP \fBNote:\fP .RS 4 This is useful for applications that need to inquire the library\&. .RE .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_INVALID_VALUE if the input parameter is null\&. .PP CU_FILE_DRIVER_VERSION_READ_ERROR if the version is not available\&. .RE .PP .SS "void cuFileHandleDeregister (\fBCUfileHandle_t\fP fh)" .PP releases a registered filehandle from cuFile .PP \fBParameters:\fP .RS 4 \fIfh\fP \fBCUfileHandle_t\fP file handle .RE .PP \fBReturns:\fP .RS 4 void .RE .PP \fBSee also:\fP .RS 4 \fBcuFileHandleRegister\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileHandleRegister (\fBCUfileHandle_t\fP * fh, \fBCUfileDescr_t\fP * descr)" .PP cuFileHandleRegister is required, and performs extra checking that is memoized to provide increased performance on later cuFile operations\&. .PP \fBParameters:\fP .RS 4 \fIfh\fP \fBCUfileHandle_t\fP opaque file handle for IO operations .br \fIdescr\fP \fBCUfileDescr_t\fP file descriptor (OS agnostic) .RE .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on successful completion\&. fh will be updated for use in \fBcuFileRead\fP, \fBcuFileWrite\fP, \fBcuFileHandleDeregister\fP .PP CU_FILE_DRIVER_NOT_INITIALIZED on failure to load driver .PP CU_FILE_IO_NOT_SUPPORTED - if filesystem is not supported .PP CU_FILE_INVALID_VALUE if null or bad api arguments .PP CU_FILE_INVALID_FILE_OPEN_FLAG if file is opened with unsupported modes like no O_DIRECT .PP CU_FILE_INVALID_FILE_TYPE if filepath is not valid or is not a regular file .PP CU_FILE_HANDLE_ALREADY_REGISTERED if file handle/descriptor is already registered .RE .PP \fBDescription\fP cuFileHandleRegister registers the open file descriptor for use with cuFile IO operations\&. .PP This API will ensure that the file’s descriptor is checked for GPUDirect Storage support and returns a valid file handle on CU_FILE_SUCCESS\&. .PP \fBNote:\fP .RS 4 the file needs to be opened in O_DIRECT mode to support GPUDirect Storage\&. .RE .PP \fBSee also:\fP .RS 4 \fBcuFileRead\fP .PP \fBcuFileWrite\fP .PP \fBcuFileHandleDeregister\fP .RE .PP .SS "ssize_t cuFileRead (\fBCUfileHandle_t\fP fh, void * bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset)" .PP read data from a registered file handle to a specified device or host memory .PP \fBParameters:\fP .RS 4 \fIfh\fP \fBCUfileHandle_t\fP opaque file handle .br \fIbufPtr_base\fP base address of buffer in device or host memory .br \fIsize\fP size bytes to read .br \fIfile_offset\fP file-offset from begining of the file .br \fIbufPtr_offset\fP offset relative to the bufPtr_base pointer to read into\&. .RE .PP \fBReturns:\fP .RS 4 size of bytes successfully read .PP -1 on error, in which case errno is set to indicate filesystem errors\&. .PP all other errors will return a negative integer value of \fBCUfileOpError\fP enum value\&. .RE .PP \fBNote:\fP .RS 4 If the bufPtr is not registered with \fBcuFileBufRegister\fP, the data will be buffered through preallocated pinned buffers if needed\&. .PP This is useful for applications that need to perform IO to unaligned file offsets and/or size\&. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory\&. .RE .PP \fBSee also:\fP .RS 4 \fBcuFileBufRegister\fP .PP \fBcuFileHandleRegister\fP .PP \fBcuFileWrite\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileReadAsync (\fBCUfileHandle_t\fP fh, void * bufPtr_base, size_t * size_p, off_t * file_offset_p, off_t * bufPtr_offset_p, ssize_t * bytes_read_p, CUstream stream)" .PP \fBParameters:\fP .RS 4 \fIfh\fP The cuFile handle for the file\&. .br \fIbufPtr_base\fP base address of buffer in device or host memory .br \fIsize_p\fP pointer to size bytes to read .RE .PP \fBNote:\fP .RS 4 *size_p if the size is not known at the time of submission, then must provide the max possible size for I/O request\&. .RE .PP \fBParameters:\fP .RS 4 \fIfile_offset_p\fP pointer to file-offset from begining of the file .br \fIbufPtr_offset_p\fP pointer to offset relative to the bufPtr_base pointer to read into\&. .br \fIbytes_read_p\fP pointer to the number of bytes that were successfully read\&. .br \fICUstream\fP stream cuda stream for the operation\&. .RE .PP \fBReturns:\fP .RS 4 size of bytes successfully read in *bytes_read_p .PP -1 on error, in which case errno is set to indicate filesystem errors\&. .PP all other errors will return a negative integer value of \fBCUfileOpError\fP enum value\&. .RE .PP \fBNote:\fP .RS 4 If the bufPtr_base is not registered with \fBcuFileBufRegister\fP, the data will be buffered through preallocated pinned buffers\&. .PP This is useful for applications that need to perform IO to unaligned file offsets and/or size\&. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory\&. .PP If the stream is registered with cuFileStreamRegister, the IO setup and teardown overhead will be reduced\&. .PP on cuda stream errors, the user must call cuFileStreamDeregister to release any outstanding cuFile resources for the stream\&. .RE .PP \fBSee also:\fP .RS 4 \fBcuFileBufRegister\fP .PP \fBcuFileHandleRegister\fP .PP \fBcuFileRead\fP .PP \fBcuFileStreamRegister\fP .PP \fBcuFileStreamDeregister\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileStreamDeregister (CUstream stream)" .PP \fBParameters:\fP .RS 4 \fICUstream\fP cuda stream for the operation\&. .RE .PP \fBNote:\fP .RS 4 deallocates resources used by previous cuFile asynchronous operations for the cuda stream .PP highly recommend to call after cuda stream errors to release any outstanding cuFile resources for this stream .PP must be called before cuStreamDestroy call for the specified stream\&. .PP This is useful for applications that need to perform IO to unaligned file offsets and/or size\&. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory\&. .RE .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized .PP CU_FILE_INVALID_VALUE if the stream is invalid .RE .PP \fBSee also:\fP .RS 4 \fBcuFileReadAsync\fP .PP \fBcuFileWriteAsync\fP .PP \fBcuFileStreamRegister\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileStreamRegister (CUstream stream, unsigned flags)" .PP \fBParameters:\fP .RS 4 \fICUstream\fP cuda stream for the operation\&. .br \fIflags\fP for the stream to improve the stream execution of IO based on input parameters\&. .RE .PP \fBNote:\fP .RS 4 supported FLAGS are .PP CU_FILE_STREAM_FIXED_BUF_OFFSET - buffer pointer offset is set at submission time .PP CU_FILE_STREAM_FIXED_FILE_OFFSET - file offset is set at submission time .PP CU_FILE_STREAM_FIXED_FILE_SIZE - file size is set at submission time .PP CU_FILE_STREAM_PAGE_ALIGNED_INPUTS - size, offset and buffer offset are 4k aligned .PP allocates resources needed to support cuFile operations asynchronously for the cuda stream .PP This is useful for applications that need to perform IO to unaligned file offsets and/or size\&. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory\&. .RE .PP \fBReturns:\fP .RS 4 CU_FILE_SUCCESS on success .PP CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized .PP CU_FILE_INVALID_VALUE if the stream is invalid .RE .PP \fBSee also:\fP .RS 4 \fBcuFileReadAsync\fP .PP \fBcuFileWriteAsync\fP .PP \fBcuFileStreamDeregister\fP .RE .PP .SS "long cuFileUseCount (void)" .PP returns use count of cufile drivers at that moment by the process\&. .SS "ssize_t cuFileWrite (\fBCUfileHandle_t\fP fh, const void * bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset)" .PP write data from a specified device or host memory to a registered file handle .PP \fBParameters:\fP .RS 4 \fIfh\fP \fBCUfileHandle_t\fP opaque file handle .br \fIbufPtr_base\fP base address of buffer in device or host memory .br \fIsize\fP size bytes to write .br \fIfile_offset\fP file-offset from begining of the file .br \fIbufPtr_offset\fP offset relative to the bufPtr_base pointer to write from\&. .RE .PP \fBReturns:\fP .RS 4 size of bytes successfully written .PP -1 on error, in which case errno is set to indicate filesystem errors\&. .PP all other errors will return a negative integer value of \fBCUfileOpError\fP enum value\&. .RE .PP \fBNote:\fP .RS 4 If the bufPtr is not registered with \fBcuFileBufRegister\fP, the data will be buffered through preallocated pinned buffers if needed\&. .PP This is useful for applications that need to perform IO to unaligned file offsets and/or size\&. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory\&. .RE .PP \fBSee also:\fP .RS 4 \fBcuFileBufRegister\fP .PP \fBcuFileHandleRegister\fP .PP \fBcuFileRead\fP .RE .PP .SS "\fBCUfileError_t\fP cuFileWriteAsync (\fBCUfileHandle_t\fP fh, void * bufPtr_base, size_t * size_p, off_t * file_offset_p, off_t * bufPtr_offset_p, ssize_t * bytes_written_p, CUstream stream)" .PP \fBParameters:\fP .RS 4 \fIfh\fP The cuFile handle for the file\&. .br \fIbufPtr_base\fP base address of buffer in device or host memory .br \fIsize_p\fP pointer to size bytes to write\&. .RE .PP \fBNote:\fP .RS 4 *size_p if the size is not known at the time of submission, then must provide the max possible size for I/O request\&. .RE .PP \fBParameters:\fP .RS 4 \fIfile_offset_p\fP pointer to file-offset from begining of the file .br \fIbufPtr_offset_p\fP pointer to offset relative to the bufPtr_base pointer to write from\&. .br \fIbytes_written_p\fP pointer to the number of bytes that were successfully written\&. .br \fICUstream\fP cuda stream for the operation\&. .RE .PP \fBReturns:\fP .RS 4 size of bytes successfully written in *bytes_written_p .PP -1 on error, in which case errno is set to indicate filesystem errors\&. .PP all other errors will return a negative integer value of \fBCUfileOpError\fP enum value\&. .RE .PP \fBNote:\fP .RS 4 If the bufPtr_base is not registered with \fBcuFileBufRegister\fP, the data will be buffered through preallocated pinned buffers\&. .PP This is useful for applications that need to perform IO to unaligned file offsets and/or size\&. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory\&. .PP If the stream is registered with cuFileStreamRegister prior to this call, the IO setup and teardown overhead will be reduced\&. .PP on cuda stream errors, the user must call cuFileStreamDeregister to release any outstanding cuFile resources for the stream\&. .RE .PP \fBSee also:\fP .RS 4 \fBcuFileBufRegister\fP .PP \fBcuFileHandleRegister\fP .PP \fBcuFileWrite\fP .PP \fBcuFileStreamRegister\fP .PP \fBcuFileStreamDeregister\fP .RE .PP .SH "Variable Documentation" .PP .SS "\fBCUfileOpError\fP" .SH "Author" .PP Generated automatically by Doxygen for GPUDirect Storage from the source code\&.