/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2021 HiSilicon Limited * Copyright(c) 2021 Intel Corporation * Copyright(c) 2021 Marvell International Ltd * Copyright(c) 2021 SmartShare Systems */ #ifndef RTE_DMADEV_H #define RTE_DMADEV_H /** * @file rte_dmadev.h * * DMA (Direct Memory Access) device API. * * The DMA framework is built on the following model: * * --------------- --------------- --------------- * | virtual DMA | | virtual DMA | | virtual DMA | * | channel | | channel | | channel | * --------------- --------------- --------------- * | | | * ------------------ | * | | * ------------ ------------ * | dmadev | | dmadev | * ------------ ------------ * | | * ------------------ ------------------ * | HW DMA channel | | HW DMA channel | * ------------------ ------------------ * | | * -------------------------------- * | * --------------------- * | HW DMA Controller | * --------------------- * * The DMA controller could have multiple HW-DMA-channels (aka. HW-DMA-queues), * each HW-DMA-channel should be represented by a dmadev. * * The dmadev could create multiple virtual DMA channels, each virtual DMA * channel represents a different transfer context. The DMA operation request * must be submitted to the virtual DMA channel. e.g. Application could create * virtual DMA channel 0 for memory-to-memory transfer scenario, and create * virtual DMA channel 1 for memory-to-device transfer scenario. * * This framework uses 'int16_t dev_id' as the device identifier of a dmadev, * and 'uint16_t vchan' as the virtual DMA channel identifier in one dmadev. * * The functions exported by the dmadev API to setup a device designated by its * device identifier must be invoked in the following order: * - rte_dma_configure() * - rte_dma_vchan_setup() * - rte_dma_start() * * Then, the application can invoke dataplane functions to process jobs. * * If the application wants to change the configuration (i.e. invoke * rte_dma_configure() or rte_dma_vchan_setup()), it must invoke * rte_dma_stop() first to stop the device and then do the reconfiguration * before invoking rte_dma_start() again. The dataplane functions should not * be invoked when the device is stopped. * * Finally, an application can close a dmadev by invoking the rte_dma_close() * function. * * The dataplane APIs include two parts: * The first part is the submission of operation requests: * - rte_dma_copy() * - rte_dma_copy_sg() * - rte_dma_fill() * - rte_dma_submit() * * These APIs could work with different virtual DMA channels which have * different contexts. * * The first three APIs are used to submit the operation request to the virtual * DMA channel, if the submission is successful, a positive * ring_idx <= UINT16_MAX is returned, otherwise a negative number is returned. * * The last API is used to issue doorbell to hardware, and also there are flags * (@see RTE_DMA_OP_FLAG_SUBMIT) parameter of the first three APIs could do the * same work. * @note When enqueuing a set of jobs to the device, having a separate submit * outside a loop makes for clearer code than having a check for the last * iteration inside the loop to set a special submit flag. However, for cases * where one item alone is to be submitted or there is a small set of jobs to * be submitted sequentially, having a submit flag provides a lower-overhead * way of doing the submission while still keeping the code clean. * * The second part is to obtain the result of requests: * - rte_dma_completed() * - return the number of operation requests completed successfully. * - rte_dma_completed_status() * - return the number of operation requests completed. * * @note If the dmadev works in silent mode (@see RTE_DMA_CAPA_SILENT), * application does not invoke the above two completed APIs. * * About the ring_idx which enqueue APIs (e.g. rte_dma_copy(), rte_dma_fill()) * return, the rules are as follows: * - ring_idx for each virtual DMA channel are independent. * - For a virtual DMA channel, the ring_idx is monotonically incremented, * when it reach UINT16_MAX, it wraps back to zero. * - This ring_idx can be used by applications to track per-operation * metadata in an application-defined circular ring. * - The initial ring_idx of a virtual DMA channel is zero, after the * device is stopped, the ring_idx needs to be reset to zero. * * One example: * - step-1: start one dmadev * - step-2: enqueue a copy operation, the ring_idx return is 0 * - step-3: enqueue a copy operation again, the ring_idx return is 1 * - ... * - step-101: stop the dmadev * - step-102: start the dmadev * - step-103: enqueue a copy operation, the ring_idx return is 0 * - ... * - step-x+0: enqueue a fill operation, the ring_idx return is 65535 * - step-x+1: enqueue a copy operation, the ring_idx return is 0 * - ... * * The DMA operation address used in enqueue APIs (i.e. rte_dma_copy(), * rte_dma_copy_sg(), rte_dma_fill()) is defined as rte_iova_t type. * * The dmadev supports two types of address: memory address and device address. * * - memory address: the source and destination address of the memory-to-memory * transfer type, or the source address of the memory-to-device transfer type, * or the destination address of the device-to-memory transfer type. * @note If the device support SVA (@see RTE_DMA_CAPA_SVA), the memory address * can be any VA address, otherwise it must be an IOVA address. * * - device address: the source and destination address of the device-to-device * transfer type, or the source address of the device-to-memory transfer type, * or the destination address of the memory-to-device transfer type. * * About MT-safe, all the functions of the dmadev API implemented by a PMD are * lock-free functions which assume to not be invoked in parallel on different * logical cores to work on the same target dmadev object. * @note Different virtual DMA channels on the same dmadev *DO NOT* support * parallel invocation because these virtual DMA channels share the same * HW-DMA-channel. */ #include #include #include #include #ifdef __cplusplus extern "C" { #endif /** Maximum number of devices if rte_dma_dev_max() is not called. */ #define RTE_DMADEV_DEFAULT_MAX 64 /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Configure the maximum number of dmadevs. * @note This function can be invoked before the primary process rte_eal_init() * to change the maximum number of dmadevs. If not invoked, the maximum number * of dmadevs is @see RTE_DMADEV_DEFAULT_MAX * * @param dev_max * maximum number of dmadevs. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental int rte_dma_dev_max(size_t dev_max); /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Get the device identifier for the named DMA device. * * @param name * DMA device name. * * @return * Returns DMA device identifier on success. * - <0: Failure to find named DMA device. */ __rte_experimental int rte_dma_get_dev_id_by_name(const char *name); /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Check whether the dev_id is valid. * * @param dev_id * DMA device index. * * @return * - If the device index is valid (true) or not (false). */ __rte_experimental bool rte_dma_is_valid(int16_t dev_id); /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Get the total number of DMA devices that have been successfully * initialised. * * @return * The total number of usable DMA devices. */ __rte_experimental uint16_t rte_dma_count_avail(void); /** * Iterates over valid dmadev instances. * * @param start_dev_id * The id of the next possible dmadev. * @return * Next valid dmadev, UINT16_MAX if there is none. */ __rte_experimental int16_t rte_dma_next_dev(int16_t start_dev_id); /** Utility macro to iterate over all available dmadevs */ #define RTE_DMA_FOREACH_DEV(p) \ for (p = rte_dma_next_dev(0); \ p != -1; \ p = rte_dma_next_dev(p + 1)) /**@{@name DMA capability * @see struct rte_dma_info::dev_capa */ /** Support memory-to-memory transfer */ #define RTE_DMA_CAPA_MEM_TO_MEM RTE_BIT64(0) /** Support memory-to-device transfer. */ #define RTE_DMA_CAPA_MEM_TO_DEV RTE_BIT64(1) /** Support device-to-memory transfer. */ #define RTE_DMA_CAPA_DEV_TO_MEM RTE_BIT64(2) /** Support device-to-device transfer. */ #define RTE_DMA_CAPA_DEV_TO_DEV RTE_BIT64(3) /** Support SVA which could use VA as DMA address. * If device support SVA then application could pass any VA address like memory * from rte_malloc(), rte_memzone(), malloc, stack memory. * If device don't support SVA, then application should pass IOVA address which * from rte_malloc(), rte_memzone(). */ #define RTE_DMA_CAPA_SVA RTE_BIT64(4) /** Support work in silent mode. * In this mode, application don't required to invoke rte_dma_completed*() * API. * @see struct rte_dma_conf::silent_mode */ #define RTE_DMA_CAPA_SILENT RTE_BIT64(5) /** Supports error handling * * With this bit set, invalid input addresses will be reported as operation failures * to the user but other operations can continue. * Without this bit set, invalid data is not handled by either HW or driver, so user * must ensure that all memory addresses are valid and accessible by HW. */ #define RTE_DMA_CAPA_HANDLES_ERRORS RTE_BIT64(6) /** Support copy operation. * This capability start with index of 32, so that it could leave gap between * normal capability and ops capability. */ #define RTE_DMA_CAPA_OPS_COPY RTE_BIT64(32) /** Support scatter-gather list copy operation. */ #define RTE_DMA_CAPA_OPS_COPY_SG RTE_BIT64(33) /** Support fill operation. */ #define RTE_DMA_CAPA_OPS_FILL RTE_BIT64(34) /**@}*/ /** * A structure used to retrieve the information of a DMA device. * * @see rte_dma_info_get */ struct rte_dma_info { const char *dev_name; /**< Unique device name. */ /** Device capabilities (RTE_DMA_CAPA_*). */ uint64_t dev_capa; /** Maximum number of virtual DMA channels supported. */ uint16_t max_vchans; /** Maximum allowed number of virtual DMA channel descriptors. */ uint16_t max_desc; /** Minimum allowed number of virtual DMA channel descriptors. */ uint16_t min_desc; /** Maximum number of source or destination scatter-gather entry * supported. * If the device does not support COPY_SG capability, this value can be * zero. * If the device supports COPY_SG capability, then rte_dma_copy_sg() * parameter nb_src/nb_dst should not exceed this value. */ uint16_t max_sges; /** NUMA node connection, -1 if unknown. */ int16_t numa_node; /** Number of virtual DMA channel configured. */ uint16_t nb_vchans; }; /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Retrieve information of a DMA device. * * @param dev_id * The identifier of the device. * @param[out] dev_info * A pointer to a structure of type *rte_dma_info* to be filled with the * information of the device. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental int rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info); /** * A structure used to configure a DMA device. * * @see rte_dma_configure */ struct rte_dma_conf { /** The number of virtual DMA channels to set up for the DMA device. * This value cannot be greater than the field 'max_vchans' of struct * rte_dma_info which get from rte_dma_info_get(). */ uint16_t nb_vchans; /** Indicates whether to enable silent mode. * false-default mode, true-silent mode. * This value can be set to true only when the SILENT capability is * supported. * * @see RTE_DMA_CAPA_SILENT */ bool enable_silent; }; /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Configure a DMA device. * * This function must be invoked first before any other function in the * API. This function can also be re-invoked when a device is in the * stopped state. * * @param dev_id * The identifier of the device to configure. * @param dev_conf * The DMA device configuration structure encapsulated into rte_dma_conf * object. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental int rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf); /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Start a DMA device. * * The device start step is the last one and consists of setting the DMA * to start accepting jobs. * * @param dev_id * The identifier of the device. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental int rte_dma_start(int16_t dev_id); /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Stop a DMA device. * * The device can be restarted with a call to rte_dma_start(). * * @param dev_id * The identifier of the device. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental int rte_dma_stop(int16_t dev_id); /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Close a DMA device. * * The device cannot be restarted after this call. * * @param dev_id * The identifier of the device. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental int rte_dma_close(int16_t dev_id); /** * DMA transfer direction defines. * * @see struct rte_dma_vchan_conf::direction */ enum rte_dma_direction { /** DMA transfer direction - from memory to memory. * * @see struct rte_dma_vchan_conf::direction */ RTE_DMA_DIR_MEM_TO_MEM, /** DMA transfer direction - from memory to device. * In a typical scenario, the SoCs are installed on host servers as * iNICs through the PCIe interface. In this case, the SoCs works in * EP(endpoint) mode, it could initiate a DMA move request from memory * (which is SoCs memory) to device (which is host memory). * * @see struct rte_dma_vchan_conf::direction */ RTE_DMA_DIR_MEM_TO_DEV, /** DMA transfer direction - from device to memory. * In a typical scenario, the SoCs are installed on host servers as * iNICs through the PCIe interface. In this case, the SoCs works in * EP(endpoint) mode, it could initiate a DMA move request from device * (which is host memory) to memory (which is SoCs memory). * * @see struct rte_dma_vchan_conf::direction */ RTE_DMA_DIR_DEV_TO_MEM, /** DMA transfer direction - from device to device. * In a typical scenario, the SoCs are installed on host servers as * iNICs through the PCIe interface. In this case, the SoCs works in * EP(endpoint) mode, it could initiate a DMA move request from device * (which is host memory) to the device (which is another host memory). * * @see struct rte_dma_vchan_conf::direction */ RTE_DMA_DIR_DEV_TO_DEV, }; /** * DMA access port type defines. * * @see struct rte_dma_port_param::port_type */ enum rte_dma_port_type { RTE_DMA_PORT_NONE, RTE_DMA_PORT_PCIE, /**< The DMA access port is PCIe. */ }; /** * A structure used to descript DMA access port parameters. * * @see struct rte_dma_vchan_conf::src_port * @see struct rte_dma_vchan_conf::dst_port */ struct rte_dma_port_param { /** The device access port type. * * @see enum rte_dma_port_type */ enum rte_dma_port_type port_type; RTE_STD_C11 union { /** PCIe access port parameters. * * The following model shows SoC's PCIe module connects to * multiple PCIe hosts and multiple endpoints. The PCIe module * has an integrated DMA controller. * * If the DMA wants to access the memory of host A, it can be * initiated by PF1 in core0, or by VF0 of PF0 in core0. * * \code{.unparsed} * System Bus * | ----------PCIe module---------- * | Bus * | Interface * | ----- ------------------ * | | | | PCIe Core0 | * | | | | | ----------- * | | | | PF-0 -- VF-0 | | Host A | * | | |--------| |- VF-1 |--------| Root | * | | | | PF-1 | | Complex | * | | | | PF-2 | ----------- * | | | ------------------ * | | | * | | | ------------------ * | | | | PCIe Core1 | * | | | | | ----------- * | | | | PF-0 -- VF-0 | | Host B | * |-----| |--------| PF-1 -- VF-0 |--------| Root | * | | | | |- VF-1 | | Complex | * | | | | PF-2 | ----------- * | | | ------------------ * | | | * | | | ------------------ * | |DMA| | | ------ * | | | | |--------| EP | * | | |--------| PCIe Core2 | ------ * | | | | | ------ * | | | | |--------| EP | * | | | | | ------ * | ----- ------------------ * * \endcode * * @note If some fields can not be supported by the * hardware/driver, then the driver ignores those fields. * Please check driver-specific documentation for limitations * and capabilities. */ __extension__ struct { uint64_t coreid : 4; /**< PCIe core id used. */ uint64_t pfid : 8; /**< PF id used. */ uint64_t vfen : 1; /**< VF enable bit. */ uint64_t vfid : 16; /**< VF id used. */ /** The pasid filed in TLP packet. */ uint64_t pasid : 20; /** The attributes filed in TLP packet. */ uint64_t attr : 3; /** The processing hint filed in TLP packet. */ uint64_t ph : 2; /** The steering tag filed in TLP packet. */ uint64_t st : 16; } pcie; }; uint64_t reserved[2]; /**< Reserved for future fields. */ }; /** * A structure used to configure a virtual DMA channel. * * @see rte_dma_vchan_setup */ struct rte_dma_vchan_conf { /** Transfer direction * * @see enum rte_dma_direction */ enum rte_dma_direction direction; /** Number of descriptor for the virtual DMA channel */ uint16_t nb_desc; /** 1) Used to describes the device access port parameter in the * device-to-memory transfer scenario. * 2) Used to describes the source device access port parameter in the * device-to-device transfer scenario. * * @see struct rte_dma_port_param */ struct rte_dma_port_param src_port; /** 1) Used to describes the device access port parameter in the * memory-to-device transfer scenario. * 2) Used to describes the destination device access port parameter in * the device-to-device transfer scenario. * * @see struct rte_dma_port_param */ struct rte_dma_port_param dst_port; }; /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Allocate and set up a virtual DMA channel. * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. The value must be in the range * [0, nb_vchans - 1] previously supplied to rte_dma_configure(). * @param conf * The virtual DMA channel configuration structure encapsulated into * rte_dma_vchan_conf object. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental int rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan, const struct rte_dma_vchan_conf *conf); /** * A structure used to retrieve statistics. * * @see rte_dma_stats_get */ struct rte_dma_stats { /** Count of operations which were submitted to hardware. */ uint64_t submitted; /** Count of operations which were completed, including successful and * failed completions. */ uint64_t completed; /** Count of operations which failed to complete. */ uint64_t errors; }; /** * Special ID, which is used to represent all virtual DMA channels. * * @see rte_dma_stats_get * @see rte_dma_stats_reset */ #define RTE_DMA_ALL_VCHAN 0xFFFFu /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Retrieve basic statistics of a or all virtual DMA channel(s). * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. * If equal RTE_DMA_ALL_VCHAN means all channels. * @param[out] stats * The basic statistics structure encapsulated into rte_dma_stats * object. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental int rte_dma_stats_get(int16_t dev_id, uint16_t vchan, struct rte_dma_stats *stats); /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Reset basic statistics of a or all virtual DMA channel(s). * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. * If equal RTE_DMA_ALL_VCHAN means all channels. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan); /** * device vchannel status * * Enum with the options for the channel status, either idle, active or halted due to error * @see rte_dma_vchan_status */ enum rte_dma_vchan_status { RTE_DMA_VCHAN_IDLE, /**< not processing, awaiting ops */ RTE_DMA_VCHAN_ACTIVE, /**< currently processing jobs */ RTE_DMA_VCHAN_HALTED_ERROR, /**< not processing due to error, cannot accept new ops */ }; /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Determine if all jobs have completed on a device channel. * This function is primarily designed for testing use, as it allows a process to check if * all jobs are completed, without actually gathering completions from those jobs. * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. * @param[out] status * The vchan status * @return * 0 - call completed successfully * < 0 - error code indicating there was a problem calling the API */ __rte_experimental int rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *status); /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Dump DMA device info. * * @param dev_id * The identifier of the device. * @param f * The file to write the output to. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental int rte_dma_dump(int16_t dev_id, FILE *f); /** * DMA transfer result status code defines. * * @see rte_dma_completed_status */ enum rte_dma_status_code { /** The operation completed successfully. */ RTE_DMA_STATUS_SUCCESSFUL, /** The operation failed to complete due abort by user. * This is mainly used when processing dev_stop, user could modify the * descriptors (e.g. change one bit to tell hardware abort this job), * it allows outstanding requests to be complete as much as possible, * so reduce the time to stop the device. */ RTE_DMA_STATUS_USER_ABORT, /** The operation failed to complete due to following scenarios: * The jobs in a particular batch are not attempted because they * appeared after a fence where a previous job failed. In some HW * implementation it's possible for jobs from later batches would be * completed, though, so report the status from the not attempted jobs * before reporting those newer completed jobs. */ RTE_DMA_STATUS_NOT_ATTEMPTED, /** The operation failed to complete due invalid source address. */ RTE_DMA_STATUS_INVALID_SRC_ADDR, /** The operation failed to complete due invalid destination address. */ RTE_DMA_STATUS_INVALID_DST_ADDR, /** The operation failed to complete due invalid source or destination * address, cover the case that only knows the address error, but not * sure which address error. */ RTE_DMA_STATUS_INVALID_ADDR, /** The operation failed to complete due invalid length. */ RTE_DMA_STATUS_INVALID_LENGTH, /** The operation failed to complete due invalid opcode. * The DMA descriptor could have multiple format, which are * distinguished by the opcode field. */ RTE_DMA_STATUS_INVALID_OPCODE, /** The operation failed to complete due bus read error. */ RTE_DMA_STATUS_BUS_READ_ERROR, /** The operation failed to complete due bus write error. */ RTE_DMA_STATUS_BUS_WRITE_ERROR, /** The operation failed to complete due bus error, cover the case that * only knows the bus error, but not sure which direction error. */ RTE_DMA_STATUS_BUS_ERROR, /** The operation failed to complete due data poison. */ RTE_DMA_STATUS_DATA_POISION, /** The operation failed to complete due descriptor read error. */ RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR, /** The operation failed to complete due device link error. * Used to indicates that the link error in the memory-to-device/ * device-to-memory/device-to-device transfer scenario. */ RTE_DMA_STATUS_DEV_LINK_ERROR, /** The operation failed to complete due lookup page fault. */ RTE_DMA_STATUS_PAGE_FAULT, /** The operation failed to complete due unknown reason. * The initial value is 256, which reserves space for future errors. */ RTE_DMA_STATUS_ERROR_UNKNOWN = 0x100, }; /** * A structure used to hold scatter-gather DMA operation request entry. * * @see rte_dma_copy_sg */ struct rte_dma_sge { rte_iova_t addr; /**< The DMA operation address. */ uint32_t length; /**< The DMA operation length. */ }; #include "rte_dmadev_core.h" /**@{@name DMA operation flag * @see rte_dma_copy() * @see rte_dma_copy_sg() * @see rte_dma_fill() */ /** Fence flag. * It means the operation with this flag must be processed only after all * previous operations are completed. * If the specify DMA HW works in-order (it means it has default fence between * operations), this flag could be NOP. */ #define RTE_DMA_OP_FLAG_FENCE RTE_BIT64(0) /** Submit flag. * It means the operation with this flag must issue doorbell to hardware after * enqueued jobs. */ #define RTE_DMA_OP_FLAG_SUBMIT RTE_BIT64(1) /** Write data to low level cache hint. * Used for performance optimization, this is just a hint, and there is no * capability bit for this, driver should not return error if this flag was set. */ #define RTE_DMA_OP_FLAG_LLC RTE_BIT64(2) /**@}*/ /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Enqueue a copy operation onto the virtual DMA channel. * * This queues up a copy operation to be performed by hardware, if the 'flags' * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin * this operation, otherwise do not trigger doorbell. * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. * @param src * The address of the source buffer. * @param dst * The address of the destination buffer. * @param length * The length of the data to be copied. * @param flags * An flags for this operation. * @see RTE_DMA_OP_FLAG_* * * @return * - 0..UINT16_MAX: index of enqueued job. * - -ENOSPC: if no space left to enqueue. * - other values < 0 on failure. */ __rte_experimental static inline int rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst, uint32_t length, uint64_t flags) { struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; #ifdef RTE_DMADEV_DEBUG if (!rte_dma_is_valid(dev_id) || length == 0) return -EINVAL; if (*obj->copy == NULL) return -ENOTSUP; #endif return (*obj->copy)(obj->dev_private, vchan, src, dst, length, flags); } /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Enqueue a scatter-gather list copy operation onto the virtual DMA channel. * * This queues up a scatter-gather list copy operation to be performed by * hardware, if the 'flags' parameter contains RTE_DMA_OP_FLAG_SUBMIT then * trigger doorbell to begin this operation, otherwise do not trigger doorbell. * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. * @param src * The pointer of source scatter-gather entry array. * @param dst * The pointer of destination scatter-gather entry array. * @param nb_src * The number of source scatter-gather entry. * @see struct rte_dma_info::max_sges * @param nb_dst * The number of destination scatter-gather entry. * @see struct rte_dma_info::max_sges * @param flags * An flags for this operation. * @see RTE_DMA_OP_FLAG_* * * @return * - 0..UINT16_MAX: index of enqueued job. * - -ENOSPC: if no space left to enqueue. * - other values < 0 on failure. */ __rte_experimental static inline int rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src, struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, uint64_t flags) { struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; #ifdef RTE_DMADEV_DEBUG if (!rte_dma_is_valid(dev_id) || src == NULL || dst == NULL || nb_src == 0 || nb_dst == 0) return -EINVAL; if (*obj->copy_sg == NULL) return -ENOTSUP; #endif return (*obj->copy_sg)(obj->dev_private, vchan, src, dst, nb_src, nb_dst, flags); } /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Enqueue a fill operation onto the virtual DMA channel. * * This queues up a fill operation to be performed by hardware, if the 'flags' * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin * this operation, otherwise do not trigger doorbell. * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. * @param pattern * The pattern to populate the destination buffer with. * @param dst * The address of the destination buffer. * @param length * The length of the destination buffer. * @param flags * An flags for this operation. * @see RTE_DMA_OP_FLAG_* * * @return * - 0..UINT16_MAX: index of enqueued job. * - -ENOSPC: if no space left to enqueue. * - other values < 0 on failure. */ __rte_experimental static inline int rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern, rte_iova_t dst, uint32_t length, uint64_t flags) { struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; #ifdef RTE_DMADEV_DEBUG if (!rte_dma_is_valid(dev_id) || length == 0) return -EINVAL; if (*obj->fill == NULL) return -ENOTSUP; #endif return (*obj->fill)(obj->dev_private, vchan, pattern, dst, length, flags); } /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Trigger hardware to begin performing enqueued operations. * * Writes the "doorbell" to the hardware to trigger it * to begin the operations previously enqueued by rte_dma_copy/fill(). * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. * * @return * 0 on success. Otherwise negative value is returned. */ __rte_experimental static inline int rte_dma_submit(int16_t dev_id, uint16_t vchan) { struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; #ifdef RTE_DMADEV_DEBUG if (!rte_dma_is_valid(dev_id)) return -EINVAL; if (*obj->submit == NULL) return -ENOTSUP; #endif return (*obj->submit)(obj->dev_private, vchan); } /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Return the number of operations that have been successfully completed. * Once an operation has been reported as completed, the results of that * operation will be visible to all cores on the system. * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. * @param nb_cpls * The maximum number of completed operations that can be processed. * @param[out] last_idx * The last completed operation's ring_idx. * If not required, NULL can be passed in. * @param[out] has_error * Indicates if there are transfer error. * If not required, NULL can be passed in. * * @return * The number of operations that successfully completed. This return value * must be less than or equal to the value of nb_cpls. */ __rte_experimental static inline uint16_t rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls, uint16_t *last_idx, bool *has_error) { struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; uint16_t idx; bool err; #ifdef RTE_DMADEV_DEBUG if (!rte_dma_is_valid(dev_id) || nb_cpls == 0) return 0; if (*obj->completed == NULL) return 0; #endif /* Ensure the pointer values are non-null to simplify drivers. * In most cases these should be compile time evaluated, since this is * an inline function. * - If NULL is explicitly passed as parameter, then compiler knows the * value is NULL * - If address of local variable is passed as parameter, then compiler * can know it's non-NULL. */ if (last_idx == NULL) last_idx = &idx; if (has_error == NULL) has_error = &err; *has_error = false; return (*obj->completed)(obj->dev_private, vchan, nb_cpls, last_idx, has_error); } /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Return the number of operations that have been completed, and the operations * result may succeed or fail. * Once an operation has been reported as completed successfully, the results of that * operation will be visible to all cores on the system. * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. * @param nb_cpls * Indicates the size of status array. * @param[out] last_idx * The last completed operation's ring_idx. * If not required, NULL can be passed in. * @param[out] status * This is a pointer to an array of length 'nb_cpls' that holds the completion * status code of each operation. * @see enum rte_dma_status_code * * @return * The number of operations that completed. This return value must be less * than or equal to the value of nb_cpls. * If this number is greater than zero (assuming n), then n values in the * status array are also set. */ __rte_experimental static inline uint16_t rte_dma_completed_status(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls, uint16_t *last_idx, enum rte_dma_status_code *status) { struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; uint16_t idx; #ifdef RTE_DMADEV_DEBUG if (!rte_dma_is_valid(dev_id) || nb_cpls == 0 || status == NULL) return 0; if (*obj->completed_status == NULL) return 0; #endif if (last_idx == NULL) last_idx = &idx; return (*obj->completed_status)(obj->dev_private, vchan, nb_cpls, last_idx, status); } /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * * Check remaining capacity in descriptor ring for the current burst. * * @param dev_id * The identifier of the device. * @param vchan * The identifier of virtual DMA channel. * * @return * - Remaining space in the descriptor ring for the current burst. * - 0 on error */ __rte_experimental static inline uint16_t rte_dma_burst_capacity(int16_t dev_id, uint16_t vchan) { struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id]; #ifdef RTE_DMADEV_DEBUG if (!rte_dma_is_valid(dev_id)) return 0; if (*obj->burst_capacity == NULL) return 0; #endif return (*obj->burst_capacity)(obj->dev_private, vchan); } #ifdef __cplusplus } #endif #endif /* RTE_DMADEV_H */