8f1d23ece0
Those macros have no real value and are easily replaced with a simple if() block. Existing users have been converted using a new cocci script. Deprecate them. Signed-off-by: David Marchand <david.marchand@redhat.com>
1149 lines
36 KiB
C
1149 lines
36 KiB
C
/* SPDX-License-Identifier: BSD-3-Clause
|
|
* Copyright(c) 2021 HiSilicon Limited
|
|
* Copyright(c) 2021 Intel Corporation
|
|
* Copyright(c) 2021 Marvell International Ltd
|
|
* Copyright(c) 2021 SmartShare Systems
|
|
*/
|
|
|
|
#ifndef RTE_DMADEV_H
|
|
#define RTE_DMADEV_H
|
|
|
|
/**
|
|
* @file rte_dmadev.h
|
|
*
|
|
* DMA (Direct Memory Access) device API.
|
|
*
|
|
* The DMA framework is built on the following model:
|
|
*
|
|
* --------------- --------------- ---------------
|
|
* | virtual DMA | | virtual DMA | | virtual DMA |
|
|
* | channel | | channel | | channel |
|
|
* --------------- --------------- ---------------
|
|
* | | |
|
|
* ------------------ |
|
|
* | |
|
|
* ------------ ------------
|
|
* | dmadev | | dmadev |
|
|
* ------------ ------------
|
|
* | |
|
|
* ------------------ ------------------
|
|
* | HW DMA channel | | HW DMA channel |
|
|
* ------------------ ------------------
|
|
* | |
|
|
* --------------------------------
|
|
* |
|
|
* ---------------------
|
|
* | HW DMA Controller |
|
|
* ---------------------
|
|
*
|
|
* The DMA controller could have multiple HW-DMA-channels (aka. HW-DMA-queues),
|
|
* each HW-DMA-channel should be represented by a dmadev.
|
|
*
|
|
* The dmadev could create multiple virtual DMA channels, each virtual DMA
|
|
* channel represents a different transfer context. The DMA operation request
|
|
* must be submitted to the virtual DMA channel. e.g. Application could create
|
|
* virtual DMA channel 0 for memory-to-memory transfer scenario, and create
|
|
* virtual DMA channel 1 for memory-to-device transfer scenario.
|
|
*
|
|
* This framework uses 'int16_t dev_id' as the device identifier of a dmadev,
|
|
* and 'uint16_t vchan' as the virtual DMA channel identifier in one dmadev.
|
|
*
|
|
* The functions exported by the dmadev API to setup a device designated by its
|
|
* device identifier must be invoked in the following order:
|
|
* - rte_dma_configure()
|
|
* - rte_dma_vchan_setup()
|
|
* - rte_dma_start()
|
|
*
|
|
* Then, the application can invoke dataplane functions to process jobs.
|
|
*
|
|
* If the application wants to change the configuration (i.e. invoke
|
|
* rte_dma_configure() or rte_dma_vchan_setup()), it must invoke
|
|
* rte_dma_stop() first to stop the device and then do the reconfiguration
|
|
* before invoking rte_dma_start() again. The dataplane functions should not
|
|
* be invoked when the device is stopped.
|
|
*
|
|
* Finally, an application can close a dmadev by invoking the rte_dma_close()
|
|
* function.
|
|
*
|
|
* The dataplane APIs include two parts:
|
|
* The first part is the submission of operation requests:
|
|
* - rte_dma_copy()
|
|
* - rte_dma_copy_sg()
|
|
* - rte_dma_fill()
|
|
* - rte_dma_submit()
|
|
*
|
|
* These APIs could work with different virtual DMA channels which have
|
|
* different contexts.
|
|
*
|
|
* The first three APIs are used to submit the operation request to the virtual
|
|
* DMA channel, if the submission is successful, a positive
|
|
* ring_idx <= UINT16_MAX is returned, otherwise a negative number is returned.
|
|
*
|
|
* The last API is used to issue doorbell to hardware, and also there are flags
|
|
* (@see RTE_DMA_OP_FLAG_SUBMIT) parameter of the first three APIs could do the
|
|
* same work.
|
|
* @note When enqueuing a set of jobs to the device, having a separate submit
|
|
* outside a loop makes for clearer code than having a check for the last
|
|
* iteration inside the loop to set a special submit flag. However, for cases
|
|
* where one item alone is to be submitted or there is a small set of jobs to
|
|
* be submitted sequentially, having a submit flag provides a lower-overhead
|
|
* way of doing the submission while still keeping the code clean.
|
|
*
|
|
* The second part is to obtain the result of requests:
|
|
* - rte_dma_completed()
|
|
* - return the number of operation requests completed successfully.
|
|
* - rte_dma_completed_status()
|
|
* - return the number of operation requests completed.
|
|
*
|
|
* @note If the dmadev works in silent mode (@see RTE_DMA_CAPA_SILENT),
|
|
* application does not invoke the above two completed APIs.
|
|
*
|
|
* About the ring_idx which enqueue APIs (e.g. rte_dma_copy(), rte_dma_fill())
|
|
* return, the rules are as follows:
|
|
* - ring_idx for each virtual DMA channel are independent.
|
|
* - For a virtual DMA channel, the ring_idx is monotonically incremented,
|
|
* when it reach UINT16_MAX, it wraps back to zero.
|
|
* - This ring_idx can be used by applications to track per-operation
|
|
* metadata in an application-defined circular ring.
|
|
* - The initial ring_idx of a virtual DMA channel is zero, after the
|
|
* device is stopped, the ring_idx needs to be reset to zero.
|
|
*
|
|
* One example:
|
|
* - step-1: start one dmadev
|
|
* - step-2: enqueue a copy operation, the ring_idx return is 0
|
|
* - step-3: enqueue a copy operation again, the ring_idx return is 1
|
|
* - ...
|
|
* - step-101: stop the dmadev
|
|
* - step-102: start the dmadev
|
|
* - step-103: enqueue a copy operation, the ring_idx return is 0
|
|
* - ...
|
|
* - step-x+0: enqueue a fill operation, the ring_idx return is 65535
|
|
* - step-x+1: enqueue a copy operation, the ring_idx return is 0
|
|
* - ...
|
|
*
|
|
* The DMA operation address used in enqueue APIs (i.e. rte_dma_copy(),
|
|
* rte_dma_copy_sg(), rte_dma_fill()) is defined as rte_iova_t type.
|
|
*
|
|
* The dmadev supports two types of address: memory address and device address.
|
|
*
|
|
* - memory address: the source and destination address of the memory-to-memory
|
|
* transfer type, or the source address of the memory-to-device transfer type,
|
|
* or the destination address of the device-to-memory transfer type.
|
|
* @note If the device support SVA (@see RTE_DMA_CAPA_SVA), the memory address
|
|
* can be any VA address, otherwise it must be an IOVA address.
|
|
*
|
|
* - device address: the source and destination address of the device-to-device
|
|
* transfer type, or the source address of the device-to-memory transfer type,
|
|
* or the destination address of the memory-to-device transfer type.
|
|
*
|
|
* About MT-safe, all the functions of the dmadev API implemented by a PMD are
|
|
* lock-free functions which assume to not be invoked in parallel on different
|
|
* logical cores to work on the same target dmadev object.
|
|
* @note Different virtual DMA channels on the same dmadev *DO NOT* support
|
|
* parallel invocation because these virtual DMA channels share the same
|
|
* HW-DMA-channel.
|
|
*/
|
|
|
|
#include <stdint.h>
|
|
|
|
#include <rte_bitops.h>
|
|
#include <rte_common.h>
|
|
#include <rte_compat.h>
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
/** Maximum number of devices if rte_dma_dev_max() is not called. */
|
|
#define RTE_DMADEV_DEFAULT_MAX 64
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Configure the maximum number of dmadevs.
|
|
* @note This function can be invoked before the primary process rte_eal_init()
|
|
* to change the maximum number of dmadevs. If not invoked, the maximum number
|
|
* of dmadevs is @see RTE_DMADEV_DEFAULT_MAX
|
|
*
|
|
* @param dev_max
|
|
* maximum number of dmadevs.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_dev_max(size_t dev_max);
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Get the device identifier for the named DMA device.
|
|
*
|
|
* @param name
|
|
* DMA device name.
|
|
*
|
|
* @return
|
|
* Returns DMA device identifier on success.
|
|
* - <0: Failure to find named DMA device.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_get_dev_id_by_name(const char *name);
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Check whether the dev_id is valid.
|
|
*
|
|
* @param dev_id
|
|
* DMA device index.
|
|
*
|
|
* @return
|
|
* - If the device index is valid (true) or not (false).
|
|
*/
|
|
__rte_experimental
|
|
bool rte_dma_is_valid(int16_t dev_id);
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Get the total number of DMA devices that have been successfully
|
|
* initialised.
|
|
*
|
|
* @return
|
|
* The total number of usable DMA devices.
|
|
*/
|
|
__rte_experimental
|
|
uint16_t rte_dma_count_avail(void);
|
|
|
|
/**
|
|
* Iterates over valid dmadev instances.
|
|
*
|
|
* @param start_dev_id
|
|
* The id of the next possible dmadev.
|
|
* @return
|
|
* Next valid dmadev, UINT16_MAX if there is none.
|
|
*/
|
|
__rte_experimental
|
|
int16_t rte_dma_next_dev(int16_t start_dev_id);
|
|
|
|
/** Utility macro to iterate over all available dmadevs */
|
|
#define RTE_DMA_FOREACH_DEV(p) \
|
|
for (p = rte_dma_next_dev(0); \
|
|
p != -1; \
|
|
p = rte_dma_next_dev(p + 1))
|
|
|
|
|
|
/**@{@name DMA capability
|
|
* @see struct rte_dma_info::dev_capa
|
|
*/
|
|
/** Support memory-to-memory transfer */
|
|
#define RTE_DMA_CAPA_MEM_TO_MEM RTE_BIT64(0)
|
|
/** Support memory-to-device transfer. */
|
|
#define RTE_DMA_CAPA_MEM_TO_DEV RTE_BIT64(1)
|
|
/** Support device-to-memory transfer. */
|
|
#define RTE_DMA_CAPA_DEV_TO_MEM RTE_BIT64(2)
|
|
/** Support device-to-device transfer. */
|
|
#define RTE_DMA_CAPA_DEV_TO_DEV RTE_BIT64(3)
|
|
/** Support SVA which could use VA as DMA address.
|
|
* If device support SVA then application could pass any VA address like memory
|
|
* from rte_malloc(), rte_memzone(), malloc, stack memory.
|
|
* If device don't support SVA, then application should pass IOVA address which
|
|
* from rte_malloc(), rte_memzone().
|
|
*/
|
|
#define RTE_DMA_CAPA_SVA RTE_BIT64(4)
|
|
/** Support work in silent mode.
|
|
* In this mode, application don't required to invoke rte_dma_completed*()
|
|
* API.
|
|
* @see struct rte_dma_conf::silent_mode
|
|
*/
|
|
#define RTE_DMA_CAPA_SILENT RTE_BIT64(5)
|
|
/** Supports error handling
|
|
*
|
|
* With this bit set, invalid input addresses will be reported as operation failures
|
|
* to the user but other operations can continue.
|
|
* Without this bit set, invalid data is not handled by either HW or driver, so user
|
|
* must ensure that all memory addresses are valid and accessible by HW.
|
|
*/
|
|
#define RTE_DMA_CAPA_HANDLES_ERRORS RTE_BIT64(6)
|
|
/** Support copy operation.
|
|
* This capability start with index of 32, so that it could leave gap between
|
|
* normal capability and ops capability.
|
|
*/
|
|
#define RTE_DMA_CAPA_OPS_COPY RTE_BIT64(32)
|
|
/** Support scatter-gather list copy operation. */
|
|
#define RTE_DMA_CAPA_OPS_COPY_SG RTE_BIT64(33)
|
|
/** Support fill operation. */
|
|
#define RTE_DMA_CAPA_OPS_FILL RTE_BIT64(34)
|
|
/**@}*/
|
|
|
|
/**
|
|
* A structure used to retrieve the information of a DMA device.
|
|
*
|
|
* @see rte_dma_info_get
|
|
*/
|
|
struct rte_dma_info {
|
|
const char *dev_name; /**< Unique device name. */
|
|
/** Device capabilities (RTE_DMA_CAPA_*). */
|
|
uint64_t dev_capa;
|
|
/** Maximum number of virtual DMA channels supported. */
|
|
uint16_t max_vchans;
|
|
/** Maximum allowed number of virtual DMA channel descriptors. */
|
|
uint16_t max_desc;
|
|
/** Minimum allowed number of virtual DMA channel descriptors. */
|
|
uint16_t min_desc;
|
|
/** Maximum number of source or destination scatter-gather entry
|
|
* supported.
|
|
* If the device does not support COPY_SG capability, this value can be
|
|
* zero.
|
|
* If the device supports COPY_SG capability, then rte_dma_copy_sg()
|
|
* parameter nb_src/nb_dst should not exceed this value.
|
|
*/
|
|
uint16_t max_sges;
|
|
/** NUMA node connection, -1 if unknown. */
|
|
int16_t numa_node;
|
|
/** Number of virtual DMA channel configured. */
|
|
uint16_t nb_vchans;
|
|
};
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Retrieve information of a DMA device.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param[out] dev_info
|
|
* A pointer to a structure of type *rte_dma_info* to be filled with the
|
|
* information of the device.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info);
|
|
|
|
/**
|
|
* A structure used to configure a DMA device.
|
|
*
|
|
* @see rte_dma_configure
|
|
*/
|
|
struct rte_dma_conf {
|
|
/** The number of virtual DMA channels to set up for the DMA device.
|
|
* This value cannot be greater than the field 'max_vchans' of struct
|
|
* rte_dma_info which get from rte_dma_info_get().
|
|
*/
|
|
uint16_t nb_vchans;
|
|
/** Indicates whether to enable silent mode.
|
|
* false-default mode, true-silent mode.
|
|
* This value can be set to true only when the SILENT capability is
|
|
* supported.
|
|
*
|
|
* @see RTE_DMA_CAPA_SILENT
|
|
*/
|
|
bool enable_silent;
|
|
};
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Configure a DMA device.
|
|
*
|
|
* This function must be invoked first before any other function in the
|
|
* API. This function can also be re-invoked when a device is in the
|
|
* stopped state.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device to configure.
|
|
* @param dev_conf
|
|
* The DMA device configuration structure encapsulated into rte_dma_conf
|
|
* object.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf);
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Start a DMA device.
|
|
*
|
|
* The device start step is the last one and consists of setting the DMA
|
|
* to start accepting jobs.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_start(int16_t dev_id);
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Stop a DMA device.
|
|
*
|
|
* The device can be restarted with a call to rte_dma_start().
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_stop(int16_t dev_id);
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Close a DMA device.
|
|
*
|
|
* The device cannot be restarted after this call.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_close(int16_t dev_id);
|
|
|
|
/**
|
|
* DMA transfer direction defines.
|
|
*
|
|
* @see struct rte_dma_vchan_conf::direction
|
|
*/
|
|
enum rte_dma_direction {
|
|
/** DMA transfer direction - from memory to memory.
|
|
*
|
|
* @see struct rte_dma_vchan_conf::direction
|
|
*/
|
|
RTE_DMA_DIR_MEM_TO_MEM,
|
|
/** DMA transfer direction - from memory to device.
|
|
* In a typical scenario, the SoCs are installed on host servers as
|
|
* iNICs through the PCIe interface. In this case, the SoCs works in
|
|
* EP(endpoint) mode, it could initiate a DMA move request from memory
|
|
* (which is SoCs memory) to device (which is host memory).
|
|
*
|
|
* @see struct rte_dma_vchan_conf::direction
|
|
*/
|
|
RTE_DMA_DIR_MEM_TO_DEV,
|
|
/** DMA transfer direction - from device to memory.
|
|
* In a typical scenario, the SoCs are installed on host servers as
|
|
* iNICs through the PCIe interface. In this case, the SoCs works in
|
|
* EP(endpoint) mode, it could initiate a DMA move request from device
|
|
* (which is host memory) to memory (which is SoCs memory).
|
|
*
|
|
* @see struct rte_dma_vchan_conf::direction
|
|
*/
|
|
RTE_DMA_DIR_DEV_TO_MEM,
|
|
/** DMA transfer direction - from device to device.
|
|
* In a typical scenario, the SoCs are installed on host servers as
|
|
* iNICs through the PCIe interface. In this case, the SoCs works in
|
|
* EP(endpoint) mode, it could initiate a DMA move request from device
|
|
* (which is host memory) to the device (which is another host memory).
|
|
*
|
|
* @see struct rte_dma_vchan_conf::direction
|
|
*/
|
|
RTE_DMA_DIR_DEV_TO_DEV,
|
|
};
|
|
|
|
/**
|
|
* DMA access port type defines.
|
|
*
|
|
* @see struct rte_dma_port_param::port_type
|
|
*/
|
|
enum rte_dma_port_type {
|
|
RTE_DMA_PORT_NONE,
|
|
RTE_DMA_PORT_PCIE, /**< The DMA access port is PCIe. */
|
|
};
|
|
|
|
/**
|
|
* A structure used to descript DMA access port parameters.
|
|
*
|
|
* @see struct rte_dma_vchan_conf::src_port
|
|
* @see struct rte_dma_vchan_conf::dst_port
|
|
*/
|
|
struct rte_dma_port_param {
|
|
/** The device access port type.
|
|
*
|
|
* @see enum rte_dma_port_type
|
|
*/
|
|
enum rte_dma_port_type port_type;
|
|
RTE_STD_C11
|
|
union {
|
|
/** PCIe access port parameters.
|
|
*
|
|
* The following model shows SoC's PCIe module connects to
|
|
* multiple PCIe hosts and multiple endpoints. The PCIe module
|
|
* has an integrated DMA controller.
|
|
*
|
|
* If the DMA wants to access the memory of host A, it can be
|
|
* initiated by PF1 in core0, or by VF0 of PF0 in core0.
|
|
*
|
|
* \code{.unparsed}
|
|
* System Bus
|
|
* | ----------PCIe module----------
|
|
* | Bus
|
|
* | Interface
|
|
* | ----- ------------------
|
|
* | | | | PCIe Core0 |
|
|
* | | | | | -----------
|
|
* | | | | PF-0 -- VF-0 | | Host A |
|
|
* | | |--------| |- VF-1 |--------| Root |
|
|
* | | | | PF-1 | | Complex |
|
|
* | | | | PF-2 | -----------
|
|
* | | | ------------------
|
|
* | | |
|
|
* | | | ------------------
|
|
* | | | | PCIe Core1 |
|
|
* | | | | | -----------
|
|
* | | | | PF-0 -- VF-0 | | Host B |
|
|
* |-----| |--------| PF-1 -- VF-0 |--------| Root |
|
|
* | | | | |- VF-1 | | Complex |
|
|
* | | | | PF-2 | -----------
|
|
* | | | ------------------
|
|
* | | |
|
|
* | | | ------------------
|
|
* | |DMA| | | ------
|
|
* | | | | |--------| EP |
|
|
* | | |--------| PCIe Core2 | ------
|
|
* | | | | | ------
|
|
* | | | | |--------| EP |
|
|
* | | | | | ------
|
|
* | ----- ------------------
|
|
*
|
|
* \endcode
|
|
*
|
|
* @note If some fields can not be supported by the
|
|
* hardware/driver, then the driver ignores those fields.
|
|
* Please check driver-specific documentation for limitations
|
|
* and capabilities.
|
|
*/
|
|
__extension__
|
|
struct {
|
|
uint64_t coreid : 4; /**< PCIe core id used. */
|
|
uint64_t pfid : 8; /**< PF id used. */
|
|
uint64_t vfen : 1; /**< VF enable bit. */
|
|
uint64_t vfid : 16; /**< VF id used. */
|
|
/** The pasid filed in TLP packet. */
|
|
uint64_t pasid : 20;
|
|
/** The attributes filed in TLP packet. */
|
|
uint64_t attr : 3;
|
|
/** The processing hint filed in TLP packet. */
|
|
uint64_t ph : 2;
|
|
/** The steering tag filed in TLP packet. */
|
|
uint64_t st : 16;
|
|
} pcie;
|
|
};
|
|
uint64_t reserved[2]; /**< Reserved for future fields. */
|
|
};
|
|
|
|
/**
|
|
* A structure used to configure a virtual DMA channel.
|
|
*
|
|
* @see rte_dma_vchan_setup
|
|
*/
|
|
struct rte_dma_vchan_conf {
|
|
/** Transfer direction
|
|
*
|
|
* @see enum rte_dma_direction
|
|
*/
|
|
enum rte_dma_direction direction;
|
|
/** Number of descriptor for the virtual DMA channel */
|
|
uint16_t nb_desc;
|
|
/** 1) Used to describes the device access port parameter in the
|
|
* device-to-memory transfer scenario.
|
|
* 2) Used to describes the source device access port parameter in the
|
|
* device-to-device transfer scenario.
|
|
*
|
|
* @see struct rte_dma_port_param
|
|
*/
|
|
struct rte_dma_port_param src_port;
|
|
/** 1) Used to describes the device access port parameter in the
|
|
* memory-to-device transfer scenario.
|
|
* 2) Used to describes the destination device access port parameter in
|
|
* the device-to-device transfer scenario.
|
|
*
|
|
* @see struct rte_dma_port_param
|
|
*/
|
|
struct rte_dma_port_param dst_port;
|
|
};
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Allocate and set up a virtual DMA channel.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel. The value must be in the range
|
|
* [0, nb_vchans - 1] previously supplied to rte_dma_configure().
|
|
* @param conf
|
|
* The virtual DMA channel configuration structure encapsulated into
|
|
* rte_dma_vchan_conf object.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan,
|
|
const struct rte_dma_vchan_conf *conf);
|
|
|
|
/**
|
|
* A structure used to retrieve statistics.
|
|
*
|
|
* @see rte_dma_stats_get
|
|
*/
|
|
struct rte_dma_stats {
|
|
/** Count of operations which were submitted to hardware. */
|
|
uint64_t submitted;
|
|
/** Count of operations which were completed, including successful and
|
|
* failed completions.
|
|
*/
|
|
uint64_t completed;
|
|
/** Count of operations which failed to complete. */
|
|
uint64_t errors;
|
|
};
|
|
|
|
/**
|
|
* Special ID, which is used to represent all virtual DMA channels.
|
|
*
|
|
* @see rte_dma_stats_get
|
|
* @see rte_dma_stats_reset
|
|
*/
|
|
#define RTE_DMA_ALL_VCHAN 0xFFFFu
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Retrieve basic statistics of a or all virtual DMA channel(s).
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel.
|
|
* If equal RTE_DMA_ALL_VCHAN means all channels.
|
|
* @param[out] stats
|
|
* The basic statistics structure encapsulated into rte_dma_stats
|
|
* object.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_stats_get(int16_t dev_id, uint16_t vchan,
|
|
struct rte_dma_stats *stats);
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Reset basic statistics of a or all virtual DMA channel(s).
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel.
|
|
* If equal RTE_DMA_ALL_VCHAN means all channels.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan);
|
|
|
|
/**
|
|
* device vchannel status
|
|
*
|
|
* Enum with the options for the channel status, either idle, active or halted due to error
|
|
* @see rte_dma_vchan_status
|
|
*/
|
|
enum rte_dma_vchan_status {
|
|
RTE_DMA_VCHAN_IDLE, /**< not processing, awaiting ops */
|
|
RTE_DMA_VCHAN_ACTIVE, /**< currently processing jobs */
|
|
RTE_DMA_VCHAN_HALTED_ERROR, /**< not processing due to error, cannot accept new ops */
|
|
};
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Determine if all jobs have completed on a device channel.
|
|
* This function is primarily designed for testing use, as it allows a process to check if
|
|
* all jobs are completed, without actually gathering completions from those jobs.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel.
|
|
* @param[out] status
|
|
* The vchan status
|
|
* @return
|
|
* 0 - call completed successfully
|
|
* < 0 - error code indicating there was a problem calling the API
|
|
*/
|
|
__rte_experimental
|
|
int
|
|
rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *status);
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Dump DMA device info.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param f
|
|
* The file to write the output to.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
int rte_dma_dump(int16_t dev_id, FILE *f);
|
|
|
|
/**
|
|
* DMA transfer result status code defines.
|
|
*
|
|
* @see rte_dma_completed_status
|
|
*/
|
|
enum rte_dma_status_code {
|
|
/** The operation completed successfully. */
|
|
RTE_DMA_STATUS_SUCCESSFUL,
|
|
/** The operation failed to complete due abort by user.
|
|
* This is mainly used when processing dev_stop, user could modify the
|
|
* descriptors (e.g. change one bit to tell hardware abort this job),
|
|
* it allows outstanding requests to be complete as much as possible,
|
|
* so reduce the time to stop the device.
|
|
*/
|
|
RTE_DMA_STATUS_USER_ABORT,
|
|
/** The operation failed to complete due to following scenarios:
|
|
* The jobs in a particular batch are not attempted because they
|
|
* appeared after a fence where a previous job failed. In some HW
|
|
* implementation it's possible for jobs from later batches would be
|
|
* completed, though, so report the status from the not attempted jobs
|
|
* before reporting those newer completed jobs.
|
|
*/
|
|
RTE_DMA_STATUS_NOT_ATTEMPTED,
|
|
/** The operation failed to complete due invalid source address. */
|
|
RTE_DMA_STATUS_INVALID_SRC_ADDR,
|
|
/** The operation failed to complete due invalid destination address. */
|
|
RTE_DMA_STATUS_INVALID_DST_ADDR,
|
|
/** The operation failed to complete due invalid source or destination
|
|
* address, cover the case that only knows the address error, but not
|
|
* sure which address error.
|
|
*/
|
|
RTE_DMA_STATUS_INVALID_ADDR,
|
|
/** The operation failed to complete due invalid length. */
|
|
RTE_DMA_STATUS_INVALID_LENGTH,
|
|
/** The operation failed to complete due invalid opcode.
|
|
* The DMA descriptor could have multiple format, which are
|
|
* distinguished by the opcode field.
|
|
*/
|
|
RTE_DMA_STATUS_INVALID_OPCODE,
|
|
/** The operation failed to complete due bus read error. */
|
|
RTE_DMA_STATUS_BUS_READ_ERROR,
|
|
/** The operation failed to complete due bus write error. */
|
|
RTE_DMA_STATUS_BUS_WRITE_ERROR,
|
|
/** The operation failed to complete due bus error, cover the case that
|
|
* only knows the bus error, but not sure which direction error.
|
|
*/
|
|
RTE_DMA_STATUS_BUS_ERROR,
|
|
/** The operation failed to complete due data poison. */
|
|
RTE_DMA_STATUS_DATA_POISION,
|
|
/** The operation failed to complete due descriptor read error. */
|
|
RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR,
|
|
/** The operation failed to complete due device link error.
|
|
* Used to indicates that the link error in the memory-to-device/
|
|
* device-to-memory/device-to-device transfer scenario.
|
|
*/
|
|
RTE_DMA_STATUS_DEV_LINK_ERROR,
|
|
/** The operation failed to complete due lookup page fault. */
|
|
RTE_DMA_STATUS_PAGE_FAULT,
|
|
/** The operation failed to complete due unknown reason.
|
|
* The initial value is 256, which reserves space for future errors.
|
|
*/
|
|
RTE_DMA_STATUS_ERROR_UNKNOWN = 0x100,
|
|
};
|
|
|
|
/**
|
|
* A structure used to hold scatter-gather DMA operation request entry.
|
|
*
|
|
* @see rte_dma_copy_sg
|
|
*/
|
|
struct rte_dma_sge {
|
|
rte_iova_t addr; /**< The DMA operation address. */
|
|
uint32_t length; /**< The DMA operation length. */
|
|
};
|
|
|
|
#include "rte_dmadev_core.h"
|
|
|
|
/**@{@name DMA operation flag
|
|
* @see rte_dma_copy()
|
|
* @see rte_dma_copy_sg()
|
|
* @see rte_dma_fill()
|
|
*/
|
|
/** Fence flag.
|
|
* It means the operation with this flag must be processed only after all
|
|
* previous operations are completed.
|
|
* If the specify DMA HW works in-order (it means it has default fence between
|
|
* operations), this flag could be NOP.
|
|
*/
|
|
#define RTE_DMA_OP_FLAG_FENCE RTE_BIT64(0)
|
|
/** Submit flag.
|
|
* It means the operation with this flag must issue doorbell to hardware after
|
|
* enqueued jobs.
|
|
*/
|
|
#define RTE_DMA_OP_FLAG_SUBMIT RTE_BIT64(1)
|
|
/** Write data to low level cache hint.
|
|
* Used for performance optimization, this is just a hint, and there is no
|
|
* capability bit for this, driver should not return error if this flag was set.
|
|
*/
|
|
#define RTE_DMA_OP_FLAG_LLC RTE_BIT64(2)
|
|
/**@}*/
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Enqueue a copy operation onto the virtual DMA channel.
|
|
*
|
|
* This queues up a copy operation to be performed by hardware, if the 'flags'
|
|
* parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin
|
|
* this operation, otherwise do not trigger doorbell.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel.
|
|
* @param src
|
|
* The address of the source buffer.
|
|
* @param dst
|
|
* The address of the destination buffer.
|
|
* @param length
|
|
* The length of the data to be copied.
|
|
* @param flags
|
|
* An flags for this operation.
|
|
* @see RTE_DMA_OP_FLAG_*
|
|
*
|
|
* @return
|
|
* - 0..UINT16_MAX: index of enqueued job.
|
|
* - -ENOSPC: if no space left to enqueue.
|
|
* - other values < 0 on failure.
|
|
*/
|
|
__rte_experimental
|
|
static inline int
|
|
rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
|
|
uint32_t length, uint64_t flags)
|
|
{
|
|
struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
|
|
|
|
#ifdef RTE_DMADEV_DEBUG
|
|
if (!rte_dma_is_valid(dev_id) || length == 0)
|
|
return -EINVAL;
|
|
if (*obj->copy == NULL)
|
|
return -ENOTSUP;
|
|
#endif
|
|
|
|
return (*obj->copy)(obj->dev_private, vchan, src, dst, length, flags);
|
|
}
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Enqueue a scatter-gather list copy operation onto the virtual DMA channel.
|
|
*
|
|
* This queues up a scatter-gather list copy operation to be performed by
|
|
* hardware, if the 'flags' parameter contains RTE_DMA_OP_FLAG_SUBMIT then
|
|
* trigger doorbell to begin this operation, otherwise do not trigger doorbell.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel.
|
|
* @param src
|
|
* The pointer of source scatter-gather entry array.
|
|
* @param dst
|
|
* The pointer of destination scatter-gather entry array.
|
|
* @param nb_src
|
|
* The number of source scatter-gather entry.
|
|
* @see struct rte_dma_info::max_sges
|
|
* @param nb_dst
|
|
* The number of destination scatter-gather entry.
|
|
* @see struct rte_dma_info::max_sges
|
|
* @param flags
|
|
* An flags for this operation.
|
|
* @see RTE_DMA_OP_FLAG_*
|
|
*
|
|
* @return
|
|
* - 0..UINT16_MAX: index of enqueued job.
|
|
* - -ENOSPC: if no space left to enqueue.
|
|
* - other values < 0 on failure.
|
|
*/
|
|
__rte_experimental
|
|
static inline int
|
|
rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src,
|
|
struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst,
|
|
uint64_t flags)
|
|
{
|
|
struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
|
|
|
|
#ifdef RTE_DMADEV_DEBUG
|
|
if (!rte_dma_is_valid(dev_id) || src == NULL || dst == NULL ||
|
|
nb_src == 0 || nb_dst == 0)
|
|
return -EINVAL;
|
|
if (*obj->copy_sg == NULL)
|
|
return -ENOTSUP;
|
|
#endif
|
|
|
|
return (*obj->copy_sg)(obj->dev_private, vchan, src, dst, nb_src,
|
|
nb_dst, flags);
|
|
}
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Enqueue a fill operation onto the virtual DMA channel.
|
|
*
|
|
* This queues up a fill operation to be performed by hardware, if the 'flags'
|
|
* parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin
|
|
* this operation, otherwise do not trigger doorbell.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel.
|
|
* @param pattern
|
|
* The pattern to populate the destination buffer with.
|
|
* @param dst
|
|
* The address of the destination buffer.
|
|
* @param length
|
|
* The length of the destination buffer.
|
|
* @param flags
|
|
* An flags for this operation.
|
|
* @see RTE_DMA_OP_FLAG_*
|
|
*
|
|
* @return
|
|
* - 0..UINT16_MAX: index of enqueued job.
|
|
* - -ENOSPC: if no space left to enqueue.
|
|
* - other values < 0 on failure.
|
|
*/
|
|
__rte_experimental
|
|
static inline int
|
|
rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern,
|
|
rte_iova_t dst, uint32_t length, uint64_t flags)
|
|
{
|
|
struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
|
|
|
|
#ifdef RTE_DMADEV_DEBUG
|
|
if (!rte_dma_is_valid(dev_id) || length == 0)
|
|
return -EINVAL;
|
|
if (*obj->fill == NULL)
|
|
return -ENOTSUP;
|
|
#endif
|
|
|
|
return (*obj->fill)(obj->dev_private, vchan, pattern, dst, length,
|
|
flags);
|
|
}
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Trigger hardware to begin performing enqueued operations.
|
|
*
|
|
* Writes the "doorbell" to the hardware to trigger it
|
|
* to begin the operations previously enqueued by rte_dma_copy/fill().
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel.
|
|
*
|
|
* @return
|
|
* 0 on success. Otherwise negative value is returned.
|
|
*/
|
|
__rte_experimental
|
|
static inline int
|
|
rte_dma_submit(int16_t dev_id, uint16_t vchan)
|
|
{
|
|
struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
|
|
|
|
#ifdef RTE_DMADEV_DEBUG
|
|
if (!rte_dma_is_valid(dev_id))
|
|
return -EINVAL;
|
|
if (*obj->submit == NULL)
|
|
return -ENOTSUP;
|
|
#endif
|
|
|
|
return (*obj->submit)(obj->dev_private, vchan);
|
|
}
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Return the number of operations that have been successfully completed.
|
|
* Once an operation has been reported as completed, the results of that
|
|
* operation will be visible to all cores on the system.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel.
|
|
* @param nb_cpls
|
|
* The maximum number of completed operations that can be processed.
|
|
* @param[out] last_idx
|
|
* The last completed operation's ring_idx.
|
|
* If not required, NULL can be passed in.
|
|
* @param[out] has_error
|
|
* Indicates if there are transfer error.
|
|
* If not required, NULL can be passed in.
|
|
*
|
|
* @return
|
|
* The number of operations that successfully completed. This return value
|
|
* must be less than or equal to the value of nb_cpls.
|
|
*/
|
|
__rte_experimental
|
|
static inline uint16_t
|
|
rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls,
|
|
uint16_t *last_idx, bool *has_error)
|
|
{
|
|
struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
|
|
uint16_t idx;
|
|
bool err;
|
|
|
|
#ifdef RTE_DMADEV_DEBUG
|
|
if (!rte_dma_is_valid(dev_id) || nb_cpls == 0)
|
|
return 0;
|
|
if (*obj->completed == NULL)
|
|
return 0;
|
|
#endif
|
|
|
|
/* Ensure the pointer values are non-null to simplify drivers.
|
|
* In most cases these should be compile time evaluated, since this is
|
|
* an inline function.
|
|
* - If NULL is explicitly passed as parameter, then compiler knows the
|
|
* value is NULL
|
|
* - If address of local variable is passed as parameter, then compiler
|
|
* can know it's non-NULL.
|
|
*/
|
|
if (last_idx == NULL)
|
|
last_idx = &idx;
|
|
if (has_error == NULL)
|
|
has_error = &err;
|
|
|
|
*has_error = false;
|
|
return (*obj->completed)(obj->dev_private, vchan, nb_cpls, last_idx,
|
|
has_error);
|
|
}
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Return the number of operations that have been completed, and the operations
|
|
* result may succeed or fail.
|
|
* Once an operation has been reported as completed successfully, the results of that
|
|
* operation will be visible to all cores on the system.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel.
|
|
* @param nb_cpls
|
|
* Indicates the size of status array.
|
|
* @param[out] last_idx
|
|
* The last completed operation's ring_idx.
|
|
* If not required, NULL can be passed in.
|
|
* @param[out] status
|
|
* This is a pointer to an array of length 'nb_cpls' that holds the completion
|
|
* status code of each operation.
|
|
* @see enum rte_dma_status_code
|
|
*
|
|
* @return
|
|
* The number of operations that completed. This return value must be less
|
|
* than or equal to the value of nb_cpls.
|
|
* If this number is greater than zero (assuming n), then n values in the
|
|
* status array are also set.
|
|
*/
|
|
__rte_experimental
|
|
static inline uint16_t
|
|
rte_dma_completed_status(int16_t dev_id, uint16_t vchan,
|
|
const uint16_t nb_cpls, uint16_t *last_idx,
|
|
enum rte_dma_status_code *status)
|
|
{
|
|
struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
|
|
uint16_t idx;
|
|
|
|
#ifdef RTE_DMADEV_DEBUG
|
|
if (!rte_dma_is_valid(dev_id) || nb_cpls == 0 || status == NULL)
|
|
return 0;
|
|
if (*obj->completed_status == NULL)
|
|
return 0;
|
|
#endif
|
|
|
|
if (last_idx == NULL)
|
|
last_idx = &idx;
|
|
|
|
return (*obj->completed_status)(obj->dev_private, vchan, nb_cpls,
|
|
last_idx, status);
|
|
}
|
|
|
|
/**
|
|
* @warning
|
|
* @b EXPERIMENTAL: this API may change without prior notice.
|
|
*
|
|
* Check remaining capacity in descriptor ring for the current burst.
|
|
*
|
|
* @param dev_id
|
|
* The identifier of the device.
|
|
* @param vchan
|
|
* The identifier of virtual DMA channel.
|
|
*
|
|
* @return
|
|
* - Remaining space in the descriptor ring for the current burst.
|
|
* - 0 on error
|
|
*/
|
|
__rte_experimental
|
|
static inline uint16_t
|
|
rte_dma_burst_capacity(int16_t dev_id, uint16_t vchan)
|
|
{
|
|
struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
|
|
|
|
#ifdef RTE_DMADEV_DEBUG
|
|
if (!rte_dma_is_valid(dev_id))
|
|
return 0;
|
|
if (*obj->burst_capacity == NULL)
|
|
return 0;
|
|
#endif
|
|
return (*obj->burst_capacity)(obj->dev_private, vchan);
|
|
}
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif /* RTE_DMADEV_H */
|