d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Microsoft Azure Network Adapter(MANA) VF support

Browse Source 
MANA is the new network adapter from Microsoft which will be available
in Azure public cloud. It provides SRIOV NIC as virtual function to
guest OS running on Hyper-V.

The code can be divided into two major parts. Gdma_main.c is the one to
bring up the hardware board and drives all underlying hardware queue
infrastructure. Mana_en.c contains all main ethernet driver code.
It has only tested and supported on amd64 architecture.

PR:		256336
Reviewed by:	decui@microsoft.com
Tested by:	whu
MFC after:	2 week
Relnotes:	yes
Sponsored by:	Microsoft
Differential Revision:	https://reviews.freebsd.org/D31150
This commit is contained in:
Wei Hu 2021-08-20 08:43:10 +00:00
parent 464a166c27
commit ce110ea12f
15 changed files with 8243 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
sys/conf/files.x86
View File

@ -264,6 +264,12 @@ dev/isci/scil/scif_sas_task_request_states.c optional isci
dev/isci/scil/scif_sas_timer.c optional isci
dev/itwd/itwd.c optional itwd
dev/kvm_clock/kvm_clock.c optional kvm_clock
dev/mana/gdma_main.c optional mana
dev/mana/mana_en.c optional mana
dev/mana/mana_sysctl.c optional mana
dev/mana/shm_channel.c optional mana
dev/mana/hw_channel.c optional mana
dev/mana/gdma_util.c optional mana
dev/qat/qat.c optional qat
dev/qat/qat_ae.c optional qat
dev/qat/qat_c2xxx.c optional qat

744
sys/dev/mana/gdma.h Normal file
View File

@ -0,0 +1,744 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Microsoft Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD$
*
*/
#ifndef _GDMA_H
#define _GDMA_H
#include <sys/bus.h>
#include <sys/bus_dma.h>
#include <sys/types.h>
#include <sys/limits.h>
#include <sys/sx.h>
#include "gdma_util.h"
#include "shm_channel.h"
/* Structures labeled with "HW DATA" are exchanged with the hardware. All of
* them are naturally aligned and hence don't need __packed.
*/
#define GDMA_BAR0 0
#define GDMA_IRQNAME_SZ 40
struct gdma_bus {
bus_space_handle_t bar0_h;
bus_space_tag_t bar0_t;
};
struct gdma_msix_entry {
int entry;
int vector;
};
enum gdma_request_type {
GDMA_VERIFY_VF_DRIVER_VERSION = 1,
GDMA_QUERY_MAX_RESOURCES = 2,
GDMA_LIST_DEVICES = 3,
GDMA_REGISTER_DEVICE = 4,
GDMA_DEREGISTER_DEVICE = 5,
GDMA_GENERATE_TEST_EQE = 10,
GDMA_CREATE_QUEUE = 12,
GDMA_DISABLE_QUEUE = 13,
GDMA_CREATE_DMA_REGION = 25,
GDMA_DMA_REGION_ADD_PAGES = 26,
GDMA_DESTROY_DMA_REGION = 27,
};
enum gdma_queue_type {
GDMA_INVALID_QUEUE,
GDMA_SQ,
GDMA_RQ,
GDMA_CQ,
GDMA_EQ,
};
enum gdma_work_request_flags {
GDMA_WR_NONE = 0,
GDMA_WR_OOB_IN_SGL = BIT(0),
GDMA_WR_PAD_BY_SGE0 = BIT(1),
};
enum gdma_eqe_type {
GDMA_EQE_COMPLETION = 3,
GDMA_EQE_TEST_EVENT = 64,
GDMA_EQE_HWC_INIT_EQ_ID_DB = 129,
GDMA_EQE_HWC_INIT_DATA = 130,
GDMA_EQE_HWC_INIT_DONE = 131,
};
enum {
GDMA_DEVICE_NONE = 0,
GDMA_DEVICE_HWC = 1,
GDMA_DEVICE_MANA = 2,
};
struct gdma_resource {
/* Protect the bitmap */
struct mtx lock_spin;
/* The bitmap size in bits. */
uint32_t size;
/* The bitmap tracks the resources. */
unsigned long *map;
};
union gdma_doorbell_entry {
uint64_t as_uint64;
struct {
uint64_t id : 24;
uint64_t reserved : 8;
uint64_t tail_ptr : 31;
uint64_t arm : 1;
} cq;
struct {
uint64_t id : 24;
uint64_t wqe_cnt : 8;
uint64_t tail_ptr : 32;
} rq;
struct {
uint64_t id : 24;
uint64_t reserved : 8;
uint64_t tail_ptr : 32;
} sq;
struct {
uint64_t id : 16;
uint64_t reserved : 16;
uint64_t tail_ptr : 31;
uint64_t arm : 1;
} eq;
}; /* HW DATA */
struct gdma_msg_hdr {
uint32_t hdr_type;
uint32_t msg_type;
uint16_t msg_version;
uint16_t hwc_msg_id;
uint32_t msg_size;
}; /* HW DATA */
struct gdma_dev_id {
union {
struct {
uint16_t type;
uint16_t instance;
};
uint32_t as_uint32;
};
}; /* HW DATA */
struct gdma_req_hdr {
struct gdma_msg_hdr req;
struct gdma_msg_hdr resp; /* The expected response */
struct gdma_dev_id dev_id;
uint32_t activity_id;
}; /* HW DATA */
struct gdma_resp_hdr {
struct gdma_msg_hdr response;
struct gdma_dev_id dev_id;
uint32_t activity_id;
uint32_t status;
uint32_t reserved;
}; /* HW DATA */
struct gdma_general_req {
struct gdma_req_hdr hdr;
}; /* HW DATA */
#define GDMA_MESSAGE_V1 1
struct gdma_general_resp {
struct gdma_resp_hdr hdr;
}; /* HW DATA */
#define GDMA_STANDARD_HEADER_TYPE 0
static inline void
mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, uint32_t code,
uint32_t req_size, uint32_t resp_size)
{
hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
hdr->req.msg_type = code;
hdr->req.msg_version = GDMA_MESSAGE_V1;
hdr->req.msg_size = req_size;
hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
hdr->resp.msg_type = code;
hdr->resp.msg_version = GDMA_MESSAGE_V1;
hdr->resp.msg_size = resp_size;
}
/* The 16-byte struct is part of the GDMA work queue entry (WQE). */
struct gdma_sge {
uint64_t address;
uint32_t mem_key;
uint32_t size;
}; /* HW DATA */
struct gdma_wqe_request {
struct gdma_sge *sgl;
uint32_t num_sge;
uint32_t inline_oob_size;
const void *inline_oob_data;
uint32_t flags;
uint32_t client_data_unit;
};
enum gdma_page_type {
GDMA_PAGE_TYPE_4K,
};
#define GDMA_INVALID_DMA_REGION 0
struct gdma_mem_info {
device_t dev;
bus_dma_tag_t dma_tag;
bus_dmamap_t dma_map;
bus_addr_t dma_handle; /* Physical address */
void *virt_addr; /* Virtual address */
uint64_t length;
/* Allocated by the PF driver */
uint64_t gdma_region;
};
#define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
struct gdma_dev {
struct gdma_context *gdma_context;
struct gdma_dev_id dev_id;
uint32_t pdid;
uint32_t doorbell;
uint32_t gpa_mkey;
/* GDMA driver specific pointer */
void *driver_data;
};
#define MINIMUM_SUPPORTED_PAGE_SIZE PAGE_SIZE
#define GDMA_CQE_SIZE 64
#define GDMA_EQE_SIZE 16
#define GDMA_MAX_SQE_SIZE 512
#define GDMA_MAX_RQE_SIZE 256
#define GDMA_COMP_DATA_SIZE 0x3C
#define GDMA_EVENT_DATA_SIZE 0xC
/* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */
#define GDMA_WQE_BU_SIZE 32
#define INVALID_PDID UINT_MAX
#define INVALID_DOORBELL UINT_MAX
#define INVALID_MEM_KEY UINT_MAX
#define INVALID_QUEUE_ID UINT_MAX
#define INVALID_PCI_MSIX_INDEX UINT_MAX
struct gdma_comp {
uint32_t cqe_data[GDMA_COMP_DATA_SIZE / 4];
uint32_t wq_num;
bool is_sq;
};
struct gdma_event {
uint32_t details[GDMA_EVENT_DATA_SIZE / 4];
uint8_t type;
};
struct gdma_queue;
#define CQE_POLLING_BUFFER 512
typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
struct gdma_event *e);
typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
/* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE
* (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the
* driver increases the 'head' in BUs rather than in bytes, and notifies
* the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track
* the HW head, and increases the 'head' by 1 for every processed EQE/CQE.
*
* The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is
* processed, the driver increases the 'tail' to indicate that WQEs have
* been consumed by the HW, so the driver can post new WQEs into the SQ/RQ.
*
* The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures
* that the EQ/CQ is big enough so they can't overflow, and the driver uses
* the owner bits mechanism to detect if the queue has become empty.
*/
struct gdma_queue {
struct gdma_dev *gdma_dev;
enum gdma_queue_type type;
uint32_t id;
struct gdma_mem_info mem_info;
void *queue_mem_ptr;
uint32_t queue_size;
bool monitor_avl_buf;
uint32_t head;
uint32_t tail;
/* Extra fields specific to EQ/CQ. */
union {
struct {
bool disable_needed;
gdma_eq_callback *callback;
void *context;
unsigned int msix_index;
uint32_t log2_throttle_limit;
struct task cleanup_task;
struct taskqueue *cleanup_tq;
int cpu;
bool do_not_ring_db;
int work_done;
int budget;
} eq;
struct {
gdma_cq_callback *callback;
void *context;
/* For CQ/EQ relationship */
struct gdma_queue *parent;
} cq;
};
};
struct gdma_queue_spec {
enum gdma_queue_type type;
bool monitor_avl_buf;
unsigned int queue_size;
/* Extra fields specific to EQ/CQ. */
union {
struct {
gdma_eq_callback *callback;
void *context;
unsigned long log2_throttle_limit;
/* Only used by the MANA device. */
struct ifnet *ndev;
} eq;
struct {
gdma_cq_callback *callback;
void *context;
struct gdma_queue *parent_eq;
} cq;
};
};
struct mana_eq {
struct gdma_queue *eq;
struct gdma_comp cqe_poll[CQE_POLLING_BUFFER];
};
struct gdma_irq_context {
struct gdma_msix_entry msix_e;
struct resource *res;
driver_intr_t *handler;
void *arg;
void *cookie;
bool requested;
int cpu;
char name[GDMA_IRQNAME_SZ];
};
struct gdma_context {
device_t dev;
struct gdma_bus gd_bus;
/* Per-vPort max number of queues */
unsigned int max_num_queues;
unsigned int max_num_msix;
unsigned int num_msix_usable;
struct gdma_resource msix_resource;
struct gdma_irq_context *irq_contexts;
/* This maps a CQ index to the queue structure. */
unsigned int max_num_cqs;
struct gdma_queue **cq_table;
/* Protect eq_test_event and test_event_eq_id */
struct sx eq_test_event_sx;
struct completion eq_test_event;
uint32_t test_event_eq_id;
struct resource *bar0;
struct resource *msix;
int msix_rid;
void __iomem *shm_base;
void __iomem *db_page_base;
uint32_t db_page_size;
/* Shared memory chanenl (used to bootstrap HWC) */
struct shm_channel shm_channel;
/* Hardware communication channel (HWC) */
struct gdma_dev hwc;
/* Azure network adapter */
struct gdma_dev mana;
};
#define MAX_NUM_GDMA_DEVICES 4
static inline bool mana_gd_is_mana(struct gdma_dev *gd)
{
return gd->dev_id.type == GDMA_DEVICE_MANA;
}
static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
{
return gd->dev_id.type == GDMA_DEVICE_HWC;
}
uint8_t *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, uint32_t wqe_offset);
uint32_t mana_gd_wq_avail_space(struct gdma_queue *wq);
int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
int mana_gd_create_hwc_queue(struct gdma_dev *gd,
const struct gdma_queue_spec *spec,
struct gdma_queue **queue_ptr);
int mana_gd_create_mana_eq(struct gdma_dev *gd,
const struct gdma_queue_spec *spec,
struct gdma_queue **queue_ptr);
int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
const struct gdma_queue_spec *spec,
struct gdma_queue **queue_ptr);
void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
void mana_gd_arm_cq(struct gdma_queue *cq);
struct gdma_wqe {
uint32_t reserved :24;
uint32_t last_vbytes :8;
union {
uint32_t flags;
struct {
uint32_t num_sge :8;
uint32_t inline_oob_size_div4 :3;
uint32_t client_oob_in_sgl :1;
uint32_t reserved1 :4;
uint32_t client_data_unit :14;
uint32_t reserved2 :2;
};
};
}; /* HW DATA */
#define INLINE_OOB_SMALL_SIZE 8
#define INLINE_OOB_LARGE_SIZE 24
#define MAX_TX_WQE_SIZE 512
#define MAX_RX_WQE_SIZE 256
struct gdma_cqe {
uint32_t cqe_data[GDMA_COMP_DATA_SIZE / 4];
union {
uint32_t as_uint32;
struct {
uint32_t wq_num :24;
uint32_t is_sq :1;
uint32_t reserved :4;
uint32_t owner_bits :3;
};
} cqe_info;
}; /* HW DATA */
#define GDMA_CQE_OWNER_BITS 3
#define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
#define SET_ARM_BIT 1
#define GDMA_EQE_OWNER_BITS 3
union gdma_eqe_info {
uint32_t as_uint32;
struct {
uint32_t type : 8;
uint32_t reserved1 : 8;
uint32_t client_id : 2;
uint32_t reserved2 : 11;
uint32_t owner_bits : 3;
};
}; /* HW DATA */
#define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
#define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
struct gdma_eqe {
uint32_t details[GDMA_EVENT_DATA_SIZE / 4];
uint32_t eqe_info;
}; /* HW DATA */
#define GDMA_REG_DB_PAGE_OFFSET 8
#define GDMA_REG_DB_PAGE_SIZE 0x10
#define GDMA_REG_SHM_OFFSET 0x18
struct gdma_posted_wqe_info {
uint32_t wqe_size_in_bu;
};
/* GDMA_GENERATE_TEST_EQE */
struct gdma_generate_test_event_req {
struct gdma_req_hdr hdr;
uint32_t queue_index;
}; /* HW DATA */
/* GDMA_VERIFY_VF_DRIVER_VERSION */
enum {
GDMA_PROTOCOL_V1 = 1,
GDMA_PROTOCOL_FIRST = GDMA_PROTOCOL_V1,
GDMA_PROTOCOL_LAST = GDMA_PROTOCOL_V1,
};
struct gdma_verify_ver_req {
struct gdma_req_hdr hdr;
/* Mandatory fields required for protocol establishment */
uint64_t protocol_ver_min;
uint64_t protocol_ver_max;
uint64_t drv_cap_flags1;
uint64_t drv_cap_flags2;
uint64_t drv_cap_flags3;
uint64_t drv_cap_flags4;
/* Advisory fields */
uint64_t drv_ver;
uint32_t os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */
uint32_t reserved;
uint32_t os_ver_major;
uint32_t os_ver_minor;
uint32_t os_ver_build;
uint32_t os_ver_platform;
uint64_t reserved_2;
uint8_t os_ver_str1[128];
uint8_t os_ver_str2[128];
uint8_t os_ver_str3[128];
uint8_t os_ver_str4[128];
}; /* HW DATA */
struct gdma_verify_ver_resp {
struct gdma_resp_hdr hdr;
uint64_t gdma_protocol_ver;
uint64_t pf_cap_flags1;
uint64_t pf_cap_flags2;
uint64_t pf_cap_flags3;
uint64_t pf_cap_flags4;
}; /* HW DATA */
/* GDMA_QUERY_MAX_RESOURCES */
struct gdma_query_max_resources_resp {
struct gdma_resp_hdr hdr;
uint32_t status;
uint32_t max_sq;
uint32_t max_rq;
uint32_t max_cq;
uint32_t max_eq;
uint32_t max_db;
uint32_t max_mst;
uint32_t max_cq_mod_ctx;
uint32_t max_mod_cq;
uint32_t max_msix;
}; /* HW DATA */
/* GDMA_LIST_DEVICES */
struct gdma_list_devices_resp {
struct gdma_resp_hdr hdr;
uint32_t num_of_devs;
uint32_t reserved;
struct gdma_dev_id devs[64];
}; /* HW DATA */
/* GDMA_REGISTER_DEVICE */
struct gdma_register_device_resp {
struct gdma_resp_hdr hdr;
uint32_t pdid;
uint32_t gpa_mkey;
uint32_t db_id;
}; /* HW DATA */
/* GDMA_CREATE_QUEUE */
struct gdma_create_queue_req {
struct gdma_req_hdr hdr;
uint32_t type;
uint32_t reserved1;
uint32_t pdid;
uint32_t doolbell_id;
uint64_t gdma_region;
uint32_t reserved2;
uint32_t queue_size;
uint32_t log2_throttle_limit;
uint32_t eq_pci_msix_index;
uint32_t cq_mod_ctx_id;
uint32_t cq_parent_eq_id;
uint8_t rq_drop_on_overrun;
uint8_t rq_err_on_wqe_overflow;
uint8_t rq_chain_rec_wqes;
uint8_t sq_hw_db;
uint32_t reserved3;
}; /* HW DATA */
struct gdma_create_queue_resp {
struct gdma_resp_hdr hdr;
uint32_t queue_index;
}; /* HW DATA */
/* GDMA_DISABLE_QUEUE */
struct gdma_disable_queue_req {
struct gdma_req_hdr hdr;
uint32_t type;
uint32_t queue_index;
uint32_t alloc_res_id_on_creation;
}; /* HW DATA */
/* GDMA_CREATE_DMA_REGION */
struct gdma_create_dma_region_req {
struct gdma_req_hdr hdr;
/* The total size of the DMA region */
uint64_t length;
/* The offset in the first page */
uint32_t offset_in_page;
/* enum gdma_page_type */
uint32_t gdma_page_type;
/* The total number of pages */
uint32_t page_count;
/* If page_addr_list_len is smaller than page_count,
* the remaining page addresses will be added via the
* message GDMA_DMA_REGION_ADD_PAGES.
*/
uint32_t page_addr_list_len;
uint64_t page_addr_list[];
}; /* HW DATA */
struct gdma_create_dma_region_resp {
struct gdma_resp_hdr hdr;
uint64_t gdma_region;
}; /* HW DATA */
/* GDMA_DMA_REGION_ADD_PAGES */
struct gdma_dma_region_add_pages_req {
struct gdma_req_hdr hdr;
uint64_t gdma_region;
uint32_t page_addr_list_len;
uint32_t reserved3;
uint64_t page_addr_list[];
}; /* HW DATA */
/* GDMA_DESTROY_DMA_REGION */
struct gdma_destroy_dma_region_req {
struct gdma_req_hdr hdr;
uint64_t gdma_region;
}; /* HW DATA */
int mana_gd_verify_vf_version(device_t dev);
int mana_gd_register_device(struct gdma_dev *gd);
int mana_gd_deregister_device(struct gdma_dev *gd);
int mana_gd_post_work_request(struct gdma_queue *wq,
const struct gdma_wqe_request *wqe_req,
struct gdma_posted_wqe_info *wqe_info);
int mana_gd_post_and_ring(struct gdma_queue *queue,
const struct gdma_wqe_request *wqe,
struct gdma_posted_wqe_info *wqe_info);
int mana_gd_alloc_res_map(uint32_t res_avil, struct gdma_resource *r,
const char *lock_name);
void mana_gd_free_res_map(struct gdma_resource *r);
void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
struct gdma_queue *queue);
int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
struct gdma_mem_info *gmi);
void mana_gd_free_memory(struct gdma_mem_info *gmi);
void mana_gd_dma_map_paddr(void *arg, bus_dma_segment_t *segs,
int nseg, int error);
int mana_gd_send_request(struct gdma_context *gc, uint32_t req_len,
const void *req, uint32_t resp_len, void *resp);
#endif /* _GDMA_H */

1961
sys/dev/mana/gdma_main.c Normal file
View File

File diff suppressed because it is too large Load Diff

96
sys/dev/mana/gdma_util.c Normal file
View File

@ -0,0 +1,96 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Microsoft Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/types.h>
#include <sys/mutex.h>
#include <sys/systm.h>
#include "gdma_util.h"
void
init_completion(struct completion *c)
{
memset(c, 0, sizeof(*c));
mtx_init(&c->lock, "gdma_completion", NULL, MTX_DEF);
c->done = 0;
}
void
free_completion(struct completion *c)
{
mtx_destroy(&c->lock);
}
void
complete(struct completion *c)
{
mtx_lock(&c->lock);
c->done++;
mtx_unlock(&c->lock);
wakeup(c);
}
void
wait_for_completion(struct completion *c)
{
mtx_lock(&c->lock);
while (c->done == 0)
mtx_sleep(c, &c->lock, 0, "gdma_wfc", 0);
c->done--;
mtx_unlock(&c->lock);
}
/*
* Return: 0 if completed, a non-zero value if timed out.
*/
int
wait_for_completion_timeout(struct completion *c, int timeout)
{
int ret;
mtx_lock(&c->lock);
if (c->done == 0)
mtx_sleep(c, &c->lock, 0, "gdma_wfc", timeout);
if (c->done > 0) {
c->done--;
ret = 0;
} else {
ret = 1;
}
mtx_unlock(&c->lock);
return (ret);
}

206
sys/dev/mana/gdma_util.h Normal file
View File

@ -0,0 +1,206 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Microsoft Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD$
*
*/
#ifndef _GDMA_UTIL_H_
#define _GDMA_UTIL_H_
#include <sys/types.h>
#include <sys/param.h>
/* Log Levels */
#define MANA_ALERT (1 << 0) /* Alerts are providing more error info. */
#define MANA_WARNING (1 << 1) /* Driver output is more error sensitive. */
#define MANA_INFO (1 << 2) /* Provides additional driver info. */
#define MANA_DBG (1 << 3) /* Driver output for debugging. */
extern int mana_log_level;
#define mana_trace_raw(ctx, level, fmt, args...) \
do { \
((void)(ctx)); \
if (((level) & mana_log_level) != (level)) \
break; \
printf(fmt, ##args); \
} while (0)
#define mana_trace(ctx, level, fmt, args...) \
mana_trace_raw(ctx, level, "%s() [TID:%d]: " \
fmt, __func__, curthread->td_tid, ##args)
#define mana_dbg(ctx, format, arg...) \
mana_trace(ctx, MANA_DBG, format, ##arg)
#define mana_info(ctx, format, arg...) \
mana_trace(ctx, MANA_INFO, format, ##arg)
#define mana_warn(ctx, format, arg...) \
mana_trace(ctx, MANA_WARNING, format, ##arg)
#define mana_err(ctx, format, arg...) \
mana_trace(ctx, MANA_ALERT, format, ##arg)
#define unlikely(x) __predict_false(!!(x))
#define likely(x) __predict_true(!!(x))
#define BITS_PER_LONG (sizeof(long) * NBBY)
#define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) % BITS_PER_LONG))
#define BITMAP_LAST_WORD_MASK(n) (~0UL >> (BITS_PER_LONG - (n)))
#define BITS_TO_LONGS(n) howmany((n), BITS_PER_LONG)
#define BIT_MASK(nr) (1UL << ((nr) & (BITS_PER_LONG - 1)))
#define BIT_WORD(nr) ((nr) / BITS_PER_LONG)
#undef ALIGN
#define ALIGN(x, y) roundup2((x), (y))
#define IS_ALIGNED(x, a) (((x) & ((__typeof(x))(a) - 1)) == 0)
#define BIT(n) (1ULL << (n))
#define PHYS_PFN(x) ((unsigned long)((x) >> PAGE_SHIFT))
#define offset_in_page(x) ((x) & PAGE_MASK)
#define min_t(type, _x, _y) \
((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y))
#define test_bit(i, a) \
((((volatile const unsigned long *)(a))[BIT_WORD(i)]) & BIT_MASK(i))
typedef volatile uint32_t atomic_t;
#define atomic_add_return(v, p) (atomic_fetchadd_int(p, v) + (v))
#define atomic_sub_return(v, p) (atomic_fetchadd_int(p, -(v)) - (v))
#define atomic_inc_return(p) atomic_add_return(1, p)
#define atomic_dec_return(p) atomic_sub_return(1, p)
#define atomic_read(p) atomic_add_return(0, p)
#define usleep_range(_1, _2) \
pause_sbt("gdma-usleep-range", SBT_1US * _1, SBT_1US * 1, C_ABSOLUTE)
static inline void
gdma_msleep(unsigned int ms)
{
if (ms == 0)
ms = 1;
pause_sbt("gdma-msleep", mstosbt(ms), 0, C_HARDCLOCK);
}
static inline void
bitmap_set(unsigned long *map, unsigned int start, int nr)
{
const unsigned int size = start + nr;
int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG);
unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start);
map += BIT_WORD(start);
while (nr - bits_to_set >= 0) {
*map |= mask_to_set;
nr -= bits_to_set;
bits_to_set = BITS_PER_LONG;
mask_to_set = ~0UL;
map++;
}
if (nr) {
mask_to_set &= BITMAP_LAST_WORD_MASK(size);
*map |= mask_to_set;
}
}
static inline void
bitmap_clear(unsigned long *map, unsigned int start, int nr)
{
const unsigned int size = start + nr;
int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG);
unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start);
map += BIT_WORD(start);
while (nr - bits_to_clear >= 0) {
*map &= ~mask_to_clear;
nr -= bits_to_clear;
bits_to_clear = BITS_PER_LONG;
mask_to_clear = ~0UL;
map++;
}
if (nr) {
mask_to_clear &= BITMAP_LAST_WORD_MASK(size);
*map &= ~mask_to_clear;
}
}
static inline unsigned long
find_first_zero_bit(const unsigned long *p, unsigned long max)
{
unsigned long i, n;
for (i = 0; i < max / BITS_PER_LONG + 1; i++) {
n = ~p[i];
if (n != 0)
return (i * BITS_PER_LONG + ffsl(n) - 1);
}
return (max);
}
static inline unsigned long
ilog2(unsigned long x)
{
unsigned long log = x;
while (x >>= 1)
log++;
return (log);
}
static inline unsigned long
roundup_pow_of_two(unsigned long x)
{
return (1UL << flsl(x - 1));
}
static inline int
is_power_of_2(unsigned long n)
{
return (n == roundup_pow_of_two(n));
}
struct completion {
unsigned int done;
struct mtx lock;
};
void init_completion(struct completion *c);
void free_completion(struct completion *c);
void complete(struct completion *c);
void wait_for_completion(struct completion *c);
int wait_for_completion_timeout(struct completion *c, int timeout);
#endif /* _GDMA_UTIL_H_ */

950
sys/dev/mana/hw_channel.c Normal file
View File

@ -0,0 +1,950 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Microsoft Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/bus.h>
#include <machine/bus.h>
#include "mana.h"
#include "hw_channel.h"
static int
mana_hwc_get_msg_index(struct hw_channel_context *hwc, uint16_t *msg_id)
{
struct gdma_resource *r = &hwc->inflight_msg_res;
uint32_t index;
sema_wait(&hwc->sema);
mtx_lock_spin(&r->lock_spin);
index = find_first_zero_bit(hwc->inflight_msg_res.map,
hwc->inflight_msg_res.size);
bitmap_set(hwc->inflight_msg_res.map, index, 1);
mtx_unlock_spin(&r->lock_spin);
*msg_id = index;
return 0;
}
static void
mana_hwc_put_msg_index(struct hw_channel_context *hwc, uint16_t msg_id)
{
struct gdma_resource *r = &hwc->inflight_msg_res;
mtx_lock_spin(&r->lock_spin);
bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1);
mtx_unlock_spin(&r->lock_spin);
sema_post(&hwc->sema);
}
static int
mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx,
const struct gdma_resp_hdr *resp_msg,
uint32_t resp_len)
{
if (resp_len < sizeof(*resp_msg))
return EPROTO;
if (resp_len > caller_ctx->output_buflen)
return EPROTO;
return 0;
}
static void
mana_hwc_handle_resp(struct hw_channel_context *hwc, uint32_t resp_len,
const struct gdma_resp_hdr *resp_msg)
{
struct hwc_caller_ctx *ctx;
int err;
if (!test_bit(resp_msg->response.hwc_msg_id,
hwc->inflight_msg_res.map)) {
device_printf(hwc->dev, "hwc_rx: invalid msg_id = %u\n",
resp_msg->response.hwc_msg_id);
return;
}
ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id;
err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);
if (err)
goto out;
ctx->status_code = resp_msg->status;
memcpy(ctx->output_buf, resp_msg, resp_len);
out:
ctx->error = err;
complete(&ctx->comp_event);
}
static int
mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
struct hwc_work_request *req)
{
device_t dev = hwc_rxq->hwc->dev;
struct gdma_sge *sge;
int err;
sge = &req->sge;
sge->address = (uint64_t)req->buf_sge_addr;
sge->mem_key = hwc_rxq->msg_buf->gpa_mkey;
sge->size = req->buf_len;
memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
req->wqe_req.sgl = sge;
req->wqe_req.num_sge = 1;
req->wqe_req.client_data_unit = 0;
err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL);
if (err)
device_printf(dev,
"Failed to post WQE on HWC RQ: %d\n", err);
return err;
}
static void
mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,
struct gdma_event *event)
{
struct hw_channel_context *hwc = ctx;
struct gdma_dev *gd = hwc->gdma_dev;
union hwc_init_type_data type_data;
union hwc_init_eq_id_db eq_db;
uint32_t type, val;
switch (event->type) {
case GDMA_EQE_HWC_INIT_EQ_ID_DB:
eq_db.as_uint32 = event->details[0];
hwc->cq->gdma_eq->id = eq_db.eq_id;
gd->doorbell = eq_db.doorbell;
break;
case GDMA_EQE_HWC_INIT_DATA:
type_data.as_uint32 = event->details[0];
type = type_data.type;
val = type_data.value;
switch (type) {
case HWC_INIT_DATA_CQID:
hwc->cq->gdma_cq->id = val;
break;
case HWC_INIT_DATA_RQID:
hwc->rxq->gdma_wq->id = val;
break;
case HWC_INIT_DATA_SQID:
hwc->txq->gdma_wq->id = val;
break;
case HWC_INIT_DATA_QUEUE_DEPTH:
hwc->hwc_init_q_depth_max = (uint16_t)val;
break;
case HWC_INIT_DATA_MAX_REQUEST:
hwc->hwc_init_max_req_msg_size = val;
break;
case HWC_INIT_DATA_MAX_RESPONSE:
hwc->hwc_init_max_resp_msg_size = val;
break;
case HWC_INIT_DATA_MAX_NUM_CQS:
gd->gdma_context->max_num_cqs = val;
break;
case HWC_INIT_DATA_PDID:
hwc->gdma_dev->pdid = val;
break;
case HWC_INIT_DATA_GPA_MKEY:
hwc->rxq->msg_buf->gpa_mkey = val;
hwc->txq->msg_buf->gpa_mkey = val;
break;
}
break;
case GDMA_EQE_HWC_INIT_DONE:
complete(&hwc->hwc_init_eqe_comp);
break;
default:
/* Ignore unknown events, which should never happen. */
break;
}
}
static void
mana_hwc_rx_event_handler(void *ctx, uint32_t gdma_rxq_id,
const struct hwc_rx_oob *rx_oob)
{
struct hw_channel_context *hwc = ctx;
struct hwc_wq *hwc_rxq = hwc->rxq;
struct hwc_work_request *rx_req;
struct gdma_resp_hdr *resp;
struct gdma_wqe *dma_oob;
struct gdma_queue *rq;
struct gdma_sge *sge;
uint64_t rq_base_addr;
uint64_t rx_req_idx;
uint8_t *wqe;
if (hwc_rxq->gdma_wq->id != gdma_rxq_id) {
mana_warn(NULL, "unmatched rx queue %u != %u\n",
hwc_rxq->gdma_wq->id, gdma_rxq_id);
return;
}
rq = hwc_rxq->gdma_wq;
wqe = mana_gd_get_wqe_ptr(rq, rx_oob->wqe_offset / GDMA_WQE_BU_SIZE);
dma_oob = (struct gdma_wqe *)wqe;
bus_dmamap_sync(rq->mem_info.dma_tag, rq->mem_info.dma_map,
BUS_DMASYNC_POSTREAD);
sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4);
/* Select the RX work request for virtual address and for reposting. */
rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle;
rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size;
bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag,
hwc_rxq->msg_buf->mem_info.dma_map,
BUS_DMASYNC_POSTREAD);
rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx];
resp = (struct gdma_resp_hdr *)rx_req->buf_va;
if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) {
device_printf(hwc->dev, "HWC RX: wrong msg_id=%u\n",
resp->response.hwc_msg_id);
return;
}
mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, resp);
/* Do no longer use 'resp', because the buffer is posted to the HW
* in the below mana_hwc_post_rx_wqe().
*/
resp = NULL;
bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag,
hwc_rxq->msg_buf->mem_info.dma_map,
BUS_DMASYNC_PREREAD);
mana_hwc_post_rx_wqe(hwc_rxq, rx_req);
}
static void
mana_hwc_tx_event_handler(void *ctx, uint32_t gdma_txq_id,
const struct hwc_rx_oob *rx_oob)
{
struct hw_channel_context *hwc = ctx;
struct hwc_wq *hwc_txq = hwc->txq;
if (!hwc_txq || hwc_txq->gdma_wq->id != gdma_txq_id) {
mana_warn(NULL, "unmatched tx queue %u != %u\n",
hwc_txq->gdma_wq->id, gdma_txq_id);
}
bus_dmamap_sync(hwc_txq->gdma_wq->mem_info.dma_tag,
hwc_txq->gdma_wq->mem_info.dma_map,
BUS_DMASYNC_POSTWRITE);
}
static int
mana_hwc_create_gdma_wq(struct hw_channel_context *hwc,
enum gdma_queue_type type, uint64_t queue_size,
struct gdma_queue **queue)
{
struct gdma_queue_spec spec = {};
if (type != GDMA_SQ && type != GDMA_RQ)
return EINVAL;
spec.type = type;
spec.monitor_avl_buf = false;
spec.queue_size = queue_size;
return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);
}
static int
mana_hwc_create_gdma_cq(struct hw_channel_context *hwc,
uint64_t queue_size,
void *ctx, gdma_cq_callback *cb,
struct gdma_queue *parent_eq,
struct gdma_queue **queue)
{
struct gdma_queue_spec spec = {};
spec.type = GDMA_CQ;
spec.monitor_avl_buf = false;
spec.queue_size = queue_size;
spec.cq.context = ctx;
spec.cq.callback = cb;
spec.cq.parent_eq = parent_eq;
return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);
}
static int
mana_hwc_create_gdma_eq(struct hw_channel_context *hwc,
uint64_t queue_size,
void *ctx, gdma_eq_callback *cb,
struct gdma_queue **queue)
{
struct gdma_queue_spec spec = {};
spec.type = GDMA_EQ;
spec.monitor_avl_buf = false;
spec.queue_size = queue_size;
spec.eq.context = ctx;
spec.eq.callback = cb;
spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ;
return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);
}
static void
mana_hwc_comp_event(void *ctx, struct gdma_queue *q_self)
{
struct hwc_rx_oob comp_data = {};
struct gdma_comp *completions;
struct hwc_cq *hwc_cq = ctx;
int comp_read, i;
completions = hwc_cq->comp_buf;
comp_read = mana_gd_poll_cq(q_self, completions, hwc_cq->queue_depth);
for (i = 0; i < comp_read; ++i) {
comp_data = *(struct hwc_rx_oob *)completions[i].cqe_data;
if (completions[i].is_sq)
hwc_cq->tx_event_handler(hwc_cq->tx_event_ctx,
completions[i].wq_num,
&comp_data);
else
hwc_cq->rx_event_handler(hwc_cq->rx_event_ctx,
completions[i].wq_num,
&comp_data);
}
bus_dmamap_sync(q_self->mem_info.dma_tag, q_self->mem_info.dma_map,
BUS_DMASYNC_POSTREAD);
mana_gd_arm_cq(q_self);
}
static void
mana_hwc_destroy_cq(struct gdma_context *gc, struct hwc_cq *hwc_cq)
{
if (!hwc_cq)
return;
if (hwc_cq->comp_buf)
free(hwc_cq->comp_buf, M_DEVBUF);
if (hwc_cq->gdma_cq)
mana_gd_destroy_queue(gc, hwc_cq->gdma_cq);
if (hwc_cq->gdma_eq)
mana_gd_destroy_queue(gc, hwc_cq->gdma_eq);
free(hwc_cq, M_DEVBUF);
}
static int
mana_hwc_create_cq(struct hw_channel_context *hwc,
uint16_t q_depth,
gdma_eq_callback *callback, void *ctx,
hwc_rx_event_handler_t *rx_ev_hdlr, void *rx_ev_ctx,
hwc_tx_event_handler_t *tx_ev_hdlr, void *tx_ev_ctx,
struct hwc_cq **hwc_cq_ptr)
{
struct gdma_queue *eq, *cq;
struct gdma_comp *comp_buf;
struct hwc_cq *hwc_cq;
uint32_t eq_size, cq_size;
int err;
eq_size = roundup_pow_of_two(GDMA_EQE_SIZE * q_depth);
if (eq_size < MINIMUM_SUPPORTED_PAGE_SIZE)
eq_size = MINIMUM_SUPPORTED_PAGE_SIZE;
cq_size = roundup_pow_of_two(GDMA_CQE_SIZE * q_depth);
if (cq_size < MINIMUM_SUPPORTED_PAGE_SIZE)
cq_size = MINIMUM_SUPPORTED_PAGE_SIZE;
hwc_cq = malloc(sizeof(*hwc_cq), M_DEVBUF, M_WAITOK | M_ZERO);
if (!hwc_cq)
return ENOMEM;
err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq);
if (err) {
device_printf(hwc->dev,
"Failed to create HWC EQ for RQ: %d\n", err);
goto out;
}
hwc_cq->gdma_eq = eq;
err = mana_hwc_create_gdma_cq(hwc, cq_size, hwc_cq,
mana_hwc_comp_event, eq, &cq);
if (err) {
device_printf(hwc->dev,
"Failed to create HWC CQ for RQ: %d\n", err);
goto out;
}
hwc_cq->gdma_cq = cq;
comp_buf = mallocarray(q_depth, sizeof(struct gdma_comp),
M_DEVBUF, M_WAITOK | M_ZERO);
if (!comp_buf) {
err = ENOMEM;
goto out;
}
hwc_cq->hwc = hwc;
hwc_cq->comp_buf = comp_buf;
hwc_cq->queue_depth = q_depth;
hwc_cq->rx_event_handler = rx_ev_hdlr;
hwc_cq->rx_event_ctx = rx_ev_ctx;
hwc_cq->tx_event_handler = tx_ev_hdlr;
hwc_cq->tx_event_ctx = tx_ev_ctx;
*hwc_cq_ptr = hwc_cq;
return 0;
out:
mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq);
return err;
}
static int
mana_hwc_alloc_dma_buf(struct hw_channel_context *hwc, uint16_t q_depth,
uint32_t max_msg_size,
struct hwc_dma_buf **dma_buf_ptr)
{
struct gdma_context *gc = hwc->gdma_dev->gdma_context;
struct hwc_work_request *hwc_wr;
struct hwc_dma_buf *dma_buf;
struct gdma_mem_info *gmi;
uint32_t buf_size;
uint8_t *base_pa;
void *virt_addr;
uint16_t i;
int err;
dma_buf = malloc(sizeof(*dma_buf) +
q_depth * sizeof(struct hwc_work_request),
M_DEVBUF, M_WAITOK | M_ZERO);
if (!dma_buf)
return ENOMEM;
dma_buf->num_reqs = q_depth;
buf_size = ALIGN(q_depth * max_msg_size, PAGE_SIZE);
gmi = &dma_buf->mem_info;
err = mana_gd_alloc_memory(gc, buf_size, gmi);
if (err) {
device_printf(hwc->dev,
"Failed to allocate DMA buffer: %d\n", err);
goto out;
}
virt_addr = dma_buf->mem_info.virt_addr;
base_pa = (uint8_t *)dma_buf->mem_info.dma_handle;
for (i = 0; i < q_depth; i++) {
hwc_wr = &dma_buf->reqs[i];
hwc_wr->buf_va = (char *)virt_addr + i * max_msg_size;
hwc_wr->buf_sge_addr = base_pa + i * max_msg_size;
hwc_wr->buf_len = max_msg_size;
}
*dma_buf_ptr = dma_buf;
return 0;
out:
free(dma_buf, M_DEVBUF);
return err;
}
static void
mana_hwc_dealloc_dma_buf(struct hw_channel_context *hwc,
struct hwc_dma_buf *dma_buf)
{
if (!dma_buf)
return;
mana_gd_free_memory(&dma_buf->mem_info);
free(dma_buf, M_DEVBUF);
}
static void
mana_hwc_destroy_wq(struct hw_channel_context *hwc,
struct hwc_wq *hwc_wq)
{
if (!hwc_wq)
return;
mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf);
if (hwc_wq->gdma_wq)
mana_gd_destroy_queue(hwc->gdma_dev->gdma_context,
hwc_wq->gdma_wq);
free(hwc_wq, M_DEVBUF);
}
static int
mana_hwc_create_wq(struct hw_channel_context *hwc,
enum gdma_queue_type q_type, uint16_t q_depth,
uint32_t max_msg_size, struct hwc_cq *hwc_cq,
struct hwc_wq **hwc_wq_ptr)
{
struct gdma_queue *queue;
struct hwc_wq *hwc_wq;
uint32_t queue_size;
int err;
if (q_type != GDMA_SQ && q_type != GDMA_RQ) {
/* XXX should fail and return error? */
mana_warn(NULL, "Invalid q_type %u\n", q_type);
}
if (q_type == GDMA_RQ)
queue_size = roundup_pow_of_two(GDMA_MAX_RQE_SIZE * q_depth);
else
queue_size = roundup_pow_of_two(GDMA_MAX_SQE_SIZE * q_depth);
if (queue_size < MINIMUM_SUPPORTED_PAGE_SIZE)
queue_size = MINIMUM_SUPPORTED_PAGE_SIZE;
hwc_wq = malloc(sizeof(*hwc_wq), M_DEVBUF, M_WAITOK | M_ZERO);
if (!hwc_wq)
return ENOMEM;
err = mana_hwc_create_gdma_wq(hwc, q_type, queue_size, &queue);
if (err)
goto out;
err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size,
&hwc_wq->msg_buf);
if (err)
goto out;
hwc_wq->hwc = hwc;
hwc_wq->gdma_wq = queue;
hwc_wq->queue_depth = q_depth;
hwc_wq->hwc_cq = hwc_cq;
*hwc_wq_ptr = hwc_wq;
return 0;
out:
if (err)
mana_hwc_destroy_wq(hwc, hwc_wq);
return err;
}
static int
mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq,
struct hwc_work_request *req,
uint32_t dest_virt_rq_id, uint32_t dest_virt_rcq_id,
bool dest_pf)
{
device_t dev = hwc_txq->hwc->dev;
struct hwc_tx_oob *tx_oob;
struct gdma_sge *sge;
int err;
if (req->msg_size == 0 || req->msg_size > req->buf_len) {
device_printf(dev, "wrong msg_size: %u, buf_len: %u\n",
req->msg_size, req->buf_len);
return EINVAL;
}
tx_oob = &req->tx_oob;
tx_oob->vrq_id = dest_virt_rq_id;
tx_oob->dest_vfid = 0;
tx_oob->vrcq_id = dest_virt_rcq_id;
tx_oob->vscq_id = hwc_txq->hwc_cq->gdma_cq->id;
tx_oob->loopback = false;
tx_oob->lso_override = false;
tx_oob->dest_pf = dest_pf;
tx_oob->vsq_id = hwc_txq->gdma_wq->id;
sge = &req->sge;
sge->address = (uint64_t)req->buf_sge_addr;
sge->mem_key = hwc_txq->msg_buf->gpa_mkey;
sge->size = req->msg_size;
memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
req->wqe_req.sgl = sge;
req->wqe_req.num_sge = 1;
req->wqe_req.inline_oob_size = sizeof(struct hwc_tx_oob);
req->wqe_req.inline_oob_data = tx_oob;
req->wqe_req.client_data_unit = 0;
err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL);
if (err)
device_printf(dev,
"Failed to post WQE on HWC SQ: %d\n", err);
return err;
}
static int
mana_hwc_init_inflight_msg(struct hw_channel_context *hwc, uint16_t num_msg)
{
int err;
sema_init(&hwc->sema, num_msg, "gdma hwc sema");
err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res,
"gdma hwc res lock");
if (err)
device_printf(hwc->dev,
"Failed to init inflight_msg_res: %d\n", err);
return (err);
}
static int
mana_hwc_test_channel(struct hw_channel_context *hwc, uint16_t q_depth,
uint32_t max_req_msg_size, uint32_t max_resp_msg_size)
{
struct gdma_context *gc = hwc->gdma_dev->gdma_context;
struct hwc_wq *hwc_rxq = hwc->rxq;
struct hwc_work_request *req;
struct hwc_caller_ctx *ctx;
int err;
int i;
/* Post all WQEs on the RQ */
for (i = 0; i < q_depth; i++) {
req = &hwc_rxq->msg_buf->reqs[i];
err = mana_hwc_post_rx_wqe(hwc_rxq, req);
if (err)
return err;
}
ctx = malloc(q_depth * sizeof(struct hwc_caller_ctx),
M_DEVBUF, M_WAITOK | M_ZERO);
if (!ctx)
return ENOMEM;
for (i = 0; i < q_depth; ++i)
init_completion(&ctx[i].comp_event);
hwc->caller_ctx = ctx;
return mana_gd_test_eq(gc, hwc->cq->gdma_eq);
}
static int
mana_hwc_establish_channel(struct gdma_context *gc, uint16_t *q_depth,
uint32_t *max_req_msg_size,
uint32_t *max_resp_msg_size)
{
struct hw_channel_context *hwc = gc->hwc.driver_data;
struct gdma_queue *rq = hwc->rxq->gdma_wq;
struct gdma_queue *sq = hwc->txq->gdma_wq;
struct gdma_queue *eq = hwc->cq->gdma_eq;
struct gdma_queue *cq = hwc->cq->gdma_cq;
int err;
init_completion(&hwc->hwc_init_eqe_comp);
err = mana_smc_setup_hwc(&gc->shm_channel, false,
eq->mem_info.dma_handle,
cq->mem_info.dma_handle,
rq->mem_info.dma_handle,
sq->mem_info.dma_handle,
eq->eq.msix_index);
if (err)
return err;
if (wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * hz))
return ETIMEDOUT;
*q_depth = hwc->hwc_init_q_depth_max;
*max_req_msg_size = hwc->hwc_init_max_req_msg_size;
*max_resp_msg_size = hwc->hwc_init_max_resp_msg_size;
if (cq->id >= gc->max_num_cqs) {
mana_warn(NULL, "invalid cq id %u > %u\n",
cq->id, gc->max_num_cqs);
return EPROTO;
}
gc->cq_table = malloc(gc->max_num_cqs * sizeof(struct gdma_queue *),
M_DEVBUF, M_WAITOK | M_ZERO);
if (!gc->cq_table)
return ENOMEM;
gc->cq_table[cq->id] = cq;
return 0;
}
static int
mana_hwc_init_queues(struct hw_channel_context *hwc, uint16_t q_depth,
uint32_t max_req_msg_size, uint32_t max_resp_msg_size)
{
struct hwc_wq *hwc_rxq = NULL;
struct hwc_wq *hwc_txq = NULL;
struct hwc_cq *hwc_cq = NULL;
int err;
err = mana_hwc_init_inflight_msg(hwc, q_depth);
if (err)
return err;
/* CQ is shared by SQ and RQ, so CQ's queue depth is the sum of SQ
* queue depth and RQ queue depth.
*/
err = mana_hwc_create_cq(hwc, q_depth * 2,
mana_hwc_init_event_handler, hwc,
mana_hwc_rx_event_handler, hwc,
mana_hwc_tx_event_handler, hwc, &hwc_cq);
if (err) {
device_printf(hwc->dev, "Failed to create HWC CQ: %d\n", err);
goto out;
}
hwc->cq = hwc_cq;
err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size,
hwc_cq, &hwc_rxq);
if (err) {
device_printf(hwc->dev, "Failed to create HWC RQ: %d\n", err);
goto out;
}
hwc->rxq = hwc_rxq;
err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size,
hwc_cq, &hwc_txq);
if (err) {
device_printf(hwc->dev, "Failed to create HWC SQ: %d\n", err);
goto out;
}
hwc->txq = hwc_txq;
hwc->num_inflight_msg = q_depth;
hwc->max_req_msg_size = max_req_msg_size;
return 0;
out:
if (hwc_txq)
mana_hwc_destroy_wq(hwc, hwc_txq);
if (hwc_rxq)
mana_hwc_destroy_wq(hwc, hwc_rxq);
if (hwc_cq)
mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq);
mana_gd_free_res_map(&hwc->inflight_msg_res);
return err;
}
int
mana_hwc_create_channel(struct gdma_context *gc)
{
uint32_t max_req_msg_size, max_resp_msg_size;
struct gdma_dev *gd = &gc->hwc;
struct hw_channel_context *hwc;
uint16_t q_depth_max;
int err;
hwc = malloc(sizeof(*hwc), M_DEVBUF, M_WAITOK | M_ZERO);
if (!hwc)
return ENOMEM;
gd->gdma_context = gc;
gd->driver_data = hwc;
hwc->gdma_dev = gd;
hwc->dev = gc->dev;
/* HWC's instance number is always 0. */
gd->dev_id.as_uint32 = 0;
gd->dev_id.type = GDMA_DEVICE_HWC;
gd->pdid = INVALID_PDID;
gd->doorbell = INVALID_DOORBELL;
err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,
HW_CHANNEL_MAX_REQUEST_SIZE,
HW_CHANNEL_MAX_RESPONSE_SIZE);
if (err) {
device_printf(hwc->dev, "Failed to initialize HWC: %d\n",
err);
goto out;
}
err = mana_hwc_establish_channel(gc, &q_depth_max, &max_req_msg_size,
&max_resp_msg_size);
if (err) {
device_printf(hwc->dev, "Failed to establish HWC: %d\n", err);
goto out;
}
err = mana_hwc_test_channel(gc->hwc.driver_data,
HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,
max_req_msg_size, max_resp_msg_size);
if (err) {
/* Test failed, but the channel has been established */
device_printf(hwc->dev, "Failed to test HWC: %d\n", err);
return EIO;
}
return 0;
out:
free(hwc, M_DEVBUF);
return (err);
}
void
mana_hwc_destroy_channel(struct gdma_context *gc)
{
struct hw_channel_context *hwc = gc->hwc.driver_data;
struct hwc_caller_ctx *ctx;
mana_smc_teardown_hwc(&gc->shm_channel, false);
ctx = hwc->caller_ctx;
free(ctx, M_DEVBUF);
hwc->caller_ctx = NULL;
mana_hwc_destroy_wq(hwc, hwc->txq);
hwc->txq = NULL;
mana_hwc_destroy_wq(hwc, hwc->rxq);
hwc->rxq = NULL;
mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq);
hwc->cq = NULL;
mana_gd_free_res_map(&hwc->inflight_msg_res);
hwc->num_inflight_msg = 0;
if (hwc->gdma_dev->pdid != INVALID_PDID) {
hwc->gdma_dev->doorbell = INVALID_DOORBELL;
hwc->gdma_dev->pdid = INVALID_PDID;
}
free(hwc, M_DEVBUF);
gc->hwc.driver_data = NULL;
gc->hwc.gdma_context = NULL;
}
int
mana_hwc_send_request(struct hw_channel_context *hwc, uint32_t req_len,
const void *req, uint32_t resp_len, void *resp)
{
struct hwc_work_request *tx_wr;
struct hwc_wq *txq = hwc->txq;
struct gdma_req_hdr *req_msg;
struct hwc_caller_ctx *ctx;
uint16_t msg_id;
int err;
mana_hwc_get_msg_index(hwc, &msg_id);
tx_wr = &txq->msg_buf->reqs[msg_id];
if (req_len > tx_wr->buf_len) {
device_printf(hwc->dev,
"HWC: req msg size: %d > %d\n", req_len,
tx_wr->buf_len);
err = EINVAL;
goto out;
}
ctx = hwc->caller_ctx + msg_id;
ctx->output_buf = resp;
ctx->output_buflen = resp_len;
req_msg = (struct gdma_req_hdr *)tx_wr->buf_va;
if (req)
memcpy(req_msg, req, req_len);
req_msg->req.hwc_msg_id = msg_id;
tx_wr->msg_size = req_len;
err = mana_hwc_post_tx_wqe(txq, tx_wr, 0, 0, false);
if (err) {
device_printf(hwc->dev,
"HWC: Failed to post send WQE: %d\n", err);
goto out;
}
if (wait_for_completion_timeout(&ctx->comp_event, 30 * hz)) {
device_printf(hwc->dev, "HWC: Request timed out!\n");
err = ETIMEDOUT;
goto out;
}
if (ctx->error) {
err = ctx->error;
goto out;
}
if (ctx->status_code) {
device_printf(hwc->dev,
"HWC: Failed hw_channel req: 0x%x\n", ctx->status_code);
err = EPROTO;
goto out;
}
out:
mana_hwc_put_msg_index(hwc, msg_id);
return err;
}

222
sys/dev/mana/hw_channel.h Normal file
View File

@ -0,0 +1,222 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Microsoft Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD$
*
*/
#ifndef _HW_CHANNEL_H
#define _HW_CHANNEL_H
#include <sys/sema.h>
#define DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ 4
#define HW_CHANNEL_MAX_REQUEST_SIZE 0x1000
#define HW_CHANNEL_MAX_RESPONSE_SIZE 0x1000
#define HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH 1
#define HWC_INIT_DATA_CQID 1
#define HWC_INIT_DATA_RQID 2
#define HWC_INIT_DATA_SQID 3
#define HWC_INIT_DATA_QUEUE_DEPTH 4
#define HWC_INIT_DATA_MAX_REQUEST 5
#define HWC_INIT_DATA_MAX_RESPONSE 6
#define HWC_INIT_DATA_MAX_NUM_CQS 7
#define HWC_INIT_DATA_PDID 8
#define HWC_INIT_DATA_GPA_MKEY 9
/* Structures labeled with "HW DATA" are exchanged with the hardware. All of
* them are naturally aligned and hence don't need __packed.
*/
union hwc_init_eq_id_db {
uint32_t as_uint32;
struct {
uint32_t eq_id : 16;
uint32_t doorbell: 16;
};
}; /* HW DATA */
union hwc_init_type_data {
uint32_t as_uint32;
struct {
uint32_t value : 24;
uint32_t type : 8;
};
}; /* HW DATA */
struct hwc_rx_oob {
uint32_t type : 6;
uint32_t eom : 1;
uint32_t som : 1;
uint32_t vendor_err : 8;
uint32_t reserved1 : 16;
uint32_t src_virt_wq : 24;
uint32_t src_vfid : 8;
uint32_t reserved2;
union {
uint32_t wqe_addr_low;
uint32_t wqe_offset;
};
uint32_t wqe_addr_high;
uint32_t client_data_unit : 14;
uint32_t reserved3 : 18;
uint32_t tx_oob_data_size;
uint32_t chunk_offset : 21;
uint32_t reserved4 : 11;
}; /* HW DATA */
struct hwc_tx_oob {
uint32_t reserved1;
uint32_t reserved2;
uint32_t vrq_id : 24;
uint32_t dest_vfid : 8;
uint32_t vrcq_id : 24;
uint32_t reserved3 : 8;
uint32_t vscq_id : 24;
uint32_t loopback : 1;
uint32_t lso_override: 1;
uint32_t dest_pf : 1;
uint32_t reserved4 : 5;
uint32_t vsq_id : 24;
uint32_t reserved5 : 8;
}; /* HW DATA */
struct hwc_work_request {
void *buf_va;
void *buf_sge_addr;
uint32_t buf_len;
uint32_t msg_size;
struct gdma_wqe_request wqe_req;
struct hwc_tx_oob tx_oob;
struct gdma_sge sge;
};
/* hwc_dma_buf represents the array of in-flight WQEs.
* mem_info as know as the GDMA mapped memory is partitioned and used by
* in-flight WQEs.
* The number of WQEs is determined by the number of in-flight messages.
*/
struct hwc_dma_buf {
struct gdma_mem_info mem_info;
uint32_t gpa_mkey;
uint32_t num_reqs;
struct hwc_work_request reqs[];
};
typedef void hwc_rx_event_handler_t(void *ctx, uint32_t gdma_rxq_id,
const struct hwc_rx_oob *rx_oob);
typedef void hwc_tx_event_handler_t(void *ctx, uint32_t gdma_txq_id,
const struct hwc_rx_oob *rx_oob);
struct hwc_cq {
struct hw_channel_context *hwc;
struct gdma_queue *gdma_cq;
struct gdma_queue *gdma_eq;
struct gdma_comp *comp_buf;
uint16_t queue_depth;
hwc_rx_event_handler_t *rx_event_handler;
void *rx_event_ctx;
hwc_tx_event_handler_t *tx_event_handler;
void *tx_event_ctx;
};
struct hwc_wq {
struct hw_channel_context *hwc;
struct gdma_queue *gdma_wq;
struct hwc_dma_buf *msg_buf;
uint16_t queue_depth;
struct hwc_cq *hwc_cq;
};
struct hwc_caller_ctx {
struct completion comp_event;
void *output_buf;
uint32_t output_buflen;
uint32_t error; /* Error code */
uint32_t status_code;
};
struct hw_channel_context {
struct gdma_dev *gdma_dev;
device_t dev;
uint16_t num_inflight_msg;
uint32_t max_req_msg_size;
uint16_t hwc_init_q_depth_max;
uint32_t hwc_init_max_req_msg_size;
uint32_t hwc_init_max_resp_msg_size;
struct completion hwc_init_eqe_comp;
struct hwc_wq *rxq;
struct hwc_wq *txq;
struct hwc_cq *cq;
struct sema sema;
struct gdma_resource inflight_msg_res;
struct hwc_caller_ctx *caller_ctx;
};
int mana_hwc_create_channel(struct gdma_context *gc);
void mana_hwc_destroy_channel(struct gdma_context *gc);
int mana_hwc_send_request(struct hw_channel_context *hwc, uint32_t req_len,
const void *req, uint32_t resp_len, void *resp);
#endif /* _HW_CHANNEL_H */

689
sys/dev/mana/mana.h Normal file
View File

@ -0,0 +1,689 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Microsoft Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD$
*
*/
#ifndef _MANA_H
#define _MANA_H
#include <sys/types.h>
#include <sys/proc.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/counter.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_media.h>
#include <netinet/tcp_lro.h>
#include "gdma.h"
#include "hw_channel.h"
/* Microsoft Azure Network Adapter (MANA)'s definitions
*
* Structures labeled with "HW DATA" are exchanged with the hardware. All of
* them are naturally aligned and hence don't need __packed.
*/
/* MANA protocol version */
#define MANA_MAJOR_VERSION 0
#define MANA_MINOR_VERSION 1
#define MANA_MICRO_VERSION 1
#define DRV_MODULE_NAME "mana"
#ifndef DRV_MODULE_VERSION
#define DRV_MODULE_VERSION \
__XSTRING(MANA_MAJOR_VERSION) "." \
__XSTRING(MANA_MINOR_VERSION) "." \
__XSTRING(MANA_MICRO_VERSION)
#endif
#define DEVICE_NAME "Microsoft Azure Network Adapter (MANA)"
#define DEVICE_DESC "MANA adapter"
/*
* Supported PCI vendor and devices IDs
*/
#ifndef PCI_VENDOR_ID_MICROSOFT
#define PCI_VENDOR_ID_MICROSOFT 0x1414
#endif
#define PCI_DEV_ID_MANA_VF 0x00ba
typedef struct _mana_vendor_id_t {
uint16_t vendor_id;
uint16_t device_id;
} mana_vendor_id_t;
typedef uint64_t mana_handle_t;
#define INVALID_MANA_HANDLE ((mana_handle_t)-1)
enum TRI_STATE {
TRI_STATE_UNKNOWN = -1,
TRI_STATE_FALSE = 0,
TRI_STATE_TRUE = 1
};
/* Number of entries for hardware indirection table must be in power of 2 */
#define MANA_INDIRECT_TABLE_SIZE 64
#define MANA_INDIRECT_TABLE_MASK (MANA_INDIRECT_TABLE_SIZE - 1)
/* The Toeplitz hash key's length in bytes: should be multiple of 8 */
#define MANA_HASH_KEY_SIZE 40
#define COMP_ENTRY_SIZE 64
#define MIN_FRAME_SIZE 146
#define ADAPTER_MTU_SIZE 1500
#define DEFAULT_FRAME_SIZE (ADAPTER_MTU_SIZE + 14)
#define MAX_FRAME_SIZE 4096
#define RX_BUFFERS_PER_QUEUE 512
#define MAX_SEND_BUFFERS_PER_QUEUE 256
#define EQ_SIZE (8 * PAGE_SIZE)
#define LOG2_EQ_THROTTLE 3
#if 1 /* XXX */
#define MAX_PORTS_IN_MANA_DEV 1
#else
#define MAX_PORTS_IN_MANA_DEV 16
#endif
struct mana_send_buf_info {
struct mbuf *mbuf;
bus_dmamap_t dma_map;
/* Required to store the result of mana_gd_post_work_request.
* gdma_posted_wqe_info.wqe_size_in_bu is required for progressing the
* work queue when the WQE is consumed.
*/
struct gdma_posted_wqe_info wqe_inf;
};
struct mana_stats {
counter_u64_t packets; /* rx, tx */
counter_u64_t bytes; /* rx, tx */
counter_u64_t stop; /* tx */
counter_u64_t wakeup; /* tx */
counter_u64_t collapse; /* tx */
counter_u64_t collapse_err; /* tx */
counter_u64_t dma_mapping_err; /* rx, tx */
counter_u64_t mbuf_alloc_fail; /* rx */
counter_u64_t alt_chg; /* tx */
counter_u64_t alt_reset; /* tx */
};
struct mana_txq {
struct gdma_queue *gdma_sq;
union {
uint32_t gdma_txq_id;
struct {
uint32_t reserved1 :10;
uint32_t vsq_frame :14;
uint32_t reserved2 :8;
};
};
uint16_t vp_offset;
struct ifnet *ndev;
/* Store index to the array of tx_qp in port structure */
int idx;
/* The alternative txq idx when this txq is under heavy load */
int alt_txq_idx;
/* The mbufs are sent to the HW and we are waiting for the CQEs. */
struct mana_send_buf_info *tx_buf_info;
uint16_t next_to_use;
uint16_t next_to_complete;
atomic_t pending_sends;
struct buf_ring *txq_br;
struct mtx txq_mtx;
char txq_mtx_name[16];
struct task enqueue_task;
struct taskqueue *enqueue_tq;
struct mana_stats stats;
};
/*
* Max WQE size is 512B. The first 8B is for GDMA Out of Band (OOB),
* next is the Client OOB can be either 8B or 24B. Thus, the max
* space for SGL entries in a singel WQE is 512 - 8 - 8 = 496B. Since each
* SGL is 16B in size, the max number of SGLs in a WQE is 496/16 = 31.
* Save one for emergency use, set the MAX_MBUF_FRAGS allowed to 30.
*/
#define MAX_MBUF_FRAGS 30
#define MANA_TSO_MAXSEG_SZ PAGE_SIZE
/* mbuf data and frags dma mappings */
struct mana_mbuf_head {
bus_addr_t dma_handle[MAX_MBUF_FRAGS + 1];
uint32_t size[MAX_MBUF_FRAGS + 1];
};
#define MANA_HEADROOM sizeof(struct mana_mbuf_head)
enum mana_tx_pkt_format {
MANA_SHORT_PKT_FMT = 0,
MANA_LONG_PKT_FMT = 1,
};
struct mana_tx_short_oob {
uint32_t pkt_fmt :2;
uint32_t is_outer_ipv4 :1;
uint32_t is_outer_ipv6 :1;
uint32_t comp_iphdr_csum :1;
uint32_t comp_tcp_csum :1;
uint32_t comp_udp_csum :1;
uint32_t supress_txcqe_gen :1;
uint32_t vcq_num :24;
uint32_t trans_off :10; /* Transport header offset */
uint32_t vsq_frame :14;
uint32_t short_vp_offset :8;
}; /* HW DATA */
struct mana_tx_long_oob {
uint32_t is_encap :1;
uint32_t inner_is_ipv6 :1;
uint32_t inner_tcp_opt :1;
uint32_t inject_vlan_pri_tag :1;
uint32_t reserved1 :12;
uint32_t pcp :3; /* 802.1Q */
uint32_t dei :1; /* 802.1Q */
uint32_t vlan_id :12; /* 802.1Q */
uint32_t inner_frame_offset :10;
uint32_t inner_ip_rel_offset :6;
uint32_t long_vp_offset :12;
uint32_t reserved2 :4;
uint32_t reserved3;
uint32_t reserved4;
}; /* HW DATA */
struct mana_tx_oob {
struct mana_tx_short_oob s_oob;
struct mana_tx_long_oob l_oob;
}; /* HW DATA */
enum mana_cq_type {
MANA_CQ_TYPE_RX,
MANA_CQ_TYPE_TX,
};
enum mana_cqe_type {
CQE_INVALID = 0,
CQE_RX_OKAY = 1,
CQE_RX_COALESCED_4 = 2,
CQE_RX_OBJECT_FENCE = 3,
CQE_RX_TRUNCATED = 4,
CQE_TX_OKAY = 32,
CQE_TX_SA_DROP = 33,
CQE_TX_MTU_DROP = 34,
CQE_TX_INVALID_OOB = 35,
CQE_TX_INVALID_ETH_TYPE = 36,
CQE_TX_HDR_PROCESSING_ERROR = 37,
CQE_TX_VF_DISABLED = 38,
CQE_TX_VPORT_IDX_OUT_OF_RANGE = 39,
CQE_TX_VPORT_DISABLED = 40,
CQE_TX_VLAN_TAGGING_VIOLATION = 41,
};
#define MANA_CQE_COMPLETION 1
struct mana_cqe_header {
uint32_t cqe_type :6;
uint32_t client_type :2;
uint32_t vendor_err :24;
}; /* HW DATA */
/* NDIS HASH Types */
#define NDIS_HASH_IPV4 BIT(0)
#define NDIS_HASH_TCP_IPV4 BIT(1)
#define NDIS_HASH_UDP_IPV4 BIT(2)
#define NDIS_HASH_IPV6 BIT(3)
#define NDIS_HASH_TCP_IPV6 BIT(4)
#define NDIS_HASH_UDP_IPV6 BIT(5)
#define NDIS_HASH_IPV6_EX BIT(6)
#define NDIS_HASH_TCP_IPV6_EX BIT(7)
#define NDIS_HASH_UDP_IPV6_EX BIT(8)
#define MANA_HASH_L3 (NDIS_HASH_IPV4 | NDIS_HASH_IPV6 | NDIS_HASH_IPV6_EX)
#define MANA_HASH_L4 \
(NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | NDIS_HASH_TCP_IPV6 | \
NDIS_HASH_UDP_IPV6 | NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX)
#define NDIS_HASH_IPV4_L3_MASK (NDIS_HASH_IPV4)
#define NDIS_HASH_IPV4_L4_MASK (NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4)
#define NDIS_HASH_IPV6_L3_MASK (NDIS_HASH_IPV6 | NDIS_HASH_IPV6_EX)
#define NDIS_HASH_IPV6_L4_MASK \
(NDIS_HASH_TCP_IPV6 | NDIS_HASH_UDP_IPV6 | \
NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX)
#define NDIS_HASH_IPV4_MASK \
(NDIS_HASH_IPV4_L3_MASK | NDIS_HASH_IPV4_L4_MASK)
#define NDIS_HASH_IPV6_MASK \
(NDIS_HASH_IPV6_L3_MASK | NDIS_HASH_IPV6_L4_MASK)
struct mana_rxcomp_perpkt_info {
uint32_t pkt_len :16;
uint32_t reserved1 :16;
uint32_t reserved2;
uint32_t pkt_hash;
}; /* HW DATA */
#define MANA_RXCOMP_OOB_NUM_PPI 4
/* Receive completion OOB */
struct mana_rxcomp_oob {
struct mana_cqe_header cqe_hdr;
uint32_t rx_vlan_id :12;
uint32_t rx_vlantag_present :1;
uint32_t rx_outer_iphdr_csum_succeed :1;
uint32_t rx_outer_iphdr_csum_fail :1;
uint32_t reserved1 :1;
uint32_t rx_hashtype :9;
uint32_t rx_iphdr_csum_succeed :1;
uint32_t rx_iphdr_csum_fail :1;
uint32_t rx_tcp_csum_succeed :1;
uint32_t rx_tcp_csum_fail :1;
uint32_t rx_udp_csum_succeed :1;
uint32_t rx_udp_csum_fail :1;
uint32_t reserved2 :1;
struct mana_rxcomp_perpkt_info ppi[MANA_RXCOMP_OOB_NUM_PPI];
uint32_t rx_wqe_offset;
}; /* HW DATA */
struct mana_tx_comp_oob {
struct mana_cqe_header cqe_hdr;
uint32_t tx_data_offset;
uint32_t tx_sgl_offset :5;
uint32_t tx_wqe_offset :27;
uint32_t reserved[12];
}; /* HW DATA */
struct mana_rxq;
struct mana_cq {
struct gdma_queue *gdma_cq;
/* Cache the CQ id (used to verify if each CQE comes to the right CQ. */
uint32_t gdma_id;
/* Type of the CQ: TX or RX */
enum mana_cq_type type;
/* Pointer to the mana_rxq that is pushing RX CQEs to the queue.
* Only and must be non-NULL if type is MANA_CQ_TYPE_RX.
*/
struct mana_rxq *rxq;
/* Pointer to the mana_txq that is pushing TX CQEs to the queue.
* Only and must be non-NULL if type is MANA_CQ_TYPE_TX.
*/
struct mana_txq *txq;
/* Pointer to a buffer which the CQ handler can copy the CQE's into. */
struct gdma_comp *gdma_comp_buf;
};
#define GDMA_MAX_RQE_SGES 15
struct mana_recv_buf_oob {
/* A valid GDMA work request representing the data buffer. */
struct gdma_wqe_request wqe_req;
struct mbuf *mbuf;
bus_dmamap_t dma_map;
/* SGL of the buffer going to be sent as part of the work request. */
uint32_t num_sge;
struct gdma_sge sgl[GDMA_MAX_RQE_SGES];
/* Required to store the result of mana_gd_post_work_request.
* gdma_posted_wqe_info.wqe_size_in_bu is required for progressing the
* work queue when the WQE is consumed.
*/
struct gdma_posted_wqe_info wqe_inf;
};
struct mana_rxq {
struct gdma_queue *gdma_rq;
/* Cache the gdma receive queue id */
uint32_t gdma_id;
/* Index of RQ in the vPort, not gdma receive queue id */
uint32_t rxq_idx;
uint32_t datasize;
mana_handle_t rxobj;
struct mana_cq rx_cq;
struct ifnet *ndev;
struct lro_ctrl lro;
/* Total number of receive buffers to be allocated */
uint32_t num_rx_buf;
uint32_t buf_index;
struct mana_stats stats;
/* MUST BE THE LAST MEMBER:
* Each receive buffer has an associated mana_recv_buf_oob.
*/
struct mana_recv_buf_oob rx_oobs[];
};
struct mana_tx_qp {
struct mana_txq txq;
struct mana_cq tx_cq;
mana_handle_t tx_object;
};
struct mana_port_stats {
counter_u64_t rx_packets;
counter_u64_t tx_packets;
counter_u64_t rx_bytes;
counter_u64_t tx_bytes;
counter_u64_t rx_drops;
counter_u64_t tx_drops;
counter_u64_t stop_queue;
counter_u64_t wake_queue;
};
struct mana_context {
struct gdma_dev *gdma_dev;
uint16_t num_ports;
struct ifnet *ports[MAX_PORTS_IN_MANA_DEV];
};
struct mana_port_context {
struct mana_context *ac;
struct ifnet *ndev;
struct ifmedia media;
struct sx apc_lock;
/* DMA tag used for queue bufs of the entire port */
bus_dma_tag_t rx_buf_tag;
bus_dma_tag_t tx_buf_tag;
uint8_t mac_addr[ETHER_ADDR_LEN];
struct mana_eq *eqs;
enum TRI_STATE rss_state;
mana_handle_t default_rxobj;
bool tx_shortform_allowed;
uint16_t tx_vp_offset;
struct mana_tx_qp *tx_qp;
/* Indirection Table for RX & TX. The values are queue indexes */
uint32_t indir_table[MANA_INDIRECT_TABLE_SIZE];
/* Indirection table containing RxObject Handles */
mana_handle_t rxobj_table[MANA_INDIRECT_TABLE_SIZE];
/* Hash key used by the NIC */
uint8_t hashkey[MANA_HASH_KEY_SIZE];
/* This points to an array of num_queues of RQ pointers. */
struct mana_rxq **rxqs;
/* Create num_queues EQs, SQs, SQ-CQs, RQs and RQ-CQs, respectively. */
unsigned int max_queues;
unsigned int num_queues;
mana_handle_t port_handle;
uint16_t port_idx;
uint16_t frame_size;
bool port_is_up;
bool port_st_save; /* Saved port state */
bool enable_tx_altq;
bool bind_cleanup_thread_cpu;
struct mana_port_stats port_stats;
struct sysctl_oid_list *port_list;
struct sysctl_ctx_list que_sysctl_ctx;
};
#define MANA_APC_LOCK_INIT(apc) \
sx_init(&(apc)->apc_lock, "MANA port lock")
#define MANA_APC_LOCK_DESTROY(apc) sx_destroy(&(apc)->apc_lock)
#define MANA_APC_LOCK_LOCK(apc) sx_xlock(&(apc)->apc_lock)
#define MANA_APC_LOCK_UNLOCK(apc) sx_unlock(&(apc)->apc_lock)
int mana_config_rss(struct mana_port_context *ac, enum TRI_STATE rx,
bool update_hash, bool update_tab);
int mana_alloc_queues(struct ifnet *ndev);
int mana_attach(struct ifnet *ndev);
int mana_detach(struct ifnet *ndev);
int mana_probe(struct gdma_dev *gd);
void mana_remove(struct gdma_dev *gd);
struct mana_obj_spec {
uint32_t queue_index;
uint64_t gdma_region;
uint32_t queue_size;
uint32_t attached_eq;
uint32_t modr_ctx_id;
};
enum mana_command_code {
MANA_QUERY_DEV_CONFIG = 0x20001,
MANA_QUERY_GF_STAT = 0x20002,
MANA_CONFIG_VPORT_TX = 0x20003,
MANA_CREATE_WQ_OBJ = 0x20004,
MANA_DESTROY_WQ_OBJ = 0x20005,
MANA_FENCE_RQ = 0x20006,
MANA_CONFIG_VPORT_RX = 0x20007,
MANA_QUERY_VPORT_CONFIG = 0x20008,
};
/* Query Device Configuration */
struct mana_query_device_cfg_req {
struct gdma_req_hdr hdr;
/* Driver Capability flags */
uint64_t drv_cap_flags1;
uint64_t drv_cap_flags2;
uint64_t drv_cap_flags3;
uint64_t drv_cap_flags4;
uint32_t proto_major_ver;
uint32_t proto_minor_ver;
uint32_t proto_micro_ver;
uint32_t reserved;
}; /* HW DATA */
struct mana_query_device_cfg_resp {
struct gdma_resp_hdr hdr;
uint64_t pf_cap_flags1;
uint64_t pf_cap_flags2;
uint64_t pf_cap_flags3;
uint64_t pf_cap_flags4;
uint16_t max_num_vports;
uint16_t reserved;
uint32_t max_num_eqs;
}; /* HW DATA */
/* Query vPort Configuration */
struct mana_query_vport_cfg_req {
struct gdma_req_hdr hdr;
uint32_t vport_index;
}; /* HW DATA */
struct mana_query_vport_cfg_resp {
struct gdma_resp_hdr hdr;
uint32_t max_num_sq;
uint32_t max_num_rq;
uint32_t num_indirection_ent;
uint32_t reserved1;
uint8_t mac_addr[6];
uint8_t reserved2[2];
mana_handle_t vport;
}; /* HW DATA */
/* Configure vPort */
struct mana_config_vport_req {
struct gdma_req_hdr hdr;
mana_handle_t vport;
uint32_t pdid;
uint32_t doorbell_pageid;
}; /* HW DATA */
struct mana_config_vport_resp {
struct gdma_resp_hdr hdr;
uint16_t tx_vport_offset;
uint8_t short_form_allowed;
uint8_t reserved;
}; /* HW DATA */
/* Create WQ Object */
struct mana_create_wqobj_req {
struct gdma_req_hdr hdr;
mana_handle_t vport;
uint32_t wq_type;
uint32_t reserved;
uint64_t wq_gdma_region;
uint64_t cq_gdma_region;
uint32_t wq_size;
uint32_t cq_size;
uint32_t cq_moderation_ctx_id;
uint32_t cq_parent_qid;
}; /* HW DATA */
struct mana_create_wqobj_resp {
struct gdma_resp_hdr hdr;
uint32_t wq_id;
uint32_t cq_id;
mana_handle_t wq_obj;
}; /* HW DATA */
/* Destroy WQ Object */
struct mana_destroy_wqobj_req {
struct gdma_req_hdr hdr;
uint32_t wq_type;
uint32_t reserved;
mana_handle_t wq_obj_handle;
}; /* HW DATA */
struct mana_destroy_wqobj_resp {
struct gdma_resp_hdr hdr;
}; /* HW DATA */
/* Fence RQ */
struct mana_fence_rq_req {
struct gdma_req_hdr hdr;
mana_handle_t wq_obj_handle;
}; /* HW DATA */
struct mana_fence_rq_resp {
struct gdma_resp_hdr hdr;
}; /* HW DATA */
/* Configure vPort Rx Steering */
struct mana_cfg_rx_steer_req {
struct gdma_req_hdr hdr;
mana_handle_t vport;
uint16_t num_indir_entries;
uint16_t indir_tab_offset;
uint32_t rx_enable;
uint32_t rss_enable;
uint8_t update_default_rxobj;
uint8_t update_hashkey;
uint8_t update_indir_tab;
uint8_t reserved;
mana_handle_t default_rxobj;
uint8_t hashkey[MANA_HASH_KEY_SIZE];
}; /* HW DATA */
struct mana_cfg_rx_steer_resp {
struct gdma_resp_hdr hdr;
}; /* HW DATA */
#define MANA_MAX_NUM_QUEUES 16
#define MANA_SHORT_VPORT_OFFSET_MAX ((1U << 8) - 1)
struct mana_tx_package {
struct gdma_wqe_request wqe_req;
struct gdma_sge sgl_array[MAX_MBUF_FRAGS];
struct mana_tx_oob tx_oob;
struct gdma_posted_wqe_info wqe_info;
};
int mana_restart(struct mana_port_context *apc);
#endif /* _MANA_H */

2699
sys/dev/mana/mana_en.c Normal file
View File

File diff suppressed because it is too large Load Diff

219
sys/dev/mana/mana_sysctl.c Normal file
View File

@ -0,0 +1,219 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Microsoft Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "mana_sysctl.h"
static int mana_sysctl_cleanup_thread_cpu(SYSCTL_HANDLER_ARGS);
int mana_log_level = MANA_ALERT | MANA_WARNING | MANA_INFO;
SYSCTL_NODE(_hw, OID_AUTO, mana, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"MANA driver parameters");
/*
* Logging level for changing verbosity of the output
*/
SYSCTL_INT(_hw_mana, OID_AUTO, log_level, CTLFLAG_RWTUN,
&mana_log_level, 0, "Logging level indicating verbosity of the logs");
SYSCTL_CONST_STRING(_hw_mana, OID_AUTO, driver_version, CTLFLAG_RD,
DRV_MODULE_VERSION, "MANA driver version");
void
mana_sysctl_add_port(struct mana_port_context *apc)
{
struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
device_t dev = gc->dev;
struct sysctl_ctx_list *ctx;
struct sysctl_oid *tree;
struct sysctl_oid_list *child;
struct mana_port_stats *port_stats;
char node_name[32];
struct sysctl_oid *port_node, *stats_node;
struct sysctl_oid_list *stats_list;
ctx = device_get_sysctl_ctx(dev);
tree = device_get_sysctl_tree(dev);
child = SYSCTL_CHILDREN(tree);
port_stats = &apc->port_stats;
snprintf(node_name, 32, "port%d", apc->port_idx);
port_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO,
node_name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Port Name");
apc->port_list = SYSCTL_CHILDREN(port_node);
SYSCTL_ADD_BOOL(ctx, apc->port_list, OID_AUTO,
"enable_altq", CTLFLAG_RW, &apc->enable_tx_altq, 0,
"Choose alternative txq under heavy load");
SYSCTL_ADD_PROC(ctx, apc->port_list, OID_AUTO,
"bind_cleanup_thread_cpu",
CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE,
apc, 0, mana_sysctl_cleanup_thread_cpu, "I",
"Bind cleanup thread to a cpu. 0 disables it.");
stats_node = SYSCTL_ADD_NODE(ctx, apc->port_list, OID_AUTO,
"port_stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
"Statistics of port");
stats_list = SYSCTL_CHILDREN(stats_node);
SYSCTL_ADD_COUNTER_U64(ctx, stats_list, OID_AUTO, "rx_packets",
CTLFLAG_RD, &port_stats->rx_packets, "Packets received");
SYSCTL_ADD_COUNTER_U64(ctx, stats_list, OID_AUTO, "tx_packets",
CTLFLAG_RD, &port_stats->tx_packets, "Packets transmitted");
SYSCTL_ADD_COUNTER_U64(ctx, stats_list, OID_AUTO, "rx_bytes",
CTLFLAG_RD, &port_stats->rx_bytes, "Bytes received");
SYSCTL_ADD_COUNTER_U64(ctx, stats_list, OID_AUTO, "tx_bytes",
CTLFLAG_RD, &port_stats->tx_bytes, "Bytes transmitted");
SYSCTL_ADD_COUNTER_U64(ctx, stats_list, OID_AUTO, "rx_drops",
CTLFLAG_RD, &port_stats->rx_drops, "Receive packet drops");
SYSCTL_ADD_COUNTER_U64(ctx, stats_list, OID_AUTO, "tx_drops",
CTLFLAG_RD, &port_stats->tx_drops, "Transmit packet drops");
}
void
mana_sysctl_add_queues(struct mana_port_context *apc)
{
struct sysctl_ctx_list *ctx = &apc->que_sysctl_ctx;
struct sysctl_oid_list *child = apc->port_list;
struct sysctl_oid *queue_node, *tx_node, *rx_node;
struct sysctl_oid_list *queue_list, *tx_list, *rx_list;
struct mana_txq *txq;
struct mana_rxq *rxq;
struct mana_stats *tx_stats, *rx_stats;
char que_name[32];
int i;
sysctl_ctx_init(ctx);
for (i = 0; i < apc->num_queues; i++) {
rxq = apc->rxqs[i];
txq = &apc->tx_qp[i].txq;
snprintf(que_name, 32, "queue%d", i);
queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO,
que_name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name");
queue_list = SYSCTL_CHILDREN(queue_node);
/* TX stats */
tx_node = SYSCTL_ADD_NODE(ctx, queue_list, OID_AUTO,
"txq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TX queue");
tx_list = SYSCTL_CHILDREN(tx_node);
tx_stats = &txq->stats;
SYSCTL_ADD_COUNTER_U64(ctx, tx_list, OID_AUTO, "count",
CTLFLAG_RD, &tx_stats->packets, "Packets sent");
SYSCTL_ADD_COUNTER_U64(ctx, tx_list, OID_AUTO, "bytes",
CTLFLAG_RD, &tx_stats->bytes, "Bytes sent");
SYSCTL_ADD_COUNTER_U64(ctx, tx_list, OID_AUTO, "queue_wakeups",
CTLFLAG_RD, &tx_stats->wakeup, "Queue wakeups");
SYSCTL_ADD_COUNTER_U64(ctx, tx_list, OID_AUTO, "queue_stops",
CTLFLAG_RD, &tx_stats->stop, "Queue stops");
SYSCTL_ADD_COUNTER_U64(ctx, tx_list, OID_AUTO, "mbuf_collapse",
CTLFLAG_RD, &tx_stats->collapse, "Mbuf collapse count");
SYSCTL_ADD_COUNTER_U64(ctx, tx_list, OID_AUTO,
"mbuf_collapse_err", CTLFLAG_RD,
&tx_stats->collapse_err, "Mbuf collapse failures");
SYSCTL_ADD_COUNTER_U64(ctx, tx_list, OID_AUTO,
"dma_mapping_err", CTLFLAG_RD,
&tx_stats->dma_mapping_err, "DMA mapping failures");
SYSCTL_ADD_COUNTER_U64(ctx, tx_list, OID_AUTO,
"alt_chg", CTLFLAG_RD,
&tx_stats->alt_chg, "Switch to alternative txq");
SYSCTL_ADD_COUNTER_U64(ctx, tx_list, OID_AUTO,
"alt_reset", CTLFLAG_RD,
&tx_stats->alt_reset, "Reset to self txq");
/* RX stats */
rx_node = SYSCTL_ADD_NODE(ctx, queue_list, OID_AUTO,
"rxq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "RX queue");
rx_list = SYSCTL_CHILDREN(rx_node);
rx_stats = &rxq->stats;
SYSCTL_ADD_COUNTER_U64(ctx, rx_list, OID_AUTO, "count",
CTLFLAG_RD, &rx_stats->packets, "Packets received");
SYSCTL_ADD_COUNTER_U64(ctx, rx_list, OID_AUTO, "bytes",
CTLFLAG_RD, &rx_stats->bytes, "Bytes received");
SYSCTL_ADD_COUNTER_U64(ctx, rx_list, OID_AUTO,
"mbuf_alloc_fail", CTLFLAG_RD,
&rx_stats->mbuf_alloc_fail, "Failed mbuf allocs");
SYSCTL_ADD_COUNTER_U64(ctx, rx_list, OID_AUTO,
"dma_mapping_err", CTLFLAG_RD,
&rx_stats->dma_mapping_err, "DMA mapping errors");
}
}
/*
* Free all queues' sysctl trees attached to the port's tree.
*/
void
mana_sysctl_free_queues(struct mana_port_context *apc)
{
sysctl_ctx_free(&apc->que_sysctl_ctx);
}
static int
mana_sysctl_cleanup_thread_cpu(SYSCTL_HANDLER_ARGS)
{
struct mana_port_context *apc = arg1;
bool bind_cpu = false;
uint8_t val;
int err;
val = 0;
err = sysctl_wire_old_buffer(req, sizeof(val));
if (err == 0) {
val = apc->bind_cleanup_thread_cpu;
err = sysctl_handle_8(oidp, &val, 0, req);
}
if (err != 0 || req->newptr == NULL)
return (err);
if (val != 0)
bind_cpu = true;
if (bind_cpu != apc->bind_cleanup_thread_cpu) {
apc->bind_cleanup_thread_cpu = bind_cpu;
err = mana_restart(apc);
}
return (err);
}

48
sys/dev/mana/mana_sysctl.h Normal file
View File

@ -0,0 +1,48 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD$
*
*/
#ifndef MANA_SYSCTL_H
#define MANA_SYSCTL_H
#include <sys/param.h>
#include <sys/types.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include "mana.h"
void mana_sysctl_add_port(struct mana_port_context *apc);
void mana_sysctl_add_queues(struct mana_port_context *apc);
void mana_sysctl_free_queues(struct mana_port_context *apc);
#endif /* !(MANA_SYSCTL_H) */

337
sys/dev/mana/shm_channel.c Normal file
View File

@ -0,0 +1,337 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Microsoft Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include "mana.h"
#include "shm_channel.h"
#include "gdma_util.h"
#define PAGE_FRAME_L48_WIDTH_BYTES 6
#define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8)
#define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF
#define PAGE_FRAME_H4_WIDTH_BITS 4
#define VECTOR_MASK 0xFFFF
#define SHMEM_VF_RESET_STATE ((uint32_t)-1)
#define SMC_MSG_TYPE_ESTABLISH_HWC 1
#define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0
#define SMC_MSG_TYPE_DESTROY_HWC 2
#define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0
#define SMC_MSG_DIRECTION_REQUEST 0
#define SMC_MSG_DIRECTION_RESPONSE 1
/* Structures labeled with "HW DATA" are exchanged with the hardware. All of
* them are naturally aligned and hence don't need __packed.
*/
/* Shared memory channel protocol header
*
* msg_type: set on request and response; response matches request.
* msg_version: newer PF writes back older response (matching request)
* older PF acts on latest version known and sets that version in result
* (less than request).
* direction: 0 for request, VF->PF; 1 for response, PF->VF.
* status: 0 on request,
* operation result on response (success = 0, failure = 1 or greater).
* reset_vf: If set on either establish or destroy request, indicates perform
* FLR before/after the operation.
* owner_is_pf: 1 indicates PF owned, 0 indicates VF owned.
*/
union smc_proto_hdr {
uint32_t as_uint32;
struct {
uint8_t msg_type : 3;
uint8_t msg_version : 3;
uint8_t reserved_1 : 1;
uint8_t direction : 1;
uint8_t status;
uint8_t reserved_2;
uint8_t reset_vf : 1;
uint8_t reserved_3 : 6;
uint8_t owner_is_pf : 1;
};
}; /* HW DATA */
#define SMC_APERTURE_BITS 256
#define SMC_BASIC_UNIT (sizeof(uint32_t))
#define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8))
#define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1)
static int
mana_smc_poll_register(void __iomem *base, bool reset)
{
void __iomem *ptr = (uint8_t *)base + SMC_LAST_DWORD * SMC_BASIC_UNIT;
volatile uint32_t last_dword;
int i;
/* Poll the hardware for the ownership bit. This should be pretty fast,
* but let's do it in a loop just in case the hardware or the PF
* driver are temporarily busy.
*/
for (i = 0; i < 20 * 1000; i++) {
last_dword = readl(ptr);
/* shmem reads as 0xFFFFFFFF in the reset case */
if (reset && last_dword == SHMEM_VF_RESET_STATE)
return 0;
/* If bit_31 is set, the PF currently owns the SMC. */
if (!(last_dword & BIT(31)))
return 0;
DELAY(1000);
}
return ETIMEDOUT;
}
static int
mana_smc_read_response(struct shm_channel *sc, uint32_t msg_type,
uint32_t msg_version, bool reset_vf)
{
void __iomem *base = sc->base;
union smc_proto_hdr hdr;
int err;
/* Wait for PF to respond. */
err = mana_smc_poll_register(base, reset_vf);
if (err)
return err;
hdr.as_uint32 =
readl((uint8_t *)base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
mana_dbg(NULL, "shm response 0x%x\n", hdr.as_uint32);
if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE)
return 0;
/* Validate protocol fields from the PF driver */
if (hdr.msg_type != msg_type || hdr.msg_version > msg_version ||
hdr.direction != SMC_MSG_DIRECTION_RESPONSE) {
device_printf(sc->dev,
"Wrong SMC response 0x%x, type=%d, ver=%d\n",
hdr.as_uint32, msg_type, msg_version);
return EPROTO;
}
/* Validate the operation result */
if (hdr.status != 0) {
device_printf(sc->dev,
"SMC operation failed: 0x%x\n", hdr.status);
return EPROTO;
}
return 0;
}
void
mana_smc_init(struct shm_channel *sc, device_t dev, void __iomem *base)
{
sc->dev = dev;
sc->base = base;
}
int
mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, uint64_t eq_addr,
uint64_t cq_addr, uint64_t rq_addr, uint64_t sq_addr,
uint32_t eq_msix_index)
{
union smc_proto_hdr *hdr;
uint16_t all_addr_h4bits = 0;
uint16_t frame_addr_seq = 0;
uint64_t frame_addr = 0;
uint8_t shm_buf[32];
uint64_t *shmem;
uint32_t *dword;
uint8_t *ptr;
int err;
int i;
/* Ensure VF already has possession of shared memory */
err = mana_smc_poll_register(sc->base, false);
if (err) {
device_printf(sc->dev,
"Timeout when setting up HWC: %d\n", err);
return err;
}
if (!IS_ALIGNED(eq_addr, PAGE_SIZE) ||
!IS_ALIGNED(cq_addr, PAGE_SIZE) ||
!IS_ALIGNED(rq_addr, PAGE_SIZE) ||
!IS_ALIGNED(sq_addr, PAGE_SIZE))
return EINVAL;
if ((eq_msix_index & VECTOR_MASK) != eq_msix_index)
return EINVAL;
/* Scheme for packing four addresses and extra info into 256 bits.
*
* Addresses must be page frame aligned, so only frame address bits
* are transferred.
*
* 52-bit frame addresses are split into the lower 48 bits and upper
* 4 bits. Lower 48 bits of 4 address are written sequentially from
* the start of the 256-bit shared memory region followed by 16 bits
* containing the upper 4 bits of the 4 addresses in sequence.
*
* A 16 bit EQ vector number fills out the next-to-last 32-bit dword.
*
* The final 32-bit dword is used for protocol control information as
* defined in smc_proto_hdr.
*/
memset(shm_buf, 0, sizeof(shm_buf));
ptr = shm_buf;
/* EQ addr: low 48 bits of frame address */
shmem = (uint64_t *)ptr;
frame_addr = PHYS_PFN(eq_addr);
*shmem = frame_addr & PAGE_FRAME_L48_MASK;
all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
(frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
ptr += PAGE_FRAME_L48_WIDTH_BYTES;
/* CQ addr: low 48 bits of frame address */
shmem = (uint64_t *)ptr;
frame_addr = PHYS_PFN(cq_addr);
*shmem = frame_addr & PAGE_FRAME_L48_MASK;
all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
(frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
ptr += PAGE_FRAME_L48_WIDTH_BYTES;
/* RQ addr: low 48 bits of frame address */
shmem = (uint64_t *)ptr;
frame_addr = PHYS_PFN(rq_addr);
*shmem = frame_addr & PAGE_FRAME_L48_MASK;
all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
(frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
ptr += PAGE_FRAME_L48_WIDTH_BYTES;
/* SQ addr: low 48 bits of frame address */
shmem = (uint64_t *)ptr;
frame_addr = PHYS_PFN(sq_addr);
*shmem = frame_addr & PAGE_FRAME_L48_MASK;
all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
(frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
ptr += PAGE_FRAME_L48_WIDTH_BYTES;
/* High 4 bits of the four frame addresses */
*((uint16_t *)ptr) = all_addr_h4bits;
ptr += sizeof(uint16_t);
/* EQ MSIX vector number */
*((uint16_t *)ptr) = (uint16_t)eq_msix_index;
ptr += sizeof(uint16_t);
/* 32-bit protocol header in final dword */
*((uint32_t *)ptr) = 0;
hdr = (union smc_proto_hdr *)ptr;
hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC;
hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION;
hdr->direction = SMC_MSG_DIRECTION_REQUEST;
hdr->reset_vf = reset_vf;
/* Write 256-message buffer to shared memory (final 32-bit write
* triggers HW to set possession bit to PF).
*/
dword = (uint32_t *)shm_buf;
for (i = 0; i < SMC_APERTURE_DWORDS; i++) {
mana_dbg(NULL, "write shm_buf %d, val: 0x%x\n",
i, *dword);
writel((char *)sc->base + i * SMC_BASIC_UNIT, *dword++);
}
/* Read shmem response (polling for VF possession) and validate.
* For setup, waiting for response on shared memory is not strictly
* necessary, since wait occurs later for results to appear in EQE's.
*/
err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC,
SMC_MSG_TYPE_ESTABLISH_HWC_VERSION, reset_vf);
if (err) {
device_printf(sc->dev,
"Error when setting up HWC: %d\n", err);
return err;
}
return 0;
}
int
mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf)
{
union smc_proto_hdr hdr = {};
int err;
/* Ensure already has possession of shared memory */
err = mana_smc_poll_register(sc->base, false);
if (err) {
device_printf(sc->dev, "Timeout when tearing down HWC\n");
return err;
}
/* Set up protocol header for HWC destroy message */
hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC;
hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION;
hdr.direction = SMC_MSG_DIRECTION_REQUEST;
hdr.reset_vf = reset_vf;
/* Write message in high 32 bits of 256-bit shared memory, causing HW
* to set possession bit to PF.
*/
writel((char *)sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT,
hdr.as_uint32);
/* Read shmem response (polling for VF possession) and validate.
* For teardown, waiting for response is required to ensure hardware
* invalidates MST entries before software frees memory.
*/
err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC,
SMC_MSG_TYPE_DESTROY_HWC_VERSION, reset_vf);
if (err) {
device_printf(sc->dev,
"Error when tearing down HWC: %d\n", err);
return err;
}
return 0;
}

52
sys/dev/mana/shm_channel.h Normal file
View File

@ -0,0 +1,52 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Microsoft Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD$
*
*/
#ifndef _SHM_CHANNEL_H
#define _SHM_CHANNEL_H
#define __iomem
struct shm_channel {
device_t dev;
void __iomem *base;
};
void mana_smc_init(struct shm_channel *sc, device_t dev, void __iomem *base);
int mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf,
uint64_t eq_addr, uint64_t cq_addr, uint64_t rq_addr, uint64_t sq_addr,
uint32_t eq_msix_index);
int mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf);
#endif /* _SHM_CHANNEL_H */

2
sys/modules/Makefile
View File

@ -231,6 +231,7 @@ SUBDIR= \
mac_stub \
mac_test \
${_malo} \
${_mana} \
md \
mdio \
mem \
@ -648,6 +649,7 @@ _ixv= ixv
.if ${MK_SOURCELESS_UCODE} != "no"
_lio= lio
.endif
_mana= mana
_nctgpio= nctgpio
_ntb= ntb
_ocs_fc= ocs_fc

12
sys/modules/mana/Makefile Normal file
View File

@ -0,0 +1,12 @@
# $FreeBSD$
.PATH: ${SRCTOP}/sys/dev/mana
KMOD = if_mana
SRCS = gdma_main.c mana_sysctl.c shm_channel.c
SRCS += mana_en.c gdma_util.c hw_channel.c
SRCS += device_if.h bus_if.h pci_if.h
CFLAGS += -I${SRCTOP}/sys/dev/mana
.include <bsd.kmod.mk>