net/thunderx/base: add secondary queue set support

Changes:
 - add new message sqs_alloc in mailbox
 - add a queue container to hold secondary qsets.
 - add nicvf_mbox_request_sqs
 - handle new mailbox messages for secondary queue set support
 - register secondary queue sets for further reuse
 - register the number secondary queue sets in MSG_QS_CFG

Signed-off-by: Maciej Czekaj <maciej.czekaj@caviumnetworks.com>
Signed-off-by: Kamil Rytarowski <kamil.rytarowski@caviumnetworks.com>
Signed-off-by: Zyta Szpak <zyta.szpak@semihalf.com>
Signed-off-by: Slawomir Rosek <slawomir.rosek@semihalf.com>
Signed-off-by: Radoslaw Biernacki <rad@semihalf.com>
Signed-off-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

drivers/net/thunderx/base/nicvf_hw_defs.h

@@ -207,6 +207,7 @@
 #define NICVF_CQE_RX2_RBPTR_WORD        (7)
 
 #define NICVF_STATIC_ASSERT(s) _Static_assert(s, #s)
+#define assert_primary(nic) assert((nic)->sqs_mode == 0)
 
 typedef uint64_t nicvf_phys_addr_t;
 

drivers/net/thunderx/base/nicvf_mbox.c

@@ -62,6 +62,9 @@ static const char *mbox_message[NIC_MBOX_MSG_MAX] =  {
 	[NIC_MBOX_MSG_RESET_STAT_COUNTER] = "NIC_MBOX_MSG_RESET_STAT_COUNTER",
 	[NIC_MBOX_MSG_CFG_DONE]           = "NIC_MBOX_MSG_CFG_DONE",
 	[NIC_MBOX_MSG_SHUTDOWN]           = "NIC_MBOX_MSG_SHUTDOWN",
+	[NIC_MBOX_MSG_RES_BIT]            = "NIC_MBOX_MSG_RES_BIT",
+	[NIC_MBOX_MSG_RSS_SIZE_RES_BIT]   = "NIC_MBOX_MSG_RSS_SIZE",
+	[NIC_MBOX_MSG_ALLOC_SQS_RES_BIT]  = "NIC_MBOX_MSG_ALLOC_SQS",
 };
 
 static inline const char * __attribute__((unused))
@@ -173,7 +176,7 @@ nicvf_handle_mbx_intr(struct nicvf *nic)
 	case NIC_MBOX_MSG_NACK:
 		nic->pf_nacked = true;
 		break;
-	case NIC_MBOX_MSG_RSS_SIZE:
+	case NIC_MBOX_MSG_RSS_SIZE_RES_BIT:
 		nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
 		nic->pf_acked = true;
 		break;
@@ -183,6 +186,26 @@ nicvf_handle_mbx_intr(struct nicvf *nic)
 		nic->speed = mbx.link_status.speed;
 		nic->pf_acked = true;
 		break;
+	case NIC_MBOX_MSG_ALLOC_SQS_RES_BIT:
+		assert_primary(nic);
+		if (mbx.sqs_alloc.qs_count != nic->sqs_count) {
+			nicvf_log_error("Received %" PRIu8 "/%" PRIu8
+			                " secondary qsets",
+			                mbx.sqs_alloc.qs_count,
+			                nic->sqs_count);
+			abort();
+		}
+		for (i = 0; i < mbx.sqs_alloc.qs_count; i++) {
+			if (mbx.sqs_alloc.svf[i] != nic->snicvf[i]->vf_id) {
+				nicvf_log_error("Received secondary qset[%zu] "
+				                "ID %" PRIu8 " expected %"
+				                PRIu8, i, mbx.sqs_alloc.svf[i],
+				                nic->snicvf[i]->vf_id);
+				abort();
+			}
+		}
+		nic->pf_acked = true;
+		break;
 	default:
 		nicvf_log_error("Invalid message from PF, msg_id=0x%hhx %s",
 				mbx.msg.msg, nicvf_mbox_msg_str(mbx.msg.msg));
@@ -314,10 +337,32 @@ nicvf_mbox_qset_config(struct nicvf *nic, struct pf_qs_cfg *qs_cfg)
 	/* Send a mailbox msg to PF to config Qset */
 	mbx.msg.msg = NIC_MBOX_MSG_QS_CFG;
 	mbx.qs.num = nic->vf_id;
+	mbx.qs.sqs_count = nic->sqs_count;
 	mbx.qs.cfg = qs_cfg->value;
 	return nicvf_mbox_send_msg_to_pf(nic, &mbx);
 }
 
+int
+nicvf_mbox_request_sqs(struct nicvf *nic)
+{
+	struct nic_mbx mbx = { .msg = { 0 } };
+	size_t i;
+
+	assert_primary(nic);
+	assert(nic->sqs_count > 0);
+	assert(nic->sqs_count <= MAX_SQS_PER_VF);
+
+	mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS;
+	mbx.sqs_alloc.spec = 1;
+	mbx.sqs_alloc.qs_count = nic->sqs_count;
+
+	/* Set no of Rx/Tx queues in each of the SQsets */
+	for (i = 0; i < nic->sqs_count; i++)
+		mbx.sqs_alloc.svf[i] = nic->snicvf[i]->vf_id;
+
+	return nicvf_mbox_send_msg_to_pf(nic, &mbx);
+}
+
 int
 nicvf_mbox_rq_drop_config(struct nicvf *nic, uint16_t qidx, bool enable)
 {

drivers/net/thunderx/base/nicvf_mbox.h

@@ -36,6 +36,7 @@
 #include <stdint.h>
 
 #include "nicvf_plat.h"
+#include "../nicvf_struct.h"
 
 /* PF <--> VF Mailbox communication
  * Two 64bit registers are shared between PF and VF for each VF
@@ -67,10 +68,16 @@
 #define	NIC_MBOX_MSG_ALLOC_SQS		0x12	/* Allocate secondary Qset */
 #define	NIC_MBOX_MSG_LOOPBACK		0x16	/* Set interface in loopback */
 #define	NIC_MBOX_MSG_RESET_STAT_COUNTER 0x17	/* Reset statistics counters */
-#define	NIC_MBOX_MSG_CFG_DONE		0xF0	/* VF configuration done */
+#define	NIC_MBOX_MSG_CFG_DONE		0x7E	/* VF configuration done */
-#define	NIC_MBOX_MSG_SHUTDOWN		0xF1	/* VF is being shutdown */
+#define	NIC_MBOX_MSG_SHUTDOWN		0x7F	/* VF is being shutdown */
+#define	NIC_MBOX_MSG_RES_BIT		0x80	/* Reset bit from PF */
 #define	NIC_MBOX_MSG_MAX		0x100	/* Maximum number of messages */
 
+#define NIC_MBOX_MSG_RSS_SIZE_RES_BIT \
+	(NIC_MBOX_MSG_RSS_SIZE | NIC_MBOX_MSG_RES_BIT)
+#define NIC_MBOX_MSG_ALLOC_SQS_RES_BIT \
+	(NIC_MBOX_MSG_ALLOC_SQS | NIC_MBOX_MSG_RES_BIT)
+
 /* Get vNIC VF configuration */
 struct nic_cfg_msg {
 	uint8_t    msg;
@@ -155,6 +162,14 @@ struct bgx_link_status {
 	uint32_t   speed;
 };
 
+/* Allocate additional SQS to VF */
+struct sqs_alloc {
+	uint8_t    msg;
+	uint8_t    spec;
+	uint8_t    qs_count;
+	uint8_t    svf[MAX_SQS_PER_VF];
+};
+
 /* Set interface in loopback mode */
 struct set_loopback {
 	uint8_t    msg;
@@ -201,6 +216,7 @@ union {
 	struct rss_sz_msg	rss_size;
 	struct rss_cfg_msg	rss_cfg;
 	struct bgx_link_status  link_status;
+	struct sqs_alloc	sqs_alloc;
 	struct set_loopback	lbk;
 	struct reset_stat_cfg	reset_stat;
 };
@@ -211,6 +227,7 @@ NICVF_STATIC_ASSERT(sizeof(struct nic_mbx) <= 16);
 int nicvf_handle_mbx_intr(struct nicvf *nic);
 int nicvf_mbox_check_pf_ready(struct nicvf *nic);
 int nicvf_mbox_qset_config(struct nicvf *nic, struct pf_qs_cfg *qs_cfg);
+int nicvf_mbox_request_sqs(struct nicvf *nic);
 int nicvf_mbox_rq_config(struct nicvf *nic, uint16_t qidx,
 			 struct pf_rq_cfg *pf_rq_cfg);
 int nicvf_mbox_sq_config(struct nicvf *nic, uint16_t qidx);

drivers/net/thunderx/nicvf_struct.h

@@ -119,6 +119,11 @@ struct nicvf {
 	uint16_t mtu;
 	bool vlan_filter_en;
 	uint8_t mac_addr[ETHER_ADDR_LEN];
+	/* secondary queue set support */
+	uint8_t sqs_id;
+	uint8_t sqs_count;
+#define MAX_SQS_PER_VF 11
+	struct nicvf *snicvf[MAX_SQS_PER_VF];
 } __rte_cache_aligned;
 
 #endif /* _THUNDERX_NICVF_STRUCT_H */