From ec6f8f42dbbf60b318056f4de52ae9af8f211c1d Mon Sep 17 00:00:00 2001
From: Zbigniew Bodek <zbb@FreeBSD.org>
Date: Wed, 11 May 2016 13:20:29 +0000
Subject: [PATCH] Bind CQ interrupts and tasks to separate CPUs in VNIC

Delegate interrupts and completion tasks on separate CPUs
for each VNIC.

Reviewed by:	wma
Obtained from:	Semihalf
Sponsored by:	Cavium
Differential Revision: https://reviews.freebsd.org/D6229
---
 sys/dev/vnic/nicvf_main.c   | 15 +++++++++++++++
 sys/dev/vnic/nicvf_queues.h |  1 +
 2 files changed, 16 insertions(+)

diff --git a/sys/dev/vnic/nicvf_main.c b/sys/dev/vnic/nicvf_main.c
index 5b74ac9ea8f3..f9a733b9990d 100644
--- a/sys/dev/vnic/nicvf_main.c
+++ b/sys/dev/vnic/nicvf_main.c
@@ -1296,6 +1296,7 @@ nicvf_release_net_interrupts(struct nicvf *nic)
 static int
 nicvf_allocate_net_interrupts(struct nicvf *nic)
 {
+	u_int cpuid;
 	int irq, rid;
 	int qidx;
 	int ret = 0;
@@ -1332,6 +1333,20 @@ nicvf_allocate_net_interrupts(struct nicvf *nic)
 			    (irq - NICVF_INTR_ID_CQ), device_get_unit(nic->dev));
 			goto error;
 		}
+		cpuid = (device_get_unit(nic->dev) * CMP_QUEUE_CNT) + qidx;
+		cpuid %= mp_ncpus;
+		/*
+		 * Save CPU ID for later use when system-wide RSS is enabled.
+		 * It will be used to pit the CQ task to the same CPU that got
+		 * interrupted.
+		 */
+		nic->qs->cq[qidx].cmp_cpuid = cpuid;
+		if (bootverbose) {
+			device_printf(nic->dev, "bind CQ%d IRQ to CPU%d\n",
+			    qidx, cpuid);
+		}
+		/* Bind interrupts to the given CPU */
+		bus_bind_intr(nic->dev, nic->msix_entries[irq].irq_res, cpuid);
 	}
 
 	/* Register RBDR interrupt */
diff --git a/sys/dev/vnic/nicvf_queues.h b/sys/dev/vnic/nicvf_queues.h
index 0fe8e3b88e05..8a828157741d 100644
--- a/sys/dev/vnic/nicvf_queues.h
+++ b/sys/dev/vnic/nicvf_queues.h
@@ -296,6 +296,7 @@ struct cmp_queue {
 
 	struct task		cmp_task;
 	struct taskqueue	*cmp_taskq;
+	u_int			cmp_cpuid; /* CPU to which bind the CQ task */
 
 	void			*desc;
 	struct q_desc_mem	dmem;