hyperv/vmbus: Cleanup cpu based channel selection.

And create cpu to channel map at device attach time for storvsc(4).

MFC after:	1 week
Sponsored by:	Microsoft OSTC
Differential Revision:	https://reviews.freebsd.org/D7229

sys/dev/hyperv/include/hyperv.h

@@ -281,8 +281,6 @@ int		hv_vmbus_channel_open(struct hv_vmbus_channel *chan,
 		    vmbus_chan_callback_t cb, void *cbarg);
 void		hv_vmbus_channel_close(hv_vmbus_channel *channel);
 
-struct hv_vmbus_channel* vmbus_select_outgoing_channel(struct hv_vmbus_channel *promary);
-
 /**
  * @brief Get physical address from virtual
  */

sys/dev/hyperv/include/vmbus.h

@@ -96,13 +96,14 @@ int	vmbus_chan_gpadl_disconnect(struct hv_vmbus_channel *chan,
 
 void	vmbus_chan_cpu_set(struct hv_vmbus_channel *chan, int cpu);
 void	vmbus_chan_cpu_rr(struct hv_vmbus_channel *chan);
+struct hv_vmbus_channel *
+	vmbus_chan_cpu2chan(struct hv_vmbus_channel *chan, int cpu);
 
 struct hv_vmbus_channel **
 	vmbus_subchan_get(struct hv_vmbus_channel *pri_chan, int subchan_cnt);
 void	vmbus_subchan_rel(struct hv_vmbus_channel **subchan, int subchan_cnt);
 void	vmbus_subchan_drain(struct hv_vmbus_channel *pri_chan);
 
-
 int	vmbus_chan_recv(struct hv_vmbus_channel *chan, void *data, int *dlen,
 	    uint64_t *xactid);
 int	vmbus_chan_recv_pkt(struct hv_vmbus_channel *chan,

sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c

@@ -147,6 +147,8 @@ struct storvsc_softc {
 	struct hv_storvsc_request	hs_init_req;
 	struct hv_storvsc_request	hs_reset_req;
 	device_t			hs_dev;
+
+	struct hv_vmbus_channel		*hs_cpu2chan[MAXCPU];
 };
 
 
@@ -664,7 +666,7 @@ hv_storvsc_io_request(struct storvsc_softc *sc,
 
 	vstor_packet->operation = VSTOR_OPERATION_EXECUTESRB;
 
-	outgoing_channel = vmbus_select_outgoing_channel(sc->hs_chan);
+	outgoing_channel = sc->hs_cpu2chan[curcpu];
 
 	mtx_unlock(&request->softc->hs_lock);
 	if (request->prp_list.gpa_range.gpa_len) {
@@ -870,6 +872,20 @@ storvsc_probe(device_t dev)
 	return (ret);
 }
 
+static void
+storvsc_create_cpu2chan(struct storvsc_softc *sc)
+{
+	int cpu;
+
+	CPU_FOREACH(cpu) {
+		sc->hs_cpu2chan[cpu] = vmbus_chan_cpu2chan(sc->hs_chan, cpu);
+		if (bootverbose) {
+			device_printf(sc->hs_dev, "cpu%d -> chan%u\n",
+			    cpu, sc->hs_cpu2chan[cpu]->ch_id);
+		}
+	}
+}
+
 /**
  * @brief StorVSC attach function
  *
@@ -967,6 +983,9 @@ storvsc_attach(device_t dev)
 		goto cleanup;
 	}
 
+	/* Construct cpu to channel mapping */
+	storvsc_create_cpu2chan(sc);
+
 	/*
 	 * Create the device queue.
 	 * Hyper-V maps each target to one SCSI HBA

sys/dev/hyperv/vmbus/hv_channel.c

@@ -1250,61 +1250,63 @@ vmbus_chan_destroy_all(struct vmbus_softc *sc)
 	mtx_unlock(&sc->vmbus_prichan_lock);
 }
 
-/**
- * @brief Select the best outgoing channel
- * 
+/*
  * The channel whose vcpu binding is closest to the currect vcpu will
  * be selected.
- * If no multi-channel, always select primary channel
- * 
- * @param primary - primary channel
+ * If no multi-channel, always select primary channel.
  */
 struct hv_vmbus_channel *
-vmbus_select_outgoing_channel(struct hv_vmbus_channel *primary)
+vmbus_chan_cpu2chan(struct hv_vmbus_channel *prichan, int cpu)
 {
-	hv_vmbus_channel *new_channel = NULL;
-	hv_vmbus_channel *outgoing_channel = primary;
-	int old_cpu_distance = 0;
-	int new_cpu_distance = 0;
-	int cur_vcpu = 0;
-	int smp_pro_id = PCPU_GET(cpuid);
+	struct hv_vmbus_channel *sel, *chan;
+	uint32_t vcpu, sel_dist;
 
-	if (TAILQ_EMPTY(&primary->ch_subchans)) {
-		return outgoing_channel;
-	}
+	KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpuid %d", cpu));
+	if (TAILQ_EMPTY(&prichan->ch_subchans))
+		return prichan;
 
-	if (smp_pro_id >= MAXCPU) {
-		return outgoing_channel;
-	}
+	vcpu = VMBUS_PCPU_GET(prichan->vmbus_sc, vcpuid, cpu);
 
-	cur_vcpu = VMBUS_PCPU_GET(primary->vmbus_sc, vcpuid, smp_pro_id);
-	
-	/* XXX need lock */
-	TAILQ_FOREACH(new_channel, &primary->ch_subchans, ch_sublink) {
-		if ((new_channel->ch_stflags & VMBUS_CHAN_ST_OPENED) == 0) {
+#define CHAN_VCPU_DIST(ch, vcpu)		\
+	(((ch)->ch_vcpuid > (vcpu)) ?		\
+	 ((ch)->ch_vcpuid - (vcpu)) : ((vcpu) - (ch)->ch_vcpuid))
+
+#define CHAN_SELECT(ch)				\
+do {						\
+	sel = ch;				\
+	sel_dist = CHAN_VCPU_DIST(ch, vcpu);	\
+} while (0)
+
+	CHAN_SELECT(prichan);
+
+	mtx_lock(&prichan->ch_subchan_lock);
+	TAILQ_FOREACH(chan, &prichan->ch_subchans, ch_sublink) {
+		uint32_t dist;
+
+		KASSERT(chan->ch_stflags & VMBUS_CHAN_ST_OPENED,
+		    ("chan%u is not opened", chan->ch_id));
+
+		if (chan->ch_vcpuid == vcpu) {
+			/* Exact match; done */
+			CHAN_SELECT(chan);
+			break;
+		}
+
+		dist = CHAN_VCPU_DIST(chan, vcpu);
+		if (sel_dist <= dist) {
+			/* Far or same distance; skip */
 			continue;
 		}
 
-		if (new_channel->ch_vcpuid == cur_vcpu){
-			return new_channel;
-		}
-
-		old_cpu_distance = ((outgoing_channel->ch_vcpuid > cur_vcpu) ?
-		    (outgoing_channel->ch_vcpuid - cur_vcpu) :
-		    (cur_vcpu - outgoing_channel->ch_vcpuid));
-
-		new_cpu_distance = ((new_channel->ch_vcpuid > cur_vcpu) ?
-		    (new_channel->ch_vcpuid - cur_vcpu) :
-		    (cur_vcpu - new_channel->ch_vcpuid));
-
-		if (old_cpu_distance < new_cpu_distance) {
-			continue;
-		}
-
-		outgoing_channel = new_channel;
+		/* Select the closer channel. */
+		CHAN_SELECT(chan);
 	}
+	mtx_unlock(&prichan->ch_subchan_lock);
 
-	return(outgoing_channel);
+#undef CHAN_SELECT
+#undef CHAN_VCPU_DIST
+
+	return sel;
 }
 
 struct hv_vmbus_channel **