cxgbe(4): Overhaul CLIP (Compressed Local IPv6) table management.

- Process the list of local IPs once instead of once per adapter.  Add
  addresses from all VNETs to the driver's list but leave hardware
  updates for later when the global VNET/IFADDR list locks have been
  released.

- Add address to the hardware table synchronously when a CLIP entry is
  requested for an address that's not already in there.

- Provide ioctls that allow userspace tools to manage addresses in the
  CLIP table.

- Add a knob (hw.cxgbe.clip_db_auto) that controls whether local IPs are
  automatically added to the CLIP table or not.

MFC after:	2 weeks
Sponsored by:	Chelsio Communications

sys/dev/cxgbe/adapter.h

@@ -50,6 +50,7 @@
 #include <machine/bus.h>
 #include <sys/socket.h>
 #include <sys/sysctl.h>
+#include <sys/taskqueue.h>
 #include <net/ethernet.h>
 #include <net/if.h>
 #include <net/if_var.h>
@@ -68,6 +69,15 @@ MALLOC_DECLARE(M_CXGBE);
 #define CXGBE_UNIMPLEMENTED(s) \
     panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__)
 
+/*
+ * Same as LIST_HEAD from queue.h.  This is to avoid conflict with LinuxKPI's
+ * LIST_HEAD when building iw_cxgbe.
+ */
+#define	CXGBE_LIST_HEAD(name, type)					\
+struct name {								\
+	struct type *lh_first;	/* first element */			\
+}
+
 #ifndef SYSCTL_ADD_UQUAD
 #define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD
 #define sysctl_handle_64 sysctl_handle_quad
@@ -886,9 +896,11 @@ struct adapter {
 	struct port_info *port[MAX_NPORTS];
 	uint8_t chan_map[MAX_NCHAN];		/* channel -> port */
 
-	struct mtx clip_table_lock;
+	CXGBE_LIST_HEAD(, clip_entry) *clip_table;
-	TAILQ_HEAD(, clip_entry) clip_table;
+	TAILQ_HEAD(, clip_entry) clip_pending;	/* these need hw update. */
+	u_long clip_mask;
 	int clip_gen;
+	struct timeout_task clip_task;
 
 	void *tom_softc;	/* (struct tom_data *) */
 	struct tom_tunables tt;

sys/dev/cxgbe/crypto/t4_kern_tls.c

@@ -379,7 +379,7 @@ send_ktls_act_open_req(struct adapter *sc, struct vi_info *vi,
 
 	isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
 	if (isipv6) {
-		tlsp->ce = t4_hold_lip(sc, &inp->in6p_laddr, NULL);
+		tlsp->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true);
 		if (tlsp->ce == NULL)
 			return (ENOENT);
 	}
@@ -2333,7 +2333,7 @@ cxgbe_tls_tag_free(struct m_snd_tag *mst)
 	if (tlsp->tid >= 0)
 		release_tid(sc, tlsp->tid, tlsp->ctrlq);
 	if (tlsp->ce)
-		t4_release_lip(sc, tlsp->ce);
+		t4_release_clip_entry(sc, tlsp->ce);
 	if (tlsp->tx_key_addr >= 0)
 		free_keyid(tlsp, tlsp->tx_key_addr);
 

sys/dev/cxgbe/t4_clip.h

@@ -32,19 +32,18 @@
 #ifndef __T4_CLIP_H
 #define	__T4_CLIP_H
 
-struct clip_entry {
+#define CLIP_HASH_SIZE 32
-	TAILQ_ENTRY(clip_entry) link;
+struct clip_entry;
-	struct in6_addr lip;	/* local IPv6 address */
+struct in6_addr;
-	u_int refcount;
-};
 
 void t4_clip_modload(void);
 void t4_clip_modunload(void);
 void t4_init_clip_table(struct adapter *);
 void t4_destroy_clip_table(struct adapter *);
-struct clip_entry *t4_hold_lip(struct adapter *, struct in6_addr *,
+struct clip_entry *t4_get_clip_entry(struct adapter *, struct in6_addr *, bool);
-    struct clip_entry *);
+void t4_hold_clip_entry(struct adapter *, struct clip_entry *);
-void t4_release_lip(struct adapter *, struct clip_entry *);
+void t4_release_clip_entry(struct adapter *, struct clip_entry *);
+int t4_release_clip_addr(struct adapter *, struct in6_addr *);
 
 int sysctl_clip(SYSCTL_HANDLER_ARGS);
 

sys/dev/cxgbe/t4_ioctl.h

@@ -64,6 +64,8 @@ enum {
 	T4_LOAD_BOOTCFG,		/* flash bootcfg */
 	T4_CUDBG_DUMP,			/* debug dump of chip state */
 	T4_SET_FILTER_MASK,		/* set filter mask (hashfilter mode) */
+	T4_HOLD_CLIP_ADDR,		/* add ref on an IP in the CLIP */
+	T4_RELEASE_CLIP_ADDR,		/* remove ref from an IP in the CLIP */
 };
 
 struct t4_reg {
@@ -405,6 +407,12 @@ struct t4_offload_policy {
 	struct offload_rule *rule;
 };
 
+/* Address/mask entry in the CLIP.  FW_CLIP2_CMD is aware of the mask. */
+struct t4_clip_addr {
+	uint8_t addr[16];
+	uint8_t mask[16];
+};
+
 #define CHELSIO_T4_GETREG	_IOWR('f', T4_GETREG, struct t4_reg)
 #define CHELSIO_T4_SETREG	_IOW('f', T4_SETREG, struct t4_reg)
 #define CHELSIO_T4_REGDUMP	_IOWR('f', T4_REGDUMP, struct t4_regdump)
@@ -431,4 +439,6 @@ struct t4_offload_policy {
 #define CHELSIO_T4_CUDBG_DUMP	_IOWR('f', T4_CUDBG_DUMP, struct t4_cudbg_dump)
 #define CHELSIO_T4_SET_OFLD_POLICY _IOW('f', T4_SET_OFLD_POLICY, struct t4_offload_policy)
 #define CHELSIO_T4_SET_FILTER_MASK _IOW('f', T4_SET_FILTER_MASK, uint32_t)
+#define CHELSIO_T4_HOLD_CLIP_ADDR _IOW('f', T4_HOLD_CLIP_ADDR, struct t4_clip_addr)
+#define CHELSIO_T4_RELEASE_CLIP_ADDR _IOW('f', T4_RELEASE_CLIP_ADDR, struct t4_clip_addr)
 #endif

sys/dev/cxgbe/t4_main.c

@@ -838,6 +838,8 @@ static int set_offload_policy(struct adapter *, struct t4_offload_policy *);
 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 static int clear_stats(struct adapter *, u_int);
+static int hold_clip_addr(struct adapter *, struct t4_clip_addr *);
+static int release_clip_addr(struct adapter *, struct t4_clip_addr *);
 #ifdef TCP_OFFLOAD
 static int toe_capability(struct vi_info *, bool);
 static void t4_async_event(void *, int);
@@ -11910,6 +11912,35 @@ clear_stats(struct adapter *sc, u_int port_id)
 	return (0);
 }
 
+static int
+hold_clip_addr(struct adapter *sc, struct t4_clip_addr *ca)
+{
+#ifdef INET6
+	struct in6_addr in6;
+
+	bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr));
+	if (t4_get_clip_entry(sc, &in6, true) != NULL)
+		return (0);
+	else
+		return (EIO);
+#else
+	return (ENOTSUP);
+#endif
+}
+
+static int
+release_clip_addr(struct adapter *sc, struct t4_clip_addr *ca)
+{
+#ifdef INET6
+	struct in6_addr in6;
+
+	bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr));
+	return (t4_release_clip_addr(sc, &in6));
+#else
+	return (ENOTSUP);
+#endif
+}
+
 int
 t4_os_find_pci_capability(struct adapter *sc, int cap)
 {
@@ -12181,6 +12212,12 @@ t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
 	case CHELSIO_T4_SET_OFLD_POLICY:
 		rc = set_offload_policy(sc, (struct t4_offload_policy *)data);
 		break;
+	case CHELSIO_T4_HOLD_CLIP_ADDR:
+		rc = hold_clip_addr(sc, (struct t4_clip_addr *)data);
+		break;
+	case CHELSIO_T4_RELEASE_CLIP_ADDR:
+		rc = release_clip_addr(sc, (struct t4_clip_addr *)data);
+		break;
 	default:
 		rc = ENOTTY;
 	}

sys/dev/cxgbe/tom/t4_connect.c

@@ -300,7 +300,7 @@ t4_connect(struct toedev *tod, struct socket *so, struct nhop_object *nh,
 		if ((inp->inp_vflag & INP_IPV6) == 0)
 			DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
 
-		toep->ce = t4_hold_lip(sc, &inp->in6p_laddr, NULL);
+		toep->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true);
 		if (toep->ce == NULL)
 			DONT_OFFLOAD_ACTIVE_OPEN(ENOENT);
 
@@ -394,7 +394,7 @@ t4_connect(struct toedev *tod, struct socket *so, struct nhop_object *nh,
 		if (toep->l2te)
 			t4_l2t_release(toep->l2te);
 		if (toep->ce)
-			t4_release_lip(sc, toep->ce);
+			t4_release_clip_entry(sc, toep->ce);
 		free_toepcb(toep);
 	}
 

sys/dev/cxgbe/tom/t4_listen.c

@@ -211,7 +211,7 @@ alloc_lctx(struct adapter *sc, struct inpcb *inp, struct vi_info *vi)
 
 	if (inp->inp_vflag & INP_IPV6 &&
 	    !IN6_ARE_ADDR_EQUAL(&in6addr_any, &inp->in6p_laddr)) {
-		lctx->ce = t4_hold_lip(sc, &inp->in6p_laddr, NULL);
+		lctx->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true);
 		if (lctx->ce == NULL) {
 			free(lctx, M_CXGBE);
 			return (NULL);
@@ -244,7 +244,7 @@ free_lctx(struct adapter *sc, struct listen_ctx *lctx)
 	    __func__, lctx->stid, lctx, lctx->inp);
 
 	if (lctx->ce)
-		t4_release_lip(sc, lctx->ce);
+		t4_release_clip_entry(sc, lctx->ce);
 	free_stid(sc, lctx);
 	free(lctx, M_CXGBE);
 
@@ -1522,8 +1522,18 @@ do_pass_establish(struct sge_iq *iq, const struct rss_header *rss,
 
 	/* Come up with something that syncache_expand should be ok with. */
 	synqe_to_protohdrs(sc, synqe, cpl, &inc, &th, &to);
-	if (inc.inc_flags & INC_ISIPV6)
+	if (inc.inc_flags & INC_ISIPV6) {
-		toep->ce = t4_hold_lip(sc, &inc.inc6_laddr, lctx->ce);
+		if (lctx->ce == NULL) {
+			toep->ce = t4_get_clip_entry(sc, &inc.inc6_laddr, true);
+			if (toep->ce == NULL) {
+				free_toepcb(toep);
+				goto reset;	/* RST without a CLIP entry? */
+			}
+		} else {
+			t4_hold_clip_entry(sc, lctx->ce);
+			toep->ce = lctx->ce;
+		}
+	}
 	so = inp->inp_socket;
 	KASSERT(so != NULL, ("%s: socket is NULL", __func__));
 

sys/dev/cxgbe/tom/t4_tom.c

@@ -348,7 +348,7 @@ release_offload_resources(struct toepcb *toep)
 	}
 
 	if (toep->ce)
-		t4_release_lip(sc, toep->ce);
+		t4_release_clip_entry(sc, toep->ce);
 
 	if (toep->params.tc_idx != -1)
 		t4_release_cl_rl(sc, toep->vi->pi->port_id, toep->params.tc_idx);