Add new a M_START() mbuf macro that returns a pointer to the start of

an mbuf's storage (internal or external).

Add a new M_SIZE() mbuf macro that returns the size of an mbuf's
storage (internal or external).

These contrast with m_data and m_len, which are with respect to data
in the buffer, rather than the buffer itself.

Rewrite M_LEADINGSPACE() and M_TRAILINGSPACE() in terms of M_START()
and M_SIZE().

This is done as we currently have many instances of using mbuf flags
to generate pointers or lengths for internal storage in header and
regular mbufs, as well as to external storage. Rather than replicate
this logic throughout the network stack, centralising the
implementation will make it easier for us to refine mbuf storage.
This should also help reduce bugs by limiting the amount of
mbuf-type-specific pointer arithmetic.  Followup changes will
propagate use of the macros throughout the stack.

M_SIZE() conflicts with one macro in the Chelsio driver; rename that
macro in a slightly unsatisfying way to eliminate the collision.

MFC after:	3 days
Obtained from:	jeff (with enhancements)
Sponsored by:	EMC / Isilon Storage Division
Reviewed by:	bz, glebius, np
Differential Revision:	https://reviews.freebsd.org/D753

sys/dev/cxgbe/common/t4_regs.h

@@ -1073,9 +1073,9 @@
 #define A_SGE_FL_BUFFER_SIZE0 0x1044
 
 #define S_SIZE    4
-#define M_SIZE    0xfffffffU
+#define CXGBE_M_SIZE    0xfffffffU
 #define V_SIZE(x) ((x) << S_SIZE)
-#define G_SIZE(x) (((x) >> S_SIZE) & M_SIZE)
+#define G_SIZE(x) (((x) >> S_SIZE) & CXGBE_M_SIZE)
 
 #define A_SGE_FL_BUFFER_SIZE1 0x1048
 #define A_SGE_FL_BUFFER_SIZE2 0x104c

sys/sys/mbuf.h

@@ -842,30 +842,51 @@ m_last(struct mbuf *m)
 	    ~(sizeof(long) - 1); 					\
 } while (0)
 
+/*
+ * Return the address of the start of the buffer associated with an mbuf,
+ * handling external storage, packet-header mbufs, and regular data mbufs.
+ */
+#define	M_START(m)							\
+	(((m)->m_flags & M_EXT) ? (m)->m_ext.ext_buf :			\
+	 ((m)->m_flags & M_PKTHDR) ? &(m)->m_pktdat[0] :		\
+	 &(m)->m_dat[0])
+
+/*
+ * Return the size of the buffer associated with an mbuf, handling external
+ * storage, packet-header mbufs, and regular data mbufs.
+ */
+#define	M_SIZE(m)							\
+	(((m)->m_flags & M_EXT) ? (m)->m_ext.ext_size :			\
+	 ((m)->m_flags & M_PKTHDR) ? MHLEN :				\
+	 MLEN)
+
 /*
  * Compute the amount of space available before the current start of data in
  * an mbuf.
  *
  * The M_WRITABLE() is a temporary, conservative safety measure: the burden
  * of checking writability of the mbuf data area rests solely with the caller.
+ *
+ * NB: In previous versions, M_LEADINGSPACE() would only check M_WRITABLE()
+ * for mbufs with external storage.  We now allow mbuf-embedded data to be
+ * read-only as well.
  */
 #define	M_LEADINGSPACE(m)						\
-	((m)->m_flags & M_EXT ?						\
-	    (M_WRITABLE(m) ? (m)->m_data - (m)->m_ext.ext_buf : 0):	\
-	    (m)->m_flags & M_PKTHDR ? (m)->m_data - (m)->m_pktdat :	\
-	    (m)->m_data - (m)->m_dat)
+	(M_WRITABLE(m) ? ((m)->m_data - M_START(m)) : 0)
 
 /*
  * Compute the amount of space available after the end of data in an mbuf.
  *
  * The M_WRITABLE() is a temporary, conservative safety measure: the burden
  * of checking writability of the mbuf data area rests solely with the caller.
+ *
+ * NB: In previous versions, M_TRAILINGSPACE() would only check M_WRITABLE()
+ * for mbufs with external storage.  We now allow mbuf-embedded data to be
+ * read-only as well.
  */
 #define	M_TRAILINGSPACE(m)						\
-	((m)->m_flags & M_EXT ?						\
-	    (M_WRITABLE(m) ? (m)->m_ext.ext_buf + (m)->m_ext.ext_size	\
-		- ((m)->m_data + (m)->m_len) : 0) :			\
-	    &(m)->m_dat[MLEN] - ((m)->m_data + (m)->m_len))
+	(M_WRITABLE(m) ?						\
+	    ((M_START(m) + M_SIZE(m)) - ((m)->m_data + (m)->m_len)) : 0)
 
 /*
  * Arrange to prepend space of size plen to mbuf m.  If a new mbuf must be