Merge from projects/counters:

Pad struct pcpu so that its size is denominator of PAGE_SIZE. This
is done to reduce memory waste in UMA_PCPU_ZONE zones.

Sponsored by:	Nginx, Inc.

sys/amd64/include/pcpu.h

@@ -77,7 +77,9 @@
 	/* Pointer to the CPU TSS descriptor */				\
 	struct system_segment_descriptor *pc_tss;			\
 	u_int	pc_cmci_mask		/* MCx banks for CMCI */	\
-	PCPU_XEN_FIELDS
+	PCPU_XEN_FIELDS;						\
+	char	__pad[293]		/* be divisor of PAGE_SIZE	\
+					   after cache alignment */
 
 #ifdef _KERNEL
 

sys/arm/include/pcpu.h

@@ -48,12 +48,13 @@ struct vmspace;
 	unsigned int pc_vfpmvfr0;					\
 	unsigned int pc_vfpmvfr1;					\
 	struct thread *pc_vfpcthread;					\
-	struct pmap *pc_curpmap;
+	struct pmap *pc_curpmap;					\
+	char __pad[133]
 #else
-#define PCPU_MD_FIELDS
+#define PCPU_MD_FIELDS							\
+	char __pad[157]
 #endif
 
-
 #ifdef _KERNEL
 
 struct pcb;

sys/i386/include/pcpu.h

@@ -74,18 +74,22 @@ struct shadow_time_info {
 	int	pc_resched_irq;						\
 	int	pc_callfunc_irq;					\
 	int	pc_virq_to_irq[NR_VIRQS];				\
-	int	pc_ipi_to_irq[NR_IPIS]	
+	int	pc_ipi_to_irq[NR_IPIS];					\
+	char	__pad[77]
 
 #elif defined(XENHVM)
 
 #define	PCPU_XEN_FIELDS							\
 	;								\
 	unsigned int pc_last_processed_l1i;				\
-	unsigned int pc_last_processed_l2i
+	unsigned int pc_last_processed_l2i;				\
+	char	__pad[229]
 
 #else /* !XEN && !XENHVM */
 
-#define PCPU_XEN_FIELDS
+#define PCPU_XEN_FIELDS							\
+	;								\
+	char	__pad[237]
 
 #endif
 

sys/ia64/include/pcpu.h

@@ -64,7 +64,8 @@ struct pcpu_md {
 
 #define	PCPU_MD_FIELDS							\
 	uint32_t	pc_acpi_id;		/* ACPI CPU id. */	\
-	struct pcpu_md	pc_md			/* MD fields. */
+	struct pcpu_md	pc_md;			/* MD fields. */	\
+	char		__pad[1265]
 
 #ifdef _KERNEL
 

sys/mips/include/pcpu.h

@@ -33,13 +33,29 @@
 
 #include <machine/pte.h>
 
-#define	PCPU_MD_FIELDS							\
+#define	PCPU_MD_COMMON_FIELDS						\
 	pd_entry_t	*pc_segbase;		/* curthread segbase */	\
 	struct	pmap	*pc_curpmap;		/* pmap of curthread */	\
 	u_int32_t	pc_next_asid;		/* next ASID to alloc */ \
 	u_int32_t	pc_asid_generation;	/* current ASID generation */ \
 	u_int		pc_pending_ipis;	/* IPIs pending to this CPU */
 
+#ifdef	__mips_n64
+#define	PCPU_MD_MIPS64_FIELDS						\
+	PCPU_MD_COMMON_FIELDS						\
+	char		__pad[61]
+#else
+#define	PCPU_MD_MIPS32_FIELDS						\
+	PCPU_MD_COMMON_FIELDS						\
+	char		__pad[133]
+#endif
+
+#ifdef	__mips_n64
+#define	PCPU_MD_FIELDS	PCPU_MD_MIPS64_FIELDS
+#else
+#define	PCPU_MD_FIELDS	PCPU_MD_MIPS32_FIELDS
+#endif
+
 #ifdef _KERNEL
 
 extern char pcpu_space[MAXCPU][PAGE_SIZE * 2];

sys/powerpc/include/pcpu.h

@@ -51,13 +51,15 @@ struct pmap;
 	register_t	pc_disisave[CPUSAVE_LEN];			\
 	register_t	pc_dbsave[CPUSAVE_LEN];
 
-#define PCPU_MD_AIM32_FIELDS
+#define PCPU_MD_AIM32_FIELDS						\
+	/* char		__pad[0] */
 
 #define PCPU_MD_AIM64_FIELDS						\
 	struct slb	pc_slb[64];					\
 	struct slb	**pc_userslb;					\
 	register_t	pc_slbsave[18];					\
-	uint8_t		pc_slbstack[1024];
+	uint8_t		pc_slbstack[1024];				\
+	char		__pad[1137]
 
 #ifdef __powerpc64__
 #define PCPU_MD_AIM_FIELDS	PCPU_MD_AIM64_FIELDS
@@ -76,7 +78,8 @@ struct pmap;
 	register_t	pc_booke_tlbsave[BOOKE_TLBSAVE_LEN];		\
 	register_t	pc_booke_tlb_level;				\
 	uint32_t	*pc_booke_tlb_lock;				\
-	int		pc_tid_next;
+	int		pc_tid_next;					\
+	char		__pad[173]
 
 /* Definitions for register offsets within the exception tmp save areas */
 #define	CPUSAVE_R27	0		/* where r27 gets saved */

sys/sparc64/include/pcpu.h

@@ -60,7 +60,8 @@ struct pmap;
 	u_int	pc_node;						\
 	u_int	pc_tlb_ctx;						\
 	u_int	pc_tlb_ctx_max;						\
-	u_int	pc_tlb_ctx_min
+	u_int	pc_tlb_ctx_min;						\
+	char	__pad[405]
 
 #ifdef _KERNEL
 

sys/sys/pcpu.h

@@ -180,6 +180,14 @@ struct pcpu {
 	PCPU_MD_FIELDS;
 } __aligned(CACHE_LINE_SIZE);
 
+#ifdef CTASSERT
+/*
+ * To minimize memory waste in per-cpu UMA zones, size of struct pcpu
+ * should be denominator of PAGE_SIZE.
+ */
+CTASSERT((PAGE_SIZE / sizeof(struct pcpu)) * sizeof(struct pcpu) == PAGE_SIZE);
+#endif
+
 #ifdef _KERNEL
 
 STAILQ_HEAD(cpuhead, pcpu);
@@ -194,6 +202,14 @@ extern struct pcpu *cpuid_to_pcpu[];
 #endif
 #define	curvidata	PCPU_GET(vidata)
 
+/* Accessor to elements allocated via UMA_ZONE_PCPU zone. */
+static inline void *
+zpcpu_get(void *base)
+{
+
+	return ((char *)(base) + sizeof(struct pcpu) * curcpu);
+}
+
 /*
  * Machine dependent callouts.  cpu_pcpu_init() is responsible for
  * initializing machine dependent fields of struct pcpu, and