Add a new cache maintenance function, idcache_inv_all, to the table, and

implementations for each of the chips we support.  Most chips up through
armv6 can use the armv4 implementation which has a single coprocessor
opcode for this operation.  The rather more complex armv7 implementation
comes from netbsd.

sys/arm/arm/cpufunc.c

@@ -146,6 +146,7 @@ struct cpu_functions arm7tdmi_cpufuncs = {
 	(void *)arm7tdmi_cache_flushID,	/* dcache_inv_range	*/
 	(void *)cpufunc_nullop,		/* dcache_wb_range	*/
 
+	cpufunc_nullop,			/* idcache_inv_all	*/
 	arm7tdmi_cache_flushID,		/* idcache_wbinv_all	*/
 	(void *)arm7tdmi_cache_flushID,	/* idcache_wbinv_range	*/
 	cpufunc_nullop,			/* l2cache_wbinv_all	*/
@@ -208,6 +209,7 @@ struct cpu_functions arm8_cpufuncs = {
 /*XXX*/	(void *)arm8_cache_purgeID,	/* dcache_inv_range	*/
 	(void *)arm8_cache_cleanID,	/* dcache_wb_range	*/
 
+	cpufunc_nullop,			/* idcache_inv_all	*/
 	arm8_cache_purgeID,		/* idcache_wbinv_all	*/
 	(void *)arm8_cache_purgeID,	/* idcache_wbinv_range	*/
 	cpufunc_nullop,			/* l2cache_wbinv_all	*/
@@ -269,6 +271,7 @@ struct cpu_functions arm9_cpufuncs = {
 	arm9_dcache_inv_range,		/* dcache_inv_range	*/
 	arm9_dcache_wb_range,		/* dcache_wb_range	*/
 
+	armv4_idcache_inv_all,		/* idcache_inv_all	*/
 	arm9_idcache_wbinv_all,		/* idcache_wbinv_all	*/
 	arm9_idcache_wbinv_range,	/* idcache_wbinv_range	*/
 	cpufunc_nullop,			/* l2cache_wbinv_all	*/
@@ -331,6 +334,7 @@ struct cpu_functions armv5_ec_cpufuncs = {
 	armv5_ec_dcache_inv_range,	/* dcache_inv_range	*/
 	armv5_ec_dcache_wb_range,	/* dcache_wb_range	*/
 
+	armv4_idcache_inv_all,		/* idcache_inv_all	*/
 	armv5_ec_idcache_wbinv_all,	/* idcache_wbinv_all	*/
 	armv5_ec_idcache_wbinv_range,	/* idcache_wbinv_range	*/
 
@@ -392,6 +396,7 @@ struct cpu_functions sheeva_cpufuncs = {
 	sheeva_dcache_inv_range,	/* dcache_inv_range	*/
 	sheeva_dcache_wb_range,		/* dcache_wb_range	*/
 
+	armv4_idcache_inv_all,		/* idcache_inv_all	*/
 	armv5_ec_idcache_wbinv_all,	/* idcache_wbinv_all	*/
 	sheeva_idcache_wbinv_range,	/* idcache_wbinv_all	*/
 
@@ -454,6 +459,7 @@ struct cpu_functions arm10_cpufuncs = {
 	arm10_dcache_inv_range,		/* dcache_inv_range	*/
 	arm10_dcache_wb_range,		/* dcache_wb_range	*/
 
+	armv4_idcache_inv_all,		/* idcache_inv_all	*/
 	arm10_idcache_wbinv_all,	/* idcache_wbinv_all	*/
 	arm10_idcache_wbinv_range,	/* idcache_wbinv_range	*/
 	cpufunc_nullop,			/* l2cache_wbinv_all	*/
@@ -515,6 +521,7 @@ struct cpu_functions pj4bv7_cpufuncs = {
 	armv7_dcache_inv_range,		/* dcache_inv_range	*/
 	armv7_dcache_wb_range,		/* dcache_wb_range	*/
 
+	armv7_idcache_inv_all,		/* idcache_inv_all	*/
 	armv7_idcache_wbinv_all,	/* idcache_wbinv_all	*/
 	armv7_idcache_wbinv_range,	/* idcache_wbinv_all	*/
 
@@ -577,6 +584,7 @@ struct cpu_functions sa110_cpufuncs = {
 /*XXX*/	sa1_cache_purgeD_rng,		/* dcache_inv_range	*/
 	sa1_cache_cleanD_rng,		/* dcache_wb_range	*/
 
+	sa1_cache_flushID,		/* idcache_inv_all	*/
 	sa1_cache_purgeID,		/* idcache_wbinv_all	*/
 	sa1_cache_purgeID_rng,		/* idcache_wbinv_range	*/
 	cpufunc_nullop,			/* l2cache_wbinv_all	*/
@@ -638,6 +646,7 @@ struct cpu_functions sa11x0_cpufuncs = {
 /*XXX*/	sa1_cache_purgeD_rng,		/* dcache_inv_range	*/
 	sa1_cache_cleanD_rng,		/* dcache_wb_range	*/
 
+	sa1_cache_flushID,		/* idcache_inv_all	*/
 	sa1_cache_purgeID,		/* idcache_wbinv_all	*/
 	sa1_cache_purgeID_rng,		/* idcache_wbinv_range	*/
 	cpufunc_nullop,			/* l2cache_wbinv_all	*/
@@ -699,6 +708,7 @@ struct cpu_functions ixp12x0_cpufuncs = {
 /*XXX*/	sa1_cache_purgeD_rng,		/* dcache_inv_range	*/
 	sa1_cache_cleanD_rng,		/* dcache_wb_range	*/
 
+	sa1_cache_flushID,		/* idcache_inv_all	*/
 	sa1_cache_purgeID,		/* idcache_wbinv_all	*/
 	sa1_cache_purgeID_rng,		/* idcache_wbinv_range	*/
 	cpufunc_nullop,			/* l2cache_wbinv_all	*/
@@ -763,6 +773,7 @@ struct cpu_functions xscale_cpufuncs = {
 	xscale_cache_flushD_rng,	/* dcache_inv_range	*/
 	xscale_cache_cleanD_rng,	/* dcache_wb_range	*/
 
+	xscale_cache_flushID,		/* idcache_inv_all	*/
 	xscale_cache_purgeID,		/* idcache_wbinv_all	*/
 	xscale_cache_purgeID_rng,	/* idcache_wbinv_range	*/
 	cpufunc_nullop,			/* l2cache_wbinv_all 	*/
@@ -826,6 +837,7 @@ struct cpu_functions xscalec3_cpufuncs = {
 	xscale_cache_flushD_rng,	/* dcache_inv_range	*/
 	xscalec3_cache_cleanD_rng,	/* dcache_wb_range	*/
 
+	xscale_cache_flushID,		/* idcache_inv_all	*/
 	xscalec3_cache_purgeID,		/* idcache_wbinv_all	*/
 	xscalec3_cache_purgeID_rng,	/* idcache_wbinv_range	*/
 	xscalec3_l2cache_purge,		/* l2cache_wbinv_all	*/
@@ -888,6 +900,7 @@ struct cpu_functions fa526_cpufuncs = {
 	fa526_dcache_inv_range,		/* dcache_inv_range	*/
 	fa526_dcache_wb_range,		/* dcache_wb_range	*/
 
+	armv4_idcache_inv_all,		/* idcache_inv_all	*/
 	fa526_idcache_wbinv_all,	/* idcache_wbinv_all	*/
 	fa526_idcache_wbinv_range,	/* idcache_wbinv_range	*/
 	cpufunc_nullop,			/* l2cache_wbinv_all	*/
@@ -949,6 +962,7 @@ struct cpu_functions arm1136_cpufuncs = {
 	armv6_dcache_inv_range,         /* dcache_inv_range     */
 	armv6_dcache_wb_range,          /* dcache_wb_range      */
 	
+	armv6_idcache_inv_all,		/* idcache_inv_all	*/
 	arm11x6_idcache_wbinv_all,      /* idcache_wbinv_all    */
 	arm11x6_idcache_wbinv_range,    /* idcache_wbinv_range  */
 	
@@ -1010,6 +1024,7 @@ struct cpu_functions arm1176_cpufuncs = {
 	armv6_dcache_inv_range,         /* dcache_inv_range     */
 	armv6_dcache_wb_range,          /* dcache_wb_range      */
 	
+	armv6_idcache_inv_all,		/* idcache_inv_all	*/
 	arm11x6_idcache_wbinv_all,      /* idcache_wbinv_all    */
 	arm11x6_idcache_wbinv_range,    /* idcache_wbinv_range  */
 	
@@ -1072,6 +1087,7 @@ struct cpu_functions cortexa_cpufuncs = {
 	armv7_dcache_inv_range,         /* dcache_inv_range     */
 	armv7_dcache_wb_range,          /* dcache_wb_range      */
 	
+	armv7_idcache_inv_all,		/* idcache_inv_all	*/
 	armv7_idcache_wbinv_all,        /* idcache_wbinv_all    */
 	armv7_idcache_wbinv_range,      /* idcache_wbinv_range  */
 	

sys/arm/arm/cpufunc_asm_armv4.S

@@ -71,3 +71,9 @@ ENTRY(armv4_drain_writebuf)
 	RET
 END(armv4_drain_writebuf)
 
+ENTRY(armv4_idcache_inv_all)
+	mov	r0, #0
+	mcr	p15, 0, r0, c7, c7, 0	/* invalidate all I+D cache */
+	RET
+END(armv4_drain_writebuf)
+

sys/arm/arm/cpufunc_asm_armv6.S

@@ -148,3 +148,9 @@ ENTRY(armv6_dcache_wbinv_all)
 END(armv6_idcache_wbinv_all)
 END(armv6_dcache_wbinv_all)
 
+ENTRY(armv6_idcache_inv_all)
+	mov	r0, #0
+	mcr	p15, 0, r0, c7, c7, 0	/* invalidate all I+D cache */
+	RET
+END(armv6_idcache_inv_all)
+

sys/arm/arm/cpufunc_asm_armv7.S

@@ -1,4 +1,5 @@
 /*-
+ * Copyright (c) 2010 Per Odlund <per.odlund@armagedon.se>
  * Copyright (C) 2011 MARVELL INTERNATIONAL LTD.
  * All rights reserved.
  *
@@ -305,3 +306,40 @@ ENTRY(armv7_auxctrl)
 	RET
 END(armv7_auxctrl)
 
+ENTRY(armv7_idcache_inv_all)
+	mov     r0, #0
+	mcr     p15, 2, r0, c0, c0, 0   @ set cache level to L1
+	mrc     p15, 1, r0, c0, c0, 0   @ read CCSIDR
+
+	ubfx    r2, r0, #13, #15        @ get num sets - 1 from CCSIDR
+	ubfx    r3, r0, #3, #10         @ get numways - 1 from CCSIDR
+	clz     r1, r3                  @ number of bits to MSB of way
+	lsl     r3, r3, r1              @ shift into position
+	mov     ip, #1                  @
+	lsl     ip, ip, r1              @ ip now contains the way decr
+
+	ubfx    r0, r0, #0, #3          @ get linesize from CCSIDR
+	add     r0, r0, #4              @ apply bias
+	lsl     r2, r2, r0              @ shift sets by log2(linesize)
+	add     r3, r3, r2              @ merge numsets - 1 with numways - 1
+	sub     ip, ip, r2              @ subtract numsets - 1 from way decr
+	mov     r1, #1
+	lsl     r1, r1, r0              @ r1 now contains the set decr
+	mov     r2, ip                  @ r2 now contains set way decr
+
+	/* r3 = ways/sets, r2 = way decr, r1 = set decr, r0 and ip are free */
+1:      mcr     p15, 0, r3, c7, c6, 2   @ invalidate line
+	movs    r0, r3                  @ get current way/set
+	beq     2f                      @ at 0 means we are done.
+	movs    r0, r0, lsl #10         @ clear way bits leaving only set bits
+	subne   r3, r3, r1              @ non-zero?, decrement set #
+	subeq   r3, r3, r2              @ zero?, decrement way # and restore set count
+	b       1b
+
+2:	dsb                             @ wait for stores to finish
+	mov     r0, #0                  @ and ...
+	mcr     p15, 0, r0, c7, c5, 0   @ invalidate instruction+branch cache
+	isb                             @ instruction sync barrier
+	bx      lr                      @ return
+END(armv7_l1cache_inv_all)
+

sys/arm/include/cpufunc.h

@@ -104,6 +104,12 @@ struct cpu_functions {
 	 *
 	 * There are some rules that must be followed:
 	 *
+	 *	ID-cache Invalidate All:
+	 *		Unlike other functions, this one must never write back.
+	 *		It is used to intialize the MMU when it is in an unknown
+	 *		state (such as when it may have lines tagged as valid
+	 *		that belong to a previous set of mappings).
+	 *                                          
 	 *	I-cache Synch (all or range):
 	 *		The goal is to synchronize the instruction stream,
 	 *		so you may beed to write-back dirty D-cache blocks
@@ -138,6 +144,7 @@ struct cpu_functions {
 	void	(*cf_dcache_inv_range)	(vm_offset_t, vm_size_t);
 	void	(*cf_dcache_wb_range)	(vm_offset_t, vm_size_t);
 
+	void	(*cf_idcache_inv_all)	(void);
 	void	(*cf_idcache_wbinv_all)	(void);
 	void	(*cf_idcache_wbinv_range) (vm_offset_t, vm_size_t);
 	void	(*cf_l2cache_wbinv_all) (void);
@@ -238,6 +245,7 @@ void tlb_broadcast(int);
 #define	cpu_dcache_inv_range(a, s) cpufuncs.cf_dcache_inv_range((a), (s))
 #define	cpu_dcache_wb_range(a, s) cpufuncs.cf_dcache_wb_range((a), (s))
 
+#define	cpu_idcache_inv_all()	cpufuncs.cf_idcache_inv_all()
 #define	cpu_idcache_wbinv_all()	cpufuncs.cf_idcache_wbinv_all()
 #define	cpu_idcache_wbinv_range(a, s) cpufuncs.cf_idcache_wbinv_range((a), (s))
 #define cpu_l2cache_wbinv_all()	cpufuncs.cf_l2cache_wbinv_all()
@@ -495,6 +503,7 @@ void	armv6_dcache_wbinv_range	(vm_offset_t, vm_size_t);
 void	armv6_dcache_inv_range		(vm_offset_t, vm_size_t);
 void	armv6_dcache_wb_range		(vm_offset_t, vm_size_t);
 
+void	armv6_idcache_inv_all		(void);
 void	armv6_idcache_wbinv_all		(void);
 void	armv6_idcache_wbinv_range	(vm_offset_t, vm_size_t);
 
@@ -503,6 +512,7 @@ void	armv7_tlb_flushID		(void);
 void	armv7_tlb_flushID_SE		(u_int);
 void	armv7_icache_sync_range		(vm_offset_t, vm_size_t);
 void	armv7_idcache_wbinv_range	(vm_offset_t, vm_size_t);
+void	armv7_idcache_inv_all		(void);
 void	armv7_dcache_wbinv_all		(void);
 void	armv7_idcache_wbinv_all		(void);
 void	armv7_dcache_wbinv_range	(vm_offset_t, vm_size_t);
@@ -587,6 +597,7 @@ void	armv4_tlb_flushD	(void);
 void	armv4_tlb_flushD_SE	(u_int va);
 
 void	armv4_drain_writebuf	(void);
+void	armv4_idcache_inv_all	(void);
 #endif
 
 #if defined(CPU_IXP12X0)