dc43978aa5
Stop using smp_ipi_mtx to protect global shootdown state, and move/multiply the global state into pcpu. Now each CPU can initiate shootdown IPI independently from other CPUs. Initiator enters critical section, then fills its local PCPU shootdown info (pc_smp_tlb_XXX), then clears scoreboard generation at location (cpu, my_cpuid) for each target cpu. After that IPI is sent to all targets which scan for zeroed scoreboard generation words. Upon finding such word the shootdown data is read from corresponding cpu' pcpu, and generation is set. Meantime initiator loops waiting for all zeroed generations in scoreboard to update. Initiator does not disable interrupts, which should allow non-invalidation IPIs from deadlocking, it only needs to disable preemption to pin itself to the instance of the pcpu smp_tlb data. The generation is set before the actual invalidation is performed in handler. It is safe because target CPU cannot return to userspace before handler finishes. In principle only NMI can preempt the handler, but NMI would see the kernel handler frame and not touch not-invalidated user page table. Handlers loop until they do not see zeroed scoreboard generations. This, together with hardware keeping one pending IPI in LAPIC IRR should prevent lost shootdowns. Notes. 1. The code does protect writes to LAPIC ICR with exclusion. I believe this is fine because we in fact do not send IPIs from interrupt handlers. More for !x2APIC mode where ICR access for write requires two registers write, we disable interrupts around it. If considered incorrect, I can add per-cpu spinlock around ipi_send(). 2. Scoreboard lines owned by given target CPU can be padded to the cache line, to reduce ping-pong. Reviewed by: markj (previous version) Discussed with: alc Tested by: pho Sponsored by: The FreeBSD Foundation MFC after: 3 weeks Differential revision: https://reviews.freebsd.org/D25510
49 lines
1.2 KiB
C
49 lines
1.2 KiB
C
/*-
|
|
* ----------------------------------------------------------------------------
|
|
* "THE BEER-WARE LICENSE" (Revision 42):
|
|
* <phk@FreeBSD.org> wrote this file. As long as you retain this notice you
|
|
* can do whatever you want with this stuff. If we meet some day, and you think
|
|
* this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
|
|
* ----------------------------------------------------------------------------
|
|
*
|
|
* $FreeBSD$
|
|
*
|
|
*/
|
|
|
|
#ifndef _MACHINE_SMP_H_
|
|
#define _MACHINE_SMP_H_
|
|
|
|
#ifdef _KERNEL
|
|
|
|
#ifdef SMP
|
|
|
|
#ifndef LOCORE
|
|
|
|
#include <x86/x86_smp.h>
|
|
|
|
#include <sys/bus.h>
|
|
#include <machine/frame.h>
|
|
#include <machine/intr_machdep.h>
|
|
#include <x86/apicvar.h>
|
|
#include <machine/pcb.h>
|
|
|
|
inthand_t
|
|
IDTVEC(invltlb), /* TLB shootdowns - global */
|
|
IDTVEC(invlpg), /* TLB shootdowns - 1 page */
|
|
IDTVEC(invlrng), /* TLB shootdowns - page range */
|
|
IDTVEC(invlcache); /* Write back and invalidate cache */
|
|
|
|
/* functions in mpboot.s */
|
|
void bootMP(void);
|
|
|
|
void invltlb_handler(void);
|
|
void invlpg_handler(void);
|
|
void invlrng_handler(void);
|
|
void invlcache_handler(void);
|
|
|
|
#endif /* !LOCORE */
|
|
#endif /* SMP */
|
|
|
|
#endif /* _KERNEL */
|
|
#endif /* _MACHINE_SMP_H_ */
|