dc43978aa5
Stop using smp_ipi_mtx to protect global shootdown state, and move/multiply the global state into pcpu. Now each CPU can initiate shootdown IPI independently from other CPUs. Initiator enters critical section, then fills its local PCPU shootdown info (pc_smp_tlb_XXX), then clears scoreboard generation at location (cpu, my_cpuid) for each target cpu. After that IPI is sent to all targets which scan for zeroed scoreboard generation words. Upon finding such word the shootdown data is read from corresponding cpu' pcpu, and generation is set. Meantime initiator loops waiting for all zeroed generations in scoreboard to update. Initiator does not disable interrupts, which should allow non-invalidation IPIs from deadlocking, it only needs to disable preemption to pin itself to the instance of the pcpu smp_tlb data. The generation is set before the actual invalidation is performed in handler. It is safe because target CPU cannot return to userspace before handler finishes. In principle only NMI can preempt the handler, but NMI would see the kernel handler frame and not touch not-invalidated user page table. Handlers loop until they do not see zeroed scoreboard generations. This, together with hardware keeping one pending IPI in LAPIC IRR should prevent lost shootdowns. Notes. 1. The code does protect writes to LAPIC ICR with exclusion. I believe this is fine because we in fact do not send IPIs from interrupt handlers. More for !x2APIC mode where ICR access for write requires two registers write, we disable interrupts around it. If considered incorrect, I can add per-cpu spinlock around ipi_send(). 2. Scoreboard lines owned by given target CPU can be padded to the cache line, to reduce ping-pong. Reviewed by: markj (previous version) Discussed with: alc Tested by: pho Sponsored by: The FreeBSD Foundation MFC after: 3 weeks Differential revision: https://reviews.freebsd.org/D25510
113 lines
3.3 KiB
C
113 lines
3.3 KiB
C
/*-
|
|
* SPDX-License-Identifier: Beerware
|
|
*
|
|
* ----------------------------------------------------------------------------
|
|
* "THE BEER-WARE LICENSE" (Revision 42):
|
|
* <phk@FreeBSD.org> wrote this file. As long as you retain this notice you
|
|
* can do whatever you want with this stuff. If we meet some day, and you think
|
|
* this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
|
|
* ----------------------------------------------------------------------------
|
|
*
|
|
* $FreeBSD$
|
|
*
|
|
*/
|
|
|
|
#ifndef _X86_X86_SMP_H_
|
|
#define _X86_X86_SMP_H_
|
|
|
|
#include <sys/bus.h>
|
|
#include <machine/frame.h>
|
|
#include <machine/intr_machdep.h>
|
|
#include <x86/apicvar.h>
|
|
#include <machine/pcb.h>
|
|
|
|
struct pmap;
|
|
|
|
/* global data in mp_x86.c */
|
|
extern int mp_naps;
|
|
extern int boot_cpu_id;
|
|
extern struct pcb stoppcbs[];
|
|
extern int cpu_apic_ids[];
|
|
extern int bootAP;
|
|
extern void *dpcpu;
|
|
extern char *bootSTK;
|
|
extern void *bootstacks[];
|
|
extern unsigned int boot_address;
|
|
extern unsigned int bootMP_size;
|
|
extern volatile int aps_ready;
|
|
extern struct mtx ap_boot_mtx;
|
|
extern int cpu_logical;
|
|
extern int cpu_cores;
|
|
extern volatile uint32_t smp_tlb_generation;
|
|
extern struct pmap *smp_tlb_pmap;
|
|
extern vm_offset_t smp_tlb_addr1, smp_tlb_addr2;
|
|
extern u_int xhits_gbl[];
|
|
extern u_int xhits_pg[];
|
|
extern u_int xhits_rng[];
|
|
extern u_int ipi_global;
|
|
extern u_int ipi_page;
|
|
extern u_int ipi_range;
|
|
extern u_int ipi_range_size;
|
|
|
|
extern int nmi_kdb_lock;
|
|
extern int nmi_is_broadcast;
|
|
|
|
struct cpu_info {
|
|
int cpu_present:1;
|
|
int cpu_bsp:1;
|
|
int cpu_disabled:1;
|
|
int cpu_hyperthread:1;
|
|
};
|
|
extern struct cpu_info *cpu_info;
|
|
|
|
/*
|
|
* Set if MWAIT does not reliably wake when the MONITORed address is written.
|
|
*/
|
|
extern bool mwait_cpustop_broken;
|
|
|
|
#ifdef COUNT_IPIS
|
|
extern u_long *ipi_invltlb_counts[MAXCPU];
|
|
extern u_long *ipi_invlrng_counts[MAXCPU];
|
|
extern u_long *ipi_invlpg_counts[MAXCPU];
|
|
extern u_long *ipi_invlcache_counts[MAXCPU];
|
|
extern u_long *ipi_rendezvous_counts[MAXCPU];
|
|
#endif
|
|
|
|
/* IPI handlers */
|
|
inthand_t
|
|
IDTVEC(ipi_intr_bitmap_handler), /* Bitmap based IPIs */
|
|
IDTVEC(cpustop), /* CPU stops & waits to be restarted */
|
|
IDTVEC(cpususpend), /* CPU suspends & waits to be resumed */
|
|
IDTVEC(rendezvous); /* handle CPU rendezvous */
|
|
|
|
typedef void (*smp_invl_cb_t)(struct pmap *, vm_offset_t addr1,
|
|
vm_offset_t addr2);
|
|
|
|
/* functions in x86_mp.c */
|
|
void assign_cpu_ids(void);
|
|
void cpu_add(u_int apic_id, char boot_cpu);
|
|
void cpustop_handler(void);
|
|
void cpususpend_handler(void);
|
|
void alloc_ap_trampoline(vm_paddr_t *physmap, unsigned int *physmap_idx);
|
|
void init_secondary_tail(void);
|
|
void init_secondary(void);
|
|
void ipi_startup(int apic_id, int vector);
|
|
void ipi_all_but_self(u_int ipi);
|
|
void ipi_bitmap_handler(struct trapframe frame);
|
|
void ipi_cpu(int cpu, u_int ipi);
|
|
int ipi_nmi_handler(void);
|
|
void ipi_selected(cpuset_t cpus, u_int ipi);
|
|
void set_interrupt_apic_ids(void);
|
|
void smp_cache_flush(smp_invl_cb_t curcpu_cb);
|
|
void smp_masked_invlpg(cpuset_t mask, vm_offset_t addr, struct pmap *pmap,
|
|
smp_invl_cb_t curcpu_cb);
|
|
void smp_masked_invlpg_range(cpuset_t mask, vm_offset_t startva,
|
|
vm_offset_t endva, struct pmap *pmap, smp_invl_cb_t curcpu_cb);
|
|
void smp_masked_invltlb(cpuset_t mask, struct pmap *pmap,
|
|
smp_invl_cb_t curcpu_cb);
|
|
void mem_range_AP_init(void);
|
|
void topo_probe(void);
|
|
void ipi_send_cpu(int cpu, u_int ipi);
|
|
|
|
#endif
|