x86: store pending bitmapped IPIs in per-cpu areas
This gets rid of the global cpu_ipi_pending array. While replace cmpset with fcmpset in the delivery code and opportunistically check if given IPI is already pending. Sponsored by: The FreeBSD Foundation
This commit is contained in:
parent
8eae2be460
commit
a8c2fcb287
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=347502
@ -193,7 +193,6 @@ cpu_mp_start(void)
|
||||
/* Initialize the logical ID to APIC ID table. */
|
||||
for (i = 0; i < MAXCPU; i++) {
|
||||
cpu_apic_ids[i] = -1;
|
||||
cpu_ipi_pending[i] = 0;
|
||||
}
|
||||
|
||||
/* Install an inter-CPU IPI for TLB invalidation */
|
||||
|
@ -84,7 +84,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
|
||||
uint32_t pc_pcid_gen; \
|
||||
uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \
|
||||
uint32_t pc_ibpb_set; \
|
||||
char __pad[3288] /* pad to UMA_PCPU_ALLOC_SIZE */
|
||||
u_int pc_ipi_bitmap; \
|
||||
char __pad[3284] /* pad to UMA_PCPU_ALLOC_SIZE */
|
||||
|
||||
#define PC_DBREG_CMD_NONE 0
|
||||
#define PC_DBREG_CMD_LOAD 1
|
||||
|
@ -153,7 +153,6 @@ cpu_mp_start(void)
|
||||
/* Initialize the logical ID to APIC ID table. */
|
||||
for (i = 0; i < MAXCPU; i++) {
|
||||
cpu_apic_ids[i] = -1;
|
||||
cpu_ipi_pending[i] = 0;
|
||||
}
|
||||
|
||||
/* Install an inter-CPU IPI for TLB invalidation */
|
||||
|
@ -87,7 +87,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
|
||||
caddr_t pc_pmap_eh_ptep; \
|
||||
uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \
|
||||
uint32_t pc_ibpb_set; \
|
||||
char __pad[3610]
|
||||
u_int pc_ipi_bitmap; \
|
||||
char __pad[3606]
|
||||
|
||||
#ifdef _KERNEL
|
||||
|
||||
|
@ -34,7 +34,6 @@ extern char *bootSTK;
|
||||
extern void *bootstacks[];
|
||||
extern unsigned int boot_address;
|
||||
extern unsigned int bootMP_size;
|
||||
extern volatile u_int cpu_ipi_pending[];
|
||||
extern volatile int aps_ready;
|
||||
extern struct mtx ap_boot_mtx;
|
||||
extern int cpu_logical;
|
||||
|
@ -137,9 +137,6 @@ _Static_assert(MAXCPU <= MAX_APIC_ID,
|
||||
_Static_assert(xAPIC_MAX_APIC_ID <= MAX_APIC_ID,
|
||||
"xAPIC_MAX_APIC_ID cannot be larger that MAX_APIC_ID");
|
||||
|
||||
/* Holds pending bitmap based IPIs per CPU */
|
||||
volatile u_int cpu_ipi_pending[MAXCPU];
|
||||
|
||||
static void release_aps(void *dummy);
|
||||
static void cpustop_handler_post(u_int cpu);
|
||||
|
||||
@ -1224,19 +1221,24 @@ ipi_startup(int apic_id, int vector)
|
||||
void
|
||||
ipi_send_cpu(int cpu, u_int ipi)
|
||||
{
|
||||
u_int bitmap, old_pending, new_pending;
|
||||
u_int bitmap, old, new;
|
||||
u_int *cpu_bitmap;
|
||||
|
||||
KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
|
||||
|
||||
if (IPI_IS_BITMAPED(ipi)) {
|
||||
bitmap = 1 << ipi;
|
||||
ipi = IPI_BITMAP_VECTOR;
|
||||
do {
|
||||
old_pending = cpu_ipi_pending[cpu];
|
||||
new_pending = old_pending | bitmap;
|
||||
} while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
|
||||
old_pending, new_pending));
|
||||
if (old_pending)
|
||||
cpu_bitmap = &cpuid_to_pcpu[cpu]->pc_ipi_bitmap;
|
||||
old = *cpu_bitmap;
|
||||
for (;;) {
|
||||
if ((old & bitmap) == bitmap)
|
||||
break;
|
||||
new = old | bitmap;
|
||||
if (atomic_fcmpset_int(cpu_bitmap, &old, new))
|
||||
break;
|
||||
}
|
||||
if (old)
|
||||
return;
|
||||
}
|
||||
lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
|
||||
@ -1255,7 +1257,7 @@ ipi_bitmap_handler(struct trapframe frame)
|
||||
td->td_intr_nesting_level++;
|
||||
oldframe = td->td_intr_frame;
|
||||
td->td_intr_frame = &frame;
|
||||
ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
|
||||
ipi_bitmap = atomic_readandclear_int(&cpuid_to_pcpu[cpu]->pc_ipi_bitmap);
|
||||
if (ipi_bitmap & (1 << IPI_PREEMPT)) {
|
||||
#ifdef COUNT_IPIS
|
||||
(*ipi_preempt_counts[cpu])++;
|
||||
|
Loading…
Reference in New Issue
Block a user