d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

x86: store pending bitmapped IPIs in per-cpu areas

Browse Source 
This gets rid of the global cpu_ipi_pending array.

While replace cmpset with fcmpset in the delivery code and opportunistically
check if given IPI is already pending.

Sponsored by:	The FreeBSD Foundation
This commit is contained in:
Mateusz Guzik 2019-05-12 06:36:54 +00:00
parent 8eae2be460
commit a8c2fcb287
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=347502
6 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sys/amd64/amd64/mp_machdep.c
View File

@ -193,7 +193,6 @@ cpu_mp_start(void)
/* Initialize the logical ID to APIC ID table. */
for (i = 0; i < MAXCPU; i++) {
cpu_apic_ids[i] = -1;
cpu_ipi_pending[i] = 0;
}
/* Install an inter-CPU IPI for TLB invalidation */

3
sys/amd64/include/pcpu.h
View File

@ -84,7 +84,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
uint32_t pc_pcid_gen; \
uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \
uint32_t pc_ibpb_set; \
char __pad[3288] /* pad to UMA_PCPU_ALLOC_SIZE */
u_int pc_ipi_bitmap; \
char __pad[3284] /* pad to UMA_PCPU_ALLOC_SIZE */
#define PC_DBREG_CMD_NONE 0
#define PC_DBREG_CMD_LOAD 1

1
sys/i386/i386/mp_machdep.c
View File

@ -153,7 +153,6 @@ cpu_mp_start(void)
/* Initialize the logical ID to APIC ID table. */
for (i = 0; i < MAXCPU; i++) {
cpu_apic_ids[i] = -1;
cpu_ipi_pending[i] = 0;
}
/* Install an inter-CPU IPI for TLB invalidation */

3
sys/i386/include/pcpu.h
View File

@ -87,7 +87,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
caddr_t pc_pmap_eh_ptep; \
uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \
uint32_t pc_ibpb_set; \
char __pad[3610]
u_int pc_ipi_bitmap; \
char __pad[3606]
#ifdef _KERNEL

1
sys/x86/include/x86_smp.h
View File

@ -34,7 +34,6 @@ extern char *bootSTK;
extern void *bootstacks[];
extern unsigned int boot_address;
extern unsigned int bootMP_size;
extern volatile u_int cpu_ipi_pending[];
extern volatile int aps_ready;
extern struct mtx ap_boot_mtx;
extern int cpu_logical;

24
sys/x86/x86/mp_x86.c
View File

@ -137,9 +137,6 @@ _Static_assert(MAXCPU <= MAX_APIC_ID,
_Static_assert(xAPIC_MAX_APIC_ID <= MAX_APIC_ID,
"xAPIC_MAX_APIC_ID cannot be larger that MAX_APIC_ID");
/* Holds pending bitmap based IPIs per CPU */
volatile u_int cpu_ipi_pending[MAXCPU];
static void release_aps(void *dummy);
static void cpustop_handler_post(u_int cpu);
@ -1224,19 +1221,24 @@ ipi_startup(int apic_id, int vector)
void
ipi_send_cpu(int cpu, u_int ipi)
{
u_int bitmap, old_pending, new_pending;
u_int bitmap, old, new;
u_int *cpu_bitmap;
KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
if (IPI_IS_BITMAPED(ipi)) {
bitmap = 1 << ipi;
ipi = IPI_BITMAP_VECTOR;
do {
old_pending = cpu_ipi_pending[cpu];
new_pending = old_pending | bitmap;
} while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
old_pending, new_pending));
if (old_pending)
cpu_bitmap = &cpuid_to_pcpu[cpu]->pc_ipi_bitmap;
old = *cpu_bitmap;
for (;;) {
if ((old & bitmap) == bitmap)
break;
new = old | bitmap;
if (atomic_fcmpset_int(cpu_bitmap, &old, new))
break;
}
if (old)
return;
}
lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
@ -1255,7 +1257,7 @@ ipi_bitmap_handler(struct trapframe frame)
td->td_intr_nesting_level++;
oldframe = td->td_intr_frame;
td->td_intr_frame = &frame;
ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
ipi_bitmap = atomic_readandclear_int(&cpuid_to_pcpu[cpu]->pc_ipi_bitmap);
if (ipi_bitmap & (1 << IPI_PREEMPT)) {
#ifdef COUNT_IPIS
(*ipi_preempt_counts[cpu])++;