d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cpuset(9): Add CPU_FOREACH_IS(SET|CLR) and modify consumers to use it

Browse Source 
This implementation is faster and doesn't modify the cpuset, so it lets
us avoid some unnecessary copying as well.  No functional change
intended.

This is a re-application of commit
9068f6ea69.

Reviewed by:	cem, kib, jhb
MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D32029
This commit is contained in:
Mark Johnston 2021-09-21 11:36:55 -04:00
parent 51425cb210
commit de8554295b
6 changed files with 11 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
sys/amd64/amd64/mp_machdep.c
View File

@ -618,7 +618,7 @@ static void
smp_targeted_tlb_shootdown(cpuset_t mask, pmap_t pmap, vm_offset_t addr1, smp_targeted_tlb_shootdown(cpuset_t mask, pmap_t pmap, vm_offset_t addr1,
vm_offset_t addr2, smp_invl_cb_t curcpu_cb, enum invl_op_codes op) vm_offset_t addr2, smp_invl_cb_t curcpu_cb, enum invl_op_codes op)
{ {
cpuset_t other_cpus, mask1; cpuset_t other_cpus;
uint32_t generation, *p_cpudone; uint32_t generation, *p_cpudone;
int cpu; int cpu;
bool is_all; bool is_all;
@ -662,10 +662,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask, pmap_t pmap, vm_offset_t addr1,
/* Fence between filling smp_tlb fields and clearing scoreboard. */ /* Fence between filling smp_tlb fields and clearing scoreboard. */
atomic_thread_fence_rel(); atomic_thread_fence_rel();
mask1 = mask; CPU_FOREACH_ISSET(cpu, &mask) {
while ((cpu = CPU_FFS(&mask1)) != 0) {
cpu--;
CPU_CLR(cpu, &mask1);
KASSERT(*invl_scoreboard_slot(cpu) != 0, KASSERT(*invl_scoreboard_slot(cpu) != 0,
("IPI scoreboard is zero, initiator %d target %d", ("IPI scoreboard is zero, initiator %d target %d",
PCPU_GET(cpuid), cpu)); PCPU_GET(cpuid), cpu));
@ -686,9 +683,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask, pmap_t pmap, vm_offset_t addr1,
ipi_selected(mask, IPI_INVLOP); ipi_selected(mask, IPI_INVLOP);
} }
curcpu_cb(pmap, addr1, addr2); curcpu_cb(pmap, addr1, addr2);
while ((cpu = CPU_FFS(&other_cpus)) != 0) { CPU_FOREACH_ISSET(cpu, &other_cpus) {
cpu--;
CPU_CLR(cpu, &other_cpus);
p_cpudone = invl_scoreboard_slot(cpu); p_cpudone = invl_scoreboard_slot(cpu);
while (atomic_load_int(p_cpudone) != generation) while (atomic_load_int(p_cpudone) != generation)
ia32_pause(); ia32_pause();

13
sys/amd64/vmm/io/vlapic.c
View File

@ -860,10 +860,7 @@ vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
*/ */
CPU_ZERO(dmask); CPU_ZERO(dmask);
amask = vm_active_cpus(vm); amask = vm_active_cpus(vm);
while ((vcpuid = CPU_FFS(&amask)) != 0) { CPU_FOREACH_ISSET(vcpuid, &amask) {
vcpuid--;
CPU_CLR(vcpuid, &amask);
vlapic = vm_lapic(vm, vcpuid); vlapic = vm_lapic(vm, vcpuid);
dfr = vlapic->apic_page->dfr; dfr = vlapic->apic_page->dfr;
ldr = vlapic->apic_page->ldr; ldr = vlapic->apic_page->ldr;
@ -1003,9 +1000,7 @@ vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
break; break;
} }
while ((i = CPU_FFS(&dmask)) != 0) { CPU_FOREACH_ISSET(i, &dmask) {
i--;
CPU_CLR(i, &dmask);
if (mode == APIC_DELMODE_FIXED) { if (mode == APIC_DELMODE_FIXED) {
lapic_intr_edge(vlapic->vm, i, vec); lapic_intr_edge(vlapic->vm, i, vec);
vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
@ -1554,9 +1549,7 @@ vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
*/ */
vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false); vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
while ((vcpuid = CPU_FFS(&dmask)) != 0) { CPU_FOREACH_ISSET(vcpuid, &dmask) {
vcpuid--;
CPU_CLR(vcpuid, &dmask);
if (delmode == IOART_DELEXINT) { if (delmode == IOART_DELEXINT) {
vm_inject_extint(vm, vcpuid); vm_inject_extint(vm, vcpuid);
} else { } else {

4
sys/amd64/vmm/vmm_lapic.c
View File

@ -87,9 +87,7 @@ lapic_set_local_intr(struct vm *vm, int cpu, int vector)
else else
CPU_SETOF(cpu, &dmask); CPU_SETOF(cpu, &dmask);
error = 0; error = 0;
while ((cpu = CPU_FFS(&dmask)) != 0) { CPU_FOREACH_ISSET(cpu, &dmask) {
cpu--;
CPU_CLR(cpu, &dmask);
vlapic = vm_lapic(vm, cpu); vlapic = vm_lapic(vm, cpu);
error = vlapic_trigger_lvt(vlapic, vector); error = vlapic_trigger_lvt(vlapic, vector);
if (error) if (error)

4
sys/i386/i386/mp_machdep.c
View File

@ -598,9 +598,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap,
ipi_selected(mask, vector); ipi_selected(mask, vector);
} }
curcpu_cb(pmap, addr1, addr2); curcpu_cb(pmap, addr1, addr2);
while ((cpu = CPU_FFS(&other_cpus)) != 0) { CPU_FOREACH_ISSET(cpu, &other_cpus) {
cpu--;
CPU_CLR(cpu, &other_cpus);
p_cpudone = &cpuid_to_pcpu[cpu]->pc_smp_tlb_done; p_cpudone = &cpuid_to_pcpu[cpu]->pc_smp_tlb_done;
while (*p_cpudone != generation) while (*p_cpudone != generation)
ia32_pause(); ia32_pause();

2
sys/sys/cpuset.h
View File

@ -66,6 +66,8 @@
#define CPU_COPY_STORE_REL(f, t) BIT_COPY_STORE_REL(CPU_SETSIZE, f, t) #define CPU_COPY_STORE_REL(f, t) BIT_COPY_STORE_REL(CPU_SETSIZE, f, t)
#define CPU_FFS(p) BIT_FFS(CPU_SETSIZE, p) #define CPU_FFS(p) BIT_FFS(CPU_SETSIZE, p)
#define CPU_FLS(p) BIT_FLS(CPU_SETSIZE, p) #define CPU_FLS(p) BIT_FLS(CPU_SETSIZE, p)
#define CPU_FOREACH_ISSET(i, p) BIT_FOREACH_ISSET(CPU_SETSIZE, i, p)
#define CPU_FOREACH_ISCLR(i, p) BIT_FOREACH_ISCLR(CPU_SETSIZE, i, p)
#define CPU_COUNT(p) ((int)BIT_COUNT(CPU_SETSIZE, p)) #define CPU_COUNT(p) ((int)BIT_COUNT(CPU_SETSIZE, p))
#define CPUSET_FSET BITSET_FSET(_NCPUWORDS) #define CPUSET_FSET BITSET_FSET(_NCPUWORDS)
#define CPUSET_T_INITIALIZER BITSET_T_INITIALIZER #define CPUSET_T_INITIALIZER BITSET_T_INITIALIZER

4
sys/x86/x86/mp_x86.c
View File

@ -1346,9 +1346,7 @@ ipi_selected(cpuset_t cpus, u_int ipi)
if (ipi == IPI_STOP_HARD) if (ipi == IPI_STOP_HARD)
CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &cpus); CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &cpus);
while ((cpu = CPU_FFS(&cpus)) != 0) { CPU_FOREACH_ISSET(cpu, &cpus) {
cpu--;
CPU_CLR(cpu, &cpus);
CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi); CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
ipi_send_cpu(cpu, ipi); ipi_send_cpu(cpu, ipi);
} }