2005-01-06 23:35:40 +00:00
|
|
|
/*-
|
2017-11-27 15:20:12 +00:00
|
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
|
|
*
|
2008-11-03 21:17:02 +00:00
|
|
|
* Copyright (c) 2001, John Baldwin <jhb@FreeBSD.org>.
|
1997-04-26 11:46:25 +00:00
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
2001-04-27 19:28:25 +00:00
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
1997-04-26 11:46:25 +00:00
|
|
|
*
|
2008-11-03 21:17:02 +00:00
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
1997-04-26 11:46:25 +00:00
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
2008-11-03 21:17:02 +00:00
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
1997-04-26 11:46:25 +00:00
|
|
|
*/
|
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
/*
|
|
|
|
* This module holds the global variables and machine independent functions
|
2001-05-10 17:45:49 +00:00
|
|
|
* used for the kernel SMP support.
|
2001-04-27 19:28:25 +00:00
|
|
|
*/
|
1997-12-08 23:00:24 +00:00
|
|
|
|
2003-06-11 00:56:59 +00:00
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
1997-08-25 21:28:08 +00:00
|
|
|
#include <sys/param.h>
|
1997-04-26 11:46:25 +00:00
|
|
|
#include <sys/systm.h>
|
2001-05-10 17:45:49 +00:00
|
|
|
#include <sys/kernel.h>
|
2001-04-27 19:28:25 +00:00
|
|
|
#include <sys/ktr.h>
|
1997-08-26 18:10:38 +00:00
|
|
|
#include <sys/proc.h>
|
2004-03-09 03:37:21 +00:00
|
|
|
#include <sys/bus.h>
|
2001-04-27 19:28:25 +00:00
|
|
|
#include <sys/lock.h>
|
2016-04-04 16:09:29 +00:00
|
|
|
#include <sys/malloc.h>
|
2000-10-20 07:58:15 +00:00
|
|
|
#include <sys/mutex.h>
|
2001-05-10 17:45:49 +00:00
|
|
|
#include <sys/pcpu.h>
|
2012-11-15 00:51:57 +00:00
|
|
|
#include <sys/sched.h>
|
2001-04-27 19:28:25 +00:00
|
|
|
#include <sys/smp.h>
|
|
|
|
#include <sys/sysctl.h>
|
1997-04-26 11:46:25 +00:00
|
|
|
|
2007-07-03 18:37:06 +00:00
|
|
|
#include <machine/cpu.h>
|
2002-03-07 04:43:51 +00:00
|
|
|
#include <machine/smp.h>
|
|
|
|
|
2004-09-01 06:42:02 +00:00
|
|
|
#include "opt_sched.h"
|
|
|
|
|
2016-04-04 16:09:29 +00:00
|
|
|
#ifdef SMP
|
|
|
|
MALLOC_DEFINE(M_TOPO, "toponodes", "SMP topology data");
|
|
|
|
|
2011-05-18 15:50:12 +00:00
|
|
|
volatile cpuset_t stopped_cpus;
|
|
|
|
volatile cpuset_t started_cpus;
|
2012-06-09 00:37:26 +00:00
|
|
|
volatile cpuset_t suspended_cpus;
|
2011-05-18 15:50:12 +00:00
|
|
|
cpuset_t hlt_cpus_mask;
|
|
|
|
cpuset_t logical_cpus_mask;
|
1997-04-26 11:46:25 +00:00
|
|
|
|
2002-03-19 21:25:46 +00:00
|
|
|
void (*cpustop_restartfunc)(void);
|
2003-12-03 14:55:31 +00:00
|
|
|
#endif
|
2014-04-26 20:27:54 +00:00
|
|
|
|
|
|
|
static int sysctl_kern_smp_active(SYSCTL_HANDLER_ARGS);
|
|
|
|
|
2004-09-03 07:42:31 +00:00
|
|
|
/* This is used in modules that need to work in both SMP and UP. */
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
cpuset_t all_cpus;
|
2003-12-03 14:55:31 +00:00
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
int mp_ncpus;
|
2003-12-23 13:54:16 +00:00
|
|
|
/* export this for libkvm consumers. */
|
|
|
|
int mp_maxcpus = MAXCPU;
|
1997-04-26 11:46:25 +00:00
|
|
|
|
2001-10-31 20:43:38 +00:00
|
|
|
volatile int smp_started;
|
2002-03-05 10:01:46 +00:00
|
|
|
u_int mp_maxid;
|
1997-04-26 11:46:25 +00:00
|
|
|
|
2020-02-26 14:26:36 +00:00
|
|
|
static SYSCTL_NODE(_kern, OID_AUTO, smp,
|
|
|
|
CTLFLAG_RD | CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL,
|
2011-11-07 15:43:11 +00:00
|
|
|
"Kernel SMP");
|
1997-04-26 11:46:25 +00:00
|
|
|
|
2011-07-17 23:05:24 +00:00
|
|
|
SYSCTL_INT(_kern_smp, OID_AUTO, maxid, CTLFLAG_RD|CTLFLAG_CAPRD, &mp_maxid, 0,
|
2008-05-23 04:05:26 +00:00
|
|
|
"Max CPU ID.");
|
|
|
|
|
2011-07-17 23:05:24 +00:00
|
|
|
SYSCTL_INT(_kern_smp, OID_AUTO, maxcpus, CTLFLAG_RD|CTLFLAG_CAPRD, &mp_maxcpus,
|
|
|
|
0, "Max number of CPUs that the system was compiled for.");
|
2003-12-23 13:54:16 +00:00
|
|
|
|
2017-10-18 22:00:44 +00:00
|
|
|
SYSCTL_PROC(_kern_smp, OID_AUTO, active, CTLFLAG_RD|CTLTYPE_INT|CTLFLAG_MPSAFE,
|
|
|
|
NULL, 0, sysctl_kern_smp_active, "I",
|
|
|
|
"Indicates system is running in SMP mode");
|
1997-04-26 11:46:25 +00:00
|
|
|
|
2002-12-28 23:21:13 +00:00
|
|
|
int smp_disabled = 0; /* has smp been disabled? */
|
2011-07-17 23:05:24 +00:00
|
|
|
SYSCTL_INT(_kern_smp, OID_AUTO, disabled, CTLFLAG_RDTUN|CTLFLAG_CAPRD,
|
|
|
|
&smp_disabled, 0, "SMP has been disabled from the loader");
|
2002-12-28 23:21:13 +00:00
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
int smp_cpus = 1; /* how many cpu's running */
|
2011-07-17 23:05:24 +00:00
|
|
|
SYSCTL_INT(_kern_smp, OID_AUTO, cpus, CTLFLAG_RD|CTLFLAG_CAPRD, &smp_cpus, 0,
|
2003-06-12 19:46:51 +00:00
|
|
|
"Number of CPUs online");
|
1997-04-26 11:46:25 +00:00
|
|
|
|
2019-01-04 18:31:17 +00:00
|
|
|
int smp_threads_per_core = 1; /* how many SMT threads are running per core */
|
|
|
|
SYSCTL_INT(_kern_smp, OID_AUTO, threads_per_core, CTLFLAG_RD|CTLFLAG_CAPRD,
|
|
|
|
&smp_threads_per_core, 0, "Number of SMT threads online per core");
|
|
|
|
|
|
|
|
int mp_ncores = -1; /* how many physical cores running */
|
|
|
|
SYSCTL_INT(_kern_smp, OID_AUTO, cores, CTLFLAG_RD|CTLFLAG_CAPRD, &mp_ncores, 0,
|
2020-08-03 17:17:17 +00:00
|
|
|
"Number of physical cores online");
|
2019-01-04 18:31:17 +00:00
|
|
|
|
2008-03-02 07:58:42 +00:00
|
|
|
int smp_topology = 0; /* Which topology we're using. */
|
2014-06-28 03:56:17 +00:00
|
|
|
SYSCTL_INT(_kern_smp, OID_AUTO, topology, CTLFLAG_RDTUN, &smp_topology, 0,
|
2008-03-02 07:58:42 +00:00
|
|
|
"Topology override setting; 0 is default provided by hardware.");
|
|
|
|
|
2003-12-03 14:55:31 +00:00
|
|
|
#ifdef SMP
|
2001-04-27 19:28:25 +00:00
|
|
|
/* Enable forwarding of a signal to a process running on a different CPU */
|
|
|
|
static int forward_signal_enabled = 1;
|
|
|
|
SYSCTL_INT(_kern_smp, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
|
2003-06-12 19:46:51 +00:00
|
|
|
&forward_signal_enabled, 0,
|
|
|
|
"Forwarding of a signal to a process on a different CPU");
|
1997-04-26 11:46:25 +00:00
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
/* Variables needed for SMP rendezvous. */
|
2008-08-27 18:23:55 +00:00
|
|
|
static volatile int smp_rv_ncpus;
|
2007-11-08 14:47:55 +00:00
|
|
|
static void (*volatile smp_rv_setup_func)(void *arg);
|
|
|
|
static void (*volatile smp_rv_action_func)(void *arg);
|
2008-01-02 17:09:15 +00:00
|
|
|
static void (*volatile smp_rv_teardown_func)(void *arg);
|
2009-01-26 15:32:39 +00:00
|
|
|
static void *volatile smp_rv_func_arg;
|
2011-07-30 20:29:39 +00:00
|
|
|
static volatile int smp_rv_waiters[4];
|
2004-08-23 21:39:29 +00:00
|
|
|
|
2023-01-29 13:18:17 +00:00
|
|
|
/*
|
2004-08-23 21:39:29 +00:00
|
|
|
* Shared mutex to restrict busywaits between smp_rendezvous() and
|
|
|
|
* smp(_targeted)_tlb_shootdown(). A deadlock occurs if both of these
|
|
|
|
* functions trigger at once and cause multiple CPUs to busywait with
|
2023-01-29 13:18:17 +00:00
|
|
|
* interrupts disabled.
|
2004-08-23 21:39:29 +00:00
|
|
|
*/
|
2004-08-28 00:49:55 +00:00
|
|
|
struct mtx smp_ipi_mtx;
|
1997-04-26 11:46:25 +00:00
|
|
|
|
|
|
|
/*
|
2003-11-21 22:23:26 +00:00
|
|
|
* Let the MD SMP code initialize mp_maxid very early if it can.
|
2002-03-05 10:01:46 +00:00
|
|
|
*/
|
|
|
|
static void
|
2003-11-21 22:23:26 +00:00
|
|
|
mp_setmaxid(void *dummy)
|
2002-03-05 10:01:46 +00:00
|
|
|
{
|
2015-11-08 14:26:50 +00:00
|
|
|
|
2003-11-21 22:23:26 +00:00
|
|
|
cpu_mp_setmaxid();
|
2015-11-08 14:26:50 +00:00
|
|
|
|
|
|
|
KASSERT(mp_ncpus >= 1, ("%s: CPU count < 1", __func__));
|
|
|
|
KASSERT(mp_ncpus > 1 || mp_maxid == 0,
|
|
|
|
("%s: one CPU but mp_maxid is not zero", __func__));
|
|
|
|
KASSERT(mp_maxid >= mp_ncpus - 1,
|
|
|
|
("%s: counters out of sync: max %d, count %d", __func__,
|
|
|
|
mp_maxid, mp_ncpus));
|
2023-01-29 12:35:18 +00:00
|
|
|
|
|
|
|
cpusetsizemin = howmany(mp_maxid + 1, NBBY);
|
2002-03-05 10:01:46 +00:00
|
|
|
}
|
2008-03-16 10:58:09 +00:00
|
|
|
SYSINIT(cpu_mp_setmaxid, SI_SUB_TUNABLES, SI_ORDER_FIRST, mp_setmaxid, NULL);
|
2002-03-05 10:01:46 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Call the MD SMP initialization code.
|
1997-04-26 11:46:25 +00:00
|
|
|
*/
|
2001-04-27 19:28:25 +00:00
|
|
|
static void
|
|
|
|
mp_start(void *dummy)
|
1997-04-26 11:46:25 +00:00
|
|
|
{
|
1997-04-28 00:25:00 +00:00
|
|
|
|
2010-05-11 15:36:16 +00:00
|
|
|
mtx_init(&smp_ipi_mtx, "smp rendezvous", NULL, MTX_SPIN);
|
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
/* Probe for MP hardware. */
|
2003-11-21 22:23:26 +00:00
|
|
|
if (smp_disabled != 0 || cpu_mp_probe() == 0) {
|
2019-01-04 18:31:17 +00:00
|
|
|
mp_ncores = 1;
|
2003-10-30 21:44:01 +00:00
|
|
|
mp_ncpus = 1;
|
2011-06-13 13:28:31 +00:00
|
|
|
CPU_SETOF(PCPU_GET(cpuid), &all_cpus);
|
2001-04-27 19:28:25 +00:00
|
|
|
return;
|
2003-10-30 21:44:01 +00:00
|
|
|
}
|
1997-04-26 11:46:25 +00:00
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
cpu_mp_start();
|
|
|
|
printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n",
|
|
|
|
mp_ncpus);
|
2019-01-04 18:31:17 +00:00
|
|
|
|
|
|
|
/* Provide a default for most architectures that don't have SMT/HTT. */
|
|
|
|
if (mp_ncores < 0)
|
|
|
|
mp_ncores = mp_ncpus;
|
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
cpu_mp_announce();
|
1997-04-26 11:46:25 +00:00
|
|
|
}
|
2008-03-16 10:58:09 +00:00
|
|
|
SYSINIT(cpu_mp, SI_SUB_CPU, SI_ORDER_THIRD, mp_start, NULL);
|
1997-06-27 23:33:17 +00:00
|
|
|
|
1997-12-08 23:00:24 +00:00
|
|
|
void
|
2001-09-12 08:38:13 +00:00
|
|
|
forward_signal(struct thread *td)
|
1998-03-03 20:55:26 +00:00
|
|
|
{
|
|
|
|
int id;
|
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
/*
|
2022-07-18 16:39:17 +00:00
|
|
|
* signotify() has already set TDA_AST and TDA_SIG on td_ast for
|
2003-03-31 22:49:17 +00:00
|
|
|
* this thread, so all we need to do is poke it if it is currently
|
2002-04-05 10:00:37 +00:00
|
|
|
* executing so that it executes ast().
|
1998-03-03 20:55:26 +00:00
|
|
|
*/
|
Commit 14/14 of sched_lock decomposition.
- Use thread_lock() rather than sched_lock for per-thread scheduling
sychronization.
- Use the per-process spinlock rather than the sched_lock for per-process
scheduling synchronization.
Tested by: kris, current@
Tested on: i386, amd64, ULE, 4BSD, libthr, libkse, PREEMPTION, etc.
Discussed with: kris, attilio, kmacy, jhb, julian, bde (small parts each)
2007-06-05 00:00:57 +00:00
|
|
|
THREAD_LOCK_ASSERT(td, MA_OWNED);
|
2002-09-11 08:13:56 +00:00
|
|
|
KASSERT(TD_IS_RUNNING(td),
|
Part 1 of KSE-III
The ability to schedule multiple threads per process
(one one cpu) by making ALL system calls optionally asynchronous.
to come: ia64 and power-pc patches, patches for gdb, test program (in tools)
Reviewed by: Almost everyone who counts
(at various times, peter, jhb, matt, alfred, mini, bernd,
and a cast of thousands)
NOTE: this is still Beta code, and contains lots of debugging stuff.
expect slight instability in signals..
2002-06-29 17:26:22 +00:00
|
|
|
("forward_signal: thread is not TDS_RUNNING"));
|
1998-03-03 20:55:26 +00:00
|
|
|
|
2001-09-12 08:38:13 +00:00
|
|
|
CTR1(KTR_SMP, "forward_signal(%p)", td->td_proc);
|
2001-01-24 09:48:52 +00:00
|
|
|
|
2020-01-12 06:07:54 +00:00
|
|
|
if (!smp_started || cold || KERNEL_PANICKED())
|
1998-03-03 20:55:26 +00:00
|
|
|
return;
|
|
|
|
if (!forward_signal_enabled)
|
|
|
|
return;
|
2001-04-27 19:28:25 +00:00
|
|
|
|
|
|
|
/* No need to IPI ourself. */
|
2001-09-12 08:38:13 +00:00
|
|
|
if (td == curthread)
|
2001-04-27 19:28:25 +00:00
|
|
|
return;
|
|
|
|
|
2003-04-10 17:35:44 +00:00
|
|
|
id = td->td_oncpu;
|
2001-04-27 19:28:25 +00:00
|
|
|
if (id == NOCPU)
|
|
|
|
return;
|
2010-08-06 15:36:59 +00:00
|
|
|
ipi_cpu(id, IPI_AST);
|
1998-03-03 20:55:26 +00:00
|
|
|
}
|
1998-03-03 22:56:30 +00:00
|
|
|
|
2001-01-24 09:48:52 +00:00
|
|
|
/*
|
|
|
|
* When called the executing CPU will send an IPI to all other CPUs
|
|
|
|
* requesting that they halt execution.
|
|
|
|
*
|
|
|
|
* Usually (but not necessarily) called with 'other_cpus' as its arg.
|
|
|
|
*
|
|
|
|
* - Signals all CPUs in map to stop.
|
|
|
|
* - Waits for each to stop.
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* -1: error
|
|
|
|
* 0: NA
|
|
|
|
* 1: ok
|
|
|
|
*
|
|
|
|
*/
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#if defined(__amd64__) || defined(__i386__)
|
|
|
|
#define X86 1
|
|
|
|
#else
|
|
|
|
#define X86 0
|
|
|
|
#endif
|
2009-08-13 17:09:45 +00:00
|
|
|
static int
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
generic_stop_cpus(cpuset_t map, u_int type)
|
2001-01-24 09:48:52 +00:00
|
|
|
{
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
#ifdef KTR
|
|
|
|
char cpusetbuf[CPUSETBUFSIZ];
|
|
|
|
#endif
|
2010-10-12 17:40:45 +00:00
|
|
|
static volatile u_int stopping_cpu = NOCPU;
|
2001-04-27 19:28:25 +00:00
|
|
|
int i;
|
2012-06-11 18:47:26 +00:00
|
|
|
volatile cpuset_t *cpus;
|
2001-01-24 09:48:52 +00:00
|
|
|
|
2010-10-12 17:40:45 +00:00
|
|
|
KASSERT(
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
type == IPI_STOP || type == IPI_STOP_HARD
|
|
|
|
#if X86
|
|
|
|
|| type == IPI_SUSPEND
|
2010-10-12 17:40:45 +00:00
|
|
|
#endif
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
, ("%s: invalid stop type", __func__));
|
2009-08-13 17:09:45 +00:00
|
|
|
|
2001-01-24 09:48:52 +00:00
|
|
|
if (!smp_started)
|
2010-10-12 17:40:45 +00:00
|
|
|
return (0);
|
2001-01-24 09:48:52 +00:00
|
|
|
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
CTR2(KTR_SMP, "stop_cpus(%s) with %u type",
|
|
|
|
cpusetobj_strprint(cpusetbuf, &map), type);
|
2001-04-27 19:28:25 +00:00
|
|
|
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#if X86
|
2013-09-20 05:06:03 +00:00
|
|
|
/*
|
2013-09-20 22:59:22 +00:00
|
|
|
* When suspending, ensure there are are no IPIs in progress.
|
|
|
|
* IPIs that have been issued, but not yet delivered (e.g.
|
|
|
|
* not pending on a vCPU when running under virtualization)
|
|
|
|
* will be lost, violating FreeBSD's assumption of reliable
|
|
|
|
* IPI delivery.
|
2013-09-20 05:06:03 +00:00
|
|
|
*/
|
|
|
|
if (type == IPI_SUSPEND)
|
|
|
|
mtx_lock_spin(&smp_ipi_mtx);
|
2013-09-22 02:46:13 +00:00
|
|
|
#endif
|
2013-09-20 05:06:03 +00:00
|
|
|
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#if X86
|
|
|
|
if (!nmi_is_broadcast || nmi_kdb_lock == 0) {
|
|
|
|
#endif
|
2010-10-12 17:40:45 +00:00
|
|
|
if (stopping_cpu != PCPU_GET(cpuid))
|
|
|
|
while (atomic_cmpset_int(&stopping_cpu, NOCPU,
|
|
|
|
PCPU_GET(cpuid)) == 0)
|
|
|
|
while (stopping_cpu != NOCPU)
|
|
|
|
cpu_spinwait(); /* spin */
|
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
/* send the stop IPI to all CPUs in map */
|
2009-08-13 17:09:45 +00:00
|
|
|
ipi_selected(map, type);
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#if X86
|
|
|
|
}
|
|
|
|
#endif
|
2004-03-27 18:21:24 +00:00
|
|
|
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#if X86
|
2012-06-11 18:47:26 +00:00
|
|
|
if (type == IPI_SUSPEND)
|
|
|
|
cpus = &suspended_cpus;
|
|
|
|
else
|
|
|
|
#endif
|
|
|
|
cpus = &stopped_cpus;
|
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
i = 0;
|
2012-06-11 18:47:26 +00:00
|
|
|
while (!CPU_SUBSET(cpus, &map)) {
|
2001-04-27 19:28:25 +00:00
|
|
|
/* spin */
|
2007-07-03 18:37:06 +00:00
|
|
|
cpu_spinwait();
|
2001-04-27 19:28:25 +00:00
|
|
|
i++;
|
2011-06-25 10:01:43 +00:00
|
|
|
if (i == 100000000) {
|
2001-04-27 19:28:25 +00:00
|
|
|
printf("timeout stopping cpus\n");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2001-01-24 09:48:52 +00:00
|
|
|
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#if X86
|
2013-09-20 05:06:03 +00:00
|
|
|
if (type == IPI_SUSPEND)
|
|
|
|
mtx_unlock_spin(&smp_ipi_mtx);
|
2013-09-22 02:46:13 +00:00
|
|
|
#endif
|
2013-09-20 05:06:03 +00:00
|
|
|
|
2010-10-12 17:40:45 +00:00
|
|
|
stopping_cpu = NOCPU;
|
|
|
|
return (1);
|
2001-01-24 09:48:52 +00:00
|
|
|
}
|
|
|
|
|
2009-08-13 17:09:45 +00:00
|
|
|
int
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
stop_cpus(cpuset_t map)
|
2009-08-13 17:09:45 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
return (generic_stop_cpus(map, IPI_STOP));
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
stop_cpus_hard(cpuset_t map)
|
2009-08-13 17:09:45 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
return (generic_stop_cpus(map, IPI_STOP_HARD));
|
|
|
|
}
|
|
|
|
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#if X86
|
2009-03-17 00:48:11 +00:00
|
|
|
int
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
suspend_cpus(cpuset_t map)
|
2009-03-17 00:48:11 +00:00
|
|
|
{
|
|
|
|
|
2010-10-12 17:40:45 +00:00
|
|
|
return (generic_stop_cpus(map, IPI_SUSPEND));
|
2009-03-17 00:48:11 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2001-01-24 09:48:52 +00:00
|
|
|
/*
|
2023-01-29 13:18:17 +00:00
|
|
|
* Called by a CPU to restart stopped CPUs.
|
2001-01-24 09:48:52 +00:00
|
|
|
*
|
|
|
|
* Usually (but not necessarily) called with 'stopped_cpus' as its arg.
|
|
|
|
*
|
|
|
|
* - Signals all CPUs in map to restart.
|
|
|
|
* - Waits for each to restart.
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* -1: error
|
|
|
|
* 0: NA
|
|
|
|
* 1: ok
|
|
|
|
*/
|
2013-09-20 05:06:03 +00:00
|
|
|
static int
|
|
|
|
generic_restart_cpus(cpuset_t map, u_int type)
|
2001-01-24 09:48:52 +00:00
|
|
|
{
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
#ifdef KTR
|
|
|
|
char cpusetbuf[CPUSETBUFSIZ];
|
|
|
|
#endif
|
2013-09-20 05:06:03 +00:00
|
|
|
volatile cpuset_t *cpus;
|
|
|
|
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#if X86
|
2019-05-04 20:34:26 +00:00
|
|
|
KASSERT(type == IPI_STOP || type == IPI_STOP_HARD
|
|
|
|
|| type == IPI_SUSPEND, ("%s: invalid stop type", __func__));
|
2001-01-24 09:48:52 +00:00
|
|
|
|
|
|
|
if (!smp_started)
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
return (0);
|
2001-01-24 09:48:52 +00:00
|
|
|
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
CTR1(KTR_SMP, "restart_cpus(%s)", cpusetobj_strprint(cpusetbuf, &map));
|
2001-01-24 09:48:52 +00:00
|
|
|
|
2013-09-20 05:06:03 +00:00
|
|
|
if (type == IPI_SUSPEND)
|
Use resume_cpus() instead of restart_cpus() to resume from ACPI suspension.
restart_cpus() worked well enough by accident. Before this set of fixes,
resume_cpus() used the same cpuset (started_cpus, meaning CPUs directed to
restart) as restart_cpus(). resume_cpus() waited for the wrong cpuset
(stopped_cpus) to become empty, but since mixtures of stopped and suspended
CPUs are not close to working, stopped_cpus must be empty when resuming so
the wait is null -- restart_cpus just allows the other CPUs to restart and
returns without waiting.
Fix resume_cpus() to wait on a non-wrong cpuset for the ACPI case, and
add further kludges to try to keep it working for the XEN case. It
was only used for XEN. It waited on suspended_cpus. This works for
XEN. However, for ACPI, resuming is a 2-step process. ACPI has already
woken up the other CPUs and removed them from suspended_cpus. This
fix records the move by putting them in a new cpuset resuming_cpus.
Waiting on suspended_cpus would give the same null wait as waiting on
stopped_cpus. Wait on resuming_cpus instead.
Add a cpuset toresume_cpus to map the CPUs being told to resume to keep
this separate from the cpuset started_cpus for mapping the CPUs being told
to restart. Mixtures of stopped and suspended/resuming CPUs are still far
from working. Describe new and some old cpusets in comments.
Add further kludges to cpususpend_handler() to try to avoid breaking it
for XEN. XEN doesn't use resumectx(), so it doesn't use the second
return path for savectx(), and it goes from the suspended state directly
to the restarted state, while ACPI resume goes through the resuming state.
Enter the resuming state early for all cases so that resume_cpus can test
for being in this state and not have to worry about the intermediate
!suspended state for ACPI only.
Reviewed by: kib
2017-12-21 09:17:48 +00:00
|
|
|
cpus = &resuming_cpus;
|
2013-09-20 05:06:03 +00:00
|
|
|
else
|
|
|
|
cpus = &stopped_cpus;
|
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
/* signal other cpus to restart */
|
Use resume_cpus() instead of restart_cpus() to resume from ACPI suspension.
restart_cpus() worked well enough by accident. Before this set of fixes,
resume_cpus() used the same cpuset (started_cpus, meaning CPUs directed to
restart) as restart_cpus(). resume_cpus() waited for the wrong cpuset
(stopped_cpus) to become empty, but since mixtures of stopped and suspended
CPUs are not close to working, stopped_cpus must be empty when resuming so
the wait is null -- restart_cpus just allows the other CPUs to restart and
returns without waiting.
Fix resume_cpus() to wait on a non-wrong cpuset for the ACPI case, and
add further kludges to try to keep it working for the XEN case. It
was only used for XEN. It waited on suspended_cpus. This works for
XEN. However, for ACPI, resuming is a 2-step process. ACPI has already
woken up the other CPUs and removed them from suspended_cpus. This
fix records the move by putting them in a new cpuset resuming_cpus.
Waiting on suspended_cpus would give the same null wait as waiting on
stopped_cpus. Wait on resuming_cpus instead.
Add a cpuset toresume_cpus to map the CPUs being told to resume to keep
this separate from the cpuset started_cpus for mapping the CPUs being told
to restart. Mixtures of stopped and suspended/resuming CPUs are still far
from working. Describe new and some old cpusets in comments.
Add further kludges to cpususpend_handler() to try to avoid breaking it
for XEN. XEN doesn't use resumectx(), so it doesn't use the second
return path for savectx(), and it goes from the suspended state directly
to the restarted state, while ACPI resume goes through the resuming state.
Enter the resuming state early for all cases so that resume_cpus can test
for being in this state and not have to worry about the intermediate
!suspended state for ACPI only.
Reviewed by: kib
2017-12-21 09:17:48 +00:00
|
|
|
if (type == IPI_SUSPEND)
|
|
|
|
CPU_COPY_STORE_REL(&map, &toresume_cpus);
|
|
|
|
else
|
|
|
|
CPU_COPY_STORE_REL(&map, &started_cpus);
|
2001-01-24 09:48:52 +00:00
|
|
|
|
2019-05-04 20:34:26 +00:00
|
|
|
/*
|
|
|
|
* Wake up any CPUs stopped with MWAIT. From MI code we can't tell if
|
|
|
|
* MONITOR/MWAIT is enabled, but the potentially redundant writes are
|
|
|
|
* relatively inexpensive.
|
|
|
|
*/
|
|
|
|
if (type == IPI_STOP) {
|
|
|
|
struct monitorbuf *mb;
|
|
|
|
u_int id;
|
|
|
|
|
|
|
|
CPU_FOREACH(id) {
|
|
|
|
if (!CPU_ISSET(id, &map))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
mb = &pcpu_find(id)->pc_monitorbuf;
|
|
|
|
atomic_store_int(&mb->stop_state,
|
|
|
|
MONITOR_STOPSTATE_RUNNING);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
if (!nmi_is_broadcast || nmi_kdb_lock == 0) {
|
2019-05-04 20:34:26 +00:00
|
|
|
/* wait for each to clear its bit */
|
|
|
|
while (CPU_OVERLAP(cpus, &map))
|
|
|
|
cpu_spinwait();
|
|
|
|
}
|
|
|
|
#else /* !X86 */
|
|
|
|
KASSERT(type == IPI_STOP || type == IPI_STOP_HARD,
|
|
|
|
("%s: invalid stop type", __func__));
|
|
|
|
|
|
|
|
if (!smp_started)
|
|
|
|
return (0);
|
|
|
|
|
|
|
|
CTR1(KTR_SMP, "restart_cpus(%s)", cpusetobj_strprint(cpusetbuf, &map));
|
|
|
|
|
|
|
|
cpus = &stopped_cpus;
|
|
|
|
|
|
|
|
/* signal other cpus to restart */
|
|
|
|
CPU_COPY_STORE_REL(&map, &started_cpus);
|
|
|
|
|
2001-04-27 19:28:25 +00:00
|
|
|
/* wait for each to clear its bit */
|
2013-09-20 05:06:03 +00:00
|
|
|
while (CPU_OVERLAP(cpus, &map))
|
2007-07-03 18:37:06 +00:00
|
|
|
cpu_spinwait();
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#endif
|
|
|
|
return (1);
|
2001-01-24 09:48:52 +00:00
|
|
|
}
|
|
|
|
|
2013-09-20 05:06:03 +00:00
|
|
|
int
|
|
|
|
restart_cpus(cpuset_t map)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (generic_restart_cpus(map, IPI_STOP));
|
|
|
|
}
|
|
|
|
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#if X86
|
2013-09-20 05:06:03 +00:00
|
|
|
int
|
|
|
|
resume_cpus(cpuset_t map)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (generic_restart_cpus(map, IPI_SUSPEND));
|
|
|
|
}
|
|
|
|
#endif
|
Handle broadcast NMIs.
On several Intel chipsets, diagnostic NMIs sent from BMC or NMIs
reporting hardware errors are broadcasted to all CPUs.
When kernel is configured to enter kdb on NMI, the outcome is
problematic, because each CPU tries to enter kdb. All CPUs are
executing NMI handlers, which set the latches disabling the nested NMI
delivery; this means that stop_cpus_hard(), used by kdb_enter() to
stop other cpus by broadcasting IPI_STOP_HARD NMI, cannot work. One
indication of this is the harmless but annoying diagnostic "timeout
stopping cpus".
Much more harming behaviour is that because all CPUs try to enter kdb,
and if ddb is used as debugger, all CPUs issue prompt on console and
race for the input, not to mention the simultaneous use of the ddb
shared state.
Try to fix this by introducing a pseudo-lock for simultaneous attempts
to handle NMIs. If one core happens to enter NMI trap handler, other
cores see it and simulate reception of the IPI_STOP_HARD. More,
generic_stop_cpus() avoids sending IPI_STOP_HARD and avoids waiting
for the acknowledgement, relying on the nmi handler on other cores
suspending and then restarting the CPU.
Since it is impossible to detect at runtime whether some stray NMI is
broadcast or unicast, add a knob for administrator (really developer)
to configure debugging NMI handling mode.
The updated patch was debugged with the help from Andrey Gapon (avg)
and discussed with him.
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D8249
2016-10-24 16:40:27 +00:00
|
|
|
#undef X86
|
2013-09-20 05:06:03 +00:00
|
|
|
|
1999-07-20 06:52:35 +00:00
|
|
|
/*
|
2023-01-29 13:18:17 +00:00
|
|
|
* All-CPU rendezvous. CPUs are signalled, all execute the setup function
|
1999-07-20 06:52:35 +00:00
|
|
|
* (if specified), rendezvous, execute the action function (if specified),
|
|
|
|
* rendezvous again, execute the teardown function (if specified), and then
|
|
|
|
* resume.
|
|
|
|
*
|
|
|
|
* Note that the supplied external functions _must_ be reentrant and aware
|
|
|
|
* that they are running in parallel and in an unknown lock context.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
smp_rendezvous_action(void)
|
|
|
|
{
|
Fix an issue with critical sections and SMP rendezvous handlers.
Specifically, a critical_exit() call that drops the nesting level to zero
has a brief window where the pending preemption flag is set and the
nesting level is set to zero. This is done purposefully to avoid races
where a preemption scheduled by an interrupt could be lost otherwise (see
revision 144777). However, this does mean that if an interrupt fires
during this window and enters and exits a critical section, it may preempt
from the interrupt context. This is generally fine as the interrupt code
is careful to arrange critical sections so that they are not exited until
it is safe to preempt (e.g. interrupts EOI'd and masked if necessary).
However, the SMP rendezvous IPI handler does not quite follow this rule,
and in general a rendezvous can never be preempted. Rendezvous handlers
are also not permitted to schedule threads to execute, so they will not
typically trigger preemptions. SMP rendezvous handlers may use
spinlocks (carefully) such as the rm_cleanIPI() handler used in rmlocks,
but using a spinlock also enters and exits a critical section. If the
interrupted top-half code is in the brief window of critical_exit() where
the nesting level is zero but a preemption is pending, then releasing the
spinlock can trigger a preemption. Because we know that SMP rendezvous
handlers can never schedule a thread, we know that a critical_exit() in
an SMP rendezvous handler will only preempt in this edge case. We also
know that the top-half thread will happily handle the deferred preemption
once the SMP rendezvous has completed, so the preemption will not be lost.
This makes it safe to employ a workaround where we use a nested critical
section in the SMP rendezvous code itself around rendezvous action
routines to prevent any preemptions during an SMP rendezvous. The
workaround intentionally avoids checking for a deferred preemption
when leaving the critical section on the assumption that if there is a
pending preemption it will be handled by the interrupted top-half code.
Submitted by: mlaier (variation specific to rm_cleanIPI())
Obtained from: Isilon
MFC after: 1 week
2011-05-24 13:36:41 +00:00
|
|
|
struct thread *td;
|
2011-05-17 16:39:08 +00:00
|
|
|
void *local_func_arg;
|
|
|
|
void (*local_setup_func)(void*);
|
|
|
|
void (*local_action_func)(void*);
|
|
|
|
void (*local_teardown_func)(void*);
|
Fix an issue with critical sections and SMP rendezvous handlers.
Specifically, a critical_exit() call that drops the nesting level to zero
has a brief window where the pending preemption flag is set and the
nesting level is set to zero. This is done purposefully to avoid races
where a preemption scheduled by an interrupt could be lost otherwise (see
revision 144777). However, this does mean that if an interrupt fires
during this window and enters and exits a critical section, it may preempt
from the interrupt context. This is generally fine as the interrupt code
is careful to arrange critical sections so that they are not exited until
it is safe to preempt (e.g. interrupts EOI'd and masked if necessary).
However, the SMP rendezvous IPI handler does not quite follow this rule,
and in general a rendezvous can never be preempted. Rendezvous handlers
are also not permitted to schedule threads to execute, so they will not
typically trigger preemptions. SMP rendezvous handlers may use
spinlocks (carefully) such as the rm_cleanIPI() handler used in rmlocks,
but using a spinlock also enters and exits a critical section. If the
interrupted top-half code is in the brief window of critical_exit() where
the nesting level is zero but a preemption is pending, then releasing the
spinlock can trigger a preemption. Because we know that SMP rendezvous
handlers can never schedule a thread, we know that a critical_exit() in
an SMP rendezvous handler will only preempt in this edge case. We also
know that the top-half thread will happily handle the deferred preemption
once the SMP rendezvous has completed, so the preemption will not be lost.
This makes it safe to employ a workaround where we use a nested critical
section in the SMP rendezvous code itself around rendezvous action
routines to prevent any preemptions during an SMP rendezvous. The
workaround intentionally avoids checking for a deferred preemption
when leaving the critical section on the assumption that if there is a
pending preemption it will be handled by the interrupted top-half code.
Submitted by: mlaier (variation specific to rm_cleanIPI())
Obtained from: Isilon
MFC after: 1 week
2011-05-24 13:36:41 +00:00
|
|
|
#ifdef INVARIANTS
|
|
|
|
int owepreempt;
|
|
|
|
#endif
|
2008-01-02 17:09:15 +00:00
|
|
|
|
2007-07-03 18:37:06 +00:00
|
|
|
/* Ensure we have up-to-date values. */
|
|
|
|
atomic_add_acq_int(&smp_rv_waiters[0], 1);
|
2008-08-27 18:23:55 +00:00
|
|
|
while (smp_rv_waiters[0] < smp_rv_ncpus)
|
2007-07-03 18:37:06 +00:00
|
|
|
cpu_spinwait();
|
|
|
|
|
2011-05-17 16:39:08 +00:00
|
|
|
/* Fetch rendezvous parameters after acquire barrier. */
|
|
|
|
local_func_arg = smp_rv_func_arg;
|
|
|
|
local_setup_func = smp_rv_setup_func;
|
|
|
|
local_action_func = smp_rv_action_func;
|
|
|
|
local_teardown_func = smp_rv_teardown_func;
|
|
|
|
|
Fix an issue with critical sections and SMP rendezvous handlers.
Specifically, a critical_exit() call that drops the nesting level to zero
has a brief window where the pending preemption flag is set and the
nesting level is set to zero. This is done purposefully to avoid races
where a preemption scheduled by an interrupt could be lost otherwise (see
revision 144777). However, this does mean that if an interrupt fires
during this window and enters and exits a critical section, it may preempt
from the interrupt context. This is generally fine as the interrupt code
is careful to arrange critical sections so that they are not exited until
it is safe to preempt (e.g. interrupts EOI'd and masked if necessary).
However, the SMP rendezvous IPI handler does not quite follow this rule,
and in general a rendezvous can never be preempted. Rendezvous handlers
are also not permitted to schedule threads to execute, so they will not
typically trigger preemptions. SMP rendezvous handlers may use
spinlocks (carefully) such as the rm_cleanIPI() handler used in rmlocks,
but using a spinlock also enters and exits a critical section. If the
interrupted top-half code is in the brief window of critical_exit() where
the nesting level is zero but a preemption is pending, then releasing the
spinlock can trigger a preemption. Because we know that SMP rendezvous
handlers can never schedule a thread, we know that a critical_exit() in
an SMP rendezvous handler will only preempt in this edge case. We also
know that the top-half thread will happily handle the deferred preemption
once the SMP rendezvous has completed, so the preemption will not be lost.
This makes it safe to employ a workaround where we use a nested critical
section in the SMP rendezvous code itself around rendezvous action
routines to prevent any preemptions during an SMP rendezvous. The
workaround intentionally avoids checking for a deferred preemption
when leaving the critical section on the assumption that if there is a
pending preemption it will be handled by the interrupted top-half code.
Submitted by: mlaier (variation specific to rm_cleanIPI())
Obtained from: Isilon
MFC after: 1 week
2011-05-24 13:36:41 +00:00
|
|
|
/*
|
|
|
|
* Use a nested critical section to prevent any preemptions
|
|
|
|
* from occurring during a rendezvous action routine.
|
|
|
|
* Specifically, if a rendezvous handler is invoked via an IPI
|
|
|
|
* and the interrupted thread was in the critical_exit()
|
|
|
|
* function after setting td_critnest to 0 but before
|
|
|
|
* performing a deferred preemption, this routine can be
|
|
|
|
* invoked with td_critnest set to 0 and td_owepreempt true.
|
|
|
|
* In that case, a critical_exit() during the rendezvous
|
|
|
|
* action would trigger a preemption which is not permitted in
|
|
|
|
* a rendezvous action. To fix this, wrap all of the
|
|
|
|
* rendezvous action handlers in a critical section. We
|
|
|
|
* cannot use a regular critical section however as having
|
|
|
|
* critical_exit() preempt from this routine would also be
|
|
|
|
* problematic (the preemption must not occur before the IPI
|
2011-05-24 19:55:57 +00:00
|
|
|
* has been acknowledged via an EOI). Instead, we
|
Fix an issue with critical sections and SMP rendezvous handlers.
Specifically, a critical_exit() call that drops the nesting level to zero
has a brief window where the pending preemption flag is set and the
nesting level is set to zero. This is done purposefully to avoid races
where a preemption scheduled by an interrupt could be lost otherwise (see
revision 144777). However, this does mean that if an interrupt fires
during this window and enters and exits a critical section, it may preempt
from the interrupt context. This is generally fine as the interrupt code
is careful to arrange critical sections so that they are not exited until
it is safe to preempt (e.g. interrupts EOI'd and masked if necessary).
However, the SMP rendezvous IPI handler does not quite follow this rule,
and in general a rendezvous can never be preempted. Rendezvous handlers
are also not permitted to schedule threads to execute, so they will not
typically trigger preemptions. SMP rendezvous handlers may use
spinlocks (carefully) such as the rm_cleanIPI() handler used in rmlocks,
but using a spinlock also enters and exits a critical section. If the
interrupted top-half code is in the brief window of critical_exit() where
the nesting level is zero but a preemption is pending, then releasing the
spinlock can trigger a preemption. Because we know that SMP rendezvous
handlers can never schedule a thread, we know that a critical_exit() in
an SMP rendezvous handler will only preempt in this edge case. We also
know that the top-half thread will happily handle the deferred preemption
once the SMP rendezvous has completed, so the preemption will not be lost.
This makes it safe to employ a workaround where we use a nested critical
section in the SMP rendezvous code itself around rendezvous action
routines to prevent any preemptions during an SMP rendezvous. The
workaround intentionally avoids checking for a deferred preemption
when leaving the critical section on the assumption that if there is a
pending preemption it will be handled by the interrupted top-half code.
Submitted by: mlaier (variation specific to rm_cleanIPI())
Obtained from: Isilon
MFC after: 1 week
2011-05-24 13:36:41 +00:00
|
|
|
* intentionally ignore td_owepreempt when leaving the
|
2011-05-24 19:55:57 +00:00
|
|
|
* critical section. This should be harmless because we do
|
|
|
|
* not permit rendezvous action routines to schedule threads,
|
|
|
|
* and thus td_owepreempt should never transition from 0 to 1
|
Fix an issue with critical sections and SMP rendezvous handlers.
Specifically, a critical_exit() call that drops the nesting level to zero
has a brief window where the pending preemption flag is set and the
nesting level is set to zero. This is done purposefully to avoid races
where a preemption scheduled by an interrupt could be lost otherwise (see
revision 144777). However, this does mean that if an interrupt fires
during this window and enters and exits a critical section, it may preempt
from the interrupt context. This is generally fine as the interrupt code
is careful to arrange critical sections so that they are not exited until
it is safe to preempt (e.g. interrupts EOI'd and masked if necessary).
However, the SMP rendezvous IPI handler does not quite follow this rule,
and in general a rendezvous can never be preempted. Rendezvous handlers
are also not permitted to schedule threads to execute, so they will not
typically trigger preemptions. SMP rendezvous handlers may use
spinlocks (carefully) such as the rm_cleanIPI() handler used in rmlocks,
but using a spinlock also enters and exits a critical section. If the
interrupted top-half code is in the brief window of critical_exit() where
the nesting level is zero but a preemption is pending, then releasing the
spinlock can trigger a preemption. Because we know that SMP rendezvous
handlers can never schedule a thread, we know that a critical_exit() in
an SMP rendezvous handler will only preempt in this edge case. We also
know that the top-half thread will happily handle the deferred preemption
once the SMP rendezvous has completed, so the preemption will not be lost.
This makes it safe to employ a workaround where we use a nested critical
section in the SMP rendezvous code itself around rendezvous action
routines to prevent any preemptions during an SMP rendezvous. The
workaround intentionally avoids checking for a deferred preemption
when leaving the critical section on the assumption that if there is a
pending preemption it will be handled by the interrupted top-half code.
Submitted by: mlaier (variation specific to rm_cleanIPI())
Obtained from: Isilon
MFC after: 1 week
2011-05-24 13:36:41 +00:00
|
|
|
* during this routine.
|
|
|
|
*/
|
|
|
|
td = curthread;
|
|
|
|
td->td_critnest++;
|
|
|
|
#ifdef INVARIANTS
|
|
|
|
owepreempt = td->td_owepreempt;
|
|
|
|
#endif
|
2020-09-01 22:12:32 +00:00
|
|
|
|
2011-05-17 16:39:08 +00:00
|
|
|
/*
|
|
|
|
* If requested, run a setup function before the main action
|
|
|
|
* function. Ensure all CPUs have completed the setup
|
|
|
|
* function before moving on to the action function.
|
|
|
|
*/
|
2017-04-09 02:00:03 +00:00
|
|
|
if (local_setup_func != smp_no_rendezvous_barrier) {
|
2022-06-06 15:21:33 +00:00
|
|
|
if (local_setup_func != NULL)
|
|
|
|
local_setup_func(local_func_arg);
|
2007-11-08 14:47:55 +00:00
|
|
|
atomic_add_int(&smp_rv_waiters[1], 1);
|
2008-08-27 18:23:55 +00:00
|
|
|
while (smp_rv_waiters[1] < smp_rv_ncpus)
|
2007-11-08 14:47:55 +00:00
|
|
|
cpu_spinwait();
|
|
|
|
}
|
2007-07-03 18:37:06 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
if (local_action_func != NULL)
|
|
|
|
local_action_func(local_func_arg);
|
|
|
|
|
2017-04-09 02:00:03 +00:00
|
|
|
if (local_teardown_func != smp_no_rendezvous_barrier) {
|
2011-07-30 20:29:39 +00:00
|
|
|
/*
|
|
|
|
* Signal that the main action has been completed. If a
|
|
|
|
* full exit rendezvous is requested, then all CPUs will
|
|
|
|
* wait here until all CPUs have finished the main action.
|
|
|
|
*/
|
|
|
|
atomic_add_int(&smp_rv_waiters[2], 1);
|
|
|
|
while (smp_rv_waiters[2] < smp_rv_ncpus)
|
Fix an issue with critical sections and SMP rendezvous handlers.
Specifically, a critical_exit() call that drops the nesting level to zero
has a brief window where the pending preemption flag is set and the
nesting level is set to zero. This is done purposefully to avoid races
where a preemption scheduled by an interrupt could be lost otherwise (see
revision 144777). However, this does mean that if an interrupt fires
during this window and enters and exits a critical section, it may preempt
from the interrupt context. This is generally fine as the interrupt code
is careful to arrange critical sections so that they are not exited until
it is safe to preempt (e.g. interrupts EOI'd and masked if necessary).
However, the SMP rendezvous IPI handler does not quite follow this rule,
and in general a rendezvous can never be preempted. Rendezvous handlers
are also not permitted to schedule threads to execute, so they will not
typically trigger preemptions. SMP rendezvous handlers may use
spinlocks (carefully) such as the rm_cleanIPI() handler used in rmlocks,
but using a spinlock also enters and exits a critical section. If the
interrupted top-half code is in the brief window of critical_exit() where
the nesting level is zero but a preemption is pending, then releasing the
spinlock can trigger a preemption. Because we know that SMP rendezvous
handlers can never schedule a thread, we know that a critical_exit() in
an SMP rendezvous handler will only preempt in this edge case. We also
know that the top-half thread will happily handle the deferred preemption
once the SMP rendezvous has completed, so the preemption will not be lost.
This makes it safe to employ a workaround where we use a nested critical
section in the SMP rendezvous code itself around rendezvous action
routines to prevent any preemptions during an SMP rendezvous. The
workaround intentionally avoids checking for a deferred preemption
when leaving the critical section on the assumption that if there is a
pending preemption it will be handled by the interrupted top-half code.
Submitted by: mlaier (variation specific to rm_cleanIPI())
Obtained from: Isilon
MFC after: 1 week
2011-05-24 13:36:41 +00:00
|
|
|
cpu_spinwait();
|
|
|
|
|
|
|
|
if (local_teardown_func != NULL)
|
|
|
|
local_teardown_func(local_func_arg);
|
|
|
|
}
|
2008-01-02 17:09:15 +00:00
|
|
|
|
2011-07-30 20:29:39 +00:00
|
|
|
/*
|
|
|
|
* Signal that the rendezvous is fully completed by this CPU.
|
|
|
|
* This means that no member of smp_rv_* pseudo-structure will be
|
|
|
|
* accessed by this target CPU after this point; in particular,
|
|
|
|
* memory pointed by smp_rv_func_arg.
|
2015-07-21 22:56:46 +00:00
|
|
|
*
|
|
|
|
* The release semantic ensures that all accesses performed by
|
|
|
|
* the current CPU are visible when smp_rendezvous_cpus()
|
|
|
|
* returns, by synchronizing with the
|
|
|
|
* atomic_load_acq_int(&smp_rv_waiters[3]).
|
2011-07-30 20:29:39 +00:00
|
|
|
*/
|
2015-07-21 22:56:46 +00:00
|
|
|
atomic_add_rel_int(&smp_rv_waiters[3], 1);
|
2011-07-30 20:29:39 +00:00
|
|
|
|
Fix an issue with critical sections and SMP rendezvous handlers.
Specifically, a critical_exit() call that drops the nesting level to zero
has a brief window where the pending preemption flag is set and the
nesting level is set to zero. This is done purposefully to avoid races
where a preemption scheduled by an interrupt could be lost otherwise (see
revision 144777). However, this does mean that if an interrupt fires
during this window and enters and exits a critical section, it may preempt
from the interrupt context. This is generally fine as the interrupt code
is careful to arrange critical sections so that they are not exited until
it is safe to preempt (e.g. interrupts EOI'd and masked if necessary).
However, the SMP rendezvous IPI handler does not quite follow this rule,
and in general a rendezvous can never be preempted. Rendezvous handlers
are also not permitted to schedule threads to execute, so they will not
typically trigger preemptions. SMP rendezvous handlers may use
spinlocks (carefully) such as the rm_cleanIPI() handler used in rmlocks,
but using a spinlock also enters and exits a critical section. If the
interrupted top-half code is in the brief window of critical_exit() where
the nesting level is zero but a preemption is pending, then releasing the
spinlock can trigger a preemption. Because we know that SMP rendezvous
handlers can never schedule a thread, we know that a critical_exit() in
an SMP rendezvous handler will only preempt in this edge case. We also
know that the top-half thread will happily handle the deferred preemption
once the SMP rendezvous has completed, so the preemption will not be lost.
This makes it safe to employ a workaround where we use a nested critical
section in the SMP rendezvous code itself around rendezvous action
routines to prevent any preemptions during an SMP rendezvous. The
workaround intentionally avoids checking for a deferred preemption
when leaving the critical section on the assumption that if there is a
pending preemption it will be handled by the interrupted top-half code.
Submitted by: mlaier (variation specific to rm_cleanIPI())
Obtained from: Isilon
MFC after: 1 week
2011-05-24 13:36:41 +00:00
|
|
|
td->td_critnest--;
|
|
|
|
KASSERT(owepreempt == td->td_owepreempt,
|
|
|
|
("rendezvous action changed td_owepreempt"));
|
1999-07-20 06:52:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
smp_rendezvous_cpus(cpuset_t map,
|
2008-05-23 04:05:26 +00:00
|
|
|
void (* setup_func)(void *),
|
|
|
|
void (* action_func)(void *),
|
|
|
|
void (* teardown_func)(void *),
|
|
|
|
void *arg)
|
1999-07-20 06:52:35 +00:00
|
|
|
{
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
int curcpumap, i, ncpus = 0;
|
2001-01-24 12:35:55 +00:00
|
|
|
|
2020-01-30 20:02:14 +00:00
|
|
|
/* See comments in the !SMP case. */
|
2001-04-27 19:28:25 +00:00
|
|
|
if (!smp_started) {
|
2011-11-03 14:36:56 +00:00
|
|
|
spinlock_enter();
|
2001-04-27 19:28:25 +00:00
|
|
|
if (setup_func != NULL)
|
|
|
|
setup_func(arg);
|
|
|
|
if (action_func != NULL)
|
|
|
|
action_func(arg);
|
|
|
|
if (teardown_func != NULL)
|
|
|
|
teardown_func(arg);
|
2011-11-03 14:36:56 +00:00
|
|
|
spinlock_exit();
|
2001-04-27 19:28:25 +00:00
|
|
|
return;
|
|
|
|
}
|
2008-05-23 04:05:26 +00:00
|
|
|
|
2020-01-30 19:38:51 +00:00
|
|
|
/*
|
|
|
|
* Make sure we come here with interrupts enabled. Otherwise we
|
2020-01-30 20:02:14 +00:00
|
|
|
* livelock if smp_ipi_mtx is owned by a thread which sent us an IPI.
|
2020-01-30 19:38:51 +00:00
|
|
|
*/
|
|
|
|
MPASS(curthread->td_md.md_spinlock_count == 0);
|
|
|
|
|
2010-06-11 18:46:34 +00:00
|
|
|
CPU_FOREACH(i) {
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
if (CPU_ISSET(i, &map))
|
2008-05-23 04:05:26 +00:00
|
|
|
ncpus++;
|
2010-06-11 18:46:34 +00:00
|
|
|
}
|
2009-03-01 14:26:24 +00:00
|
|
|
if (ncpus == 0)
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
panic("ncpus is 0 with non-zero map");
|
2008-08-27 18:23:55 +00:00
|
|
|
|
2004-08-28 00:49:55 +00:00
|
|
|
mtx_lock_spin(&smp_ipi_mtx);
|
1999-07-20 06:52:35 +00:00
|
|
|
|
2011-05-17 16:39:08 +00:00
|
|
|
/* Pass rendezvous parameters via global variables. */
|
2008-08-27 18:23:55 +00:00
|
|
|
smp_rv_ncpus = ncpus;
|
1999-07-20 06:52:35 +00:00
|
|
|
smp_rv_setup_func = setup_func;
|
|
|
|
smp_rv_action_func = action_func;
|
|
|
|
smp_rv_teardown_func = teardown_func;
|
|
|
|
smp_rv_func_arg = arg;
|
|
|
|
smp_rv_waiters[1] = 0;
|
2007-07-03 18:37:06 +00:00
|
|
|
smp_rv_waiters[2] = 0;
|
2011-07-30 20:29:39 +00:00
|
|
|
smp_rv_waiters[3] = 0;
|
2007-07-03 18:37:06 +00:00
|
|
|
atomic_store_rel_int(&smp_rv_waiters[0], 0);
|
1999-07-20 06:52:35 +00:00
|
|
|
|
2011-05-17 16:39:08 +00:00
|
|
|
/*
|
|
|
|
* Signal other processors, which will enter the IPI with
|
|
|
|
* interrupts off.
|
|
|
|
*/
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
curcpumap = CPU_ISSET(curcpu, &map);
|
|
|
|
CPU_CLR(curcpu, &map);
|
|
|
|
ipi_selected(map, IPI_RENDEZVOUS);
|
1999-07-20 06:52:35 +00:00
|
|
|
|
2008-05-23 04:05:26 +00:00
|
|
|
/* Check if the current CPU is in the map */
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
if (curcpumap != 0)
|
2008-05-23 04:05:26 +00:00
|
|
|
smp_rendezvous_action();
|
1999-07-20 06:52:35 +00:00
|
|
|
|
2011-05-17 16:39:08 +00:00
|
|
|
/*
|
2011-07-30 20:29:39 +00:00
|
|
|
* Ensure that the master CPU waits for all the other
|
|
|
|
* CPUs to finish the rendezvous, so that smp_rv_*
|
|
|
|
* pseudo-structure and the arg are guaranteed to not
|
|
|
|
* be in use.
|
2015-07-21 22:56:46 +00:00
|
|
|
*
|
|
|
|
* Load acquire synchronizes with the release add in
|
|
|
|
* smp_rendezvous_action(), which ensures that our caller sees
|
|
|
|
* all memory actions done by the called functions on other
|
|
|
|
* CPUs.
|
2011-05-17 16:39:08 +00:00
|
|
|
*/
|
2011-07-30 20:29:39 +00:00
|
|
|
while (atomic_load_acq_int(&smp_rv_waiters[3]) < ncpus)
|
|
|
|
cpu_spinwait();
|
2008-01-02 17:09:15 +00:00
|
|
|
|
2004-08-28 00:49:55 +00:00
|
|
|
mtx_unlock_spin(&smp_ipi_mtx);
|
1999-07-20 06:52:35 +00:00
|
|
|
}
|
2003-12-03 14:55:31 +00:00
|
|
|
|
2008-05-23 04:05:26 +00:00
|
|
|
void
|
|
|
|
smp_rendezvous(void (* setup_func)(void *),
|
|
|
|
void (* action_func)(void *),
|
|
|
|
void (* teardown_func)(void *),
|
|
|
|
void *arg)
|
|
|
|
{
|
|
|
|
smp_rendezvous_cpus(all_cpus, setup_func, action_func, teardown_func, arg);
|
|
|
|
}
|
|
|
|
|
Refactor/optimize cpu_search_*().
Remove cpu_search_both(), unused for many years. Without it there is
less sense for the trick of compiling common cpu_search() into separate
cpu_search_lowest() and cpu_search_highest(), so split them completely,
making code more readable. While there, split iteration over children
groups and CPUs, complicating code for very small deduplication.
Stop passing cpuset_t arguments by value and avoid some manipulations.
Since MAXCPU bump from 64 to 256, what was a single register turned
into 32-byte memory array, requiring memory allocation and accesses.
Splitting struct cpu_search into parameter and result parts allows to
even more reduce stack usage, since the first can be passed through
on recursion.
Remove CPU_FFS() from the hot paths, precalculating first and last CPU
for each CPU group in advance during initialization. Again, it was
not a problem for 64 CPUs before, but for 256 FFS needs much more code.
With these changes on 80-thread system doing ~260K uncached ZFS reads
per second I observe ~30% reduction of time spent in cpu_search_*().
MFC after: 1 month
2021-07-29 01:18:50 +00:00
|
|
|
static void
|
|
|
|
smp_topo_fill(struct cpu_group *cg)
|
|
|
|
{
|
|
|
|
int c;
|
|
|
|
|
|
|
|
for (c = 0; c < cg->cg_children; c++)
|
|
|
|
smp_topo_fill(&cg->cg_child[c]);
|
|
|
|
cg->cg_first = CPU_FFS(&cg->cg_mask) - 1;
|
|
|
|
cg->cg_last = CPU_FLS(&cg->cg_mask) - 1;
|
|
|
|
}
|
|
|
|
|
2008-03-02 07:58:42 +00:00
|
|
|
struct cpu_group *
|
|
|
|
smp_topo(void)
|
2003-12-03 14:55:31 +00:00
|
|
|
{
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
char cpusetbuf[CPUSETBUFSIZ], cpusetbuf2[CPUSETBUFSIZ];
|
2008-03-02 07:58:42 +00:00
|
|
|
struct cpu_group *top;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check for a fake topology request for debugging purposes.
|
|
|
|
*/
|
|
|
|
switch (smp_topology) {
|
|
|
|
case 1:
|
|
|
|
/* Dual core with no sharing. */
|
|
|
|
top = smp_topo_1level(CG_SHARE_NONE, 2, 0);
|
|
|
|
break;
|
2008-03-10 01:38:53 +00:00
|
|
|
case 2:
|
|
|
|
/* No topology, all cpus are equal. */
|
|
|
|
top = smp_topo_none();
|
|
|
|
break;
|
2008-03-02 07:58:42 +00:00
|
|
|
case 3:
|
|
|
|
/* Dual core with shared L2. */
|
|
|
|
top = smp_topo_1level(CG_SHARE_L2, 2, 0);
|
|
|
|
break;
|
|
|
|
case 4:
|
|
|
|
/* quad core, shared l3 among each package, private l2. */
|
|
|
|
top = smp_topo_1level(CG_SHARE_L3, 4, 0);
|
|
|
|
break;
|
|
|
|
case 5:
|
|
|
|
/* quad core, 2 dualcore parts on each package share l2. */
|
|
|
|
top = smp_topo_2level(CG_SHARE_NONE, 2, CG_SHARE_L2, 2, 0);
|
|
|
|
break;
|
|
|
|
case 6:
|
|
|
|
/* Single-core 2xHTT */
|
|
|
|
top = smp_topo_1level(CG_SHARE_L1, 2, CG_FLAG_HTT);
|
|
|
|
break;
|
|
|
|
case 7:
|
|
|
|
/* quad core with a shared l3, 8 threads sharing L2. */
|
|
|
|
top = smp_topo_2level(CG_SHARE_L3, 4, CG_SHARE_L2, 8,
|
2009-04-29 03:15:43 +00:00
|
|
|
CG_FLAG_SMT);
|
2008-03-02 07:58:42 +00:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
/* Default, ask the system what it wants. */
|
|
|
|
top = cpu_topo();
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Verify the returned topology.
|
|
|
|
*/
|
|
|
|
if (top->cg_count != mp_ncpus)
|
|
|
|
panic("Built bad topology at %p. CPU count %d != %d",
|
|
|
|
top, top->cg_count, mp_ncpus);
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
if (CPU_CMP(&top->cg_mask, &all_cpus))
|
|
|
|
panic("Built bad topology at %p. CPU mask (%s) != (%s)",
|
|
|
|
top, cpusetobj_strprint(cpusetbuf, &top->cg_mask),
|
|
|
|
cpusetobj_strprint(cpusetbuf2, &all_cpus));
|
2017-08-27 05:14:48 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Collapse nonsense levels that may be created out of convenience by
|
|
|
|
* the MD layers. They cause extra work in the search functions.
|
|
|
|
*/
|
|
|
|
while (top->cg_children == 1) {
|
|
|
|
top = &top->cg_child[0];
|
|
|
|
top->cg_parent = NULL;
|
|
|
|
}
|
Refactor/optimize cpu_search_*().
Remove cpu_search_both(), unused for many years. Without it there is
less sense for the trick of compiling common cpu_search() into separate
cpu_search_lowest() and cpu_search_highest(), so split them completely,
making code more readable. While there, split iteration over children
groups and CPUs, complicating code for very small deduplication.
Stop passing cpuset_t arguments by value and avoid some manipulations.
Since MAXCPU bump from 64 to 256, what was a single register turned
into 32-byte memory array, requiring memory allocation and accesses.
Splitting struct cpu_search into parameter and result parts allows to
even more reduce stack usage, since the first can be passed through
on recursion.
Remove CPU_FFS() from the hot paths, precalculating first and last CPU
for each CPU group in advance during initialization. Again, it was
not a problem for 64 CPUs before, but for 256 FFS needs much more code.
With these changes on 80-thread system doing ~260K uncached ZFS reads
per second I observe ~30% reduction of time spent in cpu_search_*().
MFC after: 1 month
2021-07-29 01:18:50 +00:00
|
|
|
smp_topo_fill(top);
|
2008-03-02 07:58:42 +00:00
|
|
|
return (top);
|
2003-12-03 14:55:31 +00:00
|
|
|
}
|
2008-03-02 07:58:42 +00:00
|
|
|
|
2016-04-04 16:09:29 +00:00
|
|
|
struct cpu_group *
|
|
|
|
smp_topo_alloc(u_int count)
|
|
|
|
{
|
2023-05-05 20:24:22 +00:00
|
|
|
static struct cpu_group *group = NULL;
|
2016-04-04 16:09:29 +00:00
|
|
|
static u_int index;
|
|
|
|
u_int curr;
|
|
|
|
|
2023-05-05 20:24:22 +00:00
|
|
|
if (group == NULL) {
|
|
|
|
group = mallocarray((mp_maxid + 1) * MAX_CACHE_LEVELS + 1,
|
|
|
|
sizeof(*group), M_DEVBUF, M_WAITOK | M_ZERO);
|
|
|
|
}
|
2016-04-04 16:09:29 +00:00
|
|
|
curr = index;
|
|
|
|
index += count;
|
|
|
|
return (&group[curr]);
|
|
|
|
}
|
|
|
|
|
2008-03-02 07:58:42 +00:00
|
|
|
struct cpu_group *
|
|
|
|
smp_topo_none(void)
|
|
|
|
{
|
|
|
|
struct cpu_group *top;
|
|
|
|
|
2023-05-05 20:24:22 +00:00
|
|
|
top = smp_topo_alloc(1);
|
2008-03-02 07:58:42 +00:00
|
|
|
top->cg_parent = NULL;
|
|
|
|
top->cg_child = NULL;
|
2011-02-11 22:43:10 +00:00
|
|
|
top->cg_mask = all_cpus;
|
2008-03-02 07:58:42 +00:00
|
|
|
top->cg_count = mp_ncpus;
|
|
|
|
top->cg_children = 0;
|
|
|
|
top->cg_level = CG_SHARE_NONE;
|
|
|
|
top->cg_flags = 0;
|
2020-09-01 22:12:32 +00:00
|
|
|
|
2008-03-02 07:58:42 +00:00
|
|
|
return (top);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
smp_topo_addleaf(struct cpu_group *parent, struct cpu_group *child, int share,
|
|
|
|
int count, int flags, int start)
|
|
|
|
{
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
char cpusetbuf[CPUSETBUFSIZ], cpusetbuf2[CPUSETBUFSIZ];
|
|
|
|
cpuset_t mask;
|
2008-03-02 07:58:42 +00:00
|
|
|
int i;
|
|
|
|
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
CPU_ZERO(&mask);
|
|
|
|
for (i = 0; i < count; i++, start++)
|
|
|
|
CPU_SET(start, &mask);
|
2008-03-02 07:58:42 +00:00
|
|
|
child->cg_parent = parent;
|
|
|
|
child->cg_child = NULL;
|
|
|
|
child->cg_children = 0;
|
|
|
|
child->cg_level = share;
|
|
|
|
child->cg_count = count;
|
|
|
|
child->cg_flags = flags;
|
|
|
|
child->cg_mask = mask;
|
|
|
|
parent->cg_children++;
|
|
|
|
for (; parent != NULL; parent = parent->cg_parent) {
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
if (CPU_OVERLAP(&parent->cg_mask, &child->cg_mask))
|
|
|
|
panic("Duplicate children in %p. mask (%s) child (%s)",
|
|
|
|
parent,
|
|
|
|
cpusetobj_strprint(cpusetbuf, &parent->cg_mask),
|
|
|
|
cpusetobj_strprint(cpusetbuf2, &child->cg_mask));
|
Make CPU_SET macros compliant with other implementations
The introduction of <sched.h> improved compatibility with some 3rd
party software, but caused the configure scripts of some ports to
assume that they were run in a GLIBC compatible environment.
Parts of sched.h were made conditional on -D_WITH_CPU_SET_T being
added to ports, but there still were compatibility issues due to
invalid assumptions made in autoconfigure scripts.
The differences between the FreeBSD version of macros like CPU_AND,
CPU_OR, etc. and the GLIBC versions was in the number of arguments:
FreeBSD used a 2-address scheme (one source argument is also used as
the destination of the operation), while GLIBC uses a 3-adderess
scheme (2 source operands and a separately passed destination).
The GLIBC scheme provides a super-set of the functionality of the
FreeBSD macros, since it does not prevent passing the same variable
as source and destination arguments. In code that wanted to preserve
both source arguments, the FreeBSD macros required a temporary copy of
one of the source arguments.
This patch set allows to unconditionally provide functions and macros
expected by 3rd party software written for GLIBC based systems, but
breaks builds of externally maintained sources that use any of the
following macros: CPU_AND, CPU_ANDNOT, CPU_OR, CPU_XOR.
One contributed driver (contrib/ofed/libmlx5) has been patched to
support both the old and the new CPU_OR signatures. If this commit
is merged to -STABLE, the version test will have to be extended to
cover more ranges.
Ports that have added -D_WITH_CPU_SET_T to build on -CURRENT do
no longer require that option.
The FreeBSD version has been bumped to 1400046 to reflect this
incompatible change.
Reviewed by: kib
MFC after: 2 weeks
Relnotes: yes
Differential Revision: https://reviews.freebsd.org/D33451
2021-12-30 11:20:32 +00:00
|
|
|
CPU_OR(&parent->cg_mask, &parent->cg_mask, &child->cg_mask);
|
2008-03-02 07:58:42 +00:00
|
|
|
parent->cg_count += child->cg_count;
|
|
|
|
}
|
|
|
|
|
|
|
|
return (start);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct cpu_group *
|
|
|
|
smp_topo_1level(int share, int count, int flags)
|
|
|
|
{
|
|
|
|
struct cpu_group *child;
|
|
|
|
struct cpu_group *top;
|
|
|
|
int packages;
|
|
|
|
int cpu;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
cpu = 0;
|
2023-05-05 20:24:22 +00:00
|
|
|
top = smp_topo_alloc(1);
|
2008-03-02 07:58:42 +00:00
|
|
|
packages = mp_ncpus / count;
|
2023-05-05 20:24:22 +00:00
|
|
|
top->cg_child = child = top + 1;
|
2008-03-02 07:58:42 +00:00
|
|
|
top->cg_level = CG_SHARE_NONE;
|
|
|
|
for (i = 0; i < packages; i++, child++)
|
|
|
|
cpu = smp_topo_addleaf(top, child, share, count, flags, cpu);
|
|
|
|
return (top);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct cpu_group *
|
|
|
|
smp_topo_2level(int l2share, int l2count, int l1share, int l1count,
|
|
|
|
int l1flags)
|
|
|
|
{
|
|
|
|
struct cpu_group *top;
|
|
|
|
struct cpu_group *l1g;
|
|
|
|
struct cpu_group *l2g;
|
|
|
|
int cpu;
|
|
|
|
int i;
|
|
|
|
int j;
|
|
|
|
|
|
|
|
cpu = 0;
|
2023-05-05 20:24:22 +00:00
|
|
|
top = smp_topo_alloc(1);
|
|
|
|
l2g = top + 1;
|
2008-03-02 07:58:42 +00:00
|
|
|
top->cg_child = l2g;
|
|
|
|
top->cg_level = CG_SHARE_NONE;
|
|
|
|
top->cg_children = mp_ncpus / (l2count * l1count);
|
|
|
|
l1g = l2g + top->cg_children;
|
|
|
|
for (i = 0; i < top->cg_children; i++, l2g++) {
|
|
|
|
l2g->cg_parent = top;
|
|
|
|
l2g->cg_child = l1g;
|
|
|
|
l2g->cg_level = l2share;
|
|
|
|
for (j = 0; j < l2count; j++, l1g++)
|
|
|
|
cpu = smp_topo_addleaf(l2g, l1g, l1share, l1count,
|
|
|
|
l1flags, cpu);
|
|
|
|
}
|
|
|
|
return (top);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct cpu_group *
|
|
|
|
smp_topo_find(struct cpu_group *top, int cpu)
|
|
|
|
{
|
|
|
|
struct cpu_group *cg;
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
cpuset_t mask;
|
2008-03-02 07:58:42 +00:00
|
|
|
int children;
|
|
|
|
int i;
|
|
|
|
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
CPU_SETOF(cpu, &mask);
|
2008-03-02 07:58:42 +00:00
|
|
|
cg = top;
|
|
|
|
for (;;) {
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
if (!CPU_OVERLAP(&cg->cg_mask, &mask))
|
2008-03-02 07:58:42 +00:00
|
|
|
return (NULL);
|
|
|
|
if (cg->cg_children == 0)
|
|
|
|
return (cg);
|
|
|
|
children = cg->cg_children;
|
|
|
|
for (i = 0, cg = cg->cg_child; i < children; cg++, i++)
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
if (CPU_OVERLAP(&cg->cg_mask, &mask))
|
2008-03-02 07:58:42 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
#else /* !SMP */
|
2003-12-03 14:55:31 +00:00
|
|
|
|
2008-05-23 04:05:26 +00:00
|
|
|
void
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
smp_rendezvous_cpus(cpuset_t map,
|
2008-05-23 04:05:26 +00:00
|
|
|
void (*setup_func)(void *),
|
|
|
|
void (*action_func)(void *),
|
|
|
|
void (*teardown_func)(void *),
|
|
|
|
void *arg)
|
|
|
|
{
|
2011-11-03 14:36:56 +00:00
|
|
|
/*
|
|
|
|
* In the !SMP case we just need to ensure the same initial conditions
|
|
|
|
* as the SMP case.
|
|
|
|
*/
|
|
|
|
spinlock_enter();
|
2008-05-23 04:05:26 +00:00
|
|
|
if (setup_func != NULL)
|
|
|
|
setup_func(arg);
|
|
|
|
if (action_func != NULL)
|
|
|
|
action_func(arg);
|
|
|
|
if (teardown_func != NULL)
|
|
|
|
teardown_func(arg);
|
2011-11-03 14:36:56 +00:00
|
|
|
spinlock_exit();
|
2008-05-23 04:05:26 +00:00
|
|
|
}
|
|
|
|
|
2003-12-03 14:55:31 +00:00
|
|
|
void
|
2008-01-02 17:09:15 +00:00
|
|
|
smp_rendezvous(void (*setup_func)(void *),
|
|
|
|
void (*action_func)(void *),
|
|
|
|
void (*teardown_func)(void *),
|
2003-12-03 14:55:31 +00:00
|
|
|
void *arg)
|
|
|
|
{
|
|
|
|
|
2016-08-30 19:26:07 +00:00
|
|
|
smp_rendezvous_cpus(all_cpus, setup_func, action_func, teardown_func,
|
|
|
|
arg);
|
2003-12-03 14:55:31 +00:00
|
|
|
}
|
2008-03-02 07:58:42 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Provide dummy SMP support for UP kernels. Modules that need to use SMP
|
|
|
|
* APIs will still work using this dummy support.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
mp_setvariables_for_up(void *dummy)
|
|
|
|
{
|
|
|
|
mp_ncpus = 1;
|
2019-01-04 18:31:17 +00:00
|
|
|
mp_ncores = 1;
|
2008-03-02 07:58:42 +00:00
|
|
|
mp_maxid = PCPU_GET(cpuid);
|
2011-06-13 13:28:31 +00:00
|
|
|
CPU_SETOF(mp_maxid, &all_cpus);
|
2008-03-02 07:58:42 +00:00
|
|
|
KASSERT(PCPU_GET(cpuid) == 0, ("UP must have a CPU ID of zero"));
|
|
|
|
}
|
|
|
|
SYSINIT(cpu_mp_setvariables, SI_SUB_TUNABLES, SI_ORDER_FIRST,
|
2008-03-16 10:58:09 +00:00
|
|
|
mp_setvariables_for_up, NULL);
|
2003-12-03 14:55:31 +00:00
|
|
|
#endif /* SMP */
|
2008-05-23 04:05:26 +00:00
|
|
|
|
|
|
|
void
|
2017-04-09 02:00:03 +00:00
|
|
|
smp_no_rendezvous_barrier(void *dummy)
|
2008-05-23 04:05:26 +00:00
|
|
|
{
|
|
|
|
#ifdef SMP
|
2017-04-09 02:00:03 +00:00
|
|
|
KASSERT((!smp_started),("smp_no_rendezvous called and smp is started"));
|
2008-05-23 04:05:26 +00:00
|
|
|
#endif
|
|
|
|
}
|
2012-11-15 00:51:57 +00:00
|
|
|
|
2020-02-12 11:16:55 +00:00
|
|
|
void
|
|
|
|
smp_rendezvous_cpus_retry(cpuset_t map,
|
|
|
|
void (* setup_func)(void *),
|
|
|
|
void (* action_func)(void *),
|
|
|
|
void (* teardown_func)(void *),
|
|
|
|
void (* wait_func)(void *, int),
|
|
|
|
struct smp_rendezvous_cpus_retry_arg *arg)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
|
2021-05-03 16:43:00 +00:00
|
|
|
CPU_COPY(&map, &arg->cpus);
|
|
|
|
|
2020-11-19 04:27:51 +00:00
|
|
|
/*
|
|
|
|
* Only one CPU to execute on.
|
|
|
|
*/
|
|
|
|
if (!smp_started) {
|
|
|
|
spinlock_enter();
|
|
|
|
if (setup_func != NULL)
|
|
|
|
setup_func(arg);
|
|
|
|
if (action_func != NULL)
|
|
|
|
action_func(arg);
|
|
|
|
if (teardown_func != NULL)
|
|
|
|
teardown_func(arg);
|
|
|
|
spinlock_exit();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-02-12 11:16:55 +00:00
|
|
|
/*
|
|
|
|
* Execute an action on all specified CPUs while retrying until they
|
|
|
|
* all acknowledge completion.
|
|
|
|
*/
|
|
|
|
for (;;) {
|
|
|
|
smp_rendezvous_cpus(
|
|
|
|
arg->cpus,
|
|
|
|
setup_func,
|
|
|
|
action_func,
|
|
|
|
teardown_func,
|
|
|
|
arg);
|
|
|
|
|
|
|
|
if (CPU_EMPTY(&arg->cpus))
|
|
|
|
break;
|
|
|
|
|
|
|
|
CPU_FOREACH(cpu) {
|
|
|
|
if (!CPU_ISSET(cpu, &arg->cpus))
|
|
|
|
continue;
|
|
|
|
wait_func(arg, cpu);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
smp_rendezvous_cpus_done(struct smp_rendezvous_cpus_retry_arg *arg)
|
|
|
|
{
|
|
|
|
|
|
|
|
CPU_CLR_ATOMIC(curcpu, &arg->cpus);
|
|
|
|
}
|
|
|
|
|
2012-11-15 00:51:57 +00:00
|
|
|
/*
|
2021-05-28 17:10:47 +00:00
|
|
|
* If (prio & PDROP) == 0:
|
2017-08-15 02:21:02 +00:00
|
|
|
* Wait for specified idle threads to switch once. This ensures that even
|
2012-11-15 00:51:57 +00:00
|
|
|
* preempted threads have cycled through the switch function once,
|
|
|
|
* exiting their codepaths. This allows us to change global pointers
|
|
|
|
* with no other synchronization.
|
2021-05-28 17:10:47 +00:00
|
|
|
* If (prio & PDROP) != 0:
|
|
|
|
* Force the specified CPUs to switch context at least once.
|
2012-11-15 00:51:57 +00:00
|
|
|
*/
|
|
|
|
int
|
|
|
|
quiesce_cpus(cpuset_t map, const char *wmesg, int prio)
|
|
|
|
{
|
|
|
|
struct pcpu *pcpu;
|
2021-05-28 17:10:47 +00:00
|
|
|
u_int *gen;
|
2012-11-15 00:51:57 +00:00
|
|
|
int error;
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
error = 0;
|
2021-05-28 17:10:47 +00:00
|
|
|
if ((prio & PDROP) == 0) {
|
2023-05-05 20:21:38 +00:00
|
|
|
gen = mallocarray(sizeof(u_int), mp_maxid + 1, M_TEMP,
|
|
|
|
M_WAITOK);
|
2021-05-28 17:10:47 +00:00
|
|
|
for (cpu = 0; cpu <= mp_maxid; cpu++) {
|
|
|
|
if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu))
|
|
|
|
continue;
|
|
|
|
pcpu = pcpu_find(cpu);
|
|
|
|
gen[cpu] = pcpu->pc_idlethread->td_generation;
|
|
|
|
}
|
2012-11-15 00:51:57 +00:00
|
|
|
}
|
|
|
|
for (cpu = 0; cpu <= mp_maxid; cpu++) {
|
|
|
|
if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu))
|
|
|
|
continue;
|
|
|
|
pcpu = pcpu_find(cpu);
|
|
|
|
thread_lock(curthread);
|
|
|
|
sched_bind(curthread, cpu);
|
|
|
|
thread_unlock(curthread);
|
2021-05-28 17:10:47 +00:00
|
|
|
if ((prio & PDROP) != 0)
|
|
|
|
continue;
|
2012-11-15 00:51:57 +00:00
|
|
|
while (gen[cpu] == pcpu->pc_idlethread->td_generation) {
|
2021-05-28 17:10:47 +00:00
|
|
|
error = tsleep(quiesce_cpus, prio & ~PDROP, wmesg, 1);
|
2012-12-19 20:08:06 +00:00
|
|
|
if (error != EWOULDBLOCK)
|
2012-11-15 00:51:57 +00:00
|
|
|
goto out;
|
2012-12-19 20:08:06 +00:00
|
|
|
error = 0;
|
2012-11-15 00:51:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
out:
|
|
|
|
thread_lock(curthread);
|
|
|
|
sched_unbind(curthread);
|
|
|
|
thread_unlock(curthread);
|
2021-05-28 17:10:47 +00:00
|
|
|
if ((prio & PDROP) == 0)
|
|
|
|
free(gen, M_TEMP);
|
2012-11-15 00:51:57 +00:00
|
|
|
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
quiesce_all_cpus(const char *wmesg, int prio)
|
|
|
|
{
|
|
|
|
|
|
|
|
return quiesce_cpus(all_cpus, wmesg, prio);
|
|
|
|
}
|
2014-04-26 20:27:54 +00:00
|
|
|
|
2019-11-30 17:22:10 +00:00
|
|
|
/*
|
|
|
|
* Observe all CPUs not executing in critical section.
|
|
|
|
* We are not in one so the check for us is safe. If the found
|
|
|
|
* thread changes to something else we know the section was
|
|
|
|
* exited as well.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
quiesce_all_critical(void)
|
|
|
|
{
|
|
|
|
struct thread *td, *newtd;
|
|
|
|
struct pcpu *pcpu;
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
MPASS(curthread->td_critnest == 0);
|
|
|
|
|
|
|
|
CPU_FOREACH(cpu) {
|
|
|
|
pcpu = cpuid_to_pcpu[cpu];
|
|
|
|
td = pcpu->pc_curthread;
|
|
|
|
for (;;) {
|
|
|
|
if (td->td_critnest == 0)
|
|
|
|
break;
|
|
|
|
cpu_spinwait();
|
|
|
|
newtd = (struct thread *)
|
2019-11-30 19:33:02 +00:00
|
|
|
atomic_load_acq_ptr((void *)pcpu->pc_curthread);
|
2019-11-30 17:22:10 +00:00
|
|
|
if (td != newtd)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
cpus_fence_seq_cst_issue(void *arg __unused)
|
|
|
|
{
|
|
|
|
|
|
|
|
atomic_thread_fence_seq_cst();
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Send an IPI forcing a sequentially consistent fence.
|
|
|
|
*
|
|
|
|
* Allows replacement of an explicitly fence with a compiler barrier.
|
|
|
|
* Trades speed up during normal execution for a significant slowdown when
|
|
|
|
* the barrier is needed.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
cpus_fence_seq_cst(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
#ifdef SMP
|
|
|
|
smp_rendezvous(
|
|
|
|
smp_no_rendezvous_barrier,
|
|
|
|
cpus_fence_seq_cst_issue,
|
|
|
|
smp_no_rendezvous_barrier,
|
|
|
|
NULL
|
|
|
|
);
|
|
|
|
#else
|
|
|
|
cpus_fence_seq_cst_issue(NULL);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2014-04-26 20:27:54 +00:00
|
|
|
/* Extra care is taken with this sysctl because the data type is volatile */
|
|
|
|
static int
|
|
|
|
sysctl_kern_smp_active(SYSCTL_HANDLER_ARGS)
|
|
|
|
{
|
|
|
|
int error, active;
|
|
|
|
|
|
|
|
active = smp_started;
|
|
|
|
error = SYSCTL_OUT(req, &active, sizeof(active));
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
2016-04-04 16:09:29 +00:00
|
|
|
#ifdef SMP
|
|
|
|
void
|
|
|
|
topo_init_node(struct topo_node *node)
|
|
|
|
{
|
|
|
|
|
|
|
|
bzero(node, sizeof(*node));
|
|
|
|
TAILQ_INIT(&node->children);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
topo_init_root(struct topo_node *root)
|
|
|
|
{
|
|
|
|
|
|
|
|
topo_init_node(root);
|
|
|
|
root->type = TOPO_TYPE_SYSTEM;
|
|
|
|
}
|
|
|
|
|
2016-04-05 10:36:40 +00:00
|
|
|
/*
|
|
|
|
* Add a child node with the given ID under the given parent.
|
|
|
|
* Do nothing if there is already a child with that ID.
|
|
|
|
*/
|
2016-04-04 16:09:29 +00:00
|
|
|
struct topo_node *
|
|
|
|
topo_add_node_by_hwid(struct topo_node *parent, int hwid,
|
|
|
|
topo_node_type type, uintptr_t subtype)
|
|
|
|
{
|
|
|
|
struct topo_node *node;
|
|
|
|
|
|
|
|
TAILQ_FOREACH_REVERSE(node, &parent->children,
|
|
|
|
topo_children, siblings) {
|
|
|
|
if (node->hwid == hwid
|
|
|
|
&& node->type == type && node->subtype == subtype) {
|
|
|
|
return (node);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
node = malloc(sizeof(*node), M_TOPO, M_WAITOK);
|
|
|
|
topo_init_node(node);
|
|
|
|
node->parent = parent;
|
|
|
|
node->hwid = hwid;
|
|
|
|
node->type = type;
|
|
|
|
node->subtype = subtype;
|
|
|
|
TAILQ_INSERT_TAIL(&parent->children, node, siblings);
|
|
|
|
parent->nchildren++;
|
|
|
|
|
|
|
|
return (node);
|
|
|
|
}
|
|
|
|
|
2016-04-05 10:36:40 +00:00
|
|
|
/*
|
|
|
|
* Find a child node with the given ID under the given parent.
|
|
|
|
*/
|
2016-04-04 16:09:29 +00:00
|
|
|
struct topo_node *
|
|
|
|
topo_find_node_by_hwid(struct topo_node *parent, int hwid,
|
|
|
|
topo_node_type type, uintptr_t subtype)
|
|
|
|
{
|
|
|
|
|
|
|
|
struct topo_node *node;
|
|
|
|
|
|
|
|
TAILQ_FOREACH(node, &parent->children, siblings) {
|
|
|
|
if (node->hwid == hwid
|
|
|
|
&& node->type == type && node->subtype == subtype) {
|
|
|
|
return (node);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
|
2016-04-05 10:36:40 +00:00
|
|
|
/*
|
|
|
|
* Given a node change the order of its parent's child nodes such
|
|
|
|
* that the node becomes the firt child while preserving the cyclic
|
|
|
|
* order of the children. In other words, the given node is promoted
|
|
|
|
* by rotation.
|
|
|
|
*/
|
2016-04-04 16:09:29 +00:00
|
|
|
void
|
|
|
|
topo_promote_child(struct topo_node *child)
|
|
|
|
{
|
|
|
|
struct topo_node *next;
|
|
|
|
struct topo_node *node;
|
|
|
|
struct topo_node *parent;
|
|
|
|
|
|
|
|
parent = child->parent;
|
|
|
|
next = TAILQ_NEXT(child, siblings);
|
|
|
|
TAILQ_REMOVE(&parent->children, child, siblings);
|
|
|
|
TAILQ_INSERT_HEAD(&parent->children, child, siblings);
|
|
|
|
|
|
|
|
while (next != NULL) {
|
|
|
|
node = next;
|
|
|
|
next = TAILQ_NEXT(node, siblings);
|
|
|
|
TAILQ_REMOVE(&parent->children, node, siblings);
|
|
|
|
TAILQ_INSERT_AFTER(&parent->children, child, node, siblings);
|
|
|
|
child = node;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-05 10:36:40 +00:00
|
|
|
/*
|
|
|
|
* Iterate to the next node in the depth-first search (traversal) of
|
|
|
|
* the topology tree.
|
|
|
|
*/
|
2016-04-04 16:09:29 +00:00
|
|
|
struct topo_node *
|
|
|
|
topo_next_node(struct topo_node *top, struct topo_node *node)
|
|
|
|
{
|
|
|
|
struct topo_node *next;
|
|
|
|
|
|
|
|
if ((next = TAILQ_FIRST(&node->children)) != NULL)
|
|
|
|
return (next);
|
|
|
|
|
|
|
|
if ((next = TAILQ_NEXT(node, siblings)) != NULL)
|
|
|
|
return (next);
|
|
|
|
|
2017-08-22 00:10:15 +00:00
|
|
|
while (node != top && (node = node->parent) != top)
|
2016-04-04 16:09:29 +00:00
|
|
|
if ((next = TAILQ_NEXT(node, siblings)) != NULL)
|
|
|
|
return (next);
|
|
|
|
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
|
2016-04-05 10:36:40 +00:00
|
|
|
/*
|
|
|
|
* Iterate to the next node in the depth-first search of the topology tree,
|
|
|
|
* but without descending below the current node.
|
|
|
|
*/
|
2016-04-04 16:09:29 +00:00
|
|
|
struct topo_node *
|
|
|
|
topo_next_nonchild_node(struct topo_node *top, struct topo_node *node)
|
|
|
|
{
|
|
|
|
struct topo_node *next;
|
|
|
|
|
|
|
|
if ((next = TAILQ_NEXT(node, siblings)) != NULL)
|
|
|
|
return (next);
|
|
|
|
|
2017-08-22 00:10:15 +00:00
|
|
|
while (node != top && (node = node->parent) != top)
|
2016-04-04 16:09:29 +00:00
|
|
|
if ((next = TAILQ_NEXT(node, siblings)) != NULL)
|
|
|
|
return (next);
|
|
|
|
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
|
2016-04-05 10:36:40 +00:00
|
|
|
/*
|
|
|
|
* Assign the given ID to the given topology node that represents a logical
|
|
|
|
* processor.
|
|
|
|
*/
|
2016-04-04 16:09:29 +00:00
|
|
|
void
|
|
|
|
topo_set_pu_id(struct topo_node *node, cpuid_t id)
|
|
|
|
{
|
|
|
|
|
|
|
|
KASSERT(node->type == TOPO_TYPE_PU,
|
|
|
|
("topo_set_pu_id: wrong node type: %u", node->type));
|
|
|
|
KASSERT(CPU_EMPTY(&node->cpuset) && node->cpu_count == 0,
|
|
|
|
("topo_set_pu_id: cpuset already not empty"));
|
|
|
|
node->id = id;
|
|
|
|
CPU_SET(id, &node->cpuset);
|
|
|
|
node->cpu_count = 1;
|
|
|
|
node->subtype = 1;
|
|
|
|
|
|
|
|
while ((node = node->parent) != NULL) {
|
2016-04-08 11:59:11 +00:00
|
|
|
KASSERT(!CPU_ISSET(id, &node->cpuset),
|
|
|
|
("logical ID %u is already set in node %p", id, node));
|
2016-04-04 16:09:29 +00:00
|
|
|
CPU_SET(id, &node->cpuset);
|
|
|
|
node->cpu_count++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-22 00:10:15 +00:00
|
|
|
static struct topology_spec {
|
|
|
|
topo_node_type type;
|
|
|
|
bool match_subtype;
|
|
|
|
uintptr_t subtype;
|
|
|
|
} topology_level_table[TOPO_LEVEL_COUNT] = {
|
|
|
|
[TOPO_LEVEL_PKG] = { .type = TOPO_TYPE_PKG, },
|
|
|
|
[TOPO_LEVEL_GROUP] = { .type = TOPO_TYPE_GROUP, },
|
|
|
|
[TOPO_LEVEL_CACHEGROUP] = {
|
|
|
|
.type = TOPO_TYPE_CACHE,
|
|
|
|
.match_subtype = true,
|
|
|
|
.subtype = CG_SHARE_L3,
|
|
|
|
},
|
|
|
|
[TOPO_LEVEL_CORE] = { .type = TOPO_TYPE_CORE, },
|
|
|
|
[TOPO_LEVEL_THREAD] = { .type = TOPO_TYPE_PU, },
|
|
|
|
};
|
|
|
|
|
|
|
|
static bool
|
|
|
|
topo_analyze_table(struct topo_node *root, int all, enum topo_level level,
|
|
|
|
struct topo_analysis *results)
|
2016-04-04 16:09:29 +00:00
|
|
|
{
|
2017-08-22 00:10:15 +00:00
|
|
|
struct topology_spec *spec;
|
|
|
|
struct topo_node *node;
|
|
|
|
int count;
|
|
|
|
|
|
|
|
if (level >= TOPO_LEVEL_COUNT)
|
|
|
|
return (true);
|
|
|
|
|
|
|
|
spec = &topology_level_table[level];
|
|
|
|
count = 0;
|
|
|
|
node = topo_next_node(root, root);
|
|
|
|
|
|
|
|
while (node != NULL) {
|
|
|
|
if (node->type != spec->type ||
|
|
|
|
(spec->match_subtype && node->subtype != spec->subtype)) {
|
|
|
|
node = topo_next_node(root, node);
|
2016-04-04 16:09:29 +00:00
|
|
|
continue;
|
|
|
|
}
|
2017-08-22 00:10:15 +00:00
|
|
|
if (!all && CPU_EMPTY(&node->cpuset)) {
|
|
|
|
node = topo_next_nonchild_node(root, node);
|
2016-04-04 16:09:29 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2017-08-22 00:10:15 +00:00
|
|
|
count++;
|
2016-04-04 16:09:29 +00:00
|
|
|
|
2017-08-22 00:10:15 +00:00
|
|
|
if (!topo_analyze_table(node, all, level + 1, results))
|
|
|
|
return (false);
|
2016-04-04 16:09:29 +00:00
|
|
|
|
2017-08-22 00:10:15 +00:00
|
|
|
node = topo_next_nonchild_node(root, node);
|
2016-04-04 16:09:29 +00:00
|
|
|
}
|
|
|
|
|
2017-08-22 00:10:15 +00:00
|
|
|
/* No explicit subgroups is essentially one subgroup. */
|
|
|
|
if (count == 0) {
|
|
|
|
count = 1;
|
|
|
|
|
|
|
|
if (!topo_analyze_table(root, all, level + 1, results))
|
|
|
|
return (false);
|
2016-04-04 16:09:29 +00:00
|
|
|
}
|
|
|
|
|
2017-08-22 00:10:15 +00:00
|
|
|
if (results->entities[level] == -1)
|
|
|
|
results->entities[level] = count;
|
|
|
|
else if (results->entities[level] != count)
|
|
|
|
return (false);
|
|
|
|
|
|
|
|
return (true);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check if the topology is uniform, that is, each package has the same number
|
|
|
|
* of cores in it and each core has the same number of threads (logical
|
|
|
|
* processors) in it. If so, calculate the number of packages, the number of
|
|
|
|
* groups per package, the number of cachegroups per group, and the number of
|
|
|
|
* logical processors per cachegroup. 'all' parameter tells whether to include
|
|
|
|
* administratively disabled logical processors into the analysis.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
topo_analyze(struct topo_node *topo_root, int all,
|
|
|
|
struct topo_analysis *results)
|
|
|
|
{
|
|
|
|
|
|
|
|
results->entities[TOPO_LEVEL_PKG] = -1;
|
|
|
|
results->entities[TOPO_LEVEL_CORE] = -1;
|
|
|
|
results->entities[TOPO_LEVEL_THREAD] = -1;
|
|
|
|
results->entities[TOPO_LEVEL_GROUP] = -1;
|
|
|
|
results->entities[TOPO_LEVEL_CACHEGROUP] = -1;
|
|
|
|
|
|
|
|
if (!topo_analyze_table(topo_root, all, TOPO_LEVEL_PKG, results))
|
|
|
|
return (0);
|
|
|
|
|
|
|
|
KASSERT(results->entities[TOPO_LEVEL_PKG] > 0,
|
|
|
|
("bug in topology or analysis"));
|
|
|
|
|
2016-04-04 16:09:29 +00:00
|
|
|
return (1);
|
|
|
|
}
|
2017-08-22 00:10:15 +00:00
|
|
|
|
2016-04-04 16:09:29 +00:00
|
|
|
#endif /* SMP */
|