Use atomic load & store for TSC frequency. It may be overkill for amd64 but
safer for i386 because it can be easily over 4 GHz now. More worse, it can be easily changed by user with 'machdep.tsc_freq' tunable (directly) or cpufreq(4) (indirectly). Note it is intentionally not used in performance critical paths to avoid performance regression (but we should, in theory). Alternatively, we may add "virtual TSC" with lower frequency if maximum frequency overflows 32 bits (and ignore possible incoherency as we do now).
This commit is contained in:
parent
cb379161b1
commit
3453537fa5
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=220433
@ -321,7 +321,8 @@ cpu_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
|
||||
break;
|
||||
case CPU_IVAR_NOMINAL_MHZ:
|
||||
if (tsc_is_invariant) {
|
||||
*result = (uintptr_t)(tsc_freq / 1000000);
|
||||
*result = (uintptr_t)(atomic_load_acq_64(&tsc_freq) /
|
||||
1000000);
|
||||
break;
|
||||
}
|
||||
/* FALLTHROUGH */
|
||||
|
@ -546,18 +546,19 @@ int
|
||||
cpu_est_clockrate(int cpu_id, uint64_t *rate)
|
||||
{
|
||||
register_t reg;
|
||||
uint64_t tsc1, tsc2;
|
||||
uint64_t freq, tsc1, tsc2;
|
||||
|
||||
if (pcpu_find(cpu_id) == NULL || rate == NULL)
|
||||
return (EINVAL);
|
||||
freq = atomic_load_acq_64(&tsc_freq);
|
||||
|
||||
/* If TSC is P-state invariant, DELAY(9) based logic fails. */
|
||||
if (tsc_is_invariant && tsc_freq != 0)
|
||||
if (tsc_is_invariant && freq != 0)
|
||||
return (EOPNOTSUPP);
|
||||
|
||||
/* If we're booting, trust the rate calibrated moments ago. */
|
||||
if (cold && tsc_freq != 0) {
|
||||
*rate = tsc_freq;
|
||||
if (cold && freq != 0) {
|
||||
*rate = freq;
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -586,7 +587,7 @@ cpu_est_clockrate(int cpu_id, uint64_t *rate)
|
||||
#endif
|
||||
|
||||
tsc2 -= tsc1;
|
||||
if (tsc_freq != 0) {
|
||||
if (freq != 0) {
|
||||
*rate = tsc2 * 1000;
|
||||
return (0);
|
||||
}
|
||||
|
@ -311,18 +311,22 @@ void
|
||||
startguprof(gp)
|
||||
struct gmonparam *gp;
|
||||
{
|
||||
uint64_t freq;
|
||||
|
||||
freq = atomic_load_acq_64(&tsc_freq);
|
||||
if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) {
|
||||
cputime_clock = CPUTIME_CLOCK_I8254;
|
||||
if (tsc_freq != 0 && mp_ncpus == 1)
|
||||
if (freq != 0 && mp_ncpus == 1)
|
||||
cputime_clock = CPUTIME_CLOCK_TSC;
|
||||
else
|
||||
cputime_clock = CPUTIME_CLOCK_I8254;
|
||||
}
|
||||
gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
|
||||
if (cputime_clock == CPUTIME_CLOCK_TSC) {
|
||||
gp->profrate = tsc_freq >> 1;
|
||||
gp->profrate = freq >> 1;
|
||||
cputime_prof_active = 1;
|
||||
}
|
||||
} else
|
||||
gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
|
||||
#if defined(PERFMON) && defined(I586_PMC_GUPROF)
|
||||
else if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
|
||||
if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
|
||||
if (perfmon_avail() &&
|
||||
perfmon_setup(0, cputime_clock_pmc_conf) == 0) {
|
||||
if (perfmon_start(0) != 0)
|
||||
|
@ -403,7 +403,7 @@ dtrace_gethrtime_init(void *arg)
|
||||
* Otherwise tick->time conversion will be inaccurate, but
|
||||
* will preserve monotonic property of TSC.
|
||||
*/
|
||||
tsc_f = tsc_freq;
|
||||
tsc_f = atomic_load_acq_64(&tsc_freq);
|
||||
|
||||
/*
|
||||
* The following line checks that nsec_scale calculated below
|
||||
|
@ -403,7 +403,7 @@ dtrace_gethrtime_init(void *arg)
|
||||
* Otherwise tick->time conversion will be inaccurate, but
|
||||
* will preserve monotonic property of TSC.
|
||||
*/
|
||||
tsc_f = tsc_freq;
|
||||
tsc_f = atomic_load_acq_64(&tsc_freq);
|
||||
|
||||
/*
|
||||
* The following line checks that nsec_scale calculated below
|
||||
|
@ -221,6 +221,7 @@ linprocfs_docpuinfo(PFS_FILL_ARGS)
|
||||
{
|
||||
int hw_model[2];
|
||||
char model[128];
|
||||
uint64_t freq;
|
||||
size_t size;
|
||||
int class, fqmhz, fqkhz;
|
||||
int i;
|
||||
@ -303,9 +304,10 @@ linprocfs_docpuinfo(PFS_FILL_ARGS)
|
||||
if (cpu_feature & (1 << i))
|
||||
sbuf_printf(sb, " %s", flags[i]);
|
||||
sbuf_cat(sb, "\n");
|
||||
if (class >= 5) {
|
||||
fqmhz = (tsc_freq + 4999) / 1000000;
|
||||
fqkhz = ((tsc_freq + 4999) / 10000) % 100;
|
||||
freq = atomic_load_acq_64(&tsc_freq);
|
||||
if (freq != 0) {
|
||||
fqmhz = (freq + 4999) / 1000000;
|
||||
fqkhz = ((freq + 4999) / 10000) % 100;
|
||||
sbuf_printf(sb,
|
||||
"cpu MHz\t\t: %d.%02d\n"
|
||||
"bogomips\t: %d.%02d\n",
|
||||
|
@ -929,7 +929,8 @@ init_machclk_setup(void)
|
||||
#if defined(__amd64__) || defined(__i386__)
|
||||
/* check if TSC is available */
|
||||
#ifdef __FreeBSD__
|
||||
if ((cpu_feature & CPUID_TSC) == 0 || tsc_freq == 0)
|
||||
if ((cpu_feature & CPUID_TSC) == 0 ||
|
||||
atomic_load_acq_64(&tsc_freq) == 0)
|
||||
#else
|
||||
if ((cpu_feature & CPUID_TSC) == 0)
|
||||
#endif
|
||||
@ -964,7 +965,7 @@ init_machclk(void)
|
||||
*/
|
||||
#if defined(__amd64__) || defined(__i386__)
|
||||
#ifdef __FreeBSD__
|
||||
machclk_freq = tsc_freq;
|
||||
machclk_freq = atomic_load_acq_64(&tsc_freq);
|
||||
#elif defined(__NetBSD__)
|
||||
machclk_freq = (u_int32_t)cpu_tsc_freq;
|
||||
#elif defined(__OpenBSD__) && (defined(I586_CPU) || defined(I686_CPU))
|
||||
|
@ -516,7 +516,7 @@ acpi_cpu_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
|
||||
#if defined(__amd64__) || defined(__i386__)
|
||||
case CPU_IVAR_NOMINAL_MHZ:
|
||||
if (tsc_is_invariant) {
|
||||
*result = (uintptr_t)(tsc_freq / 1000000);
|
||||
*result = (uintptr_t)(atomic_load_acq_64(&tsc_freq) / 1000000);
|
||||
break;
|
||||
}
|
||||
/* FALLTHROUGH */
|
||||
|
@ -342,7 +342,8 @@ cpu_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
|
||||
break;
|
||||
case CPU_IVAR_NOMINAL_MHZ:
|
||||
if (tsc_is_invariant) {
|
||||
*result = (uintptr_t)(tsc_freq / 1000000);
|
||||
*result = (uintptr_t)(atomic_load_acq_64(&tsc_freq) /
|
||||
1000000);
|
||||
break;
|
||||
}
|
||||
/* FALLTHROUGH */
|
||||
|
@ -1137,20 +1137,21 @@ int
|
||||
cpu_est_clockrate(int cpu_id, uint64_t *rate)
|
||||
{
|
||||
register_t reg;
|
||||
uint64_t tsc1, tsc2;
|
||||
uint64_t freq, tsc1, tsc2;
|
||||
|
||||
if (pcpu_find(cpu_id) == NULL || rate == NULL)
|
||||
return (EINVAL);
|
||||
if ((cpu_feature & CPUID_TSC) == 0)
|
||||
return (EOPNOTSUPP);
|
||||
freq = atomic_load_acq_64(&tsc_freq);
|
||||
|
||||
/* If TSC is P-state invariant, DELAY(9) based logic fails. */
|
||||
if (tsc_is_invariant && tsc_freq != 0)
|
||||
if (tsc_is_invariant && freq != 0)
|
||||
return (EOPNOTSUPP);
|
||||
|
||||
/* If we're booting, trust the rate calibrated moments ago. */
|
||||
if (cold && tsc_freq != 0) {
|
||||
*rate = tsc_freq;
|
||||
if (cold && freq != 0) {
|
||||
*rate = freq;
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -1179,7 +1180,7 @@ cpu_est_clockrate(int cpu_id, uint64_t *rate)
|
||||
#endif
|
||||
|
||||
tsc2 -= tsc1;
|
||||
if (tsc_freq != 0) {
|
||||
if (freq != 0) {
|
||||
*rate = tsc2 * 1000;
|
||||
return (0);
|
||||
}
|
||||
|
@ -336,6 +336,7 @@ perfmon_ioctl(struct cdev *dev, u_long cmd, caddr_t param, int flags, struct thr
|
||||
struct pmc *pmc;
|
||||
struct pmc_data *pmcd;
|
||||
struct pmc_tstamp *pmct;
|
||||
uint64_t freq;
|
||||
int *ip;
|
||||
int rv;
|
||||
|
||||
@ -386,13 +387,14 @@ perfmon_ioctl(struct cdev *dev, u_long cmd, caddr_t param, int flags, struct thr
|
||||
break;
|
||||
|
||||
case PMIOTSTAMP:
|
||||
if (!tsc_freq) {
|
||||
freq = atomic_load_acq_64(&tsc_freq);
|
||||
if (freq == 0) {
|
||||
rv = ENOTTY;
|
||||
break;
|
||||
}
|
||||
pmct = (struct pmc_tstamp *)param;
|
||||
/* XXX interface loses precision. */
|
||||
pmct->pmct_rate = tsc_freq / 1000000;
|
||||
pmct->pmct_rate = freq / 1000000;
|
||||
pmct->pmct_value = rdtsc();
|
||||
rv = 0;
|
||||
break;
|
||||
|
@ -170,8 +170,8 @@ cputime()
|
||||
{
|
||||
u_int count;
|
||||
int delta;
|
||||
#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) && \
|
||||
defined(PERFMON) && defined(I586_PMC_GUPROF)
|
||||
#if (defined(I586_CPU) || defined(I686_CPU)) && \
|
||||
defined(PERFMON) && defined(I586_PMC_GUPROF) && !defined(SMP)
|
||||
u_quad_t event_count;
|
||||
#endif
|
||||
u_char high, low;
|
||||
@ -286,21 +286,23 @@ void
|
||||
startguprof(gp)
|
||||
struct gmonparam *gp;
|
||||
{
|
||||
#if defined(I586_CPU) || defined(I686_CPU)
|
||||
uint64_t freq;
|
||||
|
||||
freq = atomic_load_acq_64(&tsc_freq);
|
||||
if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) {
|
||||
cputime_clock = CPUTIME_CLOCK_I8254;
|
||||
#if defined(I586_CPU) || defined(I686_CPU)
|
||||
if (tsc_freq != 0 && mp_ncpus == 1)
|
||||
if (freq != 0 && mp_ncpus == 1)
|
||||
cputime_clock = CPUTIME_CLOCK_TSC;
|
||||
#endif
|
||||
else
|
||||
cputime_clock = CPUTIME_CLOCK_I8254;
|
||||
}
|
||||
gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
|
||||
#if defined(I586_CPU) || defined(I686_CPU)
|
||||
if (cputime_clock == CPUTIME_CLOCK_TSC) {
|
||||
gp->profrate = tsc_freq >> 1;
|
||||
gp->profrate = freq >> 1;
|
||||
cputime_prof_active = 1;
|
||||
}
|
||||
} else
|
||||
gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
|
||||
#if defined(PERFMON) && defined(I586_PMC_GUPROF)
|
||||
else if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
|
||||
if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
|
||||
if (perfmon_avail() &&
|
||||
perfmon_setup(0, cputime_clock_pmc_conf) == 0) {
|
||||
if (perfmon_start(0) != 0)
|
||||
@ -325,6 +327,10 @@ startguprof(gp)
|
||||
}
|
||||
}
|
||||
#endif /* PERFMON && I586_PMC_GUPROF */
|
||||
#else /* !(I586_CPU || I686_CPU) */
|
||||
if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED)
|
||||
cputime_clock = CPUTIME_CLOCK_I8254;
|
||||
gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
|
||||
#endif /* I586_CPU || I686_CPU */
|
||||
cputime_bias = 0;
|
||||
cputime();
|
||||
|
@ -1072,16 +1072,17 @@ int
|
||||
cpu_est_clockrate(int cpu_id, uint64_t *rate)
|
||||
{
|
||||
register_t reg;
|
||||
uint64_t tsc1, tsc2;
|
||||
uint64_t freq, tsc1, tsc2;
|
||||
|
||||
if (pcpu_find(cpu_id) == NULL || rate == NULL)
|
||||
return (EINVAL);
|
||||
if ((cpu_feature & CPUID_TSC) == 0)
|
||||
return (EOPNOTSUPP);
|
||||
freq = atomic_load_acq_64(&tsc_freq);
|
||||
|
||||
/* If we're booting, trust the rate calibrated moments ago. */
|
||||
if (cold && tsc_freq != 0) {
|
||||
*rate = tsc_freq;
|
||||
if (cold && freq != 0) {
|
||||
*rate = freq;
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -1110,7 +1111,7 @@ cpu_est_clockrate(int cpu_id, uint64_t *rate)
|
||||
#endif
|
||||
|
||||
tsc2 -= tsc1;
|
||||
if (tsc_freq != 0) {
|
||||
if (freq != 0) {
|
||||
*rate = tsc2 * 1000;
|
||||
return (0);
|
||||
}
|
||||
|
@ -1215,7 +1215,7 @@ est_msr_info(device_t dev, uint64_t msr, freq_info **freqs)
|
||||
return (EOPNOTSUPP);
|
||||
|
||||
/* Figure out the bus clock. */
|
||||
freq = tsc_freq / 1000000;
|
||||
freq = atomic_load_acq_64(&tsc_freq) / 1000000;
|
||||
id = msr >> 32;
|
||||
bus = freq / (id >> 8);
|
||||
device_printf(dev, "Guessed bus clock (high) of %d MHz\n", bus);
|
||||
|
@ -246,13 +246,13 @@ getit(void)
|
||||
}
|
||||
|
||||
static __inline void
|
||||
delay_tsc(int n)
|
||||
delay_tsc(int n, uint64_t freq)
|
||||
{
|
||||
uint64_t start, end, now;
|
||||
|
||||
sched_pin();
|
||||
start = rdtsc();
|
||||
end = start + (tsc_freq * n) / 1000000;
|
||||
end = start + (freq * n) / 1000000;
|
||||
do {
|
||||
cpu_spinwait();
|
||||
now = rdtsc();
|
||||
@ -290,6 +290,7 @@ void
|
||||
DELAY(int n)
|
||||
{
|
||||
struct timecounter *tc;
|
||||
uint64_t freq;
|
||||
int delta, prev_tick, tick, ticks_left;
|
||||
|
||||
#ifdef DELAYDEBUG
|
||||
@ -298,8 +299,9 @@ DELAY(int n)
|
||||
static int state = 0;
|
||||
#endif
|
||||
|
||||
if (tsc_freq != 0) {
|
||||
delay_tsc(n);
|
||||
freq = atomic_load_acq_64(&tsc_freq);
|
||||
if (freq != 0) {
|
||||
delay_tsc(n, freq);
|
||||
return;
|
||||
}
|
||||
tc = timecounter;
|
||||
|
@ -245,14 +245,16 @@ tsc_freq_changing(void *arg, const struct cf_level *level, int *status)
|
||||
static void
|
||||
tsc_freq_changed(void *arg, const struct cf_level *level, int status)
|
||||
{
|
||||
uint64_t freq;
|
||||
|
||||
/* If there was an error during the transition, don't do anything. */
|
||||
if (tsc_disabled || status != 0)
|
||||
return;
|
||||
|
||||
/* Total setting for this level gives the new frequency in MHz. */
|
||||
tsc_freq = (uint64_t)level->total_set.freq * 1000000;
|
||||
tsc_timecounter.tc_frequency = tsc_freq;
|
||||
freq = (uint64_t)level->total_set.freq * 1000000;
|
||||
atomic_store_rel_64(&tsc_freq, freq);
|
||||
atomic_store_rel_64(&tsc_timecounter.tc_frequency, freq);
|
||||
}
|
||||
|
||||
static int
|
||||
@ -261,13 +263,13 @@ sysctl_machdep_tsc_freq(SYSCTL_HANDLER_ARGS)
|
||||
int error;
|
||||
uint64_t freq;
|
||||
|
||||
if (tsc_timecounter.tc_frequency == 0)
|
||||
freq = atomic_load_acq_64(&tsc_freq);
|
||||
if (freq == 0)
|
||||
return (EOPNOTSUPP);
|
||||
freq = tsc_freq;
|
||||
error = sysctl_handle_64(oidp, &freq, 0, req);
|
||||
if (error == 0 && req->newptr != NULL) {
|
||||
tsc_freq = freq;
|
||||
tsc_timecounter.tc_frequency = tsc_freq;
|
||||
atomic_store_rel_64(&tsc_freq, freq);
|
||||
atomic_store_rel_64(&tsc_timecounter.tc_frequency, freq);
|
||||
}
|
||||
return (error);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user