From 12d053032b6bf51e0563c2720c82a4dba0cc8b82 Mon Sep 17 00:00:00 2001
From: Andrew Turner <andrew@FreeBSD.org>
Date: Tue, 29 Dec 2020 12:44:45 +0000
Subject: [PATCH] Support overflow count in hwpmc on arm64

We increment the overflow count when receiving an overflow interrupt
with special care to check if it happens while reading the event counter.

Sponsored by:	Innovate UK
---
 sys/dev/hwpmc/hwpmc_arm64.c | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/sys/dev/hwpmc/hwpmc_arm64.c b/sys/dev/hwpmc/hwpmc_arm64.c
index 4ae8645abaf6..15ea4f892ed0 100644
--- a/sys/dev/hwpmc/hwpmc_arm64.c
+++ b/sys/dev/hwpmc/hwpmc_arm64.c
@@ -195,6 +195,8 @@ arm64_read_pmc(int cpu, int ri, pmc_value_t *v)
 {
 	pmc_value_t tmp;
 	struct pmc *pm;
+	register_t s;
+	int reg;
 
 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
 	    ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
@@ -203,7 +205,23 @@ arm64_read_pmc(int cpu, int ri, pmc_value_t *v)
 
 	pm  = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc;
 
+	/*
+	 * Ensure we don't get interrupted while updating the overflow count.
+	 */
+	s = intr_disable();
 	tmp = arm64_pmcn_read(ri);
+	reg = (1 << ri);
+	if ((READ_SPECIALREG(pmovsclr_el0) & reg) != 0) {
+		/* Clear Overflow Flag */
+		WRITE_SPECIALREG(pmovsclr_el0, reg);
+		if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
+			pm->pm_overflowcnt++;
+
+		/* Reread counter in case we raced. */
+		tmp = arm64_pmcn_read(ri);
+	}
+	tmp += 0x100000000llu * pm->pm_overflowcnt;
+	intr_restore(s);
 
 	PMCDBG2(MDP, REA, 2, "arm64-read id=%d -> %jd", ri, tmp);
 	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
@@ -231,6 +249,7 @@ arm64_write_pmc(int cpu, int ri, pmc_value_t v)
 
 	PMCDBG3(MDP, WRI, 1, "arm64-write cpu=%d ri=%d v=%jx", cpu, ri, v);
 
+	pm->pm_overflowcnt = v >> 32;
 	arm64_pmcn_write(ri, v);
 
 	return 0;
@@ -342,9 +361,6 @@ arm64_intr(struct trapframe *tf)
 		pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc;
 		if (pm == NULL)
 			continue;
-		if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
-			continue;
-
 		/* Check if counter is overflowed */
 		reg = (1 << ri);
 		if ((READ_SPECIALREG(pmovsclr_el0) & reg) == 0)
@@ -355,6 +371,12 @@ arm64_intr(struct trapframe *tf)
 		isb();
 
 		retval = 1; /* Found an interrupting PMC. */
+
+		if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
+			pm->pm_overflowcnt += 1;
+			continue;
+		}
+
 		if (pm->pm_state != PMC_STATE_RUNNING)
 			continue;