powerpc/powernv: Add asynchronous token management for powernv

The OPAL firmware only supports a finite number of in-flight asynchronous
operations.  Rather than have each subsystem try to manage its own, use a
central management service to hand out tokens.

More work can be done to improve asynchronous behavior, such as funneling
things through a future OPAL heartbeat handler, but capabilities will be
added as needed.

Augment the existing consumers (i2c and sensors) to use this new API.

MFC after:	4 weeks

sys/conf/files.powerpc

@@ -189,6 +189,7 @@ powerpc/powermac/uninorth.c	optional	powermac
 powerpc/powermac/uninorthpci.c	optional	powermac pci
 powerpc/powermac/vcoregpio.c	optional	powermac 
 powerpc/powernv/opal.c		optional	powernv
+powerpc/powernv/opal_async.c	optional	powernv
 powerpc/powernv/opal_console.c	optional	powernv
 powerpc/powernv/opal_dev.c	optional	powernv
 powerpc/powernv/opal_i2c.c	optional	iicbus fdt powernv

sys/powerpc/powernv/opal.h

@@ -168,4 +168,9 @@ struct opal_ipmi_msg {
 	uint8_t data[];
 };
 
+int	opal_init_async_tokens(int);
+int	opal_alloc_async_token(void);
+void	opal_free_async_token(int);
+int	opal_wait_completion(void *, uint64_t, uint64_t);
+
 #endif

sys/powerpc/powernv/opal_async.c

@@ -0,0 +1,94 @@
+/*-
+ * Copyright (C) 2019 Justin Hibbits
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/vmem.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include "opal.h"
+
+/*
+ * Manage asynchronous tokens for the OPAL abstraction layer.
+ *
+ * Only a finite number of in-flight tokens are supported by OPAL, so we must be
+ * careful managing this.  The basic design uses the vmem subsystem as a general
+ * purpose allocator, with wrappers to manage expected behaviors and
+ * requirements.
+ */
+static vmem_t *async_token_pool;
+
+/* Setup the token pool. */
+int
+opal_init_async_tokens(int count)
+{
+	/* Only allow one initialization */
+	if (async_token_pool != NULL)
+		return (EINVAL);
+
+	async_token_pool = vmem_create("OPAL Async", 0, count, 1, 1,
+	    M_WAITOK | M_FIRSTFIT);
+
+	return (0);
+}
+
+int
+opal_alloc_async_token(void)
+{
+	vmem_addr_t token;
+
+	vmem_alloc(async_token_pool, 1, M_FIRSTFIT | M_WAITOK, &token);
+
+	return (token);
+}
+
+void
+opal_free_async_token(int token)
+{
+
+	vmem_free(async_token_pool, token, 1);
+}
+
+/*
+ * Wait for the operation watched by the token to complete.  Return the result
+ * of the operation, error if it returns early.
+ */
+int
+opal_wait_completion(void *buf, uint64_t size, uint64_t token)
+{
+	int err;
+
+	do {
+		err = opal_call(OPAL_CHECK_ASYNC_COMPLETION,
+		    vtophys(buf), size, token);
+	} while (err == OPAL_BUSY);
+
+	return (err);
+}

sys/powerpc/powernv/opal_dev.c

@@ -135,6 +135,7 @@ opaldev_attach(device_t dev)
 	device_t cdev;
 	uint64_t junk;
 	int i, rv;
+	uint32_t async_count;
 	struct ofw_bus_devinfo *dinfo;
 	struct resource *irq;
 
@@ -159,6 +160,10 @@ opaldev_attach(device_t dev)
 		    INTR_ENTROPY, NULL, opal_intr, (void *)rman_get_start(irq),
 		    NULL);
 
+	OF_getencprop(ofw_bus_get_node(dev), "opal-msg-async-num",
+	    &async_count, sizeof(async_count));
+	opal_init_async_tokens(async_count);
+
 	for (child = OF_child(ofw_bus_get_node(dev)); child != 0;
 	    child = OF_peer(child)) {
 		dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_WAITOK | M_ZERO);

sys/powerpc/powernv/opal_i2c.c

@@ -168,35 +168,28 @@ i2c_opal_send_request(uint32_t bus_id, struct opal_i2c_request *req)
 {
 	struct opal_msg msg;
 	uint64_t token;
-	uint64_t msg_addr;
 	int rc;
 
-	/*
-	 * XXX:
-	 * Async tokens should be managed globally. Since there are only a very
-	 * few places now, use a punning of the stack address of the message.
-	 */
-	token = (uintptr_t)&msg;
+	token = opal_alloc_async_token();
 
 	memset(&msg, 0, sizeof(msg));
-	msg_addr = pmap_kextract((vm_offset_t)&msg);
 
 	rc = opal_call(OPAL_I2C_REQUEST, token, bus_id,
-	    pmap_kextract((uint64_t)req));
+	    vtophys(req));
 	if (rc != OPAL_ASYNC_COMPLETION)
-		return (rc);
+		goto out;
 
-	do {
-		rc = opal_call(OPAL_CHECK_ASYNC_COMPLETION,
-		    msg_addr, sizeof(msg), token);
-	} while (rc == OPAL_BUSY);
+	rc = opal_wait_completion(&msg, sizeof(msg), token);
 
 	if (rc != OPAL_SUCCESS)
-		return (rc);
+		goto out;
 
 	rc = opal_get_async_rc(msg);
 
-	return rc;
+out:
+	opal_free_async_token(token);
+
+	return (rc);
 }
 
 static int

sys/powerpc/powernv/opal_sensor.c

@@ -95,33 +95,28 @@ opal_sensor_get_val(uint32_t key, uint64_t *val)
 {
 	struct opal_msg msg;
 	uint32_t val32;
-	int i, rv;
+	int rv, token;
 
-	rv = opal_call(OPAL_SENSOR_READ, key, key, vtophys(&val32));
+	token = opal_alloc_async_token();
+	rv = opal_call(OPAL_SENSOR_READ, key, token, vtophys(&val32));
 
 	if (rv == OPAL_ASYNC_COMPLETION) {
 		/* Sleep a little to let things settle. */
 		DELAY(100);
 		bzero(&msg, sizeof(msg));
-		i = 0;
-		do {
-			rv = opal_call(OPAL_CHECK_ASYNC_COMPLETION,
-			    vtophys(&msg), sizeof(msg), key);
-			/* Sleep for ~100us if necessary. */
-			if (rv == OPAL_BUSY)
-				DELAY(100);
-		} while (rv == OPAL_BUSY && ++i < 10);
-		if (rv != OPAL_SUCCESS)
-			return (EIO);
-		val32 = msg.params[0];
+		rv = opal_wait_completion(&msg, sizeof(msg), token);
+
+		if (rv == OPAL_SUCCESS)
+			val32 = msg.params[0];
 	}
 
-	if (rv != OPAL_SUCCESS)
-		return (EIO);
-
-	*val = val32;
+	if (rv == OPAL_SUCCESS)
+		*val = val32;
+	else
+		rv = EIO;
 	
-	return (0);
+	opal_free_async_token(token);
+	return (rv);
 }
 
 static int