From 98844e99d40a90ae89d84762e07150af3a8f89bd Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Wed, 15 Feb 2023 13:32:52 -0800
Subject: [PATCH] aio: Fix more synchronization issues in aio_biowakeup.

- Use atomic_store to set job->error.  atomic_set does an or
  operation, not assignment.

- Use refcount_* to manage job->nbio.

  This ensures proper memory barriers are present so that the last bio
  won't see a possibly stale value of job->error.

- Don't re-read job->error after reading it via atomic_load.

Reported by:	markj (1)
Reviewed by:	mjg, markj
Differential Revision:	https://reviews.freebsd.org/D38611
---
 sys/kern/vfs_aio.c | 11 ++++++-----
 sys/sys/aio.h      |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c
index 9868b8770747..d32e6fea15ca 100644
--- a/sys/kern/vfs_aio.c
+++ b/sys/kern/vfs_aio.c
@@ -1281,7 +1281,7 @@ aio_qbio(struct proc *p, struct kaiocb *job)
 	}
 
 	bios = malloc(sizeof(struct bio *) * iovcnt, M_TEMP, M_WAITOK);
-	atomic_store_int(&job->nbio, iovcnt);
+	refcount_init(&job->nbio, iovcnt);
 	for (i = 0; i < iovcnt; i++) {
 		struct vm_page** pages;
 		struct bio *bp;
@@ -2489,15 +2489,16 @@ aio_biowakeup(struct bio *bp)
 	 * error of whichever failed bio completed last.
 	 */
 	if (flags & BIO_ERROR)
-		atomic_set_int(&job->error, bio_error);
+		atomic_store_int(&job->error, bio_error);
 	if (opcode & LIO_WRITE)
 		atomic_add_int(&job->outblock, nblks);
 	else
 		atomic_add_int(&job->inblock, nblks);
 
-	if (atomic_fetchadd_int(&job->nbio, -1) == 1) {
-		if (atomic_load_int(&job->error))
-			aio_complete(job, -1, job->error);
+	if (refcount_release(&job->nbio)) {
+		bio_error = atomic_load_int(&job->error);
+		if (bio_error != 0)
+			aio_complete(job, -1, bio_error);
 		else
 			aio_complete(job, atomic_load_long(&job->nbytes), 0);
 	}
diff --git a/sys/sys/aio.h b/sys/sys/aio.h
index 7b931d04febc..272132acf91e 100644
--- a/sys/sys/aio.h
+++ b/sys/sys/aio.h
@@ -151,7 +151,7 @@ struct kaiocb {
 	aio_handle_fn_t *handle_fn;	/* (c) backend handle function */
 	union {				/* Backend-specific data fields */
 		struct {		/* BIO backend */
-			int	nbio;	/* Number of remaining bios */
+			volatile u_int nbio; /* Number of remaining bios */
 			int	error;	/* Worst error of all bios */
 			long	nbytes;	/* Bytes completed so far */
 		};