From cb5debd34fd20768247322904944cd3c69836b10 Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Fri, 6 Jan 2017 23:41:45 +0000
Subject: [PATCH] Set MORETOCOME for AIO write requests on a socket.

Add a MSG_MOREOTOCOME message flag. When this flag is set, sosend*
set PRUS_MOREOTOCOME when invoking the protocol send method. The aio
worker tasks for sending on a socket set this flag when there are
additional write jobs waiting on the socket buffer.

Reviewed by:	adrian
MFC after:	1 month
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D8955
---
 sys/kern/sys_socket.c  | 2 ++
 sys/kern/uipc_socket.c | 2 ++
 sys/sys/socket.h       | 1 +
 3 files changed, 5 insertions(+)

diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c
index aab972b06e38..441a628b7f78 100644
--- a/sys/kern/sys_socket.c
+++ b/sys/kern/sys_socket.c
@@ -604,6 +604,8 @@ soaio_process_job(struct socket *so, struct sockbuf *sb, struct kaiocb *job)
 		if (td->td_ru.ru_msgrcv != ru_before)
 			job->msgrcv = 1;
 	} else {
+		if (!TAILQ_EMPTY(&sb->sb_aiojobq))
+			flags |= MSG_MORETOCOME;
 		uio.uio_rw = UIO_WRITE;
 		ru_before = td->td_ru.ru_msgsnd;
 #ifdef MAC
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 7cd2a75ff688..23910c5bc1d3 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -1182,6 +1182,7 @@ sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio,
 	     (resid <= 0)) ?
 		PRUS_EOF :
 		/* If there is more to send set PRUS_MORETOCOME */
+		(flags & MSG_MORETOCOME) ||
 		(resid > 0 && space > 0) ? PRUS_MORETOCOME : 0,
 		top, addr, control, td);
 	if (dontroute) {
@@ -1368,6 +1369,7 @@ sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio,
 			     (resid <= 0)) ?
 				PRUS_EOF :
 			/* If there is more to send set PRUS_MORETOCOME. */
+			    (flags & MSG_MORETOCOME) ||
 			    (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0,
 			    top, addr, control, td);
 			if (dontroute) {
diff --git a/sys/sys/socket.h b/sys/sys/socket.h
index ea3d9b6bee34..9429f5a1f078 100644
--- a/sys/sys/socket.h
+++ b/sys/sys/socket.h
@@ -435,6 +435,7 @@ struct msghdr {
 #endif
 #ifdef _KERNEL
 #define	MSG_SOCALLBCK   0x10000		/* for use by socket callbacks - soreceive (TCP) */
+#define	MSG_MORETOCOME	0x20000		/* additional data pending */
 #endif
 
 /*