From 49587b8fb4d12166d30efb7111d32f4606a1e73b Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Thu, 8 Sep 2022 23:15:32 +0300
Subject: [PATCH] i386 copyout/in_fast: handle page fault from KVA access

by delegating the work to the slow path.

Some kernel memory, like pipe buffers, is pageable.  We must not enable
interrupts, and consequently, preemption, while in critical section in
the fast copyout path, because we use pcpu buffers.  If page fault
occurs while copying from the pcpu copyout_buf to kernel memory, abort
fast path and delegate work to the slow implementation.

In collaboration with:	pho, tijl
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
---
 sys/i386/i386/copyout_fast.s | 21 +++++++++------------
 sys/i386/i386/exception.s    | 25 +++++++++++++++----------
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/sys/i386/i386/copyout_fast.s b/sys/i386/i386/copyout_fast.s
index 70647fe7613d..4391f36b18e4 100644
--- a/sys/i386/i386/copyout_fast.s
+++ b/sys/i386/i386/copyout_fast.s
@@ -47,32 +47,30 @@ ENTRY(copyout_fast)
 	pushl	%ebx
 
 	movl	20(%ebp),%ebx		/* KCR3 */
-	movl	PCPU(CURPCB),%edx
-	movl	PCB_CR3(%edx),%edx	/* UCR3 */
 	/* bcopy(%esi = kaddr, %edi = PCPU(copyout_buf), %ecx = len) */
 	movl	16(%ebp),%ecx
 	movl	8(%ebp),%esi
+	movl	%esp,%eax
+	movl	$copyout_fault,%edx
+
 	cli
 	movl	PCPU(COPYOUT_BUF),%edi
-	rep; movsb
+pf_y1:	rep; movsb
 
 	movl	16(%ebp),%ecx		/* len */
 	movl	PCPU(COPYOUT_BUF),%esi	/* kaddr */
 	movl	12(%ebp),%edi		/* uaddr */
-
-	movl	%esp,%eax
 	movl	PCPU(TRAMPSTK),%esp
-
+	movl	PCPU(CURPCB),%edx
+	movl	PCB_CR3(%edx),%edx	/* UCR3 */
 	movl	%edx,%cr3
 	movl	$copyout_fault,%edx
-
 	/* bcopy(%esi = PCPU(copyout_buf), %edi = udaddr, %ecx = len) */
 pf_x1:	rep; movsb
 
 	movl	%ebx,%cr3
 	movl	%eax,%esp
 	sti
-
 	xorl	%eax,%eax
 	popl	%ebx
 	popl	%edi
@@ -93,10 +91,10 @@ ENTRY(copyin_fast)
 	movl	PCB_CR3(%eax),%edx	/* UCR3 */
 	movl	16(%ebp),%ecx		/* len */
 	movl	8(%ebp),%esi		/* udaddr */
+	movl	%esp,%eax
+
 	cli
 	movl	PCPU(COPYOUT_BUF),%edi	/* kaddr */
-
-	movl	%esp,%eax
 	movl	PCPU(TRAMPSTK),%esp
 	movl	%edx,%cr3
 	movl	$copyout_fault,%edx
@@ -110,10 +108,9 @@ pf_x2:	rep; movsb
 	movl	16(%ebp),%ecx
 	movl	12(%ebp),%edi
 	movl	PCPU(COPYOUT_BUF),%esi
-	rep; movsb
+pf_y2:	rep; movsb
 
 	sti
-
 	xorl	%eax,%eax
 	popl	%ebx
 	popl	%edi
diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s
index f8d61c884a21..e83d95e6afe1 100644
--- a/sys/i386/i386/exception.s
+++ b/sys/i386/i386/exception.s
@@ -130,15 +130,15 @@ IDTVEC(prot)
 	jmp	irettraps
 IDTVEC(page)
 	testl	$PSL_VM, TF_EFLAGS-TF_ERR(%esp)
-	jnz	4f
+	jnz	5f
 	testb	$SEL_RPL_MASK, TF_CS-TF_ERR(%esp)
-	jnz	4f
+	jnz	5f
 	cmpl	$PMAP_TRM_MIN_ADDRESS, TF_EIP-TF_ERR(%esp)
-	jb	4f
+	jb	5f
 	pushl	%eax
 	movl	TF_EIP-TF_ERR+4(%esp), %eax
 	addl	$1f, %eax
-	call	5f
+	call	6f
 1:	cmpl	$pf_x1, %eax
 	je	2f
 	cmpl	$pf_x2, %eax
@@ -154,16 +154,21 @@ IDTVEC(page)
 	cmpl	$pf_x7, %eax
 	je	2f
 	cmpl	$pf_x8, %eax
-	jne	3f
-2:	popl	%eax
-	movl	%ebx, %cr3
+	je	2f
+	cmpl	$pf_y1, %eax
+	je	3f
+	cmpl	$pf_y2, %eax
+	je	3f
+	jmp	4f
+2:	movl	%ebx, %cr3
+3:	popl	%eax
 	movl	%edx, TF_EIP-TF_ERR(%esp)
 	addl	$4, %esp
 	iret
-3:	popl	%eax
-4:	pushl	$T_PAGEFLT
+4:	popl	%eax
+5:	pushl	$T_PAGEFLT
 	jmp	alltraps
-5:	subl	(%esp), %eax
+6:	subl	(%esp), %eax
 	retl
 IDTVEC(rsvd_pti)
 IDTVEC(rsvd)