d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Crypto(4) driver for AESNI.

Browse Source 
The aeskeys_{amd64,i386}.S content was mostly obtained from OpenBSD,
no objections to the license from core.

Hardware provided by:	Sentex Communications
Tested by:	fabient, pho (previous versions)
MFC after:	1 month
This commit is contained in:
Konstantin Belousov 2010-07-23 11:00:46 +00:00
parent 959059ef9e
commit 5f270659fd
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=210409
11 changed files with 1489 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sys/conf/files.amd64
View File

@ -138,6 +138,10 @@ amd64/amd64/uma_machdep.c standard
amd64/amd64/vm_machdep.c standard
amd64/pci/pci_bus.c optional pci
amd64/pci/pci_cfgreg.c optional pci
crypto/aesni/aesencdec_amd64.S optional aesni
crypto/aesni/aeskeys_amd64.S optional aesni
crypto/aesni/aesni.c optional aesni
crypto/aesni/aesni_wrap.c optional aesni
crypto/blowfish/bf_enc.c optional crypto | ipsec
crypto/des/des_enc.c optional crypto | ipsec | netsmb
crypto/via/padlock.c optional padlock

4
sys/conf/files.i386
View File

@ -112,6 +112,10 @@ bf_enc.o optional crypto | ipsec \
dependency "$S/crypto/blowfish/arch/i386/bf_enc.S $S/crypto/blowfish/arch/i386/bf_enc_586.S $S/crypto/blowfish/arch/i386/bf_enc_686.S" \
compile-with "${CC} -c -I$S/crypto/blowfish/arch/i386 ${ASM_CFLAGS} ${WERROR} ${.IMPSRC}" \
no-implicit-rule
crypto/aesni/aesencdec_i386.S optional aesni
crypto/aesni/aeskeys_i386.S optional aesni
crypto/aesni/aesni.c optional aesni
crypto/aesni/aesni_wrap.c optional aesni
crypto/des/arch/i386/des_enc.S optional crypto | ipsec | netsmb
crypto/via/padlock.c optional padlock
crypto/via/padlock_cipher.c optional padlock

135
sys/crypto/aesni/aesencdec_amd64.S Normal file
View File

@ -0,0 +1,135 @@
/*-
* Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <machine/asmacros.h>
.text
ENTRY(aesni_enc)
.cfi_startproc
movdqu (%rdx),%xmm0
cmpq $0,%r8
je 1f
movdqu (%r8),%xmm1 /* unaligned load into reg */
pxor %xmm1,%xmm0 /* pxor otherwise can fault on iv */
1:
pxor (%rsi),%xmm0
2:
addq $0x10,%rsi
// aesenc (%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xdc,0x06
decl %edi
jne 2b
addq $0x10,%rsi
// aesenclast (%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xdd,0x06
movdqu %xmm0,(%rcx)
retq
.cfi_endproc
END(aesni_enc)
ENTRY(aesni_dec)
.cfi_startproc
movdqu (%rdx),%xmm0
pxor (%rsi),%xmm0
1:
addq $0x10,%rsi
// aesdec (%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x06
decl %edi
jne 1b
addq $0x10,%rsi
// aesdeclast (%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xdf,0x06
cmpq $0,%r8
je 2f
movdqu (%r8),%xmm1
pxor %xmm1,%xmm0
2:
movdqu %xmm0,(%rcx)
retq
.cfi_endproc
END(aesni_dec)
ENTRY(aesni_decrypt_cbc)
.cfi_startproc
shrq $4,%rdx
movdqu (%r8),%xmm1
1:
movdqu (%rcx),%xmm0
movdqa %xmm0,%xmm2
pxor (%rsi),%xmm0
cmpl $12,%edi
// aesdec 0x10(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x46,0x10
// aesdec 0x20(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x46,0x20
// aesdec 0x30(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x46,0x30
// aesdec 0x40(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x46,0x40
// aesdec 0x50(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x46,0x50
// aesdec 0x60(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x46,0x60
// aesdec 0x70(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x46,0x70
// aesdec 0x80(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x86,0x80,0x00,0x00,0x00
// aesdec 0x90(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x86,0x90,0x00,0x00,0x00
jge 2f
// aesdeclast 0xa0(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xdf,0x86,0xa0,0x00,0x00,0x00
jmp 4f
2:
// aesdec 0xa0(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x86,0xa0,0x00,0x00,0x00
// aesdec 0xb0(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x86,0xb0,0x00,0x00,0x00
jg 3f
// aesdeclast 0xc0(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xdf,0x86,0xc0,0x00,0x00,0x00
jmp 4f
3:
// aesdec 0xc0(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x86,0xc0,0x00,0x00,0x00
// aesdec 0xd0(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x86,0xd0,0x00,0x00,0x00
// aesdeclast 0xe0(%rsi),%xmm0
.byte 0x66,0x0f,0x38,0xdf,0x86,0xe0,0x00,0x00,0x00
4:
pxor %xmm1,%xmm0
movdqu %xmm0,(%rcx)
movdqa %xmm2,%xmm1 // iv
addq $0x10,%rcx
decq %rdx
jne 1b
retq
.cfi_endproc
END(aesni_decrypt_cbc)
.ident "$FreeBSD$"

166
sys/crypto/aesni/aesencdec_i386.S Normal file
View File

@ -0,0 +1,166 @@
/*-
* Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <machine/asmacros.h>
ENTRY(aesni_enc)
.cfi_startproc
pushl %ebp
.cfi_adjust_cfa_offset 4
movl %esp,%ebp
movl 8(%ebp),%ecx /* rounds */
movl 16(%ebp),%edx
movdqu (%edx),%xmm0 /* from */
movl 24(%ebp),%eax /* iv */
cmpl $0,%eax
je 1f
movdqu (%eax),%xmm1
pxor %xmm1,%xmm0
1:
movl 12(%ebp),%eax /* key */
pxor (%eax),%xmm0
2:
addl $0x10,%eax
// aesenc (%eax),%xmm0
.byte 0x66,0x0f,0x38,0xdc,0x00
loopne 2b
addl $0x10,%eax
// aesenclast (%eax),%xmm0
.byte 0x66,0x0f,0x38,0xdd,0x00
movl 20(%ebp),%eax
movdqu %xmm0,(%eax) /* to */
leave
.cfi_adjust_cfa_offset -4
retl
.cfi_endproc
END(aesni_enc)
ENTRY(aesni_dec)
.cfi_startproc
pushl %ebp
.cfi_adjust_cfa_offset 4
movl %esp,%ebp
movl 8(%ebp),%ecx /* rounds */
movl 16(%ebp),%edx
movdqu (%edx),%xmm0 /* from */
movl 12(%ebp),%eax /* key */
pxor (%eax),%xmm0
1:
addl $0x10,%eax
// aesdec (%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x00
loopne 1b
addl $0x10,%eax
// aesdeclast (%eax),%xmm0
.byte 0x66,0x0f,0x38,0xdf,0x00
movl 24(%ebp),%eax
cmpl $0,%eax /* iv */
je 2f
movdqu (%eax),%xmm1
pxor %xmm1,%xmm0
2:
movl 20(%ebp),%eax
movdqu %xmm0,(%eax) /* to */
leave
.cfi_adjust_cfa_offset -4
retl
.cfi_endproc
END(aesni_dec)
ENTRY(aesni_decrypt_cbc)
.cfi_startproc
pushl %ebp
.cfi_adjust_cfa_offset 4
movl %esp,%ebp
pushl %ebx
pushl %esi
movl 12(%ebp),%eax /* key */
movl 16(%ebp),%ecx /* length */
shrl $4,%ecx
movl 20(%ebp),%ebx /* buf */
movl 24(%ebp),%esi
movdqu (%esi),%xmm1 /* iv */
movl 8(%ebp),%esi /* rounds */
1:
movdqu (%ebx),%xmm0
movdqa %xmm0,%xmm2
pxor (%eax),%xmm0
cmpl $12,%esi
// aesdec 0x10(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x40,0x10
// aesdec 0x20(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x40,0x20
// aesdec 0x30(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x40,0x30
// aesdec 0x40(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x40,0x40
// aesdec 0x50(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x40,0x50
// aesdec 0x60(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x40,0x60
// aesdec 0x70(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x40,0x70
// aesdec 0x80(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x80,0x80,0x00,0x00,0x00
// aesdec 0x90(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x80,0x90,0x00,0x00,0x00
jge 2f
// aesdeclast 0xa0(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xdf,0x80,0xa0,0x00,0x00,0x00
jmp 4f
2:
// aesdec 0xa0(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x80,0xa0,0x00,0x00,0x00
// aesdec 0xb0(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x80,0xb0,0x00,0x00,0x00
jg 3f
// aesdeclast 0xc0(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xdf,0x80,0xc0,0x00,0x00,0x00
jmp 4f
3:
// aesdec 0xc0(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x80,0xc0,0x00,0x00,0x00
// aesdec 0xd0(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xde,0x80,0xd0,0x00,0x00,0x00
// aesdeclast 0xe0(%eax),%xmm0
.byte 0x66,0x0f,0x38,0xdf,0x80,0xe0,0x00,0x00,0x00
4:
pxor %xmm1,%xmm0
movdqu %xmm0,(%ebx)
movdqa %xmm2,%xmm1
addl $0x10,%ebx
decl %ecx
jne 1b
popl %esi
popl %ebx
leave
.cfi_adjust_cfa_offset -4
retl
.cfi_endproc
END(aesni_decrypt_cbc)
.ident "$FreeBSD$"

255
sys/crypto/aesni/aeskeys_amd64.S Normal file
View File

@ -0,0 +1,255 @@
/*-
* The white paper of AES-NI instructions can be downloaded from:
* http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
*
* Copyright (C) 2008-2010, Intel Corporation
* Author: Huang Ying <ying.huang@intel.com>
* Vinodh Gopal <vinodh.gopal@intel.com>
* Kahraman Akdemir
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the
* distribution.
*
* - Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products
* derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <machine/asmacros.h>
.text
ENTRY(_key_expansion_128)
_key_expansion_256a:
.cfi_startproc
pshufd $0b11111111,%xmm1,%xmm1
shufps $0b00010000,%xmm0,%xmm4
pxor %xmm4,%xmm0
shufps $0b10001100,%xmm0,%xmm4
pxor %xmm4,%xmm0
pxor %xmm1,%xmm0
movaps %xmm0,(%rsi)
addq $0x10,%rsi
retq
.cfi_endproc
END(_key_expansion_128)
ENTRY(_key_expansion_192a)
.cfi_startproc
pshufd $0b01010101,%xmm1,%xmm1
shufps $0b00010000,%xmm0,%xmm4
pxor %xmm4,%xmm0
shufps $0b10001100,%xmm0,%xmm4
pxor %xmm4,%xmm0
pxor %xmm1,%xmm0
movaps %xmm2,%xmm5
movaps %xmm2,%xmm6
pslldq $4,%xmm5
pshufd $0b11111111,%xmm0,%xmm3
pxor %xmm3,%xmm2
pxor %xmm5,%xmm2
movaps %xmm0,%xmm1
shufps $0b01000100,%xmm0,%xmm6
movaps %xmm6,(%rsi)
shufps $0b01001110,%xmm2,%xmm1
movaps %xmm1,0x10(%rsi)
addq $0x20,%rsi
retq
.cfi_endproc
END(_key_expansion_192a)
ENTRY(_key_expansion_192b)
.cfi_startproc
pshufd $0b01010101,%xmm1,%xmm1
shufps $0b00010000,%xmm0,%xmm4
pxor %xmm4,%xmm0
shufps $0b10001100,%xmm0,%xmm4
pxor %xmm4,%xmm0
pxor %xmm1,%xmm0
movaps %xmm2,%xmm5
pslldq $4,%xmm5
pshufd $0b11111111,%xmm0,%xmm3
pxor %xmm3,%xmm2
pxor %xmm5,%xmm2
movaps %xmm0,(%rsi)
addq $0x10,%rsi
retq
.cfi_endproc
END(_key_expansion_192b)
ENTRY(_key_expansion_256b)
.cfi_startproc
pshufd $0b10101010,%xmm1,%xmm1
shufps $0b00010000,%xmm2,%xmm4
pxor %xmm4,%xmm2
shufps $0b10001100,%xmm2,%xmm4
pxor %xmm4,%xmm2
pxor %xmm1,%xmm2
movaps %xmm2,(%rsi)
addq $0x10,%rsi
retq
.cfi_endproc
END(_key_expansion_256b)
ENTRY(aesni_set_enckey)
.cfi_startproc
movups (%rdi),%xmm0 # user key (first 16 bytes)
movaps %xmm0,(%rsi)
addq $0x10,%rsi # key addr
pxor %xmm4,%xmm4 # xmm4 is assumed 0 in _key_expansion_x
cmpl $12,%edx
jb .Lenc_key128
je .Lenc_key192
movups 0x10(%rdi),%xmm2 # other user key
movaps %xmm2,(%rsi)
addq $0x10,%rsi
// aeskeygenassist $0x1,%xmm2,%xmm1 # round 1
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x01
call _key_expansion_256a
// aeskeygenassist $0x1,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x01
call _key_expansion_256b
// aeskeygenassist $0x2,%xmm2,%xmm1 # round 2
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x02
call _key_expansion_256a
// aeskeygenassist $0x2,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x02
call _key_expansion_256b
// aeskeygenassist $0x4,%xmm2,%xmm1 # round 3
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x04
call _key_expansion_256a
// aeskeygenassist $0x4,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x04
call _key_expansion_256b
// aeskeygenassist $0x8,%xmm2,%xmm1 # round 4
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x08
call _key_expansion_256a
// aeskeygenassist $0x8,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x08
call _key_expansion_256b
// aeskeygenassist $0x10,%xmm2,%xmm1 # round 5
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x10
call _key_expansion_256a
// aeskeygenassist $0x10,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x10
call _key_expansion_256b
// aeskeygenassist $0x20,%xmm2,%xmm1 # round 6
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x20
call _key_expansion_256a
// aeskeygenassist $0x20,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x20
call _key_expansion_256b
// aeskeygenassist $0x40,%xmm2,%xmm1 # round 7
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x20
call _key_expansion_256a
retq
.Lenc_key192:
movq 0x10(%rdi),%xmm2 # other user key
// aeskeygenassist $0x1,%xmm2,%xmm1 # round 1
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x01
call _key_expansion_192a
// aeskeygenassist $0x2,%xmm2,%xmm1 # round 2
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x02
call _key_expansion_192b
// aeskeygenassist $0x4,%xmm2,%xmm1 # round 3
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x04
call _key_expansion_192a
// aeskeygenassist $0x8,%xmm2,%xmm1 # round 4
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x08
call _key_expansion_192b
// aeskeygenassist $0x10,%xmm2,%xmm1 # round 5
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x10
call _key_expansion_192a
// aeskeygenassist $0x20,%xmm2,%xmm1 # round 6
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x20
call _key_expansion_192b
// aeskeygenassist $0x40,%xmm2,%xmm1 # round 7
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x40
call _key_expansion_192a
// aeskeygenassist $0x80,%xmm2,%xmm1 # round 8
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x80
call _key_expansion_192b
retq
.Lenc_key128:
// aeskeygenassist $0x1,%xmm0,%xmm1 # round 1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x01
call _key_expansion_128
// aeskeygenassist $0x2,%xmm0,%xmm1 # round 2
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x02
call _key_expansion_128
// aeskeygenassist $0x4,%xmm0,%xmm1 # round 3
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x04
call _key_expansion_128
// aeskeygenassist $0x8,%xmm0,%xmm1 # round 4
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x08
call _key_expansion_128
// aeskeygenassist $0x10,%xmm0,%xmm1 # round 5
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x10
call _key_expansion_128
// aeskeygenassist $0x20,%xmm0,%xmm1 # round 6
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x20
call _key_expansion_128
// aeskeygenassist $0x40,%xmm0,%xmm1 # round 7
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x40
call _key_expansion_128
// aeskeygenassist $0x80,%xmm0,%xmm1 # round 8
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x80
call _key_expansion_128
// aeskeygenassist $0x1b,%xmm0,%xmm1 # round 9
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x1b
call _key_expansion_128
// aeskeygenassist $0x36,%xmm0,%xmm1 # round 10
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x36
call _key_expansion_128
retq
.cfi_endproc
END(aesni_set_enckey)
ENTRY(aesni_set_deckey)
.cfi_startproc
movslq %edx,%rax
shlq $4,%rax
addq %rax,%rdi
movdqa (%rdi),%xmm0
movdqa %xmm0,(%rsi)
decl %edx
1:
addq $0x10,%rsi
subq $0x10,%rdi
// aesimc (%rdi),%xmm1
.byte 0x66,0x0f,0x38,0xdb,0x0f
movdqa %xmm1,(%rsi)
decl %edx
jne 1b
addq $0x10,%rsi
subq $0x10,%rdi
movdqa (%rdi),%xmm0
movdqa %xmm0,(%rsi)
retq
.cfi_endproc
END(aesni_set_deckey)
.ident "$FreeBSD$"

273
sys/crypto/aesni/aeskeys_i386.S Normal file
View File

@ -0,0 +1,273 @@
/*-
* The white paper of AES-NI instructions can be downloaded from:
* http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
*
* Copyright (C) 2008-2010, Intel Corporation
* Author: Huang Ying <ying.huang@intel.com>
* Vinodh Gopal <vinodh.gopal@intel.com>
* Kahraman Akdemir
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the
* distribution.
*
* - Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products
* derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <machine/asmacros.h>
.text
ENTRY(_key_expansion_128)
_key_expansion_256a:
.cfi_startproc
pshufd $0b11111111,%xmm1,%xmm1
shufps $0b00010000,%xmm0,%xmm4
pxor %xmm4,%xmm0
shufps $0b10001100,%xmm0,%xmm4
pxor %xmm4,%xmm0
pxor %xmm1,%xmm0
movaps %xmm0,(%edx)
addl $0x10,%edx
retq
.cfi_endproc
END(_key_expansion_128)
ENTRY(_key_expansion_192a)
.cfi_startproc
pshufd $0b01010101,%xmm1,%xmm1
shufps $0b00010000,%xmm0,%xmm4
pxor %xmm4,%xmm0
shufps $0b10001100,%xmm0,%xmm4
pxor %xmm4,%xmm0
pxor %xmm1,%xmm0
movaps %xmm2,%xmm5
movaps %xmm2,%xmm6
pslldq $4,%xmm5
pshufd $0b11111111,%xmm0,%xmm3
pxor %xmm3,%xmm2
pxor %xmm5,%xmm2
movaps %xmm0,%xmm1
shufps $0b01000100,%xmm0,%xmm6
movaps %xmm6,(%edx)
shufps $0b01001110,%xmm2,%xmm1
movaps %xmm1,0x10(%edx)
addl $0x20,%edx
retq
.cfi_endproc
END(_key_expansion_192a)
ENTRY(_key_expansion_192b)
.cfi_startproc
pshufd $0b01010101,%xmm1,%xmm1
shufps $0b00010000,%xmm0,%xmm4
pxor %xmm4,%xmm0
shufps $0b10001100,%xmm0,%xmm4
pxor %xmm4,%xmm0
pxor %xmm1,%xmm0
movaps %xmm2,%xmm5
pslldq $4,%xmm5
pshufd $0b11111111,%xmm0,%xmm3
pxor %xmm3,%xmm2
pxor %xmm5,%xmm2
movaps %xmm0,(%edx)
addl $0x10,%edx
retl
.cfi_endproc
END(_key_expansion_192b)
ENTRY(_key_expansion_256b)
.cfi_startproc
pshufd $0b10101010,%xmm1,%xmm1
shufps $0b00010000,%xmm2,%xmm4
pxor %xmm4,%xmm2
shufps $0b10001100,%xmm2,%xmm4
pxor %xmm4,%xmm2
pxor %xmm1,%xmm2
movaps %xmm2,(%edx)
addl $0x10,%edx
retl
.cfi_endproc
END(_key_expansion_256b)
ENTRY(aesni_set_enckey)
.cfi_startproc
pushl %ebp
.cfi_adjust_cfa_offset 4
movl %esp,%ebp
movl 8(%ebp),%ecx
movl 12(%ebp),%edx
movups (%ecx),%xmm0 # user key (first 16 bytes)
movaps %xmm0,(%edx)
addl $0x10,%edx # key addr
pxor %xmm4,%xmm4 # xmm4 is assumed 0 in _key_expansion_x
cmpl $12,16(%ebp) # rounds
jb .Lenc_key128
je .Lenc_key192
movups 0x10(%ecx),%xmm2 # other user key
movaps %xmm2,(%edx)
addl $0x10,%edx
// aeskeygenassist $0x1,%xmm2,%xmm1 # round 1
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x01
call _key_expansion_256a
// aeskeygenassist $0x1,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x01
call _key_expansion_256b
// aeskeygenassist $0x2,%xmm2,%xmm1 # round 2
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x02
call _key_expansion_256a
// aeskeygenassist $0x2,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x02
call _key_expansion_256b
// aeskeygenassist $0x4,%xmm2,%xmm1 # round 3
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x04
call _key_expansion_256a
// aeskeygenassist $0x4,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x04
call _key_expansion_256b
// aeskeygenassist $0x8,%xmm2,%xmm1 # round 4
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x08
call _key_expansion_256a
// aeskeygenassist $0x8,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x08
call _key_expansion_256b
// aeskeygenassist $0x10,%xmm2,%xmm1 # round 5
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x10
call _key_expansion_256a
// aeskeygenassist $0x10,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x10
call _key_expansion_256b
// aeskeygenassist $0x20,%xmm2,%xmm1 # round 6
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x20
call _key_expansion_256a
// aeskeygenassist $0x20,%xmm0,%xmm1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x20
call _key_expansion_256b
// aeskeygenassist $0x40,%xmm2,%xmm1 # round 7
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x20
call _key_expansion_256a
.cfi_adjust_cfa_offset -4
leave
retl
.Lenc_key192:
movq 0x10(%ecx),%xmm2 # other user key
// aeskeygenassist $0x1,%xmm2,%xmm1 # round 1
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x01
call _key_expansion_192a
// aeskeygenassist $0x2,%xmm2,%xmm1 # round 2
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x02
call _key_expansion_192b
// aeskeygenassist $0x4,%xmm2,%xmm1 # round 3
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x04
call _key_expansion_192a
// aeskeygenassist $0x8,%xmm2,%xmm1 # round 4
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x08
call _key_expansion_192b
// aeskeygenassist $0x10,%xmm2,%xmm1 # round 5
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x10
call _key_expansion_192a
// aeskeygenassist $0x20,%xmm2,%xmm1 # round 6
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x20
call _key_expansion_192b
// aeskeygenassist $0x40,%xmm2,%xmm1 # round 7
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x40
call _key_expansion_192a
// aeskeygenassist $0x80,%xmm2,%xmm1 # round 8
.byte 0x66,0x0f,0x3a,0xdf,0xca,0x80
call _key_expansion_192b
leave
.cfi_adjust_cfa_offset -4
retl
.Lenc_key128:
// aeskeygenassist $0x1,%xmm0,%xmm1 # round 1
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x01
call _key_expansion_128
// aeskeygenassist $0x2,%xmm0,%xmm1 # round 2
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x02
call _key_expansion_128
// aeskeygenassist $0x4,%xmm0,%xmm1 # round 3
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x04
call _key_expansion_128
// aeskeygenassist $0x8,%xmm0,%xmm1 # round 4
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x08
call _key_expansion_128
// aeskeygenassist $0x10,%xmm0,%xmm1 # round 5
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x10
call _key_expansion_128
// aeskeygenassist $0x20,%xmm0,%xmm1 # round 6
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x20
call _key_expansion_128
// aeskeygenassist $0x40,%xmm0,%xmm1 # round 7
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x40
call _key_expansion_128
// aeskeygenassist $0x80,%xmm0,%xmm1 # round 8
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x80
call _key_expansion_128
// aeskeygenassist $0x1b,%xmm0,%xmm1 # round 9
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x1b
call _key_expansion_128
// aeskeygenassist $0x36,%xmm0,%xmm1 # round 10
.byte 0x66,0x0f,0x3a,0xdf,0xc8,0x36
call _key_expansion_128
leave
.cfi_adjust_cfa_offset -4
retl
.cfi_endproc
END(aesni_set_enckey)
ENTRY(aesni_set_deckey)
.cfi_startproc
pushl %ebp
.cfi_adjust_cfa_offset 4
movl %esp,%ebp
movl 16(%ebp),%eax /* rounds */
movl %eax,%ecx
shll $4,%ecx
addl 8(%ebp),%ecx /* encrypt_schedule last quad */
movl 12(%ebp),%edx /* decrypt_schedule */
movdqa (%ecx),%xmm0
movdqa %xmm0,(%edx)
decl %eax
1:
addl $0x10,%edx
subl $0x10,%ecx
// aesimc (%ecx),%xmm1
.byte 0x66,0x0f,0x38,0xdb,0x09
movdqa %xmm1,(%edx)
decl %eax
jne 1b
addl $0x10,%edx
subl $0x10,%ecx
movdqa (%ecx),%xmm0
movdqa %xmm0,(%edx)
leave
.cfi_adjust_cfa_offset -4
retl
.cfi_endproc
END(aesni_set_deckey)
.ident "$FreeBSD$"

338
sys/crypto/aesni/aesni.c Normal file
View File

@ -0,0 +1,338 @@
/*-
* Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
* Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/kobj.h>
#include <sys/libkern.h>
#include <sys/lock.h>
#include <sys/module.h>
#include <sys/malloc.h>
#include <sys/rwlock.h>
#include <sys/bus.h>
#include <sys/uio.h>
#include <crypto/aesni/aesni.h>
#include "cryptodev_if.h"
struct aesni_softc {
int32_t cid;
uint32_t sid;
TAILQ_HEAD(aesni_sessions_head, aesni_session) sessions;
struct rwlock lock;
};
static int aesni_newsession(device_t, uint32_t *sidp, struct cryptoini *cri);
static int aesni_freesession(device_t, uint64_t tid);
static void aesni_freesession_locked(struct aesni_softc *sc,
struct aesni_session *ses);
MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
static void
aesni_identify(driver_t *drv, device_t parent)
{
/* NB: order 10 is so we get attached after h/w devices */
if (device_find_child(parent, "aesni", -1) == NULL &&
BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0)
panic("aesni: could not attach");
}
static int
aesni_probe(device_t dev)
{
char capp[32];
if ((cpu_feature2 & CPUID2_AESNI) == 0) {
device_printf(dev, "No AESNI support.\n");
return (EINVAL);
}
strlcpy(capp, "AES-CBC", sizeof(capp));
device_set_desc_copy(dev, capp);
return (0);
}
static int
aesni_attach(device_t dev)
{
struct aesni_softc *sc;
sc = device_get_softc(dev);
TAILQ_INIT(&sc->sessions);
sc->sid = 1;
sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
if (sc->cid < 0) {
device_printf(dev, "Could not get crypto driver id.\n");
return (ENOMEM);
}
rw_init(&sc->lock, "aesni_lock");
crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
return (0);
}
static int
aesni_detach(device_t dev)
{
struct aesni_softc *sc;
struct aesni_session *ses;
sc = device_get_softc(dev);
rw_wlock(&sc->lock);
TAILQ_FOREACH(ses, &sc->sessions, next) {
if (ses->used) {
rw_wunlock(&sc->lock);
device_printf(dev,
"Cannot detach, sessions still active.\n");
return (EBUSY);
}
}
while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) {
TAILQ_REMOVE(&sc->sessions, ses, next);
free(ses, M_AESNI);
}
rw_wunlock(&sc->lock);
rw_destroy(&sc->lock);
crypto_unregister_all(sc->cid);
return (0);
}
static int
aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
{
struct aesni_softc *sc;
struct aesni_session *ses;
struct cryptoini *encini;
int error;
if (sidp == NULL || cri == NULL)
return (EINVAL);
sc = device_get_softc(dev);
ses = NULL;
encini = NULL;
for (; cri != NULL; cri = cri->cri_next) {
switch (cri->cri_alg) {
case CRYPTO_AES_CBC:
if (encini != NULL)
return (EINVAL);
encini = cri;
break;
default:
return (EINVAL);
}
}
if (encini == NULL)
return (EINVAL);
rw_wlock(&sc->lock);
/*
* Free sessions goes first, so if first session is used, we need to
* allocate one.
*/
ses = TAILQ_FIRST(&sc->sessions);
if (ses == NULL || ses->used) {
ses = malloc(sizeof(*ses), M_AESNI, M_NOWAIT | M_ZERO);
if (ses == NULL) {
rw_wunlock(&sc->lock);
return (ENOMEM);
}
KASSERT(((uintptr_t)ses) % 0x10 == 0,
("malloc returned unaligned pointer"));
ses->id = sc->sid++;
} else {
TAILQ_REMOVE(&sc->sessions, ses, next);
}
ses->used = 1;
TAILQ_INSERT_TAIL(&sc->sessions, ses, next);
rw_wunlock(&sc->lock);
error = aesni_cipher_setup(ses, encini);
if (error != 0) {
rw_wlock(&sc->lock);
aesni_freesession_locked(sc, ses);
rw_wunlock(&sc->lock);
return (error);
}
*sidp = ses->id;
return (0);
}
static void
aesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses)
{
uint32_t sid;
sid = ses->id;
TAILQ_REMOVE(&sc->sessions, ses, next);
bzero(ses, sizeof(*ses));
ses->id = sid;
TAILQ_INSERT_HEAD(&sc->sessions, ses, next);
}
static int
aesni_freesession(device_t dev, uint64_t tid)
{
struct aesni_softc *sc;
struct aesni_session *ses;
uint32_t sid;
sc = device_get_softc(dev);
sid = ((uint32_t)tid) & 0xffffffff;
rw_wlock(&sc->lock);
TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
if (ses->id == sid)
break;
}
if (ses == NULL) {
rw_wunlock(&sc->lock);
return (EINVAL);
}
aesni_freesession_locked(sc, ses);
rw_wunlock(&sc->lock);
return (0);
}
static int
aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
{
struct aesni_softc *sc = device_get_softc(dev);
struct aesni_session *ses = NULL;
struct cryptodesc *crd, *enccrd;
int error;
error = 0;
enccrd = NULL;
/* Sanity check. */
if (crp == NULL)
return (EINVAL);
if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
error = EINVAL;
goto out;
}
for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
switch (crd->crd_alg) {
case CRYPTO_AES_CBC:
if (enccrd != NULL) {
error = EINVAL;
goto out;
}
enccrd = crd;
break;
default:
return (EINVAL);
}
}
if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
error = EINVAL;
goto out;
}
rw_rlock(&sc->lock);
TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
if (ses->id == (crp->crp_sid & 0xffffffff))
break;
}
rw_runlock(&sc->lock);
if (ses == NULL) {
error = EINVAL;
goto out;
}
error = aesni_cipher_process(ses, enccrd, crp);
if (error != 0)
goto out;
out:
crp->crp_etype = error;
crypto_done(crp);
return (error);
}
uint8_t *
aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
int *allocated)
{
struct uio *uio;
struct iovec *iov;
uint8_t *addr;
if (crp->crp_flags & CRYPTO_F_IMBUF)
goto alloc;
else if (crp->crp_flags & CRYPTO_F_IOV) {
uio = (struct uio *)crp->crp_buf;
if (uio->uio_iovcnt != 1)
goto alloc;
iov = uio->uio_iov;
addr = (u_char *)iov->iov_base + enccrd->crd_skip;
} else
addr = (u_char *)crp->crp_buf;
*allocated = 0;
return (addr);
alloc:
addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
if (addr != NULL) {
*allocated = 1;
crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
enccrd->crd_len, addr);
} else
*allocated = 0;
return (addr);
}
static device_method_t aesni_methods[] = {
DEVMETHOD(device_identify, aesni_identify),
DEVMETHOD(device_probe, aesni_probe),
DEVMETHOD(device_attach, aesni_attach),
DEVMETHOD(device_detach, aesni_detach),
DEVMETHOD(cryptodev_newsession, aesni_newsession),
DEVMETHOD(cryptodev_freesession, aesni_freesession),
DEVMETHOD(cryptodev_process, aesni_process),
{0, 0},
};
static driver_t aesni_driver = {
"aesni",
aesni_methods,
sizeof(struct aesni_softc),
};
static devclass_t aesni_devclass;
DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0);
MODULE_VERSION(aesni, 1);
MODULE_DEPEND(aesni, crypto, 1, 1, 1);

103
sys/crypto/aesni/aesni.h Normal file
View File

@ -0,0 +1,103 @@
/*-
* Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _AESNI_H_
#define _AESNI_H_
#include <sys/types.h>
#include <sys/malloc.h>
#include <sys/queue.h>
#include <opencrypto/cryptodev.h>
#if defined(__amd64__) || (defined(__i386__) && !defined(PC98))
#include <machine/cpufunc.h>
#include <machine/cputypes.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
#endif
#if defined(__i386__)
#include <machine/npx.h>
#elif defined(__amd64__)
#include <machine/fpu.h>
#endif
#define AES128_ROUNDS 10
#define AES192_ROUNDS 12
#define AES256_ROUNDS 14
#define AES_SCHED_LEN ((AES256_ROUNDS + 1) * AES_BLOCK_LEN)
struct aesni_session {
uint8_t enc_schedule[AES_SCHED_LEN] __aligned(16);
uint8_t dec_schedule[AES_SCHED_LEN] __aligned(16);
uint8_t iv[AES_BLOCK_LEN];
int rounds;
/* uint8_t *ses_ictx; */
/* uint8_t *ses_octx; */
/* int ses_mlen; */
int used;
uint32_t id;
TAILQ_ENTRY(aesni_session) next;
struct fpu_kern_ctx fpu_ctx;
};
/*
* Internal functions, implemented in assembler.
*/
void aesni_enc(int rounds, const uint8_t *key_schedule,
const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN],
const uint8_t iv[AES_BLOCK_LEN]);
void aesni_dec(int rounds, const uint8_t *key_schedule,
const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN],
const uint8_t iv[AES_BLOCK_LEN]);
void aesni_set_enckey(const uint8_t *userkey, uint8_t *encrypt_schedule,
int number_of_rounds);
void aesni_set_deckey(const uint8_t *encrypt_schedule,
uint8_t *decrypt_schedule, int number_of_rounds);
/*
* Slightly more public interfaces.
*/
void aesni_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN]);
void aesni_decrypt_cbc(int rounds, const void *key_schedule, size_t len,
const uint8_t *from, const uint8_t iv[AES_BLOCK_LEN]);
void aesni_encrypt_ecb(int rounds, const void *key_schedule, size_t len,
const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN]);
void aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len,
const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN]);
int aesni_cipher_setup(struct aesni_session *ses,
struct cryptoini *encini);
int aesni_cipher_process(struct aesni_session *ses,
struct cryptodesc *enccrd, struct cryptop *crp);
uint8_t *aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
int *allocated);
#endif

194
sys/crypto/aesni/aesni_wrap.c Normal file
View File

@ -0,0 +1,194 @@
/*-
* Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/libkern.h>
#include <sys/malloc.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <crypto/aesni/aesni.h>
MALLOC_DECLARE(M_AESNI);
#ifdef DEBUG
static void
ps_len(const char *string, const uint8_t *data, int length)
{
int i;
printf("%-12s[0x", string);
for(i = 0; i < length; i++) {
if (i % AES_BLOCK_LEN == 0 && i > 0)
printf("+");
printf("%02x", data[i]);
}
printf("]\n");
}
#endif
void
aesni_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN])
{
const uint8_t *ivp;
size_t i;
#ifdef DEBUG
ps_len("AES CBC encrypt iv:", iv, AES_BLOCK_LEN);
ps_len("from:", from, len);
#endif
len /= AES_BLOCK_LEN;
ivp = iv;
for (i = 0; i < len; i++) {
aesni_enc(rounds - 1, key_schedule, from, to, ivp);
ivp = to;
from += AES_BLOCK_LEN;
to += AES_BLOCK_LEN;
}
#ifdef DEBUG
ps_len("to:", to - len * AES_BLOCK_LEN, len * AES_BLOCK_LEN);
#endif
}
void
aesni_encrypt_ecb(int rounds, const void *key_schedule, size_t len,
const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN])
{
size_t i;
len /= AES_BLOCK_LEN;
for (i = 0; i < len; i++) {
aesni_enc(rounds - 1, key_schedule, from, to, NULL);
from += AES_BLOCK_LEN;
to += AES_BLOCK_LEN;
}
}
void
aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len,
const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN])
{
size_t i;
len /= AES_BLOCK_LEN;
for (i = 0; i < len; i++) {
aesni_dec(rounds - 1, key_schedule, from, to, NULL);
from += AES_BLOCK_LEN;
to += AES_BLOCK_LEN;
}
}
int
aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini)
{
struct thread *td;
int error;
switch (encini->cri_klen) {
case 128:
ses->rounds = AES128_ROUNDS;
break;
case 192:
ses->rounds = AES192_ROUNDS;
break;
case 256:
ses->rounds = AES256_ROUNDS;
break;
default:
return (EINVAL);
}
td = curthread;
error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL);
if (error == 0) {
aesni_set_enckey(encini->cri_key, ses->enc_schedule,
ses->rounds);
aesni_set_deckey(ses->enc_schedule, ses->dec_schedule,
ses->rounds);
arc4rand(ses->iv, sizeof(ses->iv), 0);
fpu_kern_leave(td, &ses->fpu_ctx);
}
return (error);
}
int
aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
struct cryptop *crp)
{
struct thread *td;
uint8_t *buf;
int error, allocated;
buf = aesni_cipher_alloc(enccrd, crp, &allocated);
if (buf == NULL) {
error = ENOMEM;
goto out;
}
td = curthread;
error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL);
if (error != 0)
goto out1;
if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
crypto_copyback(crp->crp_flags, crp->crp_buf,
enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
enccrd->crd_len, buf, buf, ses->iv);
} else {
if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
else
crypto_copydata(crp->crp_flags, crp->crp_buf,
enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
enccrd->crd_len, buf, ses->iv);
}
fpu_kern_leave(td, &ses->fpu_ctx);
if (allocated)
crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
enccrd->crd_len, buf);
if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0)
crypto_copydata(crp->crp_flags, crp->crp_buf,
enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN,
AES_BLOCK_LEN, ses->iv);
out1:
if (allocated) {
bzero(buf, enccrd->crd_len);
free(buf, M_AESNI);
}
out:
return (error);
}

7
sys/modules/Makefile
View File

@ -10,6 +10,7 @@ SUBDIR= ${_3dfx} \
accf_http \
${_acpi} \
ae \
${_aesni} \
age \
${_agp} \
aha \
@ -439,6 +440,9 @@ _zfs= zfs
.if ${MACHINE} == "i386"
_aac= aac
_acpi= acpi
.if ${MK_CRYPT} != "no" || defined(ALL_MODULES)
_aesni= aesni
.endif
_ahb= ahb
_amdsbwd= amdsbwd
_amdtemp= amdtemp
@ -494,6 +498,9 @@ _snc= snc
.if ${MACHINE_ARCH} == "amd64"
_aac= aac
_acpi= acpi
.if ${MK_CRYPT} != "no" || defined(ALL_MODULES)
_aesni= aesni
.endif
_agp= agp
_an= an
_amdsbwd= amdsbwd

10
sys/modules/aesni/Makefile Normal file
View File

@ -0,0 +1,10 @@
# $FreeBSD$
.PATH: ${.CURDIR}/../../crypto/aesni
KMOD= aesni
SRCS= aesni.c aesni_wrap.c
SRCS+= aesencdec_$(MACHINE_CPUARCH).S aeskeys_$(MACHINE_CPUARCH).S
SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h
.include <bsd.kmod.mk>