From 8083f14fc2267ab70ecd34b28e670378e100ce96 Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Sun, 16 Mar 2014 01:43:23 +0000 Subject: [PATCH] replace the kernel's version w/ cperciva's implementation... In all my tests, it is faster ~20%, even on an old IXP425 533MHz it is ~45% faster... This is partly due to loop unrolling, so the code size does significantly increase... I do plan on committing a version that rolls up the loops again for smaller code size for embedded systems where size is more important than absolute performance (it'll save ~6k code)... The kernel implementation is now shared w/ userland's libcrypt and libmd... We drop support for sha256 from sha2.c, so now sha2.c only contains sha384 and sha512... Reviewed by: secteam@ --- lib/libcrypt/Makefile | 2 +- lib/libmd/Makefile | 2 +- sys/conf/files | 2 + sys/crypto/sha2/sha2.c | 348 +------------------------------------- sys/crypto/sha2/sha2.h | 59 +------ sys/crypto/sha2/sha256.h | 8 +- sys/crypto/sha2/sha256c.c | 4 + 7 files changed, 19 insertions(+), 406 deletions(-) diff --git a/lib/libcrypt/Makefile b/lib/libcrypt/Makefile index 798b4b6350d4..aca24d8f8af3 100644 --- a/lib/libcrypt/Makefile +++ b/lib/libcrypt/Makefile @@ -9,7 +9,7 @@ SHLIBDIR?= /lib SHLIB_MAJOR= 5 LIB= crypt -.PATH: ${.CURDIR}/../libmd +.PATH: ${.CURDIR}/../libmd ${.CURDIR}/../../sys/crypto/sha2 SRCS= crypt.c misc.c \ crypt-md5.c md5c.c \ crypt-nthash.c md4c.c \ diff --git a/lib/libmd/Makefile b/lib/libmd/Makefile index d0fe7054fa4d..727fd8621716 100644 --- a/lib/libmd/Makefile +++ b/lib/libmd/Makefile @@ -43,7 +43,7 @@ CLEANFILES+= md[245]hl.c md[245].ref md[245].3 mddriver \ sha256.ref sha256hl.c sha512.ref sha512hl.c CFLAGS+= -I${.CURDIR} -.PATH: ${.CURDIR}/${MACHINE_ARCH} +.PATH: ${.CURDIR}/${MACHINE_ARCH} ${.CURDIR}/../../sys/crypto/sha2 .if exists(${MACHINE_ARCH}/sha.S) SRCS+= sha.S diff --git a/sys/conf/files b/sys/conf/files index 7decc156ac6b..596cc692d501 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -522,6 +522,8 @@ crypto/sha1.c optional carp | crypto | ipsec | \ netgraph_mppc_encryption | sctp crypto/sha2/sha2.c optional crypto | geom_bde | ipsec | random | \ sctp | zfs +crypto/sha2/sha256c.c optional crypto | geom_bde | ipsec | random | \ + sctp | zfs crypto/siphash/siphash.c optional inet | inet6 crypto/siphash/siphash_test.c optional inet | inet6 ddb/db_access.c optional ddb diff --git a/sys/crypto/sha2/sha2.c b/sys/crypto/sha2/sha2.c index 66ec6e9132c0..5bb52f38ba66 100644 --- a/sys/crypto/sha2/sha2.c +++ b/sys/crypto/sha2/sha2.c @@ -121,20 +121,10 @@ __FBSDID("$FreeBSD$"); * Thank you, Jun-ichiro itojun Hagino, for suggesting using u_intXX_t * types and pointing out recent ANSI C support for uintXX_t in inttypes.h. */ -#if 0 /*def SHA2_USE_INTTYPES_H*/ - typedef uint8_t sha2_byte; /* Exactly 1 byte */ typedef uint32_t sha2_word32; /* Exactly 4 bytes */ typedef uint64_t sha2_word64; /* Exactly 8 bytes */ -#else /* SHA2_USE_INTTYPES_H */ - -typedef u_int8_t sha2_byte; /* Exactly 1 byte */ -typedef u_int32_t sha2_word32; /* Exactly 4 bytes */ -typedef u_int64_t sha2_word64; /* Exactly 8 bytes */ - -#endif /* SHA2_USE_INTTYPES_H */ - /*** SHA-256/384/512 Various Length Definitions ***********************/ /* NOTE: Most of these are in sha2.h */ @@ -183,8 +173,6 @@ typedef u_int64_t sha2_word64; /* Exactly 8 bytes */ */ /* Shift-right (used in SHA-256, SHA-384, and SHA-512): */ #define R(b,x) ((x) >> (b)) -/* 32-bit Rotate-right (used in SHA-256): */ -#define S32(b,x) (((x) >> (b)) | ((x) << (32 - (b)))) /* 64-bit Rotate-right (used in SHA-384 and SHA-512): */ #define S64(b,x) (((x) >> (b)) | ((x) << (64 - (b)))) @@ -192,12 +180,6 @@ typedef u_int64_t sha2_word64; /* Exactly 8 bytes */ #define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) #define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) -/* Four of six logical functions used in SHA-256: */ -#define Sigma0_256(x) (S32(2, (x)) ^ S32(13, (x)) ^ S32(22, (x))) -#define Sigma1_256(x) (S32(6, (x)) ^ S32(11, (x)) ^ S32(25, (x))) -#define sigma0_256(x) (S32(7, (x)) ^ S32(18, (x)) ^ R(3 , (x))) -#define sigma1_256(x) (S32(17, (x)) ^ S32(19, (x)) ^ R(10, (x))) - /* Four of six logical functions used in SHA-384 and SHA-512: */ #define Sigma0_512(x) (S64(28, (x)) ^ S64(34, (x)) ^ S64(39, (x))) #define Sigma1_512(x) (S64(14, (x)) ^ S64(18, (x)) ^ S64(41, (x))) @@ -210,43 +192,10 @@ typedef u_int64_t sha2_word64; /* Exactly 8 bytes */ * only. */ static void SHA512_Last(SHA512_CTX*); -static void SHA256_Transform(SHA256_CTX*, const sha2_word32*); static void SHA512_Transform(SHA512_CTX*, const sha2_word64*); /*** SHA-XYZ INITIAL HASH VALUES AND CONSTANTS ************************/ -/* Hash constant words K for SHA-256: */ -static const sha2_word32 K256[64] = { - 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, - 0x3956c25bUL, 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, - 0xd807aa98UL, 0x12835b01UL, 0x243185beUL, 0x550c7dc3UL, - 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, 0xc19bf174UL, - 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, - 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, - 0x983e5152UL, 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, - 0xc6e00bf3UL, 0xd5a79147UL, 0x06ca6351UL, 0x14292967UL, - 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, 0x53380d13UL, - 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, - 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, - 0xd192e819UL, 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, - 0x19a4c116UL, 0x1e376c08UL, 0x2748774cUL, 0x34b0bcb5UL, - 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, 0x682e6ff3UL, - 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, - 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL -}; - -/* Initial hash value H for SHA-256: */ -static const sha2_word32 sha256_initial_hash_value[8] = { - 0x6a09e667UL, - 0xbb67ae85UL, - 0x3c6ef372UL, - 0xa54ff53aUL, - 0x510e527fUL, - 0x9b05688cUL, - 0x1f83d9abUL, - 0x5be0cd19UL -}; - /* Hash constant words K for SHA-384 and SHA-512: */ static const sha2_word64 K512[80] = { 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, @@ -323,301 +272,6 @@ static const char *sha2_hex_digits = "0123456789abcdef"; /*** SHA-256: *********************************************************/ -void SHA256_Init(SHA256_CTX* context) { - if (context == (SHA256_CTX*)0) { - return; - } - bcopy(sha256_initial_hash_value, context->state, SHA256_DIGEST_LENGTH); - bzero(context->buffer, SHA256_BLOCK_LENGTH); - context->bitcount = 0; -} - -#ifdef SHA2_UNROLL_TRANSFORM - -/* Unrolled SHA-256 round macros: */ - -#if BYTE_ORDER == LITTLE_ENDIAN - -#define ROUND256_0_TO_15(a,b,c,d,e,f,g,h) \ - REVERSE32(*data++, W256[j]); \ - T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + \ - K256[j] + W256[j]; \ - (d) += T1; \ - (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \ - j++ - - -#else /* BYTE_ORDER == LITTLE_ENDIAN */ - -#define ROUND256_0_TO_15(a,b,c,d,e,f,g,h) \ - T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + \ - K256[j] + (W256[j] = *data++); \ - (d) += T1; \ - (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \ - j++ - -#endif /* BYTE_ORDER == LITTLE_ENDIAN */ - -#define ROUND256(a,b,c,d,e,f,g,h) \ - s0 = W256[(j+1)&0x0f]; \ - s0 = sigma0_256(s0); \ - s1 = W256[(j+14)&0x0f]; \ - s1 = sigma1_256(s1); \ - T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + K256[j] + \ - (W256[j&0x0f] += s1 + W256[(j+9)&0x0f] + s0); \ - (d) += T1; \ - (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \ - j++ - -static void SHA256_Transform(SHA256_CTX* context, const sha2_word32* data) { - sha2_word32 a, b, c, d, e, f, g, h, s0, s1; - sha2_word32 T1, *W256; - int j; - - W256 = (sha2_word32*)context->buffer; - - /* Initialize registers with the prev. intermediate value */ - a = context->state[0]; - b = context->state[1]; - c = context->state[2]; - d = context->state[3]; - e = context->state[4]; - f = context->state[5]; - g = context->state[6]; - h = context->state[7]; - - j = 0; - do { - /* Rounds 0 to 15 (unrolled): */ - ROUND256_0_TO_15(a,b,c,d,e,f,g,h); - ROUND256_0_TO_15(h,a,b,c,d,e,f,g); - ROUND256_0_TO_15(g,h,a,b,c,d,e,f); - ROUND256_0_TO_15(f,g,h,a,b,c,d,e); - ROUND256_0_TO_15(e,f,g,h,a,b,c,d); - ROUND256_0_TO_15(d,e,f,g,h,a,b,c); - ROUND256_0_TO_15(c,d,e,f,g,h,a,b); - ROUND256_0_TO_15(b,c,d,e,f,g,h,a); - } while (j < 16); - - /* Now for the remaining rounds to 64: */ - do { - ROUND256(a,b,c,d,e,f,g,h); - ROUND256(h,a,b,c,d,e,f,g); - ROUND256(g,h,a,b,c,d,e,f); - ROUND256(f,g,h,a,b,c,d,e); - ROUND256(e,f,g,h,a,b,c,d); - ROUND256(d,e,f,g,h,a,b,c); - ROUND256(c,d,e,f,g,h,a,b); - ROUND256(b,c,d,e,f,g,h,a); - } while (j < 64); - - /* Compute the current intermediate hash value */ - context->state[0] += a; - context->state[1] += b; - context->state[2] += c; - context->state[3] += d; - context->state[4] += e; - context->state[5] += f; - context->state[6] += g; - context->state[7] += h; - - /* Clean up */ - a = b = c = d = e = f = g = h = T1 = 0; -} - -#else /* SHA2_UNROLL_TRANSFORM */ - -static void SHA256_Transform(SHA256_CTX* context, const sha2_word32* data) { - sha2_word32 a, b, c, d, e, f, g, h, s0, s1; - sha2_word32 T1, T2, *W256; - int j; - - W256 = (sha2_word32*)context->buffer; - - /* Initialize registers with the prev. intermediate value */ - a = context->state[0]; - b = context->state[1]; - c = context->state[2]; - d = context->state[3]; - e = context->state[4]; - f = context->state[5]; - g = context->state[6]; - h = context->state[7]; - - j = 0; - do { -#if BYTE_ORDER == LITTLE_ENDIAN - /* Copy data while converting to host byte order */ - REVERSE32(*data++,W256[j]); - /* Apply the SHA-256 compression function to update a..h */ - T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + W256[j]; -#else /* BYTE_ORDER == LITTLE_ENDIAN */ - /* Apply the SHA-256 compression function to update a..h with copy */ - T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + (W256[j] = *data++); -#endif /* BYTE_ORDER == LITTLE_ENDIAN */ - T2 = Sigma0_256(a) + Maj(a, b, c); - h = g; - g = f; - f = e; - e = d + T1; - d = c; - c = b; - b = a; - a = T1 + T2; - - j++; - } while (j < 16); - - do { - /* Part of the message block expansion: */ - s0 = W256[(j+1)&0x0f]; - s0 = sigma0_256(s0); - s1 = W256[(j+14)&0x0f]; - s1 = sigma1_256(s1); - - /* Apply the SHA-256 compression function to update a..h */ - T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + - (W256[j&0x0f] += s1 + W256[(j+9)&0x0f] + s0); - T2 = Sigma0_256(a) + Maj(a, b, c); - h = g; - g = f; - f = e; - e = d + T1; - d = c; - c = b; - b = a; - a = T1 + T2; - - j++; - } while (j < 64); - - /* Compute the current intermediate hash value */ - context->state[0] += a; - context->state[1] += b; - context->state[2] += c; - context->state[3] += d; - context->state[4] += e; - context->state[5] += f; - context->state[6] += g; - context->state[7] += h; - - /* Clean up */ - a = b = c = d = e = f = g = h = T1 = T2 = 0; -} - -#endif /* SHA2_UNROLL_TRANSFORM */ - -void SHA256_Update(SHA256_CTX* context, const sha2_byte *data, size_t len) { - unsigned int freespace, usedspace; - - if (len == 0) { - /* Calling with no data is valid - we do nothing */ - return; - } - - /* Sanity check: */ - assert(context != (SHA256_CTX*)0 && data != (sha2_byte*)0); - - usedspace = (context->bitcount >> 3) % SHA256_BLOCK_LENGTH; - if (usedspace > 0) { - /* Calculate how much free space is available in the buffer */ - freespace = SHA256_BLOCK_LENGTH - usedspace; - - if (len >= freespace) { - /* Fill the buffer completely and process it */ - bcopy(data, &context->buffer[usedspace], freespace); - context->bitcount += freespace << 3; - len -= freespace; - data += freespace; - SHA256_Transform(context, (sha2_word32*)context->buffer); - } else { - /* The buffer is not yet full */ - bcopy(data, &context->buffer[usedspace], len); - context->bitcount += len << 3; - /* Clean up: */ - usedspace = freespace = 0; - return; - } - } - while (len >= SHA256_BLOCK_LENGTH) { - /* Process as many complete blocks as we can */ - SHA256_Transform(context, (const sha2_word32*)data); - context->bitcount += SHA256_BLOCK_LENGTH << 3; - len -= SHA256_BLOCK_LENGTH; - data += SHA256_BLOCK_LENGTH; - } - if (len > 0) { - /* There's left-overs, so save 'em */ - bcopy(data, context->buffer, len); - context->bitcount += len << 3; - } - /* Clean up: */ - usedspace = freespace = 0; -} - -void SHA256_Final(sha2_byte digest[], SHA256_CTX* context) { - sha2_word32 *d = (sha2_word32*)digest; - unsigned int usedspace; - - /* Sanity check: */ - assert(context != (SHA256_CTX*)0); - - /* If no digest buffer is passed, we don't bother doing this: */ - if (digest != (sha2_byte*)0) { - usedspace = (context->bitcount >> 3) % SHA256_BLOCK_LENGTH; -#if BYTE_ORDER == LITTLE_ENDIAN - /* Convert FROM host byte order */ - REVERSE64(context->bitcount,context->bitcount); -#endif - if (usedspace > 0) { - /* Begin padding with a 1 bit: */ - context->buffer[usedspace++] = 0x80; - - if (usedspace <= SHA256_SHORT_BLOCK_LENGTH) { - /* Set-up for the last transform: */ - bzero(&context->buffer[usedspace], SHA256_SHORT_BLOCK_LENGTH - usedspace); - } else { - if (usedspace < SHA256_BLOCK_LENGTH) { - bzero(&context->buffer[usedspace], SHA256_BLOCK_LENGTH - usedspace); - } - /* Do second-to-last transform: */ - SHA256_Transform(context, (sha2_word32*)context->buffer); - - /* And set-up for the last transform: */ - bzero(context->buffer, SHA256_SHORT_BLOCK_LENGTH); - } - } else { - /* Set-up for the last transform: */ - bzero(context->buffer, SHA256_SHORT_BLOCK_LENGTH); - - /* Begin padding with a 1 bit: */ - *context->buffer = 0x80; - } - /* Set the bit count: */ - *(sha2_word64*)&context->buffer[SHA256_SHORT_BLOCK_LENGTH] = context->bitcount; - - /* Final transform: */ - SHA256_Transform(context, (sha2_word32*)context->buffer); - -#if BYTE_ORDER == LITTLE_ENDIAN - { - /* Convert TO host byte order */ - int j; - for (j = 0; j < 8; j++) { - REVERSE32(context->state[j],context->state[j]); - *d++ = context->state[j]; - } - } -#else - bcopy(context->state, d, SHA256_DIGEST_LENGTH); -#endif - } - - /* Clean up state data: */ - bzero(context, sizeof(*context)); - usedspace = 0; -} - char *SHA256_End(SHA256_CTX* context, char buffer[]) { sha2_byte digest[SHA256_DIGEST_LENGTH], *d = digest; int i; @@ -641,7 +295,7 @@ char *SHA256_End(SHA256_CTX* context, char buffer[]) { return buffer; } -char* SHA256_Data(const sha2_byte* data, size_t len, char digest[SHA256_DIGEST_STRING_LENGTH]) { +char* SHA256_Data(const void *data, unsigned int len, char *digest) { SHA256_CTX context; SHA256_Init(&context); diff --git a/sys/crypto/sha2/sha2.h b/sys/crypto/sha2/sha2.h index 639d58ba69f4..59e91f64711b 100644 --- a/sys/crypto/sha2/sha2.h +++ b/sys/crypto/sha2/sha2.h @@ -56,70 +56,17 @@ extern "C" { #define SHA512_DIGEST_STRING_LENGTH (SHA512_DIGEST_LENGTH * 2 + 1) -/*** SHA-256/384/512 Context Structures *******************************/ -/* NOTE: If your architecture does not define either u_intXX_t types or - * uintXX_t (from inttypes.h), you may need to define things by hand - * for your system: - */ -#if 0 -typedef unsigned char u_int8_t; /* 1-byte (8-bits) */ -typedef unsigned int u_int32_t; /* 4-bytes (32-bits) */ -typedef unsigned long long u_int64_t; /* 8-bytes (64-bits) */ -#endif -/* - * Most BSD systems already define u_intXX_t types, as does Linux. - * Some systems, however, like Compaq's Tru64 Unix instead can use - * uintXX_t types defined by very recent ANSI C standards and included - * in the file: - * - * #include - * - * If you choose to use then please define: - * - * #define SHA2_USE_INTTYPES_H - * - * Or on the command line during compile: - * - * cc -DSHA2_USE_INTTYPES_H ... - */ -#if 0 /*def SHA2_USE_INTTYPES_H*/ - -typedef struct _SHA256_CTX { - uint32_t state[8]; - uint64_t bitcount; - uint8_t buffer[SHA256_BLOCK_LENGTH]; -} SHA256_CTX; +/*** SHA-384/512 Context Structures *******************************/ typedef struct _SHA512_CTX { uint64_t state[8]; uint64_t bitcount[2]; uint8_t buffer[SHA512_BLOCK_LENGTH]; } SHA512_CTX; -#else /* SHA2_USE_INTTYPES_H */ - -typedef struct _SHA256_CTX { - u_int32_t state[8]; - u_int64_t bitcount; - u_int8_t buffer[SHA256_BLOCK_LENGTH]; -} SHA256_CTX; -typedef struct _SHA512_CTX { - u_int64_t state[8]; - u_int64_t bitcount[2]; - u_int8_t buffer[SHA512_BLOCK_LENGTH]; -} SHA512_CTX; - -#endif /* SHA2_USE_INTTYPES_H */ - typedef SHA512_CTX SHA384_CTX; -/*** SHA-256/384/512 Function Prototypes ******************************/ - -void SHA256_Init(SHA256_CTX *); -void SHA256_Update(SHA256_CTX*, const u_int8_t*, size_t); -void SHA256_Final(u_int8_t[SHA256_DIGEST_LENGTH], SHA256_CTX*); -char* SHA256_End(SHA256_CTX*, char[SHA256_DIGEST_STRING_LENGTH]); -char* SHA256_Data(const u_int8_t*, size_t, char[SHA256_DIGEST_STRING_LENGTH]); +/*** SHA-384/512 Function Prototypes ******************************/ void SHA384_Init(SHA384_CTX*); void SHA384_Update(SHA384_CTX*, const u_int8_t*, size_t); @@ -137,4 +84,6 @@ char* SHA512_Data(const u_int8_t*, size_t, char[SHA512_DIGEST_STRING_LENGTH]); } #endif /* __cplusplus */ +#include "sha256.h" + #endif /* __SHA2_H__ */ diff --git a/sys/crypto/sha2/sha256.h b/sys/crypto/sha2/sha256.h index 1b6a4f4c5755..80765ef8749d 100644 --- a/sys/crypto/sha2/sha256.h +++ b/sys/crypto/sha2/sha256.h @@ -29,12 +29,14 @@ #ifndef _SHA256_H_ #define _SHA256_H_ +#ifndef _KERNEL #include +#endif typedef struct SHA256Context { uint32_t state[8]; uint64_t count; - unsigned char buf[64]; + uint8_t buf[64]; } SHA256_CTX; __BEGIN_DECLS @@ -42,9 +44,11 @@ void SHA256_Init(SHA256_CTX *); void SHA256_Update(SHA256_CTX *, const void *, size_t); void SHA256_Final(unsigned char [32], SHA256_CTX *); char *SHA256_End(SHA256_CTX *, char *); +char *SHA256_Data(const void *, unsigned int, char *); +#ifndef _KERNEL char *SHA256_File(const char *, char *); char *SHA256_FileChunk(const char *, char *, off_t, off_t); -char *SHA256_Data(const void *, unsigned int, char *); +#endif __END_DECLS #endif /* !_SHA256_H_ */ diff --git a/sys/crypto/sha2/sha256c.c b/sys/crypto/sha2/sha256c.c index 76cec8ec3815..9c666df3ee25 100644 --- a/sys/crypto/sha2/sha256c.c +++ b/sys/crypto/sha2/sha256c.c @@ -30,7 +30,11 @@ __FBSDID("$FreeBSD$"); #include #include +#ifdef _KERNEL +#include +#else #include +#endif #include "sha256.h"