696c3895ae
a preference for memory load instructions over large code footprints with embedded immediate variables. On amd64 CPUs from 2007-2008 there is not a significant change, but amd64 CPUs from 2009-2010 get roughly 10% more throughput with this code; amd64 CPUs from 2011-2012 get roughly 15% more throughput; and AMD64 CPUs from 2013-2015 get 20-25% more throughput. The Raspberry Pi 2 increases its throughput by 6-8%. Sponsored by: Tarsnap Backup Inc. Performance tested by: allanjude MFC after: 3 weeks
317 lines
8.5 KiB
C
317 lines
8.5 KiB
C
/*-
|
|
* Copyright 2005 Colin Percival
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include <sys/endian.h>
|
|
#include <sys/types.h>
|
|
|
|
#ifdef _KERNEL
|
|
#include <sys/systm.h>
|
|
#else
|
|
#include <string.h>
|
|
#endif
|
|
|
|
#include "sha256.h"
|
|
|
|
#if BYTE_ORDER == BIG_ENDIAN
|
|
|
|
/* Copy a vector of big-endian uint32_t into a vector of bytes */
|
|
#define be32enc_vect(dst, src, len) \
|
|
memcpy((void *)dst, (const void *)src, (size_t)len)
|
|
|
|
/* Copy a vector of bytes into a vector of big-endian uint32_t */
|
|
#define be32dec_vect(dst, src, len) \
|
|
memcpy((void *)dst, (const void *)src, (size_t)len)
|
|
|
|
#else /* BYTE_ORDER != BIG_ENDIAN */
|
|
|
|
/*
|
|
* Encode a length len/4 vector of (uint32_t) into a length len vector of
|
|
* (unsigned char) in big-endian form. Assumes len is a multiple of 4.
|
|
*/
|
|
static void
|
|
be32enc_vect(unsigned char *dst, const uint32_t *src, size_t len)
|
|
{
|
|
size_t i;
|
|
|
|
for (i = 0; i < len / 4; i++)
|
|
be32enc(dst + i * 4, src[i]);
|
|
}
|
|
|
|
/*
|
|
* Decode a big-endian length len vector of (unsigned char) into a length
|
|
* len/4 vector of (uint32_t). Assumes len is a multiple of 4.
|
|
*/
|
|
static void
|
|
be32dec_vect(uint32_t *dst, const unsigned char *src, size_t len)
|
|
{
|
|
size_t i;
|
|
|
|
for (i = 0; i < len / 4; i++)
|
|
dst[i] = be32dec(src + i * 4);
|
|
}
|
|
|
|
#endif /* BYTE_ORDER != BIG_ENDIAN */
|
|
|
|
/* SHA256 round constants. */
|
|
static const uint32_t K[64] = {
|
|
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
|
|
0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
|
|
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
|
|
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
|
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
|
|
0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
|
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
|
|
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
|
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
|
|
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
|
|
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
|
|
0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
|
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
|
|
0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
|
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
|
|
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
|
|
};
|
|
|
|
/* Elementary functions used by SHA256 */
|
|
#define Ch(x, y, z) ((x & (y ^ z)) ^ z)
|
|
#define Maj(x, y, z) ((x & (y | z)) | (y & z))
|
|
#define SHR(x, n) (x >> n)
|
|
#define ROTR(x, n) ((x >> n) | (x << (32 - n)))
|
|
#define S0(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22))
|
|
#define S1(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25))
|
|
#define s0(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHR(x, 3))
|
|
#define s1(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHR(x, 10))
|
|
|
|
/* SHA256 round function */
|
|
#define RND(a, b, c, d, e, f, g, h, k) \
|
|
h += S1(e) + Ch(e, f, g) + k; \
|
|
d += h; \
|
|
h += S0(a) + Maj(a, b, c);
|
|
|
|
/* Adjusted round function for rotating state */
|
|
#define RNDr(S, W, i, ii) \
|
|
RND(S[(64 - i) % 8], S[(65 - i) % 8], \
|
|
S[(66 - i) % 8], S[(67 - i) % 8], \
|
|
S[(68 - i) % 8], S[(69 - i) % 8], \
|
|
S[(70 - i) % 8], S[(71 - i) % 8], \
|
|
W[i + ii] + K[i + ii])
|
|
|
|
/* Message schedule computation */
|
|
#define MSCH(W, ii, i) \
|
|
W[i + ii + 16] = s1(W[i + ii + 14]) + W[i + ii + 9] + s0(W[i + ii + 1]) + W[i + ii]
|
|
|
|
/*
|
|
* SHA256 block compression function. The 256-bit state is transformed via
|
|
* the 512-bit input block to produce a new state.
|
|
*/
|
|
static void
|
|
SHA256_Transform(uint32_t * state, const unsigned char block[64])
|
|
{
|
|
uint32_t W[64];
|
|
uint32_t S[8];
|
|
int i;
|
|
|
|
/* 1. Prepare the first part of the message schedule W. */
|
|
be32dec_vect(W, block, 64);
|
|
|
|
/* 2. Initialize working variables. */
|
|
memcpy(S, state, 32);
|
|
|
|
/* 3. Mix. */
|
|
for (i = 0; i < 64; i += 16) {
|
|
RNDr(S, W, 0, i);
|
|
RNDr(S, W, 1, i);
|
|
RNDr(S, W, 2, i);
|
|
RNDr(S, W, 3, i);
|
|
RNDr(S, W, 4, i);
|
|
RNDr(S, W, 5, i);
|
|
RNDr(S, W, 6, i);
|
|
RNDr(S, W, 7, i);
|
|
RNDr(S, W, 8, i);
|
|
RNDr(S, W, 9, i);
|
|
RNDr(S, W, 10, i);
|
|
RNDr(S, W, 11, i);
|
|
RNDr(S, W, 12, i);
|
|
RNDr(S, W, 13, i);
|
|
RNDr(S, W, 14, i);
|
|
RNDr(S, W, 15, i);
|
|
|
|
if (i == 48)
|
|
break;
|
|
MSCH(W, 0, i);
|
|
MSCH(W, 1, i);
|
|
MSCH(W, 2, i);
|
|
MSCH(W, 3, i);
|
|
MSCH(W, 4, i);
|
|
MSCH(W, 5, i);
|
|
MSCH(W, 6, i);
|
|
MSCH(W, 7, i);
|
|
MSCH(W, 8, i);
|
|
MSCH(W, 9, i);
|
|
MSCH(W, 10, i);
|
|
MSCH(W, 11, i);
|
|
MSCH(W, 12, i);
|
|
MSCH(W, 13, i);
|
|
MSCH(W, 14, i);
|
|
MSCH(W, 15, i);
|
|
}
|
|
|
|
/* 4. Mix local working variables into global state */
|
|
for (i = 0; i < 8; i++)
|
|
state[i] += S[i];
|
|
}
|
|
|
|
static unsigned char PAD[64] = {
|
|
0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
};
|
|
|
|
/* Add padding and terminating bit-count. */
|
|
static void
|
|
SHA256_Pad(SHA256_CTX * ctx)
|
|
{
|
|
size_t r;
|
|
|
|
/* Figure out how many bytes we have buffered. */
|
|
r = (ctx->count >> 3) & 0x3f;
|
|
|
|
/* Pad to 56 mod 64, transforming if we finish a block en route. */
|
|
if (r < 56) {
|
|
/* Pad to 56 mod 64. */
|
|
memcpy(&ctx->buf[r], PAD, 56 - r);
|
|
} else {
|
|
/* Finish the current block and mix. */
|
|
memcpy(&ctx->buf[r], PAD, 64 - r);
|
|
SHA256_Transform(ctx->state, ctx->buf);
|
|
|
|
/* The start of the final block is all zeroes. */
|
|
memset(&ctx->buf[0], 0, 56);
|
|
}
|
|
|
|
/* Add the terminating bit-count. */
|
|
be64enc(&ctx->buf[56], ctx->count);
|
|
|
|
/* Mix in the final block. */
|
|
SHA256_Transform(ctx->state, ctx->buf);
|
|
}
|
|
|
|
/* SHA-256 initialization. Begins a SHA-256 operation. */
|
|
void
|
|
SHA256_Init(SHA256_CTX * ctx)
|
|
{
|
|
|
|
/* Zero bits processed so far */
|
|
ctx->count = 0;
|
|
|
|
/* Magic initialization constants */
|
|
ctx->state[0] = 0x6A09E667;
|
|
ctx->state[1] = 0xBB67AE85;
|
|
ctx->state[2] = 0x3C6EF372;
|
|
ctx->state[3] = 0xA54FF53A;
|
|
ctx->state[4] = 0x510E527F;
|
|
ctx->state[5] = 0x9B05688C;
|
|
ctx->state[6] = 0x1F83D9AB;
|
|
ctx->state[7] = 0x5BE0CD19;
|
|
}
|
|
|
|
/* Add bytes into the hash */
|
|
void
|
|
SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len)
|
|
{
|
|
uint64_t bitlen;
|
|
uint32_t r;
|
|
const unsigned char *src = in;
|
|
|
|
/* Number of bytes left in the buffer from previous updates */
|
|
r = (ctx->count >> 3) & 0x3f;
|
|
|
|
/* Convert the length into a number of bits */
|
|
bitlen = len << 3;
|
|
|
|
/* Update number of bits */
|
|
ctx->count += bitlen;
|
|
|
|
/* Handle the case where we don't need to perform any transforms */
|
|
if (len < 64 - r) {
|
|
memcpy(&ctx->buf[r], src, len);
|
|
return;
|
|
}
|
|
|
|
/* Finish the current block */
|
|
memcpy(&ctx->buf[r], src, 64 - r);
|
|
SHA256_Transform(ctx->state, ctx->buf);
|
|
src += 64 - r;
|
|
len -= 64 - r;
|
|
|
|
/* Perform complete blocks */
|
|
while (len >= 64) {
|
|
SHA256_Transform(ctx->state, src);
|
|
src += 64;
|
|
len -= 64;
|
|
}
|
|
|
|
/* Copy left over data into buffer */
|
|
memcpy(ctx->buf, src, len);
|
|
}
|
|
|
|
/*
|
|
* SHA-256 finalization. Pads the input data, exports the hash value,
|
|
* and clears the context state.
|
|
*/
|
|
void
|
|
SHA256_Final(unsigned char digest[static SHA256_DIGEST_LENGTH], SHA256_CTX *ctx)
|
|
{
|
|
|
|
/* Add padding */
|
|
SHA256_Pad(ctx);
|
|
|
|
/* Write the hash */
|
|
be32enc_vect(digest, ctx->state, SHA256_DIGEST_LENGTH);
|
|
|
|
/* Clear the context state */
|
|
memset(ctx, 0, sizeof(*ctx));
|
|
}
|
|
|
|
#ifdef WEAK_REFS
|
|
/* When building libmd, provide weak references. Note: this is not
|
|
activated in the context of compiling these sources for internal
|
|
use in libcrypt.
|
|
*/
|
|
#undef SHA256_Init
|
|
__weak_reference(_libmd_SHA256_Init, SHA256_Init);
|
|
#undef SHA256_Update
|
|
__weak_reference(_libmd_SHA256_Update, SHA256_Update);
|
|
#undef SHA256_Final
|
|
__weak_reference(_libmd_SHA256_Final, SHA256_Final);
|
|
#undef SHA256_Transform
|
|
__weak_reference(_libmd_SHA256_Transform, SHA256_Transform);
|
|
#endif
|