42ddb8f1db
Two optimizations to MI strlen(3) inspired by David S. Miller's blog posting [1]. - Use word-sized test for unaligned pointer before working the hard way. Memory page boundary is always integral multiple of a word alignment boundary. Therefore, if we can access memory referenced by pointer p, then (p & ~word mask) must be also accessible. - Better utilization of multi-issue processor's ability of concurrency. The previous implementation utilized a formular that must be executed sequentially. However, the ~, & and - operations can actually be caculated at the same time when the operand were different and unrelated. The original Hacker's Delight formular also offered consistent performance regardless whether the input would contain characters with their highest-bit set, as it catches real nul characters only. These two optimizations has shown further improvements over the previous implementation on microbenchmarks on i386 and amd64 CPU including Pentium 4, Core Duo 2 and i7. [1] http://vger.kernel.org/~davem/cgi-bin/blog.cgi/2010/03/08#strlen_1
131 lines
3.8 KiB
C
131 lines
3.8 KiB
C
/*-
|
|
* Copyright (c) 2009, 2010 Xin LI <delphij@FreeBSD.org>
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include <sys/limits.h>
|
|
#include <sys/types.h>
|
|
#include <string.h>
|
|
|
|
/*
|
|
* Portable strlen() for 32-bit and 64-bit systems.
|
|
*
|
|
* Rationale: it is generally much more efficient to do word length
|
|
* operations and avoid branches on modern computer systems, as
|
|
* compared to byte-length operations with a lot of branches.
|
|
*
|
|
* The expression:
|
|
*
|
|
* ((x - 0x01....01) & ~x & 0x80....80)
|
|
*
|
|
* would evaluate to a non-zero value iff any of the bytes in the
|
|
* original word is zero.
|
|
*
|
|
* On multi-issue processors, we can divide the above expression into:
|
|
* a) (x - 0x01....01)
|
|
* b) (~x & 0x80....80)
|
|
* c) a & b
|
|
*
|
|
* Where, a) and b) can be partially computed in parallel.
|
|
*
|
|
* The algorithm above is found on "Hacker's Delight" by
|
|
* Henry S. Warren, Jr.
|
|
*/
|
|
|
|
/* Magic numbers for the algorithm */
|
|
#if LONG_BIT == 32
|
|
static const unsigned long mask01 = 0x01010101;
|
|
static const unsigned long mask80 = 0x80808080;
|
|
#elif LONG_BIT == 64
|
|
static const unsigned long mask01 = 0x0101010101010101;
|
|
static const unsigned long mask80 = 0x8080808080808080;
|
|
#else
|
|
#error Unsupported word size
|
|
#endif
|
|
|
|
#define LONGPTR_MASK (sizeof(long) - 1)
|
|
|
|
/*
|
|
* Helper macro to return string length if we caught the zero
|
|
* byte.
|
|
*/
|
|
#define testbyte(x) \
|
|
do { \
|
|
if (p[x] == '\0') \
|
|
return (p - str + x); \
|
|
} while (0)
|
|
|
|
size_t
|
|
strlen(const char *str)
|
|
{
|
|
const char *p;
|
|
const unsigned long *lp;
|
|
long va, vb;
|
|
|
|
/*
|
|
* Before trying the hard (unaligned byte-by-byte access) way
|
|
* to figure out whether there is a nul character, try to see
|
|
* if there is a nul character is within this accessible word
|
|
* first.
|
|
*
|
|
* p and (p & ~LONGPTR_MASK) must be equally accessible since
|
|
* they always fall in the same memory page, as long as page
|
|
* boundaries is integral multiple of word size.
|
|
*/
|
|
lp = (const unsigned long *)((uintptr_t)str & ~LONGPTR_MASK);
|
|
va = (*lp - mask01);
|
|
vb = ((~*lp) & mask80);
|
|
lp++;
|
|
if (va & vb)
|
|
/* Check if we have \0 in the first part */
|
|
for (p = str; p < (const char *)lp; p++)
|
|
if (*p == '\0')
|
|
return (p - str);
|
|
|
|
/* Scan the rest of the string using word sized operation */
|
|
for (; ; lp++) {
|
|
va = (*lp - mask01);
|
|
vb = ((~*lp) & mask80);
|
|
if (va & vb) {
|
|
p = (const char *)(lp);
|
|
testbyte(0);
|
|
testbyte(1);
|
|
testbyte(2);
|
|
testbyte(3);
|
|
#if (LONG_BIT >= 64)
|
|
testbyte(4);
|
|
testbyte(5);
|
|
testbyte(6);
|
|
testbyte(7);
|
|
#endif
|
|
}
|
|
}
|
|
|
|
/* NOTREACHED */
|
|
return (0);
|
|
}
|