04b6fa8330
facilities as well as support for the Octeon 2 family of SoCs.
XXX Note that with our antediluvian assembler, we can't support some Octeon 2
instructions and fall back to using the old ones instead.
787 lines
22 KiB
C
787 lines
22 KiB
C
/***********************license start***************
|
|
* Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
|
|
* reserved.
|
|
*
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are
|
|
* met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
*
|
|
* * Redistributions in binary form must reproduce the above
|
|
* copyright notice, this list of conditions and the following
|
|
* disclaimer in the documentation and/or other materials provided
|
|
* with the distribution.
|
|
|
|
* * Neither the name of Cavium Networks nor the names of
|
|
* its contributors may be used to endorse or promote products
|
|
* derived from this software without specific prior written
|
|
* permission.
|
|
|
|
* This Software, including technical data, may be subject to U.S. export control
|
|
* laws, including the U.S. Export Administration Act and its associated
|
|
* regulations, and may be subject to export or import regulations in other
|
|
* countries.
|
|
|
|
* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
|
|
* AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
|
|
* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
|
|
* THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
|
|
* DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
|
|
* SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
|
|
* MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
|
|
* VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
|
|
* CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
|
|
* PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
|
|
***********************license end**************************************/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
* @file
|
|
*
|
|
* This file provides atomic operations
|
|
*
|
|
* <hr>$Revision: 49448 $<hr>
|
|
*
|
|
*
|
|
*/
|
|
|
|
|
|
#ifndef __CVMX_ATOMIC_H__
|
|
#define __CVMX_ATOMIC_H__
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 32 bit (aligned) memory location.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints. (This should NOT be used for reference counting -
|
|
* use the standard version instead.)
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*/
|
|
static inline void cvmx_atomic_add32_nosync(int32_t *ptr, int32_t incr)
|
|
{
|
|
if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
|
|
{
|
|
uint32_t tmp;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: ll %[tmp], %[val] \n"
|
|
" addu %[tmp], %[inc] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp)
|
|
: [inc] "r" (incr)
|
|
: "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
" saa %[inc], (%[base]) \n"
|
|
: "+m" (*ptr)
|
|
: [inc] "r" (incr), [base] "r" (ptr)
|
|
: "memory");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 32 bit (aligned) memory location.
|
|
*
|
|
* Memory access ordering is enforced before/after the atomic operation,
|
|
* so no additional 'sync' instructions are required.
|
|
*
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*/
|
|
static inline void cvmx_atomic_add32(int32_t *ptr, int32_t incr)
|
|
{
|
|
CVMX_SYNCWS;
|
|
cvmx_atomic_add32_nosync(ptr, incr);
|
|
CVMX_SYNCWS;
|
|
}
|
|
|
|
/**
|
|
* Atomically sets a 32 bit (aligned) memory location to a value
|
|
*
|
|
* @param ptr address of memory to set
|
|
* @param value value to set memory location to.
|
|
*/
|
|
static inline void cvmx_atomic_set32(int32_t *ptr, int32_t value)
|
|
{
|
|
CVMX_SYNCWS;
|
|
*ptr = value;
|
|
CVMX_SYNCWS;
|
|
}
|
|
|
|
/**
|
|
* Returns the current value of a 32 bit (aligned) memory
|
|
* location.
|
|
*
|
|
* @param ptr Address of memory to get
|
|
* @return Value of the memory
|
|
*/
|
|
static inline int32_t cvmx_atomic_get32(int32_t *ptr)
|
|
{
|
|
return *(volatile int32_t *)ptr;
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 64 bit (aligned) memory location.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints. (This should NOT be used for reference counting -
|
|
* use the standard version instead.)
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*/
|
|
static inline void cvmx_atomic_add64_nosync(int64_t *ptr, int64_t incr)
|
|
{
|
|
if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
|
|
{
|
|
uint64_t tmp;
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: lld %[tmp], %[val] \n"
|
|
" daddu %[tmp], %[inc] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp)
|
|
: [inc] "r" (incr)
|
|
: "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
" saad %[inc], (%[base]) \n"
|
|
: "+m" (*ptr)
|
|
: [inc] "r" (incr), [base] "r" (ptr)
|
|
: "memory");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 64 bit (aligned) memory location.
|
|
*
|
|
* Memory access ordering is enforced before/after the atomic operation,
|
|
* so no additional 'sync' instructions are required.
|
|
*
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*/
|
|
static inline void cvmx_atomic_add64(int64_t *ptr, int64_t incr)
|
|
{
|
|
CVMX_SYNCWS;
|
|
cvmx_atomic_add64_nosync(ptr, incr);
|
|
CVMX_SYNCWS;
|
|
}
|
|
|
|
/**
|
|
* Atomically sets a 64 bit (aligned) memory location to a value
|
|
*
|
|
* @param ptr address of memory to set
|
|
* @param value value to set memory location to.
|
|
*/
|
|
static inline void cvmx_atomic_set64(int64_t *ptr, int64_t value)
|
|
{
|
|
CVMX_SYNCWS;
|
|
*ptr = value;
|
|
CVMX_SYNCWS;
|
|
}
|
|
|
|
/**
|
|
* Returns the current value of a 64 bit (aligned) memory
|
|
* location.
|
|
*
|
|
* @param ptr Address of memory to get
|
|
* @return Value of the memory
|
|
*/
|
|
static inline int64_t cvmx_atomic_get64(int64_t *ptr)
|
|
{
|
|
return *(volatile int64_t *)ptr;
|
|
}
|
|
|
|
/**
|
|
* Atomically compares the old value with the value at ptr, and if they match,
|
|
* stores new_val to ptr.
|
|
* If *ptr and old don't match, function returns failure immediately.
|
|
* If *ptr and old match, function spins until *ptr updated to new atomically, or
|
|
* until *ptr and old no longer match
|
|
*
|
|
* Does no memory synchronization.
|
|
*
|
|
* @return 1 on success (match and store)
|
|
* 0 on no match
|
|
*/
|
|
static inline uint32_t cvmx_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
|
|
{
|
|
uint32_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: ll %[tmp], %[val] \n"
|
|
" li %[ret], 0 \n"
|
|
" bne %[tmp], %[old], 2f \n"
|
|
" move %[tmp], %[new_val] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" li %[ret], 1 \n"
|
|
"2: nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [old] "r" (old_val), [new_val] "r" (new_val)
|
|
: "memory");
|
|
|
|
return(ret);
|
|
|
|
}
|
|
|
|
/**
|
|
* Atomically compares the old value with the value at ptr, and if they match,
|
|
* stores new_val to ptr.
|
|
* If *ptr and old don't match, function returns failure immediately.
|
|
* If *ptr and old match, function spins until *ptr updated to new atomically, or
|
|
* until *ptr and old no longer match
|
|
*
|
|
* Does memory synchronization that is required to use this as a locking primitive.
|
|
*
|
|
* @return 1 on success (match and store)
|
|
* 0 on no match
|
|
*/
|
|
static inline uint32_t cvmx_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
|
|
{
|
|
uint32_t ret;
|
|
CVMX_SYNCWS;
|
|
ret = cvmx_atomic_compare_and_store32_nosync(ptr, old_val, new_val);
|
|
CVMX_SYNCWS;
|
|
return ret;
|
|
|
|
|
|
}
|
|
|
|
/**
|
|
* Atomically compares the old value with the value at ptr, and if they match,
|
|
* stores new_val to ptr.
|
|
* If *ptr and old don't match, function returns failure immediately.
|
|
* If *ptr and old match, function spins until *ptr updated to new atomically, or
|
|
* until *ptr and old no longer match
|
|
*
|
|
* Does no memory synchronization.
|
|
*
|
|
* @return 1 on success (match and store)
|
|
* 0 on no match
|
|
*/
|
|
static inline uint64_t cvmx_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
|
|
{
|
|
uint64_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: lld %[tmp], %[val] \n"
|
|
" li %[ret], 0 \n"
|
|
" bne %[tmp], %[old], 2f \n"
|
|
" move %[tmp], %[new_val] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" li %[ret], 1 \n"
|
|
"2: nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [old] "r" (old_val), [new_val] "r" (new_val)
|
|
: "memory");
|
|
|
|
return(ret);
|
|
|
|
}
|
|
|
|
/**
|
|
* Atomically compares the old value with the value at ptr, and if they match,
|
|
* stores new_val to ptr.
|
|
* If *ptr and old don't match, function returns failure immediately.
|
|
* If *ptr and old match, function spins until *ptr updated to new atomically, or
|
|
* until *ptr and old no longer match
|
|
*
|
|
* Does memory synchronization that is required to use this as a locking primitive.
|
|
*
|
|
* @return 1 on success (match and store)
|
|
* 0 on no match
|
|
*/
|
|
static inline uint64_t cvmx_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
|
|
{
|
|
uint64_t ret;
|
|
CVMX_SYNCWS;
|
|
ret = cvmx_atomic_compare_and_store64_nosync(ptr, old_val, new_val);
|
|
CVMX_SYNCWS;
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 64 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints. (This should NOT be used for reference counting -
|
|
* use the standard version instead.)
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*
|
|
* @return Value of memory location before increment
|
|
*/
|
|
static inline int64_t cvmx_atomic_fetch_and_add64_nosync(int64_t *ptr, int64_t incr)
|
|
{
|
|
uint64_t tmp, ret;
|
|
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
|
|
{
|
|
CVMX_PUSH_OCTEON2;
|
|
if (__builtin_constant_p(incr) && incr == 1)
|
|
{
|
|
__asm__ __volatile__(
|
|
"laid %0,(%2)"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
|
|
}
|
|
else if (__builtin_constant_p(incr) && incr == -1)
|
|
{
|
|
__asm__ __volatile__(
|
|
"ladd %0,(%2)"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
"laad %0,(%2),%3"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory");
|
|
}
|
|
CVMX_POP_OCTEON2;
|
|
}
|
|
else
|
|
{
|
|
#endif
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: lld %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" daddu %[tmp], %[inc] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [inc] "r" (incr)
|
|
: "memory");
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
}
|
|
#endif
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 64 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* Memory access ordering is enforced before/after the atomic operation,
|
|
* so no additional 'sync' instructions are required.
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*
|
|
* @return Value of memory location before increment
|
|
*/
|
|
static inline int64_t cvmx_atomic_fetch_and_add64(int64_t *ptr, int64_t incr)
|
|
{
|
|
uint64_t ret;
|
|
CVMX_SYNCWS;
|
|
ret = cvmx_atomic_fetch_and_add64_nosync(ptr, incr);
|
|
CVMX_SYNCWS;
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 32 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints. (This should NOT be used for reference counting -
|
|
* use the standard version instead.)
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*
|
|
* @return Value of memory location before increment
|
|
*/
|
|
static inline int32_t cvmx_atomic_fetch_and_add32_nosync(int32_t *ptr, int32_t incr)
|
|
{
|
|
uint32_t tmp, ret;
|
|
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
|
|
{
|
|
CVMX_PUSH_OCTEON2;
|
|
if (__builtin_constant_p(incr) && incr == 1)
|
|
{
|
|
__asm__ __volatile__(
|
|
"lai %0,(%2)"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
|
|
}
|
|
else if (__builtin_constant_p(incr) && incr == -1)
|
|
{
|
|
__asm__ __volatile__(
|
|
"lad %0,(%2)"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
"laa %0,(%2),%3"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory");
|
|
}
|
|
CVMX_POP_OCTEON2;
|
|
}
|
|
else
|
|
{
|
|
#endif
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: ll %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" addu %[tmp], %[inc] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [inc] "r" (incr)
|
|
: "memory");
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
}
|
|
#endif
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 32 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* Memory access ordering is enforced before/after the atomic operation,
|
|
* so no additional 'sync' instructions are required.
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*
|
|
* @return Value of memory location before increment
|
|
*/
|
|
static inline int32_t cvmx_atomic_fetch_and_add32(int32_t *ptr, int32_t incr)
|
|
{
|
|
uint32_t ret;
|
|
CVMX_SYNCWS;
|
|
ret = cvmx_atomic_fetch_and_add32_nosync(ptr, incr);
|
|
CVMX_SYNCWS;
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* Atomically set bits in a 64 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param mask mask of bits to set
|
|
*
|
|
* @return Value of memory location before setting bits
|
|
*/
|
|
static inline uint64_t cvmx_atomic_fetch_and_bset64_nosync(uint64_t *ptr, uint64_t mask)
|
|
{
|
|
uint64_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: lld %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" or %[tmp], %[msk] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [msk] "r" (mask)
|
|
: "memory");
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically set bits in a 32 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param mask mask of bits to set
|
|
*
|
|
* @return Value of memory location before setting bits
|
|
*/
|
|
static inline uint32_t cvmx_atomic_fetch_and_bset32_nosync(uint32_t *ptr, uint32_t mask)
|
|
{
|
|
uint32_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: ll %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" or %[tmp], %[msk] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [msk] "r" (mask)
|
|
: "memory");
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically clear bits in a 64 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param mask mask of bits to clear
|
|
*
|
|
* @return Value of memory location before clearing bits
|
|
*/
|
|
static inline uint64_t cvmx_atomic_fetch_and_bclr64_nosync(uint64_t *ptr, uint64_t mask)
|
|
{
|
|
uint64_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
" nor %[msk], 0 \n"
|
|
"1: lld %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" and %[tmp], %[msk] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask)
|
|
: : "memory");
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically clear bits in a 32 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param mask mask of bits to clear
|
|
*
|
|
* @return Value of memory location before clearing bits
|
|
*/
|
|
static inline uint32_t cvmx_atomic_fetch_and_bclr32_nosync(uint32_t *ptr, uint32_t mask)
|
|
{
|
|
uint32_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
" nor %[msk], 0 \n"
|
|
"1: ll %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" and %[tmp], %[msk] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask)
|
|
: : "memory");
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically swaps value in 64 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param new_val new value to write
|
|
*
|
|
* @return Value of memory location before swap operation
|
|
*/
|
|
static inline uint64_t cvmx_atomic_swap64_nosync(uint64_t *ptr, uint64_t new_val)
|
|
{
|
|
uint64_t tmp, ret;
|
|
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
|
|
{
|
|
CVMX_PUSH_OCTEON2;
|
|
if (__builtin_constant_p(new_val) && new_val == 0)
|
|
{
|
|
__asm__ __volatile__(
|
|
"lacd %0,(%1)"
|
|
: "=r" (ret) : "r" (ptr) : "memory");
|
|
}
|
|
else if (__builtin_constant_p(new_val) && new_val == ~0ull)
|
|
{
|
|
__asm__ __volatile__(
|
|
"lasd %0,(%1)"
|
|
: "=r" (ret) : "r" (ptr) : "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
"lawd %0,(%1),%2"
|
|
: "=r" (ret) : "r" (ptr), "r" (new_val) : "memory");
|
|
}
|
|
CVMX_POP_OCTEON2;
|
|
}
|
|
else
|
|
{
|
|
#endif
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: lld %[ret], %[val] \n"
|
|
" move %[tmp], %[new_val] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [new_val] "r" (new_val)
|
|
: "memory");
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
}
|
|
#endif
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically swaps value in 32 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param new_val new value to write
|
|
*
|
|
* @return Value of memory location before swap operation
|
|
*/
|
|
static inline uint32_t cvmx_atomic_swap32_nosync(uint32_t *ptr, uint32_t new_val)
|
|
{
|
|
uint32_t tmp, ret;
|
|
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
|
|
{
|
|
CVMX_PUSH_OCTEON2;
|
|
if (__builtin_constant_p(new_val) && new_val == 0)
|
|
{
|
|
__asm__ __volatile__(
|
|
"lac %0,(%1)"
|
|
: "=r" (ret) : "r" (ptr) : "memory");
|
|
}
|
|
else if (__builtin_constant_p(new_val) && new_val == ~0u)
|
|
{
|
|
__asm__ __volatile__(
|
|
"las %0,(%1)"
|
|
: "=r" (ret) : "r" (ptr) : "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
"law %0,(%1),%2"
|
|
: "=r" (ret) : "r" (ptr), "r" (new_val) : "memory");
|
|
}
|
|
CVMX_POP_OCTEON2;
|
|
}
|
|
else
|
|
{
|
|
#endif
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: ll %[ret], %[val] \n"
|
|
" move %[tmp], %[new_val] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [new_val] "r" (new_val)
|
|
: "memory");
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
}
|
|
#endif
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* This atomic operation is now named cvmx_atomic_compare_and_store32_nosync
|
|
* and the (deprecated) macro is provided for backward compatibility.
|
|
* @deprecated
|
|
*/
|
|
#define cvmx_atomic_compare_and_store_nosync32 cvmx_atomic_compare_and_store32_nosync
|
|
|
|
/**
|
|
* This atomic operation is now named cvmx_atomic_compare_and_store64_nosync
|
|
* and the (deprecated) macro is provided for backward compatibility.
|
|
* @deprecated
|
|
*/
|
|
#define cvmx_atomic_compare_and_store_nosync64 cvmx_atomic_compare_and_store64_nosync
|
|
|
|
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif /* __CVMX_ATOMIC_H__ */
|