dc4ee6ca91
make use of it where possible. This primarily brings in support for newer hardware, and FreeBSD is not yet able to support the abundance of IRQs on new hardware and many features in the Ethernet driver. Because of the changes to IRQs in the Simple Executive, we have to maintain our own list of Octeon IRQs now, which probably can be pared-down and be specific to the CIU interrupt unit soon, and when other interrupt mechanisms are added they can maintain their own definitions. Remove unmasking of interrupts from within the UART device now that the function used is no longer present in the Simple Executive. The unmasking seems to have been gratuitous as this is more properly handled by the buses above the UART device, and seems to work on that basis.
771 lines
22 KiB
C
771 lines
22 KiB
C
/***********************license start***************
|
|
* Copyright (c) 2003-2010 Cavium Inc. (support@cavium.com). All rights
|
|
* reserved.
|
|
*
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are
|
|
* met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
*
|
|
* * Redistributions in binary form must reproduce the above
|
|
* copyright notice, this list of conditions and the following
|
|
* disclaimer in the documentation and/or other materials provided
|
|
* with the distribution.
|
|
|
|
* * Neither the name of Cavium Inc. nor the names of
|
|
* its contributors may be used to endorse or promote products
|
|
* derived from this software without specific prior written
|
|
* permission.
|
|
|
|
* This Software, including technical data, may be subject to U.S. export control
|
|
* laws, including the U.S. Export Administration Act and its associated
|
|
* regulations, and may be subject to export or import regulations in other
|
|
* countries.
|
|
|
|
* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
|
|
* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
|
|
* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
|
|
* THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
|
|
* DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
|
|
* SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
|
|
* MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
|
|
* VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
|
|
* CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
|
|
* PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
|
|
***********************license end**************************************/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
* @file
|
|
*
|
|
* This file provides atomic operations
|
|
*
|
|
* <hr>$Revision: 70030 $<hr>
|
|
*
|
|
*
|
|
*/
|
|
|
|
|
|
#ifndef __CVMX_ATOMIC_H__
|
|
#define __CVMX_ATOMIC_H__
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 32 bit (aligned) memory location.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints. (This should NOT be used for reference counting -
|
|
* use the standard version instead.)
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*/
|
|
static inline void cvmx_atomic_add32_nosync(int32_t *ptr, int32_t incr)
|
|
{
|
|
if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
|
|
{
|
|
uint32_t tmp;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: ll %[tmp], %[val] \n"
|
|
" addu %[tmp], %[inc] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp)
|
|
: [inc] "r" (incr)
|
|
: "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
" saa %[inc], (%[base]) \n"
|
|
: "+m" (*ptr)
|
|
: [inc] "r" (incr), [base] "r" (ptr)
|
|
: "memory");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 32 bit (aligned) memory location.
|
|
*
|
|
* Memory access ordering is enforced before/after the atomic operation,
|
|
* so no additional 'sync' instructions are required.
|
|
*
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*/
|
|
static inline void cvmx_atomic_add32(int32_t *ptr, int32_t incr)
|
|
{
|
|
CVMX_SYNCWS;
|
|
cvmx_atomic_add32_nosync(ptr, incr);
|
|
CVMX_SYNCWS;
|
|
}
|
|
|
|
/**
|
|
* Atomically sets a 32 bit (aligned) memory location to a value
|
|
*
|
|
* @param ptr address of memory to set
|
|
* @param value value to set memory location to.
|
|
*/
|
|
static inline void cvmx_atomic_set32(int32_t *ptr, int32_t value)
|
|
{
|
|
CVMX_SYNCWS;
|
|
*ptr = value;
|
|
CVMX_SYNCWS;
|
|
}
|
|
|
|
/**
|
|
* Returns the current value of a 32 bit (aligned) memory
|
|
* location.
|
|
*
|
|
* @param ptr Address of memory to get
|
|
* @return Value of the memory
|
|
*/
|
|
static inline int32_t cvmx_atomic_get32(int32_t *ptr)
|
|
{
|
|
return *(volatile int32_t *)ptr;
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 64 bit (aligned) memory location.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints. (This should NOT be used for reference counting -
|
|
* use the standard version instead.)
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*/
|
|
static inline void cvmx_atomic_add64_nosync(int64_t *ptr, int64_t incr)
|
|
{
|
|
if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
|
|
{
|
|
uint64_t tmp;
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: lld %[tmp], %[val] \n"
|
|
" daddu %[tmp], %[inc] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp)
|
|
: [inc] "r" (incr)
|
|
: "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
" saad %[inc], (%[base]) \n"
|
|
: "+m" (*ptr)
|
|
: [inc] "r" (incr), [base] "r" (ptr)
|
|
: "memory");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 64 bit (aligned) memory location.
|
|
*
|
|
* Memory access ordering is enforced before/after the atomic operation,
|
|
* so no additional 'sync' instructions are required.
|
|
*
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*/
|
|
static inline void cvmx_atomic_add64(int64_t *ptr, int64_t incr)
|
|
{
|
|
CVMX_SYNCWS;
|
|
cvmx_atomic_add64_nosync(ptr, incr);
|
|
CVMX_SYNCWS;
|
|
}
|
|
|
|
/**
|
|
* Atomically sets a 64 bit (aligned) memory location to a value
|
|
*
|
|
* @param ptr address of memory to set
|
|
* @param value value to set memory location to.
|
|
*/
|
|
static inline void cvmx_atomic_set64(int64_t *ptr, int64_t value)
|
|
{
|
|
CVMX_SYNCWS;
|
|
*ptr = value;
|
|
CVMX_SYNCWS;
|
|
}
|
|
|
|
/**
|
|
* Returns the current value of a 64 bit (aligned) memory
|
|
* location.
|
|
*
|
|
* @param ptr Address of memory to get
|
|
* @return Value of the memory
|
|
*/
|
|
static inline int64_t cvmx_atomic_get64(int64_t *ptr)
|
|
{
|
|
return *(volatile int64_t *)ptr;
|
|
}
|
|
|
|
/**
|
|
* Atomically compares the old value with the value at ptr, and if they match,
|
|
* stores new_val to ptr.
|
|
* If *ptr and old don't match, function returns failure immediately.
|
|
* If *ptr and old match, function spins until *ptr updated to new atomically, or
|
|
* until *ptr and old no longer match
|
|
*
|
|
* Does no memory synchronization.
|
|
*
|
|
* @return 1 on success (match and store)
|
|
* 0 on no match
|
|
*/
|
|
static inline uint32_t cvmx_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
|
|
{
|
|
uint32_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: ll %[tmp], %[val] \n"
|
|
" li %[ret], 0 \n"
|
|
" bne %[tmp], %[old], 2f \n"
|
|
" move %[tmp], %[new_val] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" li %[ret], 1 \n"
|
|
"2: nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [old] "r" (old_val), [new_val] "r" (new_val)
|
|
: "memory");
|
|
|
|
return(ret);
|
|
|
|
}
|
|
|
|
/**
|
|
* Atomically compares the old value with the value at ptr, and if they match,
|
|
* stores new_val to ptr.
|
|
* If *ptr and old don't match, function returns failure immediately.
|
|
* If *ptr and old match, function spins until *ptr updated to new atomically, or
|
|
* until *ptr and old no longer match
|
|
*
|
|
* Does memory synchronization that is required to use this as a locking primitive.
|
|
*
|
|
* @return 1 on success (match and store)
|
|
* 0 on no match
|
|
*/
|
|
static inline uint32_t cvmx_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
|
|
{
|
|
uint32_t ret;
|
|
CVMX_SYNCWS;
|
|
ret = cvmx_atomic_compare_and_store32_nosync(ptr, old_val, new_val);
|
|
CVMX_SYNCWS;
|
|
return ret;
|
|
|
|
|
|
}
|
|
|
|
/**
|
|
* Atomically compares the old value with the value at ptr, and if they match,
|
|
* stores new_val to ptr.
|
|
* If *ptr and old don't match, function returns failure immediately.
|
|
* If *ptr and old match, function spins until *ptr updated to new atomically, or
|
|
* until *ptr and old no longer match
|
|
*
|
|
* Does no memory synchronization.
|
|
*
|
|
* @return 1 on success (match and store)
|
|
* 0 on no match
|
|
*/
|
|
static inline uint64_t cvmx_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
|
|
{
|
|
uint64_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: lld %[tmp], %[val] \n"
|
|
" li %[ret], 0 \n"
|
|
" bne %[tmp], %[old], 2f \n"
|
|
" move %[tmp], %[new_val] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" li %[ret], 1 \n"
|
|
"2: nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [old] "r" (old_val), [new_val] "r" (new_val)
|
|
: "memory");
|
|
|
|
return(ret);
|
|
|
|
}
|
|
|
|
/**
|
|
* Atomically compares the old value with the value at ptr, and if they match,
|
|
* stores new_val to ptr.
|
|
* If *ptr and old don't match, function returns failure immediately.
|
|
* If *ptr and old match, function spins until *ptr updated to new atomically, or
|
|
* until *ptr and old no longer match
|
|
*
|
|
* Does memory synchronization that is required to use this as a locking primitive.
|
|
*
|
|
* @return 1 on success (match and store)
|
|
* 0 on no match
|
|
*/
|
|
static inline uint64_t cvmx_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
|
|
{
|
|
uint64_t ret;
|
|
CVMX_SYNCWS;
|
|
ret = cvmx_atomic_compare_and_store64_nosync(ptr, old_val, new_val);
|
|
CVMX_SYNCWS;
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 64 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints. (This should NOT be used for reference counting -
|
|
* use the standard version instead.)
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*
|
|
* @return Value of memory location before increment
|
|
*/
|
|
static inline int64_t cvmx_atomic_fetch_and_add64_nosync(int64_t *ptr, int64_t incr)
|
|
{
|
|
uint64_t tmp, ret;
|
|
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
|
|
{
|
|
CVMX_PUSH_OCTEON2;
|
|
if (__builtin_constant_p(incr) && incr == 1)
|
|
{
|
|
__asm__ __volatile__(
|
|
"laid %0,(%2)"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
|
|
}
|
|
else if (__builtin_constant_p(incr) && incr == -1)
|
|
{
|
|
__asm__ __volatile__(
|
|
"ladd %0,(%2)"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
"laad %0,(%2),%3"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory");
|
|
}
|
|
CVMX_POP_OCTEON2;
|
|
}
|
|
else
|
|
{
|
|
#endif
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: lld %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" daddu %[tmp], %[inc] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [inc] "r" (incr)
|
|
: "memory");
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
}
|
|
#endif
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 64 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* Memory access ordering is enforced before/after the atomic operation,
|
|
* so no additional 'sync' instructions are required.
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*
|
|
* @return Value of memory location before increment
|
|
*/
|
|
static inline int64_t cvmx_atomic_fetch_and_add64(int64_t *ptr, int64_t incr)
|
|
{
|
|
uint64_t ret;
|
|
CVMX_SYNCWS;
|
|
ret = cvmx_atomic_fetch_and_add64_nosync(ptr, incr);
|
|
CVMX_SYNCWS;
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 32 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints. (This should NOT be used for reference counting -
|
|
* use the standard version instead.)
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*
|
|
* @return Value of memory location before increment
|
|
*/
|
|
static inline int32_t cvmx_atomic_fetch_and_add32_nosync(int32_t *ptr, int32_t incr)
|
|
{
|
|
uint32_t tmp, ret;
|
|
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
|
|
{
|
|
CVMX_PUSH_OCTEON2;
|
|
if (__builtin_constant_p(incr) && incr == 1)
|
|
{
|
|
__asm__ __volatile__(
|
|
"lai %0,(%2)"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
|
|
}
|
|
else if (__builtin_constant_p(incr) && incr == -1)
|
|
{
|
|
__asm__ __volatile__(
|
|
"lad %0,(%2)"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
"laa %0,(%2),%3"
|
|
: "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory");
|
|
}
|
|
CVMX_POP_OCTEON2;
|
|
}
|
|
else
|
|
{
|
|
#endif
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: ll %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" addu %[tmp], %[inc] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [inc] "r" (incr)
|
|
: "memory");
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
}
|
|
#endif
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically adds a signed value to a 32 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* Memory access ordering is enforced before/after the atomic operation,
|
|
* so no additional 'sync' instructions are required.
|
|
*
|
|
* @param ptr address in memory to add incr to
|
|
* @param incr amount to increment memory location by (signed)
|
|
*
|
|
* @return Value of memory location before increment
|
|
*/
|
|
static inline int32_t cvmx_atomic_fetch_and_add32(int32_t *ptr, int32_t incr)
|
|
{
|
|
uint32_t ret;
|
|
CVMX_SYNCWS;
|
|
ret = cvmx_atomic_fetch_and_add32_nosync(ptr, incr);
|
|
CVMX_SYNCWS;
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* Atomically set bits in a 64 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param mask mask of bits to set
|
|
*
|
|
* @return Value of memory location before setting bits
|
|
*/
|
|
static inline uint64_t cvmx_atomic_fetch_and_bset64_nosync(uint64_t *ptr, uint64_t mask)
|
|
{
|
|
uint64_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: lld %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" or %[tmp], %[msk] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [msk] "r" (mask)
|
|
: "memory");
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically set bits in a 32 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param mask mask of bits to set
|
|
*
|
|
* @return Value of memory location before setting bits
|
|
*/
|
|
static inline uint32_t cvmx_atomic_fetch_and_bset32_nosync(uint32_t *ptr, uint32_t mask)
|
|
{
|
|
uint32_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: ll %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" or %[tmp], %[msk] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [msk] "r" (mask)
|
|
: "memory");
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically clear bits in a 64 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param mask mask of bits to clear
|
|
*
|
|
* @return Value of memory location before clearing bits
|
|
*/
|
|
static inline uint64_t cvmx_atomic_fetch_and_bclr64_nosync(uint64_t *ptr, uint64_t mask)
|
|
{
|
|
uint64_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
" nor %[msk], 0 \n"
|
|
"1: lld %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" and %[tmp], %[msk] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask)
|
|
: : "memory");
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically clear bits in a 32 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param mask mask of bits to clear
|
|
*
|
|
* @return Value of memory location before clearing bits
|
|
*/
|
|
static inline uint32_t cvmx_atomic_fetch_and_bclr32_nosync(uint32_t *ptr, uint32_t mask)
|
|
{
|
|
uint32_t tmp, ret;
|
|
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
" nor %[msk], 0 \n"
|
|
"1: ll %[tmp], %[val] \n"
|
|
" move %[ret], %[tmp] \n"
|
|
" and %[tmp], %[msk] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask)
|
|
: : "memory");
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically swaps value in 64 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param new_val new value to write
|
|
*
|
|
* @return Value of memory location before swap operation
|
|
*/
|
|
static inline uint64_t cvmx_atomic_swap64_nosync(uint64_t *ptr, uint64_t new_val)
|
|
{
|
|
uint64_t tmp, ret;
|
|
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
|
|
{
|
|
CVMX_PUSH_OCTEON2;
|
|
if (__builtin_constant_p(new_val) && new_val == 0)
|
|
{
|
|
__asm__ __volatile__(
|
|
"lacd %0,(%1)"
|
|
: "=r" (ret) : "r" (ptr) : "memory");
|
|
}
|
|
else if (__builtin_constant_p(new_val) && new_val == ~0ull)
|
|
{
|
|
__asm__ __volatile__(
|
|
"lasd %0,(%1)"
|
|
: "=r" (ret) : "r" (ptr) : "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
"lawd %0,(%1),%2"
|
|
: "=r" (ret) : "r" (ptr), "r" (new_val) : "memory");
|
|
}
|
|
CVMX_POP_OCTEON2;
|
|
}
|
|
else
|
|
{
|
|
#endif
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: lld %[ret], %[val] \n"
|
|
" move %[tmp], %[new_val] \n"
|
|
" scd %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [new_val] "r" (new_val)
|
|
: "memory");
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
}
|
|
#endif
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/**
|
|
* Atomically swaps value in 32 bit (aligned) memory location,
|
|
* and returns previous value.
|
|
*
|
|
* This version does not perform 'sync' operations to enforce memory
|
|
* operations. This should only be used when there are no memory operation
|
|
* ordering constraints.
|
|
*
|
|
* @param ptr address in memory
|
|
* @param new_val new value to write
|
|
*
|
|
* @return Value of memory location before swap operation
|
|
*/
|
|
static inline uint32_t cvmx_atomic_swap32_nosync(uint32_t *ptr, uint32_t new_val)
|
|
{
|
|
uint32_t tmp, ret;
|
|
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
|
|
{
|
|
CVMX_PUSH_OCTEON2;
|
|
if (__builtin_constant_p(new_val) && new_val == 0)
|
|
{
|
|
__asm__ __volatile__(
|
|
"lac %0,(%1)"
|
|
: "=r" (ret) : "r" (ptr) : "memory");
|
|
}
|
|
else if (__builtin_constant_p(new_val) && new_val == ~0u)
|
|
{
|
|
__asm__ __volatile__(
|
|
"las %0,(%1)"
|
|
: "=r" (ret) : "r" (ptr) : "memory");
|
|
}
|
|
else
|
|
{
|
|
__asm__ __volatile__(
|
|
"law %0,(%1),%2"
|
|
: "=r" (ret) : "r" (ptr), "r" (new_val) : "memory");
|
|
}
|
|
CVMX_POP_OCTEON2;
|
|
}
|
|
else
|
|
{
|
|
#endif
|
|
__asm__ __volatile__(
|
|
".set noreorder \n"
|
|
"1: ll %[ret], %[val] \n"
|
|
" move %[tmp], %[new_val] \n"
|
|
" sc %[tmp], %[val] \n"
|
|
" beqz %[tmp], 1b \n"
|
|
" nop \n"
|
|
".set reorder \n"
|
|
: [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
|
|
: [new_val] "r" (new_val)
|
|
: "memory");
|
|
#if !defined(__FreeBSD__) || !defined(_KERNEL)
|
|
}
|
|
#endif
|
|
|
|
return (ret);
|
|
}
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif /* __CVMX_ATOMIC_H__ */
|