mirror of https://github.com/F-Stack/f-stack.git
158 lines
4.1 KiB
C
158 lines
4.1 KiB
C
/* SPDX-License-Identifier: BSD-3-Clause
|
|
* Copyright(c) 2017 Cavium, Inc
|
|
* Copyright(c) 2019 Arm Limited
|
|
*/
|
|
|
|
#ifndef _RTE_PAUSE_ARM64_H_
|
|
#define _RTE_PAUSE_ARM64_H_
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
#include <rte_common.h>
|
|
|
|
#ifdef RTE_ARM_USE_WFE
|
|
#define RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
|
|
#endif
|
|
|
|
#include "generic/rte_pause.h"
|
|
|
|
static inline void rte_pause(void)
|
|
{
|
|
asm volatile("yield" ::: "memory");
|
|
}
|
|
|
|
#ifdef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
|
|
|
|
/* Send an event to quit WFE. */
|
|
#define __SEVL() { asm volatile("sevl" : : : "memory"); }
|
|
|
|
/* Put processor into low power WFE(Wait For Event) state. */
|
|
#define __WFE() { asm volatile("wfe" : : : "memory"); }
|
|
|
|
static __rte_always_inline void
|
|
rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
|
|
int memorder)
|
|
{
|
|
uint16_t value;
|
|
|
|
assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
|
|
|
|
/*
|
|
* Atomic exclusive load from addr, it returns the 16-bit content of
|
|
* *addr while making it 'monitored',when it is written by someone
|
|
* else, the 'monitored' state is cleared and a event is generated
|
|
* implicitly to exit WFE.
|
|
*/
|
|
#define __LOAD_EXC_16(src, dst, memorder) { \
|
|
if (memorder == __ATOMIC_RELAXED) { \
|
|
asm volatile("ldxrh %w[tmp], [%x[addr]]" \
|
|
: [tmp] "=&r" (dst) \
|
|
: [addr] "r"(src) \
|
|
: "memory"); \
|
|
} else { \
|
|
asm volatile("ldaxrh %w[tmp], [%x[addr]]" \
|
|
: [tmp] "=&r" (dst) \
|
|
: [addr] "r"(src) \
|
|
: "memory"); \
|
|
} }
|
|
|
|
__LOAD_EXC_16(addr, value, memorder)
|
|
if (value != expected) {
|
|
__SEVL()
|
|
do {
|
|
__WFE()
|
|
__LOAD_EXC_16(addr, value, memorder)
|
|
} while (value != expected);
|
|
}
|
|
#undef __LOAD_EXC_16
|
|
}
|
|
|
|
static __rte_always_inline void
|
|
rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
|
|
int memorder)
|
|
{
|
|
uint32_t value;
|
|
|
|
assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
|
|
|
|
/*
|
|
* Atomic exclusive load from addr, it returns the 32-bit content of
|
|
* *addr while making it 'monitored',when it is written by someone
|
|
* else, the 'monitored' state is cleared and a event is generated
|
|
* implicitly to exit WFE.
|
|
*/
|
|
#define __LOAD_EXC_32(src, dst, memorder) { \
|
|
if (memorder == __ATOMIC_RELAXED) { \
|
|
asm volatile("ldxr %w[tmp], [%x[addr]]" \
|
|
: [tmp] "=&r" (dst) \
|
|
: [addr] "r"(src) \
|
|
: "memory"); \
|
|
} else { \
|
|
asm volatile("ldaxr %w[tmp], [%x[addr]]" \
|
|
: [tmp] "=&r" (dst) \
|
|
: [addr] "r"(src) \
|
|
: "memory"); \
|
|
} }
|
|
|
|
__LOAD_EXC_32(addr, value, memorder)
|
|
if (value != expected) {
|
|
__SEVL()
|
|
do {
|
|
__WFE()
|
|
__LOAD_EXC_32(addr, value, memorder)
|
|
} while (value != expected);
|
|
}
|
|
#undef __LOAD_EXC_32
|
|
}
|
|
|
|
static __rte_always_inline void
|
|
rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
|
|
int memorder)
|
|
{
|
|
uint64_t value;
|
|
|
|
assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
|
|
|
|
/*
|
|
* Atomic exclusive load from addr, it returns the 64-bit content of
|
|
* *addr while making it 'monitored',when it is written by someone
|
|
* else, the 'monitored' state is cleared and a event is generated
|
|
* implicitly to exit WFE.
|
|
*/
|
|
#define __LOAD_EXC_64(src, dst, memorder) { \
|
|
if (memorder == __ATOMIC_RELAXED) { \
|
|
asm volatile("ldxr %x[tmp], [%x[addr]]" \
|
|
: [tmp] "=&r" (dst) \
|
|
: [addr] "r"(src) \
|
|
: "memory"); \
|
|
} else { \
|
|
asm volatile("ldaxr %x[tmp], [%x[addr]]" \
|
|
: [tmp] "=&r" (dst) \
|
|
: [addr] "r"(src) \
|
|
: "memory"); \
|
|
} }
|
|
|
|
__LOAD_EXC_64(addr, value, memorder)
|
|
if (value != expected) {
|
|
__SEVL()
|
|
do {
|
|
__WFE()
|
|
__LOAD_EXC_64(addr, value, memorder)
|
|
} while (value != expected);
|
|
}
|
|
}
|
|
#undef __LOAD_EXC_64
|
|
|
|
#undef __SEVL
|
|
#undef __WFE
|
|
|
|
#endif
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif /* _RTE_PAUSE_ARM64_H_ */
|