23#define DISPENSO_MAJOR_VERSION 1
24#define DISPENSO_MINOR_VERSION 3
26#if defined(DISPENSO_SHARED_LIB)
29#if defined(DISPENSO_LIB_EXPORT)
30#define DISPENSO_DLL_ACCESS __declspec(dllexport)
32#define DISPENSO_DLL_ACCESS __declspec(dllimport)
35#elif defined(__clang__) || defined(__GNUC__)
36#define DISPENSO_DLL_ACCESS __attribute__((visibility("default")))
40#if !defined(DISPENSO_DLL_ACCESS)
41#define DISPENSO_DLL_ACCESS
44using ssize_t = std::make_signed<std::size_t>::type;
46#if defined(__clang__) || defined(__GNUC__)
47#define DISPENSO_INLINE __attribute__((always_inline)) inline
48#elif defined(_MSC_VER) || defined(__INTEL_COMPILER)
49#define DISPENSO_INLINE __forceinline
51#define DISPENSO_INLINE inline
58constexpr size_t kCacheLineSize = 64;
67#define DISPENSO_THREAD_LOCAL __declspec(thread)
68#elif defined(__GNUC__) || defined(__clang__)
69#define DISPENSO_THREAD_LOCAL __thread
71#error Supply lightweight thread-locals for this compiler. Can define to thread_local if lightweight not available
74#if (defined(__GNUC__) || defined(__clang__))
75#define DISPENSO_EXPECT(a, b) __builtin_expect(a, b)
77#define DISPENSO_EXPECT(a, b) a
81#if (defined(__GNUC__) || defined(__clang__))
82#define DO_PRAGMA(X) _Pragma(#X)
83#define DISPENSO_DISABLE_WARNING_PUSH DO_PRAGMA(GCC diagnostic push)
84#define DISPENSO_DISABLE_WARNING_POP DO_PRAGMA(GCC diagnostic pop)
85#define DISPENSO_DISABLE_WARNING(warningName) DO_PRAGMA(GCC diagnostic ignored #warningName)
86#if !defined(__clang__)
87#define DISPENSO_DISABLE_WARNING_ZERO_VARIADIC_MACRO_ARGUMENTS
88#define DISPENSO_DISABLE_WARNING_GLOBAL_CONSTRUCTORS
90#define DISPENSO_DISABLE_WARNING_ZERO_VARIADIC_MACRO_ARGUMENTS \
91 DISPENSO_DISABLE_WARNING(-Wgnu-zero-variadic-macro-arguments)
92#define DISPENSO_DISABLE_WARNING_GLOBAL_CONSTRUCTORS \
93 DISPENSO_DISABLE_WARNING(-Wglobal-constructors)
95#elif defined(_MSC_VER)
96#define DISPENSO_DISABLE_WARNING_PUSH __pragma(warning(push))
97#define DISPENSO_DISABLE_WARNING_POP __pragma(warning(pop))
98#define DISPENSO_DISABLE_WARNING(warningNumber) __pragma(warning(disable : warningNumber))
99#define DISPENSO_DISABLE_WARNING_ZERO_VARIADIC_MACRO_ARGUMENTS
100#define DISPENSO_DISABLE_WARNING_GLOBAL_CONSTRUCTORS
102#define DISPENSO_DISABLE_WARNING_PUSH
103#define DISPENSO_DISABLE_WARNING_POP
104#define DISPENSO_DISABLE_WARNING_ZERO_VARIADIC_MACRO_ARGUMENTS
105#define DISPENSO_DISABLE_WARNING_GLOBAL_CONSTRUCTORS
118 operator const T&()
const {
123 alignas(kCacheLineSize)
T t_;
129struct AlignedBuffer {
130 alignas(
alignof(
T))
char b[
sizeof(
T)];
134struct alignas(kCacheLineSize) AlignedAtomic :
public std::atomic<T*> {};
136inline void* alignedMalloc(
size_t bytes,
size_t alignment) {
137 alignment = std::max(alignment,
sizeof(uintptr_t));
138 char* ptr =
reinterpret_cast<char*
>(::malloc(bytes + alignment));
139 uintptr_t base =
reinterpret_cast<uintptr_t
>(ptr);
140 uintptr_t oldBase = base;
141 uintptr_t mask = alignment - 1;
145 uintptr_t* recovery =
reinterpret_cast<uintptr_t*
>(base -
sizeof(uintptr_t));
147 return reinterpret_cast<void*
>(base);
150inline void* alignedMalloc(
size_t bytes) {
151 return alignedMalloc(bytes, kCacheLineSize);
154inline void alignedFree(
void* ptr) {
158 char* p =
reinterpret_cast<char*
>(ptr);
159 uintptr_t recovered = *
reinterpret_cast<uintptr_t*
>(p -
sizeof(uintptr_t));
160 ::free(
reinterpret_cast<void*
>(recovered));
164struct AlignedFreeDeleter {
165 void operator()(T* ptr) {
167 detail::alignedFree(ptr);
171struct AlignedFreeDeleter<void> {
172 void operator()(
void* ptr) {
173 detail::alignedFree(ptr);
177template <
typename T,
class... Args>
178std::shared_ptr<T> make_shared(Args&&... args) {
179 void* tv = alignedMalloc(
sizeof(T),
alignof(T));
180 T* t =
new (tv) T(std::forward<Args>(args)...);
181 return std::shared_ptr<T>(t, AlignedFreeDeleter<T>());
184inline constexpr uintptr_t alignToCacheLine(uintptr_t val) {
185 constexpr uintptr_t kMask = kCacheLineSize - 1;
191#if defined __x86_64__ || defined __i386__
192inline void cpuRelax() {
193 asm volatile(
"pause" :::
"memory");
195#elif defined __arm64__ || defined __aarch64__
196inline void cpuRelax() {
197 asm volatile(
"yield" :::
"memory");
199#elif defined __powerpc__ || defined __POWERPC__
201inline void cpuRelax() {
202 asm volatile(
"or r27,r27,r27" :::
"memory");
205inline void cpuRelax() {
206 asm volatile(
"or 27,27,27" :::
"memory");
211inline void cpuRelax() {}
220struct StaticChunking {
221 ssize_t transitionTaskIndex;
222 ssize_t ceilChunkSize;
225inline StaticChunking staticChunkSize(ssize_t items, ssize_t chunks) {
227 StaticChunking chunking;
228 chunking.ceilChunkSize = (items + chunks - 1) / chunks;
229 ssize_t numLeft = chunking.ceilChunkSize * chunks - items;
230 chunking.transitionTaskIndex = chunks - numLeft;
detail::OpResult< T > OpResult