diff --git a/rumur/resources/header.c b/rumur/resources/header.c index 7b4b6c31..10036547 100644 --- a/rumur/resources/header.c +++ b/rumur/resources/header.c @@ -2462,6 +2462,7 @@ static dword_t atomic_read(dword_t *p) { * ¹ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 */ #ifdef __x86_64__ +#ifndef __ILP32__ #ifdef __SSE2__ #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 #ifdef __has_include @@ -2488,6 +2489,7 @@ static dword_t atomic_read(dword_t *p) { #endif #endif #endif +#endif #if defined(__x86_64__) || defined(__i386__) || \ (defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)) @@ -2522,6 +2524,7 @@ static void atomic_write(dword_t *p, dword_t v) { * ¹ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 */ #ifdef __x86_64__ +#ifndef __ILP32__ #ifdef __SSE2__ #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 #ifdef __has_include @@ -2549,6 +2552,7 @@ static void atomic_write(dword_t *p, dword_t v) { #endif #endif #endif +#endif #if defined(__x86_64__) || defined(__i386__) || \ (defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)) diff --git a/rumur/src/rumur-run b/rumur/src/rumur-run index fe1f637f..30ece710 100644 --- a/rumur/src/rumur-run +++ b/rumur/src/rumur-run @@ -101,6 +101,14 @@ def needs_libatomic(): // replicate what is in ../resources/header.c +#ifdef __x86_64__ +#ifdef __has_include +#if __has_include() +#include +#endif +#endif +#endif + #define THREADS 2 #if __SIZEOF_POINTER__ <= 4 @@ -117,6 +125,40 @@ static dword_t atomic_read(dword_t *p) { return *p; } + /* 128-bit AVX loads are atomic.¹ So use that when possible. + * + * ¹ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 + */ +#ifdef __x86_64__ +#ifndef __ILP32__ +#ifdef __SSE2__ +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 +#ifdef __has_include +#if __has_include() +#ifdef __has_feature + /* TSan (falsely, I believe) considers a 128-bit load on a shared variable to + * be a data race + */ +#if !__has_feature(thread_sanitizer) + /* This is the only reliable way I have found of emitting a MOVDQA/MOVAPS. + * Surprisingly the Intel intrinsics for these do not reliably lower to the + * instruction they claim to, and inline assembly results in inefficient + * surrounding logic. + */ + { + typedef __m128i __attribute__((may_alias)) avx128_t; + volatile const avx128_t *const ptr = (const avx128_t *)p; + return (dword_t)*ptr; + } +#endif +#endif +#endif +#endif +#endif +#endif +#endif +#endif + #if defined(__x86_64__) || defined(__i386__) || \ (defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)) /* x86-64: MOV is not guaranteed to be atomic on 128-bit naturally aligned @@ -145,6 +187,41 @@ static void atomic_write(dword_t *p, dword_t v) { return; } + /* 128-bit AVX stores are atomic.¹ So use that when possible. + * + * ¹ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 + */ +#ifdef __x86_64__ +#ifndef __ILP32__ +#ifdef __SSE2__ +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 +#ifdef __has_include +#if __has_include() +#ifdef __has_feature + /* TSan (falsely, I believe) considers a 128-bit store on a shared variable to + * be a data race + */ +#if !__has_feature(thread_sanitizer) + /* This is the only reliable way I have found of emitting a MOVDQA/MOVAPS. + * Surprisingly the Intel intrinsics for these do not reliably lower to the + * instruction they claim to, and inline assembly results in inefficient + * surrounding logic. + */ + { + typedef __m128i __attribute__((may_alias)) avx128_t; + volatile avx128_t *const ptr = (avx128_t *)p; + *ptr = (__m128i)v; + return; + } +#endif +#endif +#endif +#endif +#endif +#endif +#endif +#endif + #if defined(__x86_64__) || defined(__i386__) || \ (defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)) /* As explained above, we need some extra gymnastics to avoid a call to diff --git a/tests/tests.py b/tests/tests.py index 9d062e96..87177085 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -130,6 +130,14 @@ def needs_libatomic(): // replicate what is in ../rumur/resources/header.c +#ifdef __x86_64__ +#ifdef __has_include +#if __has_include() +#include +#endif +#endif +#endif + #define THREADS 2 #if __SIZEOF_POINTER__ <= 4 @@ -146,6 +154,40 @@ def needs_libatomic(): return *p; } + /* 128-bit AVX loads are atomic.¹ So use that when possible. + * + * ¹ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 + */ +#ifdef __x86_64__ +#ifndef __ILP32__ +#ifdef __SSE2__ +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 +#ifdef __has_include +#if __has_include() +#ifdef __has_feature + /* TSan (falsely, I believe) considers a 128-bit load on a shared variable to + * be a data race + */ +#if !__has_feature(thread_sanitizer) + /* This is the only reliable way I have found of emitting a MOVDQA/MOVAPS. + * Surprisingly the Intel intrinsics for these do not reliably lower to the + * instruction they claim to, and inline assembly results in inefficient + * surrounding logic. + */ + { + typedef __m128i __attribute__((may_alias)) avx128_t; + volatile const avx128_t *const ptr = (const avx128_t *)p; + return (dword_t)*ptr; + } +#endif +#endif +#endif +#endif +#endif +#endif +#endif +#endif + #if defined(__x86_64__) || defined(__i386__) || \\ (defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)) /* x86-64: MOV is not guaranteed to be atomic on 128-bit naturally aligned @@ -174,6 +216,41 @@ def needs_libatomic(): return; } + /* 128-bit AVX stores are atomic.¹ So use that when possible. + * + * ¹ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 + */ +#ifdef __x86_64__ +#ifndef __ILP32__ +#ifdef __SSE2__ +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 +#ifdef __has_include +#if __has_include() +#ifdef __has_feature + /* TSan (falsely, I believe) considers a 128-bit store on a shared variable to + * be a data race + */ +#if !__has_feature(thread_sanitizer) + /* This is the only reliable way I have found of emitting a MOVDQA/MOVAPS. + * Surprisingly the Intel intrinsics for these do not reliably lower to the + * instruction they claim to, and inline assembly results in inefficient + * surrounding logic. + */ + { + typedef __m128i __attribute__((may_alias)) avx128_t; + volatile avx128_t *const ptr = (avx128_t *)p; + *ptr = (__m128i)v; + return; + } +#endif +#endif +#endif +#endif +#endif +#endif +#endif +#endif + #if defined(__x86_64__) || defined(__i386__) || \\ (defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)) /* As explained above, we need some extra gymnastics to avoid a call to