New Upstream Release - pthreadpool
Ready changes
Summary
Merged new upstream version: 0.0~git20230818.b1b6536 (was: 0.0~git20210507.1787867).
Resulting package
Built on 2023-08-18T17:52 (took 6m17s)
The resulting binary packages can be installed (if you have the apt repository enabled) by running one of:
apt install -t fresh-releases libpthreadpool-devapt install -t fresh-releases libpthreadpool0-dbgsymapt install -t fresh-releases libpthreadpool0
Diff
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c1cba55..7fa4285 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,7 +1,7 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.5 FATAL_ERROR)
# ---[ Project
-PROJECT(pthreadpool C CXX)
+PROJECT(pthreadpool C)
# ---[ Options.
SET(PTHREADPOOL_LIBRARY_TYPE "default" CACHE STRING "Type of library (shared, static, or default) to build")
@@ -25,6 +25,10 @@ ENDIF()
# ---[ CMake options
INCLUDE(GNUInstallDirs)
+IF(PTHREADPOOL_BUILD_TESTS OR PTHREADPOOL_BUILD_BENCHMARKS)
+ ENABLE_LANGUAGE(CXX)
+ENDIF()
+
IF(PTHREADPOOL_BUILD_TESTS)
ENABLE_TESTING()
ENDIF()
diff --git a/cmake/DownloadGoogleTest.cmake b/cmake/DownloadGoogleTest.cmake
index 2231ff7..d2d9ce4 100644
--- a/cmake/DownloadGoogleTest.cmake
+++ b/cmake/DownloadGoogleTest.cmake
@@ -4,8 +4,8 @@ PROJECT(googletest-download NONE)
INCLUDE(ExternalProject)
ExternalProject_Add(googletest
- URL https://github.com/google/googletest/archive/release-1.10.0.zip
- URL_HASH SHA256=94c634d499558a76fa649edb13721dce6e98fb1e7018dfaeba3cd7a083945e91
+ URL https://github.com/google/googletest/archive/release-1.12.0.zip
+ URL_HASH SHA256=ce7366fe57eb49928311189cb0e40e0a8bf3d3682fca89af30d884c25e983786
SOURCE_DIR "${CMAKE_BINARY_DIR}/googletest-source"
BINARY_DIR "${CMAKE_BINARY_DIR}/googletest"
CONFIGURE_COMMAND ""
diff --git a/debian/changelog b/debian/changelog
index 2e4c90b..659164c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+pthreadpool (0.0~git20230818.b1b6536-1) UNRELEASED; urgency=low
+
+ * New upstream snapshot.
+
+ -- Debian Janitor <janitor@jelmer.uk> Fri, 18 Aug 2023 17:46:38 -0000
+
pthreadpool (0.0~git20210507.1787867-1) unstable; urgency=medium
* Upload to unstable.
diff --git a/debian/patches/nodownload b/debian/patches/nodownload
index 853515f..d389d32 100644
--- a/debian/patches/nodownload
+++ b/debian/patches/nodownload
@@ -1,6 +1,8 @@
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -37,13 +37,7 @@
+Index: pthreadpool.git/CMakeLists.txt
+===================================================================
+--- pthreadpool.git.orig/CMakeLists.txt
++++ pthreadpool.git/CMakeLists.txt
+@@ -41,13 +41,7 @@ ENDMACRO()
# ---[ Download deps
IF(NOT DEFINED FXDIV_SOURCE_DIR)
@@ -15,7 +17,7 @@
ENDIF()
IF(PTHREADPOOL_BUILD_TESTS AND NOT DEFINED GOOGLETEST_SOURCE_DIR)
-@@ -57,13 +51,9 @@
+@@ -61,13 +55,9 @@ IF(PTHREADPOOL_BUILD_TESTS AND NOT DEFIN
ENDIF()
IF(PTHREADPOOL_BUILD_BENCHMARKS AND NOT DEFINED GOOGLEBENCHMARK_SOURCE_DIR)
@@ -32,7 +34,7 @@
ENDIF()
# ---[ pthreadpool library
-@@ -131,6 +121,7 @@
+@@ -135,6 +125,7 @@ ENDIF()
SET_TARGET_PROPERTIES(pthreadpool PROPERTIES
C_STANDARD 11
C_EXTENSIONS NO)
@@ -40,7 +42,7 @@
TARGET_LINK_LIBRARIES(pthreadpool PUBLIC pthreadpool_interface)
TARGET_INCLUDE_DIRECTORIES(pthreadpool PRIVATE src)
IF(NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
-@@ -149,16 +140,6 @@
+@@ -153,16 +144,6 @@ IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
TARGET_COMPILE_DEFINITIONS(pthreadpool PRIVATE _GNU_SOURCE=1)
ENDIF()
diff --git a/include/pthreadpool.h b/include/pthreadpool.h
index d02e64d..2e20b66 100644
--- a/include/pthreadpool.h
+++ b/include/pthreadpool.h
@@ -13,6 +13,7 @@ typedef void (*pthreadpool_task_2d_tile_1d_t)(void*, size_t, size_t, size_t);
typedef void (*pthreadpool_task_2d_tile_2d_t)(void*, size_t, size_t, size_t, size_t);
typedef void (*pthreadpool_task_3d_t)(void*, size_t, size_t, size_t);
typedef void (*pthreadpool_task_3d_tile_1d_t)(void*, size_t, size_t, size_t, size_t);
+typedef void (*pthreadpool_task_3d_tile_1d_with_thread_t)(void*, size_t, size_t, size_t, size_t, size_t);
typedef void (*pthreadpool_task_3d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t);
typedef void (*pthreadpool_task_4d_t)(void*, size_t, size_t, size_t, size_t);
typedef void (*pthreadpool_task_4d_tile_1d_t)(void*, size_t, size_t, size_t, size_t, size_t);
@@ -25,10 +26,15 @@ typedef void (*pthreadpool_task_6d_tile_1d_t)(void*, size_t, size_t, size_t, siz
typedef void (*pthreadpool_task_6d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t);
typedef void (*pthreadpool_task_1d_with_id_t)(void*, uint32_t, size_t);
+typedef void (*pthreadpool_task_2d_tile_1d_with_id_t)(void*, uint32_t, size_t, size_t, size_t);
typedef void (*pthreadpool_task_2d_tile_2d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t);
+typedef void (*pthreadpool_task_3d_tile_1d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t);
typedef void (*pthreadpool_task_3d_tile_2d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t, size_t);
typedef void (*pthreadpool_task_4d_tile_2d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t, size_t, size_t);
+typedef void (*pthreadpool_task_2d_tile_1d_with_id_with_thread_t)(void*, uint32_t, size_t, size_t, size_t, size_t);
+typedef void (*pthreadpool_task_3d_tile_1d_with_id_with_thread_t)(void*, uint32_t, size_t, size_t, size_t, size_t, size_t);
+
/**
* Disable support for denormalized numbers to the maximum extent possible for
@@ -265,6 +271,109 @@ void pthreadpool_parallelize_2d_tile_1d(
size_t tile_j,
uint32_t flags);
+/**
+ * Process items on a 2D grid with the specified maximum tile size along the
+ * last grid dimension using a microarchitecture-aware task function.
+ *
+ * The function implements a parallel version of the following snippet:
+ *
+ * uint32_t uarch_index = cpuinfo_initialize() ?
+ * cpuinfo_get_current_uarch_index() : default_uarch_index;
+ * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index;
+ * for (size_t i = 0; i < range_i; i++)
+ * for (size_t j = 0; j < range_j; j += tile_j)
+ * function(context, uarch_index, i, j, min(range_j - j, tile_j));
+ *
+ * When the function returns, all items have been processed and the thread pool
+ * is ready for a new task.
+ *
+ * @note If multiple threads call this function with the same thread pool, the
+ * calls are serialized.
+ *
+ * @param threadpool the thread pool to use for parallelisation. If threadpool
+ * is NULL, all items are processed serially on the calling thread.
+ * @param function the function to call for each tile.
+ * @param context the first argument passed to the specified function.
+ * @param default_uarch_index the microarchitecture index to use when
+ * pthreadpool is configured without cpuinfo, cpuinfo initialization failed,
+ * or index returned by cpuinfo_get_current_uarch_index() exceeds the
+ * max_uarch_index value.
+ * @param max_uarch_index the maximum microarchitecture index expected by
+ * the specified function. If the index returned by
+ * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index
+ * will be used instead. default_uarch_index can exceed max_uarch_index.
+ * @param range_i the number of items to process along the first dimension
+ * of the 2D grid.
+ * @param range_j the number of items to process along the second dimension
+ * of the 2D grid.
+ * @param tile_j the maximum number of items along the second dimension of
+ * the 2D grid to process in one function call.
+ * @param flags a bitwise combination of zero or more optional flags
+ * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS)
+ */
+void pthreadpool_parallelize_2d_tile_1d_with_uarch(
+ pthreadpool_t threadpool,
+ pthreadpool_task_2d_tile_1d_with_id_t function,
+ void* context,
+ uint32_t default_uarch_index,
+ uint32_t max_uarch_index,
+ size_t range_i,
+ size_t range_j,
+ size_t tile_j,
+ uint32_t flags);
+
+/**
+ * Process items on a 2D grid with the specified maximum tile size along the
+ * last grid dimension using a microarchitecture-aware task function and passing
+ * along the current thread id.
+ *
+ * The function implements a parallel version of the following snippet:
+ *
+ * uint32_t uarch_index = cpuinfo_initialize() ?
+ * cpuinfo_get_current_uarch_index() : default_uarch_index;
+ * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index;
+ * for (size_t i = 0; i < range_i; i++)
+ * for (size_t j = 0; j < range_j; j += tile_j)
+ * function(context, uarch_index, thread_index, i, j, min(range_j - j, tile_j));
+ *
+ * When the function returns, all items have been processed and the thread pool
+ * is ready for a new task.
+ *
+ * @note If multiple threads call this function with the same thread pool, the
+ * calls are serialized.
+ *
+ * @param threadpool the thread pool to use for parallelisation. If threadpool
+ * is NULL, all items are processed serially on the calling thread.
+ * @param function the function to call for each tile.
+ * @param context the first argument passed to the specified function.
+ * @param default_uarch_index the microarchitecture index to use when
+ * pthreadpool is configured without cpuinfo, cpuinfo initialization failed,
+ * or index returned by cpuinfo_get_current_uarch_index() exceeds the
+ * max_uarch_index value.
+ * @param max_uarch_index the maximum microarchitecture index expected by
+ * the specified function. If the index returned by
+ * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index
+ * will be used instead. default_uarch_index can exceed max_uarch_index.
+ * @param range_i the number of items to process along the first dimension
+ * of the 2D grid.
+ * @param range_j the number of items to process along the second dimension
+ * of the 2D grid.
+ * @param tile_j the maximum number of items along the second dimension of
+ * the 2D grid to process in one function call.
+ * @param flags a bitwise combination of zero or more optional flags
+ * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS)
+ */
+void pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
+ pthreadpool_t threadpool,
+ pthreadpool_task_2d_tile_1d_with_id_with_thread_t function,
+ void* context,
+ uint32_t default_uarch_index,
+ uint32_t max_uarch_index,
+ size_t range_i,
+ size_t range_j,
+ size_t tile_j,
+ uint32_t flags);
+
/**
* Process items on a 2D grid with the specified maximum tile size along each
* grid dimension.
@@ -447,6 +556,165 @@ void pthreadpool_parallelize_3d_tile_1d(
size_t tile_k,
uint32_t flags);
+/**
+ * Process items on a 3D grid with the specified maximum tile size along the
+ * last grid dimension and passing along the current thread id.
+ *
+ * The function implements a parallel version of the following snippet:
+ *
+ * for (size_t i = 0; i < range_i; i++)
+ * for (size_t j = 0; j < range_j; j++)
+ * for (size_t k = 0; k < range_k; k += tile_k)
+ * function(context, thread_index, i, j, k, min(range_k - k, tile_k));
+ *
+ * When the function returns, all items have been processed and the thread pool
+ * is ready for a new task.
+ *
+ * @note If multiple threads call this function with the same thread pool, the
+ * calls are serialized.
+ *
+ * @param threadpool the thread pool to use for parallelisation. If threadpool
+ * is NULL, all items are processed serially on the calling thread.
+ * @param function the function to call for each tile.
+ * @param context the first argument passed to the specified function.
+ * @param range_i the number of items to process along the first dimension
+ * of the 3D grid.
+ * @param range_j the number of items to process along the second dimension
+ * of the 3D grid.
+ * @param range_k the number of items to process along the third dimension
+ * of the 3D grid.
+ * @param tile_k the maximum number of items along the third dimension of
+ * the 3D grid to process in one function call.
+ * @param flags a bitwise combination of zero or more optional flags
+ * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS)
+ */
+void pthreadpool_parallelize_3d_tile_1d_with_thread(
+ pthreadpool_t threadpool,
+ pthreadpool_task_3d_tile_1d_with_thread_t function,
+ void* context,
+ size_t range_i,
+ size_t range_j,
+ size_t range_k,
+ size_t tile_k,
+ uint32_t flags);
+
+/**
+ * Process items on a 3D grid with the specified maximum tile size along the
+ * last grid dimension using a microarchitecture-aware task function.
+ *
+ * The function implements a parallel version of the following snippet:
+ *
+ * uint32_t uarch_index = cpuinfo_initialize() ?
+ * cpuinfo_get_current_uarch_index() : default_uarch_index;
+ * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index;
+ * for (size_t i = 0; i < range_i; i++)
+ * for (size_t j = 0; j < range_j; j++)
+ * for (size_t k = 0; k < range_k; k += tile_k)
+ * function(context, uarch_index, i, j, k, min(range_k - k, tile_k));
+ *
+ * When the function returns, all items have been processed and the thread pool
+ * is ready for a new task.
+ *
+ * @note If multiple threads call this function with the same thread pool, the
+ * calls are serialized.
+ *
+ * @param threadpool the thread pool to use for parallelisation. If
+ * threadpool is NULL, all items are processed serially on the calling
+ * thread.
+ * @param function the function to call for each tile.
+ * @param context the first argument passed to the specified
+ * function.
+ * @param default_uarch_index the microarchitecture index to use when
+ * pthreadpool is configured without cpuinfo, cpuinfo initialization failed,
+ * or index returned by cpuinfo_get_current_uarch_index() exceeds the
+ * max_uarch_index value.
+ * @param max_uarch_index the maximum microarchitecture index expected by
+ * the specified function. If the index returned by
+ * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index
+ * will be used instead. default_uarch_index can exceed max_uarch_index.
+ * @param range_i the number of items to process along the first
+ * dimension of the 3D grid.
+ * @param range_j the number of items to process along the second
+ * dimension of the 3D grid.
+ * @param range_k the number of items to process along the third
+ * dimension of the 3D grid.
+ * @param tile_k the maximum number of items along the third
+ * dimension of the 3D grid to process in one function call.
+ * @param flags a bitwise combination of zero or more optional
+ * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or
+ * PTHREADPOOL_FLAG_YIELD_WORKERS)
+ */
+void pthreadpool_parallelize_3d_tile_1d_with_uarch(
+ pthreadpool_t threadpool,
+ pthreadpool_task_3d_tile_1d_with_id_t function,
+ void* context,
+ uint32_t default_uarch_index,
+ uint32_t max_uarch_index,
+ size_t range_i,
+ size_t range_j,
+ size_t range_k,
+ size_t tile_k,
+ uint32_t flags);
+
+/**
+ * Process items on a 3D grid with the specified maximum tile size along the
+ * last grid dimension using a microarchitecture-aware task function and passing
+ * along the current thread id.
+ *
+ * The function implements a parallel version of the following snippet:
+ *
+ * uint32_t uarch_index = cpuinfo_initialize() ?
+ * cpuinfo_get_current_uarch_index() : default_uarch_index;
+ * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index;
+ * for (size_t i = 0; i < range_i; i++)
+ * for (size_t j = 0; j < range_j; j++)
+ * for (size_t k = 0; k < range_k; k += tile_k)
+ * function(context, uarch_index, thread_index, i, j, k, min(range_k - k, tile_k));
+ *
+ * When the function returns, all items have been processed and the thread pool
+ * is ready for a new task.
+ *
+ * @note If multiple threads call this function with the same thread pool, the
+ * calls are serialized.
+ *
+ * @param threadpool the thread pool to use for parallelisation. If
+ * threadpool is NULL, all items are processed serially on the calling
+ * thread.
+ * @param function the function to call for each tile.
+ * @param context the first argument passed to the specified
+ * function.
+ * @param default_uarch_index the microarchitecture index to use when
+ * pthreadpool is configured without cpuinfo, cpuinfo initialization failed,
+ * or index returned by cpuinfo_get_current_uarch_index() exceeds the
+ * max_uarch_index value.
+ * @param max_uarch_index the maximum microarchitecture index expected by
+ * the specified function. If the index returned by
+ * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index
+ * will be used instead. default_uarch_index can exceed max_uarch_index.
+ * @param range_i the number of items to process along the first
+ * dimension of the 3D grid.
+ * @param range_j the number of items to process along the second
+ * dimension of the 3D grid.
+ * @param range_k the number of items to process along the third
+ * dimension of the 3D grid.
+ * @param tile_k the maximum number of items along the third
+ * dimension of the 3D grid to process in one function call.
+ * @param flags a bitwise combination of zero or more optional
+ * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or
+ * PTHREADPOOL_FLAG_YIELD_WORKERS)
+ */
+void pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
+ pthreadpool_t threadpool,
+ pthreadpool_task_3d_tile_1d_with_id_with_thread_t function,
+ void* context,
+ uint32_t default_uarch_index,
+ uint32_t max_uarch_index,
+ size_t range_i,
+ size_t range_j,
+ size_t range_k,
+ size_t tile_k,
+ uint32_t flags);
+
/**
* Process items on a 3D grid with the specified maximum tile size along the
* last two grid dimensions.
diff --git a/src/fastpath.c b/src/fastpath.c
index b914ff0..876037d 100644
--- a/src/fastpath.c
+++ b/src/fastpath.c
@@ -241,6 +241,125 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_1d_fastpath(
pthreadpool_fence_release();
}
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_1d_with_uarch_fastpath(
+ struct pthreadpool* threadpool,
+ struct thread_info* thread)
+{
+ assert(threadpool != NULL);
+ assert(thread != NULL);
+
+ const pthreadpool_task_2d_tile_1d_with_id_t task = (pthreadpool_task_2d_tile_1d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+ void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+ const uint32_t default_uarch_index = threadpool->params.parallelize_2d_tile_1d_with_uarch.default_uarch_index;
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > threadpool->params.parallelize_2d_tile_1d_with_uarch.max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ const size_t threads_count = threadpool->threads_count.value;
+ const size_t range_threshold = -threads_count;
+
+ /* Process thread's own range of items */
+ const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+ const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.tile_range_j;
+ const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(range_start, tile_range_j);
+ const size_t tile_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.tile_j;
+ size_t i = tile_index_i_j.quotient;
+ size_t start_j = tile_index_i_j.remainder * tile_j;
+
+ const size_t range_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.range_j;
+ while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+ task(argument, uarch_index, i, start_j, min(range_j - start_j, tile_j));
+ start_j += tile_j;
+ if (start_j >= range_j) {
+ start_j = 0;
+ i += 1;
+ }
+ }
+
+ /* There still may be other threads with work */
+ const size_t thread_number = thread->thread_number;
+ for (size_t tid = modulo_decrement(thread_number, threads_count);
+ tid != thread_number;
+ tid = modulo_decrement(tid, threads_count))
+ {
+ struct thread_info* other_thread = &threadpool->threads[tid];
+ while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+ const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+ const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(linear_index, tile_range_j);
+ const size_t start_j = tile_index_i_j.remainder * tile_j;
+ task(argument, uarch_index, tile_index_i_j.quotient, start_j, min(range_j - start_j, tile_j));
+ }
+ }
+
+ /* Make changes by this thread visible to other threads */
+ pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_1d_with_uarch_with_thread_fastpath(
+ struct pthreadpool* threadpool,
+ struct thread_info* thread)
+{
+ assert(threadpool != NULL);
+ assert(thread != NULL);
+
+ const pthreadpool_task_2d_tile_1d_with_id_with_thread_t task =
+ (pthreadpool_task_2d_tile_1d_with_id_with_thread_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+ void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+ const uint32_t default_uarch_index = threadpool->params.parallelize_2d_tile_1d_with_uarch.default_uarch_index;
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > threadpool->params.parallelize_2d_tile_1d_with_uarch.max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ const size_t threads_count = threadpool->threads_count.value;
+ const size_t range_threshold = -threads_count;
+
+ /* Process thread's own range of items */
+ const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+ const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.tile_range_j;
+ const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(range_start, tile_range_j);
+ const size_t tile_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.tile_j;
+ size_t i = tile_index_i_j.quotient;
+ size_t start_j = tile_index_i_j.remainder * tile_j;
+
+ const size_t range_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.range_j;
+ const size_t thread_number = thread->thread_number;
+ while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+ task(argument, uarch_index, thread_number, i, start_j, min(range_j - start_j, tile_j));
+ start_j += tile_j;
+ if (start_j >= range_j) {
+ start_j = 0;
+ i += 1;
+ }
+ }
+
+ /* There still may be other threads with work */
+ for (size_t tid = modulo_decrement(thread_number, threads_count);
+ tid != thread_number;
+ tid = modulo_decrement(tid, threads_count))
+ {
+ struct thread_info* other_thread = &threadpool->threads[tid];
+ while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+ const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+ const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(linear_index, tile_range_j);
+ const size_t start_j = tile_index_i_j.remainder * tile_j;
+ task(argument, uarch_index, tid, tile_index_i_j.quotient, start_j, min(range_j - start_j, tile_j));
+ }
+ }
+
+ /* Make changes by this thread visible to other threads */
+ pthreadpool_fence_release();
+}
+
PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_fastpath(
struct pthreadpool* threadpool,
struct thread_info* thread)
@@ -466,6 +585,196 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_1d_fastpath(
pthreadpool_fence_release();
}
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_1d_with_thread_fastpath(
+ struct pthreadpool* threadpool,
+ struct thread_info* thread)
+{
+ assert(threadpool != NULL);
+ assert(thread != NULL);
+
+ const pthreadpool_task_3d_tile_1d_with_thread_t task = (pthreadpool_task_3d_tile_1d_with_thread_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+ void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+ const size_t threads_count = threadpool->threads_count.value;
+ const size_t range_threshold = -threads_count;
+
+ /* Process thread's own range of items */
+ const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+ const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_1d.tile_range_k;
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k);
+ const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_3d_tile_1d.range_j;
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t tile_k = threadpool->params.parallelize_3d_tile_1d.tile_k;
+ size_t i = index_i_j.quotient;
+ size_t j = index_i_j.remainder;
+ size_t start_k = tile_index_ij_k.remainder * tile_k;
+
+ const size_t range_k = threadpool->params.parallelize_3d_tile_1d.range_k;
+ const size_t thread_number = thread->thread_number;
+ while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+ task(argument, thread_number, i, j, start_k, min(range_k - start_k, tile_k));
+ start_k += tile_k;
+ if (start_k >= range_k) {
+ start_k = 0;
+ if (++j == range_j.value) {
+ j = 0;
+ i += 1;
+ }
+ }
+ }
+
+ /* There still may be other threads with work */
+ for (size_t tid = modulo_decrement(thread_number, threads_count);
+ tid != thread_number;
+ tid = modulo_decrement(tid, threads_count))
+ {
+ struct thread_info* other_thread = &threadpool->threads[tid];
+ while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+ const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k);
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t start_k = tile_index_ij_k.remainder * tile_k;
+ task(argument, tid, index_i_j.quotient, index_i_j.remainder, start_k, min(range_k - start_k, tile_k));
+ }
+ }
+
+ /* Make changes by this thread visible to other threads */
+ pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_1d_with_uarch_fastpath(
+ struct pthreadpool* threadpool,
+ struct thread_info* thread)
+{
+ assert(threadpool != NULL);
+ assert(thread != NULL);
+
+ const pthreadpool_task_3d_tile_1d_with_id_t task = (pthreadpool_task_3d_tile_1d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+ void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+ const uint32_t default_uarch_index = threadpool->params.parallelize_3d_tile_1d_with_uarch.default_uarch_index;
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > threadpool->params.parallelize_3d_tile_1d_with_uarch.max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ const size_t threads_count = threadpool->threads_count.value;
+ const size_t range_threshold = -threads_count;
+
+ /* Process thread's own range of items */
+ const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+ const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.tile_range_k;
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k);
+ const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_3d_tile_1d_with_uarch.range_j;
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t tile_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.tile_k;
+ size_t i = index_i_j.quotient;
+ size_t j = index_i_j.remainder;
+ size_t start_k = tile_index_ij_k.remainder * tile_k;
+
+ const size_t range_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.range_k;
+ while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+ task(argument, uarch_index, i, j, start_k, min(range_k - start_k, tile_k));
+ start_k += tile_k;
+ if (start_k >= range_k) {
+ start_k = 0;
+ if (++j == range_j.value) {
+ j = 0;
+ i += 1;
+ }
+ }
+ }
+
+ /* There still may be other threads with work */
+ const size_t thread_number = thread->thread_number;
+ for (size_t tid = modulo_decrement(thread_number, threads_count);
+ tid != thread_number;
+ tid = modulo_decrement(tid, threads_count))
+ {
+ struct thread_info* other_thread = &threadpool->threads[tid];
+ while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+ const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k);
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t start_k = tile_index_ij_k.remainder * tile_k;
+ task(argument, uarch_index, index_i_j.quotient, index_i_j.remainder, start_k, min(range_k - start_k, tile_k));
+ }
+ }
+
+ /* Make changes by this thread visible to other threads */
+ pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_1d_with_uarch_with_thread_fastpath(
+ struct pthreadpool* threadpool,
+ struct thread_info* thread)
+{
+ assert(threadpool != NULL);
+ assert(thread != NULL);
+
+ const pthreadpool_task_3d_tile_1d_with_id_with_thread_t task =
+ (pthreadpool_task_3d_tile_1d_with_id_with_thread_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+ void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+ const uint32_t default_uarch_index = threadpool->params.parallelize_3d_tile_1d_with_uarch.default_uarch_index;
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > threadpool->params.parallelize_3d_tile_1d_with_uarch.max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ const size_t threads_count = threadpool->threads_count.value;
+ const size_t range_threshold = -threads_count;
+
+ /* Process thread's own range of items */
+ const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+ const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.tile_range_k;
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k);
+ const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_3d_tile_1d_with_uarch.range_j;
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t tile_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.tile_k;
+ size_t i = index_i_j.quotient;
+ size_t j = index_i_j.remainder;
+ size_t start_k = tile_index_ij_k.remainder * tile_k;
+
+ const size_t range_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.range_k;
+ const size_t thread_number = thread->thread_number;
+ while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+ task(argument, uarch_index, thread_number, i, j, start_k, min(range_k - start_k, tile_k));
+ start_k += tile_k;
+ if (start_k >= range_k) {
+ start_k = 0;
+ if (++j == range_j.value) {
+ j = 0;
+ i += 1;
+ }
+ }
+ }
+
+ /* There still may be other threads with work */
+ for (size_t tid = modulo_decrement(thread_number, threads_count);
+ tid != thread_number;
+ tid = modulo_decrement(tid, threads_count))
+ {
+ struct thread_info* other_thread = &threadpool->threads[tid];
+ while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+ const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k);
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t start_k = tile_index_ij_k.remainder * tile_k;
+ task(argument, uarch_index, tid, index_i_j.quotient, index_i_j.remainder, start_k, min(range_k - start_k, tile_k));
+ }
+ }
+
+ /* Make changes by this thread visible to other threads */
+ pthreadpool_fence_release();
+}
+
PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_fastpath(
struct pthreadpool* threadpool,
struct thread_info* thread)
diff --git a/src/portable-api.c b/src/portable-api.c
index 42d0369..983649e 100644
--- a/src/portable-api.c
+++ b/src/portable-api.c
@@ -223,6 +223,115 @@ static void thread_parallelize_2d_tile_1d(struct pthreadpool* threadpool, struct
pthreadpool_fence_release();
}
+static void thread_parallelize_2d_tile_1d_with_uarch(struct pthreadpool* threadpool, struct thread_info* thread) {
+ assert(threadpool != NULL);
+ assert(thread != NULL);
+
+ const pthreadpool_task_2d_tile_1d_with_id_t task = (pthreadpool_task_2d_tile_1d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+ void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+ const uint32_t default_uarch_index = threadpool->params.parallelize_2d_tile_1d_with_uarch.default_uarch_index;
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > threadpool->params.parallelize_2d_tile_1d_with_uarch.max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ /* Process thread's own range of items */
+ const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+ const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.tile_range_j;
+ const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(range_start, tile_range_j);
+ const size_t tile_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.tile_j;
+ size_t i = tile_index_i_j.quotient;
+ size_t start_j = tile_index_i_j.remainder * tile_j;
+
+ const size_t range_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.range_j;
+ while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) {
+ task(argument, uarch_index, i, start_j, min(range_j - start_j, tile_j));
+ start_j += tile_j;
+ if (start_j >= range_j) {
+ start_j = 0;
+ i += 1;
+ }
+ }
+
+ /* There still may be other threads with work */
+ const size_t thread_number = thread->thread_number;
+ const size_t threads_count = threadpool->threads_count.value;
+ for (size_t tid = modulo_decrement(thread_number, threads_count);
+ tid != thread_number;
+ tid = modulo_decrement(tid, threads_count))
+ {
+ struct thread_info* other_thread = &threadpool->threads[tid];
+ while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) {
+ const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+ const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(linear_index, tile_range_j);
+ const size_t start_j = tile_index_i_j.remainder * tile_j;
+ task(argument, uarch_index, tile_index_i_j.quotient, start_j, min(range_j - start_j, tile_j));
+ }
+ }
+
+ /* Make changes by this thread visible to other threads */
+ pthreadpool_fence_release();
+}
+
+static void thread_parallelize_2d_tile_1d_with_uarch_with_thread(struct pthreadpool* threadpool, struct thread_info* thread) {
+ assert(threadpool != NULL);
+ assert(thread != NULL);
+
+ const pthreadpool_task_2d_tile_1d_with_id_with_thread_t task =
+ (pthreadpool_task_2d_tile_1d_with_id_with_thread_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+ void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+ const uint32_t default_uarch_index = threadpool->params.parallelize_2d_tile_1d_with_uarch.default_uarch_index;
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > threadpool->params.parallelize_2d_tile_1d_with_uarch.max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ /* Process thread's own range of items */
+ const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+ const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.tile_range_j;
+ const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(range_start, tile_range_j);
+ const size_t tile_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.tile_j;
+ size_t i = tile_index_i_j.quotient;
+ size_t start_j = tile_index_i_j.remainder * tile_j;
+
+ const size_t thread_number = thread->thread_number;
+ const size_t range_j = threadpool->params.parallelize_2d_tile_1d_with_uarch.range_j;
+ while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) {
+ task(argument, uarch_index, thread_number, i, start_j, min(range_j - start_j, tile_j));
+ start_j += tile_j;
+ if (start_j >= range_j) {
+ start_j = 0;
+ i += 1;
+ }
+ }
+
+ /* There still may be other threads with work */
+ const size_t threads_count = threadpool->threads_count.value;
+ for (size_t tid = modulo_decrement(thread_number, threads_count);
+ tid != thread_number;
+ tid = modulo_decrement(tid, threads_count))
+ {
+ struct thread_info* other_thread = &threadpool->threads[tid];
+ while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) {
+ const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+ const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(linear_index, tile_range_j);
+ const size_t start_j = tile_index_i_j.remainder * tile_j;
+ task(argument, uarch_index, tid, tile_index_i_j.quotient, start_j, min(range_j - start_j, tile_j));
+ }
+ }
+
+ /* Make changes by this thread visible to other threads */
+ pthreadpool_fence_release();
+}
+
static void thread_parallelize_2d_tile_2d(struct pthreadpool* threadpool, struct thread_info* thread) {
assert(threadpool != NULL);
assert(thread != NULL);
@@ -428,6 +537,181 @@ static void thread_parallelize_3d_tile_1d(struct pthreadpool* threadpool, struct
pthreadpool_fence_release();
}
+static void thread_parallelize_3d_tile_1d_with_thread(struct pthreadpool* threadpool, struct thread_info* thread) {
+ assert(threadpool != NULL);
+ assert(thread != NULL);
+
+ const pthreadpool_task_3d_tile_1d_with_thread_t task = (pthreadpool_task_3d_tile_1d_with_thread_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+ void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+ /* Process thread's own range of items */
+ const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+ const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_1d.tile_range_k;
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k);
+ const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_3d_tile_1d.range_j;
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t tile_k = threadpool->params.parallelize_3d_tile_1d.tile_k;
+ size_t i = index_i_j.quotient;
+ size_t j = index_i_j.remainder;
+ size_t start_k = tile_index_ij_k.remainder * tile_k;
+
+ const size_t thread_number = thread->thread_number;
+ const size_t range_k = threadpool->params.parallelize_3d_tile_1d.range_k;
+ while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) {
+ task(argument, thread_number, i, j, start_k, min(range_k - start_k, tile_k));
+ start_k += tile_k;
+ if (start_k >= range_k) {
+ start_k = 0;
+ if (++j == range_j.value) {
+ j = 0;
+ i += 1;
+ }
+ }
+ }
+
+ /* There still may be other threads with work */
+ const size_t threads_count = threadpool->threads_count.value;
+ for (size_t tid = modulo_decrement(thread_number, threads_count);
+ tid != thread_number;
+ tid = modulo_decrement(tid, threads_count))
+ {
+ struct thread_info* other_thread = &threadpool->threads[tid];
+ while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) {
+ const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k);
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t start_k = tile_index_ij_k.remainder * tile_k;
+ task(argument, tid, index_i_j.quotient, index_i_j.remainder, start_k, min(range_k - start_k, tile_k));
+ }
+ }
+
+ /* Make changes by this thread visible to other threads */
+ pthreadpool_fence_release();
+}
+
+static void thread_parallelize_3d_tile_1d_with_uarch(struct pthreadpool* threadpool, struct thread_info* thread) {
+ assert(threadpool != NULL);
+ assert(thread != NULL);
+
+ const pthreadpool_task_3d_tile_1d_with_id_t task = (pthreadpool_task_3d_tile_1d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+ void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+ const uint32_t default_uarch_index = threadpool->params.parallelize_3d_tile_1d_with_uarch.default_uarch_index;
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > threadpool->params.parallelize_3d_tile_1d_with_uarch.max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ /* Process thread's own range of items */
+ const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+ const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.tile_range_k;
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k);
+ const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_3d_tile_1d_with_uarch.range_j;
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t tile_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.tile_k;
+ size_t i = index_i_j.quotient;
+ size_t j = index_i_j.remainder;
+ size_t start_k = tile_index_ij_k.remainder * tile_k;
+
+ const size_t range_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.range_k;
+ while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) {
+ task(argument, uarch_index, i, j, start_k, min(range_k - start_k, tile_k));
+ start_k += tile_k;
+ if (start_k >= range_k) {
+ start_k = 0;
+ if (++j == range_j.value) {
+ j = 0;
+ i += 1;
+ }
+ }
+ }
+
+ /* There still may be other threads with work */
+ const size_t thread_number = thread->thread_number;
+ const size_t threads_count = threadpool->threads_count.value;
+ for (size_t tid = modulo_decrement(thread_number, threads_count);
+ tid != thread_number;
+ tid = modulo_decrement(tid, threads_count))
+ {
+ struct thread_info* other_thread = &threadpool->threads[tid];
+ while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) {
+ const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k);
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t start_k = tile_index_ij_k.remainder * tile_k;
+ task(argument, uarch_index, index_i_j.quotient, index_i_j.remainder, start_k, min(range_k - start_k, tile_k));
+ }
+ }
+
+ /* Make changes by this thread visible to other threads */
+ pthreadpool_fence_release();
+}
+
+static void thread_parallelize_3d_tile_1d_with_uarch_with_thread(struct pthreadpool* threadpool, struct thread_info* thread) {
+ assert(threadpool != NULL);
+ assert(thread != NULL);
+
+ const pthreadpool_task_3d_tile_1d_with_id_with_thread_t task =
+ (pthreadpool_task_3d_tile_1d_with_id_with_thread_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+ void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+ const uint32_t default_uarch_index = threadpool->params.parallelize_3d_tile_1d_with_uarch.default_uarch_index;
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > threadpool->params.parallelize_3d_tile_1d_with_uarch.max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ /* Process thread's own range of items */
+ const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+ const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.tile_range_k;
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k);
+ const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_3d_tile_1d_with_uarch.range_j;
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t tile_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.tile_k;
+ size_t i = index_i_j.quotient;
+ size_t j = index_i_j.remainder;
+ size_t start_k = tile_index_ij_k.remainder * tile_k;
+
+ const size_t thread_number = thread->thread_number;
+ const size_t range_k = threadpool->params.parallelize_3d_tile_1d_with_uarch.range_k;
+ while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) {
+ task(argument, uarch_index, thread_number, i, j, start_k, min(range_k - start_k, tile_k));
+ start_k += tile_k;
+ if (start_k >= range_k) {
+ start_k = 0;
+ if (++j == range_j.value) {
+ j = 0;
+ i += 1;
+ }
+ }
+ }
+
+ /* There still may be other threads with work */
+ const size_t threads_count = threadpool->threads_count.value;
+ for (size_t tid = modulo_decrement(thread_number, threads_count);
+ tid != thread_number;
+ tid = modulo_decrement(tid, threads_count))
+ {
+ struct thread_info* other_thread = &threadpool->threads[tid];
+ while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) {
+ const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+ const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k);
+ const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j);
+ const size_t start_k = tile_index_ij_k.remainder * tile_k;
+ task(argument, uarch_index, tid, index_i_j.quotient, index_i_j.remainder, start_k, min(range_k - start_k, tile_k));
+ }
+ }
+
+ /* Make changes by this thread visible to other threads */
+ pthreadpool_fence_release();
+}
+
static void thread_parallelize_3d_tile_2d(struct pthreadpool* threadpool, struct thread_info* thread) {
assert(threadpool != NULL);
assert(thread != NULL);
@@ -1442,6 +1726,124 @@ void pthreadpool_parallelize_2d_tile_1d(
}
}
+void pthreadpool_parallelize_2d_tile_1d_with_uarch(
+ pthreadpool_t threadpool,
+ pthreadpool_task_2d_tile_1d_with_id_t task,
+ void* argument,
+ uint32_t default_uarch_index,
+ uint32_t max_uarch_index,
+ size_t range_i,
+ size_t range_j,
+ size_t tile_j,
+ uint32_t flags)
+{
+ size_t threads_count;
+ if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= 1 && range_j <= tile_j)) {
+ /* No thread pool used: execute task sequentially on the calling thread */
+
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ struct fpu_state saved_fpu_state = { 0 };
+ if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
+ saved_fpu_state = get_fpu_state();
+ disable_fpu_denormals();
+ }
+ for (size_t i = 0; i < range_i; i++) {
+ for (size_t j = 0; j < range_j; j += tile_j) {
+ task(argument, uarch_index, i, j, min(range_j - j, tile_j));
+ }
+ }
+ if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
+ set_fpu_state(saved_fpu_state);
+ }
+ } else {
+ const size_t tile_range_j = divide_round_up(range_j, tile_j);
+ const size_t tile_range = range_i * tile_range_j;
+ const struct pthreadpool_2d_tile_1d_with_uarch_params params = {
+ .default_uarch_index = default_uarch_index,
+ .max_uarch_index = max_uarch_index,
+ .range_j = range_j,
+ .tile_j = tile_j,
+ .tile_range_j = fxdiv_init_size_t(tile_range_j),
+ };
+ thread_function_t parallelize_2d_tile_1d_with_uarch = &thread_parallelize_2d_tile_1d_with_uarch;
+ #if PTHREADPOOL_USE_FASTPATH
+ const size_t range_threshold = -threads_count;
+ if (tile_range < range_threshold) {
+ parallelize_2d_tile_1d_with_uarch = &pthreadpool_thread_parallelize_2d_tile_1d_with_uarch_fastpath;
+ }
+ #endif
+ pthreadpool_parallelize(
+ threadpool, parallelize_2d_tile_1d_with_uarch, ¶ms, sizeof(params),
+ task, argument, tile_range, flags);
+ }
+}
+
+void pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
+ pthreadpool_t threadpool,
+ pthreadpool_task_2d_tile_1d_with_id_with_thread_t task,
+ void* argument,
+ uint32_t default_uarch_index,
+ uint32_t max_uarch_index,
+ size_t range_i,
+ size_t range_j,
+ size_t tile_j,
+ uint32_t flags)
+{
+ size_t threads_count;
+ if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= 1 && range_j <= tile_j)) {
+ /* No thread pool used: execute task sequentially on the calling thread */
+
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ struct fpu_state saved_fpu_state = { 0 };
+ if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
+ saved_fpu_state = get_fpu_state();
+ disable_fpu_denormals();
+ }
+ for (size_t i = 0; i < range_i; i++) {
+ for (size_t j = 0; j < range_j; j += tile_j) {
+ task(argument, uarch_index, 0, i, j, min(range_j - j, tile_j));
+ }
+ }
+ if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
+ set_fpu_state(saved_fpu_state);
+ }
+ } else {
+ const size_t tile_range_j = divide_round_up(range_j, tile_j);
+ const size_t tile_range = range_i * tile_range_j;
+ const struct pthreadpool_2d_tile_1d_with_uarch_params params = {
+ .default_uarch_index = default_uarch_index,
+ .max_uarch_index = max_uarch_index,
+ .range_j = range_j,
+ .tile_j = tile_j,
+ .tile_range_j = fxdiv_init_size_t(tile_range_j),
+ };
+ thread_function_t parallelize_2d_tile_1d_with_uarch_with_thread = &thread_parallelize_2d_tile_1d_with_uarch_with_thread;
+ #if PTHREADPOOL_USE_FASTPATH
+ const size_t range_threshold = -threads_count;
+ if (tile_range < range_threshold) {
+ parallelize_2d_tile_1d_with_uarch_with_thread = &pthreadpool_thread_parallelize_2d_tile_1d_with_uarch_with_thread_fastpath;
+ }
+ #endif
+ pthreadpool_parallelize(
+ threadpool, parallelize_2d_tile_1d_with_uarch_with_thread, ¶ms, sizeof(params),
+ task, argument, tile_range, flags);
+ }
+}
+
void pthreadpool_parallelize_2d_tile_2d(
pthreadpool_t threadpool,
pthreadpool_task_2d_tile_2d_t task,
@@ -1651,6 +2053,182 @@ void pthreadpool_parallelize_3d_tile_1d(
}
}
+void pthreadpool_parallelize_3d_tile_1d_with_thread(
+ pthreadpool_t threadpool,
+ pthreadpool_task_3d_tile_1d_with_thread_t task,
+ void* argument,
+ size_t range_i,
+ size_t range_j,
+ size_t range_k,
+ size_t tile_k,
+ uint32_t flags)
+{
+ size_t threads_count;
+ if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k)) {
+ /* No thread pool used: execute task sequentially on the calling thread */
+ struct fpu_state saved_fpu_state = { 0 };
+ if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
+ saved_fpu_state = get_fpu_state();
+ disable_fpu_denormals();
+ }
+ for (size_t i = 0; i < range_i; i++) {
+ for (size_t j = 0; j < range_j; j++) {
+ for (size_t k = 0; k < range_k; k += tile_k) {
+ task(argument, 0, i, j, k, min(range_k - k, tile_k));
+ }
+ }
+ }
+ if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
+ set_fpu_state(saved_fpu_state);
+ }
+ } else {
+ const size_t tile_range_k = divide_round_up(range_k, tile_k);
+ const size_t tile_range = range_i * range_j * tile_range_k;
+ const struct pthreadpool_3d_tile_1d_params params = {
+ .range_k = range_k,
+ .tile_k = tile_k,
+ .range_j = fxdiv_init_size_t(range_j),
+ .tile_range_k = fxdiv_init_size_t(tile_range_k),
+ };
+ thread_function_t parallelize_3d_tile_1d_with_thread = &thread_parallelize_3d_tile_1d_with_thread;
+ #if PTHREADPOOL_USE_FASTPATH
+ const size_t range_threshold = -threads_count;
+ if (tile_range < range_threshold) {
+ parallelize_3d_tile_1d_with_thread = &pthreadpool_thread_parallelize_3d_tile_1d_with_thread_fastpath;
+ }
+ #endif
+ pthreadpool_parallelize(
+ threadpool, parallelize_3d_tile_1d_with_thread, ¶ms, sizeof(params),
+ task, argument, tile_range, flags);
+ }
+}
+
+void pthreadpool_parallelize_3d_tile_1d_with_uarch(
+ pthreadpool_t threadpool,
+ pthreadpool_task_3d_tile_1d_with_id_t task,
+ void* argument,
+ uint32_t default_uarch_index,
+ uint32_t max_uarch_index,
+ size_t range_i,
+ size_t range_j,
+ size_t range_k,
+ size_t tile_k,
+ uint32_t flags)
+{
+ size_t threads_count;
+ if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k)) {
+ /* No thread pool used: execute task sequentially on the calling thread */
+
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ struct fpu_state saved_fpu_state = { 0 };
+ if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
+ saved_fpu_state = get_fpu_state();
+ disable_fpu_denormals();
+ }
+ for (size_t i = 0; i < range_i; i++) {
+ for (size_t j = 0; j < range_j; j++) {
+ for (size_t k = 0; k < range_k; k += tile_k) {
+ task(argument, uarch_index, i, j, k, min(range_k - k, tile_k));
+ }
+ }
+ }
+ if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
+ set_fpu_state(saved_fpu_state);
+ }
+ } else {
+ const size_t tile_range_k = divide_round_up(range_k, tile_k);
+ const size_t tile_range = range_i * range_j * tile_range_k;
+ const struct pthreadpool_3d_tile_1d_with_uarch_params params = {
+ .default_uarch_index = default_uarch_index,
+ .max_uarch_index = max_uarch_index,
+ .range_k = range_k,
+ .tile_k = tile_k,
+ .range_j = fxdiv_init_size_t(range_j),
+ .tile_range_k = fxdiv_init_size_t(tile_range_k),
+ };
+ thread_function_t parallelize_3d_tile_1d_with_uarch = &thread_parallelize_3d_tile_1d_with_uarch;
+ #if PTHREADPOOL_USE_FASTPATH
+ const size_t range_threshold = -threads_count;
+ if (tile_range < range_threshold) {
+ parallelize_3d_tile_1d_with_uarch = &pthreadpool_thread_parallelize_3d_tile_1d_with_uarch_fastpath;
+ }
+ #endif
+ pthreadpool_parallelize(
+ threadpool, parallelize_3d_tile_1d_with_uarch, ¶ms, sizeof(params),
+ task, argument, tile_range, flags);
+ }
+}
+
+void pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
+ pthreadpool_t threadpool,
+ pthreadpool_task_3d_tile_1d_with_id_with_thread_t task,
+ void* argument,
+ uint32_t default_uarch_index,
+ uint32_t max_uarch_index,
+ size_t range_i,
+ size_t range_j,
+ size_t range_k,
+ size_t tile_k,
+ uint32_t flags)
+{
+ size_t threads_count;
+ if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k)) {
+ /* No thread pool used: execute task sequentially on the calling thread */
+
+ uint32_t uarch_index = default_uarch_index;
+ #if PTHREADPOOL_USE_CPUINFO
+ uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index);
+ if (uarch_index > max_uarch_index) {
+ uarch_index = default_uarch_index;
+ }
+ #endif
+
+ struct fpu_state saved_fpu_state = { 0 };
+ if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
+ saved_fpu_state = get_fpu_state();
+ disable_fpu_denormals();
+ }
+ for (size_t i = 0; i < range_i; i++) {
+ for (size_t j = 0; j < range_j; j++) {
+ for (size_t k = 0; k < range_k; k += tile_k) {
+ task(argument, uarch_index, 0, i, j, k, min(range_k - k, tile_k));
+ }
+ }
+ }
+ if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
+ set_fpu_state(saved_fpu_state);
+ }
+ } else {
+ const size_t tile_range_k = divide_round_up(range_k, tile_k);
+ const size_t tile_range = range_i * range_j * tile_range_k;
+ const struct pthreadpool_3d_tile_1d_with_uarch_params params = {
+ .default_uarch_index = default_uarch_index,
+ .max_uarch_index = max_uarch_index,
+ .range_k = range_k,
+ .tile_k = tile_k,
+ .range_j = fxdiv_init_size_t(range_j),
+ .tile_range_k = fxdiv_init_size_t(tile_range_k),
+ };
+ thread_function_t parallelize_3d_tile_1d_with_uarch_with_thread = &thread_parallelize_3d_tile_1d_with_uarch_with_thread;
+ #if PTHREADPOOL_USE_FASTPATH
+ const size_t range_threshold = -threads_count;
+ if (tile_range < range_threshold) {
+ parallelize_3d_tile_1d_with_uarch_with_thread = &pthreadpool_thread_parallelize_3d_tile_1d_with_uarch_with_thread_fastpath;
+ }
+ #endif
+ pthreadpool_parallelize(
+ threadpool, parallelize_3d_tile_1d_with_uarch_with_thread, ¶ms, sizeof(params),
+ task, argument, tile_range, flags);
+ }
+}
+
void pthreadpool_parallelize_3d_tile_2d(
pthreadpool_t threadpool,
pthreadpool_task_3d_tile_2d_t task,
diff --git a/src/threadpool-atomics.h b/src/threadpool-atomics.h
index 44772e2..eaa0707 100644
--- a/src/threadpool-atomics.h
+++ b/src/threadpool-atomics.h
@@ -5,7 +5,7 @@
#include <stdint.h>
/* SSE-specific headers */
-#if defined(__i386__) || defined(__i686__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)
+#if defined(__i386__) || defined(__i686__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) && !defined(_M_ARM64EC)
#include <xmmintrin.h>
#endif
@@ -369,7 +369,7 @@
static inline void pthreadpool_fence_release() {
__sync_synchronize();
}
-#elif defined(_MSC_VER) && defined(_M_X64)
+#elif defined(_MSC_VER) && defined(_M_ARM)
typedef volatile uint32_t pthreadpool_atomic_uint32_t;
typedef volatile size_t pthreadpool_atomic_size_t;
typedef void *volatile pthreadpool_atomic_void_p;
@@ -377,26 +377,26 @@
static inline uint32_t pthreadpool_load_relaxed_uint32_t(
pthreadpool_atomic_uint32_t* address)
{
- return *address;
+ return (uint32_t) __iso_volatile_load32((const volatile __int32*) address);
}
static inline size_t pthreadpool_load_relaxed_size_t(
pthreadpool_atomic_size_t* address)
{
- return *address;
+ return (size_t) __iso_volatile_load32((const volatile __int32*) address);
}
static inline void* pthreadpool_load_relaxed_void_p(
pthreadpool_atomic_void_p* address)
{
- return *address;
+ return (void*) __iso_volatile_load32((const volatile __int32*) address);
}
static inline uint32_t pthreadpool_load_acquire_uint32_t(
pthreadpool_atomic_uint32_t* address)
{
- /* x86-64 loads always have acquire semantics; use only a compiler barrier */
- const uint32_t value = *address;
+ const uint32_t value = (uint32_t) __iso_volatile_load32((const volatile __int32*) address);
+ __dmb(_ARM_BARRIER_ISH);
_ReadBarrier();
return value;
}
@@ -404,8 +404,8 @@
static inline size_t pthreadpool_load_acquire_size_t(
pthreadpool_atomic_size_t* address)
{
- /* x86-64 loads always have acquire semantics; use only a compiler barrier */
- const size_t value = *address;
+ const size_t value = (size_t) __iso_volatile_load32((const volatile __int32*) address);
+ __dmb(_ARM_BARRIER_ISH);
_ReadBarrier();
return value;
}
@@ -414,68 +414,68 @@
pthreadpool_atomic_uint32_t* address,
uint32_t value)
{
- *address = value;
+ __iso_volatile_store32((volatile __int32*) address, (__int32) value);
}
static inline void pthreadpool_store_relaxed_size_t(
pthreadpool_atomic_size_t* address,
size_t value)
{
- *address = value;
+ __iso_volatile_store32((volatile __int32*) address, (__int32) value);
}
static inline void pthreadpool_store_relaxed_void_p(
pthreadpool_atomic_void_p* address,
void* value)
{
- *address = value;
+ __iso_volatile_store32((volatile __int32*) address, (__int32) value);
}
static inline void pthreadpool_store_release_uint32_t(
pthreadpool_atomic_uint32_t* address,
uint32_t value)
{
- /* x86-64 stores always have release semantics; use only a compiler barrier */
_WriteBarrier();
- *address = value;
+ __dmb(_ARM_BARRIER_ISH);
+ __iso_volatile_store32((volatile __int32*) address, (__int32) value);
}
static inline void pthreadpool_store_release_size_t(
pthreadpool_atomic_size_t* address,
size_t value)
{
- /* x86-64 stores always have release semantics; use only a compiler barrier */
_WriteBarrier();
- *address = value;
+ __dmb(_ARM_BARRIER_ISH);
+ __iso_volatile_store32((volatile __int32*) address, (__int32) value);
}
static inline size_t pthreadpool_decrement_fetch_relaxed_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement64((volatile __int64*) address);
+ return (size_t) _InterlockedDecrement_nf((volatile long*) address);
}
static inline size_t pthreadpool_decrement_fetch_release_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement64((volatile __int64*) address);
+ return (size_t) _InterlockedDecrement_rel((volatile long*) address);
}
static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement64((volatile __int64*) address);
+ return (size_t) _InterlockedDecrement((volatile long*) address);
}
static inline bool pthreadpool_try_decrement_relaxed_size_t(
pthreadpool_atomic_size_t* value)
{
- size_t actual_value = *value;
+ size_t actual_value = (size_t) __iso_volatile_load32((const volatile __int32*) value);
while (actual_value != 0) {
const size_t new_value = actual_value - 1;
const size_t expected_value = actual_value;
- actual_value = _InterlockedCompareExchange64(
- (volatile __int64*) value, (__int64) new_value, (__int64) expected_value);
+ actual_value = _InterlockedCompareExchange_nf(
+ (volatile long*) value, (long) new_value, (long) expected_value);
if (actual_value == expected_value) {
return true;
}
@@ -484,15 +484,15 @@
}
static inline void pthreadpool_fence_acquire() {
- _mm_lfence();
+ __dmb(_ARM_BARRIER_ISH);
_ReadBarrier();
}
static inline void pthreadpool_fence_release() {
_WriteBarrier();
- _mm_sfence();
+ __dmb(_ARM_BARRIER_ISH);
}
-#elif defined(_MSC_VER) && defined(_M_IX86)
+#elif defined(_MSC_VER) && defined(_M_ARM64)
typedef volatile uint32_t pthreadpool_atomic_uint32_t;
typedef volatile size_t pthreadpool_atomic_size_t;
typedef void *volatile pthreadpool_atomic_void_p;
@@ -500,105 +500,97 @@
static inline uint32_t pthreadpool_load_relaxed_uint32_t(
pthreadpool_atomic_uint32_t* address)
{
- return *address;
+ return (uint32_t) __iso_volatile_load32((const volatile __int32*) address);
}
static inline size_t pthreadpool_load_relaxed_size_t(
pthreadpool_atomic_size_t* address)
{
- return *address;
+ return (size_t) __iso_volatile_load64((const volatile __int64*) address);
}
static inline void* pthreadpool_load_relaxed_void_p(
pthreadpool_atomic_void_p* address)
{
- return *address;
+ return (void*) __iso_volatile_load64((const volatile __int64*) address);
}
static inline uint32_t pthreadpool_load_acquire_uint32_t(
pthreadpool_atomic_uint32_t* address)
{
- /* x86 loads always have acquire semantics; use only a compiler barrier */
- const uint32_t value = *address;
- _ReadBarrier();
- return value;
+ return (uint32_t) __ldar32((volatile unsigned __int32*) address);
}
static inline size_t pthreadpool_load_acquire_size_t(
pthreadpool_atomic_size_t* address)
{
- /* x86 loads always have acquire semantics; use only a compiler barrier */
- const size_t value = *address;
- _ReadBarrier();
- return value;
+ return (size_t) __ldar64((volatile unsigned __int64*) address);
}
static inline void pthreadpool_store_relaxed_uint32_t(
pthreadpool_atomic_uint32_t* address,
uint32_t value)
{
- *address = value;
+ __iso_volatile_store32((volatile __int32*) address, (__int32) value);
}
static inline void pthreadpool_store_relaxed_size_t(
pthreadpool_atomic_size_t* address,
size_t value)
{
- *address = value;
+ __iso_volatile_store64((volatile __int64*) address, (__int64) value);
}
static inline void pthreadpool_store_relaxed_void_p(
pthreadpool_atomic_void_p* address,
void* value)
{
- *address = value;
+ __iso_volatile_store64((volatile __int64*) address, (__int64) value);
}
static inline void pthreadpool_store_release_uint32_t(
pthreadpool_atomic_uint32_t* address,
uint32_t value)
{
- /* x86 stores always have release semantics; use only a compiler barrier */
_WriteBarrier();
- *address = value;
+ __stlr32((unsigned __int32 volatile*) address, (unsigned __int32) value);
}
static inline void pthreadpool_store_release_size_t(
pthreadpool_atomic_size_t* address,
size_t value)
{
- /* x86 stores always have release semantics; use only a compiler barrier */
_WriteBarrier();
- *address = value;
+ __stlr64((unsigned __int64 volatile*) address, (unsigned __int64) value);
}
static inline size_t pthreadpool_decrement_fetch_relaxed_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement((volatile long*) address);
+ return (size_t) _InterlockedDecrement64_nf((volatile __int64*) address);
}
static inline size_t pthreadpool_decrement_fetch_release_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement((volatile long*) address);
+ return (size_t) _InterlockedDecrement64_rel((volatile __int64*) address);
}
static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement((volatile long*) address);
+ return (size_t) _InterlockedDecrement64((volatile __int64*) address);
}
static inline bool pthreadpool_try_decrement_relaxed_size_t(
pthreadpool_atomic_size_t* value)
{
- size_t actual_value = *value;
+ size_t actual_value = (size_t) __iso_volatile_load64((const volatile __int64*) value);
while (actual_value != 0) {
const size_t new_value = actual_value - 1;
const size_t expected_value = actual_value;
- actual_value = _InterlockedCompareExchange(
- (volatile long*) value, (long) new_value, (long) expected_value);
+ actual_value = _InterlockedCompareExchange64_nf(
+ (volatile __int64*) value, (__int64) new_value, (__int64) expected_value);
if (actual_value == expected_value) {
return true;
}
@@ -607,13 +599,15 @@
}
static inline void pthreadpool_fence_acquire() {
- _mm_lfence();
+ __dmb(_ARM64_BARRIER_ISHLD);
+ _ReadBarrier();
}
static inline void pthreadpool_fence_release() {
- _mm_sfence();
+ _WriteBarrier();
+ __dmb(_ARM64_BARRIER_ISH);
}
-#elif defined(_MSC_VER) && defined(_M_ARM64)
+#elif defined(_MSC_VER) && defined(_M_IX86)
typedef volatile uint32_t pthreadpool_atomic_uint32_t;
typedef volatile size_t pthreadpool_atomic_size_t;
typedef void *volatile pthreadpool_atomic_void_p;
@@ -621,97 +615,105 @@
static inline uint32_t pthreadpool_load_relaxed_uint32_t(
pthreadpool_atomic_uint32_t* address)
{
- return (uint32_t) __iso_volatile_load32((const volatile __int32*) address);
+ return *address;
}
static inline size_t pthreadpool_load_relaxed_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) __iso_volatile_load64((const volatile __int64*) address);
+ return *address;
}
static inline void* pthreadpool_load_relaxed_void_p(
pthreadpool_atomic_void_p* address)
{
- return (void*) __iso_volatile_load64((const volatile __int64*) address);
+ return *address;
}
static inline uint32_t pthreadpool_load_acquire_uint32_t(
pthreadpool_atomic_uint32_t* address)
{
- return (uint32_t) __ldar32((volatile unsigned __int32*) address);
+ /* x86 loads always have acquire semantics; use only a compiler barrier */
+ const uint32_t value = *address;
+ _ReadBarrier();
+ return value;
}
static inline size_t pthreadpool_load_acquire_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) __ldar64((volatile unsigned __int64*) address);
+ /* x86 loads always have acquire semantics; use only a compiler barrier */
+ const size_t value = *address;
+ _ReadBarrier();
+ return value;
}
static inline void pthreadpool_store_relaxed_uint32_t(
pthreadpool_atomic_uint32_t* address,
uint32_t value)
{
- __iso_volatile_store32((volatile __int32*) address, (__int32) value);
+ *address = value;
}
static inline void pthreadpool_store_relaxed_size_t(
pthreadpool_atomic_size_t* address,
size_t value)
{
- __iso_volatile_store64((volatile __int64*) address, (__int64) value);
+ *address = value;
}
static inline void pthreadpool_store_relaxed_void_p(
pthreadpool_atomic_void_p* address,
void* value)
{
- __iso_volatile_store64((volatile __int64*) address, (__int64) value);
+ *address = value;
}
static inline void pthreadpool_store_release_uint32_t(
pthreadpool_atomic_uint32_t* address,
uint32_t value)
{
+ /* x86 stores always have release semantics; use only a compiler barrier */
_WriteBarrier();
- __stlr32((unsigned __int32 volatile*) address, (unsigned __int32) value);
+ *address = value;
}
static inline void pthreadpool_store_release_size_t(
pthreadpool_atomic_size_t* address,
size_t value)
{
+ /* x86 stores always have release semantics; use only a compiler barrier */
_WriteBarrier();
- __stlr64((unsigned __int64 volatile*) address, (unsigned __int64) value);
+ *address = value;
}
static inline size_t pthreadpool_decrement_fetch_relaxed_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement64_nf((volatile __int64*) address);
+ return (size_t) _InterlockedDecrement((volatile long*) address);
}
static inline size_t pthreadpool_decrement_fetch_release_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement64_rel((volatile __int64*) address);
+ return (size_t) _InterlockedDecrement((volatile long*) address);
}
static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement64((volatile __int64*) address);
+ return (size_t) _InterlockedDecrement((volatile long*) address);
}
static inline bool pthreadpool_try_decrement_relaxed_size_t(
pthreadpool_atomic_size_t* value)
{
- size_t actual_value = (size_t) __iso_volatile_load64((const volatile __int64*) value);
+ size_t actual_value = *value;
while (actual_value != 0) {
const size_t new_value = actual_value - 1;
const size_t expected_value = actual_value;
- actual_value = _InterlockedCompareExchange64_nf(
- (volatile __int64*) value, (__int64) new_value, (__int64) expected_value);
+ actual_value = _InterlockedCompareExchange(
+ (volatile long*) value, (long) new_value, (long) expected_value);
if (actual_value == expected_value) {
return true;
}
@@ -720,15 +722,13 @@
}
static inline void pthreadpool_fence_acquire() {
- __dmb(_ARM64_BARRIER_ISHLD);
- _ReadBarrier();
+ _mm_lfence();
}
static inline void pthreadpool_fence_release() {
- _WriteBarrier();
- __dmb(_ARM64_BARRIER_ISH);
+ _mm_sfence();
}
-#elif defined(_MSC_VER) && defined(_M_ARM)
+#elif defined(_MSC_VER) && defined(_M_X64)
typedef volatile uint32_t pthreadpool_atomic_uint32_t;
typedef volatile size_t pthreadpool_atomic_size_t;
typedef void *volatile pthreadpool_atomic_void_p;
@@ -736,26 +736,26 @@
static inline uint32_t pthreadpool_load_relaxed_uint32_t(
pthreadpool_atomic_uint32_t* address)
{
- return (uint32_t) __iso_volatile_load32((const volatile __int32*) address);
+ return *address;
}
static inline size_t pthreadpool_load_relaxed_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) __iso_volatile_load32((const volatile __int32*) address);
+ return *address;
}
static inline void* pthreadpool_load_relaxed_void_p(
pthreadpool_atomic_void_p* address)
{
- return (void*) __iso_volatile_load32((const volatile __int32*) address);
+ return *address;
}
static inline uint32_t pthreadpool_load_acquire_uint32_t(
pthreadpool_atomic_uint32_t* address)
{
- const uint32_t value = (uint32_t) __iso_volatile_load32((const volatile __int32*) address);
- __dmb(_ARM_BARRIER_ISH);
+ /* x86-64 loads always have acquire semantics; use only a compiler barrier */
+ const uint32_t value = *address;
_ReadBarrier();
return value;
}
@@ -763,8 +763,8 @@
static inline size_t pthreadpool_load_acquire_size_t(
pthreadpool_atomic_size_t* address)
{
- const size_t value = (size_t) __iso_volatile_load32((const volatile __int32*) address);
- __dmb(_ARM_BARRIER_ISH);
+ /* x86-64 loads always have acquire semantics; use only a compiler barrier */
+ const size_t value = *address;
_ReadBarrier();
return value;
}
@@ -773,68 +773,68 @@
pthreadpool_atomic_uint32_t* address,
uint32_t value)
{
- __iso_volatile_store32((volatile __int32*) address, (__int32) value);
+ *address = value;
}
static inline void pthreadpool_store_relaxed_size_t(
pthreadpool_atomic_size_t* address,
size_t value)
{
- __iso_volatile_store32((volatile __int32*) address, (__int32) value);
+ *address = value;
}
static inline void pthreadpool_store_relaxed_void_p(
pthreadpool_atomic_void_p* address,
void* value)
{
- __iso_volatile_store32((volatile __int32*) address, (__int32) value);
+ *address = value;
}
static inline void pthreadpool_store_release_uint32_t(
pthreadpool_atomic_uint32_t* address,
uint32_t value)
{
+ /* x86-64 stores always have release semantics; use only a compiler barrier */
_WriteBarrier();
- __dmb(_ARM_BARRIER_ISH);
- __iso_volatile_store32((volatile __int32*) address, (__int32) value);
+ *address = value;
}
static inline void pthreadpool_store_release_size_t(
pthreadpool_atomic_size_t* address,
size_t value)
{
+ /* x86-64 stores always have release semantics; use only a compiler barrier */
_WriteBarrier();
- __dmb(_ARM_BARRIER_ISH);
- __iso_volatile_store32((volatile __int32*) address, (__int32) value);
+ *address = value;
}
static inline size_t pthreadpool_decrement_fetch_relaxed_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement_nf((volatile long*) address);
+ return (size_t) _InterlockedDecrement64((volatile __int64*) address);
}
static inline size_t pthreadpool_decrement_fetch_release_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement_rel((volatile long*) address);
+ return (size_t) _InterlockedDecrement64((volatile __int64*) address);
}
static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t(
pthreadpool_atomic_size_t* address)
{
- return (size_t) _InterlockedDecrement((volatile long*) address);
+ return (size_t) _InterlockedDecrement64((volatile __int64*) address);
}
static inline bool pthreadpool_try_decrement_relaxed_size_t(
pthreadpool_atomic_size_t* value)
{
- size_t actual_value = (size_t) __iso_volatile_load32((const volatile __int32*) value);
+ size_t actual_value = *value;
while (actual_value != 0) {
const size_t new_value = actual_value - 1;
const size_t expected_value = actual_value;
- actual_value = _InterlockedCompareExchange_nf(
- (volatile long*) value, (long) new_value, (long) expected_value);
+ actual_value = _InterlockedCompareExchange64(
+ (volatile __int64*) value, (__int64) new_value, (__int64) expected_value);
if (actual_value == expected_value) {
return true;
}
@@ -843,23 +843,19 @@
}
static inline void pthreadpool_fence_acquire() {
- __dmb(_ARM_BARRIER_ISH);
+ _mm_lfence();
_ReadBarrier();
}
static inline void pthreadpool_fence_release() {
_WriteBarrier();
- __dmb(_ARM_BARRIER_ISH);
+ _mm_sfence();
}
#else
#error "Platform-specific implementation of threadpool-atomics.h required"
#endif
-#if defined(__i386__) || defined(__i686__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)
- static inline void pthreadpool_yield() {
- _mm_pause();
- }
-#elif defined(__ARM_ACLE) || defined(_MSC_VER) && (defined(_M_ARM) || defined(_M_ARM64))
+#if defined(__ARM_ACLE) || defined(_MSC_VER) && (defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC))
static inline void pthreadpool_yield() {
__yield();
}
@@ -867,6 +863,10 @@
static inline void pthreadpool_yield() {
__asm__ __volatile__("yield");
}
+#elif defined(__i386__) || defined(__i686__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)
+ static inline void pthreadpool_yield() {
+ _mm_pause();
+ }
#else
static inline void pthreadpool_yield() {
pthreadpool_fence_acquire();
diff --git a/src/threadpool-object.h b/src/threadpool-object.h
index 4e4c3d9..93e5619 100644
--- a/src/threadpool-object.h
+++ b/src/threadpool-object.h
@@ -128,6 +128,29 @@ struct pthreadpool_2d_tile_1d_params {
struct fxdiv_divisor_size_t tile_range_j;
};
+struct pthreadpool_2d_tile_1d_with_uarch_params {
+ /**
+ * Copy of the default_uarch_index argument passed to the pthreadpool_parallelize_2d_tile_1d_with_uarch function.
+ */
+ uint32_t default_uarch_index;
+ /**
+ * Copy of the max_uarch_index argument passed to the pthreadpool_parallelize_2d_tile_1d_with_uarch function.
+ */
+ uint32_t max_uarch_index;
+ /**
+ * Copy of the range_j argument passed to the pthreadpool_parallelize_2d_tile_1d function.
+ */
+ size_t range_j;
+ /**
+ * Copy of the tile_j argument passed to the pthreadpool_parallelize_2d_tile_1d function.
+ */
+ size_t tile_j;
+ /**
+ * FXdiv divisor for the divide_round_up(range_j, tile_j) value.
+ */
+ struct fxdiv_divisor_size_t tile_range_j;
+};
+
struct pthreadpool_2d_tile_2d_params {
/**
* Copy of the range_i argument passed to the pthreadpool_parallelize_2d_tile_2d function.
@@ -212,6 +235,33 @@ struct pthreadpool_3d_tile_1d_params {
struct fxdiv_divisor_size_t tile_range_k;
};
+struct pthreadpool_3d_tile_1d_with_uarch_params {
+ /**
+ * Copy of the default_uarch_index argument passed to the pthreadpool_parallelize_3d_tile_1d_with_uarch function.
+ */
+ uint32_t default_uarch_index;
+ /**
+ * Copy of the max_uarch_index argument passed to the pthreadpool_parallelize_3d_tile_1d_with_uarch function.
+ */
+ uint32_t max_uarch_index;
+ /**
+ * Copy of the range_k argument passed to the pthreadpool_parallelize_3d_tile_1d_with_uarch function.
+ */
+ size_t range_k;
+ /**
+ * Copy of the tile_k argument passed to the pthreadpool_parallelize_3d_tile_1d_with_uarch function.
+ */
+ size_t tile_k;
+ /**
+ * FXdiv divisor for the range_j argument passed to the pthreadpool_parallelize_3d_tile_1d_with_uarch function.
+ */
+ struct fxdiv_divisor_size_t range_j;
+ /**
+ * FXdiv divisor for the divide_round_up(range_k, tile_k) value.
+ */
+ struct fxdiv_divisor_size_t tile_range_k;
+};
+
struct pthreadpool_3d_tile_2d_params {
/**
* Copy of the range_j argument passed to the pthreadpool_parallelize_3d_tile_2d function.
@@ -627,10 +677,12 @@ struct PTHREADPOOL_CACHELINE_ALIGNED pthreadpool {
struct pthreadpool_1d_tile_1d_params parallelize_1d_tile_1d;
struct pthreadpool_2d_params parallelize_2d;
struct pthreadpool_2d_tile_1d_params parallelize_2d_tile_1d;
+ struct pthreadpool_2d_tile_1d_with_uarch_params parallelize_2d_tile_1d_with_uarch;
struct pthreadpool_2d_tile_2d_params parallelize_2d_tile_2d;
struct pthreadpool_2d_tile_2d_with_uarch_params parallelize_2d_tile_2d_with_uarch;
struct pthreadpool_3d_params parallelize_3d;
struct pthreadpool_3d_tile_1d_params parallelize_3d_tile_1d;
+ struct pthreadpool_3d_tile_1d_with_uarch_params parallelize_3d_tile_1d_with_uarch;
struct pthreadpool_3d_tile_2d_params parallelize_3d_tile_2d;
struct pthreadpool_3d_tile_2d_with_uarch_params parallelize_3d_tile_2d_with_uarch;
struct pthreadpool_4d_params parallelize_4d;
@@ -750,6 +802,14 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_1d_fastpath(
struct pthreadpool* threadpool,
struct thread_info* thread);
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_1d_with_uarch_fastpath(
+ struct pthreadpool* threadpool,
+ struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_1d_with_uarch_with_thread_fastpath(
+ struct pthreadpool* threadpool,
+ struct thread_info* thread);
+
PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_fastpath(
struct pthreadpool* threadpool,
struct thread_info* thread);
@@ -766,6 +826,18 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_1d_fastpath(
struct pthreadpool* threadpool,
struct thread_info* thread);
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_1d_with_thread_fastpath(
+ struct pthreadpool* threadpool,
+ struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_1d_with_uarch_fastpath(
+ struct pthreadpool* threadpool,
+ struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_1d_with_uarch_with_thread_fastpath(
+ struct pthreadpool* threadpool,
+ struct thread_info* thread);
+
PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_fastpath(
struct pthreadpool* threadpool,
struct thread_info* thread);
diff --git a/src/threadpool-utils.h b/src/threadpool-utils.h
index 91e2445..0b81360 100644
--- a/src/threadpool-utils.h
+++ b/src/threadpool-utils.h
@@ -4,7 +4,7 @@
#include <stddef.h>
/* SSE-specific headers */
-#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)
+#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) && !defined(_M_ARM64EC) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)
#include <xmmintrin.h>
#endif
@@ -15,12 +15,12 @@
struct fpu_state {
-#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)
- uint32_t mxcsr;
-#elif defined(__GNUC__) && defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) || defined(_MSC_VER) && defined(_M_ARM)
+#if defined(__GNUC__) && defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) || defined(_MSC_VER) && defined(_M_ARM)
uint32_t fpscr;
-#elif defined(__GNUC__) && defined(__aarch64__) || defined(_MSC_VER) && defined(_M_ARM64)
+#elif defined(__GNUC__) && defined(__aarch64__) || defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC))
uint64_t fpcr;
+#elif defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)
+ uint32_t mxcsr;
#else
char unused;
#endif
@@ -28,12 +28,12 @@ struct fpu_state {
static inline struct fpu_state get_fpu_state() {
struct fpu_state state = { 0 };
-#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)
- state.mxcsr = (uint32_t) _mm_getcsr();
-#elif defined(_MSC_VER) && defined(_M_ARM)
+#if defined(_MSC_VER) && defined(_M_ARM)
state.fpscr = (uint32_t) _MoveFromCoprocessor(10, 7, 1, 0, 0);
-#elif defined(_MSC_VER) && defined(_M_ARM64)
+#elif defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC))
state.fpcr = (uint64_t) _ReadStatusReg(0x5A20);
+#elif defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)
+ state.mxcsr = (uint32_t) _mm_getcsr();
#elif defined(__GNUC__) && defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0)
__asm__ __volatile__("VMRS %[fpscr], fpscr" : [fpscr] "=r" (state.fpscr));
#elif defined(__GNUC__) && defined(__aarch64__)
@@ -43,27 +43,25 @@ static inline struct fpu_state get_fpu_state() {
}
static inline void set_fpu_state(const struct fpu_state state) {
-#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)
- _mm_setcsr((unsigned int) state.mxcsr);
-#elif defined(_MSC_VER) && defined(_M_ARM)
+#if defined(_MSC_VER) && defined(_M_ARM)
_MoveToCoprocessor((int) state.fpscr, 10, 7, 1, 0, 0);
-#elif defined(_MSC_VER) && defined(_M_ARM64)
+#elif defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC))
_WriteStatusReg(0x5A20, (__int64) state.fpcr);
#elif defined(__GNUC__) && defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0)
__asm__ __volatile__("VMSR fpscr, %[fpscr]" : : [fpscr] "r" (state.fpscr));
#elif defined(__GNUC__) && defined(__aarch64__)
__asm__ __volatile__("MSR fpcr, %[fpcr]" : : [fpcr] "r" (state.fpcr));
+#elif defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)
+ _mm_setcsr((unsigned int) state.mxcsr);
#endif
}
static inline void disable_fpu_denormals() {
-#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)
- _mm_setcsr(_mm_getcsr() | 0x8040);
-#elif defined(_MSC_VER) && defined(_M_ARM)
+#if defined(_MSC_VER) && defined(_M_ARM)
int fpscr = _MoveFromCoprocessor(10, 7, 1, 0, 0);
fpscr |= 0x1000000;
_MoveToCoprocessor(fpscr, 10, 7, 1, 0, 0);
-#elif defined(_MSC_VER) && defined(_M_ARM64)
+#elif defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC))
__int64 fpcr = _ReadStatusReg(0x5A20);
fpcr |= 0x1080000;
_WriteStatusReg(0x5A20, fpcr);
@@ -92,6 +90,8 @@ static inline void disable_fpu_denormals() {
"ORR %w[fpcr], %w[fpcr], 0x80000\n"
"MSR fpcr, %[fpcr]\n"
: [fpcr] "=r" (fpcr));
+#elif defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1)
+ _mm_setcsr(_mm_getcsr() | 0x8040);
#endif
}
diff --git a/test/pthreadpool.cc b/test/pthreadpool.cc
index c9592ec..5cd31a6 100644
--- a/test/pthreadpool.cc
+++ b/test/pthreadpool.cc
@@ -1580,22 +1580,22 @@ TEST(Parallelize2DTile1D, MultiThreadPoolWorkStealing) {
EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
}
-static void ComputeNothing2DTile2D(void*, size_t, size_t, size_t, size_t) {
+static void ComputeNothing2DTile1DWithUArch(void*, uint32_t, size_t, size_t, size_t) {
}
-TEST(Parallelize2DTile2D, SingleThreadPoolCompletes) {
+TEST(Parallelize2DTile1DWithUArch, SingleThreadPoolCompletes) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d(threadpool.get(),
- ComputeNothing2DTile2D,
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(threadpool.get(),
+ ComputeNothing2DTile1DWithUArch,
nullptr,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-TEST(Parallelize2DTile2D, MultiThreadPoolCompletes) {
+TEST(Parallelize2DTile1DWithUArch, MultiThreadPoolCompletes) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -1603,36 +1603,35 @@ TEST(Parallelize2DTile2D, MultiThreadPoolCompletes) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- ComputeNothing2DTile2D,
+ ComputeNothing2DTile1DWithUArch,
nullptr,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-static void CheckBounds2DTile2D(void*, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- EXPECT_LT(start_i, kParallelize2DTile2DRangeI);
- EXPECT_LT(start_j, kParallelize2DTile2DRangeJ);
- EXPECT_LE(start_i + tile_i, kParallelize2DTile2DRangeI);
- EXPECT_LE(start_j + tile_j, kParallelize2DTile2DRangeJ);
+static void CheckUArch2DTile1DWithUArch(void*, uint32_t uarch_index, size_t, size_t, size_t) {
+ if (uarch_index != kDefaultUArchIndex) {
+ EXPECT_LE(uarch_index, kMaxUArchIndex);
+ }
}
-TEST(Parallelize2DTile2D, SingleThreadPoolAllItemsInBounds) {
+TEST(Parallelize2DTile1DWithUArch, SingleThreadPoolUArchInBounds) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- CheckBounds2DTile2D,
+ CheckUArch2DTile1DWithUArch,
nullptr,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-TEST(Parallelize2DTile2D, MultiThreadPoolAllItemsInBounds) {
+TEST(Parallelize2DTile1DWithUArch, MultiThreadPoolUArchInBounds) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -1640,41 +1639,72 @@ TEST(Parallelize2DTile2D, MultiThreadPoolAllItemsInBounds) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- CheckBounds2DTile2D,
+ CheckUArch2DTile1DWithUArch,
nullptr,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-static void CheckTiling2DTile2D(void*, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- EXPECT_GT(tile_i, 0);
- EXPECT_LE(tile_i, kParallelize2DTile2DTileI);
- EXPECT_EQ(start_i % kParallelize2DTile2DTileI, 0);
- EXPECT_EQ(tile_i, std::min<size_t>(kParallelize2DTile2DTileI, kParallelize2DTile2DRangeI - start_i));
+static void CheckBounds2DTile1DWithUArch(void*, uint32_t, size_t i, size_t start_j, size_t tile_j) {
+ EXPECT_LT(i, kParallelize2DTile1DRangeI);
+ EXPECT_LT(start_j, kParallelize2DTile1DRangeJ);
+ EXPECT_LE(start_j + tile_j, kParallelize2DTile1DRangeJ);
+}
+
+TEST(Parallelize2DTile1DWithUArch, SingleThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
+ threadpool.get(),
+ CheckBounds2DTile1DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
+ 0 /* flags */);
+}
+
+TEST(Parallelize2DTile1DWithUArch, MultiThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
+ threadpool.get(),
+ CheckBounds2DTile1DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
+ 0 /* flags */);
+}
+static void CheckTiling2DTile1DWithUArch(void*, uint32_t, size_t i, size_t start_j, size_t tile_j) {
EXPECT_GT(tile_j, 0);
- EXPECT_LE(tile_j, kParallelize2DTile2DTileJ);
- EXPECT_EQ(start_j % kParallelize2DTile2DTileJ, 0);
- EXPECT_EQ(tile_j, std::min<size_t>(kParallelize2DTile2DTileJ, kParallelize2DTile2DRangeJ - start_j));
+ EXPECT_LE(tile_j, kParallelize2DTile1DTileJ);
+ EXPECT_EQ(start_j % kParallelize2DTile1DTileJ, 0);
+ EXPECT_EQ(tile_j, std::min<size_t>(kParallelize2DTile1DTileJ, kParallelize2DTile1DRangeJ - start_j));
}
-TEST(Parallelize2DTile2D, SingleThreadPoolUniformTiling) {
+TEST(Parallelize2DTile1DWithUArch, SingleThreadPoolUniformTiling) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- CheckTiling2DTile2D,
+ CheckTiling2DTile1DWithUArch,
nullptr,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-TEST(Parallelize2DTile2D, MultiThreadPoolUniformTiling) {
+TEST(Parallelize2DTile1DWithUArch, MultiThreadPoolUniformTiling) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -1682,49 +1712,47 @@ TEST(Parallelize2DTile2D, MultiThreadPoolUniformTiling) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- CheckTiling2DTile2D,
+ CheckTiling2DTile1DWithUArch,
nullptr,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-static void SetTrue2DTile2D(std::atomic_bool* processed_indicators, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- for (size_t i = start_i; i < start_i + tile_i; i++) {
- for (size_t j = start_j; j < start_j + tile_j; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
- processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
- }
+static void SetTrue2DTile1DWithUArch(std::atomic_bool* processed_indicators, uint32_t, size_t i, size_t start_j, size_t tile_j) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
+ processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
}
}
-TEST(Parallelize2DTile2D, SingleThreadPoolAllItemsProcessed) {
- std::vector<std::atomic_bool> indicators(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArch, SingleThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(SetTrue2DTile2D),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_t>(SetTrue2DTile1DWithUArch),
static_cast<void*>(indicators.data()),
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
<< "Element (" << i << ", " << j << ") not processed";
}
}
}
-TEST(Parallelize2DTile2D, MultiThreadPoolAllItemsProcessed) {
- std::vector<std::atomic_bool> indicators(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArch, MultiThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -1733,49 +1761,47 @@ TEST(Parallelize2DTile2D, MultiThreadPoolAllItemsProcessed) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(SetTrue2DTile2D),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_t>(SetTrue2DTile1DWithUArch),
static_cast<void*>(indicators.data()),
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
<< "Element (" << i << ", " << j << ") not processed";
}
}
}
-static void Increment2DTile2D(std::atomic_int* processed_counters, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- for (size_t i = start_i; i < start_i + tile_i; i++) {
- for (size_t j = start_j; j < start_j + tile_j; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
- processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
- }
+static void Increment2DTile1DWithUArch(std::atomic_int* processed_counters, uint32_t, size_t i, size_t start_j, size_t tile_j) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
+ processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
}
}
-TEST(Parallelize2DTile2D, SingleThreadPoolEachItemProcessedOnce) {
- std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArch, SingleThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(Increment2DTile2D),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_t>(Increment2DTile1DWithUArch),
static_cast<void*>(counters.data()),
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
<< "Element (" << i << ", " << j << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
@@ -1783,8 +1809,8 @@ TEST(Parallelize2DTile2D, SingleThreadPoolEachItemProcessedOnce) {
}
}
-TEST(Parallelize2DTile2D, MultiThreadPoolEachItemProcessedOnce) {
- std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArch, MultiThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -1793,17 +1819,17 @@ TEST(Parallelize2DTile2D, MultiThreadPoolEachItemProcessedOnce) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(Increment2DTile2D),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_t>(Increment2DTile1DWithUArch),
static_cast<void*>(counters.data()),
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
<< "Element (" << i << ", " << j << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
@@ -1811,25 +1837,25 @@ TEST(Parallelize2DTile2D, MultiThreadPoolEachItemProcessedOnce) {
}
}
-TEST(Parallelize2DTile2D, SingleThreadPoolEachItemProcessedMultipleTimes) {
- std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArch, SingleThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(Increment2DTile2D),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_t>(Increment2DTile1DWithUArch),
static_cast<void*>(counters.data()),
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
<< "Element (" << i << ", " << j << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times "
@@ -1838,8 +1864,8 @@ TEST(Parallelize2DTile2D, SingleThreadPoolEachItemProcessedMultipleTimes) {
}
}
-TEST(Parallelize2DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) {
- std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -1849,18 +1875,18 @@ TEST(Parallelize2DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) {
}
for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(Increment2DTile2D),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_t>(Increment2DTile1DWithUArch),
static_cast<void*>(counters.data()),
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
<< "Element (" << i << ", " << j << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times "
@@ -1869,15 +1895,13 @@ TEST(Parallelize2DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) {
}
}
-static void IncrementSame2DTile2D(std::atomic_int* num_processed_items, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- for (size_t i = start_i; i < start_i + tile_i; i++) {
- for (size_t j = start_j; j < start_j + tile_j; j++) {
- num_processed_items->fetch_add(1, std::memory_order_relaxed);
- }
+static void IncrementSame2DTile1DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t start_j, size_t tile_j) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ num_processed_items->fetch_add(1, std::memory_order_relaxed);
}
}
-TEST(Parallelize2DTile2D, MultiThreadPoolHighContention) {
+TEST(Parallelize2DTile1DWithUArch, MultiThreadPoolHighContention) {
std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -1887,27 +1911,27 @@ TEST(Parallelize2DTile2D, MultiThreadPoolHighContention) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(IncrementSame2DTile2D),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_t>(IncrementSame2DTile1DWithUArch),
static_cast<void*>(&num_processed_items),
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
}
-static void WorkImbalance2DTile2D(std::atomic_int* num_processed_items, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- num_processed_items->fetch_add(tile_i * tile_j, std::memory_order_relaxed);
- if (start_i == 0 && start_j == 0) {
+static void WorkImbalance2DTile1DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t start_j, size_t tile_j) {
+ num_processed_items->fetch_add(tile_j, std::memory_order_relaxed);
+ if (i == 0 && start_j == 0) {
/* Spin-wait until all items are computed */
- while (num_processed_items->load(std::memory_order_relaxed) != kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ) {
+ while (num_processed_items->load(std::memory_order_relaxed) != kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ) {
std::atomic_thread_fence(std::memory_order_acquire);
}
}
}
-TEST(Parallelize2DTile2D, MultiThreadPoolWorkStealing) {
+TEST(Parallelize2DTile1DWithUArch, MultiThreadPoolWorkStealing) {
std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -1917,33 +1941,32 @@ TEST(Parallelize2DTile2D, MultiThreadPoolWorkStealing) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(WorkImbalance2DTile2D),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_t>(WorkImbalance2DTile1DWithUArch),
static_cast<void*>(&num_processed_items),
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
}
-static void ComputeNothing2DTile2DWithUArch(void*, uint32_t, size_t, size_t, size_t, size_t) {
+static void ComputeNothing2DTile1DWithUArchWithThread(void*, uint32_t, size_t, size_t, size_t, size_t) {
}
-TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolCompletes) {
+TEST(Parallelize2DTile1DWithUArchWithThread, SingleThreadPoolCompletes) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d_with_uarch(threadpool.get(),
- ComputeNothing2DTile2DWithUArch,
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(threadpool.get(),
+ ComputeNothing2DTile1DWithUArchWithThread,
nullptr,
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolCompletes) {
+TEST(Parallelize2DTile1DWithUArchWithThread, MultiThreadPoolCompletes) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -1951,37 +1974,35 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolCompletes) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- ComputeNothing2DTile2DWithUArch,
+ ComputeNothing2DTile1DWithUArchWithThread,
nullptr,
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-static void CheckUArch2DTile2DWithUArch(void*, uint32_t uarch_index, size_t, size_t, size_t, size_t) {
+static void CheckUArch2DTile1DWithUArchWithThread(void*, uint32_t uarch_index, size_t, size_t, size_t, size_t) {
if (uarch_index != kDefaultUArchIndex) {
EXPECT_LE(uarch_index, kMaxUArchIndex);
}
}
-TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolUArchInBounds) {
+TEST(Parallelize2DTile1DWithUArchWithThread, SingleThreadPoolUArchInBounds) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- CheckUArch2DTile2DWithUArch,
+ CheckUArch2DTile1DWithUArchWithThread,
nullptr,
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolUArchInBounds) {
+TEST(Parallelize2DTile1DWithUArchWithThread, MultiThreadPoolUArchInBounds) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -1989,38 +2010,35 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolUArchInBounds) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- CheckUArch2DTile2DWithUArch,
+ CheckUArch2DTile1DWithUArchWithThread,
nullptr,
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-static void CheckBounds2DTile2DWithUArch(void*, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- EXPECT_LT(start_i, kParallelize2DTile2DRangeI);
- EXPECT_LT(start_j, kParallelize2DTile2DRangeJ);
- EXPECT_LE(start_i + tile_i, kParallelize2DTile2DRangeI);
- EXPECT_LE(start_j + tile_j, kParallelize2DTile2DRangeJ);
+static void CheckBounds2DTile1DWithUArchWithThread(void*, uint32_t, size_t, size_t i, size_t start_j, size_t tile_j) {
+ EXPECT_LT(i, kParallelize2DTile1DRangeI);
+ EXPECT_LT(start_j, kParallelize2DTile1DRangeJ);
+ EXPECT_LE(start_j + tile_j, kParallelize2DTile1DRangeJ);
}
-TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolAllItemsInBounds) {
+TEST(Parallelize2DTile1DWithUArchWithThread, SingleThreadPoolAllItemsInBounds) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- CheckBounds2DTile2DWithUArch,
+ CheckBounds2DTile1DWithUArchWithThread,
nullptr,
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolAllItemsInBounds) {
+TEST(Parallelize2DTile1DWithUArchWithThread, MultiThreadPoolAllItemsInBounds) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2028,43 +2046,36 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolAllItemsInBounds) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- CheckBounds2DTile2DWithUArch,
+ CheckBounds2DTile1DWithUArchWithThread,
nullptr,
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-static void CheckTiling2DTile2DWithUArch(void*, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- EXPECT_GT(tile_i, 0);
- EXPECT_LE(tile_i, kParallelize2DTile2DTileI);
- EXPECT_EQ(start_i % kParallelize2DTile2DTileI, 0);
- EXPECT_EQ(tile_i, std::min<size_t>(kParallelize2DTile2DTileI, kParallelize2DTile2DRangeI - start_i));
-
+static void CheckTiling2DTile1DWithUArchWithThread(void*, uint32_t, size_t, size_t i, size_t start_j, size_t tile_j) {
EXPECT_GT(tile_j, 0);
- EXPECT_LE(tile_j, kParallelize2DTile2DTileJ);
- EXPECT_EQ(start_j % kParallelize2DTile2DTileJ, 0);
- EXPECT_EQ(tile_j, std::min<size_t>(kParallelize2DTile2DTileJ, kParallelize2DTile2DRangeJ - start_j));
+ EXPECT_LE(tile_j, kParallelize2DTile1DTileJ);
+ EXPECT_EQ(start_j % kParallelize2DTile1DTileJ, 0);
+ EXPECT_EQ(tile_j, std::min<size_t>(kParallelize2DTile1DTileJ, kParallelize2DTile1DRangeJ - start_j));
}
-TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolUniformTiling) {
+TEST(Parallelize2DTile1DWithUArchWithThread, SingleThreadPoolUniformTiling) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- CheckTiling2DTile2DWithUArch,
+ CheckTiling2DTile1DWithUArchWithThread,
nullptr,
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolUniformTiling) {
+TEST(Parallelize2DTile1DWithUArchWithThread, MultiThreadPoolUniformTiling) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2072,51 +2083,47 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolUniformTiling) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- CheckTiling2DTile2DWithUArch,
+ CheckTiling2DTile1DWithUArchWithThread,
nullptr,
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
-static void SetTrue2DTile2DWithUArch(std::atomic_bool* processed_indicators, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- for (size_t i = start_i; i < start_i + tile_i; i++) {
- for (size_t j = start_j; j < start_j + tile_j; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
- processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
- }
+static void SetTrue2DTile1DWithUArchWithThread(std::atomic_bool* processed_indicators, uint32_t, size_t, size_t i, size_t start_j, size_t tile_j) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
+ processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
}
}
-TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolAllItemsProcessed) {
- std::vector<std::atomic_bool> indicators(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArchWithThread, SingleThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(SetTrue2DTile2DWithUArch),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_with_thread_t>(SetTrue2DTile1DWithUArchWithThread),
static_cast<void*>(indicators.data()),
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
<< "Element (" << i << ", " << j << ") not processed";
}
}
}
-TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolAllItemsProcessed) {
- std::vector<std::atomic_bool> indicators(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArchWithThread, MultiThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2125,51 +2132,47 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolAllItemsProcessed) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(SetTrue2DTile2DWithUArch),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_with_thread_t>(SetTrue2DTile1DWithUArchWithThread),
static_cast<void*>(indicators.data()),
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
<< "Element (" << i << ", " << j << ") not processed";
}
}
}
-static void Increment2DTile2DWithUArch(std::atomic_int* processed_counters, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- for (size_t i = start_i; i < start_i + tile_i; i++) {
- for (size_t j = start_j; j < start_j + tile_j; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
- processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
- }
+static void Increment2DTile1DWithUArchWithThread(std::atomic_int* processed_counters, uint32_t, size_t, size_t i, size_t start_j, size_t tile_j) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
+ processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
}
}
-TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolEachItemProcessedOnce) {
- std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArchWithThread, SingleThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(Increment2DTile2DWithUArch),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_with_thread_t>(Increment2DTile1DWithUArchWithThread),
static_cast<void*>(counters.data()),
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
<< "Element (" << i << ", " << j << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
@@ -2177,8 +2180,8 @@ TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolEachItemProcessedOnce) {
}
}
-TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolEachItemProcessedOnce) {
- std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArchWithThread, MultiThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2187,18 +2190,17 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolEachItemProcessedOnce) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(Increment2DTile2DWithUArch),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_with_thread_t>(Increment2DTile1DWithUArchWithThread),
static_cast<void*>(counters.data()),
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
<< "Element (" << i << ", " << j << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
@@ -2206,26 +2208,25 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolEachItemProcessedOnce) {
}
}
-TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolEachItemProcessedMultipleTimes) {
- std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArchWithThread, SingleThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(Increment2DTile2DWithUArch),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_with_thread_t>(Increment2DTile1DWithUArchWithThread),
static_cast<void*>(counters.data()),
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
<< "Element (" << i << ", " << j << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times "
@@ -2234,8 +2235,8 @@ TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolEachItemProcessedMultipleTime
}
}
-TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes) {
- std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+TEST(Parallelize2DTile1DWithUArchWithThread, MultiThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2245,19 +2246,18 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes
}
for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(Increment2DTile2DWithUArch),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_with_thread_t>(Increment2DTile1DWithUArchWithThread),
static_cast<void*>(counters.data()),
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
}
- for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
- for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
- const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ for (size_t i = 0; i < kParallelize2DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile1DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile1DRangeJ + j;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
<< "Element (" << i << ", " << j << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times "
@@ -2266,15 +2266,13 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes
}
}
-static void IncrementSame2DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- for (size_t i = start_i; i < start_i + tile_i; i++) {
- for (size_t j = start_j; j < start_j + tile_j; j++) {
- num_processed_items->fetch_add(1, std::memory_order_relaxed);
- }
+static void IncrementSame2DTile1DWithUArchWithThread(std::atomic_int* num_processed_items, uint32_t, size_t, size_t i, size_t start_j, size_t tile_j) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ num_processed_items->fetch_add(1, std::memory_order_relaxed);
}
}
-TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolHighContention) {
+TEST(Parallelize2DTile1DWithUArchWithThread, MultiThreadPoolHighContention) {
std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -2284,28 +2282,27 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolHighContention) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(IncrementSame2DTile2DWithUArch),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_with_thread_t>(IncrementSame2DTile1DWithUArchWithThread),
static_cast<void*>(&num_processed_items),
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
}
-static void WorkImbalance2DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
- num_processed_items->fetch_add(tile_i * tile_j, std::memory_order_relaxed);
- if (start_i == 0 && start_j == 0) {
+static void WorkImbalance2DTile1DWithUArchWithThread(std::atomic_int* num_processed_items, uint32_t, size_t, size_t i, size_t start_j, size_t tile_j) {
+ num_processed_items->fetch_add(tile_j, std::memory_order_relaxed);
+ if (i == 0 && start_j == 0) {
/* Spin-wait until all items are computed */
- while (num_processed_items->load(std::memory_order_relaxed) != kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ) {
+ while (num_processed_items->load(std::memory_order_relaxed) != kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ) {
std::atomic_thread_fence(std::memory_order_acquire);
}
}
}
-TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolWorkStealing) {
+TEST(Parallelize2DTile1DWithUArchWithThread, MultiThreadPoolWorkStealing) {
std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -2315,32 +2312,51 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolWorkStealing) {
GTEST_SKIP();
}
- pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(WorkImbalance2DTile2DWithUArch),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_with_thread_t>(WorkImbalance2DTile1DWithUArchWithThread),
static_cast<void*>(&num_processed_items),
kDefaultUArchIndex, kMaxUArchIndex,
- kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
- kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
0 /* flags */);
- EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile1DRangeI * kParallelize2DTile1DRangeJ);
}
-static void ComputeNothing3D(void*, size_t, size_t, size_t) {
+static void SetThreadTrue2DTile1DWithUArchWithThread(const size_t* num_threads, uint32_t, size_t thread_index, size_t i, size_t start_j, size_t tile_j) {
+ EXPECT_LE(thread_index, *num_threads);
}
-TEST(Parallelize3D, SingleThreadPoolCompletes) {
+TEST(Parallelize2DTile1DWithUArchWithThread, MultiThreadPoolThreadIndexValid) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ size_t num_threads = pthreadpool_get_threads_count(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_1d_with_id_with_thread_t>(SetThreadTrue2DTile1DWithUArchWithThread),
+ static_cast<void*>(&num_threads),
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile1DRangeI, kParallelize2DTile1DRangeJ, kParallelize2DTile1DTileJ,
+ 0 /* flags */);
+}
+
+static void ComputeNothing2DTile2D(void*, size_t, size_t, size_t, size_t) {
+}
+
+TEST(Parallelize2DTile2D, SingleThreadPoolCompletes) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_3d(threadpool.get(),
- ComputeNothing3D,
+ pthreadpool_parallelize_2d_tile_2d(threadpool.get(),
+ ComputeNothing2DTile2D,
nullptr,
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
0 /* flags */);
}
-TEST(Parallelize3D, MultiThreadPoolCompletes) {
+TEST(Parallelize2DTile2D, MultiThreadPoolCompletes) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2348,33 +2364,36 @@ TEST(Parallelize3D, MultiThreadPoolCompletes) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_2d_tile_2d(
threadpool.get(),
- ComputeNothing3D,
+ ComputeNothing2DTile2D,
nullptr,
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
0 /* flags */);
}
-static void CheckBounds3D(void*, size_t i, size_t j, size_t k) {
- EXPECT_LT(i, kParallelize3DRangeI);
- EXPECT_LT(j, kParallelize3DRangeJ);
- EXPECT_LT(k, kParallelize3DRangeK);
+static void CheckBounds2DTile2D(void*, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ EXPECT_LT(start_i, kParallelize2DTile2DRangeI);
+ EXPECT_LT(start_j, kParallelize2DTile2DRangeJ);
+ EXPECT_LE(start_i + tile_i, kParallelize2DTile2DRangeI);
+ EXPECT_LE(start_j + tile_j, kParallelize2DTile2DRangeJ);
}
-TEST(Parallelize3D, SingleThreadPoolAllItemsInBounds) {
+TEST(Parallelize2DTile2D, SingleThreadPoolAllItemsInBounds) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_2d_tile_2d(
threadpool.get(),
- CheckBounds3D,
+ CheckBounds2DTile2D,
nullptr,
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
0 /* flags */);
}
-TEST(Parallelize3D, MultiThreadPoolAllItemsInBounds) {
+TEST(Parallelize2DTile2D, MultiThreadPoolAllItemsInBounds) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2382,36 +2401,1875 @@ TEST(Parallelize3D, MultiThreadPoolAllItemsInBounds) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_2d_tile_2d(
threadpool.get(),
- CheckBounds3D,
+ CheckBounds2DTile2D,
nullptr,
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
0 /* flags */);
}
-static void SetTrue3D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k) {
- const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
- processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
+static void CheckTiling2DTile2D(void*, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ EXPECT_GT(tile_i, 0);
+ EXPECT_LE(tile_i, kParallelize2DTile2DTileI);
+ EXPECT_EQ(start_i % kParallelize2DTile2DTileI, 0);
+ EXPECT_EQ(tile_i, std::min<size_t>(kParallelize2DTile2DTileI, kParallelize2DTile2DRangeI - start_i));
+
+ EXPECT_GT(tile_j, 0);
+ EXPECT_LE(tile_j, kParallelize2DTile2DTileJ);
+ EXPECT_EQ(start_j % kParallelize2DTile2DTileJ, 0);
+ EXPECT_EQ(tile_j, std::min<size_t>(kParallelize2DTile2DTileJ, kParallelize2DTile2DRangeJ - start_j));
+}
+
+TEST(Parallelize2DTile2D, SingleThreadPoolUniformTiling) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_2d(
+ threadpool.get(),
+ CheckTiling2DTile2D,
+ nullptr,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+}
+
+TEST(Parallelize2DTile2D, MultiThreadPoolUniformTiling) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d(
+ threadpool.get(),
+ CheckTiling2DTile2D,
+ nullptr,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+}
+
+static void SetTrue2DTile2D(std::atomic_bool* processed_indicators, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ for (size_t i = start_i; i < start_i + tile_i; i++) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
+ }
+ }
+}
+
+TEST(Parallelize2DTile2D, SingleThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_2d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(SetTrue2DTile2D),
+ static_cast<void*>(indicators.data()),
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
+ << "Element (" << i << ", " << j << ") not processed";
+ }
+ }
+}
+
+TEST(Parallelize2DTile2D, MultiThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(SetTrue2DTile2D),
+ static_cast<void*>(indicators.data()),
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
+ << "Element (" << i << ", " << j << ") not processed";
+ }
+ }
+}
+
+static void Increment2DTile2D(std::atomic_int* processed_counters, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ for (size_t i = start_i; i < start_i + tile_i; i++) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+}
+
+TEST(Parallelize2DTile2D, SingleThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_2d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(Increment2DTile2D),
+ static_cast<void*>(counters.data()),
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
+ << "Element (" << i << ", " << j << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
+ }
+ }
+}
+
+TEST(Parallelize2DTile2D, MultiThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(Increment2DTile2D),
+ static_cast<void*>(counters.data()),
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
+ << "Element (" << i << ", " << j << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
+ }
+ }
+}
+
+TEST(Parallelize2DTile2D, SingleThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
+ pthreadpool_parallelize_2d_tile_2d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(Increment2DTile2D),
+ static_cast<void*>(counters.data()),
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+ }
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
+ << "Element (" << i << ", " << j << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times "
+ << "(expected: " << kIncrementIterations << ")";
+ }
+ }
+}
+
+TEST(Parallelize2DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
+ pthreadpool_parallelize_2d_tile_2d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(Increment2DTile2D),
+ static_cast<void*>(counters.data()),
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+ }
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
+ << "Element (" << i << ", " << j << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times "
+ << "(expected: " << kIncrementIterations << ")";
+ }
+ }
+}
+
+static void IncrementSame2DTile2D(std::atomic_int* num_processed_items, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ for (size_t i = start_i; i < start_i + tile_i; i++) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ num_processed_items->fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+}
+
+TEST(Parallelize2DTile2D, MultiThreadPoolHighContention) {
+ std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(IncrementSame2DTile2D),
+ static_cast<void*>(&num_processed_items),
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+}
+
+static void WorkImbalance2DTile2D(std::atomic_int* num_processed_items, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ num_processed_items->fetch_add(tile_i * tile_j, std::memory_order_relaxed);
+ if (start_i == 0 && start_j == 0) {
+ /* Spin-wait until all items are computed */
+ while (num_processed_items->load(std::memory_order_relaxed) != kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ) {
+ std::atomic_thread_fence(std::memory_order_acquire);
+ }
+ }
+}
+
+TEST(Parallelize2DTile2D, MultiThreadPoolWorkStealing) {
+ std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_t>(WorkImbalance2DTile2D),
+ static_cast<void*>(&num_processed_items),
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+}
+
+static void ComputeNothing2DTile2DWithUArch(void*, uint32_t, size_t, size_t, size_t, size_t) {
+}
+
+TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolCompletes) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(threadpool.get(),
+ ComputeNothing2DTile2DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+}
+
+TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolCompletes) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ ComputeNothing2DTile2DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+}
+
+static void CheckUArch2DTile2DWithUArch(void*, uint32_t uarch_index, size_t, size_t, size_t, size_t) {
+ if (uarch_index != kDefaultUArchIndex) {
+ EXPECT_LE(uarch_index, kMaxUArchIndex);
+ }
+}
+
+TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolUArchInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ CheckUArch2DTile2DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+}
+
+TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolUArchInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ CheckUArch2DTile2DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+}
+
+static void CheckBounds2DTile2DWithUArch(void*, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ EXPECT_LT(start_i, kParallelize2DTile2DRangeI);
+ EXPECT_LT(start_j, kParallelize2DTile2DRangeJ);
+ EXPECT_LE(start_i + tile_i, kParallelize2DTile2DRangeI);
+ EXPECT_LE(start_j + tile_j, kParallelize2DTile2DRangeJ);
+}
+
+TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ CheckBounds2DTile2DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+}
+
+TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ CheckBounds2DTile2DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+}
+
+static void CheckTiling2DTile2DWithUArch(void*, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ EXPECT_GT(tile_i, 0);
+ EXPECT_LE(tile_i, kParallelize2DTile2DTileI);
+ EXPECT_EQ(start_i % kParallelize2DTile2DTileI, 0);
+ EXPECT_EQ(tile_i, std::min<size_t>(kParallelize2DTile2DTileI, kParallelize2DTile2DRangeI - start_i));
+
+ EXPECT_GT(tile_j, 0);
+ EXPECT_LE(tile_j, kParallelize2DTile2DTileJ);
+ EXPECT_EQ(start_j % kParallelize2DTile2DTileJ, 0);
+ EXPECT_EQ(tile_j, std::min<size_t>(kParallelize2DTile2DTileJ, kParallelize2DTile2DRangeJ - start_j));
+}
+
+TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolUniformTiling) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ CheckTiling2DTile2DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+}
+
+TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolUniformTiling) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ CheckTiling2DTile2DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+}
+
+static void SetTrue2DTile2DWithUArch(std::atomic_bool* processed_indicators, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ for (size_t i = start_i; i < start_i + tile_i; i++) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
+ }
+ }
+}
+
+TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(SetTrue2DTile2DWithUArch),
+ static_cast<void*>(indicators.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
+ << "Element (" << i << ", " << j << ") not processed";
+ }
+ }
+}
+
+TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(SetTrue2DTile2DWithUArch),
+ static_cast<void*>(indicators.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
+ << "Element (" << i << ", " << j << ") not processed";
+ }
+ }
+}
+
+static void Increment2DTile2DWithUArch(std::atomic_int* processed_counters, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ for (size_t i = start_i; i < start_i + tile_i; i++) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+}
+
+TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(Increment2DTile2DWithUArch),
+ static_cast<void*>(counters.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
+ << "Element (" << i << ", " << j << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
+ }
+ }
+}
+
+TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(Increment2DTile2DWithUArch),
+ static_cast<void*>(counters.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
+ << "Element (" << i << ", " << j << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
+ }
+ }
+}
+
+TEST(Parallelize2DTile2DWithUArch, SingleThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(Increment2DTile2DWithUArch),
+ static_cast<void*>(counters.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+ }
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
+ << "Element (" << i << ", " << j << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times "
+ << "(expected: " << kIncrementIterations << ")";
+ }
+ }
+}
+
+TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(Increment2DTile2DWithUArch),
+ static_cast<void*>(counters.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+ }
+
+ for (size_t i = 0; i < kParallelize2DTile2DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize2DTile2DRangeJ; j++) {
+ const size_t linear_idx = i * kParallelize2DTile2DRangeJ + j;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
+ << "Element (" << i << ", " << j << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times "
+ << "(expected: " << kIncrementIterations << ")";
+ }
+ }
+}
+
+static void IncrementSame2DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ for (size_t i = start_i; i < start_i + tile_i; i++) {
+ for (size_t j = start_j; j < start_j + tile_j; j++) {
+ num_processed_items->fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+}
+
+TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolHighContention) {
+ std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(IncrementSame2DTile2DWithUArch),
+ static_cast<void*>(&num_processed_items),
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+}
+
+static void WorkImbalance2DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t start_i, size_t start_j, size_t tile_i, size_t tile_j) {
+ num_processed_items->fetch_add(tile_i * tile_j, std::memory_order_relaxed);
+ if (start_i == 0 && start_j == 0) {
+ /* Spin-wait until all items are computed */
+ while (num_processed_items->load(std::memory_order_relaxed) != kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ) {
+ std::atomic_thread_fence(std::memory_order_acquire);
+ }
+ }
+}
+
+TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolWorkStealing) {
+ std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_2d_tile_2d_with_uarch(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_2d_tile_2d_with_id_t>(WorkImbalance2DTile2DWithUArch),
+ static_cast<void*>(&num_processed_items),
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize2DTile2DRangeI, kParallelize2DTile2DRangeJ,
+ kParallelize2DTile2DTileI, kParallelize2DTile2DTileJ,
+ 0 /* flags */);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ);
+}
+
+static void ComputeNothing3D(void*, size_t, size_t, size_t) {
+}
+
+TEST(Parallelize3D, SingleThreadPoolCompletes) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d(threadpool.get(),
+ ComputeNothing3D,
+ nullptr,
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3D, MultiThreadPoolCompletes) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ ComputeNothing3D,
+ nullptr,
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+}
+
+static void CheckBounds3D(void*, size_t i, size_t j, size_t k) {
+ EXPECT_LT(i, kParallelize3DRangeI);
+ EXPECT_LT(j, kParallelize3DRangeJ);
+ EXPECT_LT(k, kParallelize3DRangeK);
+}
+
+TEST(Parallelize3D, SingleThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ CheckBounds3D,
+ nullptr,
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3D, MultiThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ CheckBounds3D,
+ nullptr,
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+}
+
+static void SetTrue3D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k) {
+ const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
+}
+
+TEST(Parallelize3D, SingleThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_t>(SetTrue3D),
+ static_cast<void*>(indicators.data()),
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
+ << "Element (" << i << ", " << j << ", " << k << ") not processed";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3D, MultiThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_t>(SetTrue3D),
+ static_cast<void*>(indicators.data()),
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
+ << "Element (" << i << ", " << j << ", " << k << ") not processed";
+ }
+ }
+ }
+}
+
+static void Increment3D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k) {
+ const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
+}
+
+TEST(Parallelize3D, SingleThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_t>(Increment3D),
+ static_cast<void*>(counters.data()),
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3D, MultiThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_t>(Increment3D),
+ static_cast<void*>(counters.data()),
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3D, SingleThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_t>(Increment3D),
+ static_cast<void*>(counters.data()),
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+ }
+
+ for (size_t i = 0; i < kParallelize3DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times "
+ << "(expected: " << kIncrementIterations << ")";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3D, MultiThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_t>(Increment3D),
+ static_cast<void*>(counters.data()),
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+ }
+
+ for (size_t i = 0; i < kParallelize3DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times "
+ << "(expected: " << kIncrementIterations << ")";
+ }
+ }
+ }
+}
+
+static void IncrementSame3D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k) {
+ num_processed_items->fetch_add(1, std::memory_order_relaxed);
+}
+
+TEST(Parallelize3D, MultiThreadPoolHighContention) {
+ std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_t>(IncrementSame3D),
+ static_cast<void*>(&num_processed_items),
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+}
+
+static void WorkImbalance3D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k) {
+ num_processed_items->fetch_add(1, std::memory_order_relaxed);
+ if (i == 0 && j == 0 && k == 0) {
+ /* Spin-wait until all items are computed */
+ while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK) {
+ std::atomic_thread_fence(std::memory_order_acquire);
+ }
+ }
+}
+
+TEST(Parallelize3D, MultiThreadPoolWorkStealing) {
+ std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_t>(WorkImbalance3D),
+ static_cast<void*>(&num_processed_items),
+ kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ 0 /* flags */);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+}
+
+static void ComputeNothing3DTile1D(void*, size_t, size_t, size_t, size_t) {
+}
+
+TEST(Parallelize3DTile1D, SingleThreadPoolCompletes) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d(threadpool.get(),
+ ComputeNothing3DTile1D,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1D, MultiThreadPoolCompletes) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ ComputeNothing3DTile1D,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void CheckBounds3DTile1D(void*, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ EXPECT_LT(i, kParallelize3DTile1DRangeI);
+ EXPECT_LT(j, kParallelize3DTile1DRangeJ);
+ EXPECT_LT(start_k, kParallelize3DTile1DRangeK);
+ EXPECT_LE(start_k + tile_k, kParallelize3DTile1DRangeK);
+}
+
+TEST(Parallelize3DTile1D, SingleThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ CheckBounds3DTile1D,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ CheckBounds3DTile1D,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void CheckTiling3DTile1D(void*, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ EXPECT_GT(tile_k, 0);
+ EXPECT_LE(tile_k, kParallelize3DTile1DTileK);
+ EXPECT_EQ(start_k % kParallelize3DTile1DTileK, 0);
+ EXPECT_EQ(tile_k, std::min<size_t>(kParallelize3DTile1DTileK, kParallelize3DTile1DRangeK - start_k));
+}
+
+TEST(Parallelize3DTile1D, SingleThreadPoolUniformTiling) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ CheckTiling3DTile1D,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1D, MultiThreadPoolUniformTiling) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ CheckTiling3DTile1D,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void SetTrue3DTile1D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ for (size_t k = start_k; k < start_k + tile_k; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
+ }
+}
+
+TEST(Parallelize3DTile1D, SingleThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(SetTrue3DTile1D),
+ static_cast<void*>(indicators.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
+ << "Element (" << i << ", " << j << ", " << k << ") not processed";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(SetTrue3DTile1D),
+ static_cast<void*>(indicators.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
+ << "Element (" << i << ", " << j << ", " << k << ") not processed";
+ }
+ }
+ }
+}
+
+static void Increment3DTile1D(std::atomic_int* processed_counters, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ for (size_t k = start_k; k < start_k + tile_k; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
+ }
+}
+
+TEST(Parallelize3DTile1D, SingleThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D),
+ static_cast<void*>(counters.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D),
+ static_cast<void*>(counters.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3DTile1D, SingleThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D),
+ static_cast<void*>(counters.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+ }
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times "
+ << "(expected: " << kIncrementIterations << ")";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D),
+ static_cast<void*>(counters.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+ }
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times "
+ << "(expected: " << kIncrementIterations << ")";
+ }
+ }
+ }
+}
+
+static void IncrementSame3DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ for (size_t k = start_k; k < start_k + tile_k; k++) {
+ num_processed_items->fetch_add(1, std::memory_order_relaxed);
+ }
+}
+
+TEST(Parallelize3DTile1D, MultiThreadPoolHighContention) {
+ std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(IncrementSame3DTile1D),
+ static_cast<void*>(&num_processed_items),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+}
+
+static void WorkImbalance3DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ num_processed_items->fetch_add(tile_k, std::memory_order_relaxed);
+ if (i == 0 && j == 0 && start_k == 0) {
+ /* Spin-wait until all items are computed */
+ while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK) {
+ std::atomic_thread_fence(std::memory_order_acquire);
+ }
+ }
+}
+
+TEST(Parallelize3DTile1D, MultiThreadPoolWorkStealing) {
+ std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(WorkImbalance3DTile1D),
+ static_cast<void*>(&num_processed_items),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+}
+
+static void ComputeNothing3DTile1DWithThread(void*, size_t, size_t, size_t, size_t, size_t) {
+}
+
+TEST(Parallelize3DTile1DWithThread, SingleThreadPoolCompletes) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(threadpool.get(),
+ ComputeNothing3DTile1DWithThread,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1DWithThread, MultiThreadPoolCompletes) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ ComputeNothing3DTile1DWithThread,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void CheckBounds3DTile1DWithThread(void*, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ EXPECT_LT(i, kParallelize3DTile1DRangeI);
+ EXPECT_LT(j, kParallelize3DTile1DRangeJ);
+ EXPECT_LT(start_k, kParallelize3DTile1DRangeK);
+ EXPECT_LE(start_k + tile_k, kParallelize3DTile1DRangeK);
+}
+
+TEST(Parallelize3DTile1DWithThread, SingleThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ CheckBounds3DTile1DWithThread,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1DWithThread, MultiThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ CheckBounds3DTile1DWithThread,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void CheckTiling3DTile1DWithThread(void*, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ EXPECT_GT(tile_k, 0);
+ EXPECT_LE(tile_k, kParallelize3DTile1DTileK);
+ EXPECT_EQ(start_k % kParallelize3DTile1DTileK, 0);
+ EXPECT_EQ(tile_k, std::min<size_t>(kParallelize3DTile1DTileK, kParallelize3DTile1DRangeK - start_k));
+}
+
+TEST(Parallelize3DTile1DWithThread, SingleThreadPoolUniformTiling) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ CheckTiling3DTile1DWithThread,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1DWithThread, MultiThreadPoolUniformTiling) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ CheckTiling3DTile1DWithThread,
+ nullptr,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void SetTrue3DTile1DWithThread(std::atomic_bool* processed_indicators, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ for (size_t k = start_k; k < start_k + tile_k; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
+ }
+}
+
+TEST(Parallelize3DTile1DWithThread, SingleThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_thread_t>(SetTrue3DTile1DWithThread),
+ static_cast<void*>(indicators.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
+ << "Element (" << i << ", " << j << ", " << k << ") not processed";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3DTile1DWithThread, MultiThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_thread_t>(SetTrue3DTile1DWithThread),
+ static_cast<void*>(indicators.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
+ << "Element (" << i << ", " << j << ", " << k << ") not processed";
+ }
+ }
+ }
+}
+
+static void Increment3DTile1DWithThread(std::atomic_int* processed_counters, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ for (size_t k = start_k; k < start_k + tile_k; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
+ }
+}
+
+TEST(Parallelize3DTile1DWithThread, SingleThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_thread_t>(Increment3DTile1DWithThread),
+ static_cast<void*>(counters.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3DTile1DWithThread, MultiThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_thread_t>(Increment3DTile1DWithThread),
+ static_cast<void*>(counters.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3DTile1DWithThread, SingleThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_thread_t>(Increment3DTile1DWithThread),
+ static_cast<void*>(counters.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+ }
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times "
+ << "(expected: " << kIncrementIterations << ")";
+ }
+ }
+ }
+}
+
+TEST(Parallelize3DTile1DWithThread, MultiThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_thread_t>(Increment3DTile1DWithThread),
+ static_cast<void*>(counters.data()),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+ }
+
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
+ << "Element (" << i << ", " << j << ", " << k << ") was processed "
+ << counters[linear_idx].load(std::memory_order_relaxed) << " times "
+ << "(expected: " << kIncrementIterations << ")";
+ }
+ }
+ }
+}
+
+static void IncrementSame3DTile1DWithThread(std::atomic_int* num_processed_items, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ for (size_t k = start_k; k < start_k + tile_k; k++) {
+ num_processed_items->fetch_add(1, std::memory_order_relaxed);
+ }
+}
+
+TEST(Parallelize3DTile1DWithThread, MultiThreadPoolHighContention) {
+ std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_thread_t>(IncrementSame3DTile1DWithThread),
+ static_cast<void*>(&num_processed_items),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+}
+
+static void WorkImbalance3DTile1DWithThread(std::atomic_int* num_processed_items, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ num_processed_items->fetch_add(tile_k, std::memory_order_relaxed);
+ if (i == 0 && j == 0 && start_k == 0) {
+ /* Spin-wait until all items are computed */
+ while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK) {
+ std::atomic_thread_fence(std::memory_order_acquire);
+ }
+ }
+}
+
+TEST(Parallelize3DTile1DWithThread, MultiThreadPoolWorkStealing) {
+ std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
+
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_thread_t>(WorkImbalance3DTile1DWithThread),
+ static_cast<void*>(&num_processed_items),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
+}
+
+static void CheckThreadIndexValid3DTile1DWithThread(const size_t* num_threads, size_t thread_index, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ EXPECT_LE(thread_index, *num_threads);
+}
+
+TEST(Parallelize3DTile1DWithThread, MultiThreadPoolThreadIndexValid) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ size_t num_threads = pthreadpool_get_threads_count(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_thread_t>(CheckThreadIndexValid3DTile1DWithThread),
+ static_cast<void*>(&num_threads),
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void ComputeNothing3DTile1DWithUArch(void*, uint32_t, size_t, size_t, size_t, size_t) {
+}
+
+TEST(Parallelize3DTile1DWithUArch, SingleThreadPoolCompletes) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(threadpool.get(),
+ ComputeNothing3DTile1DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1DWithUArch, MultiThreadPoolCompletes) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
+ threadpool.get(),
+ ComputeNothing3DTile1DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void CheckUArch3DTile1DWithUArch(void*, uint32_t uarch_index, size_t, size_t, size_t, size_t) {
+ if (uarch_index != kDefaultUArchIndex) {
+ EXPECT_LE(uarch_index, kMaxUArchIndex);
+ }
+}
+
+TEST(Parallelize3DTile1DWithUArch, SingleThreadPoolUArchInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
+ threadpool.get(),
+ CheckUArch3DTile1DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1DWithUArch, MultiThreadPoolUArchInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
+ threadpool.get(),
+ CheckUArch3DTile1DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void CheckBounds3DTile1DWithUArch(void*, uint32_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ EXPECT_LT(i, kParallelize3DTile1DRangeI);
+ EXPECT_LT(j, kParallelize3DTile1DRangeJ);
+ EXPECT_LT(start_k, kParallelize3DTile1DRangeK);
+ EXPECT_LE(start_k + tile_k, kParallelize3DTile1DRangeK);
+}
+
+TEST(Parallelize3DTile1DWithUArch, SingleThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
+ threadpool.get(),
+ CheckBounds3DTile1DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1DWithUArch, MultiThreadPoolAllItemsInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
+ threadpool.get(),
+ CheckBounds3DTile1DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void CheckTiling3DTile1DWithUArch(void*, uint32_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ EXPECT_GT(tile_k, 0);
+ EXPECT_LE(tile_k, kParallelize3DTile1DTileK);
+ EXPECT_EQ(start_k % kParallelize3DTile1DTileK, 0);
+ EXPECT_EQ(tile_k, std::min<size_t>(kParallelize3DTile1DTileK, kParallelize3DTile1DRangeK - start_k));
+}
+
+TEST(Parallelize3DTile1DWithUArch, SingleThreadPoolUniformTiling) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
+ threadpool.get(),
+ CheckTiling3DTile1DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1DWithUArch, MultiThreadPoolUniformTiling) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
+ threadpool.get(),
+ CheckTiling3DTile1DWithUArch,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
}
-TEST(Parallelize3D, SingleThreadPoolAllItemsProcessed) {
- std::vector<std::atomic_bool> indicators(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+static void SetTrue3DTile1DWithUArch(std::atomic_bool* processed_indicators, uint32_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ for (size_t k = start_k; k < start_k + tile_k; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
+ }
+}
+
+TEST(Parallelize3DTile1DWithUArch, SingleThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_t>(SetTrue3D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_t>(SetTrue3DTile1DWithUArch),
static_cast<void*>(indicators.data()),
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
0 /* flags */);
- for (size_t i = 0; i < kParallelize3DRangeI; i++) {
- for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
- for (size_t k = 0; k < kParallelize3DRangeK; k++) {
- const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
<< "Element (" << i << ", " << j << ", " << k << ") not processed";
}
@@ -2419,8 +4277,8 @@ TEST(Parallelize3D, SingleThreadPoolAllItemsProcessed) {
}
}
-TEST(Parallelize3D, MultiThreadPoolAllItemsProcessed) {
- std::vector<std::atomic_bool> indicators(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+TEST(Parallelize3DTile1DWithUArch, MultiThreadPoolAllItemsProcessed) {
+ std::vector<std::atomic_bool> indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2429,17 +4287,19 @@ TEST(Parallelize3D, MultiThreadPoolAllItemsProcessed) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_t>(SetTrue3D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_t>(SetTrue3DTile1DWithUArch),
static_cast<void*>(indicators.data()),
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
0 /* flags */);
- for (size_t i = 0; i < kParallelize3DRangeI; i++) {
- for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
- for (size_t k = 0; k < kParallelize3DRangeK; k++) {
- const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed))
<< "Element (" << i << ", " << j << ", " << k << ") not processed";
}
@@ -2447,28 +4307,32 @@ TEST(Parallelize3D, MultiThreadPoolAllItemsProcessed) {
}
}
-static void Increment3D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k) {
- const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
- processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
+static void Increment3DTile1DWithUArch(std::atomic_int* processed_counters, uint32_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ for (size_t k = start_k; k < start_k + tile_k; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
+ processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
+ }
}
-TEST(Parallelize3D, SingleThreadPoolEachItemProcessedOnce) {
- std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+TEST(Parallelize3DTile1DWithUArch, SingleThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_t>(Increment3D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_t>(Increment3DTile1DWithUArch),
static_cast<void*>(counters.data()),
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
0 /* flags */);
- for (size_t i = 0; i < kParallelize3DRangeI; i++) {
- for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
- for (size_t k = 0; k < kParallelize3DRangeK; k++) {
- const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
<< "Element (" << i << ", " << j << ", " << k << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
@@ -2477,8 +4341,8 @@ TEST(Parallelize3D, SingleThreadPoolEachItemProcessedOnce) {
}
}
-TEST(Parallelize3D, MultiThreadPoolEachItemProcessedOnce) {
- std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+TEST(Parallelize3DTile1DWithUArch, MultiThreadPoolEachItemProcessedOnce) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2487,17 +4351,19 @@ TEST(Parallelize3D, MultiThreadPoolEachItemProcessedOnce) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_t>(Increment3D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_t>(Increment3DTile1DWithUArch),
static_cast<void*>(counters.data()),
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
0 /* flags */);
- for (size_t i = 0; i < kParallelize3DRangeI; i++) {
- for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
- for (size_t k = 0; k < kParallelize3DRangeK; k++) {
- const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1)
<< "Element (" << i << ", " << j << ", " << k << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)";
@@ -2506,25 +4372,27 @@ TEST(Parallelize3D, MultiThreadPoolEachItemProcessedOnce) {
}
}
-TEST(Parallelize3D, SingleThreadPoolEachItemProcessedMultipleTimes) {
- std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+TEST(Parallelize3DTile1DWithUArch, SingleThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_t>(Increment3D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_t>(Increment3DTile1DWithUArch),
static_cast<void*>(counters.data()),
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
- 0 /* flags */);
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
}
- for (size_t i = 0; i < kParallelize3DRangeI; i++) {
- for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
- for (size_t k = 0; k < kParallelize3DRangeK; k++) {
- const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
<< "Element (" << i << ", " << j << ", " << k << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times "
@@ -2534,8 +4402,8 @@ TEST(Parallelize3D, SingleThreadPoolEachItemProcessedMultipleTimes) {
}
}
-TEST(Parallelize3D, MultiThreadPoolEachItemProcessedMultipleTimes) {
- std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+TEST(Parallelize3DTile1DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes) {
+ std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2545,18 +4413,20 @@ TEST(Parallelize3D, MultiThreadPoolEachItemProcessedMultipleTimes) {
}
for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_t>(Increment3D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_t>(Increment3DTile1DWithUArch),
static_cast<void*>(counters.data()),
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
- 0 /* flags */);
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
}
- for (size_t i = 0; i < kParallelize3DRangeI; i++) {
- for (size_t j = 0; j < kParallelize3DRangeJ; j++) {
- for (size_t k = 0; k < kParallelize3DRangeK; k++) {
- const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k;
+ for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) {
+ for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) {
+ for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) {
+ const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations)
<< "Element (" << i << ", " << j << ", " << k << ") was processed "
<< counters[linear_idx].load(std::memory_order_relaxed) << " times "
@@ -2566,11 +4436,13 @@ TEST(Parallelize3D, MultiThreadPoolEachItemProcessedMultipleTimes) {
}
}
-static void IncrementSame3D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k) {
- num_processed_items->fetch_add(1, std::memory_order_relaxed);
+static void IncrementSame3DTile1DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ for (size_t k = start_k; k < start_k + tile_k; k++) {
+ num_processed_items->fetch_add(1, std::memory_order_relaxed);
+ }
}
-TEST(Parallelize3D, MultiThreadPoolHighContention) {
+TEST(Parallelize3DTile1DWithUArch, MultiThreadPoolHighContention) {
std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -2580,26 +4452,28 @@ TEST(Parallelize3D, MultiThreadPoolHighContention) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_t>(IncrementSame3D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_t>(IncrementSame3DTile1DWithUArch),
static_cast<void*>(&num_processed_items),
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
0 /* flags */);
- EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
}
-static void WorkImbalance3D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k) {
- num_processed_items->fetch_add(1, std::memory_order_relaxed);
- if (i == 0 && j == 0 && k == 0) {
+static void WorkImbalance3DTile1DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ num_processed_items->fetch_add(tile_k, std::memory_order_relaxed);
+ if (i == 0 && j == 0 && start_k == 0) {
/* Spin-wait until all items are computed */
- while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK) {
+ while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK) {
std::atomic_thread_fence(std::memory_order_acquire);
}
}
}
-TEST(Parallelize3D, MultiThreadPoolWorkStealing) {
+TEST(Parallelize3DTile1DWithUArch, MultiThreadPoolWorkStealing) {
std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -2609,31 +4483,34 @@ TEST(Parallelize3D, MultiThreadPoolWorkStealing) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_t>(WorkImbalance3D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_t>(WorkImbalance3DTile1DWithUArch),
static_cast<void*>(&num_processed_items),
- kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
0 /* flags */);
- EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK);
+ EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
}
-static void ComputeNothing3DTile1D(void*, size_t, size_t, size_t, size_t) {
+static void ComputeNothing3DTile1DWithUArchWithThread(void*, uint32_t, size_t, size_t, size_t, size_t, size_t) {
}
-TEST(Parallelize3DTile1D, SingleThreadPoolCompletes) {
+TEST(Parallelize3DTile1DWithUArchWithThread, SingleThreadPoolCompletes) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_3d_tile_1d(threadpool.get(),
- ComputeNothing3DTile1D,
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(threadpool.get(),
+ ComputeNothing3DTile1DWithUArchWithThread,
nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
}
-TEST(Parallelize3DTile1D, MultiThreadPoolCompletes) {
+TEST(Parallelize3DTile1DWithUArchWithThread, MultiThreadPoolCompletes) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2641,36 +4518,76 @@ TEST(Parallelize3DTile1D, MultiThreadPoolCompletes) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- ComputeNothing3DTile1D,
+ ComputeNothing3DTile1DWithUArchWithThread,
nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
}
-static void CheckBounds3DTile1D(void*, size_t i, size_t j, size_t start_k, size_t tile_k) {
+static void CheckUArch3DTile1DWithUArchWithThread(void*, uint32_t uarch_index, size_t, size_t, size_t, size_t, size_t) {
+ if (uarch_index != kDefaultUArchIndex) {
+ EXPECT_LE(uarch_index, kMaxUArchIndex);
+ }
+}
+
+TEST(Parallelize3DTile1DWithUArchWithThread, SingleThreadPoolUArchInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
+ threadpool.get(),
+ CheckUArch3DTile1DWithUArchWithThread,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+TEST(Parallelize3DTile1DWithUArchWithThread, MultiThreadPoolUArchInBounds) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ if (pthreadpool_get_threads_count(threadpool.get()) <= 1) {
+ GTEST_SKIP();
+ }
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
+ threadpool.get(),
+ CheckUArch3DTile1DWithUArchWithThread,
+ nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
+static void CheckBounds3DTile1DWithUArchWithThread(void*, uint32_t, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
EXPECT_LT(i, kParallelize3DTile1DRangeI);
EXPECT_LT(j, kParallelize3DTile1DRangeJ);
EXPECT_LT(start_k, kParallelize3DTile1DRangeK);
EXPECT_LE(start_k + tile_k, kParallelize3DTile1DRangeK);
}
-TEST(Parallelize3DTile1D, SingleThreadPoolAllItemsInBounds) {
+TEST(Parallelize3DTile1DWithUArchWithThread, SingleThreadPoolAllItemsInBounds) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- CheckBounds3DTile1D,
+ CheckBounds3DTile1DWithUArchWithThread,
nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
}
-TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsInBounds) {
+TEST(Parallelize3DTile1DWithUArchWithThread, MultiThreadPoolAllItemsInBounds) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2678,36 +4595,38 @@ TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsInBounds) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- CheckBounds3DTile1D,
+ CheckBounds3DTile1DWithUArchWithThread,
nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
}
-static void CheckTiling3DTile1D(void*, size_t i, size_t j, size_t start_k, size_t tile_k) {
+static void CheckTiling3DTile1DWithUArchWithThread(void*, uint32_t, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
EXPECT_GT(tile_k, 0);
EXPECT_LE(tile_k, kParallelize3DTile1DTileK);
EXPECT_EQ(start_k % kParallelize3DTile1DTileK, 0);
EXPECT_EQ(tile_k, std::min<size_t>(kParallelize3DTile1DTileK, kParallelize3DTile1DRangeK - start_k));
}
-TEST(Parallelize3DTile1D, SingleThreadPoolUniformTiling) {
+TEST(Parallelize3DTile1DWithUArchWithThread, SingleThreadPoolUniformTiling) {
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- CheckTiling3DTile1D,
+ CheckTiling3DTile1DWithUArchWithThread,
nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
}
-TEST(Parallelize3DTile1D, MultiThreadPoolUniformTiling) {
+TEST(Parallelize3DTile1DWithUArchWithThread, MultiThreadPoolUniformTiling) {
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
@@ -2715,32 +4634,34 @@ TEST(Parallelize3DTile1D, MultiThreadPoolUniformTiling) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- CheckTiling3DTile1D,
+ CheckTiling3DTile1DWithUArchWithThread,
nullptr,
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
}
-static void SetTrue3DTile1D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t start_k, size_t tile_k) {
+static void SetTrue3DTile1DWithUArchWithThread(std::atomic_bool* processed_indicators, uint32_t, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
for (size_t k = start_k; k < start_k + tile_k; k++) {
const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
processed_indicators[linear_idx].store(true, std::memory_order_relaxed);
}
}
-TEST(Parallelize3DTile1D, SingleThreadPoolAllItemsProcessed) {
+TEST(Parallelize3DTile1DWithUArchWithThread, SingleThreadPoolAllItemsProcessed) {
std::vector<std::atomic_bool> indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(SetTrue3DTile1D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_with_thread_t>(SetTrue3DTile1DWithUArchWithThread),
static_cast<void*>(indicators.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
@@ -2756,7 +4677,7 @@ TEST(Parallelize3DTile1D, SingleThreadPoolAllItemsProcessed) {
}
}
-TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsProcessed) {
+TEST(Parallelize3DTile1DWithUArchWithThread, MultiThreadPoolAllItemsProcessed) {
std::vector<std::atomic_bool> indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -2766,10 +4687,11 @@ TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsProcessed) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(SetTrue3DTile1D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_with_thread_t>(SetTrue3DTile1DWithUArchWithThread),
static_cast<void*>(indicators.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
@@ -2785,23 +4707,24 @@ TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsProcessed) {
}
}
-static void Increment3DTile1D(std::atomic_int* processed_counters, size_t i, size_t j, size_t start_k, size_t tile_k) {
+static void Increment3DTile1DWithUArchWithThread(std::atomic_int* processed_counters, uint32_t, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
for (size_t k = start_k; k < start_k + tile_k; k++) {
const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k;
processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed);
}
}
-TEST(Parallelize3DTile1D, SingleThreadPoolEachItemProcessedOnce) {
+TEST(Parallelize3DTile1DWithUArchWithThread, SingleThreadPoolEachItemProcessedOnce) {
std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_with_thread_t>(Increment3DTile1DWithUArchWithThread),
static_cast<void*>(counters.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
@@ -2818,7 +4741,7 @@ TEST(Parallelize3DTile1D, SingleThreadPoolEachItemProcessedOnce) {
}
}
-TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedOnce) {
+TEST(Parallelize3DTile1DWithUArchWithThread, MultiThreadPoolEachItemProcessedOnce) {
std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -2828,10 +4751,11 @@ TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedOnce) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_with_thread_t>(Increment3DTile1DWithUArchWithThread),
static_cast<void*>(counters.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
@@ -2848,17 +4772,18 @@ TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedOnce) {
}
}
-TEST(Parallelize3DTile1D, SingleThreadPoolEachItemProcessedMultipleTimes) {
+TEST(Parallelize3DTile1DWithUArchWithThread, SingleThreadPoolEachItemProcessedMultipleTimes) {
std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy);
ASSERT_TRUE(threadpool.get());
for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_with_thread_t>(Increment3DTile1DWithUArchWithThread),
static_cast<void*>(counters.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
@@ -2877,7 +4802,7 @@ TEST(Parallelize3DTile1D, SingleThreadPoolEachItemProcessedMultipleTimes) {
}
}
-TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedMultipleTimes) {
+TEST(Parallelize3DTile1DWithUArchWithThread, MultiThreadPoolEachItemProcessedMultipleTimes) {
std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -2888,10 +4813,11 @@ TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedMultipleTimes) {
}
for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) {
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_with_thread_t>(Increment3DTile1DWithUArchWithThread),
static_cast<void*>(counters.data()),
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
@@ -2910,13 +4836,13 @@ TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedMultipleTimes) {
}
}
-static void IncrementSame3DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t tile_k) {
+static void IncrementSame3DTile1DWithUArchWithThread(std::atomic_int* num_processed_items, uint32_t, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
for (size_t k = start_k; k < start_k + tile_k; k++) {
num_processed_items->fetch_add(1, std::memory_order_relaxed);
}
}
-TEST(Parallelize3DTile1D, MultiThreadPoolHighContention) {
+TEST(Parallelize3DTile1DWithUArchWithThread, MultiThreadPoolHighContention) {
std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -2926,17 +4852,18 @@ TEST(Parallelize3DTile1D, MultiThreadPoolHighContention) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(IncrementSame3DTile1D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_with_thread_t>(IncrementSame3DTile1DWithUArchWithThread),
static_cast<void*>(&num_processed_items),
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
}
-static void WorkImbalance3DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t tile_k) {
+static void WorkImbalance3DTile1DWithUArchWithThread(std::atomic_int* num_processed_items, uint32_t, size_t, size_t i, size_t j, size_t start_k, size_t tile_k) {
num_processed_items->fetch_add(tile_k, std::memory_order_relaxed);
if (i == 0 && j == 0 && start_k == 0) {
/* Spin-wait until all items are computed */
@@ -2946,7 +4873,7 @@ static void WorkImbalance3DTile1D(std::atomic_int* num_processed_items, size_t i
}
}
-TEST(Parallelize3DTile1D, MultiThreadPoolWorkStealing) {
+TEST(Parallelize3DTile1DWithUArchWithThread, MultiThreadPoolWorkStealing) {
std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0);
auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
@@ -2956,16 +4883,37 @@ TEST(Parallelize3DTile1D, MultiThreadPoolWorkStealing) {
GTEST_SKIP();
}
- pthreadpool_parallelize_3d_tile_1d(
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
threadpool.get(),
- reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(WorkImbalance3DTile1D),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_with_thread_t>(WorkImbalance3DTile1DWithUArchWithThread),
static_cast<void*>(&num_processed_items),
+ kDefaultUArchIndex, kMaxUArchIndex,
kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
kParallelize3DTile1DTileK,
0 /* flags */);
EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK);
}
+static void SetThreadTrue3DTile1DWithUArchWithThread(const size_t* num_threads, uint32_t, size_t thread_index, size_t i, size_t j, size_t start_k, size_t tile_k) {
+ EXPECT_LE(thread_index, *num_threads);
+}
+
+TEST(Parallelize3DTile1DWithUArchWithThread, MultiThreadPoolThreadIndexValid) {
+ auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy);
+ ASSERT_TRUE(threadpool.get());
+
+ size_t num_threads = pthreadpool_get_threads_count(threadpool.get());
+
+ pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread(
+ threadpool.get(),
+ reinterpret_cast<pthreadpool_task_3d_tile_1d_with_id_with_thread_t>(SetThreadTrue3DTile1DWithUArchWithThread),
+ static_cast<void*>(&num_threads),
+ kDefaultUArchIndex, kMaxUArchIndex,
+ kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK,
+ kParallelize3DTile1DTileK,
+ 0 /* flags */);
+}
+
static void ComputeNothing3DTile2D(void*, size_t, size_t, size_t, size_t, size_t) {
}