35 MPI_Initialized(&mpi_initialized);
37 return mpi_initialized;
44 "ERROR(yac_yaxt_init): MPI has not yet been initialised");
48 "ERROR(yac_yaxt_init): YAXT was initialised by YAC. \n"
49 "In case there are multiple instances of YAC in parallel, the user has "
50 "to initialise YAXT such that it is available on all processes that "
81 "ERROR(yac_mpi_init_core): "
82 "could not determine MPI data type for size_t "
83 "(sizeof(size_t): %zu; sizeof(%s): %zu)",
119 MPI_Comm_rank(comm, &rank);
121 char error_string[MPI_MAX_ERROR_STRING];
122 int length_of_error_string, error_class;
124 MPI_Error_class(error_code, &error_class);
125 MPI_Error_string(error_class, error_string, &length_of_error_string);
126 fprintf(stderr,
"%3d: %s\n", rank, error_string);
127 MPI_Abort(comm, error_code);
132 void const * send_buffer,
size_t const * sendcounts,
size_t const * sdispls,
133 void * recv_buffer,
size_t const * recvcounts,
size_t const * rdispls,
134 size_t dt_size, MPI_Datatype dt, MPI_Comm comm) {
136#define USE_P2P_ALLTOALLV
137#ifdef USE_P2P_ALLTOALLV
138 int comm_rank, comm_size;
143 for (
int i = 0; i < comm_size; ++i)
144 req_count += (sendcounts[i] > 0) + (recvcounts[i] > 0);
145 MPI_Request * req =
xmalloc((
size_t)req_count *
sizeof(*req));
148 for (
int j = 0, lb = comm_rank, ub = comm_size; j < 2;
149 ++j, lb = 0, ub = comm_rank) {
150 for (
int i = lb; i < ub; ++i) {
151 if (sendcounts[i] > 0) {
153 sendcounts[i] <= INT_MAX,
154 "ERROR(yac_alltoallv_p2p): sendcounts[%d] = %zu exceeds INT_MAX (%d)",
155 i, sendcounts[i], (
int)INT_MAX)
158 (
void const *)((
unsigned char *)send_buffer +
159 dt_size * sdispls[i]),
160 (
int)(sendcounts[i]), dt, i, 0,
161 comm, req + req_count), comm);
164 if (recvcounts[i] > 0) {
166 recvcounts[i] <= INT_MAX,
167 "ERROR(yac_alltoallv_p2p): recvcounts[%d] = %zu exceeds INT_MAX (%d)",
168 i, recvcounts[i], (
int)INT_MAX)
171 (
void *)((
unsigned char *)recv_buffer +
172 dt_size * rdispls[i]),
173 (
int)(recvcounts[i]), dt, i, 0,
174 comm, req + req_count), comm);
179 yac_mpi_call(MPI_Waitall(req_count, req, MPI_STATUSES_IGNORE), comm);
184 int * int_buffer =
xmalloc(4 * comm_size *
sizeof(*int_buffer));
185 int * int_sendcounts = int_buffer + 0 * comm_size;
186 int * int_sdispls = int_buffer + 1 * comm_size;
187 int * int_recvcounts = int_buffer + 2 * comm_size;
188 int * int_rdispls = int_buffer + 3 * comm_size;
189 for (
int i = 0; i < comm_size; ++i) {
191 sendcounts[i] <= INT_MAX,
192 "ERROR(yac_alltoallv_p2p): sendcounts[%d] = %zu exceeds INT_MAX (%d)",
193 i, sendcounts[i], (
int)INT_MAX)
195 sdispls[i] <= INT_MAX,
196 "ERROR(yac_alltoallv_p2p): sdispls[%d] = %zu exceeds INT_MAX (%d)",
197 i, sdispls[i], (
int)INT_MAX)
199 recvcounts[i] <= INT_MAX,
200 "ERROR(yac_alltoallv_p2p): recvcounts[%d] = %zu exceeds INT_MAX (%d)",
201 i, recvcounts[i], (
int)INT_MAX)
203 rdispls[i] <= INT_MAX,
204 "ERROR(yac_alltoallv_p2p): rdispls[%d] = %zu exceeds INT_MAX (%d)",
205 i, rdispls[i], (
int)INT_MAX)
206 int_sendcounts[i] = (int)(sendcounts[i]);
207 int_sdispls[i] = (int)(sdispls[i]);
208 int_recvcounts[i] = (int)(recvcounts[i]);
209 int_rdispls[i] = (int)(rdispls[i]);
212 MPI_Alltoallv(send_buffer, int_sendcounts, int_sdispls, dt,
213 recv_buffer, int_recvcounts, int_rdispls, dt, comm), comm);
218#define YAC_ALLTOALL_P2P_TYPE(NAME, TYPE, TYPE_SIZE, MPI_TYPE) \
219 void yac_alltoallv_ ## NAME ## _p2p( \
220 TYPE const * send_buffer, size_t const * sendcounts, size_t const * sdispls, \
221 TYPE * recv_buffer, size_t const * recvcounts, size_t const * rdispls, \
224 (void const *)send_buffer, sendcounts, sdispls, \
225 (void *)recv_buffer, recvcounts, rdispls, \
226 TYPE_SIZE, MPI_TYPE, comm); \
237 void const * send_buffer,
int const * sendcounts,
int const * sdispls,
238 void * recv_buffer,
int const * recvcounts,
int const * rdispls,
239 size_t dt_size, MPI_Datatype dt, struct
yac_group_comm group_comm) {
241 MPI_Comm comm = group_comm.comm;
244 int rank = comm_rank - group_comm.start;
247 for (
int i = 0; i < group_comm.size; ++i)
248 req_count += (sendcounts[i] > 0) + (recvcounts[i] > 0);
249 MPI_Request * req =
xmalloc((
size_t)req_count *
sizeof(*req));
252 for (
int j = 0, lb = rank, ub = group_comm.size; j < 2;
253 ++j, lb = 0, ub = rank) {
254 for (
int i = lb; i < ub; ++i) {
255 if (sendcounts[i] > 0) {
259 (
void const *)((
unsigned char *)send_buffer +
260 dt_size * (
size_t)(sdispls[i])),
261 sendcounts[i], dt, i + group_comm.start, 0,
262 comm, req + req_count), comm);
265 if (recvcounts[i] > 0) {
268 (
void *)((
unsigned char *)recv_buffer +
269 dt_size * (
size_t)(rdispls[i])),
270 recvcounts[i], dt, i + group_comm.start, 0,
271 comm, req + req_count), comm);
276 yac_mpi_call(MPI_Waitall(req_count, req, MPI_STATUSES_IGNORE), comm);
283 while(power < x) power *= 2;
295 int rank = comm_rank - group_comm.
start;
297 int rem = group_comm.
size - pof2;
299 double * recv_buffer =
xmalloc((
size_t)count *
sizeof(*recv_buffer));
301 if (rank < 2 * rem) {
306 (
void*)recv_buffer, count, MPI_DOUBLE, rank - 1 + group_comm.
start, 0,
307 group_comm.
comm, MPI_STATUS_IGNORE), group_comm.
comm);
308 for (
int i = 0; i < count; ++i) buffer[i] += recv_buffer[i];
313 (
void const *)buffer, count, MPI_DOUBLE, rank + 1 + group_comm.
start,
314 0, group_comm.
comm), group_comm.
comm);
318 my_rank = rank - rem;
322 while (mask < pof2) {
323 int newdst = my_rank ^ mask;
325 if (newdst < rem) dst = newdst * 2 + 1;
326 else dst = newdst + rem;
329 (
void const*)buffer, count, MPI_DOUBLE, dst + group_comm.
start, 0,
330 (
void*)recv_buffer, count, MPI_DOUBLE, dst + group_comm.
start, 0,
331 group_comm.
comm, MPI_STATUS_IGNORE),
333 for (
int i = 0; i < count; ++i) buffer[i] += recv_buffer[i];
338 if (rank < 2 * rem) {
342 (
void const*)buffer, count, MPI_DOUBLE, rank - 1 + group_comm.
start,
343 0, group_comm.
comm), group_comm.
comm);
347 (
void*)buffer, count, MPI_DOUBLE, rank + 1 + group_comm.
start, 0,
348 group_comm.
comm, MPI_STATUS_IGNORE), group_comm.
comm);
354 if (x <= 1)
return 0;
356 while (x >>= 1) ++l2;
362 const uint64_t * sendbuf, uint64_t * recvbuf,
int count,
367 int rank = comm_rank - group_comm.
start;
369 uint64_t * temp =
xmalloc((
size_t)group_comm.
size * (
size_t)count *
sizeof(*temp));
372 memcpy(temp, sendbuf, (
size_t)count *
sizeof(*temp));
374 int curr_len = count;
376 for (
int r = 0; r < lg2; ++r) {
377 int dst = (rank - nblk + group_comm.
size) % group_comm.
size;
378 int src = (rank + nblk) % group_comm.
size;
381 (
void const*)temp, curr_len, MPI_UINT64_T, dst + group_comm.
start, 0,
382 (
void *)(temp + (
size_t)curr_len), curr_len, MPI_UINT64_T,
383 src + group_comm.
start, 0, group_comm.
comm, MPI_STATUS_IGNORE),
388 int rest = count * group_comm.
size - curr_len;
389 int dst = (rank - nblk + group_comm.
size) % group_comm.
size;
390 int src = (rank + nblk) % group_comm.
size;
393 (
void const*)temp, rest, MPI_UINT64_T, dst + group_comm.
start, 0,
394 (
void*)(temp + (
size_t)curr_len), rest, MPI_UINT64_T,
395 src + group_comm.
start, 0, group_comm.
comm, MPI_STATUS_IGNORE),
397 memcpy(recvbuf + (
size_t)count * (
size_t)rank,
398 temp, (
size_t)count * (
size_t)(group_comm.
size - rank) *
sizeof(*temp));
399 memcpy(recvbuf, temp + (
size_t)count * (
size_t)(group_comm.
size - rank),
400 (
size_t)count * (
size_t)rank *
sizeof(*temp));
406 void * buffer,
int count, MPI_Datatype datatype,
int root,
411 int rank = comm_rank - group_comm.
start;
414 if ((root < group_comm.
start) ||
415 (root >= group_comm.
start + group_comm.
size)) {
417 if (comm_rank == root) {
420 (
void const*)buffer, count, datatype, group_comm.
start, 0,
423 }
else if (comm_rank == group_comm.
start) {
426 buffer, count, datatype, root, 0, group_comm.
comm,
427 MPI_STATUS_IGNORE), group_comm.
comm);
431 root -= group_comm.
start;
437 int temp_rank = (group_comm.
size + rank - root) % group_comm.
size;
440 while (bit <= temp_rank) bit <<= 1;
444 (((temp_rank ^ bit) + root) % group_comm.
size) + group_comm.
start;
447 MPI_Recv(buffer, count, datatype, src_rank, 0, group_comm.
comm,
448 MPI_STATUS_IGNORE), group_comm.
comm);
452 int temp_rank = (group_comm.
size + rank - root) % group_comm.
size;
453 int bit = 1, send_rank;
455 while(bit <= temp_rank) bit <<= 1;
457 while ((send_rank = temp_rank | bit) < group_comm.
size) {
461 send_rank = ((send_rank + root) % group_comm.
size) + group_comm.
start;
465 (
void const*)buffer, count, datatype, send_rank, 0, group_comm.
comm),
473 group_comm.
start = 0;
490 return group_comm.
size;
515 (split_rank >= 0) && (split_rank < group_comm.
size),
516 "ERROR(yac_group_comm_split): invalid split rank")
519 int size[2] = {split_rank, group_comm.
size - split_rank};
520 int local_idx = (comm_rank - group_comm.
start) >= split_rank;
523 local_group_comm->
size =
size[local_idx];
525 remote_group_comm->
start =
start[local_idx^1];
526 remote_group_comm->
size =
size[local_idx^1];
533 MPI_Datatype bnd_circle_dt;
534 int array_of_blocklengths[] = {3, 1, 1};
535 const MPI_Aint array_of_displacements[] =
536 {(MPI_Aint)(intptr_t)(
const void *)&(dummy.
base_vector[0]) -
537 (MPI_Aint)(intptr_t)(
const void *)&dummy,
538 (MPI_Aint)(intptr_t)(
const void *)&(dummy.
inc_angle.
sin) -
539 (MPI_Aint)(intptr_t)(
const void *)&dummy,
540 (MPI_Aint)(intptr_t)(
const void *)&(dummy.
inc_angle.
cos) -
541 (MPI_Aint)(intptr_t)(
const void *)&dummy};
542 const MPI_Datatype array_of_types[] =
543 {MPI_DOUBLE, MPI_DOUBLE, MPI_DOUBLE};
545 MPI_Type_create_struct(3, array_of_blocklengths, array_of_displacements,
546 array_of_types, &bnd_circle_dt), comm);
551 MPI_Datatype dt,
size_t new_size, MPI_Comm comm) {
553 MPI_Datatype resized_dt;
555#define OPENMPI_WORKAROUND
556#ifdef OPENMPI_WORKAROUND
558 MPI_Type_get_extent(dt, &lb, &extent);
560 MPI_Type_create_resized(dt, lb, (MPI_Aint)new_size, &resized_dt), comm);
563 MPI_Type_create_resized(dt, 0, (MPI_Aint)new_size, &resized_dt), comm);
565#undef OPENMPI_WORKAROUND
572 int count,
size_t const * sendcounts,
size_t * recvcounts,
573 size_t * sdispls,
size_t * rdispls, MPI_Comm comm) {
587 size_t iter_count = (size_t)(count * comm_size);
588 for (
size_t i = 0, saccu = 0, raccu = 0; i < iter_count; ++i) {
589 sdispls[i+1] = saccu;
591 saccu += sendcounts[i];
592 raccu += recvcounts[i];
597 int count,
size_t ** sendcounts,
size_t ** recvcounts,
598 size_t ** sdispls,
size_t ** rdispls, MPI_Comm comm) {
603 size_t * comm_buffer_;
607 4 * (
size_t)count * (
size_t)comm_size + 1);
613 (4 * (
size_t)count * (
size_t)comm_size + 1) *
sizeof(*comm_buffer_));
616 size_t offset = (size_t)count * (
size_t)comm_size;
617 *sendcounts = comm_buffer_ + 0 * offset;
618 *recvcounts = comm_buffer_ + 1 * offset;
619 *rdispls = comm_buffer_ + 2 * offset;
620 *sdispls = comm_buffer_ + 3 * offset;
624 comm_buffer_, 0, (
size_t)count * (
size_t)comm_size *
sizeof(*comm_buffer_));
628 size_t * sendcounts,
size_t * recvcounts,
629 size_t * sdispls,
size_t * rdispls) {
#define ENSURE_ARRAY_SIZE(arrayp, curr_array_size, req_size)
struct sin_cos_angle inc_angle
angle between the middle point and the boundary of the spherical cap
#define YAC_ASSERT_F(exp, format,...)
#define YAC_ASSERT(exp, msg)
static int mpi_initialised_by_yac
#define YAC_ALLTOALL_P2P_TYPE(NAME, TYPE, TYPE_SIZE, MPI_TYPE)
void yac_alltoallv_p2p_group(void const *send_buffer, int const *sendcounts, int const *sdispls, void *recv_buffer, int const *recvcounts, int const *rdispls, size_t dt_size, MPI_Datatype dt, struct yac_group_comm group_comm)
int yac_group_comm_get_global_rank(struct yac_group_comm group_comm)
void yac_generate_alltoallv_args(int count, size_t const *sendcounts, size_t *recvcounts, size_t *sdispls, size_t *rdispls, MPI_Comm comm)
static int yaxt_initialised_by_yac
void yac_free_comm_buffers(size_t *sendcounts, size_t *recvcounts, size_t *sdispls, size_t *rdispls)
int yac_group_comm_get_rank(struct yac_group_comm group_comm)
int yac_mpi_is_initialised()
void yac_group_comm_split(struct yac_group_comm group_comm, int split_rank, struct yac_group_comm *local_group_comm, struct yac_group_comm *remote_group_comm)
MPI_Datatype yac_get_bounding_circle_mpi_datatype(MPI_Comm comm)
static size_t * comm_buffer
int yac_group_comm_get_global_size(struct yac_group_comm group_comm)
void yac_yaxt_init(MPI_Comm comm)
struct yac_group_comm yac_group_comm_new(MPI_Comm comm)
void yac_mpi_error(int error_code, MPI_Comm comm)
static size_t comm_buffer_array_size
static int yaxt_init_count
void yac_allreduce_sum_dble(double *buffer, int count, struct yac_group_comm group_comm)
void yac_yaxt_init_f2c(MPI_Fint comm)
void yac_get_comm_buffers(int count, size_t **sendcounts, size_t **recvcounts, size_t **sdispls, size_t **rdispls, MPI_Comm comm)
int yac_group_comm_get_size(struct yac_group_comm group_comm)
MPI_Datatype yac_create_resized(MPI_Datatype dt, size_t new_size, MPI_Comm comm)
void yac_alltoallv_p2p(void const *send_buffer, size_t const *sendcounts, size_t const *sdispls, void *recv_buffer, size_t const *recvcounts, size_t const *rdispls, size_t dt_size, MPI_Datatype dt, MPI_Comm comm)
void yac_bcast_group(void *buffer, int count, MPI_Datatype datatype, int root, struct yac_group_comm group_comm)
void yac_allgather_uint64(const uint64_t *sendbuf, uint64_t *recvbuf, int count, struct yac_group_comm group_comm)
static int nearest_power_of_two(int x)
static int comm_buffer_in_use
static void yac_yaxt_cleanup()
void yac_group_comm_delete(struct yac_group_comm group_comm)
#define yac_mpi_call(call, comm)
#define YAC_MPI_SIZE_T_TYPE