35 MPI_Initialized(&mpi_initialized);
37 return mpi_initialized;
44 "ERROR(yac_yaxt_init): MPI has not yet been initialised");
48 "ERROR(yac_yaxt_init): YAXT was initialised by YAC. \n"
49 "In case there are multiple instances of YAC in parallel, the user has "
50 "to initialise YAXT such that it is available on all processes that "
81 "ERROR(yac_mpi_init_core): "
82 "could not determine MPI data type for size_t "
83 "(sizeof(size_t): %zu; sizeof(%s): %zu)",
118 MPI_Comm_rank(comm, &rank);
120 char error_string[MPI_MAX_ERROR_STRING];
121 int length_of_error_string, error_class;
123 MPI_Error_class(error_code, &error_class);
124 MPI_Error_string(error_class, error_string, &length_of_error_string);
125 fprintf(stderr,
"%3d: %s\n", rank, error_string);
126 MPI_Abort(comm, error_code);
130 void const * send_buffer,
size_t const * sendcounts,
size_t const * sdispls,
131 void * recv_buffer,
size_t const * recvcounts,
size_t const * rdispls,
132 size_t dt_size, MPI_Datatype dt, MPI_Comm comm) {
134#define USE_P2P_ALLTOALLV
135#ifdef USE_P2P_ALLTOALLV
136 int comm_rank, comm_size;
141 for (
int i = 0; i < comm_size; ++i)
142 req_count += (sendcounts[i] > 0) + (recvcounts[i] > 0);
143 MPI_Request * req =
xmalloc((
size_t)req_count *
sizeof(*req));
146 for (
int j = 0, lb = comm_rank, ub = comm_size; j < 2;
147 ++j, lb = 0, ub = comm_rank) {
148 for (
int i = lb; i < ub; ++i) {
149 if (sendcounts[i] > 0) {
151 sendcounts[i] <= INT_MAX,
152 "ERROR(yac_alltoallv_p2p): sendcounts[%d] = %zu exceeds INT_MAX (%d)",
153 i, sendcounts[i], (
int)INT_MAX)
156 (
void const *)((
unsigned char *)send_buffer +
157 dt_size * sdispls[i]),
158 (
int)(sendcounts[i]), dt, i, 0,
159 comm, req + req_count), comm);
162 if (recvcounts[i] > 0) {
164 recvcounts[i] <= INT_MAX,
165 "ERROR(yac_alltoallv_p2p): recvcounts[%d] = %zu exceeds INT_MAX (%d)",
166 i, recvcounts[i], (
int)INT_MAX)
169 (
void *)((
unsigned char *)recv_buffer +
170 dt_size * rdispls[i]),
171 (
int)(recvcounts[i]), dt, i, 0,
172 comm, req + req_count), comm);
177 yac_mpi_call(MPI_Waitall(req_count, req, MPI_STATUSES_IGNORE), comm);
182 int * int_buffer =
xmalloc(4 * comm_size *
sizeof(*int_buffer));
183 int * int_sendcounts = int_buffer + 0 * comm_size;
184 int * int_sdispls = int_buffer + 1 * comm_size;
185 int * int_recvcounts = int_buffer + 2 * comm_size;
186 int * int_rdispls = int_buffer + 3 * comm_size;
187 for (
int i = 0; i < comm_size; ++i) {
189 sendcounts[i] <= INT_MAX,
190 "ERROR(yac_alltoallv_p2p): sendcounts[%d] = %zu exceeds INT_MAX (%d)",
191 i, sendcounts[i], (
int)INT_MAX)
193 sdispls[i] <= INT_MAX,
194 "ERROR(yac_alltoallv_p2p): sdispls[%d] = %zu exceeds INT_MAX (%d)",
195 i, sdispls[i], (
int)INT_MAX)
197 recvcounts[i] <= INT_MAX,
198 "ERROR(yac_alltoallv_p2p): recvcounts[%d] = %zu exceeds INT_MAX (%d)",
199 i, recvcounts[i], (
int)INT_MAX)
201 rdispls[i] <= INT_MAX,
202 "ERROR(yac_alltoallv_p2p): rdispls[%d] = %zu exceeds INT_MAX (%d)",
203 i, rdispls[i], (
int)INT_MAX)
204 int_sendcounts[i] = (int)(sendcounts[i]);
205 int_sdispls[i] = (int)(sdispls[i]);
206 int_recvcounts[i] = (int)(recvcounts[i]);
207 int_rdispls[i] = (int)(rdispls[i]);
210 MPI_Alltoallv(send_buffer, int_sendcounts, int_sdispls, dt,
211 recv_buffer, int_recvcounts, int_rdispls, dt, comm), comm);
216#define YAC_ALLTOALL_P2P_TYPE(NAME, TYPE, TYPE_SIZE, MPI_TYPE) \
217 void yac_alltoallv_ ## NAME ## _p2p( \
218 TYPE const * send_buffer, size_t const * sendcounts, size_t const * sdispls, \
219 TYPE * recv_buffer, size_t const * recvcounts, size_t const * rdispls, \
222 (void const *)send_buffer, sendcounts, sdispls, \
223 (void *)recv_buffer, recvcounts, rdispls, \
224 TYPE_SIZE, MPI_TYPE, comm); \
235 void const * send_buffer,
int const * sendcounts,
int const * sdispls,
236 void * recv_buffer,
int const * recvcounts,
int const * rdispls,
237 size_t dt_size, MPI_Datatype dt, struct
yac_group_comm group_comm) {
239 MPI_Comm comm = group_comm.comm;
242 int rank = comm_rank - group_comm.start;
245 for (
int i = 0; i < group_comm.size; ++i)
246 req_count += (sendcounts[i] > 0) + (recvcounts[i] > 0);
247 MPI_Request * req =
xmalloc((
size_t)req_count *
sizeof(*req));
250 for (
int j = 0, lb = rank, ub = group_comm.size; j < 2;
251 ++j, lb = 0, ub = rank) {
252 for (
int i = lb; i < ub; ++i) {
253 if (sendcounts[i] > 0) {
257 (
void const *)((
unsigned char *)send_buffer +
258 dt_size * (
size_t)(sdispls[i])),
259 sendcounts[i], dt, i + group_comm.start, 0,
260 comm, req + req_count), comm);
263 if (recvcounts[i] > 0) {
266 (
void *)((
unsigned char *)recv_buffer +
267 dt_size * (
size_t)(rdispls[i])),
268 recvcounts[i], dt, i + group_comm.start, 0,
269 comm, req + req_count), comm);
274 yac_mpi_call(MPI_Waitall(req_count, req, MPI_STATUSES_IGNORE), comm);
281 while(power < x) power *= 2;
293 int rank = comm_rank - group_comm.
start;
295 int rem = group_comm.
size - pof2;
297 double * recv_buffer =
xmalloc((
size_t)count *
sizeof(*recv_buffer));
299 if (rank < 2 * rem) {
304 (
void*)recv_buffer, count, MPI_DOUBLE, rank - 1 + group_comm.
start, 0,
305 group_comm.
comm, MPI_STATUS_IGNORE), group_comm.
comm);
306 for (
int i = 0; i < count; ++i) buffer[i] += recv_buffer[i];
311 (
void const *)buffer, count, MPI_DOUBLE, rank + 1 + group_comm.
start,
312 0, group_comm.
comm), group_comm.
comm);
316 my_rank = rank - rem;
320 while (mask < pof2) {
321 int newdst = my_rank ^ mask;
323 if (newdst < rem) dst = newdst * 2 + 1;
324 else dst = newdst + rem;
327 (
void const*)buffer, count, MPI_DOUBLE, dst + group_comm.
start, 0,
328 (
void*)recv_buffer, count, MPI_DOUBLE, dst + group_comm.
start, 0,
329 group_comm.
comm, MPI_STATUS_IGNORE),
331 for (
int i = 0; i < count; ++i) buffer[i] += recv_buffer[i];
336 if (rank < 2 * rem) {
340 (
void const*)buffer, count, MPI_DOUBLE, rank - 1 + group_comm.
start,
341 0, group_comm.
comm), group_comm.
comm);
345 (
void*)buffer, count, MPI_DOUBLE, rank + 1 + group_comm.
start, 0,
346 group_comm.
comm, MPI_STATUS_IGNORE), group_comm.
comm);
352 if (x <= 1)
return 0;
354 while (x >>= 1) ++l2;
360 const uint64_t * sendbuf, uint64_t * recvbuf,
int count,
365 int rank = comm_rank - group_comm.
start;
367 uint64_t * temp =
xmalloc((
size_t)group_comm.
size * (
size_t)count *
sizeof(*temp));
370 memcpy(temp, sendbuf, (
size_t)count *
sizeof(*temp));
372 int curr_len = count;
374 for (
int r = 0; r < lg2; ++r) {
375 int dst = (rank - nblk + group_comm.
size) % group_comm.
size;
376 int src = (rank + nblk) % group_comm.
size;
379 (
void const*)temp, curr_len, MPI_UINT64_T, dst + group_comm.
start, 0,
380 (
void *)(temp + (
size_t)curr_len), curr_len, MPI_UINT64_T,
381 src + group_comm.
start, 0, group_comm.
comm, MPI_STATUS_IGNORE),
386 int rest = count * group_comm.
size - curr_len;
387 int dst = (rank - nblk + group_comm.
size) % group_comm.
size;
388 int src = (rank + nblk) % group_comm.
size;
391 (
void const*)temp, rest, MPI_UINT64_T, dst + group_comm.
start, 0,
392 (
void*)(temp + (
size_t)curr_len), rest, MPI_UINT64_T,
393 src + group_comm.
start, 0, group_comm.
comm, MPI_STATUS_IGNORE),
395 memcpy(recvbuf + (
size_t)count * (
size_t)rank,
396 temp, (
size_t)count * (
size_t)(group_comm.
size - rank) *
sizeof(*temp));
397 memcpy(recvbuf, temp + (
size_t)count * (
size_t)(group_comm.
size - rank),
398 (
size_t)count * (
size_t)rank *
sizeof(*temp));
404 void * buffer,
int count, MPI_Datatype datatype,
int root,
409 int rank = comm_rank - group_comm.
start;
412 if ((root < group_comm.
start) ||
413 (root >= group_comm.
start + group_comm.
size)) {
415 if (comm_rank == root) {
418 (
void const*)buffer, count, datatype, group_comm.
start, 0,
421 }
else if (comm_rank == group_comm.
start) {
424 buffer, count, datatype, root, 0, group_comm.
comm,
425 MPI_STATUS_IGNORE), group_comm.
comm);
429 root -= group_comm.
start;
435 int temp_rank = (group_comm.
size + rank - root) % group_comm.
size;
438 while (bit <= temp_rank) bit <<= 1;
442 (((temp_rank ^ bit) + root) % group_comm.
size) + group_comm.
start;
445 MPI_Recv(buffer, count, datatype, src_rank, 0, group_comm.
comm,
446 MPI_STATUS_IGNORE), group_comm.
comm);
450 int temp_rank = (group_comm.
size + rank - root) % group_comm.
size;
451 int bit = 1, send_rank;
453 while(bit <= temp_rank) bit <<= 1;
455 while ((send_rank = temp_rank | bit) < group_comm.
size) {
459 send_rank = ((send_rank + root) % group_comm.
size) + group_comm.
start;
463 (
void const*)buffer, count, datatype, send_rank, 0, group_comm.
comm),
471 group_comm.
start = 0;
488 return group_comm.
size;
513 (split_rank >= 0) && (split_rank < group_comm.
size),
514 "ERROR(yac_group_comm_split): invalid split rank")
517 int size[2] = {split_rank, group_comm.
size - split_rank};
518 int local_idx = (comm_rank - group_comm.
start) >= split_rank;
521 local_group_comm->
size =
size[local_idx];
523 remote_group_comm->
start =
start[local_idx^1];
524 remote_group_comm->
size =
size[local_idx^1];
531 MPI_Datatype bnd_circle_dt;
532 int array_of_blocklengths[] = {3, 1, 1};
533 const MPI_Aint array_of_displacements[] =
534 {(MPI_Aint)(intptr_t)(
const void *)&(dummy.
base_vector[0]) -
535 (MPI_Aint)(intptr_t)(
const void *)&dummy,
536 (MPI_Aint)(intptr_t)(
const void *)&(dummy.
inc_angle.
sin) -
537 (MPI_Aint)(intptr_t)(
const void *)&dummy,
538 (MPI_Aint)(intptr_t)(
const void *)&(dummy.
inc_angle.
cos) -
539 (MPI_Aint)(intptr_t)(
const void *)&dummy};
540 const MPI_Datatype array_of_types[] =
541 {MPI_DOUBLE, MPI_DOUBLE, MPI_DOUBLE};
543 MPI_Type_create_struct(3, array_of_blocklengths, array_of_displacements,
544 array_of_types, &bnd_circle_dt), comm);
549 MPI_Datatype dt,
size_t new_size, MPI_Comm comm) {
551 MPI_Datatype resized_dt;
553#define OPENMPI_WORKAROUND
554#ifdef OPENMPI_WORKAROUND
556 MPI_Type_get_extent(dt, &lb, &extent);
558 MPI_Type_create_resized(dt, lb, (MPI_Aint)new_size, &resized_dt), comm);
561 MPI_Type_create_resized(dt, 0, (MPI_Aint)new_size, &resized_dt), comm);
563#undef OPENMPI_WORKAROUND
570 int count,
size_t const * sendcounts,
size_t * recvcounts,
571 size_t * sdispls,
size_t * rdispls, MPI_Comm comm) {
585 for (
size_t i = 0, saccu = 0, raccu = 0; i < count * comm_size; ++i) {
586 sdispls[i+1] = saccu;
588 saccu += sendcounts[i];
589 raccu += recvcounts[i];
594 int count,
size_t ** sendcounts,
size_t ** recvcounts,
595 size_t ** sdispls,
size_t ** rdispls, MPI_Comm comm) {
600 size_t * comm_buffer_;
604 4 * (
size_t)count * (
size_t)comm_size + 1);
610 (4 * (
size_t)count * (
size_t)comm_size + 1) *
sizeof(*comm_buffer_));
613 size_t offset = (size_t)count * (
size_t)comm_size;
614 *sendcounts = comm_buffer_ + 0 * offset;
615 *recvcounts = comm_buffer_ + 1 * offset;
616 *rdispls = comm_buffer_ + 2 * offset;
617 *sdispls = comm_buffer_ + 3 * offset;
621 comm_buffer_, 0, (
size_t)count * (
size_t)comm_size *
sizeof(*comm_buffer_));
625 size_t * sendcounts,
size_t * recvcounts,
626 size_t * sdispls,
size_t * rdispls) {
#define ENSURE_ARRAY_SIZE(arrayp, curr_array_size, req_size)
struct sin_cos_angle inc_angle
angle between the middle point and the boundary of the spherical cap
#define YAC_ASSERT_F(exp, format,...)
#define YAC_ASSERT(exp, msg)
static int mpi_initialised_by_yac
#define YAC_ALLTOALL_P2P_TYPE(NAME, TYPE, TYPE_SIZE, MPI_TYPE)
void yac_alltoallv_p2p_group(void const *send_buffer, int const *sendcounts, int const *sdispls, void *recv_buffer, int const *recvcounts, int const *rdispls, size_t dt_size, MPI_Datatype dt, struct yac_group_comm group_comm)
int yac_group_comm_get_global_rank(struct yac_group_comm group_comm)
void yac_generate_alltoallv_args(int count, size_t const *sendcounts, size_t *recvcounts, size_t *sdispls, size_t *rdispls, MPI_Comm comm)
static int yaxt_initialised_by_yac
void yac_free_comm_buffers(size_t *sendcounts, size_t *recvcounts, size_t *sdispls, size_t *rdispls)
int yac_group_comm_get_rank(struct yac_group_comm group_comm)
int yac_mpi_is_initialised()
void yac_group_comm_split(struct yac_group_comm group_comm, int split_rank, struct yac_group_comm *local_group_comm, struct yac_group_comm *remote_group_comm)
MPI_Datatype yac_get_bounding_circle_mpi_datatype(MPI_Comm comm)
static size_t * comm_buffer
int yac_group_comm_get_global_size(struct yac_group_comm group_comm)
void yac_yaxt_init(MPI_Comm comm)
struct yac_group_comm yac_group_comm_new(MPI_Comm comm)
void yac_mpi_error(int error_code, MPI_Comm comm)
static size_t comm_buffer_array_size
static int yaxt_init_count
void yac_allreduce_sum_dble(double *buffer, int count, struct yac_group_comm group_comm)
void yac_yaxt_init_f2c(MPI_Fint comm)
void yac_get_comm_buffers(int count, size_t **sendcounts, size_t **recvcounts, size_t **sdispls, size_t **rdispls, MPI_Comm comm)
int yac_group_comm_get_size(struct yac_group_comm group_comm)
MPI_Datatype yac_create_resized(MPI_Datatype dt, size_t new_size, MPI_Comm comm)
void yac_alltoallv_p2p(void const *send_buffer, size_t const *sendcounts, size_t const *sdispls, void *recv_buffer, size_t const *recvcounts, size_t const *rdispls, size_t dt_size, MPI_Datatype dt, MPI_Comm comm)
void yac_bcast_group(void *buffer, int count, MPI_Datatype datatype, int root, struct yac_group_comm group_comm)
void yac_allgather_uint64(const uint64_t *sendbuf, uint64_t *recvbuf, int count, struct yac_group_comm group_comm)
static int nearest_power_of_two(int x)
static int comm_buffer_in_use
static void yac_yaxt_cleanup()
void yac_group_comm_delete(struct yac_group_comm group_comm)
#define yac_mpi_call(call, comm)
#define YAC_MPI_SIZE_T_TYPE