YetAnotherCoupler 3.2.0
Loading...
Searching...
No Matches
io_utils.c
Go to the documentation of this file.
1// Copyright (c) 2024 The YAC Authors
2//
3// SPDX-License-Identifier: BSD-3-Clause
4
5#include <string.h>
6#include "yac_mpi_internal.h"
7#include "io_utils.h"
8#include "math.h"
9
10#define IO_RANK_LIST_STR "YAC_IO_RANK_LIST"
11#define IO_MAX_NUM_RANKS_STR "YAC_IO_MAX_NUM_RANKS"
12#define IO_RANK_EXCLUDE_LIST_STR "YAC_IO_RANK_EXCLUDE_LIST"
13#define IO_MAX_NUM_RANKS_PER_NODE "YAC_IO_MAX_NUM_RANKS_PER_NODE"
14#define DEFAULT_MAX_NUM_IO_RANK_PER_NODE (1);
15
16static inline int compare_int(const void * a, const void * b) {
17
18 int const * a_ = a, * b_ = b;
19
20 return (*a_ > *b_) - (*b_ > *a_);
21}
22
23static void read_rank_list(
24 char const * env_name, MPI_Comm comm, size_t * num_ranks_, int ** ranks_) {
25
26 int rank;
27 yac_mpi_call(MPI_Comm_rank(comm, &rank), comm);
28
29 size_t num_ranks = 0;
30 int * ranks = NULL;
31
32 // environment is only checked on rank 0, results are broadcasted
33 // to other processes
34 if (rank == 0) {
35
36 // check whether the user provided a list of ranks for IO
37 char * rank_list_str = getenv(env_name);
38 if ((rank_list_str != NULL) && (rank_list_str[0] != '\0')) {
39
40 int temp_num_ranks = 1;
41 for (char * curr_char = rank_list_str + 1; *curr_char != '\0'; ++curr_char)
42 if (*curr_char == ',') ++temp_num_ranks;
43
44 char * rank_list_copy = strdup(rank_list_str);
45 num_ranks = 0;
46 int * world_ranks = xmalloc(temp_num_ranks * sizeof(*world_ranks));
47
48 // parse rank list
49 char * rank_str = strtok(rank_list_copy, ",");
50 while (rank_str != NULL) {
51 int curr_rank = atoi(rank_str);
53 curr_rank >= 0, "ERROR(read_rank_list): \"%s\" is not a valid rank",
54 rank_str);
55 world_ranks[num_ranks++] = curr_rank;
56 rank_str = strtok(NULL, ",");
57 }
58 free(rank_list_copy);
59
60 // sort ranks
61 qsort(world_ranks, num_ranks, sizeof(*world_ranks), compare_int);
62
63 // remove duplicated ranks
64 yac_remove_duplicates_int(world_ranks, &num_ranks);
65
66 // translate rank list from MPI_COMM_WORLD to comm
67 ranks = xmalloc(num_ranks * sizeof(ranks));
68 MPI_Group world_group, comm_group;
70 MPI_Comm_group(MPI_COMM_WORLD, &world_group), MPI_COMM_WORLD);
71 yac_mpi_call(MPI_Comm_group(comm, &comm_group), comm);
73 MPI_Group_translate_ranks(
74 world_group, (int)num_ranks, world_ranks,
75 comm_group, ranks), MPI_COMM_WORLD);
76 yac_mpi_call(MPI_Group_free(&comm_group), comm);
77 yac_mpi_call(MPI_Group_free(&world_group), MPI_COMM_WORLD);
78 free(world_ranks);
79
80 // remove ranks, which are not avaible in comm
81 size_t new_num_ranks = 0;
82 for (size_t i = 0; i < num_ranks; ++i) {
83 if (ranks[i] != MPI_UNDEFINED) {
84 if (i != new_num_ranks)
85 ranks[new_num_ranks] = ranks[i];
86 ++new_num_ranks;
87 }
88 }
89 num_ranks = new_num_ranks;
90
91 // sort ranks
92 qsort(ranks, num_ranks, sizeof(*ranks), compare_int);
93 }
94
95 // broadcast ranks
97 MPI_Bcast(&num_ranks, 1, YAC_MPI_SIZE_T, 0, comm), comm);
98 if (num_ranks > 0) {
99 ranks = xrealloc(ranks, num_ranks * sizeof(*ranks));
101 MPI_Bcast(ranks, (int)num_ranks, MPI_INT, 0, comm), comm);
102 } else {
103 free(ranks);
104 }
105 } else {
106
107 // receive ranks from root
109 MPI_Bcast(&num_ranks, 1, YAC_MPI_SIZE_T, 0, comm), comm);
110 if (num_ranks > 0) {
111 ranks = xmalloc(num_ranks * sizeof(*ranks));
113 MPI_Bcast(ranks, (int)num_ranks, MPI_INT, 0, comm), comm);
114 }
115 }
116 *num_ranks_ = num_ranks;
117 *ranks_ = ranks;
118}
119
121 MPI_Comm comm, size_t * num_io_ranks, int ** io_ranks) {
122
123 read_rank_list(IO_RANK_LIST_STR, comm, num_io_ranks, io_ranks);
124}
125
127 MPI_Comm comm, size_t * num_io_ranks_, int ** io_ranks_) {
128
129 int rank, size;
130 yac_mpi_call(MPI_Comm_rank(comm, &rank), comm);
131 yac_mpi_call(MPI_Comm_size(comm, &size), comm);
132
133 int max_num_io_rank_per_node;
134
135 // environment is only checked on rank 0, results are broadcasted
136 // to other processes
137 if (rank == 0) {
138
139 // check whether the user provided a maximum number of ranks per node
140 char * max_num_io_rank_per_node_str = getenv(IO_MAX_NUM_RANKS_PER_NODE);
141 if ((max_num_io_rank_per_node_str != NULL) &&
142 (max_num_io_rank_per_node_str[0] != '\0')) {
143
144 max_num_io_rank_per_node = atoi(max_num_io_rank_per_node_str);
146 (max_num_io_rank_per_node > 0) || (max_num_io_rank_per_node == -1),
147 "ERROR(check_io_max_num_ranks_per_node): "
148 "\"%s\" is not a valid value for the maximum number of io ranks "
149 "per node", max_num_io_rank_per_node_str);
150 } else {
151 max_num_io_rank_per_node = DEFAULT_MAX_NUM_IO_RANK_PER_NODE;
152 }
154 MPI_Bcast(&max_num_io_rank_per_node, 1, MPI_INT, 0, comm), comm);
155 } else {
157 MPI_Bcast(&max_num_io_rank_per_node, 1, MPI_INT, 0, comm), comm);
158 }
159
160 // if there is no limit on the number of io ranks per node
161 if (max_num_io_rank_per_node == -1) return;
162
163 // create one communicator per node
164 MPI_Comm node_comm;
166 MPI_Comm_split_type(
167 comm, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &node_comm), comm);
168
169 // determine rank within node comm
170 int node_rank;
171 yac_mpi_call(MPI_Comm_rank(node_comm, &node_rank), node_comm);
172
173 // translate io ranks to node ranks
174 int * node_ranks = xmalloc(*num_io_ranks_ * sizeof(*node_ranks));
175 MPI_Group io_group, node_group;
176 yac_mpi_call(MPI_Comm_group(comm, &io_group), comm);
177 yac_mpi_call(MPI_Comm_group(node_comm, &node_group), node_comm);
179 MPI_Group_translate_ranks(
180 io_group, (int)*num_io_ranks_, *io_ranks_,
181 node_group, node_ranks), comm);
182 yac_mpi_call(MPI_Group_free(&node_group), node_comm);
183 yac_mpi_call(MPI_Group_free(&io_group), comm);
184 yac_mpi_call(MPI_Comm_free(&node_comm), comm);
185
186 // remove MPI_UNDEFINED entries
187 size_t num_node_io_ranks = 0;
188 for (size_t i = 0; i < *num_io_ranks_; ++i) {
189 if (node_ranks[i] != MPI_UNDEFINED) {
190 if (i != num_node_io_ranks)
191 node_ranks[num_node_io_ranks] = node_ranks[i];
192 ++num_node_io_ranks;
193 }
194 }
195
196 // sort ranks
197 qsort(node_ranks, num_node_io_ranks, sizeof(*node_ranks), compare_int);
198
199 // check whether local process is within the first n io ranks on its node
200 int local_is_io_rank = 0;
201 for (size_t i = 0;
202 (i < num_node_io_ranks) &&
203 (i < (size_t)max_num_io_rank_per_node) && !local_is_io_rank; ++i)
204 if (node_ranks[i] == node_rank) local_is_io_rank = 1;
205 free(node_ranks);
206
207 // determine io ranks on all nodes
208 int * is_io_rank = xmalloc((size_t)size * sizeof(*is_io_rank));
210 MPI_Allgather(
211 &local_is_io_rank, 1, MPI_INT, is_io_rank, 1, MPI_INT, comm), comm);
212
213 // compress is_io_rank into io_ranks
214 size_t num_io_ranks = 0;
215 for (int i = 0; i < size; ++i)
216 if (is_io_rank[i]) is_io_rank[num_io_ranks++] = i;
217
218 free(*io_ranks_);
219 *num_io_ranks_ = num_io_ranks;
220 *io_ranks_ = xrealloc(is_io_rank, num_io_ranks * sizeof(**io_ranks_));
221}
222
224 MPI_Comm comm, size_t * num_io_ranks, int ** io_ranks) {
225
226 int rank;
227 yac_mpi_call(MPI_Comm_rank(comm, &rank), comm);
228
229 int max_num_ranks = -1;
230
231 // environment is only checked on rank 0, results are broadcasted
232 // to other processes
233 if (rank == 0) {
234
235 // check whether the user provided a maximum number of io ranks
236 char * max_num_ranks_str = getenv(IO_MAX_NUM_RANKS_STR);
237 if ((max_num_ranks_str != NULL) && (max_num_ranks_str[0] != '\0')) {
238 max_num_ranks = atoi(max_num_ranks_str);
240 (max_num_ranks > 0) || (max_num_ranks == -1),
241 "ERROR(check_io_max_num_ranks): "
242 "\"%s\" is not a valid value for the maximum number of io ranks",
243 max_num_ranks_str);
244 }
245
247 MPI_Bcast(&max_num_ranks, 1, MPI_INT, 0, comm), comm);
248 } else {
250 MPI_Bcast(&max_num_ranks, 1, MPI_INT, 0, comm), comm);
251 }
252
253 // if there is no limit on the number of io ranks
254 if (max_num_ranks == -1) return;
255
256 if ((max_num_ranks > 0) && ((size_t)max_num_ranks < *num_io_ranks)) {
257 *io_ranks =
258 xrealloc(*io_ranks, (size_t)max_num_ranks * sizeof(**io_ranks));
259 *num_io_ranks = (size_t)max_num_ranks;
260 }
261}
262
264 MPI_Comm comm, size_t * num_io_ranks, int ** io_ranks) {
265
266 // read in the list of ranks which are not to be used for io
267 size_t num_io_ranks_excluded;
268 int * io_ranks_excluded;
270 IO_RANK_EXCLUDE_LIST_STR, comm, &num_io_ranks_excluded, &io_ranks_excluded);
271
272 // if there are ranks to be excluded
273 if (num_io_ranks_excluded > 0) {
274
275 // sort ranks
276 qsort(*io_ranks, *num_io_ranks, sizeof(**io_ranks), compare_int);
277
278 // match exclude list with io rank list
279 size_t new_num_io_ranks = 0;
280 for (size_t i = 0, j = 0; i < *num_io_ranks; ++i) {
281
282 while ((j < num_io_ranks_excluded) &&
283 (io_ranks_excluded[j] < (*io_ranks)[i])) ++j;
284
285 if ((j >= num_io_ranks_excluded) ||
286 ((*io_ranks)[i] != io_ranks_excluded[j])) {
287
288 if (i != new_num_io_ranks)
289 (*io_ranks)[new_num_io_ranks] = (*io_ranks)[i];
290 ++new_num_io_ranks;
291 }
292 }
293
294 if (new_num_io_ranks != *num_io_ranks) {
295 *io_ranks = xrealloc(*io_ranks, new_num_io_ranks * sizeof(**io_ranks));
296 *num_io_ranks = new_num_io_ranks;
297 }
298 }
299
300 free(io_ranks_excluded);
301}
302
304 MPI_Comm comm, int * local_is_io_, int ** io_ranks_, int * num_io_ranks_) {
305
306 int rank, size;
307 yac_mpi_call(MPI_Comm_rank(comm, &rank), comm);
308 yac_mpi_call(MPI_Comm_size(comm, &size), comm);
309
310 size_t num_io_ranks = 0;
311 int * io_ranks = NULL;
312
313 // check environment for io rank list
314 read_io_rank_list(comm, &num_io_ranks, &io_ranks);
315
316 // if no rank list was provided -> generate default rank list (all processes)
317 if (num_io_ranks == 0) {
318 num_io_ranks = (size_t)size;
319 io_ranks = xmalloc(num_io_ranks * sizeof(*io_ranks));
320 for (int i = 0; i < size; ++i) io_ranks[i] = i;
321 }
322
323 // check whether we have to exclude some ranks
324 check_io_rank_exclude_list(comm, &num_io_ranks, &io_ranks);
325
326 // check for the maximum number of io ranks per node
327 check_io_max_num_ranks_per_node(comm, &num_io_ranks, &io_ranks);
328
329 // check maximum number of io ranks
330 check_io_max_num_ranks(comm, &num_io_ranks, &io_ranks);
331
333 num_io_ranks > 0, "ERROR(yac_get_io_ranks): could not determine io ranks");
334
335 int local_is_io = 0;
336 for (size_t i = 0; (i < num_io_ranks) && !local_is_io; ++i)
337 if (io_ranks[i] == rank) local_is_io = 1;
338
339 *local_is_io_ = local_is_io;
340 *io_ranks_ = io_ranks;
341 *num_io_ranks_ = (int)num_io_ranks;
342}
343
344void yac_nc_open(const char * path, int omode, int * ncidp) {
345
346#ifndef YAC_NETCDF_ENABLED
347
348 UNUSED(path);
349 UNUSED(omode);
350 UNUSED(ncidp);
351 die("ERROR(yac_nc_open): YAC is built without the NetCDF support");
352#else
353
355 yac_file_exists(path),
356 "ERROR(yac_nc_open): file \"%s\" does not exist", path);
357 YAC_HANDLE_ERROR(nc_open(path, omode, ncidp));
358#endif
359}
360
361void yac_nc_create(const char * path, int cmode, int * ncidp) {
362
363#ifndef YAC_NETCDF_ENABLED
364
365 UNUSED(path);
366 UNUSED(cmode);
367 UNUSED(ncidp);
368 die("ERROR(yac_nc_create): YAC is built without the NetCDF support");
369#else
370
371 int status = nc_create(path, cmode, ncidp);
373 status == NC_NOERR,
374 "ERROR(yac_nc_create): failed to create file \"%s\" "
375 "(NetCDF error message: \"%s\")", path, nc_strerror(status));
376#endif
377}
378
379void yac_nc_inq_dimid(int ncid, char const * name, int * dimidp) {
380
381#ifndef YAC_NETCDF_ENABLED
382
383 UNUSED(ncid);
384 UNUSED(name);
385 UNUSED(dimidp);
386 die("ERROR(yac_nc_inq_dimid): YAC is built without the NetCDF support");
387#else
388
389 int status = nc_inq_dimid(ncid, name, dimidp);
390
391 if (status == NC_EBADDIM) {
392 // GCOVR_EXCL_START
393 size_t pathlen;
394 YAC_HANDLE_ERROR(nc_inq_path(ncid, &pathlen, NULL));
395 char * path = xmalloc(pathlen * sizeof(*path));
396 YAC_HANDLE_ERROR(nc_inq_path(ncid, NULL, path));
398 0, "ERROR(yac_nc_inq_dimid): "
399 "dimension \"%s\" could not be found in file \"%s\"", name, path);
400 // GCOVR_EXCL_STOP
401 } else YAC_HANDLE_ERROR(status);
402#endif
403}
404
405void yac_nc_inq_varid(int ncid, char const * name, int * varidp) {
406
407#ifndef YAC_NETCDF_ENABLED
408
409 UNUSED(ncid);
410 UNUSED(name);
411 UNUSED(varidp);
412 die("ERROR(yac_nc_inq_varid): YAC is built without the NetCDF support");
413#else
414
415 int status = nc_inq_varid(ncid, name, varidp);
416
417 if (status == NC_ENOTVAR) {
418 // GCOVR_EXCL_START
419 size_t pathlen;
420 YAC_HANDLE_ERROR(nc_inq_path(ncid, &pathlen, NULL));
421 char * path = xmalloc(pathlen * sizeof(*path));
422 YAC_HANDLE_ERROR(nc_inq_path(ncid, NULL, path));
424 0, "ERROR(yac_nc_inq_varid): "
425 "variable \"%s\" could not be found in file \"%s\"", name, path);
426 // GCOVR_EXCL_STOP
427 } else YAC_HANDLE_ERROR(status);
428#endif
429}
#define UNUSED(x)
Definition core.h:71
#define IO_MAX_NUM_RANKS_PER_NODE
Definition io_utils.c:13
#define IO_RANK_LIST_STR
Definition io_utils.c:10
static void read_io_rank_list(MPI_Comm comm, size_t *num_io_ranks, int **io_ranks)
Definition io_utils.c:120
#define DEFAULT_MAX_NUM_IO_RANK_PER_NODE
Definition io_utils.c:14
void yac_get_io_ranks(MPI_Comm comm, int *local_is_io_, int **io_ranks_, int *num_io_ranks_)
Definition io_utils.c:303
static void read_rank_list(char const *env_name, MPI_Comm comm, size_t *num_ranks_, int **ranks_)
Definition io_utils.c:23
void yac_nc_create(const char *path, int cmode, int *ncidp)
Definition io_utils.c:361
void yac_nc_inq_varid(int ncid, char const *name, int *varidp)
Definition io_utils.c:405
#define IO_RANK_EXCLUDE_LIST_STR
Definition io_utils.c:12
static void check_io_max_num_ranks(MPI_Comm comm, size_t *num_io_ranks, int **io_ranks)
Definition io_utils.c:223
static void check_io_max_num_ranks_per_node(MPI_Comm comm, size_t *num_io_ranks_, int **io_ranks_)
Definition io_utils.c:126
#define IO_MAX_NUM_RANKS_STR
Definition io_utils.c:11
static int compare_int(const void *a, const void *b)
Definition io_utils.c:16
void yac_nc_open(const char *path, int omode, int *ncidp)
Definition io_utils.c:344
void yac_nc_inq_dimid(int ncid, char const *name, int *dimidp)
Definition io_utils.c:379
static void check_io_rank_exclude_list(MPI_Comm comm, size_t *num_io_ranks, int **io_ranks)
Definition io_utils.c:263
int yac_file_exists(const char *filename)
Definition utils_core.c:12
#define YAC_HANDLE_ERROR(exp)
Definition io_utils.h:30
#define xrealloc(ptr, size)
Definition ppm_xfuncs.h:67
#define xmalloc(size)
Definition ppm_xfuncs.h:66
static void yac_remove_duplicates_int(int *array, size_t *n)
#define YAC_ASSERT_F(exp, format,...)
Definition yac_assert.h:19
#define die(msg)
Definition yac_assert.h:12
#define YAC_ASSERT(exp, msg)
Definition yac_assert.h:16
#define yac_mpi_call(call, comm)
#define YAC_MPI_SIZE_T