52 size_t * num_elem,
size_t * num_nodes,
53 int ** num_nodes_per_elem,
int ** num_elem_per_node,
54 size_t ** elem_to_node,
size_t ** node_to_elem) {
56 int comm_rank, comm_size;
58 MPI_Comm_rank(comm, &comm_rank);
59 MPI_Comm_size(comm, &comm_size);
61 int local_is_io, * io_ranks, num_io_ranks;
64 size_t num_global_nodes, num_global_elem, num_nod_per_elem;
66 size_t read_local_start_elem = 0;
67 size_t read_num_local_elems = 0;
68 size_t read_local_start_node = 0;
69 size_t read_num_local_nodes = 0;
72 int * read_dist_elem_to_node = NULL;
76 unsigned long io_proc_idx = ULONG_MAX;
77 for (
int i = 0; (i < num_io_ranks) && (io_proc_idx == ULONG_MAX); ++i)
78 if (io_ranks[i] == comm_rank)
79 io_proc_idx = (
unsigned long)i;
87 size_t num_el_blk, num_el_in_blk1;
101 "reader currently only supports a single block of elements");
103 num_global_elem == num_el_in_blk1,
104 "total number of elements and number of elements in first block "
105 "of elements do not match");
108 read_local_start_elem =
109 ((
unsigned long)num_global_elem * io_proc_idx) / (
unsigned long)num_io_ranks;
110 read_num_local_elems =
111 ((
unsigned long)num_global_elem * (io_proc_idx+1)) / (
unsigned long)num_io_ranks -
112 (
unsigned long)read_local_start_elem;
113 read_local_start_node =
114 ((
unsigned long)num_global_nodes * io_proc_idx) / (
unsigned long)num_io_ranks;
115 read_num_local_nodes =
116 ((
unsigned long)num_global_nodes * (io_proc_idx+1)) / (
unsigned long)num_io_ranks -
117 (
unsigned long)read_local_start_node;
120 double * read_node_coord_x =
121 xmalloc(read_num_local_nodes *
sizeof(*read_node_coord_x));
122 double * read_node_coord_y =
123 xmalloc(read_num_local_nodes *
sizeof(*read_node_coord_y));
124 double * read_node_coord_z =
125 xmalloc(read_num_local_nodes *
sizeof(*read_node_coord_z));;
130 ncid, varid, (
size_t[]){0, read_local_start_node},
131 (
size_t[]){1,read_num_local_nodes}, read_node_coord_x));
134 ncid, varid, (
size_t[]){1, read_local_start_node},
135 (
size_t[]){1,read_num_local_nodes}, read_node_coord_y));
138 ncid, varid, (
size_t[]){2, read_local_start_node},
139 (
size_t[]){1,read_num_local_nodes}, read_node_coord_z));
141 xmalloc(read_num_local_nodes *
sizeof(*read_node_coords));
142 for (
size_t i = 0; i < read_num_local_nodes; ++i) {
143 read_node_coords[i][0] = read_node_coord_x[i];
144 read_node_coords[i][1] = read_node_coord_y[i];
145 read_node_coords[i][2] = read_node_coord_z[i];
147 free(read_node_coord_z);
148 free(read_node_coord_y);
149 free(read_node_coord_x);
151 read_dist_elem_to_node =
153 read_num_local_elems * num_nod_per_elem *
154 sizeof(*read_dist_elem_to_node));
158 ncid, varid, (
size_t[]){read_local_start_elem, 0},
159 (
size_t[]){read_num_local_elems, num_nod_per_elem},
160 read_dist_elem_to_node));
161 for (
size_t i = 0; i < read_num_local_elems * num_nod_per_elem; ++i)
162 read_dist_elem_to_node[i]--;
167 read_node_coords =
xmalloc(1 *
sizeof(*read_node_coords));
168 read_dist_elem_to_node =
xmalloc(1 *
sizeof(*read_dist_elem_to_node));
175 if (comm_rank == 0) tmp = num_global_nodes;
177 num_global_nodes = tmp;
178 if (comm_rank == 0) tmp = num_global_elem;
180 num_global_elem = tmp;
181 if (comm_rank == 0) tmp = num_nod_per_elem;
183 num_nod_per_elem = tmp;
187 size_t local_start_elem =
188 ((
unsigned long)num_global_elem * (
unsigned long)comm_rank) /
189 (
unsigned long)comm_size;
190 size_t num_local_elems =
191 ((
unsigned long)num_global_elem * ((
unsigned long)comm_rank+1)) /
192 (
unsigned long)comm_size - (
unsigned long)local_start_elem;
193 size_t local_start_node =
194 ((
unsigned long)num_global_nodes * (
unsigned long)comm_rank) /
195 (
unsigned long)comm_size;
196 size_t num_local_nodes =
197 ((
unsigned long)num_global_nodes * ((
unsigned long)comm_rank+1)) /
198 (
unsigned long)comm_size - (
unsigned long)local_start_node;
201 int * dist_elem_to_node =
202 xmalloc(num_local_elems * num_nod_per_elem *
sizeof(*dist_elem_to_node));
204 int * send_count =
xcalloc(comm_size,
sizeof(*send_count));
205 int * recv_count =
xcalloc(comm_size,
sizeof(*recv_count));
207 for (
size_t i = 0; i < read_num_local_elems; ++i)
210 read_local_start_elem + i, num_global_elem, comm_size)] +=
213 MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, comm);
215 int * send_displ =
xmalloc(comm_size *
sizeof(*send_displ));
216 int * recv_displ =
xmalloc(comm_size *
sizeof(*recv_displ));
217 int send_accum = 0, recv_accum = 0;
218 for (
int i = 0; i < comm_size; ++i) {
219 send_displ[i] = send_accum;
220 recv_displ[i] = recv_accum;
221 send_accum += send_count[i];
222 recv_accum += recv_count[i];
225 MPI_Alltoallv(read_dist_elem_to_node, send_count, send_displ, MPI_INT,
226 dist_elem_to_node, recv_count, recv_displ, MPI_INT, comm);
232 free(read_dist_elem_to_node);
237 xmalloc(num_local_nodes *
sizeof(*dist_node_coords));
239 int * send_count =
xcalloc(comm_size,
sizeof(*send_count));
240 int * recv_count =
xcalloc(comm_size,
sizeof(*recv_count));
242 for (
size_t i = 0; i < read_num_local_nodes; ++i)
245 read_local_start_node + i, num_global_nodes, comm_size)] += 3;
247 MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, comm);
249 int * send_displ =
xmalloc(comm_size *
sizeof(*send_displ));
250 int * recv_displ =
xmalloc(comm_size *
sizeof(*recv_displ));
251 int send_accum = 0, recv_accum = 0;
252 for (
int i = 0; i < comm_size; ++i) {
253 send_displ[i] = send_accum;
254 recv_displ[i] = recv_accum;
255 send_accum += send_count[i];
256 recv_accum += recv_count[i];
259 MPI_Alltoallv(read_node_coords, send_count, send_displ, MPI_DOUBLE,
260 dist_node_coords, recv_count, recv_displ, MPI_DOUBLE, comm);
266 free(read_node_coords);
271 size_t num_core_nodes;
273 size_t N = num_local_elems * num_nod_per_elem;
274 *node_ids =
xmalloc(
N *
sizeof(**node_ids));
275 *num_elem_per_node =
xmalloc(
N *
sizeof(**num_elem_per_node));
276 *elem_to_node =
xmalloc(
N *
sizeof(**elem_to_node));
277 *node_to_elem =
xmalloc(
N *
sizeof(**node_to_elem));
278 for (
size_t i = 0; i <
N; ++i)
279 (*node_ids)[i] = (
yac_int)dist_elem_to_node[i];
280 size_t * permutation = *node_to_elem;
281 for (
size_t i = 0; i <
N; ++i) permutation[i] = i;
284 yac_int prev_node_id = YAC_INT_MAX;
286 for (
size_t i = 0; i <
N; ++i) {
287 yac_int curr_node_id = (*node_ids)[i];
288 if (prev_node_id == curr_node_id) {
289 (*num_elem_per_node)[num_core_nodes-1]++;
291 (*num_elem_per_node)[num_core_nodes] = 1;
292 (*node_ids)[num_core_nodes] = (prev_node_id = curr_node_id);
295 (*elem_to_node)[permutation[i]] = num_core_nodes-1;
296 permutation[i] /= num_nod_per_elem;
299 xrealloc(*node_ids, num_core_nodes *
sizeof(**node_ids));
302 num_core_nodes *
sizeof(**num_elem_per_node));
303 free(dist_elem_to_node);
308 *node_coords =
xmalloc(num_core_nodes *
sizeof(**node_coords));
309 int * send_count =
xcalloc(comm_size,
sizeof(*send_count));
310 int * recv_count =
xcalloc(comm_size,
sizeof(*recv_count));
312 for (
size_t i = 0; i < num_core_nodes; ++i)
315 (*node_ids)[i], num_global_nodes, comm_size)]++;
317 MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, comm);
319 int * send_displ =
xmalloc(comm_size *
sizeof(*send_displ));
320 int * recv_displ =
xmalloc(comm_size *
sizeof(*recv_displ));
321 int send_accum = 0, recv_accum = 0;
322 for (
int i = 0; i < comm_size; ++i) {
323 send_displ[i] = send_accum;
324 recv_displ[i] = recv_accum;
325 send_accum += send_count[i];
326 recv_accum += recv_count[i];
329 int num_all_local_nodes_remote = 0;
330 for (
int i = 0; i < comm_size; ++i)
331 num_all_local_nodes_remote += recv_count[i];
334 xmalloc(num_all_local_nodes_remote *
sizeof(*remote_node_buffer));
337 *node_ids, send_count, send_displ,
yac_int_dt,
338 remote_node_buffer, recv_count, recv_displ,
yac_int_dt, comm);
341 xmalloc(num_all_local_nodes_remote *
sizeof(*send_node_coords));
343 for (
int i = 0, l = 0; i < comm_size; ++i) {
344 for (
int j = 0; j < recv_count[i]; ++j, ++l) {
345 size_t idx = (size_t)(remote_node_buffer[l]) - local_start_node;
346 send_node_coords[l][0] = dist_node_coords[idx][0];
347 send_node_coords[l][1] = dist_node_coords[idx][1];
348 send_node_coords[l][2] = dist_node_coords[idx][2];
356 free(remote_node_buffer);
357 free(dist_node_coords);
359 MPI_Alltoallv(send_node_coords, recv_count, recv_displ, MPI_DOUBLE,
360 *node_coords, send_count, send_displ, MPI_DOUBLE, comm);
362 free(send_node_coords);
370 *elem_ids =
xmalloc(num_local_elems *
sizeof(**elem_ids));
371 for (
size_t i = 0; i < num_local_elems; ++i)
372 (*elem_ids)[i] = (
yac_int)(local_start_elem + i);
375 *num_nodes_per_elem =
376 xmalloc(num_local_elems *
sizeof(**num_nodes_per_elem));
377 for (
size_t i = 0; i < num_local_elems; ++i)
378 (*num_nodes_per_elem)[i] = num_nod_per_elem;
380 *num_elem = num_local_elems;
381 *num_nodes = num_core_nodes;
422 const char * filename,
int use_ll_edges, MPI_Comm comm) {
429 int * num_nodes_per_elem;
430 int * num_elem_per_node;
431 size_t * elem_to_node;
432 size_t * node_to_elem;
435 filename, comm, &node_coords, &elem_ids, &node_ids, &num_elem, &num_nodes,
436 &num_nodes_per_elem, &num_elem_per_node, &elem_to_node, &node_to_elem);
439 size_t * elem_to_edge;
446 size_t max_num_edges = 0;
447 for (
size_t i = 0; i < num_elem; ++i)
448 max_num_edges += num_nodes_per_elem[i];
452 xmalloc(max_num_edges *
sizeof(*temp_edges));
453 for (
size_t i = 0, offset = 0, k = 0; i < num_elem; ++i) {
454 size_t * curr_elem_to_node = elem_to_node + offset;
455 size_t curr_num_edges = num_nodes_per_elem[i];
456 offset += curr_num_edges;
457 for (
size_t j = 0; j < curr_num_edges; ++j, ++k) {
459 curr_elem_to_node[j] > curr_elem_to_node[(j+1)%curr_num_edges];
460 temp_edges[k].
node[order] = curr_elem_to_node[j];
461 temp_edges[k].
node[order^1] = curr_elem_to_node[(j+1)%curr_num_edges];
465 qsort(temp_edges, max_num_edges,
469 elem_to_edge =
xmalloc(max_num_edges *
sizeof(*elem_to_edge));
472 for (
size_t i = 0, prev_indices[2] = {SIZE_MAX, SIZE_MAX};
473 i < max_num_edges; ++i) {
476 if ((prev_indices[0] != temp_edges[i].
node[0]) ||
477 (prev_indices[1] != temp_edges[i].
node[1])) {
479 prev_indices[0] = temp_edges[i].
node[0];
480 prev_indices[1] = temp_edges[i].
node[1];
481 edge_to_node[num_edges][0] = prev_indices[0];
482 edge_to_node[num_edges][1] = prev_indices[1];
486 elem_to_edge[curr_elem_to_edge_idx] = num_edges - 1;
489 xrealloc(edge_to_node, num_edges *
sizeof(*edge_to_node));
491 edge_type =
xmalloc(num_edges *
sizeof(*edge_type));
493 for (
size_t i = 0; i < num_edges; ++i) {
494 double * edge_vertex_a = node_coords[edge_to_node[i][0]];
495 double * edge_vertex_b = node_coords[edge_to_node[i][1]];
496 int vertex_a_is_pole =
check_pole(edge_vertex_a);
497 int vertex_b_is_pole =
check_pole(edge_vertex_b);
499 (vertex_a_is_pole ^ vertex_b_is_pole) ||
500 (!vertex_a_is_pole && !vertex_b_is_pole &&
503 (vertex_a_is_pole && vertex_b_is_pole) ||
506 is_lon_edge || is_lat_edge,
507 "\"use_ll_edges == true\" but edge is neither lon nor lat "
508 "((%lf,%lf,%lf),(%lf,%lf,%lf))",
509 edge_vertex_a[0], edge_vertex_a[1], edge_vertex_a[2],
510 edge_vertex_b[0], edge_vertex_b[1], edge_vertex_b[2]);
514 for (
size_t i = 0; i < num_edges; ++i)
520 grid_data.vertex_coordinates = node_coords;
531 grid_data.num_cells_per_vertex = num_elem_per_node;
541 grid_data.num_total_vertices = num_nodes;