598 char const * filename,
char const * grid_name,
size_t num_cells,
599 int num_vertices_per_cell,
int cell_center_coords_avaiable,
600 int cell_global_ids_available,
int core_cell_mask_available,
601 int vertex_global_ids_available,
int core_vertex_mask_available,
602 int edge_global_ids_available,
int core_edge_mask_available) {
604 size_t grid_name_len = strlen(grid_name) + 1;
606 char nv_dim_name[3 + grid_name_len];
607 char nc_dim_name[3 + grid_name_len];
609 char cla_var_name[4 + grid_name_len];
610 char clo_var_name[4 + grid_name_len];
611 char lat_var_name[4 + grid_name_len];
612 char lon_var_name[4 + grid_name_len];
613 char rnk_var_name[4 + grid_name_len];
614 char cmk_var_name[4 + grid_name_len];
615 char gid_var_name[4 + grid_name_len];
616 char vcmk_var_name[5 + grid_name_len];
617 char vgid_var_name[5 + grid_name_len];
618 char ecmk_var_name[5 + grid_name_len];
619 char egid_var_name[5 + grid_name_len];
620 char const * unit_att =
"units";
621 char const * coord_unit =
"degree";
623 snprintf(nv_dim_name, 3 + grid_name_len,
"nv_%s", grid_name);
624 snprintf(nc_dim_name, 3 + grid_name_len,
"nc_%s", grid_name);
626 snprintf(cla_var_name, 4 + grid_name_len,
"%s.cla", grid_name);
627 snprintf(clo_var_name, 4 + grid_name_len,
"%s.clo", grid_name);
628 snprintf(lat_var_name, 4 + grid_name_len,
"%s.lat", grid_name);
629 snprintf(lon_var_name, 4 + grid_name_len,
"%s.lon", grid_name);
630 snprintf(rnk_var_name, 4 + grid_name_len,
"%s.rnk", grid_name);
631 snprintf(cmk_var_name, 4 + grid_name_len,
"%s.cmk", grid_name);
632 snprintf(gid_var_name, 4 + grid_name_len,
"%s.gid", grid_name);
633 snprintf(vcmk_var_name, 5 + grid_name_len,
"%s.vcmk", grid_name);
634 snprintf(vgid_var_name, 5 + grid_name_len,
"%s.vgid", grid_name);
635 snprintf(ecmk_var_name, 5 + grid_name_len,
"%s.ecmk", grid_name);
636 snprintf(egid_var_name, 5 + grid_name_len,
"%s.egid", grid_name);
643 yac_nc_create(filename, NC_CLOBBER | NC_64BIT_OFFSET, &ncid);
651 def_dim(filename, ncid, nv_dim_name, (
size_t)num_vertices_per_cell, file_is_new);
653 def_dim(filename, ncid, nc_dim_name, (
size_t)num_cells, file_is_new);
656 int corner_dims[2] = {nc_dim_id, nv_dim_id};
657 int cell_dims[1] = {nc_dim_id};
659 filename, ncid, cla_var_name, NC_DOUBLE, 2, corner_dims,
660 unit_att, coord_unit, file_is_new);
662 filename, ncid, clo_var_name, NC_DOUBLE, 2, corner_dims,
663 unit_att, coord_unit, file_is_new);
664 if (cell_center_coords_avaiable) {
666 filename, ncid, lat_var_name, NC_DOUBLE, 1, cell_dims,
667 unit_att, coord_unit, file_is_new);
669 filename, ncid, lon_var_name, NC_DOUBLE, 1, cell_dims,
670 unit_att, coord_unit, file_is_new);
672 if (cell_global_ids_available)
674 filename, ncid, gid_var_name, NC_INT, 1, cell_dims, NULL, NULL, file_is_new);
675 if (core_cell_mask_available)
677 filename, ncid, cmk_var_name, NC_INT, 1, cell_dims, NULL, NULL, file_is_new);
678 if (vertex_global_ids_available)
680 filename, ncid, vgid_var_name, NC_INT, 2, corner_dims, NULL, NULL, file_is_new);
681 if (core_vertex_mask_available)
683 filename, ncid, vcmk_var_name, NC_INT, 2, corner_dims, NULL, NULL, file_is_new);
684 if (edge_global_ids_available)
686 filename, ncid, egid_var_name, NC_INT, 2, corner_dims, NULL, NULL, file_is_new);
687 if (core_edge_mask_available)
689 filename, ncid, ecmk_var_name, NC_INT, 2, corner_dims, NULL, NULL, file_is_new);
691 filename, ncid, rnk_var_name, NC_INT, 1, cell_dims, NULL, NULL, file_is_new);
693 time_t now = time(NULL);
694 char str_now_UTC[32];
696 str_now_UTC,
sizeof(str_now_UTC),
"%Y-%m-%dT%H:%M:%SZ",
699 char const * created_by =
"Created by YAC";
704 nc_put_att_text(ncid, NC_GLOBAL,
"title", strlen(created_by), created_by));
706 nc_put_att_text(ncid, NC_GLOBAL,
"description", strlen(created_by), created_by));
710 nc_put_att_text(ncid, NC_GLOBAL,
"timeStamp", strlen(str_now_UTC), str_now_UTC));
743 struct yac_basic_grid * grid,
char const * filename, MPI_Comm comm) {
745#ifndef YAC_NETCDF_ENABLED
752 "ERROR(yac_basic_grid_to_file_parallel): "
753 "YAC is built without the NetCDF support");
756 int comm_rank, comm_size;
761 int filename_len = (int)strlen(filename) + 1;
762 yac_mpi_call(MPI_Bcast(&filename_len, 1, MPI_INT, 0, comm), comm);
763 char * filename_buffer =
764 xmalloc((
size_t)filename_len *
sizeof(*filename_buffer));
765 if (comm_rank == 0) strcpy(filename_buffer, filename);
767 MPI_Bcast(filename_buffer, filename_len, MPI_CHAR, 0, comm), comm);
770 !strcmp(filename, filename_buffer),
771 "ERROR(yac_basic_grid_to_file_parallel): "
772 "inconsistent filename (\"%s\" on rank 0 != \"%s\" on rank %d)",
773 filename_buffer, filename, comm_rank);
774 free(filename_buffer);
780 ((cell_field != NULL) &&
786 uint64_t * global_cell_counts =
787 xmalloc((
size_t)comm_size *
sizeof(*global_cell_counts));
788 int max_num_vertices_per_cell = 0;
789 int cell_center_coords_avaiable =
790 (grid_data->
num_cells == 0) || (cell_center_coords != NULL);
791 int cell_global_ids_available =
793 int core_cell_mask_available =
795 int vertex_global_ids_available =
797 int core_vertex_mask_available =
799 int edge_global_ids_available =
801 int core_edge_mask_available =
810 &local_cell_count, 1, MPI_UINT64_T,
811 global_cell_counts, 1, MPI_UINT64_T, comm), comm);
814 MPI_IN_PLACE, &max_num_vertices_per_cell, 1, MPI_INT, MPI_MAX, comm),
816 int ints[7] = {cell_center_coords_avaiable,
817 cell_global_ids_available,
818 core_cell_mask_available,
819 vertex_global_ids_available,
820 core_vertex_mask_available,
821 edge_global_ids_available,
822 core_edge_mask_available};
825 MPI_IN_PLACE, ints,
sizeof(ints)/
sizeof(ints[0]),
826 MPI_INT, MPI_MIN, comm), comm);
827 cell_center_coords_avaiable = ints[0];
828 cell_global_ids_available = ints[1];
829 core_cell_mask_available = ints[2];
830 vertex_global_ids_available = ints[3];
831 core_vertex_mask_available = ints[4];
832 edge_global_ids_available = ints[5];
833 core_edge_mask_available = ints[6];
835 size_t global_cell_count = 0;
836 for (
int i = 0; i < comm_size; ++i)
837 global_cell_count += (
size_t)(global_cell_counts[i]);
840 global_cell_count > 0,
841 "ERROR(yac_basic_grid_to_file_parallel): grid \"%s\" has no cells",
847 filename, grid->
name, global_cell_count, max_num_vertices_per_cell,
848 cell_center_coords_avaiable,
849 cell_global_ids_available, core_cell_mask_available,
850 vertex_global_ids_available, core_vertex_mask_available,
851 edge_global_ids_available, core_edge_mask_available);
860 size_t recv_count = 0;
861 size_t io_start_idx = SIZE_MAX;
866 for (io_rank_idx = 0; io_rank_idx < num_io_ranks; ++io_rank_idx)
867 if (io_ranks[io_rank_idx] == comm_rank)
break;
871 ((
long long)(io_rank_idx) * (
long long)global_cell_count)/
872 (
long long)num_io_ranks);
875 ((
long long)(io_rank_idx + 1) * (
long long)global_cell_count)/
876 (
long long)num_io_ranks - (
long long)io_start_idx);
880 struct Xt_com_pos * com_pos_buffer =
881 xmalloc(5 * (
size_t)comm_size *
sizeof(*com_pos_buffer));
882 struct Xt_com_pos * cell_src_com = com_pos_buffer;
883 struct Xt_com_pos * cell_dst_com = com_pos_buffer + (size_t)comm_size;
884 struct Xt_com_pos * vertex_src_com = com_pos_buffer + 2 * (size_t)comm_size;
885 struct Xt_com_pos * vertex_dst_com = com_pos_buffer + 3 * (size_t)comm_size;
886 struct Xt_com_pos * edge_src_com = com_pos_buffer + 4 * (size_t)comm_size;
887 struct Xt_com_pos * edge_dst_com = vertex_dst_com;
888 int num_src_msg = 0, num_dst_msg = 0;
889 int * transfer_pos_buffer =
891 (
size_t)(2 * max_num_vertices_per_cell + 1) *
892 (grid->
data.
num_cells + recv_count) *
sizeof(*transfer_pos_buffer));
893 int * cell_send_pos = transfer_pos_buffer;
894 int * cell_recv_pos = transfer_pos_buffer + grid->
data.
num_cells;
895 int * vertex_send_pos =
897 int * vertex_recv_pos =
900 int * edge_send_pos =
903 (
size_t)max_num_vertices_per_cell * recv_count;
907 size_t * cell_to_vertex =
909 size_t * cell_to_edge =
913 cell_send_pos[i] = (int)i;
914 for (
int j = 0; j < num_vertices; ++j)
915 vertex_send_pos[i * (
size_t)max_num_vertices_per_cell + (size_t)j] =
917 for (
int j = num_vertices; j < max_num_vertices_per_cell; ++j)
918 vertex_send_pos[i * (
size_t)max_num_vertices_per_cell + (size_t)j] = 0;
919 for (
int j = 0; j < num_vertices; ++j)
920 edge_send_pos[i * (
size_t)max_num_vertices_per_cell + (size_t)j] =
922 for (
int j = num_vertices; j < max_num_vertices_per_cell; ++j)
923 edge_send_pos[i * (
size_t)max_num_vertices_per_cell + (size_t)j] = 0;
926 for (
size_t i = 0, k = 0; i < recv_count; ++i) {
927 cell_recv_pos[i] = (int)i;
928 for (
int j = 0; j < max_num_vertices_per_cell; ++j, ++k)
929 vertex_recv_pos[k] = k;
932 size_t io_end_idx = 0;
933 size_t global_count_accu = 0;
934 for (
int io_rank_idx = 0, curr_rank = 0; io_rank_idx < num_io_ranks;
937 int curr_io_rank = io_ranks[io_rank_idx];
938 size_t io_start_idx = io_end_idx;
941 ((
long long)(io_rank_idx + 1) * (
long long)global_cell_count)/
942 (
long long)num_io_ranks);
943 size_t io_size = io_end_idx - io_start_idx;
944 size_t curr_recv_size = 0;
946 while(global_count_accu < io_end_idx) {
950 (global_count_accu + global_cell_counts[curr_rank]) -
951 (io_start_idx + curr_recv_size), io_size - curr_recv_size);
953 if (curr_rank == comm_rank) {
954 cell_src_com[num_src_msg].transfer_pos = cell_send_pos;
955 cell_src_com[num_src_msg].num_transfer_pos = (int)count;
956 cell_src_com[num_src_msg].rank = curr_io_rank;
957 cell_send_pos += count;
959 vertex_src_com[num_src_msg].transfer_pos = vertex_send_pos;
960 vertex_src_com[num_src_msg].num_transfer_pos =
961 (int)count * max_num_vertices_per_cell;
962 vertex_src_com[num_src_msg].rank = curr_io_rank;
964 count * (size_t)max_num_vertices_per_cell;
966 edge_src_com[num_src_msg].transfer_pos = edge_send_pos;
967 edge_src_com[num_src_msg].num_transfer_pos =
968 (int)count * max_num_vertices_per_cell;
969 edge_src_com[num_src_msg].rank = curr_io_rank;
971 count * (size_t)max_num_vertices_per_cell;
976 if (curr_io_rank == comm_rank) {
977 cell_dst_com[num_dst_msg].transfer_pos = cell_recv_pos;
978 cell_dst_com[num_dst_msg].num_transfer_pos = (int)count;
979 cell_dst_com[num_dst_msg].rank = curr_rank;
980 cell_recv_pos += count;
982 vertex_dst_com[num_dst_msg].transfer_pos = vertex_recv_pos;
983 vertex_dst_com[num_dst_msg].num_transfer_pos =
984 (int)count * max_num_vertices_per_cell;
985 vertex_dst_com[num_dst_msg].rank = curr_rank;
986 vertex_recv_pos += count * (size_t)max_num_vertices_per_cell;
991 if ((global_count_accu + global_cell_counts[curr_rank]) <=
993 global_count_accu += global_cell_counts[curr_rank];
997 curr_recv_size += count;
999 if (curr_recv_size >= io_size)
break;
1003 free(global_cell_counts);
1007 size_t start[2], count[2];
1011 for (io_rank_idx = 0; io_rank_idx < num_io_ranks; ++io_rank_idx)
1012 if (io_ranks[io_rank_idx] == comm_rank)
break;
1014 start[0] = io_start_idx;
1016 count[0] = recv_count;
1017 count[1] = max_num_vertices_per_cell;
1019 yac_nc_open(filename, NC_WRITE | NC_SHARE, &ncid);
1025 void * recv_buffer =
1027 (
size_t)max_num_vertices_per_cell * recv_count *
1035 xt_xmap_intersection_pos_new(
1036 num_src_msg, cell_src_com, num_dst_msg, cell_dst_com, comm);
1038 Xt_redist cell_redist_int = xt_redist_p2p_new(cell_xmap, MPI_INT);
1041 int * num_vertices_per_cell =
1042 xmalloc(recv_count *
sizeof(*num_vertices_per_cell));
1043 xt_redist_s_exchange1(
1047 if (cell_center_coords_avaiable) {
1050 double * send_buffer_dble =
1052 double * send_buffer_lon = send_buffer_dble;
1053 double * send_buffer_lat = send_buffer_dble + grid->
data.
num_cells;
1056 cell_center_coords[i], send_buffer_lon + i, send_buffer_lat + i);
1057 send_buffer_lon[i] /=
YAC_RAD;
1058 send_buffer_lat[i] /=
YAC_RAD;
1062 Xt_redist cell_redist_dble = xt_redist_p2p_new(cell_xmap, MPI_DOUBLE);
1063 xt_redist_s_exchange1(cell_redist_dble, send_buffer_lon, recv_buffer);
1064 put_vara(ncid, grid->
name,
"lon", start, count, recv_buffer);
1065 xt_redist_s_exchange1(cell_redist_dble, send_buffer_lat, recv_buffer);
1066 put_vara(ncid, grid->
name,
"lat", start, count, recv_buffer);
1067 xt_redist_delete(cell_redist_dble);
1068 free(send_buffer_dble);
1071 int * cell_send_buffer_int =
1078 cell_send_buffer_int[i] = comm_rank;
1081 xt_redist_s_exchange1(cell_redist_int, cell_send_buffer_int, recv_buffer);
1082 put_vara(ncid, grid->
name,
"rnk", start, count, recv_buffer);
1086 if (core_cell_mask_available) {
1089 xt_redist_s_exchange1(
1091 put_vara(ncid, grid->
name,
"cmk", start, count, recv_buffer);
1095 if (cell_global_ids_available) {
1099 cell_send_buffer_int[i] = (
int)(grid->
data.
cell_ids[i]);
1102 xt_redist_s_exchange1(
1103 cell_redist_int, cell_send_buffer_int, recv_buffer);
1104 put_vara(ncid, grid->
name,
"gid", start, count, recv_buffer);
1107 xt_redist_delete(cell_redist_int);
1108 free(cell_send_buffer_int);
1110 xt_xmap_delete(cell_xmap);
1116 Xt_xmap vertex_xmap =
1117 xt_xmap_intersection_pos_new(
1118 num_src_msg, vertex_src_com, num_dst_msg, vertex_dst_com, comm);
1123 double * send_buffer_dble =
1125 double * send_buffer_lon = send_buffer_dble;
1130 send_buffer_lon + i, send_buffer_lat + i);
1131 send_buffer_lon[i] /=
YAC_RAD;
1132 send_buffer_lat[i] /=
YAC_RAD;
1136 Xt_redist vertex_redist_dble = xt_redist_p2p_new(vertex_xmap, MPI_DOUBLE);
1137 xt_redist_s_exchange1(vertex_redist_dble, send_buffer_lon, recv_buffer);
1138 for (
size_t i = 0, k = 0; i < recv_count;
1139 ++i, k += (size_t)max_num_vertices_per_cell)
1140 for (
size_t j = (
size_t)num_vertices_per_cell[i];
1141 j < (size_t)max_num_vertices_per_cell; ++j)
1142 ((
double*)recv_buffer)[k+j] = DBL_MAX;
1143 put_vara(ncid, grid->
name,
"clo", start, count, recv_buffer);
1144 xt_redist_s_exchange1(vertex_redist_dble, send_buffer_lat, recv_buffer);
1145 free(send_buffer_dble);
1146 for (
size_t i = 0, k = 0; i < recv_count;
1147 ++i, k += (size_t)max_num_vertices_per_cell)
1148 for (
size_t j = (
size_t)num_vertices_per_cell[i];
1149 j < (size_t)max_num_vertices_per_cell; ++j)
1150 ((
double*)recv_buffer)[k+j] = DBL_MAX;
1151 put_vara(ncid, grid->
name,
"cla", start, count, recv_buffer);
1152 xt_redist_delete(vertex_redist_dble);
1155 Xt_redist vertex_redist_int = xt_redist_p2p_new(vertex_xmap, MPI_INT);
1156 int * vertex_send_buffer_int =
1160 if (core_vertex_mask_available) {
1163 xt_redist_s_exchange1(
1165 for (
size_t i = 0, k = 0; i < recv_count;
1166 ++i, k += (size_t)max_num_vertices_per_cell)
1167 for (
size_t j = (
size_t)num_vertices_per_cell[i];
1168 j < (size_t)max_num_vertices_per_cell; ++j)
1169 ((
int*)recv_buffer)[k+j] = INT_MAX;
1170 put_vara(ncid, grid->
name,
"vcmk", start, count, recv_buffer);
1174 if (vertex_global_ids_available) {
1181 xt_redist_s_exchange1(
1182 vertex_redist_int, vertex_send_buffer_int, recv_buffer);
1183 for (
size_t i = 0, k = 0; i < recv_count;
1184 ++i, k += (size_t)max_num_vertices_per_cell)
1185 for (
size_t j = (
size_t)num_vertices_per_cell[i];
1186 j < (size_t)max_num_vertices_per_cell; ++j)
1187 ((
int*)recv_buffer)[k+j] = INT_MAX;
1188 put_vara(ncid, grid->
name,
"vgid", start, count, recv_buffer);
1191 free(vertex_send_buffer_int);
1192 xt_redist_delete(vertex_redist_int);
1194 xt_xmap_delete(vertex_xmap);
1200 if (core_edge_mask_available || edge_global_ids_available) {
1203 xt_xmap_intersection_pos_new(
1204 num_src_msg, edge_src_com, num_dst_msg, edge_dst_com, comm);
1206 Xt_redist edge_redist_int = xt_redist_p2p_new(edge_xmap, MPI_INT);
1207 int * edge_send_buffer_int =
1211 if (core_edge_mask_available) {
1214 xt_redist_s_exchange1(
1216 for (
size_t i = 0, k = 0; i < recv_count;
1217 ++i, k += (size_t)max_num_vertices_per_cell)
1218 for (
size_t j = (
size_t)num_vertices_per_cell[i];
1219 j < (size_t)max_num_vertices_per_cell; ++j)
1220 ((
int*)recv_buffer)[k+j] = INT_MAX;
1221 put_vara(ncid, grid->
name,
"ecmk", start, count, recv_buffer);
1225 if (edge_global_ids_available) {
1229 edge_send_buffer_int[i] = (
int)(grid->
data.
edge_ids[i]);
1232 xt_redist_s_exchange1(
1233 edge_redist_int, edge_send_buffer_int, recv_buffer);
1234 for (
size_t i = 0, k = 0; i < recv_count;
1235 ++i, k += (size_t)max_num_vertices_per_cell)
1236 for (
size_t j = (
size_t)num_vertices_per_cell[i];
1237 j < (size_t)max_num_vertices_per_cell; ++j)
1238 ((
int*)recv_buffer)[k+j] = INT_MAX;
1239 put_vara(ncid, grid->
name,
"egid", start, count, recv_buffer);
1242 free(edge_send_buffer_int);
1243 xt_redist_delete(edge_redist_int);
1245 xt_xmap_delete(edge_xmap);
1251 free(num_vertices_per_cell);
1253 free(com_pos_buffer);
1254 free(transfer_pos_buffer);