1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
3 * (C) 2014 by Argonne National Laboratory.
4 * See COPYRIGHT in top-level directory.
7 /* Wei-keng Liao (wkliao@ece.northwestern.edu) September 8, 2008 */
9 /* Uses nonblocking collective I/O.*/
20 /* rjl: I was just too lazy to compute this at run-time */
21 char compare_buf[XLEN * 4][YLEN * 4] = {
22 {'0', '1', '2', 0, 0, '3', '4', '5', 0, 0, 'D', 'E', 'F', 0, 0, 'G', 'H', 'I'},
23 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
24 {'6', '7', '8', 0, 0, '9', ':', ';', 0, 0, 'J', 'K', 'L', 0, 0, 'M', 'N', 'O'},
25 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
26 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
27 {'X', 'Y', 'Z', 0, 0, '[', '\\', ']', 0, 0, 'l', 'm', 'n', 0, 0, 'o', 'p', 'q'},
28 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
29 {'^', '_', '`', 0, 0, 'a', 'b', 'c', 0, 0, 'r', 's', 't', 0, 0, 'u', 'v', 'w'},
30 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
31 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
32 {'0', '1', '2', 0, 0, '3', '4', '5', 0, 0, 'D', 'E', 'F', 0, 0, 'G', 'H', 'I'},
33 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
34 {'6', '7', '8', 0, 0, '9', ':', ';', 0, 0, 'J', 'K', 'L', 0, 0, 'M', 'N', 'O'},
35 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
36 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
37 {'X', 'Y', 'Z', 0, 0, '[', '\\', ']', 0, 0, 'l', 'm', 'n', 0, 0, 'o', 'p', 'q'},
38 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
39 {'^', '_', '`', 0, 0, 'a', 'b', 'c', 0, 0, 'r', 's', 't', 0, 0, 'u', 'v', 'w'},
40 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
41 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
45 /* set this if you want a dump of the global array
49 /*----< main() >------------------------------------------------------------*/
50 int main(int argc, char **argv)
52 int i, j, err, rank, np, num_io;
54 int rank_dim[2], array_of_sizes[2];
55 int array_of_subsizes[2];
56 int count, *blocklengths, global_array_size;
58 MPI_Aint *displacements;
64 MPI_Offset offset = 0;
70 MPI_Init(&argc, &argv);
71 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
72 MPI_Comm_size(MPI_COMM_WORLD, &np);
76 printf("Please run with 4 processes. Exiting ...\n\n");
81 filename = (argc > 1) ? argv[1] : (char*)"testfile";
85 request = (MPI_Request *) malloc(num_io * sizeof(MPI_Request));
86 statuses = (MPI_Status *) malloc(num_io * sizeof(MPI_Status));
88 /*-----------------------------------------------------------------------*/
89 /* process rank in each dimension */
90 rank_dim[0] = rank / 2;
91 rank_dim[1] = rank % 2;
93 /* global 2D array size */
94 array_of_sizes[0] = YLEN * 2;
95 array_of_sizes[1] = XLEN * 2;
97 global_array_size = array_of_sizes[0] * array_of_sizes[1];
99 array_of_subsizes[0] = YLEN / 2;
100 array_of_subsizes[1] = XLEN * SUB_XLEN / 5;
102 offset = rank_dim[0] * YLEN * array_of_sizes[1] + rank_dim[1] * XLEN;
104 /* define data type for file view */
105 count = array_of_subsizes[0] * 2; /* 2 is the no. blocks along X */
106 blocklengths = (int *) malloc(count * sizeof(int));
107 displacements = (MPI_Aint *) malloc(count * sizeof(MPI_Aint));
108 for (i = 0; i < count; i++)
109 blocklengths[i] = array_of_subsizes[1] / 2;
110 for (i = 0; i < array_of_subsizes[0]; i++)
111 for (j = 0; j < 2; j++)
112 displacements[i * 2 + j] = offset + i * 2 * array_of_sizes[1]
114 MPI_Type_create_hindexed(count, blocklengths, displacements, MPI_CHAR, &ftype);
115 MPI_Type_commit(&ftype);
116 MPI_Type_size_x(ftype, &ftype_size);
118 /* subarray's layout in the global array
120 P0's 's layout P1's layout
121 [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9] | [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9]
122 [ 0] 0 1 2 3 4 5 | D E F G H I
124 [ 2] 6 7 8 9 : ; | J K L M N O
133 P2's 's layout P3's layout
134 [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9] | [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9]
140 [ 5] X Y Z [ \ ] | l m n o p q
142 [ 7] ^ _ ` a b c | r s t u v w
147 /* initialize the write buffer */
148 buf = (char *) malloc(array_of_subsizes[0] * array_of_subsizes[1]);
149 for (i = 0; i < array_of_subsizes[0] * array_of_subsizes[1]; i++)
150 buf[i] = '0' + rank * 20 + i % 79;
152 /* zero file contents --------------------------------------------------- */
154 char *wr_buf = (char *) calloc(num_io * global_array_size, 1);
155 MPI_File_open(MPI_COMM_SELF, filename,
156 MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
157 MPI_File_write(fh, wr_buf, num_io * global_array_size, MPI_CHAR, &status);
161 /* open the file -------------------------------------------------------- */
162 err = MPI_File_open(MPI_COMM_WORLD, filename,
163 MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
164 if (err != MPI_SUCCESS) {
165 printf("Error: MPI_File_open() filename %s\n", filename);
166 MPI_Abort(MPI_COMM_WORLD, -1);
170 /* MPI nonblocking collective write */
171 for (i = 0; i < num_io; i++) {
172 offset = i * global_array_size;
173 /* set the file view */
174 MPI_File_set_view(fh, offset, MPI_BYTE, ftype, "native", MPI_INFO_NULL);
175 MPI_File_iwrite_all(fh, buf, ftype_size, MPI_CHAR, &request[i]);
177 MPI_Waitall(num_io, request, statuses);
180 /* read and print file contents ----------------------------------------- */
183 char *rd_buf = (char *) calloc(num_io * global_array_size, 1);
184 MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
185 MPI_File_read(fh, rd_buf, num_io * global_array_size, MPI_CHAR, &status);
189 printf("-------------------------------------------------------\n");
191 for (i = 0; i < 2; i++) {
192 for (j = 0; j < XLEN; j++)
200 for (k = 0; k < num_io; k++) {
201 for (i = 0; i < 2 * YLEN; i++) {
202 printf("[%2d]", k * 2 * YLEN + i);
203 for (j = 0; j < 2 * XLEN; j++) {
204 if (j > 0 && j % XLEN == 0)
218 for (i = 0; i < 2 * YLEN * num_io; i++) {
219 for (j = 0; j < 2 * XLEN; j++) {
220 if (*ptr != compare_buf[i][j]) {
221 fprintf(stderr, "expected %d got %d at [%d][%d]\n",
222 *ptr, compare_buf[i][j], i, j);
231 fprintf(stdout, " No Errors\n");
233 fprintf(stderr, "Found %d errors\n", nr_errors);
241 MPI_Type_free(&ftype);
246 /* command-line outputs are: (the global array is written twice)
248 % mpiexec -n 4 wkl_subarray
249 -------------------------------------------------------
250 [ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 ]
252 [ 0] 0 1 2 3 4 5 D E F G H I
254 [ 2] 6 7 8 9 : ; J K L M N O
257 [ 5] X Y Z [ \ ] l m n o p q
259 [ 7] ^ _ ` a b c r s t u v w
263 [10] 0 1 2 3 4 5 D E F G H I
265 [12] 6 7 8 9 : ; J K L M N O
268 [15] X Y Z [ \ ] l m n o p q
270 [17] ^ _ ` a b c r s t u v w