1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
3 * (C) 2014 by Argonne National Laboratory.
4 * See COPYRIGHT in top-level directory.
12 /* tests noncontiguous reads/writes using nonblocking collective I/O */
14 /* this test is almost exactly like i_noncontig_coll.c with the following changes:
16 * . generalized file writing/reading to handle arbitrary number of processors
17 * . provides the "cb_config_list" hint with several permutations of the
18 * available processors.
19 * [ makes use of code copied from ROMIO's ADIO code to collect the names of
23 /* we are going to muck with this later to make it evenly divisible by however many compute nodes we have */
24 #define STARTING_SIZE 5000
26 int test_file(char *filename, int mynod, int nprocs, char *cb_hosts, const char *msg, int verbose);
28 #define ADIOI_Free free
29 #define ADIOI_Malloc malloc
30 #define FPRINTF fprintf
31 /* I have no idea what the "D" stands for; it's how things are done in adio.h
33 struct ADIO_cb_name_arrayD {
38 typedef struct ADIO_cb_name_arrayD *ADIO_cb_name_array;
40 void handle_error(int errcode, const char *str);
41 int cb_gather_name_array(MPI_Comm comm, ADIO_cb_name_array * arrayp);
42 void default_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
43 void reverse_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
44 void reverse_alternating_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
45 void simple_shuffle_str(int mynod, int len, ADIO_cb_name_array array, char *dest);
48 void handle_error(int errcode, const char *str)
50 char msg[MPI_MAX_ERROR_STRING];
52 MPI_Error_string(errcode, msg, &resultlen);
53 fprintf(stderr, "%s: %s\n", str, msg);
54 MPI_Abort(MPI_COMM_WORLD, 1);
58 /* cb_gather_name_array() - gather a list of processor names from all processes
59 * in a communicator and store them on rank 0.
61 * This is a collective call on the communicator(s) passed in.
63 * Obtains a rank-ordered list of processor names from the processes in
66 * Returns 0 on success, -1 on failure.
68 * NOTE: Needs some work to cleanly handle out of memory cases!
70 int cb_gather_name_array(MPI_Comm comm, ADIO_cb_name_array * arrayp)
72 /* this is copied from ROMIO, but since this test is for correctness,
73 * not performance, note that we have removed the parts where ROMIO
74 * uses a keyval to cache the name array. We'll just rebuild it if we
77 char my_procname[MPI_MAX_PROCESSOR_NAME], **procname = 0;
78 int *procname_len = NULL, my_procname_len, *disp = NULL, i;
79 int commsize, commrank;
80 ADIO_cb_name_array array = NULL;
82 MPI_Comm_size(comm, &commsize);
83 MPI_Comm_rank(comm, &commrank);
85 MPI_Get_processor_name(my_procname, &my_procname_len);
87 /* allocate space for everything */
88 array = (ADIO_cb_name_array) malloc(sizeof(*array));
95 /* process 0 keeps the real list */
96 array->namect = commsize;
98 array->names = (char **) ADIOI_Malloc(sizeof(char *) * commsize);
99 if (array->names == NULL) {
102 procname = array->names; /* simpler to read */
104 procname_len = (int *) ADIOI_Malloc(commsize * sizeof(int));
105 if (procname_len == NULL) {
110 /* everyone else just keeps an empty list as a placeholder */
114 /* gather lengths first */
115 MPI_Gather(&my_procname_len, 1, MPI_INT, procname_len, 1, MPI_INT, 0, comm);
118 #ifdef CB_CONFIG_LIST_DEBUG
119 for (i = 0; i < commsize; i++) {
120 FPRINTF(stderr, "len[%d] = %d\n", i, procname_len[i]);
124 for (i = 0; i < commsize; i++) {
125 /* add one to the lengths because we need to count the
126 * terminator, and we are going to use this list of lengths
127 * again in the gatherv.
130 procname[i] = malloc(procname_len[i]);
131 if (procname[i] == NULL) {
136 /* create our list of displacements for the gatherv. we're going
137 * to do everything relative to the start of the region allocated
140 * I suppose it is theoretically possible that the distance between
141 * malloc'd regions could be more than will fit in an int. We don't
144 disp = malloc(commsize * sizeof(int));
146 for (i = 1; i < commsize; i++) {
147 disp[i] = (int) (procname[i] - procname[0]);
152 /* now gather strings */
154 MPI_Gatherv(my_procname, my_procname_len + 1, MPI_CHAR,
155 procname[0], procname_len, disp, MPI_CHAR, 0, comm);
158 /* if we didn't do this, we would need to allocate procname[]
159 * on all processes...which seems a little silly.
161 MPI_Gatherv(my_procname, my_procname_len + 1, MPI_CHAR,
162 NULL, NULL, NULL, MPI_CHAR, 0, comm);
166 /* no longer need the displacements or lengths */
170 #ifdef CB_CONFIG_LIST_DEBUG
171 for (i = 0; i < commsize; i++) {
172 fprintf(stderr, "name[%d] = %s\n", i, procname[i]);
181 void default_str(int mynod, int len, ADIO_cb_name_array array, char *dest)
187 for (i = 0; i < array->namect; i++) {
188 p = snprintf(ptr, len, "%s,", array->names[i]);
191 /* chop off that last comma */
192 dest[strlen(dest) - 1] = '\0';
194 MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
197 void reverse_str(int mynod, int len, ADIO_cb_name_array array, char *dest)
203 for (i = (array->namect - 1); i >= 0; i--) {
204 p = snprintf(ptr, len, "%s,", array->names[i]);
207 dest[strlen(dest) - 1] = '\0';
209 MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
212 void reverse_alternating_str(int mynod, int len, ADIO_cb_name_array array, char *dest)
219 for (i = (array->namect - 1); i >= 0; i -= 2) {
220 p = snprintf(ptr, len, "%s,", array->names[i]);
224 for (i = (array->namect - 2); i > 0; i -= 2) {
225 p = snprintf(ptr, len, "%s,", array->names[i]);
228 dest[strlen(dest) - 1] = '\0';
230 MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
233 void simple_shuffle_str(int mynod, int len, ADIO_cb_name_array array, char *dest)
239 for (i = (array->namect / 2); i < array->namect; i++) {
240 p = snprintf(ptr, len, "%s,", array->names[i]);
243 for (i = 0; i < (array->namect / 2); i++) {
244 p = snprintf(ptr, len, "%s,", array->names[i]);
247 dest[strlen(dest) - 1] = '\0';
249 MPI_Bcast(dest, len, MPI_CHAR, 0, MPI_COMM_WORLD);
252 int main(int argc, char **argv)
254 int i, mynod, nprocs, len, errs = 0, sum_errs = 0, verbose = 0;
256 char *cb_config_string;
258 ADIO_cb_name_array array;
261 MPI_Init(&argc, &argv);
262 MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
263 MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
266 /* process 0 takes the file name as a command-line argument and
267 * broadcasts it to other processes */
269 filename = (char*)"testfile";
270 len = strlen(filename);
271 MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
272 MPI_Bcast(filename, len + 1, MPI_CHAR, 0, MPI_COMM_WORLD);
275 MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
276 filename = (char *) malloc(len + 1);
277 MPI_Bcast(filename, len + 1, MPI_CHAR, 0, MPI_COMM_WORLD);
280 /* want to hint the cb_config_list, but do so in a non-sequential way */
281 cb_gather_name_array(MPI_COMM_WORLD, &array);
285 if (array->namect < 2) {
286 fprintf(stderr, "Run this test on two or more hosts\n");
287 MPI_Abort(MPI_COMM_WORLD, 1);
290 /* get space for the permuted cb_config_string */
293 for (i = 0; i < array->namect; i++) {
294 /* +1: space for either a , or \0 if last */
295 cb_config_len += strlen(array->names[i]) + 1;
299 MPI_Bcast(&cb_config_len, 1, MPI_INT, 0, MPI_COMM_WORLD);
300 if ((cb_config_string = malloc(cb_config_len)) == NULL) {
302 MPI_Abort(MPI_COMM_WORLD, 1);
305 /* first, no hinting */
306 errs += test_file(filename, mynod, nprocs, NULL, "collective w/o hinting", verbose);
308 /* hint, but no change in order */
309 default_str(mynod, cb_config_len, array, cb_config_string);
310 errs += test_file(filename, mynod, nprocs, cb_config_string,
311 "collective w/ hinting: default order", verbose);
314 reverse_str(mynod, cb_config_len, array, cb_config_string);
315 errs += test_file(filename, mynod, nprocs, cb_config_string,
316 "collective w/ hinting: reverse order", verbose);
318 /* reverse, every other */
319 reverse_alternating_str(mynod, cb_config_len, array, cb_config_string);
320 errs += test_file(filename, mynod, nprocs, cb_config_string,
321 "collective w/ hinting: permutation1", verbose);
323 /* second half, first half */
324 simple_shuffle_str(mynod, cb_config_len, array, cb_config_string);
325 errs += test_file(filename, mynod, nprocs, cb_config_string,
326 "collective w/ hinting: permutation2", verbose);
328 MPI_Allreduce(&errs, &sum_errs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
332 fprintf(stderr, "Found %d error cases\n", sum_errs);
334 printf(" No Errors\n");
338 free(cb_config_string);
343 #define SEEDER(x,y,z) ((x)*1000000 + (y) + (x)*(z))
345 int test_file(char *filename, int mynod, int nprocs, char *cb_hosts, const char *msg, int verbose)
347 MPI_Datatype typevec, newtype, t[3];
348 int *buf, i, b[3], errcode, errors = 0;
353 int SIZE = (STARTING_SIZE / nprocs) * nprocs;
356 if (mynod == 0 && verbose)
357 fprintf(stderr, "%s\n", msg);
359 buf = (int *) malloc(SIZE * sizeof(int));
362 MPI_Abort(MPI_COMM_WORLD, -1);
366 if (cb_hosts != NULL) {
367 MPI_Info_create(&info);
368 MPI_Info_set(info, "cb_config_list", cb_hosts);
371 info = MPI_INFO_NULL;
374 MPI_Type_vector(SIZE / nprocs, 1, nprocs, MPI_INT, &typevec);
376 b[0] = b[1] = b[2] = 1;
378 d[1] = mynod * sizeof(int);
379 d[2] = SIZE * sizeof(int);
384 MPI_Type_struct(3, b, d, t, &newtype);
385 MPI_Type_commit(&newtype);
386 MPI_Type_free(&typevec);
390 fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous "
391 "in file using collective I/O\n");
392 MPI_File_delete(filename, info);
394 MPI_Barrier(MPI_COMM_WORLD);
396 errcode = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
397 if (errcode != MPI_SUCCESS) {
398 handle_error(errcode, "MPI_File_open");
401 MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
403 for (i = 0; i < SIZE; i++)
404 buf[i] = SEEDER(mynod, i, SIZE);
405 errcode = MPI_File_iwrite_all(fh, buf, 1, newtype, &request);
406 if (errcode != MPI_SUCCESS) {
407 handle_error(errcode, "nc mem - nc file: MPI_File_iwrite_all");
410 MPI_Barrier(MPI_COMM_WORLD);
411 MPI_Wait(&request, &status);
413 for (i = 0; i < SIZE; i++)
416 errcode = MPI_File_iread_at_all(fh, 0, buf, 1, newtype, &request);
417 if (errcode != MPI_SUCCESS) {
418 handle_error(errcode, "nc mem - nc file: MPI_File_iread_at_all");
420 MPI_Wait(&request, &status);
422 /* the verification for N compute nodes is tricky. Say we have 3
424 * process 0 sees: 0 -1 -1 3 -1 -1 ...
425 * process 1 sees: -1 34 -1 -1 37 -1 ...
426 * process 2 sees: -1 -1 68 -1 -1 71 ... */
428 /* verify those leading -1s exist if they should */
429 for (i = 0; i < mynod; i++) {
432 fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]);
436 /* now the modulo games are hairy. processor 0 sees real data in the 0th,
437 * 3rd, 6th... elements of the buffer (assuming nprocs==3). proc 1 sees
438 * the data in 1st, 4th, 7th..., and proc 2 sees it in 2nd, 5th, 8th */
440 for (/* 'i' set in above loop */ ; i < SIZE; i++) {
441 if (((i - mynod) % nprocs) && buf[i] != -1) {
443 fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]);
446 if (!((i - mynod) % nprocs) && buf[i] != SEEDER(mynod, i, SIZE)) {
448 fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
449 mynod, i, buf[i], SEEDER(mynod, i, SIZE));
455 MPI_Barrier(MPI_COMM_WORLD);
459 fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in "
460 "file using collective I/O\n");
461 MPI_File_delete(filename, info);
463 MPI_Barrier(MPI_COMM_WORLD);
465 MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
467 for (i = 0; i < SIZE; i++)
468 buf[i] = SEEDER(mynod, i, SIZE);
469 errcode = MPI_File_iwrite_at_all(fh, mynod * (SIZE / nprocs) * sizeof(int),
470 buf, 1, newtype, &request);
471 if (errcode != MPI_SUCCESS)
472 handle_error(errcode, "nc mem - c file: MPI_File_iwrite_at_all");
474 MPI_Barrier(MPI_COMM_WORLD);
475 MPI_Wait(&request, &status);
477 for (i = 0; i < SIZE; i++)
480 errcode = MPI_File_iread_at_all(fh, mynod * (SIZE / nprocs) * sizeof(int),
481 buf, 1, newtype, &request);
482 if (errcode != MPI_SUCCESS)
483 handle_error(errcode, "nc mem - c file: MPI_File_iread_at_all");
484 MPI_Wait(&request, &status);
486 /* just like as above */
487 for (i = 0; i < mynod; i++) {
490 fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]);
494 for (/* i set in above loop */ ; i < SIZE; i++) {
495 if (((i - mynod) % nprocs) && buf[i] != -1) {
497 fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]);
500 if (!((i - mynod) % nprocs) && buf[i] != SEEDER(mynod, i, SIZE)) {
502 fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
503 mynod, i, buf[i], SEEDER(mynod, i, SIZE));
510 MPI_Barrier(MPI_COMM_WORLD);
514 fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in "
515 "file using collective I/O\n");
516 MPI_File_delete(filename, info);
518 MPI_Barrier(MPI_COMM_WORLD);
520 MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
522 MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
524 for (i = 0; i < SIZE; i++)
525 buf[i] = SEEDER(mynod, i, SIZE);
526 errcode = MPI_File_iwrite_all(fh, buf, SIZE, MPI_INT, &request);
527 if (errcode != MPI_SUCCESS)
528 handle_error(errcode, "c mem - nc file: MPI_File_iwrite_all");
530 MPI_Barrier(MPI_COMM_WORLD);
531 MPI_Wait(&request, &status);
533 for (i = 0; i < SIZE; i++)
536 errcode = MPI_File_iread_at_all(fh, 0, buf, SIZE, MPI_INT, &request);
537 if (errcode != MPI_SUCCESS)
538 handle_error(errcode, "c mem - nc file: MPI_File_iread_at_all");
539 MPI_Wait(&request, &status);
541 /* same crazy checking */
542 for (i = 0; i < SIZE; i++) {
543 if (buf[i] != SEEDER(mynod, i, SIZE)) {
545 fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
546 mynod, i, buf[i], SEEDER(mynod, i, SIZE));
553 MPI_Type_free(&newtype);
555 if (info != MPI_INFO_NULL)
556 MPI_Info_free(&info);