1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
3 * (C) 2011 by Argonne National Laboratory.
4 * See COPYRIGHT in top-level directory.
7 /* This test attempts to execute multiple simultaneous nonblocking collective
8 * (NBC) MPI routines at the same time, and manages their completion with a
9 * variety of routines (MPI_{Wait,Test}{,_all,_any,_some}). It also throws a
10 * few point-to-point operations into the mix.
12 * Possible improvements:
13 * - post operations on multiple comms from multiple threads
28 /* Constants that control the high level test harness behavior. */
29 /* MAIN_ITERATIONS is how many NBC ops the test will attempt to issue. */
30 #define MAIN_ITERATIONS (100000)
31 /* WINDOW is the maximum number of outstanding NBC requests at any given time */
33 /* we sleep with probability 1/CHANCE_OF_SLEEP */
34 #define CHANCE_OF_SLEEP (1000)
35 /* JITTER_DELAY is denominated in microseconds (us) */
36 #define JITTER_DELAY (50000) /* 0.05 seconds */
37 /* NUM_COMMS is the number of communicators on which ops will be posted */
40 /* Constants that control behavior of the individual testing operations.
41 * Altering these can help to explore the testing space, but increasing them too
42 * much can consume too much memory (often O(n^2) usage). */
43 /* FIXME is COUNT==10 too limiting? should we try a larger count too (~500)? */
47 #define my_assert(cond_) \
52 fprintf(stderr, "assertion (%s) failed on line %d\n", #cond_, __LINE__); \
57 /* Intended to act like "rand_r", but we can be sure that it will exist and be
58 * consistent across all of comm world. Returns a number in the range
60 #define GEN_PRN_MAX (4294967291-1)
61 static unsigned int gen_prn(unsigned int x)
63 /* a simple "multiplicative congruential method" PRNG, with parameters:
64 * m=4294967291, largest 32-bit prime
65 * a=279470273, good primitive root of m from "TABLES OF LINEAR
66 * CONGRUENTIAL GENERATORS OF DIFFERENT SIZES AND GOOD
67 * LATTICE STRUCTURE", by Pierre L’Ecuyer */
68 return (279470273UL * (unsigned long)x) % 4294967291UL;
71 /* given a random unsigned int value "rndval_" from gen_prn, this evaluates to a
72 * value in the range [min_,max_) */
73 #define rand_range(rndval_,min_,max_) \
74 ((unsigned int)((min_) + ((rndval_) * (1.0 / (GEN_PRN_MAX+1.0)) * ((max_) - (min_)))))
77 static void sum_fn(void *invec, void *inoutvec, int *len, MPI_Datatype *datatype)
81 int *inout = inoutvec;
82 for (i = 0; i < *len; ++i) {
83 inout[i] = in[i] + inout[i];
87 /* used to keep track of buffers that should be freed after the corresponding
88 * operation has completed */
90 int case_num; /* which test case initiated this req/laundry */
102 static void cleanup_laundry(struct laundry *l)
105 l->comm = MPI_COMM_NULL;
106 if (l->buf) free(l->buf);
107 if (l->recvbuf) free(l->recvbuf);
108 if (l->sendcounts) free(l->sendcounts);
109 if (l->recvcounts) free(l->recvcounts);
110 if (l->sdispls) free(l->sdispls);
111 if (l->rdispls) free(l->rdispls);
112 if (l->sendtypes) free(l->sendtypes);
113 if (l->recvtypes) free(l->recvtypes);
116 /* Starts a "random" operation on "comm" corresponding to "rndnum" and returns
117 * in (*req) a request handle corresonding to that operation. This call should
118 * be considered collective over comm (with a consistent value for "rndnum"),
119 * even though the operation may only be a point-to-point request. */
120 static void start_random_nonblocking(MPI_Comm comm, unsigned int rndnum, MPI_Request *req, struct laundry *l)
126 int *sendcounts = NULL;
127 int *recvcounts = NULL;
130 int *sendtypes = NULL;
131 int *recvtypes = NULL;
132 signed char *buf_alias = NULL;
134 MPI_Comm_rank(comm, &rank);
135 MPI_Comm_size(comm, &size);
137 *req = MPI_REQUEST_NULL;
142 l->buf = buf = malloc(COUNT*size*sizeof(int));
143 l->recvbuf = recvbuf = malloc(COUNT*size*sizeof(int));
144 l->sendcounts = sendcounts = malloc(size*sizeof(int));
145 l->recvcounts = recvcounts = malloc(size*sizeof(int));
146 l->sdispls = sdispls = malloc(size*sizeof(int));
147 l->rdispls = rdispls = malloc(size*sizeof(int));
148 l->sendtypes = sendtypes = malloc(size*sizeof(MPI_Datatype));
149 l->recvtypes = recvtypes = malloc(size*sizeof(MPI_Datatype));
151 #define NUM_CASES (21)
152 l->case_num = rand_range(rndnum, 0, NUM_CASES);
153 switch (l->case_num) {
154 case 0: /* MPI_Ibcast */
155 for (i = 0; i < COUNT; ++i) {
163 MPI_Ibcast(buf, COUNT, MPI_INT, 0, comm, req);
166 case 1: /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
167 /* FIXME fiddle with PRIME and buffer allocation s.t. PRIME is much larger (1021?) */
168 buf_alias = (signed char *)buf;
169 my_assert(COUNT*size*sizeof(int) > PRIME); /* sanity */
170 for (i = 0; i < PRIME; ++i) {
176 for (i = PRIME; i < COUNT * size * sizeof(int); ++i) {
179 MPI_Ibcast(buf_alias, PRIME, MPI_SIGNED_CHAR, 0, comm, req);
182 case 2: /* MPI_Ibarrier */
183 MPI_Ibarrier(comm, req);
186 case 3: /* MPI_Ireduce */
187 for (i = 0; i < COUNT; ++i) {
189 recvbuf[i] = 0xdeadbeef;
191 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, 0, comm, req);
194 case 4: /* same again, use a user op and free it before the wait */
196 MPI_Op op = MPI_OP_NULL;
197 MPI_Op_create(sum_fn, /*commute=*/1, &op);
198 for (i = 0; i < COUNT; ++i) {
200 recvbuf[i] = 0xdeadbeef;
202 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, op, 0, comm, req);
207 case 5: /* MPI_Iallreduce */
208 for (i = 0; i < COUNT; ++i) {
210 recvbuf[i] = 0xdeadbeef;
212 MPI_Iallreduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
215 case 6: /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
216 for (i = 0; i < size; ++i) {
217 sendcounts[i] = COUNT;
218 recvcounts[i] = COUNT;
219 sdispls[i] = COUNT * i;
220 rdispls[i] = COUNT * i;
221 for (j = 0; j < COUNT; ++j) {
222 buf[i*COUNT+j] = rank + (i * j);
223 recvbuf[i*COUNT+j] = 0xdeadbeef;
226 MPI_Ialltoallv(buf, sendcounts, sdispls, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, comm, req);
229 case 7: /* MPI_Igather */
230 for (i = 0; i < size*COUNT; ++i) {
232 recvbuf[i] = 0xdeadbeef;
234 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);
237 case 8: /* same test again, just use a dup'ed datatype and free it before the wait */
239 MPI_Datatype type = MPI_DATATYPE_NULL;
240 MPI_Type_dup(MPI_INT, &type);
241 for (i = 0; i < size*COUNT; ++i) {
243 recvbuf[i] = 0xdeadbeef;
245 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, type, 0, comm, req);
246 MPI_Type_free(&type); /* should cause implementations that don't refcount
247 correctly to blow up or hang in the wait */
251 case 9: /* MPI_Iscatter */
252 for (i = 0; i < size; ++i) {
253 for (j = 0; j < COUNT; ++j) {
255 buf[i*COUNT+j] = i + j;
257 buf[i*COUNT+j] = 0xdeadbeef;
258 recvbuf[i*COUNT+j] = 0xdeadbeef;
261 MPI_Iscatter(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);
264 case 10: /* MPI_Iscatterv */
265 for (i = 0; i < size; ++i) {
266 /* weak test, just test the regular case where all counts are equal */
267 sendcounts[i] = COUNT;
268 sdispls[i] = i * COUNT;
269 for (j = 0; j < COUNT; ++j) {
271 buf[i*COUNT+j] = i + j;
273 buf[i*COUNT+j] = 0xdeadbeef;
274 recvbuf[i*COUNT+j] = 0xdeadbeef;
277 MPI_Iscatterv(buf, sendcounts, sdispls, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);
280 case 11: /* MPI_Ireduce_scatter */
281 for (i = 0; i < size; ++i) {
282 recvcounts[i] = COUNT;
283 for (j = 0; j < COUNT; ++j) {
284 buf[i*COUNT+j] = rank + i;
285 recvbuf[i*COUNT+j] = 0xdeadbeef;
288 MPI_Ireduce_scatter(buf, recvbuf, recvcounts, MPI_INT, MPI_SUM, comm, req);
291 case 12: /* MPI_Ireduce_scatter_block */
292 for (i = 0; i < size; ++i) {
293 for (j = 0; j < COUNT; ++j) {
294 buf[i*COUNT+j] = rank + i;
295 recvbuf[i*COUNT+j] = 0xdeadbeef;
298 MPI_Ireduce_scatter_block(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
301 case 13: /* MPI_Igatherv */
302 for (i = 0; i < size*COUNT; ++i) {
304 recvbuf[i] = 0xdeadbeef;
306 for (i = 0; i < COUNT; ++i) {
309 for (i = 0; i < size; ++i) {
310 recvcounts[i] = COUNT;
311 rdispls[i] = i * COUNT;
313 MPI_Igatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, 0, comm, req);
316 case 14: /* MPI_Ialltoall */
317 for (i = 0; i < size; ++i) {
318 for (j = 0; j < COUNT; ++j) {
319 buf[i*COUNT+j] = rank + (i * j);
320 recvbuf[i*COUNT+j] = 0xdeadbeef;
323 MPI_Ialltoall(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, comm, req);
326 case 15: /* MPI_Iallgather */
327 for (i = 0; i < size*COUNT; ++i) {
329 recvbuf[i] = 0xdeadbeef;
331 MPI_Iallgather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, comm, req);
334 case 16: /* MPI_Iallgatherv */
335 for (i = 0; i < size; ++i) {
336 for (j = 0; j < COUNT; ++j) {
337 recvbuf[i*COUNT+j] = 0xdeadbeef;
339 recvcounts[i] = COUNT;
340 rdispls[i] = i * COUNT;
342 for (i = 0; i < COUNT; ++i)
344 MPI_Iallgatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, comm, req);
347 case 17: /* MPI_Iscan */
348 for (i = 0; i < COUNT; ++i) {
350 recvbuf[i] = 0xdeadbeef;
352 MPI_Iscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
355 case 18: /* MPI_Iexscan */
356 for (i = 0; i < COUNT; ++i) {
358 recvbuf[i] = 0xdeadbeef;
360 MPI_Iexscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
363 case 19: /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
364 for (i = 0; i < size; ++i) {
365 sendcounts[i] = COUNT;
366 recvcounts[i] = COUNT;
367 sdispls[i] = COUNT * i * sizeof(int);
368 rdispls[i] = COUNT * i * sizeof(int);
369 sendtypes[i] = MPI_INT;
370 recvtypes[i] = MPI_INT;
371 for (j = 0; j < COUNT; ++j) {
372 buf[i*COUNT+j] = rank + (i * j);
373 recvbuf[i*COUNT+j] = 0xdeadbeef;
376 MPI_Ialltoallw(buf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, req);
379 case 20: /* basic pt2pt MPI_Isend/MPI_Irecv pairing */
380 /* even ranks send to odd ranks, but only if we have a full pair */
381 if ((rank % 2 != 0) || (rank != size-1)) {
382 for (j = 0; j < COUNT; ++j) {
384 recvbuf[j] = 0xdeadbeef;
387 MPI_Isend(buf, COUNT, MPI_INT, rank+1, 5, comm, req);
389 MPI_Irecv(recvbuf, COUNT, MPI_INT, rank-1, 5, comm, req);
394 fprintf(stderr, "unexpected value for l->case_num=%d)\n", (l->case_num));
401 static void check_after_completion(struct laundry *l)
405 MPI_Comm comm = l->comm;
407 int *recvbuf = l->recvbuf;
408 int *sendcounts = l->sendcounts;
409 int *recvcounts = l->recvcounts;
410 int *sdispls = l->sdispls;
411 int *rdispls = l->rdispls;
412 int *sendtypes = l->sendtypes;
413 int *recvtypes = l->recvtypes;
414 char *buf_alias = (char *)buf;
416 MPI_Comm_rank(comm, &rank);
417 MPI_Comm_size(comm, &size);
419 /* these cases all correspond to cases in start_random_nonblocking */
420 switch (l->case_num) {
421 case 0: /* MPI_Ibcast */
422 for (i = 0; i < COUNT; ++i) {
424 printf("buf[%d]=%d i=%d\n", i, buf[i], i);
425 my_assert(buf[i] == i);
429 case 1: /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
430 for (i = 0; i < PRIME; ++i) {
431 if (buf_alias[i] != i)
432 printf("buf_alias[%d]=%d i=%d\n", i, buf_alias[i], i);
433 my_assert(buf_alias[i] == i);
437 case 2: /* MPI_Ibarrier */
438 /* nothing to check */
441 case 3: /* MPI_Ireduce */
443 for (i = 0; i < COUNT; ++i) {
444 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
445 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
446 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
451 case 4: /* same again, use a user op and free it before the wait */
453 for (i = 0; i < COUNT; ++i) {
454 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
455 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
456 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
461 case 5: /* MPI_Iallreduce */
462 for (i = 0; i < COUNT; ++i) {
463 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
464 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
465 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
469 case 6: /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
470 for (i = 0; i < size; ++i) {
471 for (j = 0; j < COUNT; ++j) {
472 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j)));*/
473 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
478 case 7: /* MPI_Igather */
480 for (i = 0; i < size; ++i) {
481 for (j = 0; j < COUNT; ++j) {
482 my_assert(recvbuf[i*COUNT+j] == i + j);
487 for (i = 0; i < size*COUNT; ++i) {
488 my_assert(recvbuf[i] == 0xdeadbeef);
493 case 8: /* same test again, just use a dup'ed datatype and free it before the wait */
495 for (i = 0; i < size; ++i) {
496 for (j = 0; j < COUNT; ++j) {
497 my_assert(recvbuf[i*COUNT+j] == i + j);
502 for (i = 0; i < size*COUNT; ++i) {
503 my_assert(recvbuf[i] == 0xdeadbeef);
508 case 9: /* MPI_Iscatter */
509 for (j = 0; j < COUNT; ++j) {
510 my_assert(recvbuf[j] == rank + j);
513 for (i = 0; i < size*COUNT; ++i) {
514 /* check we didn't corrupt the sendbuf somehow */
515 my_assert(buf[i] == 0xdeadbeef);
520 case 10: /* MPI_Iscatterv */
521 for (j = 0; j < COUNT; ++j) {
522 my_assert(recvbuf[j] == rank + j);
525 for (i = 0; i < size*COUNT; ++i) {
526 /* check we didn't corrupt the sendbuf somehow */
527 my_assert(buf[i] == 0xdeadbeef);
530 for (i = 1; i < size; ++i) {
531 for (j = 0; j < COUNT; ++j) {
532 /* check we didn't corrupt the rest of the recvbuf */
533 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
538 case 11: /* MPI_Ireduce_scatter */
539 for (j = 0; j < COUNT; ++j) {
540 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
542 for (i = 1; i < size; ++i) {
543 for (j = 0; j < COUNT; ++j) {
544 /* check we didn't corrupt the rest of the recvbuf */
545 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
550 case 12: /* MPI_Ireduce_scatter_block */
551 for (j = 0; j < COUNT; ++j) {
552 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
554 for (i = 1; i < size; ++i) {
555 for (j = 0; j < COUNT; ++j) {
556 /* check we didn't corrupt the rest of the recvbuf */
557 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
562 case 13: /* MPI_Igatherv */
564 for (i = 0; i < size; ++i) {
565 for (j = 0; j < COUNT; ++j) {
566 my_assert(recvbuf[i*COUNT+j] == i + j);
571 for (i = 0; i < size*COUNT; ++i) {
572 my_assert(recvbuf[i] == 0xdeadbeef);
577 case 14: /* MPI_Ialltoall */
578 for (i = 0; i < size; ++i) {
579 for (j = 0; j < COUNT; ++j) {
580 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (i * j)));*/
581 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
586 case 15: /* MPI_Iallgather */
587 for (i = 0; i < size; ++i) {
588 for (j = 0; j < COUNT; ++j) {
589 my_assert(recvbuf[i*COUNT+j] == i + j);
594 case 16: /* MPI_Iallgatherv */
595 for (i = 0; i < size; ++i) {
596 for (j = 0; j < COUNT; ++j) {
597 my_assert(recvbuf[i*COUNT+j] == i + j);
602 case 17: /* MPI_Iscan */
603 for (i = 0; i < COUNT; ++i) {
604 my_assert(recvbuf[i] == ((rank * (rank+1) / 2) + (i * (rank + 1))));
608 case 18: /* MPI_Iexscan */
609 for (i = 0; i < COUNT; ++i) {
611 my_assert(recvbuf[i] == 0xdeadbeef);
613 my_assert(recvbuf[i] == ((rank * (rank+1) / 2) + (i * (rank + 1)) - (rank + i)));
617 case 19: /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
618 for (i = 0; i < size; ++i) {
619 for (j = 0; j < COUNT; ++j) {
620 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j)));*/
621 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
626 case 20: /* basic pt2pt MPI_Isend/MPI_Irecv pairing */
627 /* even ranks send to odd ranks, but only if we have a full pair */
628 if ((rank % 2 != 0) || (rank != size-1)) {
629 for (j = 0; j < COUNT; ++j) {
630 /* only odd procs did a recv */
632 my_assert(recvbuf[j] == 0xdeadbeef);
635 if (recvbuf[j] != j) printf("recvbuf[%d]=%d j=%d\n", j, recvbuf[j], j);
636 my_assert(recvbuf[j] == j);
643 printf("invalid case_num (%d) detected\n", l->case_num);
650 static void complete_something_somehow(unsigned int rndnum, int numreqs, MPI_Request reqs[], int *outcount, int indices[])
654 #define COMPLETION_CASES (8)
655 switch (rand_range(rndnum, 0, COMPLETION_CASES)) {
657 MPI_Waitall(numreqs, reqs, MPI_STATUSES_IGNORE);
659 for (i = 0; i < numreqs; ++i) {
665 MPI_Testsome(numreqs, reqs, outcount, indices, MPI_STATUS_IGNORE);
666 if (*outcount == MPI_UNDEFINED) {
672 MPI_Waitsome(numreqs, reqs, outcount, indices, MPI_STATUS_IGNORE);
673 if (*outcount == MPI_UNDEFINED) {
679 MPI_Waitany(numreqs, reqs, &idx, MPI_STATUS_IGNORE);
680 if (idx == MPI_UNDEFINED) {
690 MPI_Testany(numreqs, reqs, &idx, &flag, MPI_STATUS_IGNORE);
691 if (idx == MPI_UNDEFINED) {
701 MPI_Testall(numreqs, reqs, &flag, MPI_STATUSES_IGNORE);
704 for (i = 0; i < numreqs; ++i) {
714 /* select a new random index and wait on it */
715 rndnum = gen_prn(rndnum);
716 idx = rand_range(rndnum, 0, numreqs);
717 MPI_Wait(&reqs[idx], MPI_STATUS_IGNORE);
723 /* select a new random index and wait on it */
724 rndnum = gen_prn(rndnum);
725 idx = rand_range(rndnum, 0, numreqs);
726 MPI_Test(&reqs[idx], &flag, MPI_STATUS_IGNORE);
727 *outcount = (flag ? 1 : 0);
735 #undef COMPLETION_CASES
738 int main(int argc, char **argv)
740 int i, num_posted, num_completed;
742 unsigned int seed = 0x10bc;
743 unsigned int post_seq, complete_seq;
744 struct laundry larr[WINDOW];
745 MPI_Request reqs[WINDOW];
748 MPI_Comm comms[NUM_COMMS];
751 MPI_Init(&argc, &argv);
752 MPI_Comm_rank(MPI_COMM_WORLD, &wrank);
753 MPI_Comm_size(MPI_COMM_WORLD, &wsize);
755 /* it is critical that all processes in the communicator start with a
756 * consistent value for "post_seq" */
757 post_seq = complete_seq = gen_prn(seed);
762 /* construct all of the communicators, just dups of comm world for now */
763 for (i = 0; i < NUM_COMMS; ++i) {
764 MPI_Comm_dup(MPI_COMM_WORLD, &comms[i]);
767 /* fill the entire window of ops */
768 for (i = 0; i < WINDOW; ++i) {
769 reqs[i] = MPI_REQUEST_NULL;
770 memset(&larr[i], 0, sizeof(struct laundry));
771 larr[i].case_num = -1;
773 /* randomly select a comm, using a new seed to avoid correlating
774 * particular kinds of NBC ops with particular communicators */
775 comm = comms[rand_range(gen_prn(post_seq), 0, NUM_COMMS)];
777 start_random_nonblocking(comm, post_seq, &reqs[i], &larr[i]);
779 post_seq = gen_prn(post_seq);
782 /* now loop repeatedly, completing ops with "random" completion functions,
783 * until we've posted and completed MAIN_ITERATIONS ops */
784 while (num_completed < MAIN_ITERATIONS) {
785 complete_something_somehow(complete_seq, WINDOW, reqs, &outcount, indices);
786 complete_seq = gen_prn(complete_seq);
787 for (i = 0; i < outcount; ++i) {
788 int idx = indices[i];
789 assert(reqs[idx] == MPI_REQUEST_NULL);
790 if (larr[idx].case_num != -1) {
791 check_after_completion(&larr[idx]);
792 cleanup_laundry(&larr[idx]);
794 if (num_posted < MAIN_ITERATIONS) {
795 comm = comms[rand_range(gen_prn(post_seq), 0, NUM_COMMS)];
796 start_random_nonblocking(comm, post_seq, &reqs[idx], &larr[idx]);
798 post_seq = gen_prn(post_seq);
803 /* "randomly" and infrequently introduce some jitter into the system */
804 if (0 == rand_range(gen_prn(complete_seq + wrank), 0, CHANCE_OF_SLEEP)) {
805 usleep(JITTER_DELAY); /* take a short nap */
809 for (i = 0; i < NUM_COMMS; ++i) {
810 MPI_Comm_free(&comms[i]);
815 printf("found %d errors\n", errs);
817 printf(" No errors\n");