1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
3 * (C) 2011 by Argonne National Laboratory.
4 * See COPYRIGHT in top-level directory.
7 /* This test attempts to execute multiple simultaneous nonblocking collective
8 * (NBC) MPI routines at the same time, and manages their completion with a
9 * variety of routines (MPI_{Wait,Test}{,_all,_any,_some}). It also throws a
10 * few point-to-point operations into the mix.
12 * Possible improvements:
13 * - post operations on multiple comms from multiple threads
24 /* Constants that control the high level test harness behavior. */
25 /* MAIN_ITERATIONS is how many NBC ops the test will attempt to issue. */
26 #define MAIN_ITERATIONS (1000)
27 /* WINDOW is the maximum number of outstanding NBC requests at any given time */
29 /* we sleep with probability 1/CHANCE_OF_SLEEP */
30 #define CHANCE_OF_SLEEP (1000)
31 /* JITTER_DELAY is denominated in microseconds (us) */
32 #define JITTER_DELAY (50000) /* 0.05 seconds */
33 /* NUM_COMMS is the number of communicators on which ops will be posted */
36 /* Constants that control behavior of the individual testing operations.
37 * Altering these can help to explore the testing space, but increasing them too
38 * much can consume too much memory (often O(n^2) usage). */
39 /* FIXME is COUNT==10 too limiting? should we try a larger count too (~500)? */
43 #define my_assert(cond_) \
48 fprintf(stderr, "assertion (%s) failed on line %d\n", #cond_, __LINE__); \
53 /* Intended to act like "rand_r", but we can be sure that it will exist and be
54 * consistent across all of comm world. Returns a number in the range
56 #define GEN_PRN_MAX (4294967291-1)
57 static unsigned int gen_prn(unsigned int x)
59 /* a simple "multiplicative congruential method" PRNG, with parameters:
60 * m=4294967291, largest 32-bit prime
61 * a=279470273, good primitive root of m from "TABLES OF LINEAR
62 * CONGRUENTIAL GENERATORS OF DIFFERENT SIZES AND GOOD
63 * LATTICE STRUCTURE", by Pierre L’Ecuyer */
64 return (279470273UL * (unsigned long) x) % 4294967291UL;
67 /* given a random unsigned int value "rndval_" from gen_prn, this evaluates to a
68 * value in the range [min_,max_) */
69 #define rand_range(rndval_,min_,max_) \
70 ((unsigned int)((min_) + ((rndval_) * (1.0 / (GEN_PRN_MAX+1.0)) * ((max_) - (min_)))))
73 static void sum_fn(void *invec, void *inoutvec, int *len, MPI_Datatype * datatype)
77 int *inout = inoutvec;
78 for (i = 0; i < *len; ++i) {
79 inout[i] = in[i] + inout[i];
83 /* used to keep track of buffers that should be freed after the corresponding
84 * operation has completed */
86 int case_num; /* which test case initiated this req/laundry */
94 MPI_Datatype *sendtypes;
95 MPI_Datatype *recvtypes;
98 static void cleanup_laundry(struct laundry *l)
101 l->comm = MPI_COMM_NULL;
120 /* Starts a "random" operation on "comm" corresponding to "rndnum" and returns
121 * in (*req) a request handle corresponding to that operation. This call should
122 * be considered collective over comm (with a consistent value for "rndnum"),
123 * even though the operation may only be a point-to-point request. */
124 static void start_random_nonblocking(MPI_Comm comm, unsigned int rndnum, MPI_Request * req,
131 int *sendcounts = NULL;
132 int *recvcounts = NULL;
135 MPI_Datatype *sendtypes = NULL;
136 MPI_Datatype *recvtypes = NULL;
137 signed char *buf_alias = NULL;
139 MPI_Comm_rank(comm, &rank);
140 MPI_Comm_size(comm, &size);
142 *req = MPI_REQUEST_NULL;
147 l->buf = buf = malloc(COUNT * size * sizeof(int));
148 l->recvbuf = recvbuf = malloc(COUNT * size * sizeof(int));
149 l->sendcounts = sendcounts = malloc(size * sizeof(int));
150 l->recvcounts = recvcounts = malloc(size * sizeof(int));
151 l->sdispls = sdispls = malloc(size * sizeof(int));
152 l->rdispls = rdispls = malloc(size * sizeof(int));
153 l->sendtypes = sendtypes = malloc(size * sizeof(MPI_Datatype));
154 l->recvtypes = recvtypes = malloc(size * sizeof(MPI_Datatype));
156 #define NUM_CASES (21)
157 l->case_num = rand_range(rndnum, 0, NUM_CASES);
158 switch (l->case_num) {
159 case 0: /* MPI_Ibcast */
160 for (i = 0; i < COUNT; ++i) {
168 MPI_Ibcast(buf, COUNT, MPI_INT, 0, comm, req);
171 case 1: /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
172 /* FIXME fiddle with PRIME and buffer allocation s.t. PRIME is much larger (1021?) */
173 buf_alias = (signed char *) buf;
174 my_assert(COUNT * size * sizeof(int) > PRIME); /* sanity */
175 for (i = 0; i < PRIME; ++i) {
181 for (i = PRIME; i < COUNT * size * sizeof(int); ++i) {
184 MPI_Ibcast(buf_alias, PRIME, MPI_SIGNED_CHAR, 0, comm, req);
187 case 2: /* MPI_Ibarrier */
188 MPI_Ibarrier(comm, req);
191 case 3: /* MPI_Ireduce */
192 for (i = 0; i < COUNT; ++i) {
194 recvbuf[i] = 0xdeadbeef;
196 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, 0, comm, req);
199 case 4: /* same again, use a user op and free it before the wait */
201 MPI_Op op = MPI_OP_NULL;
202 MPI_Op_create(sum_fn, /*commute= */ 1, &op);
203 for (i = 0; i < COUNT; ++i) {
205 recvbuf[i] = 0xdeadbeef;
207 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, op, 0, comm, req);
212 case 5: /* MPI_Iallreduce */
213 for (i = 0; i < COUNT; ++i) {
215 recvbuf[i] = 0xdeadbeef;
217 MPI_Iallreduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
220 case 6: /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
221 for (i = 0; i < size; ++i) {
222 sendcounts[i] = COUNT;
223 recvcounts[i] = COUNT;
224 sdispls[i] = COUNT * i;
225 rdispls[i] = COUNT * i;
226 for (j = 0; j < COUNT; ++j) {
227 buf[i * COUNT + j] = rank + (i * j);
228 recvbuf[i * COUNT + j] = 0xdeadbeef;
231 MPI_Ialltoallv(buf, sendcounts, sdispls, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT,
235 case 7: /* MPI_Igather */
236 for (i = 0; i < size * COUNT; ++i) {
238 recvbuf[i] = 0xdeadbeef;
240 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);
243 case 8: /* same test again, just use a dup'ed datatype and free it before the wait */
245 MPI_Datatype type = MPI_DATATYPE_NULL;
246 MPI_Type_dup(MPI_INT, &type);
247 for (i = 0; i < size * COUNT; ++i) {
249 recvbuf[i] = 0xdeadbeef;
251 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, type, 0, comm, req);
252 MPI_Type_free(&type); /* should cause implementations that don't refcount
253 * correctly to blow up or hang in the wait */
257 case 9: /* MPI_Iscatter */
258 for (i = 0; i < size; ++i) {
259 for (j = 0; j < COUNT; ++j) {
261 buf[i * COUNT + j] = i + j;
263 buf[i * COUNT + j] = 0xdeadbeef;
264 recvbuf[i * COUNT + j] = 0xdeadbeef;
267 MPI_Iscatter(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);
270 case 10: /* MPI_Iscatterv */
271 for (i = 0; i < size; ++i) {
272 /* weak test, just test the regular case where all counts are equal */
273 sendcounts[i] = COUNT;
274 sdispls[i] = i * COUNT;
275 for (j = 0; j < COUNT; ++j) {
277 buf[i * COUNT + j] = i + j;
279 buf[i * COUNT + j] = 0xdeadbeef;
280 recvbuf[i * COUNT + j] = 0xdeadbeef;
283 MPI_Iscatterv(buf, sendcounts, sdispls, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);
286 case 11: /* MPI_Ireduce_scatter */
287 for (i = 0; i < size; ++i) {
288 recvcounts[i] = COUNT;
289 for (j = 0; j < COUNT; ++j) {
290 buf[i * COUNT + j] = rank + i;
291 recvbuf[i * COUNT + j] = 0xdeadbeef;
294 MPI_Ireduce_scatter(buf, recvbuf, recvcounts, MPI_INT, MPI_SUM, comm, req);
297 case 12: /* MPI_Ireduce_scatter_block */
298 for (i = 0; i < size; ++i) {
299 for (j = 0; j < COUNT; ++j) {
300 buf[i * COUNT + j] = rank + i;
301 recvbuf[i * COUNT + j] = 0xdeadbeef;
304 MPI_Ireduce_scatter_block(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
307 case 13: /* MPI_Igatherv */
308 for (i = 0; i < size * COUNT; ++i) {
310 recvbuf[i] = 0xdeadbeef;
312 for (i = 0; i < COUNT; ++i) {
315 for (i = 0; i < size; ++i) {
316 recvcounts[i] = COUNT;
317 rdispls[i] = i * COUNT;
319 MPI_Igatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, 0, comm, req);
322 case 14: /* MPI_Ialltoall */
323 for (i = 0; i < size; ++i) {
324 for (j = 0; j < COUNT; ++j) {
325 buf[i * COUNT + j] = rank + (i * j);
326 recvbuf[i * COUNT + j] = 0xdeadbeef;
329 MPI_Ialltoall(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, comm, req);
332 case 15: /* MPI_Iallgather */
333 for (i = 0; i < size * COUNT; ++i) {
335 recvbuf[i] = 0xdeadbeef;
337 MPI_Iallgather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, comm, req);
340 case 16: /* MPI_Iallgatherv */
341 for (i = 0; i < size; ++i) {
342 for (j = 0; j < COUNT; ++j) {
343 recvbuf[i * COUNT + j] = 0xdeadbeef;
345 recvcounts[i] = COUNT;
346 rdispls[i] = i * COUNT;
348 for (i = 0; i < COUNT; ++i)
350 MPI_Iallgatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, comm, req);
353 case 17: /* MPI_Iscan */
354 for (i = 0; i < COUNT; ++i) {
356 recvbuf[i] = 0xdeadbeef;
358 MPI_Iscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
361 case 18: /* MPI_Iexscan */
362 for (i = 0; i < COUNT; ++i) {
364 recvbuf[i] = 0xdeadbeef;
366 MPI_Iexscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);
369 case 19: /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
370 for (i = 0; i < size; ++i) {
371 sendcounts[i] = COUNT;
372 recvcounts[i] = COUNT;
373 sdispls[i] = COUNT * i * sizeof(int);
374 rdispls[i] = COUNT * i * sizeof(int);
375 sendtypes[i] = MPI_INT;
376 recvtypes[i] = MPI_INT;
377 for (j = 0; j < COUNT; ++j) {
378 buf[i * COUNT + j] = rank + (i * j);
379 recvbuf[i * COUNT + j] = 0xdeadbeef;
382 MPI_Ialltoallw(buf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes,
386 case 20: /* basic pt2pt MPI_Isend/MPI_Irecv pairing */
387 /* even ranks send to odd ranks, but only if we have a full pair */
388 if ((rank % 2 != 0) || (rank != size - 1)) {
389 for (j = 0; j < COUNT; ++j) {
391 recvbuf[j] = 0xdeadbeef;
394 MPI_Isend(buf, COUNT, MPI_INT, rank + 1, 5, comm, req);
396 MPI_Irecv(recvbuf, COUNT, MPI_INT, rank - 1, 5, comm, req);
401 fprintf(stderr, "unexpected value for l->case_num=%d)\n", (l->case_num));
407 static void check_after_completion(struct laundry *l)
411 MPI_Comm comm = l->comm;
413 int *recvbuf = l->recvbuf;
414 char *buf_alias = (char *) buf;
416 MPI_Comm_rank(comm, &rank);
417 MPI_Comm_size(comm, &size);
419 /* these cases all correspond to cases in start_random_nonblocking */
420 switch (l->case_num) {
421 case 0: /* MPI_Ibcast */
422 for (i = 0; i < COUNT; ++i) {
424 printf("buf[%d]=%d i=%d\n", i, buf[i], i);
425 my_assert(buf[i] == i);
429 case 1: /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
430 for (i = 0; i < PRIME; ++i) {
431 if (buf_alias[i] != i)
432 printf("buf_alias[%d]=%d i=%d\n", i, buf_alias[i], i);
433 my_assert(buf_alias[i] == i);
437 case 2: /* MPI_Ibarrier */
438 /* nothing to check */
441 case 3: /* MPI_Ireduce */
443 for (i = 0; i < COUNT; ++i) {
444 if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))
445 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],
446 ((size * (size - 1) / 2) + (i * size)));
447 my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));
452 case 4: /* same again, use a user op and free it before the wait */
454 for (i = 0; i < COUNT; ++i) {
455 if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))
456 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],
457 ((size * (size - 1) / 2) + (i * size)));
458 my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));
463 case 5: /* MPI_Iallreduce */
464 for (i = 0; i < COUNT; ++i) {
465 if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))
466 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],
467 ((size * (size - 1) / 2) + (i * size)));
468 my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));
472 case 6: /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
473 for (i = 0; i < size; ++i) {
474 for (j = 0; j < COUNT; ++j) {
475 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j))); */
476 my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));
481 case 7: /* MPI_Igather */
483 for (i = 0; i < size; ++i) {
484 for (j = 0; j < COUNT; ++j) {
485 my_assert(recvbuf[i * COUNT + j] == i + j);
490 for (i = 0; i < size * COUNT; ++i) {
491 my_assert(recvbuf[i] == 0xdeadbeef);
496 case 8: /* same test again, just use a dup'ed datatype and free it before the wait */
498 for (i = 0; i < size; ++i) {
499 for (j = 0; j < COUNT; ++j) {
500 my_assert(recvbuf[i * COUNT + j] == i + j);
505 for (i = 0; i < size * COUNT; ++i) {
506 my_assert(recvbuf[i] == 0xdeadbeef);
511 case 9: /* MPI_Iscatter */
512 for (j = 0; j < COUNT; ++j) {
513 my_assert(recvbuf[j] == rank + j);
516 for (i = 0; i < size * COUNT; ++i) {
517 /* check we didn't corrupt the sendbuf somehow */
518 my_assert(buf[i] == 0xdeadbeef);
523 case 10: /* MPI_Iscatterv */
524 for (j = 0; j < COUNT; ++j) {
525 my_assert(recvbuf[j] == rank + j);
528 for (i = 0; i < size * COUNT; ++i) {
529 /* check we didn't corrupt the sendbuf somehow */
530 my_assert(buf[i] == 0xdeadbeef);
533 for (i = 1; i < size; ++i) {
534 for (j = 0; j < COUNT; ++j) {
535 /* check we didn't corrupt the rest of the recvbuf */
536 my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);
541 case 11: /* MPI_Ireduce_scatter */
542 for (j = 0; j < COUNT; ++j) {
543 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
545 for (i = 1; i < size; ++i) {
546 for (j = 0; j < COUNT; ++j) {
547 /* check we didn't corrupt the rest of the recvbuf */
548 my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);
553 case 12: /* MPI_Ireduce_scatter_block */
554 for (j = 0; j < COUNT; ++j) {
555 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
557 for (i = 1; i < size; ++i) {
558 for (j = 0; j < COUNT; ++j) {
559 /* check we didn't corrupt the rest of the recvbuf */
560 my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);
565 case 13: /* MPI_Igatherv */
567 for (i = 0; i < size; ++i) {
568 for (j = 0; j < COUNT; ++j) {
569 my_assert(recvbuf[i * COUNT + j] == i + j);
574 for (i = 0; i < size * COUNT; ++i) {
575 my_assert(recvbuf[i] == 0xdeadbeef);
580 case 14: /* MPI_Ialltoall */
581 for (i = 0; i < size; ++i) {
582 for (j = 0; j < COUNT; ++j) {
583 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (i * j))); */
584 my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));
589 case 15: /* MPI_Iallgather */
590 for (i = 0; i < size; ++i) {
591 for (j = 0; j < COUNT; ++j) {
592 my_assert(recvbuf[i * COUNT + j] == i + j);
597 case 16: /* MPI_Iallgatherv */
598 for (i = 0; i < size; ++i) {
599 for (j = 0; j < COUNT; ++j) {
600 my_assert(recvbuf[i * COUNT + j] == i + j);
605 case 17: /* MPI_Iscan */
606 for (i = 0; i < COUNT; ++i) {
607 my_assert(recvbuf[i] == ((rank * (rank + 1) / 2) + (i * (rank + 1))));
611 case 18: /* MPI_Iexscan */
612 for (i = 0; i < COUNT; ++i) {
614 my_assert(recvbuf[i] == 0xdeadbeef);
616 my_assert(recvbuf[i] == ((rank * (rank + 1) / 2) + (i * (rank + 1)) - (rank + i)));
620 case 19: /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
621 for (i = 0; i < size; ++i) {
622 for (j = 0; j < COUNT; ++j) {
623 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j))); */
624 my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));
629 case 20: /* basic pt2pt MPI_Isend/MPI_Irecv pairing */
630 /* even ranks send to odd ranks, but only if we have a full pair */
631 if ((rank % 2 != 0) || (rank != size - 1)) {
632 for (j = 0; j < COUNT; ++j) {
633 /* only odd procs did a recv */
635 my_assert(recvbuf[j] == 0xdeadbeef);
639 printf("recvbuf[%d]=%d j=%d\n", j, recvbuf[j], j);
640 my_assert(recvbuf[j] == j);
647 printf("invalid case_num (%d) detected\n", l->case_num);
655 static void complete_something_somehow(unsigned int rndnum, int numreqs, MPI_Request reqs[],
656 int *outcount, int indices[])
660 #define COMPLETION_CASES (8)
661 switch (rand_range(rndnum, 0, COMPLETION_CASES)) {
663 MPI_Waitall(numreqs, reqs, MPI_STATUSES_IGNORE);
665 for (i = 0; i < numreqs; ++i) {
671 MPI_Testsome(numreqs, reqs, outcount, indices, MPI_STATUS_IGNORE);
672 if (*outcount == MPI_UNDEFINED) {
678 MPI_Waitsome(numreqs, reqs, outcount, indices, MPI_STATUS_IGNORE);
679 if (*outcount == MPI_UNDEFINED) {
685 MPI_Waitany(numreqs, reqs, &idx, MPI_STATUS_IGNORE);
686 if (idx == MPI_UNDEFINED) {
696 MPI_Testany(numreqs, reqs, &idx, &flag, MPI_STATUS_IGNORE);
697 if (idx == MPI_UNDEFINED) {
707 MPI_Testall(numreqs, reqs, &flag, MPI_STATUSES_IGNORE);
710 for (i = 0; i < numreqs; ++i) {
720 /* select a new random index and wait on it */
721 rndnum = gen_prn(rndnum);
722 idx = rand_range(rndnum, 0, numreqs);
723 MPI_Wait(&reqs[idx], MPI_STATUS_IGNORE);
729 /* select a new random index and wait on it */
730 rndnum = gen_prn(rndnum);
731 idx = rand_range(rndnum, 0, numreqs);
732 MPI_Test(&reqs[idx], &flag, MPI_STATUS_IGNORE);
733 *outcount = (flag ? 1 : 0);
741 #undef COMPLETION_CASES
744 int main(int argc, char **argv)
746 int i, num_posted, num_completed;
748 unsigned int seed = 0x10bc;
749 unsigned int post_seq, complete_seq;
750 struct laundry larr[WINDOW];
751 MPI_Request reqs[WINDOW];
754 MPI_Comm comms[NUM_COMMS];
757 MPI_Init(&argc, &argv);
758 MPI_Comm_rank(MPI_COMM_WORLD, &wrank);
759 MPI_Comm_size(MPI_COMM_WORLD, &wsize);
761 /* it is critical that all processes in the communicator start with a
762 * consistent value for "post_seq" */
763 post_seq = complete_seq = gen_prn(seed);
768 /* construct all of the communicators, just dups of comm world for now */
769 for (i = 0; i < NUM_COMMS; ++i) {
770 MPI_Comm_dup(MPI_COMM_WORLD, &comms[i]);
773 /* fill the entire window of ops */
774 for (i = 0; i < WINDOW; ++i) {
775 reqs[i] = MPI_REQUEST_NULL;
776 memset(&larr[i], 0, sizeof(struct laundry));
777 larr[i].case_num = -1;
779 /* randomly select a comm, using a new seed to avoid correlating
780 * particular kinds of NBC ops with particular communicators */
781 comm = comms[rand_range(gen_prn(post_seq), 0, NUM_COMMS)];
783 start_random_nonblocking(comm, post_seq, &reqs[i], &larr[i]);
785 post_seq = gen_prn(post_seq);
788 /* now loop repeatedly, completing ops with "random" completion functions,
789 * until we've posted and completed MAIN_ITERATIONS ops */
790 while (num_completed < MAIN_ITERATIONS) {
791 complete_something_somehow(complete_seq, WINDOW, reqs, &outcount, indices);
792 complete_seq = gen_prn(complete_seq);
793 for (i = 0; i < outcount; ++i) {
794 int idx = indices[i];
795 // assert(reqs[idx] == MPI_REQUEST_NULL);
796 if (larr[idx].case_num != -1) {
797 check_after_completion(&larr[idx]);
798 cleanup_laundry(&larr[idx]);
800 if (num_posted < MAIN_ITERATIONS) {
801 comm = comms[rand_range(gen_prn(post_seq), 0, NUM_COMMS)];
802 start_random_nonblocking(comm, post_seq, &reqs[idx], &larr[idx]);
804 post_seq = gen_prn(post_seq);
809 /* "randomly" and infrequently introduce some jitter into the system */
810 if (0 == rand_range(gen_prn(complete_seq + wrank), 0, CHANCE_OF_SLEEP)) {
811 usleep(JITTER_DELAY); /* take a short nap */
815 for (i = 0; i < NUM_COMMS; ++i) {
816 MPI_Comm_free(&comms[i]);
821 printf("found %d errors\n", errs);
823 printf(" No errors\n");