1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
3 * (C) 2011 by Argonne National Laboratory.
4 * See COPYRIGHT in top-level directory.
7 /* A basic test of all 17 nonblocking collective operations specified by the
8 * MPI-3 standard. It only exercises the intracommunicator functionality,
9 * does not use MPI_IN_PLACE, and only transmits/receives simple integer types
10 * with relatively small counts. It does check a few fancier issues, such as
11 * ensuring that "premature user releases" of MPI_Op and MPI_Datatype objects
12 * does not result in an error or segfault. */
21 #define my_assert(cond_) \
24 fprintf(stderr, "assertion (%s) failed, aborting\n", #cond_); \
25 MPI_Abort(MPI_COMM_WORLD, 1); \
30 static void sum_fn(void *invec, void *inoutvec, int *len, MPI_Datatype *datatype)
34 int *inout = inoutvec;
35 for (i = 0; i < *len; ++i) {
36 inout[i] = in[i] + inout[i];
41 int main(int argc, char **argv)
47 int *sendcounts = NULL;
48 int *recvcounts = NULL;
51 int *sendtypes = NULL;
52 int *recvtypes = NULL;
53 signed char *buf_alias = NULL;
56 MPI_Init(&argc, &argv);
57 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
58 MPI_Comm_size(MPI_COMM_WORLD, &size);
60 buf = malloc(COUNT*size*sizeof(int));
61 recvbuf = malloc(COUNT*size*sizeof(int));
62 sendcounts = malloc(size*sizeof(int));
63 recvcounts = malloc(size*sizeof(int));
64 sdispls = malloc(size*sizeof(int));
65 rdispls = malloc(size*sizeof(int));
66 sendtypes = malloc(size*sizeof(MPI_Datatype));
67 recvtypes = malloc(size*sizeof(MPI_Datatype));
70 for (i = 0; i < COUNT; ++i) {
78 MPI_Ibcast(buf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
79 MPI_Wait(&req, MPI_STATUS_IGNORE);
81 for (i = 0; i < COUNT; ++i) {
83 printf("buf[%d]=%d i=%d\n", i, buf[i], i);
84 my_assert(buf[i] == i);
87 /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
88 buf_alias = (signed char *)buf;
89 my_assert(COUNT*size*sizeof(int) > PRIME); /* sanity */
90 for (i = 0; i < PRIME; ++i) {
96 for (i = PRIME; i < COUNT * size * sizeof(int); ++i) {
99 MPI_Ibcast(buf_alias, PRIME, MPI_SIGNED_CHAR, 0, MPI_COMM_WORLD, &req);
100 MPI_Wait(&req, MPI_STATUS_IGNORE);
101 for (i = 0; i < PRIME; ++i) {
102 if (buf_alias[i] != i)
103 printf("buf_alias[%d]=%d i=%d\n", i, buf_alias[i], i);
104 my_assert(buf_alias[i] == i);
108 MPI_Ibarrier(MPI_COMM_WORLD, &req);
109 MPI_Wait(&req, MPI_STATUS_IGNORE);
112 for (i = 0; i < COUNT; ++i) {
114 recvbuf[i] = 0xdeadbeef;
116 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD, &req);
117 MPI_Wait(&req, MPI_STATUS_IGNORE);
119 for (i = 0; i < COUNT; ++i) {
120 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
121 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
122 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
126 /* same again, use a user op and free it before the wait */
128 MPI_Op op = MPI_OP_NULL;
129 MPI_Op_create(sum_fn, /*commute=*/1, &op);
131 for (i = 0; i < COUNT; ++i) {
133 recvbuf[i] = 0xdeadbeef;
135 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, op, 0, MPI_COMM_WORLD, &req);
137 MPI_Wait(&req, MPI_STATUS_IGNORE);
139 for (i = 0; i < COUNT; ++i) {
140 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
141 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
142 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
148 for (i = 0; i < COUNT; ++i) {
150 recvbuf[i] = 0xdeadbeef;
152 MPI_Iallreduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
153 MPI_Wait(&req, MPI_STATUS_IGNORE);
154 for (i = 0; i < COUNT; ++i) {
155 if (recvbuf[i] != ((size * (size-1) / 2) + (i * size)))
156 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i], ((size * (size-1) / 2) + (i * size)));
157 my_assert(recvbuf[i] == ((size * (size-1) / 2) + (i * size)));
160 /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
161 for (i = 0; i < size; ++i) {
162 sendcounts[i] = COUNT;
163 recvcounts[i] = COUNT;
164 sdispls[i] = COUNT * i;
165 rdispls[i] = COUNT * i;
166 for (j = 0; j < COUNT; ++j) {
167 buf[i*COUNT+j] = rank + (i * j);
168 recvbuf[i*COUNT+j] = 0xdeadbeef;
171 MPI_Ialltoallv(buf, sendcounts, sdispls, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, MPI_COMM_WORLD, &req);
172 MPI_Wait(&req, MPI_STATUS_IGNORE);
173 for (i = 0; i < size; ++i) {
174 for (j = 0; j < COUNT; ++j) {
175 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j)));*/
176 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
181 for (i = 0; i < size*COUNT; ++i) {
183 recvbuf[i] = 0xdeadbeef;
185 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
186 MPI_Wait(&req, MPI_STATUS_IGNORE);
188 for (i = 0; i < size; ++i) {
189 for (j = 0; j < COUNT; ++j) {
190 my_assert(recvbuf[i*COUNT+j] == i + j);
195 for (i = 0; i < size*COUNT; ++i) {
196 my_assert(recvbuf[i] == 0xdeadbeef);
200 /* same test again, just use a dup'ed datatype and free it before the wait */
202 MPI_Datatype type = MPI_DATATYPE_NULL;
203 MPI_Type_dup(MPI_INT, &type);
205 for (i = 0; i < size*COUNT; ++i) {
207 recvbuf[i] = 0xdeadbeef;
209 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, type, 0, MPI_COMM_WORLD, &req);
210 MPI_Type_free(&type); /* should cause implementations that don't refcount
211 correctly to blow up or hang in the wait */
212 MPI_Wait(&req, MPI_STATUS_IGNORE);
214 for (i = 0; i < size; ++i) {
215 for (j = 0; j < COUNT; ++j) {
216 my_assert(recvbuf[i*COUNT+j] == i + j);
221 for (i = 0; i < size*COUNT; ++i) {
222 my_assert(recvbuf[i] == 0xdeadbeef);
228 for (i = 0; i < size; ++i) {
229 for (j = 0; j < COUNT; ++j) {
231 buf[i*COUNT+j] = i + j;
233 buf[i*COUNT+j] = 0xdeadbeef;
234 recvbuf[i*COUNT+j] = 0xdeadbeef;
237 MPI_Iscatter(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
238 MPI_Wait(&req, MPI_STATUS_IGNORE);
239 for (j = 0; j < COUNT; ++j) {
240 my_assert(recvbuf[j] == rank + j);
243 for (i = 0; i < size*COUNT; ++i) {
244 /* check we didn't corrupt the sendbuf somehow */
245 my_assert(buf[i] == 0xdeadbeef);
250 for (i = 0; i < size; ++i) {
251 /* weak test, just test the regular case where all counts are equal */
252 sendcounts[i] = COUNT;
253 sdispls[i] = i * COUNT;
254 for (j = 0; j < COUNT; ++j) {
256 buf[i*COUNT+j] = i + j;
258 buf[i*COUNT+j] = 0xdeadbeef;
259 recvbuf[i*COUNT+j] = 0xdeadbeef;
262 MPI_Iscatterv(buf, sendcounts, sdispls, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
263 MPI_Wait(&req, MPI_STATUS_IGNORE);
264 for (j = 0; j < COUNT; ++j) {
265 my_assert(recvbuf[j] == rank + j);
268 for (i = 0; i < size*COUNT; ++i) {
269 /* check we didn't corrupt the sendbuf somehow */
270 my_assert(buf[i] == 0xdeadbeef);
273 for (i = 1; i < size; ++i) {
274 for (j = 0; j < COUNT; ++j) {
275 /* check we didn't corrupt the rest of the recvbuf */
276 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
280 /* MPI_Ireduce_scatter */
281 for (i = 0; i < size; ++i) {
282 recvcounts[i] = COUNT;
283 for (j = 0; j < COUNT; ++j) {
284 buf[i*COUNT+j] = rank + i;
285 recvbuf[i*COUNT+j] = 0xdeadbeef;
288 MPI_Ireduce_scatter(buf, recvbuf, recvcounts, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
289 MPI_Wait(&req, MPI_STATUS_IGNORE);
290 for (j = 0; j < COUNT; ++j) {
291 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
293 for (i = 1; i < size; ++i) {
294 for (j = 0; j < COUNT; ++j) {
295 /* check we didn't corrupt the rest of the recvbuf */
296 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
300 /* MPI_Ireduce_scatter_block */
301 for (i = 0; i < size; ++i) {
302 for (j = 0; j < COUNT; ++j) {
303 buf[i*COUNT+j] = rank + i;
304 recvbuf[i*COUNT+j] = 0xdeadbeef;
307 MPI_Ireduce_scatter_block(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
308 MPI_Wait(&req, MPI_STATUS_IGNORE);
309 for (j = 0; j < COUNT; ++j) {
310 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
312 for (i = 1; i < size; ++i) {
313 for (j = 0; j < COUNT; ++j) {
314 /* check we didn't corrupt the rest of the recvbuf */
315 my_assert(recvbuf[i*COUNT+j] == 0xdeadbeef);
320 for (i = 0; i < size*COUNT; ++i) {
322 recvbuf[i] = 0xdeadbeef;
324 for (i = 0; i < COUNT; ++i) {
327 for (i = 0; i < size; ++i) {
328 recvcounts[i] = COUNT;
329 rdispls[i] = i * COUNT;
331 MPI_Igatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, 0, MPI_COMM_WORLD, &req);
332 MPI_Wait(&req, MPI_STATUS_IGNORE);
334 for (i = 0; i < size; ++i) {
335 for (j = 0; j < COUNT; ++j) {
336 my_assert(recvbuf[i*COUNT+j] == i + j);
341 for (i = 0; i < size*COUNT; ++i) {
342 my_assert(recvbuf[i] == 0xdeadbeef);
347 for (i = 0; i < size; ++i) {
348 for (j = 0; j < COUNT; ++j) {
349 buf[i*COUNT+j] = rank + (i * j);
350 recvbuf[i*COUNT+j] = 0xdeadbeef;
353 MPI_Ialltoall(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, MPI_COMM_WORLD, &req);
354 MPI_Wait(&req, MPI_STATUS_IGNORE);
355 for (i = 0; i < size; ++i) {
356 for (j = 0; j < COUNT; ++j) {
357 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (i * j)));*/
358 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
363 for (i = 0; i < size*COUNT; ++i) {
365 recvbuf[i] = 0xdeadbeef;
367 MPI_Iallgather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, MPI_COMM_WORLD, &req);
368 MPI_Wait(&req, MPI_STATUS_IGNORE);
369 for (i = 0; i < size; ++i) {
370 for (j = 0; j < COUNT; ++j) {
371 my_assert(recvbuf[i*COUNT+j] == i + j);
375 /* MPI_Iallgatherv */
376 for (i = 0; i < size; ++i) {
377 for (j = 0; j < COUNT; ++j) {
378 recvbuf[i*COUNT+j] = 0xdeadbeef;
380 recvcounts[i] = COUNT;
381 rdispls[i] = i * COUNT;
383 for (i = 0; i < COUNT; ++i)
385 MPI_Iallgatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, MPI_COMM_WORLD, &req);
386 MPI_Wait(&req, MPI_STATUS_IGNORE);
387 for (i = 0; i < size; ++i) {
388 for (j = 0; j < COUNT; ++j) {
389 my_assert(recvbuf[i*COUNT+j] == i + j);
394 for (i = 0; i < COUNT; ++i) {
396 recvbuf[i] = 0xdeadbeef;
398 MPI_Iscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
399 MPI_Wait(&req, MPI_STATUS_IGNORE);
400 for (i = 0; i < COUNT; ++i) {
401 my_assert(recvbuf[i] == ((rank * (rank+1) / 2) + (i * (rank + 1))));
405 for (i = 0; i < COUNT; ++i) {
407 recvbuf[i] = 0xdeadbeef;
409 MPI_Iexscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
410 MPI_Wait(&req, MPI_STATUS_IGNORE);
411 for (i = 0; i < COUNT; ++i) {
413 my_assert(recvbuf[i] == 0xdeadbeef);
415 my_assert(recvbuf[i] == ((rank * (rank+1) / 2) + (i * (rank + 1)) - (rank + i)));
418 /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
419 for (i = 0; i < size; ++i) {
420 sendcounts[i] = COUNT;
421 recvcounts[i] = COUNT;
422 sdispls[i] = COUNT * i * sizeof(int);
423 rdispls[i] = COUNT * i * sizeof(int);
424 sendtypes[i] = MPI_INT;
425 recvtypes[i] = MPI_INT;
426 for (j = 0; j < COUNT; ++j) {
427 buf[i*COUNT+j] = rank + (i * j);
428 recvbuf[i*COUNT+j] = 0xdeadbeef;
431 MPI_Ialltoallw(buf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, MPI_COMM_WORLD, &req);
432 MPI_Wait(&req, MPI_STATUS_IGNORE);
433 for (i = 0; i < size; ++i) {
434 for (j = 0; j < COUNT; ++j) {
435 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j)));*/
436 my_assert(recvbuf[i*COUNT+j] == (i + (rank * j)));
441 printf(" No Errors\n");