1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
3 * (C) 2011 by Argonne National Laboratory.
4 * See COPYRIGHT in top-level directory.
7 /* A basic test of all 17 nonblocking collective operations specified by the
8 * MPI-3 standard. It only exercises the intracommunicator functionality,
9 * does not use MPI_IN_PLACE, and only transmits/receives simple integer types
10 * with relatively small counts. It does check a few fancier issues, such as
11 * ensuring that "premature user releases" of MPI_Op and MPI_Datatype objects
12 * does not result in an error or segfault. */
21 #define my_assert(cond_) \
24 fprintf(stderr, "assertion (%s) failed, aborting\n", #cond_); \
25 MPI_Abort(MPI_COMM_WORLD, 1); \
29 static void sum_fn(void *invec, void *inoutvec, int *len, MPI_Datatype * datatype)
33 int *inout = inoutvec;
34 for (i = 0; i < *len; ++i) {
35 inout[i] = in[i] + inout[i];
40 int main(int argc, char **argv)
46 int *sendcounts = NULL;
47 int *recvcounts = NULL;
50 MPI_Datatype *sendtypes = NULL;
51 MPI_Datatype *recvtypes = NULL;
52 signed char *buf_alias = NULL;
55 MPI_Init(&argc, &argv);
56 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
57 MPI_Comm_size(MPI_COMM_WORLD, &size);
59 buf = malloc(COUNT * size * sizeof(int));
60 recvbuf = malloc(COUNT * size * sizeof(int));
61 sendcounts = malloc(size * sizeof(int));
62 recvcounts = malloc(size * sizeof(int));
63 sdispls = malloc(size * sizeof(int));
64 rdispls = malloc(size * sizeof(int));
65 sendtypes = malloc(size * sizeof(MPI_Datatype));
66 recvtypes = malloc(size * sizeof(MPI_Datatype));
69 for (i = 0; i < COUNT; ++i) {
77 MPI_Ibcast(buf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
78 MPI_Wait(&req, MPI_STATUS_IGNORE);
80 for (i = 0; i < COUNT; ++i) {
82 printf("buf[%d]=%d i=%d\n", i, buf[i], i);
83 my_assert(buf[i] == i);
86 /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */
87 buf_alias = (signed char *) buf;
88 my_assert(COUNT * size * sizeof(int) > PRIME); /* sanity */
89 for (i = 0; i < PRIME; ++i) {
95 for (i = PRIME; i < COUNT * size * sizeof(int); ++i) {
98 MPI_Ibcast(buf_alias, PRIME, MPI_SIGNED_CHAR, 0, MPI_COMM_WORLD, &req);
99 MPI_Wait(&req, MPI_STATUS_IGNORE);
100 for (i = 0; i < PRIME; ++i) {
101 if (buf_alias[i] != i)
102 printf("buf_alias[%d]=%d i=%d\n", i, buf_alias[i], i);
103 my_assert(buf_alias[i] == i);
107 MPI_Ibarrier(MPI_COMM_WORLD, &req);
108 MPI_Wait(&req, MPI_STATUS_IGNORE);
111 for (i = 0; i < COUNT; ++i) {
113 recvbuf[i] = 0xdeadbeef;
115 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD, &req);
116 MPI_Wait(&req, MPI_STATUS_IGNORE);
118 for (i = 0; i < COUNT; ++i) {
119 if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))
120 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],
121 ((size * (size - 1) / 2) + (i * size)));
122 my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));
126 /* same again, use a user op and free it before the wait */
128 MPI_Op op = MPI_OP_NULL;
129 MPI_Op_create(sum_fn, 1, &op);
130 for (i = 0; i < COUNT; ++i) {
132 recvbuf[i] = 0xdeadbeef;
134 MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, op, 0, MPI_COMM_WORLD, &req);
136 MPI_Wait(&req, MPI_STATUS_IGNORE);
138 for (i = 0; i < COUNT; ++i) {
139 if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))
140 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],
141 ((size * (size - 1) / 2) + (i * size)));
142 my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));
148 for (i = 0; i < COUNT; ++i) {
150 recvbuf[i] = 0xdeadbeef;
152 MPI_Iallreduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
153 MPI_Wait(&req, MPI_STATUS_IGNORE);
154 for (i = 0; i < COUNT; ++i) {
155 if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))
156 printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],
157 ((size * (size - 1) / 2) + (i * size)));
158 my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));
161 /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */
162 for (i = 0; i < size; ++i) {
163 sendcounts[i] = COUNT;
164 recvcounts[i] = COUNT;
165 sdispls[i] = COUNT * i;
166 rdispls[i] = COUNT * i;
167 for (j = 0; j < COUNT; ++j) {
168 buf[i * COUNT + j] = rank + (i * j);
169 recvbuf[i * COUNT + j] = 0xdeadbeef;
172 MPI_Ialltoallv(buf, sendcounts, sdispls, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT,
173 MPI_COMM_WORLD, &req);
174 MPI_Wait(&req, MPI_STATUS_IGNORE);
175 for (i = 0; i < size; ++i) {
176 for (j = 0; j < COUNT; ++j) {
177 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j))); */
178 my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));
183 for (i = 0; i < size * COUNT; ++i) {
185 recvbuf[i] = 0xdeadbeef;
187 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
188 MPI_Wait(&req, MPI_STATUS_IGNORE);
190 for (i = 0; i < size; ++i) {
191 for (j = 0; j < COUNT; ++j) {
192 my_assert(recvbuf[i * COUNT + j] == i + j);
197 for (i = 0; i < size * COUNT; ++i) {
198 my_assert(recvbuf[i] == 0xdeadbeef);
202 /* same test again, just use a dup'ed datatype and free it before the wait */
204 MPI_Datatype type = MPI_DATATYPE_NULL;
205 MPI_Type_dup(MPI_INT, &type);
207 for (i = 0; i < size * COUNT; ++i) {
209 recvbuf[i] = 0xdeadbeef;
211 MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, type, 0, MPI_COMM_WORLD, &req);
212 MPI_Type_free(&type); /* should cause implementations that don't refcount
213 * correctly to blow up or hang in the wait */
214 MPI_Wait(&req, MPI_STATUS_IGNORE);
216 for (i = 0; i < size; ++i) {
217 for (j = 0; j < COUNT; ++j) {
218 my_assert(recvbuf[i * COUNT + j] == i + j);
223 for (i = 0; i < size * COUNT; ++i) {
224 my_assert(recvbuf[i] == 0xdeadbeef);
230 for (i = 0; i < size; ++i) {
231 for (j = 0; j < COUNT; ++j) {
233 buf[i * COUNT + j] = i + j;
235 buf[i * COUNT + j] = 0xdeadbeef;
236 recvbuf[i * COUNT + j] = 0xdeadbeef;
239 MPI_Iscatter(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD, &req);
240 MPI_Wait(&req, MPI_STATUS_IGNORE);
241 for (j = 0; j < COUNT; ++j) {
242 my_assert(recvbuf[j] == rank + j);
245 for (i = 0; i < size * COUNT; ++i) {
246 /* check we didn't corrupt the sendbuf somehow */
247 my_assert(buf[i] == 0xdeadbeef);
252 for (i = 0; i < size; ++i) {
253 /* weak test, just test the regular case where all counts are equal */
254 sendcounts[i] = COUNT;
255 sdispls[i] = i * COUNT;
256 for (j = 0; j < COUNT; ++j) {
258 buf[i * COUNT + j] = i + j;
260 buf[i * COUNT + j] = 0xdeadbeef;
261 recvbuf[i * COUNT + j] = 0xdeadbeef;
264 MPI_Iscatterv(buf, sendcounts, sdispls, MPI_INT, recvbuf, COUNT, MPI_INT, 0, MPI_COMM_WORLD,
266 MPI_Wait(&req, MPI_STATUS_IGNORE);
267 for (j = 0; j < COUNT; ++j) {
268 my_assert(recvbuf[j] == rank + j);
271 for (i = 0; i < size * COUNT; ++i) {
272 /* check we didn't corrupt the sendbuf somehow */
273 my_assert(buf[i] == 0xdeadbeef);
276 for (i = 1; i < size; ++i) {
277 for (j = 0; j < COUNT; ++j) {
278 /* check we didn't corrupt the rest of the recvbuf */
279 my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);
283 /* MPI_Ireduce_scatter */
284 for (i = 0; i < size; ++i) {
285 recvcounts[i] = COUNT;
286 for (j = 0; j < COUNT; ++j) {
287 buf[i * COUNT + j] = rank + i;
288 recvbuf[i * COUNT + j] = 0xdeadbeef;
291 MPI_Ireduce_scatter(buf, recvbuf, recvcounts, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
292 MPI_Wait(&req, MPI_STATUS_IGNORE);
293 for (j = 0; j < COUNT; ++j) {
294 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
296 for (i = 1; i < size; ++i) {
297 for (j = 0; j < COUNT; ++j) {
298 /* check we didn't corrupt the rest of the recvbuf */
299 my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);
303 /* MPI_Ireduce_scatter_block */
304 for (i = 0; i < size; ++i) {
305 for (j = 0; j < COUNT; ++j) {
306 buf[i * COUNT + j] = rank + i;
307 recvbuf[i * COUNT + j] = 0xdeadbeef;
310 MPI_Ireduce_scatter_block(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
311 MPI_Wait(&req, MPI_STATUS_IGNORE);
312 for (j = 0; j < COUNT; ++j) {
313 my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));
315 for (i = 1; i < size; ++i) {
316 for (j = 0; j < COUNT; ++j) {
317 /* check we didn't corrupt the rest of the recvbuf */
318 my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);
323 for (i = 0; i < size * COUNT; ++i) {
325 recvbuf[i] = 0xdeadbeef;
327 for (i = 0; i < COUNT; ++i) {
330 for (i = 0; i < size; ++i) {
331 recvcounts[i] = COUNT;
332 rdispls[i] = i * COUNT;
334 MPI_Igatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, 0, MPI_COMM_WORLD,
336 MPI_Wait(&req, MPI_STATUS_IGNORE);
338 for (i = 0; i < size; ++i) {
339 for (j = 0; j < COUNT; ++j) {
340 my_assert(recvbuf[i * COUNT + j] == i + j);
345 for (i = 0; i < size * COUNT; ++i) {
346 my_assert(recvbuf[i] == 0xdeadbeef);
351 for (i = 0; i < size; ++i) {
352 for (j = 0; j < COUNT; ++j) {
353 buf[i * COUNT + j] = rank + (i * j);
354 recvbuf[i * COUNT + j] = 0xdeadbeef;
357 MPI_Ialltoall(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, MPI_COMM_WORLD, &req);
358 MPI_Wait(&req, MPI_STATUS_IGNORE);
359 for (i = 0; i < size; ++i) {
360 for (j = 0; j < COUNT; ++j) {
361 /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (i * j))); */
362 my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));
367 for (i = 0; i < size * COUNT; ++i) {
369 recvbuf[i] = 0xdeadbeef;
371 MPI_Iallgather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, MPI_COMM_WORLD, &req);
372 MPI_Wait(&req, MPI_STATUS_IGNORE);
373 for (i = 0; i < size; ++i) {
374 for (j = 0; j < COUNT; ++j) {
375 my_assert(recvbuf[i * COUNT + j] == i + j);
379 /* MPI_Iallgatherv */
380 for (i = 0; i < size; ++i) {
381 for (j = 0; j < COUNT; ++j) {
382 recvbuf[i * COUNT + j] = 0xdeadbeef;
384 recvcounts[i] = COUNT;
385 rdispls[i] = i * COUNT;
387 for (i = 0; i < COUNT; ++i)
389 MPI_Iallgatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, MPI_COMM_WORLD,
391 MPI_Wait(&req, MPI_STATUS_IGNORE);
392 for (i = 0; i < size; ++i) {
393 for (j = 0; j < COUNT; ++j) {
394 my_assert(recvbuf[i * COUNT + j] == i + j);
399 for (i = 0; i < COUNT; ++i) {
401 recvbuf[i] = 0xdeadbeef;
403 MPI_Iscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
404 MPI_Wait(&req, MPI_STATUS_IGNORE);
405 for (i = 0; i < COUNT; ++i) {
406 my_assert(recvbuf[i] == ((rank * (rank + 1) / 2) + (i * (rank + 1))));
410 for (i = 0; i < COUNT; ++i) {
412 recvbuf[i] = 0xdeadbeef;
414 MPI_Iexscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &req);
415 MPI_Wait(&req, MPI_STATUS_IGNORE);
416 for (i = 0; i < COUNT; ++i) {
418 my_assert(recvbuf[i] == 0xdeadbeef);
420 my_assert(recvbuf[i] == ((rank * (rank + 1) / 2) + (i * (rank + 1)) - (rank + i)));
423 /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */
424 for (i = 0; i < size; ++i) {
425 sendcounts[i] = COUNT;
426 recvcounts[i] = COUNT;
427 sdispls[i] = COUNT * i * sizeof(int);
428 rdispls[i] = COUNT * i * sizeof(int);
429 sendtypes[i] = MPI_INT;
430 recvtypes[i] = MPI_INT;
431 for (j = 0; j < COUNT; ++j) {
432 buf[i * COUNT + j] = rank + (i * j);
433 recvbuf[i * COUNT + j] = 0xdeadbeef;
436 MPI_Ialltoallw(buf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes,
437 MPI_COMM_WORLD, &req);
438 MPI_Wait(&req, MPI_STATUS_IGNORE);
439 for (i = 0; i < size; ++i) {
440 for (j = 0; j < COUNT; ++j) {
441 /* printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j))); */
442 my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));
447 printf(" No Errors\n");