1 #include "colls_private.h"
2 //#include <star-reduction.c>
4 int smpi_coll_tuned_allreduce_rdb(void *sbuff, void *rbuff, int count,
5 MPI_Datatype dtype, MPI_Op op, MPI_Comm comm)
7 int nprocs, rank, type_size, tag = 543;
8 int mask, dst, pof2, newrank, rem, newdst;
13 #ifdef MPICH2_REDUCTION
14 MPI_User_function * uop = MPIR_Op_table[op % 16 - 1];
16 MPI_User_function *uop;
17 struct MPIR_OP *op_ptr;
18 op_ptr = MPIR_ToPointer(op);
22 nprocs=smpi_comm_size(comm);
23 rank=smpi_comm_rank(comm);
25 smpi_datatype_extent(dtype, &lb, &extent);
26 tmp_buf = (void *) xbt_malloc(count * extent);
28 smpi_mpi_sendrecv(sbuff, count, dtype, rank, 500,
29 rbuff, count, dtype, rank, 500, comm, &status);
31 type_size=smpi_datatype_size(dtype);
33 // find nearest power-of-two less than or equal to comm_size
35 while (pof2 <= nprocs)
41 // In the non-power-of-two case, all even-numbered
42 // processes of rank < 2*rem send their data to
43 // (rank+1). These even-numbered processes no longer
44 // participate in the algorithm until the very end. The
45 // remaining processes form a nice power-of-two.
51 smpi_mpi_send(rbuff, count, dtype, rank + 1, tag, comm);
53 // temporarily set the rank to -1 so that this
54 // process does not pariticipate in recursive
59 smpi_mpi_recv(tmp_buf, count, dtype, rank - 1, tag, comm, &status);
60 // do the reduction on received data. since the
61 // ordering is right, it doesn't matter whether
62 // the operation is commutative or not.
63 smpi_op_apply(op, tmp_buf, rbuff, &count, &dtype);
70 else // rank >= 2 * rem
73 // If op is user-defined or count is less than pof2, use
74 // recursive doubling algorithm. Otherwise do a reduce-scatter
75 // followed by allgather. (If op is user-defined,
76 // derived datatypes are allowed and the user could pass basic
77 // datatypes on one process and derived on another as long as
78 // the type maps are the same. Breaking up derived
79 // datatypes to do the reduce-scatter is tricky, therefore
80 // using recursive doubling in that case.)
85 newdst = newrank ^ mask;
86 // find real rank of dest
87 dst = (newdst < rem) ? newdst * 2 + 1 : newdst + rem;
89 // Send the most current data, which is in recvbuf. Recv
91 smpi_mpi_sendrecv(rbuff, count, dtype, dst, tag, tmp_buf, count, dtype,
92 dst, tag, comm, &status);
94 // tmp_buf contains data received in this step.
95 // recvbuf contains data accumulated so far
97 // op is commutative OR the order is already right
98 // we assume it is commuttive op
99 // if (op -> op_commute || (dst < rank))
101 smpi_op_apply(op, tmp_buf, rbuff, &count, &dtype);
102 } else // op is noncommutative and the order is not right
104 smpi_op_apply(op, rbuff, tmp_buf, &count, &dtype);
106 // copy result back into recvbuf
107 smpi_mpi_sendrecv(tmp_buf, count, dtype, rank, tag, rbuff, count,
108 dtype, rank, tag, comm, &status);
113 // In the non-power-of-two case, all odd-numbered processes of
114 // rank < 2 * rem send the result to (rank-1), the ranks who didn't
115 // participate above.
117 if (rank < 2 * rem) {
119 smpi_mpi_send(rbuff, count, dtype, rank - 1, tag, comm);
121 smpi_mpi_recv(rbuff, count, dtype, rank + 1, tag, comm, &status);