4 smpi_coll_tuned_allgather_RDB(void *sbuf, int send_count,
5 MPI_Datatype send_type,
6 void *rbuf, int recv_count,
7 MPI_Datatype recv_type,
12 MPI_Aint send_chunk, recv_chunk;
14 // local int variables
15 int i, j, k, dst, rank, num_procs, send_offset, recv_offset, tree_root;
16 int dst_tree_root, rank_tree_root, last_recv_count, num_procs_completed;
21 int curr_count = recv_count;
23 // local string variables
24 char *send_ptr = (char *) sbuf;
25 char *recv_ptr = (char *) rbuf;
27 // get size of the communicator, followed by rank
28 MPI_Comm_size(comm, &num_procs);
29 MPI_Comm_rank(comm, &rank);
31 // get size of single element's type for send buffer and recv buffer
32 MPI_Type_extent(send_type, &send_chunk);
33 MPI_Type_extent(recv_type, &recv_chunk);
35 // multiply size of each element by number of elements to send or recv
36 send_chunk *= send_count;
37 recv_chunk *= recv_count;
39 // perform a local copy
40 MPI_Sendrecv(send_ptr, send_count, send_type, rank, tag,
41 recv_ptr + rank * recv_chunk, recv_count, recv_type, rank, tag,
45 while (mask < num_procs) {
47 dst_tree_root = dst >> i;
49 rank_tree_root = rank >> i;
51 send_offset = rank_tree_root * send_chunk;
52 recv_offset = dst_tree_root * recv_chunk;
54 if (dst < num_procs) {
55 MPI_Sendrecv(recv_ptr + send_offset, curr_count, send_type, dst,
56 tag, recv_ptr + recv_offset, mask * recv_count,
57 recv_type, dst, tag, comm, &status);
58 MPI_Get_count(&status, recv_type, &last_recv_count);
59 curr_count += last_recv_count;
62 if (dst_tree_root + mask > num_procs) {
63 num_procs_completed = num_procs - rank_tree_root - mask;
64 /* num_procs_completed is the number of processes in this
65 subtree that have all the data. Send data to others
66 in a tree fashion. First find root of current tree
67 that is being divided into two. k is the number of
68 least-significant bits in this process's rank that
69 must be zeroed out to find the rank of the root */
79 offset = recv_chunk * (rank_tree_root + mask);
83 dst = rank ^ tmp_mask;
85 tree_root = rank >> k;
88 /* send only if this proc has data and destination
89 doesn't have data. at any step, multiple processes
90 can send if they have the data */
92 && (rank < tree_root + num_procs_completed)
93 && (dst >= tree_root + num_procs_completed)) {
94 MPI_Send(recv_ptr + offset, last_recv_count, recv_type, dst,
97 /* last_recv_cnt was set in the previous
98 receive. that's the amount of data to be
101 /* recv only if this proc. doesn't have data and sender
103 else if ((dst < rank)
104 && (dst < tree_root + num_procs_completed)
105 && (rank >= tree_root + num_procs_completed)) {
106 MPI_Recv(recv_ptr + offset,
107 recv_count * num_procs_completed,
108 recv_type, dst, tag, comm, &status);
109 // num_procs_completed is also equal to the no. of processes
110 // whose data we don't have
111 MPI_Get_count(&status, recv_type, &last_recv_count);
112 curr_count += last_recv_count;