1 /* Copyright (c) 2013-2017. The SimGrid Team.
2 * All rights reserved. */
4 /* This program is free software; you can redistribute it and/or modify it
5 * under the terms of the license (GNU LGPL) which comes with this package. */
7 #include "../colls_private.h"
10 /*****************************************************************************
12 * Function: alltoall_3dmesh_shoot
17 send_buff: send input buffer
18 send_count: number of elements to send
19 send_type: data type of elements being sent
20 recv_buff: receive output buffer
21 recv_count: number of elements to received
22 recv_type: data type of elements being received
25 * Descrp: Function realizes the alltoall operation using the 3dmesh
26 algorithm. It actually performs allgather operation in x dimension,
27 y dimension, then in z dimension. Each node then extracts the
28 needed data. The communication in all dimension is simple.
31 ****************************************************************************/
33 static int alltoall_check_is_3dmesh(int num, int *i, int *j, int *k)
39 if ((num % (x * x)) == 0) {
50 int Coll_alltoall_3dmesh::alltoall(void *send_buff, int send_count,
51 MPI_Datatype send_type,
52 void *recv_buff, int recv_count,
53 MPI_Datatype recv_type, MPI_Comm comm)
55 MPI_Request *reqs, *req_ptr;
57 MPI_Status status, *statuses;
58 int i, j, src, dst, rank, num_procs, num_reqs, X, Y, Z, block_size, count;
59 int my_z, two_dsize, my_row_base, my_col_base, my_z_base, src_row_base;
60 int src_z_base, send_offset, recv_offset, tag = COLL_TAG_ALLTOALL;
62 char *tmp_buff1, *tmp_buff2;
65 num_procs = comm->size();
66 extent = send_type->get_extent();
68 if (not alltoall_check_is_3dmesh(num_procs, &X, &Y, &Z))
78 my_z = rank / two_dsize;
80 my_row_base = (rank / X) * X;
81 my_col_base = (rank % Y) + (my_z * two_dsize);
82 my_z_base = my_z * two_dsize;
84 block_size = extent * send_count;
86 tmp_buff1 = (char *) smpi_get_tmp_sendbuffer(block_size * num_procs * two_dsize);
87 tmp_buff2 = (char *) smpi_get_tmp_recvbuffer(block_size * two_dsize);
89 statuses = (MPI_Status *) xbt_malloc(num_reqs * sizeof(MPI_Status));
90 reqs = (MPI_Request *) xbt_malloc(num_reqs * sizeof(MPI_Request));
94 recv_offset = (rank % two_dsize) * block_size * num_procs;
96 Request::sendrecv(send_buff, send_count * num_procs, send_type, rank, tag,
97 tmp_buff1 + recv_offset, num_procs * recv_count,
98 recv_type, rank, tag, comm, &status);
100 count = send_count * num_procs;
102 for (i = 0; i < Y; i++) {
103 src = i + my_row_base;
106 recv_offset = (src % two_dsize) * block_size * num_procs;
107 *(req_ptr++) = Request::irecv(tmp_buff1 + recv_offset, count, recv_type, src, tag, comm);
110 for (i = 0; i < Y; i++) {
111 dst = i + my_row_base;
114 Request::send(send_buff, count, send_type, dst, tag, comm);
117 Request::waitall(Y - 1, reqs, statuses);
121 for (i = 0; i < X; i++) {
122 src = (i * Y + my_col_base);
126 src_row_base = (src / X) * X;
128 recv_offset = (src_row_base % two_dsize) * block_size * num_procs;
129 *(req_ptr++) = Request::irecv(tmp_buff1 + recv_offset, recv_count * num_procs * Y,
130 recv_type, src, tag, comm);
133 send_offset = (my_row_base % two_dsize) * block_size * num_procs;
134 for (i = 0; i < X; i++) {
135 dst = (i * Y + my_col_base);
138 Request::send(tmp_buff1 + send_offset, send_count * num_procs * Y, send_type,
142 Request::waitall(X - 1, reqs, statuses);
145 for (i = 0; i < two_dsize; i++) {
146 send_offset = (rank * block_size) + (i * block_size * num_procs);
147 recv_offset = (my_z_base * block_size) + (i * block_size);
148 Request::sendrecv(tmp_buff1 + send_offset, send_count, send_type, rank, tag,
149 (char *) recv_buff + recv_offset, recv_count, recv_type,
150 rank, tag, comm, &status);
153 for (i = 1; i < Z; i++) {
154 src = (rank + i * two_dsize) % num_procs;
155 src_z_base = (src / two_dsize) * two_dsize;
157 recv_offset = (src_z_base * block_size);
159 *(req_ptr++) = Request::irecv((char *) recv_buff + recv_offset, recv_count * two_dsize,
160 recv_type, src, tag, comm);
163 for (i = 1; i < Z; i++) {
164 dst = (rank + i * two_dsize) % num_procs;
167 for (j = 0; j < two_dsize; j++) {
168 send_offset = (dst + j * num_procs) * block_size;
169 Request::sendrecv(tmp_buff1 + send_offset, send_count, send_type,
170 rank, tag, tmp_buff2 + recv_offset, recv_count,
171 recv_type, rank, tag, comm, &status);
173 recv_offset += block_size;
176 Request::send(tmp_buff2, send_count * two_dsize, send_type, dst, tag, comm);
180 Request::waitall(Z - 1, reqs, statuses);
184 smpi_free_tmp_buffer(tmp_buff1);
185 smpi_free_tmp_buffer(tmp_buff2);