1 /* Copyright (c) 2014-2023. The SimGrid Team. All rights reserved. */
3 /* This program is free software; you can redistribute it and/or modify it
4 * under the terms of the license (GNU LGPL) which comes with this package. */
8 #include "smpi_comm.hpp"
9 #include "smpi_topo.hpp"
10 #include "xbt/sysdep.h"
15 /* static functions */
16 static int assignnodes(int ndim, const std::vector<int>& factors, std::vector<int>& dims);
17 static int getfactors(int num, std::vector<int>& factors);
19 namespace simgrid::smpi {
21 void Topo::setComm(MPI_Comm comm)
23 xbt_assert(not comm_);
27 /*******************************************************************************
28 * Cartesian topologies
29 ******************************************************************************/
31 Topo_Cart::Topo_Cart(int ndims) : ndims_(ndims), dims_(ndims), periodic_(ndims), position_(ndims)
35 /* reorder is ignored, don't know what would be the consequences of a dumb reordering but neither do I see the point of
37 Topo_Cart::Topo_Cart(MPI_Comm comm_old, int ndims, const int dims[], const int periods[], int /*reorder*/, MPI_Comm* comm_cart)
40 int rank = comm_old->rank();
44 for (int i = 0 ; i < ndims ; i++) {
48 if(comm_cart != nullptr)
49 *comm_cart = MPI_COMM_NULL;
55 // FIXME : code duplication... See coords
57 for (int i=0; i<ndims; i++) {
59 periodic_[i] = periods[i];
60 nranks = nranks / dims[i];
61 /* FIXME: nranks could be zero (?) */
62 position_[i] = rank / nranks;
66 if(comm_cart != nullptr){
67 const Group* oldGroup = comm_old->group();
68 auto* newGroup = new Group(newSize);
69 for (int i = 0 ; i < newSize ; i++) {
70 newGroup->set_mapping(oldGroup->actor(i), i);
72 *comm_cart = new Comm(newGroup, std::shared_ptr<Topo>(this));
75 if(comm_cart != nullptr){
77 auto* group = new Group(MPI_COMM_SELF->group());
78 *comm_cart = new Comm(group, std::shared_ptr<Topo>(this));
80 *comm_cart = MPI_COMM_NULL;
84 if(comm_cart != nullptr){
89 Topo_Cart* Topo_Cart::sub(const int remain_dims[], MPI_Comm *newcomm) {
90 int oldNDims = ndims_;
91 std::vector<int> newDims;
92 std::vector<int> newPeriodic;
94 if (remain_dims == nullptr && oldNDims != 0) {
98 for (int i = 0 ; i < oldNDims ; i++) {
104 newDims.resize(newNDims);
105 newPeriodic.resize(newNDims);
107 // that should not segfault
109 for (int i = 0; i < oldNDims; i++) {
111 newDims[j] =dims_[i];
112 newPeriodic[j] =periodic_[i];
118 //split into several communicators
120 for (int i = 0; i < oldNDims; i++) {
121 if (not remain_dims[i]) {
122 color = (color * dims_[i] + position_[i]);
127 res = new Topo_Cart(getComm(), newNDims, newDims.data(), newPeriodic.data(), 0, newcomm);
129 *newcomm = getComm()->split(color, getComm()->rank());
130 auto topo = std::make_shared<Topo_Cart>(getComm(), newNDims, newDims.data(), newPeriodic.data(), 0, nullptr);
132 res->setComm(*newcomm);
133 (*newcomm)->set_topo(topo);
138 int Topo_Cart::coords(int rank, int /*maxdims*/, int coords[])
140 int nnodes = nnodes_;
141 for (int i = 0; i< ndims_; i++ ) {
142 nnodes = nnodes /dims_[i];
143 coords[i] = rank / nnodes;
144 rank = rank % nnodes;
149 int Topo_Cart::get(int maxdims, int* dims, int* periods, int* coords) {
150 int ndims=ndims_ < maxdims ?ndims_ : maxdims;
151 for(int i = 0 ; i < ndims ; i++) {
153 periods[i] =periodic_[i];
154 coords[i] =position_[i];
159 int Topo_Cart::rank(const int* coords, int* rank) {
164 for (int i=ndims-1; i >=0; i-- ) {
165 int coord = coords[i];
167 /* The user can give us whatever coordinates he wants. If one of them is out of range, either this dimension is
168 * periodic, and we consider the equivalent coordinate inside the bounds, or it's not and then it's an error
170 if (coord >=dims_[i]) {
172 coord = coord %dims_[i];
174 // Should I do that ?
178 } else if (coord < 0) {
180 coord = coord %dims_[i];
182 coord =dims_[i] + coord;
189 *rank += multiplier * coord;
190 multiplier *=dims_[i];
195 int Topo_Cart::shift(int direction, int disp, int* rank_source, int* rank_dest)
200 if (ndims_ < direction) {
204 std::vector<int> position(ndims_);
205 this->coords(getComm()->rank(), ndims_, position.data());
206 position[direction] += disp;
208 if(position[direction] < 0 ||
209 position[direction] >=dims_[direction]) {
210 if(periodic_[direction]) {
211 position[direction] %=dims_[direction];
212 this->rank(position.data(), rank_dest);
214 *rank_dest = MPI_PROC_NULL;
217 this->rank(position.data(), rank_dest);
220 position[direction] = position_[direction] - disp;
221 if(position[direction] < 0 || position[direction] >=dims_[direction]) {
222 if(periodic_[direction]) {
223 position[direction] %=dims_[direction];
224 this->rank(position.data(), rank_source);
226 *rank_source = MPI_PROC_NULL;
229 this->rank(position.data(), rank_source);
234 int Topo_Cart::dim_get(int* ndims) const
240 // Everything below has been taken from ompi, but could be easily rewritten (and partially was to follow sonar rules).
243 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
244 * University Research and Technology
245 * Corporation. All rights reserved.
246 * Copyright (c) 2004-2005 The University of Tennessee and The University
247 * of Tennessee Research Foundation. All rights
249 * Copyright (c) 2004-2014 High Performance Computing Center Stuttgart,
250 * University of Stuttgart. All rights reserved.
251 * Copyright (c) 2004-2005 The Regents of the University of California.
252 * All rights reserved.
253 * Copyright (c) 2012 Los Alamos National Security, LLC. All rights
255 * Copyright (c) 2014 Intel, Inc. All rights reserved
258 * Additional copyrights may follow
264 * This is a utility function, no need to have anything in the lower layer for this at all
266 int Topo_Cart::Dims_create(int nnodes, int ndims, int dims[])
268 /* Get # of free-to-be-assigned processes and # of free dimensions */
269 int freeprocs = nnodes;
271 for (int i = 0; i < ndims; ++i) {
274 } else if ((dims[i] < 0) || ((nnodes % dims[i]) != 0)) {
278 freeprocs /= dims[i];
283 if (freeprocs == 1) {
289 if (freeprocs == 1) {
290 for (int i = 0; i < ndims; ++i) {
298 /* Factor the number of free processes */
299 std::vector<int> factors;
300 int err = getfactors(freeprocs, factors);
301 if (MPI_SUCCESS != err)
304 /* Assign free processes to free dimensions */
305 std::vector<int> procs;
306 err = assignnodes(freedims, factors, procs);
307 if (MPI_SUCCESS != err)
310 /* Return assignment results */
311 auto p = procs.begin();
312 for (int i = 0; i < ndims; ++i) {
322 } // namespace simgrid::smpi
327 * Function: - assign processes to dimensions
328 * - get "best-balanced" grid
329 * - greedy bin-packing algorithm used
330 * - sort dimensions in decreasing order
331 * - dimensions array dynamically allocated
332 * Accepts: - # of dimensions
333 * - std::vector of prime factors
334 * - reference to std::vector of dimensions (returned value)
335 * Returns: - 0 or ERROR
337 static int assignnodes(int ndim, const std::vector<int>& factors, std::vector<int>& dims)
343 /* Allocate and initialize the bins */
345 dims.resize(ndim, 1);
347 /* Loop assigning factors from the highest to the lowest */
348 for (auto pfact = factors.crbegin(); pfact != factors.crend(); ++pfact) {
349 /* Assign a factor to the smallest bin */
350 auto pmin = std::min_element(dims.begin(), dims.end());
354 /* Sort dimensions in decreasing order */
355 std::sort(dims.begin(), dims.end(), std::greater<>());
363 * Function: - factorize a number
365 * - reference to std::vector of prime factors
366 * Returns: - MPI_SUCCESS or ERROR
368 static int getfactors(int num, std::vector<int>& factors)
375 /* determine all occurrences of factor 2 */
376 while((num % 2) == 0) {
378 factors.push_back(2);
380 /* determine all occurrences of uneven prime numbers up to sqrt(num) */
382 while ((num > 1) && (d * d < num)) {
383 while((num % d) == 0) {
385 factors.push_back(d);
389 /* as we looped only up to sqrt(num) one factor > sqrt(num) may be left over */
391 factors.push_back(num);