X-Git-Url: http://bilbo.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/ae465ab565add653931acac0fa455335fc9e75e4..9afa0d0db01da2ae64e48fb594cc87c186dde192:/src/smpi/mpi/smpi_comm.cpp diff --git a/src/smpi/mpi/smpi_comm.cpp b/src/smpi/mpi/smpi_comm.cpp index 54d494d186..ba3cf8100b 100644 --- a/src/smpi/mpi/smpi_comm.cpp +++ b/src/smpi/mpi/smpi_comm.cpp @@ -1,16 +1,17 @@ -/* Copyright (c) 2010-2021. The SimGrid Team. All rights reserved. */ +/* Copyright (c) 2010-2023. The SimGrid Team. All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ #include "smpi_comm.hpp" +#include "simgrid/host.h" #include "smpi_coll.hpp" #include "smpi_datatype.hpp" +#include "smpi_info.hpp" #include "smpi_request.hpp" #include "smpi_win.hpp" -#include "smpi_info.hpp" +#include "src/kernel/resource/HostImpl.hpp" #include "src/smpi/include/smpi_actor.hpp" -#include "src/surf/HostImpl.hpp" #include @@ -32,8 +33,7 @@ MPI_Comm MPI_COMM_WORLD = MPI_COMM_UNINITIALIZED; /* Support for cartesian topology was added, but there are 2 other types of topology, graph et dist graph. In order to * support them, we have to add a field SMPI_Topo_type, and replace the MPI_Topology field by an union. */ -namespace simgrid{ -namespace smpi{ +namespace simgrid::smpi { std::unordered_map Comm::keyvals_; int Comm::keyval_id_=0; @@ -55,7 +55,6 @@ Comm::Comm(MPI_Group group, MPI_Topology topo, bool smp, int in_id) colls::bcast(&id, 1, MPI_INT, 0, this); XBT_DEBUG("Communicator %p has id %d", this, id); id_=id;//only set here, as we don't want to change it in the middle of the bcast - colls::barrier(this); } } @@ -65,45 +64,53 @@ void Comm::destroy(Comm* comm) Comm::destroy(smpi_process()->comm_world()); return; } - if(comm != MPI_COMM_WORLD) + if (comm != MPI_COMM_WORLD && not comm->deleted()) { + comm->cleanup_attr(); comm->mark_as_deleted(); + } Comm::unref(comm); } int Comm::dup(MPI_Comm* newcomm){ - if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { - // we need to switch as the called function may silently touch global variables - smpi_switch_data_segment(s4u::Actor::self()); - } + // we need to switch as the called function may silently touch global variables + smpi_switch_data_segment(s4u::Actor::self()); + auto* cp = new Group(this->group()); (*newcomm) = new Comm(cp, this->topo()); - int ret = MPI_SUCCESS; - - for (auto const& it : attributes()) { - auto elem_it = keyvals_.find(it.first); - if (elem_it != keyvals_.end()) { - smpi_key_elem& elem = elem_it->second; - int flag = 0; - void* value_out = nullptr; - if (elem.copy_fn.comm_copy_fn != MPI_NULL_COPY_FN && elem.copy_fn.comm_copy_fn != MPI_COMM_DUP_FN) - ret = elem.copy_fn.comm_copy_fn(this, it.first, elem.extra_state, it.second, &value_out, &flag); - else if (elem.copy_fn.comm_copy_fn_fort != MPI_NULL_COPY_FN && *(int*)*elem.copy_fn.comm_copy_fn_fort != 1) { - value_out = (int*)xbt_malloc(sizeof(int)); - elem.copy_fn.comm_copy_fn_fort(this, it.first, elem.extra_state, it.second, value_out, &flag, &ret); - } - if (ret != MPI_SUCCESS) { - Comm::destroy(*newcomm); - *newcomm = MPI_COMM_NULL; - return ret; - } - if (elem.copy_fn.comm_copy_fn == MPI_COMM_DUP_FN || - ((elem.copy_fn.comm_copy_fn_fort != MPI_NULL_COPY_FN) && *(int*)*elem.copy_fn.comm_copy_fn_fort == 1)) { - elem.refcount++; - (*newcomm)->attributes().emplace(it.first, it.second); - } else if (flag) { - elem.refcount++; - (*newcomm)->attributes().emplace(it.first, value_out); + + for (auto const& [key, value] : attributes()) { + auto elem_it = keyvals_.find(key); + xbt_assert(elem_it != keyvals_.end(), "Keyval not found for Comm: %d", key); + + smpi_key_elem& elem = elem_it->second; + int ret = MPI_SUCCESS; + int flag = 0; + void* value_out = nullptr; + if (elem.copy_fn.comm_copy_fn == MPI_COMM_DUP_FN) { + value_out = value; + flag = 1; + } else if (elem.copy_fn.comm_copy_fn != MPI_NULL_COPY_FN) { + ret = elem.copy_fn.comm_copy_fn(this, key, elem.extra_state, value, &value_out, &flag); + } + if (elem.copy_fn.comm_copy_fn_fort != MPI_NULL_COPY_FN) { + value_out = xbt_new(int, 1); + if (*(int*)*elem.copy_fn.comm_copy_fn_fort == 1) { // MPI_COMM_DUP_FN + memcpy(value_out, value, sizeof(int)); + flag = 1; + } else { // not null, nor dup + elem.copy_fn.comm_copy_fn_fort(this, key, elem.extra_state, value, value_out, &flag, &ret); } + if (ret != MPI_SUCCESS) + xbt_free(value_out); + } + if (ret != MPI_SUCCESS) { + Comm::destroy(*newcomm); + *newcomm = MPI_COMM_NULL; + return ret; + } + if (flag) { + elem.refcount++; + (*newcomm)->attributes().try_emplace(key, value_out); } } //duplicate info if present @@ -114,7 +121,7 @@ int Comm::dup(MPI_Comm* newcomm){ (*newcomm)->set_errhandler(errhandlers_[this->rank()]); else (*newcomm)->set_errhandler(errhandler_); - return ret; + return MPI_SUCCESS; } int Comm::dup_with_info(MPI_Info info, MPI_Comm* newcomm){ @@ -175,12 +182,12 @@ void Comm::get_name(char* name, int* len) const std::string Comm::name() const { int size; - char name[MPI_MAX_NAME_STRING+1]; - this->get_name(name, &size); + std::array name; + this->get_name(name.data(), &size); if (name[0]=='\0') - return std::string("MPI_Comm"); + return "MPI_Comm"; else - return std::string(name); + return name.data(); } @@ -234,14 +241,14 @@ bool Comm::is_uniform() const { if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->is_uniform(); - return is_uniform_ != 0; + return is_uniform_; } bool Comm::is_blocked() const { if (this == MPI_COMM_UNINITIALIZED) return smpi_process()->comm_world()->is_blocked(); - return is_blocked_ != 0; + return is_blocked_; } bool Comm::is_smp_comm() const @@ -296,10 +303,10 @@ MPI_Comm Comm::split(int color, int key) } std::vector requests(rankmap.size()); int reqs = 0; - for (auto const& rank : rankmap) { - if (rank.second != 0) { + for (auto const& [_, rank] : rankmap) { + if (rank != 0) { group_snd[reqs]=new Group(group_out); - requests[reqs] = Request::isend(&(group_snd[reqs]), 1, MPI_PTR, rank.second, system_tag, this); + requests[reqs] = Request::isend(&(group_snd[reqs]), 1, MPI_PTR, rank, system_tag, this); reqs++; } } @@ -352,7 +359,7 @@ void Comm::unref(Comm* comm){ simgrid::smpi::Info::unref(comm->info_); if(comm->errhandlers_!=nullptr){ for (int i=0; isize(); i++) - if (comm->errhandlers_[i]!=MPI_ERRHANDLER_NULL) + if (comm->errhandlers_[i]!=MPI_ERRHANDLER_NULL) simgrid::smpi::Errhandler::unref(comm->errhandlers_[i]); delete[] comm->errhandlers_; } else if (comm->errhandler_ != MPI_ERRHANDLER_NULL) @@ -404,10 +411,9 @@ void Comm::init_smp(){ smpi_process()->set_replaying(false); } - if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { - // we need to switch as the called function may silently touch global variables - smpi_switch_data_segment(s4u::Actor::self()); - } + // we need to switch as the called function may silently touch global variables + smpi_switch_data_segment(s4u::Actor::self()); + // identify neighbors in comm MPI_Comm comm_intra = find_intra_comm(&leader); @@ -418,11 +424,6 @@ void Comm::init_smp(){ allgather__ring(&leader, 1, MPI_INT , leaders_map, 1, MPI_INT, this); - if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { - // we need to switch as the called function may silently touch global variables - smpi_switch_data_segment(s4u::Actor::self()); - } - if(leaders_map_==nullptr){ leaders_map_= leaders_map; }else{ @@ -472,9 +473,10 @@ void Comm::init_smp(){ // Are the nodes uniform ? = same number of process/node int my_local_size=comm_intra->size(); + int is_uniform; if(comm_intra->rank()==0) { - int is_uniform = 1; - int* non_uniform_map = xbt_new0(int,leader_group_size); + is_uniform = 1; + auto* non_uniform_map = xbt_new0(int, leader_group_size); allgather__ring(&my_local_size, 1, MPI_INT, non_uniform_map, 1, MPI_INT, leader_comm); for(i=0; i < leader_group_size; i++) { @@ -488,14 +490,13 @@ void Comm::init_smp(){ }else{ xbt_free(non_uniform_map); } - is_uniform_=is_uniform; } - bcast__scatter_LR_allgather(&is_uniform_, 1, MPI_INT, 0, comm_intra); + bcast__scatter_LR_allgather(&is_uniform, 1, MPI_INT, 0, comm_intra); + is_uniform_ = (is_uniform != 0); + + // we need to switch as the called function may silently touch global variables + smpi_switch_data_segment(s4u::Actor::self()); - if (smpi_cfg_privatization() == SmpiPrivStrategies::MMAP) { - // we need to switch as the called function may silently touch global variables - smpi_switch_data_segment(s4u::Actor::self()); - } // Are the ranks blocked ? = allocated contiguously on the SMP nodes int is_blocked=1; int prev = this->group()->rank(comm_intra->group()->actor(0)); @@ -511,13 +512,8 @@ void Comm::init_smp(){ int global_blocked; allreduce__default(&is_blocked, &global_blocked, 1, MPI_INT, MPI_LAND, this); - if(MPI_COMM_WORLD==MPI_COMM_UNINITIALIZED || this==MPI_COMM_WORLD){ - if(this->rank()==0){ - is_blocked_ = global_blocked; - } - }else{ - is_blocked_=global_blocked; - } + if ((MPI_COMM_WORLD != MPI_COMM_UNINITIALIZED && this != MPI_COMM_WORLD) || this->rank() == 0) + is_blocked_ = (global_blocked != 0); delete[] leader_list; if(replaying) @@ -552,19 +548,17 @@ void Comm::remove_rma_win(MPI_Win win){ void Comm::finish_rma_calls() const { + const int myrank = rank(); for (auto const& it : rma_wins_) { - if(it->rank()==this->rank()){//is it ours (for MPI_COMM_WORLD)? + if (it->rank() == myrank) { // is it ours (for MPI_COMM_WORLD)? int finished = it->finish_comms(); - XBT_DEBUG("Barrier for rank %d - Finished %d RMA calls",this->rank(), finished); + XBT_DEBUG("Barrier for rank %d - Finished %d RMA calls", myrank, finished); } } } MPI_Info Comm::info() { - if (info_ == MPI_INFO_NULL) - info_ = new Info(); - info_->ref(); return info_; } @@ -584,9 +578,12 @@ MPI_Errhandler Comm::errhandler() errhandler_->ref(); return errhandler_; } else { - if(errhandlers_==nullptr) - return MPI_ERRORS_ARE_FATAL; - else { + if(errhandlers_==nullptr){ + if (_smpi_cfg_default_errhandler_is_error) + return MPI_ERRORS_ARE_FATAL; + else + return MPI_ERRORS_RETURN; + } else { if(errhandlers_[this->rank()] != MPI_ERRHANDLER_NULL) errhandlers_[this->rank()]->ref(); return errhandlers_[this->rank()]; @@ -628,5 +625,54 @@ MPI_Comm Comm::split_type(int type, int /*key*/, const Info*) } } -} // namespace smpi -} // namespace simgrid +static inline std::string hash_message(int src, int dst, int tag){ + return std::to_string(tag) + '_' + std::to_string(src) + '_' + std::to_string(dst); +} + +unsigned int Comm::get_sent_messages_count(int src, int dst, int tag) +{ + return sent_messages_[hash_message(src, dst, tag)]; +} + +void Comm::increment_sent_messages_count(int src, int dst, int tag) +{ + sent_messages_[hash_message(src, dst, tag)]++; +} + +unsigned int Comm::get_received_messages_count(int src, int dst, int tag) +{ + return recv_messages_[hash_message(src, dst, tag)]; +} + +void Comm::increment_received_messages_count(int src, int dst, int tag) +{ + recv_messages_[hash_message(src, dst, tag)]++; +} + +unsigned int Comm::get_collectives_count() +{ + if (this==MPI_COMM_UNINITIALIZED){ + return smpi_process()->comm_world()->get_collectives_count(); + }else if(this == MPI_COMM_WORLD || this == smpi_process()->comm_world()){ + if (collectives_counts_.empty()) + collectives_counts_.resize(this->size()); + return collectives_counts_[this->rank()]; + }else{ + return collectives_count_; + } +} + +void Comm::increment_collectives_count() +{ + if (this==MPI_COMM_UNINITIALIZED){ + smpi_process()->comm_world()->increment_collectives_count(); + }else if (this == MPI_COMM_WORLD || this == smpi_process()->comm_world()){ + if (collectives_counts_.empty()) + collectives_counts_.resize(this->size()); + collectives_counts_[this->rank()]++; + }else{ + collectives_count_++; + } +} + +} // namespace simgrid::smpi