1 /* Copyright (c) 2008-2023. The SimGrid Team. All rights reserved. */
3 /* This program is free software; you can redistribute it and/or modify it
4 * under the terms of the license (GNU LGPL) which comes with this package. */
6 #if defined(_GNU_SOURCE)
7 #define DEFINED_GNUSOURCE 1
12 #if defined(__linux__)
14 //inspired by https://stackoverflow.com/a/70211227
15 #if not defined(__USE_GNU)
20 #ifndef DEFINED_GNUSOURCE
24 #include "private.hpp"
25 #include "smpi_coll.hpp"
26 #include "smpi_config.hpp"
27 #include "src/mc/mc.h"
28 #include "xbt/config.hpp"
30 #include "xbt/parse_units.hpp"
32 #include <cfloat> /* DBL_MAX */
33 #include <boost/algorithm/string.hpp> /* trim */
34 #include <boost/tokenizer.hpp>
36 #include "src/mc/mc_config.hpp"
37 #include "src/mc/mc_replay.hpp"
39 #if defined(__APPLE__)
40 # include <AvailabilityMacros.h>
41 # ifndef MAC_OS_X_VERSION_10_12
42 # define MAC_OS_X_VERSION_10_12 101200
45 # define __MAC_11_0 110000
48 constexpr bool HAVE_WORKING_MMAP = ((MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12) && (MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_11_0));
49 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__sun) || defined(__HAIKU__) || defined(__MUSL__)
50 constexpr bool HAVE_WORKING_MMAP = false;
52 constexpr bool HAVE_WORKING_MMAP = true;
55 SharedMallocType _smpi_cfg_shared_malloc = SharedMallocType::GLOBAL;
56 SmpiPrivStrategies _smpi_cfg_privatization = SmpiPrivStrategies::NONE;
57 double _smpi_cfg_host_speed;
59 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(smpi_config, smpi, "Logging specific to SMPI (config)");
61 simgrid::config::Flag<std::string> _smpi_cfg_host_speed_string{
62 "smpi/host-speed", "Speed of the host running the simulation (in flop/s). Used to bench the operations.", "20000f",
63 [](const std::string& str) {
64 _smpi_cfg_host_speed = xbt_parse_get_speed("smpi/host-speed", 1, str, "option smpi/host-speed");
65 xbt_assert(_smpi_cfg_host_speed > 0.0, "Invalid value (%s) for 'smpi/host-speed': it must be positive.",
69 simgrid::config::Flag<bool> _smpi_cfg_simulate_computation{
70 "smpi/simulate-computation", "Whether the computational part of the simulated application should be simulated.",
72 simgrid::config::Flag<std::string> _smpi_cfg_shared_malloc_string{
73 "smpi/shared-malloc", "Whether SMPI_SHARED_MALLOC is enabled. Disable it for debugging purposes.", "global",
74 [](const std::string& val) {
75 if ((val == "yes") || (val == "1") || (val == "on") || (val == "global")) {
76 _smpi_cfg_shared_malloc = SharedMallocType::GLOBAL;
77 } else if (val == "local") {
78 _smpi_cfg_shared_malloc = SharedMallocType::LOCAL;
79 } else if ((val == "no") || (val == "0") || (val == "off")) {
80 _smpi_cfg_shared_malloc = SharedMallocType::NONE;
82 xbt_die("Invalid value '%s' for option smpi/shared-malloc. Possible values: 'on' or 'global', 'local', 'off'",
87 simgrid::config::Flag<double> _smpi_cfg_cpu_threshold{
88 "smpi/cpu-threshold", "Minimal computation time (in seconds) not discarded, or -1 for infinity.", 1e-6,
89 [](const double& val){
91 _smpi_cfg_cpu_threshold = DBL_MAX;
94 simgrid::config::Flag<int> _smpi_cfg_async_small_thresh{"smpi/async-small-thresh",
95 "Maximal size of messages that are to be sent asynchronously, without waiting for the receiver",
97 simgrid::config::Flag<int> _smpi_cfg_detached_send_thresh{
98 "smpi/send-is-detached-thresh",
99 "Threshold of message size where MPI_Send stops behaving like MPI_Isend and becomes MPI_Ssend", 65536};
100 simgrid::config::Flag<bool> _smpi_cfg_grow_injected_times{"smpi/grow-injected-times",
101 "Whether we want to make the injected time in MPI_Iprobe and MPI_Test grow, to "
102 "allow faster simulation. This can make simulation less precise, though.",
104 simgrid::config::Flag<double> _smpi_cfg_iprobe_cpu_usage{"smpi/iprobe-cpu-usage",
105 "Maximum usage of CPUs by MPI_Iprobe() calls. We've observed that MPI_Iprobes "
106 "consume significantly less power than the maximum of a specific application. "
107 "This value is then (Iprobe_Usage/Max_Application_Usage).",
110 simgrid::config::Flag<bool> _smpi_cfg_trace_call_location{"smpi/trace-call-location",
111 "Should filename and linenumber of MPI calls be traced?", false};
112 simgrid::config::Flag<bool> _smpi_cfg_trace_call_use_absolute_path{"smpi/trace-call-use-absolute-path",
113 "Should filenames for trace-call tracing be absolute or not?", false};
114 simgrid::config::Flag<std::string> _smpi_cfg_comp_adjustment_file{
115 "smpi/comp-adjustment-file", "A file containing speedups or slowdowns for some parts of the code.", "",
116 [](const std::string& filename) {
117 if (not filename.empty()) {
118 std::ifstream fstream(filename);
119 xbt_assert(fstream.is_open(), "Could not open file %s. Does it exist?", filename.c_str());
121 using Tokenizer = boost::tokenizer<boost::escaped_list_separator<char>>;
122 std::getline(fstream, line); // Skip the header line
123 while (std::getline(fstream, line)) {
125 auto it = tok.begin();
126 auto end = std::next(tok.begin());
127 std::string location = *it;
128 boost::trim(location);
129 location2speedup.try_emplace(location, std::stod(*end));
134 simgrid::config::Flag<bool> _smpi_cfg_default_errhandler_is_error{
135 "smpi/errors-are-fatal", "Whether MPI errors are fatal or just return. Default is true", true };
136 simgrid::config::Flag<bool> _smpi_cfg_pedantic{
137 "smpi/pedantic", "Activate extra checks that may crash slightly incorrect codes which would not crash on actual implementations", false };
138 simgrid::config::Flag<double> _smpi_init_sleep(
139 "smpi/init", "Time to inject inside a call to MPI_Init", 0.0);
141 simgrid::config::Flag<std::string> _smpi_cfg_papi_events_file{"smpi/papi-events",
142 "This switch enables tracking the specified counters with PAPI", ""};
145 simgrid::config::Flag<double> _smpi_cfg_auto_shared_malloc_thresh("smpi/auto-shared-malloc-thresh",
146 "Threshold size for the automatic sharing of memory",
149 simgrid::config::Flag<bool> _smpi_cfg_display_alloc("smpi/display-allocs",
150 "Whether we should display a memory allocations analysis after simulation.",
153 simgrid::config::Flag<int> _smpi_cfg_list_leaks("smpi/list-leaks",
154 "Whether we should display the n first MPI handle leaks (addresses and type only) after simulation",
157 double smpi_cfg_host_speed(){
158 return _smpi_cfg_host_speed;
161 bool smpi_cfg_simulate_computation(){
162 return _smpi_cfg_simulate_computation;
165 SharedMallocType smpi_cfg_shared_malloc(){
166 return _smpi_cfg_shared_malloc;
169 double smpi_cfg_cpu_thresh(){
170 return _smpi_cfg_cpu_threshold;
173 SmpiPrivStrategies smpi_cfg_privatization(){
174 return _smpi_cfg_privatization;
177 int smpi_cfg_async_small_thresh(){
178 return _smpi_cfg_async_small_thresh;
181 int smpi_cfg_detached_send_thresh(){
182 return _smpi_cfg_detached_send_thresh;
185 bool smpi_cfg_grow_injected_times(){
186 return _smpi_cfg_grow_injected_times;
189 double smpi_cfg_iprobe_cpu_usage(){
190 return _smpi_cfg_iprobe_cpu_usage;
193 bool smpi_cfg_trace_call_location(){
194 return _smpi_cfg_trace_call_location;
197 bool smpi_cfg_trace_call_use_absolute_path(){
198 return _smpi_cfg_trace_call_use_absolute_path;
201 bool smpi_cfg_display_alloc(){
202 return _smpi_cfg_list_leaks != -1 ? true : _smpi_cfg_display_alloc;
205 std::string smpi_cfg_comp_adjustment_file(){
206 return _smpi_cfg_comp_adjustment_file;
209 std::string smpi_cfg_papi_events_file(){
210 return _smpi_cfg_papi_events_file;
213 double smpi_cfg_auto_shared_malloc_thresh(){
214 return _smpi_cfg_auto_shared_malloc_thresh;
217 // public version declared in smpi.h (without parameter, and with C linkage)
218 void smpi_init_options()
220 smpi_init_options_internal(false);
223 void smpi_init_options_internal(bool called_by_smpi_main)
225 static bool smpi_options_initialized = false;
226 static bool running_with_smpi_main = false;
228 if (called_by_smpi_main)
229 running_with_smpi_main = true;
231 // return if already called
232 if (smpi_options_initialized)
234 simgrid::config::declare_flag<bool>("smpi/display-timing", "Whether we should display the timing after simulation.", false);
235 simgrid::config::declare_flag<bool>("smpi/keep-temps", "Whether we should keep the generated temporary files.", false);
236 simgrid::config::declare_flag<std::string>("smpi/tmpdir", "tmp dir for dlopen files", "/tmp");
238 simgrid::config::declare_flag<std::string>("smpi/coll-selector", "Which collective selector to use", "default");
239 simgrid::config::declare_flag<std::string>("smpi/gather", "Which collective to use for gather", "");
240 simgrid::config::declare_flag<std::string>("smpi/allgather", "Which collective to use for allgather", "");
241 simgrid::config::declare_flag<std::string>("smpi/barrier", "Which collective to use for barrier", "");
242 simgrid::config::declare_flag<std::string>("smpi/reduce_scatter", "Which collective to use for reduce_scatter", "");
243 simgrid::config::declare_flag<std::string>("smpi/scatter", "Which collective to use for scatter", "");
244 simgrid::config::declare_flag<std::string>("smpi/allgatherv", "Which collective to use for allgatherv", "");
245 simgrid::config::declare_flag<std::string>("smpi/allreduce", "Which collective to use for allreduce", "");
246 simgrid::config::declare_flag<std::string>("smpi/alltoall", "Which collective to use for alltoall", "");
247 simgrid::config::declare_flag<std::string>("smpi/alltoallv", "Which collective to use for alltoallv", "");
248 simgrid::config::declare_flag<std::string>("smpi/bcast", "Which collective to use for bcast", "");
249 simgrid::config::declare_flag<std::string>("smpi/reduce", "Which collective to use for reduce", "");
251 const char* default_privatization = std::getenv("SMPI_PRIVATIZATION");
252 if (default_privatization == nullptr)
253 default_privatization = "no";
255 simgrid::config::declare_flag<std::string>(
256 "smpi/privatization", "How we should privatize global variable at runtime (no, yes, mmap, dlopen).",
257 default_privatization, [](const std::string& smpi_privatize_option) {
258 if (smpi_privatize_option == "no" || smpi_privatize_option == "0" || smpi_privatize_option == "OFF")
259 _smpi_cfg_privatization = SmpiPrivStrategies::NONE;
260 else if (smpi_privatize_option == "yes" || smpi_privatize_option == "1" || smpi_privatize_option == "ON")
261 _smpi_cfg_privatization = SmpiPrivStrategies::DEFAULT;
262 else if (smpi_privatize_option == "mmap")
263 _smpi_cfg_privatization = SmpiPrivStrategies::MMAP;
264 else if (smpi_privatize_option == "dlopen")
265 _smpi_cfg_privatization = SmpiPrivStrategies::DLOPEN;
267 xbt_die("Invalid value for smpi/privatization: '%s'", smpi_privatize_option.c_str());
269 if (not running_with_smpi_main) {
270 XBT_DEBUG("Running without smpi_main(); disable smpi/privatization.");
271 _smpi_cfg_privatization = SmpiPrivStrategies::NONE;
273 if (not HAVE_WORKING_MMAP && _smpi_cfg_privatization == SmpiPrivStrategies::MMAP) {
274 XBT_INFO("mmap privatization is broken on this platform, switching to dlopen privatization instead.");
275 _smpi_cfg_privatization = SmpiPrivStrategies::DLOPEN;
279 simgrid::config::declare_flag<std::string>(
280 "smpi/privatize-libs",
281 "Add libraries (; separated) to privatize (libgfortran for example)."
282 "You need to provide the full names of the files (libgfortran.so.4), or its full path",
284 simgrid::config::declare_flag<double>("smpi/shared-malloc-blocksize",
285 "Size of the bogus file which will be created for global shared allocations",
287 simgrid::config::declare_flag<std::string>("smpi/shared-malloc-hugepage",
288 "Path to a mounted hugetlbfs, to use huge pages with shared malloc.", "");
290 simgrid::config::declare_flag<std::string>(
291 "smpi/os", "Small messages timings (MPI_Send minimum time for small messages)", "0:0:0:0:0");
292 simgrid::config::declare_flag<std::string>(
293 "smpi/ois", "Small messages timings (MPI_Isend minimum time for small messages)", "0:0:0:0:0");
294 simgrid::config::declare_flag<std::string>(
295 "smpi/or", "Small messages timings (MPI_Recv minimum time for small messages)", "0:0:0:0:0");
297 simgrid::config::declare_flag<bool>("smpi/barrier-finalization", {"smpi/finalization-barrier"},
298 "Do we add a barrier in MPI_Finalize or not", false);
299 simgrid::config::declare_flag<bool>("smpi/barrier-collectives",
300 "Inject a barrier in each colllective operation, to detect some deadlocks in "
301 "incorrect MPI codes, which may not be triggered in all cases",
304 smpi_options_initialized = true;
307 void smpi_check_options()
309 if (MC_is_active() || MC_record_replay_is_active()) {
310 if (_sg_mc_buffering == "zero")
311 simgrid::config::set_value<int>("smpi/send-is-detached-thresh", 0);
312 else if (_sg_mc_buffering == "infty")
313 simgrid::config::set_value<int>("smpi/send-is-detached-thresh", INT_MAX);
318 xbt_assert(smpi_cfg_async_small_thresh() <= smpi_cfg_detached_send_thresh(),
319 "smpi/async-small-thresh (=%d) should be smaller or equal to smpi/send-is-detached-thresh (=%d)",
320 smpi_cfg_async_small_thresh(),
321 smpi_cfg_detached_send_thresh());
323 if (simgrid::config::is_default("smpi/host-speed") && not MC_is_active()) {
324 XBT_INFO("You did not set the power of the host running the simulation. "
325 "The timings will certainly not be accurate. "
326 "Use the option \"--cfg=smpi/host-speed:<flops>\" to set its value. "
328 "https://simgrid.org/doc/latest/Configuring_SimGrid.html#automatic-benchmarking-of-smpi-code for more "
332 simgrid::smpi::colls::set_collectives();
333 simgrid::smpi::colls::smpi_coll_cleanup_callback = nullptr;