1 /* Copyright (c) 2007-2023. The SimGrid Team. All rights reserved. */
3 /* This program is free software; you can redistribute it and/or modify it
4 * under the terms of the license (GNU LGPL) which comes with this package. */
6 #include "src/mc/remote/CheckerSide.hpp"
7 #include "src/mc/explo/Exploration.hpp"
8 #include "xbt/config.hpp"
9 #include "xbt/system_error.hpp"
11 #if SIMGRID_HAVE_STATEFUL_MC
12 #include "src/mc/explo/LivenessChecker.hpp"
13 #include "src/mc/sosp/RemoteProcessMemory.hpp"
17 #include <sys/prctl.h>
20 #include <boost/tokenizer.hpp>
23 #include <sys/ptrace.h>
27 #define WAITPID_CHECKED_FLAGS __WALL
29 #define WAITPID_CHECKED_FLAGS 0
32 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(mc_checkerside, mc, "MC communication with the application");
34 static simgrid::config::Flag<std::string> _sg_mc_setenv{
35 "model-check/setenv", "Extra environment variables to pass to the child process (ex: 'AZE=aze;QWE=qwe').", "",
36 [](std::string_view value) {
37 xbt_assert(value.empty() || value.find('=', 0) != std::string_view::npos,
38 "The 'model-check/setenv' parameter must be like 'AZE=aze', but it does not contain an equal sign.");
41 namespace simgrid::mc {
43 XBT_ATTRIB_NORETURN static void run_child_process(int socket, const std::vector<char*>& args, bool need_ptrace)
45 /* On startup, simix_global_init() calls simgrid::mc::Client::initialize(), which checks whether the MC_ENV_SOCKET_FD
46 * env variable is set. If so, MC mode is assumed, and the client is setup from its side
50 // Make sure we do not outlive our parent
53 xbt_assert(sigprocmask(SIG_SETMASK, &mask, nullptr) >= 0, "Could not unblock signals");
54 xbt_assert(prctl(PR_SET_PDEATHSIG, SIGHUP) == 0, "Could not PR_SET_PDEATHSIG");
57 setenv(MC_ENV_SOCKET_FD, std::to_string(socket).c_str(), 1);
59 setenv("MC_NEED_PTRACE", "1", 1);
61 /* Setup the tokenizer that parses the cfg:model-check/setenv parameter */
62 using Tokenizer = boost::tokenizer<boost::char_separator<char>>;
63 boost::char_separator<char> semicol_sep(";");
64 boost::char_separator<char> equal_sep("=");
65 Tokenizer token_vars(_sg_mc_setenv.get(), semicol_sep); /* Iterate over all FOO=foo parts */
66 for (const auto& token : token_vars) {
67 std::vector<std::string> kv;
68 Tokenizer token_kv(token, equal_sep);
69 for (const auto& t : token_kv) /* Iterate over 'FOO' and then 'foo' in that 'FOO=foo' */
71 xbt_assert(kv.size() == 2, "Parse error on 'model-check/setenv' value %s. Does it contain an equal sign?",
73 XBT_INFO("setenv '%s'='%s'", kv[0].c_str(), kv[1].c_str());
74 setenv(kv[0].c_str(), kv[1].c_str(), 1);
77 /* And now, exec the child process */
79 while (args[i] != nullptr && args[i][0] == '-')
82 xbt_assert(args[i] != nullptr,
83 "Unable to find a binary to exec on the command line. Did you only pass config flags?");
85 execvp(args[i], args.data() + i);
86 XBT_CRITICAL("The model-checked process failed to exec(%s): %s.\n"
87 " Make sure that your binary exists on disk and is executable.",
88 args[i], strerror(errno));
89 if (strchr(args[i], '=') != nullptr)
90 XBT_CRITICAL("If you want to pass environment variables to the application, please use --cfg=model-check/setenv:%s",
93 xbt_die("Aborting now.");
96 static void wait_application_process(pid_t pid)
98 XBT_DEBUG("Waiting for the model-checked process");
101 // The model-checked process SIGSTOP itself to signal it's ready:
102 xbt_assert(waitpid(pid, &status, WAITPID_CHECKED_FLAGS) == pid && WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP,
103 "Could not wait model-checked process");
107 ptrace(PTRACE_SETOPTIONS, pid, nullptr, PTRACE_O_TRACEEXIT);
108 ptrace(PTRACE_CONT, pid, 0, 0);
110 ptrace(PT_CONTINUE, pid, (caddr_t)1, 0);
112 xbt_die("no ptrace equivalent coded for this platform, stateful model-checking is impossible.");
114 xbt_assert(errno == 0,
115 "Ptrace does not seem to be usable in your setup (errno: %d). "
116 "If you run from within a docker, adding `--cap-add SYS_PTRACE` to the docker line may help. "
117 "If it does not help, please report this bug.",
119 XBT_DEBUG("%d ptrace correctly setup.", getpid());
122 void CheckerSide::setup_events(bool socket_only)
124 auto* base = event_base_new();
127 socket_event_ = event_new(
128 base, get_channel().get_socket(), EV_READ | EV_PERSIST,
129 [](evutil_socket_t, short events, void* arg) {
130 auto checker = static_cast<simgrid::mc::CheckerSide*>(arg);
131 if (events == EV_READ) {
134 std::array<char, MC_MESSAGE_LENGTH> buffer;
135 ssize_t size = checker->get_channel().receive(buffer.data(), buffer.size(), MSG_DONTWAIT);
137 XBT_ERROR("Channel::receive failure: %s", strerror(errno));
139 throw simgrid::xbt::errno_error();
142 if (size == 0) // The app closed the socket. It must be dead by now.
143 checker->handle_waitpid();
144 else if (not checker->handle_message(buffer.data(), size)) {
145 checker->break_loop();
148 } while (checker->get_channel().has_pending_data());
150 xbt_die("Unexpected event");
154 event_add(socket_event_, nullptr);
157 signal_event_ = nullptr;
159 signal_event_ = event_new(
160 base, SIGCHLD, EV_SIGNAL | EV_PERSIST,
161 [](evutil_socket_t sig, short events, void* arg) {
162 auto checker = static_cast<simgrid::mc::CheckerSide*>(arg);
163 if (events == EV_SIGNAL) {
165 checker->handle_waitpid();
167 xbt_die("Unexpected signal: %d", sig);
169 xbt_die("Unexpected event");
173 event_add(signal_event_, nullptr);
177 /* When this constructor is called, no other checkerside exists */
178 CheckerSide::CheckerSide(const std::vector<char*>& args, bool need_memory_info) : running_(true)
180 XBT_DEBUG("Create a CheckerSide. Needs_meminfo: %s", need_memory_info ? "YES" : "no");
182 // Create an AF_UNIX socketpair used for exchanging messages between the model-checker process (ancestor)
183 // and the application process (child)
185 xbt_assert(socketpair(AF_UNIX,
187 SOCK_STREAM, /* Mac OSX does not have AF_UNIX + SOCK_SEQPACKET, even if that's faster*/
192 "Could not create socketpair: %s", strerror(errno));
195 xbt_assert(pid_ >= 0, "Could not fork application process");
197 if (pid_ == 0) { // Child
199 run_child_process(sockets[0], args, need_memory_info); // We need ptrace if we need the mem info
203 // Parent (model-checker):
205 channel_.reset_socket(sockets[1]);
207 setup_events(false); /* we need a signal handler too */
208 if (need_memory_info) {
209 #if SIMGRID_HAVE_STATEFUL_MC
210 // setup ptrace and sync with the app
211 wait_application_process(pid_);
213 // Request the initial memory on need
214 channel_.send(MessageType::NEED_MEMINFO);
215 s_mc_message_need_meminfo_reply_t answer;
216 ssize_t answer_size = channel_.receive(answer);
217 xbt_assert(answer_size != -1, "Could not receive message");
218 xbt_assert(answer.type == MessageType::NEED_MEMINFO_REPLY,
219 "The received message is not the NEED_MEMINFO_REPLY I was expecting but of type %s",
220 to_c_str(answer.type));
221 xbt_assert(answer_size == sizeof answer, "Broken message (size=%zd; expected %zu)", answer_size, sizeof answer);
223 /* We now have enough info to create the memory address space */
224 remote_memory_ = std::make_unique<simgrid::mc::RemoteProcessMemory>(pid_, answer.mmalloc_default_mdp);
226 xbt_die("Cannot introspect memory without MC support");
233 CheckerSide::~CheckerSide()
235 event_del(socket_event_);
236 event_free(socket_event_);
237 if (signal_event_ != nullptr) {
238 event_del(signal_event_);
239 event_free(signal_event_);
243 /* This constructor is called when cloning a checkerside to get its application to fork away */
244 CheckerSide::CheckerSide(int socket, CheckerSide* child_checker)
245 : channel_(socket, child_checker->channel_), running_(true), child_checker_(child_checker)
247 setup_events(true); // We already have a signal handled in that case
249 s_mc_message_int_t answer;
250 ssize_t s = get_channel().receive(answer);
251 xbt_assert(s != -1, "Could not receive answer to FORK_REPLY");
252 xbt_assert(s == sizeof answer, "Broken message (size=%zd; expected %zu)", s, sizeof answer);
253 xbt_assert(answer.type == MessageType::FORK_REPLY,
254 "Received unexpected message %s (%i); expected MessageType::FORK_REPLY (%i)", to_c_str(answer.type),
255 (int)answer.type, (int)MessageType::FORK_REPLY);
261 std::unique_ptr<CheckerSide> CheckerSide::clone(int master_socket)
263 s_mc_message_int_t m = {};
264 m.type = MessageType::FORK;
266 xbt_assert(get_channel().send(m) == 0, "Could not ask the app to fork on need.");
268 int sock = accept(master_socket, nullptr /* I know who's connecting*/, nullptr);
269 xbt_assert(sock > 0, "Cannot accept the incomming connection of the forked app: %s.", strerror(errno));
271 return std::make_unique<CheckerSide>(sock, this);
274 void CheckerSide::finalize(bool terminate_asap)
276 s_mc_message_int_t m = {};
277 m.type = MessageType::FINALIZE;
278 m.value = terminate_asap;
279 xbt_assert(get_channel().send(m) == 0, "Could not ask the app to finalize on need");
281 s_mc_message_t answer;
282 ssize_t s = get_channel().receive(answer);
283 xbt_assert(s != -1, "Could not receive answer to FINALIZE");
284 xbt_assert(s == sizeof answer, "Broken message (size=%zd; expected %zu)", s, sizeof answer);
285 xbt_assert(answer.type == MessageType::FINALIZE_REPLY,
286 "Received unexpected message %s (%i); expected MessageType::FINALIZE_REPLY (%i)", to_c_str(answer.type),
287 (int)answer.type, (int)MessageType::FINALIZE_REPLY);
290 void CheckerSide::dispatch_events()
292 event_base_dispatch(base_.get());
295 void CheckerSide::break_loop() const
297 event_base_loopbreak(base_.get());
300 bool CheckerSide::handle_message(const char* buffer, ssize_t size)
302 s_mc_message_t base_message;
304 xbt_assert(size >= (ssize_t)sizeof(base_message), "Broken message. Got only %ld bytes.", size);
305 memcpy(&base_message, buffer, sizeof(base_message));
307 switch (base_message.type) {
308 case MessageType::IGNORE_HEAP: {
309 consumed = sizeof(s_mc_message_ignore_heap_t);
310 #if SIMGRID_HAVE_STATEFUL_MC
311 if (remote_memory_ != nullptr) {
312 s_mc_message_ignore_heap_t message;
313 xbt_assert(size >= static_cast<ssize_t>(sizeof(message)), "Broken message");
314 memcpy(&message, buffer, sizeof(message));
316 IgnoredHeapRegion region;
317 region.block = message.block;
318 region.fragment = message.fragment;
319 region.address = message.address;
320 region.size = message.size;
321 get_remote_memory()->ignore_heap(region);
324 XBT_INFO("Ignoring a IGNORE_HEAP message because we don't need to introspect memory.");
328 case MessageType::UNIGNORE_HEAP: {
329 consumed = sizeof(s_mc_message_ignore_memory_t);
330 #if SIMGRID_HAVE_STATEFUL_MC
331 if (remote_memory_ != nullptr) {
332 s_mc_message_ignore_memory_t message;
333 xbt_assert(size == static_cast<ssize_t>(sizeof(message)), "Broken message");
334 memcpy(&message, buffer, sizeof(message));
335 get_remote_memory()->unignore_heap((void*)message.addr, message.size);
338 XBT_INFO("Ignoring an UNIGNORE_HEAP message because we don't need to introspect memory.");
342 case MessageType::IGNORE_MEMORY: {
343 consumed = sizeof(s_mc_message_ignore_memory_t);
344 #if SIMGRID_HAVE_STATEFUL_MC
345 if (remote_memory_ != nullptr) {
346 s_mc_message_ignore_memory_t message;
347 xbt_assert(size >= static_cast<ssize_t>(sizeof(message)), "Broken message");
348 memcpy(&message, buffer, sizeof(message));
349 get_remote_memory()->ignore_region(message.addr, message.size);
352 XBT_INFO("Ignoring an IGNORE_MEMORY message because we don't need to introspect memory.");
356 case MessageType::STACK_REGION: {
357 consumed = sizeof(s_mc_message_stack_region_t);
358 #if SIMGRID_HAVE_STATEFUL_MC
359 if (remote_memory_ != nullptr) {
360 s_mc_message_stack_region_t message;
361 xbt_assert(size >= static_cast<ssize_t>(sizeof(message)), "Broken message");
362 memcpy(&message, buffer, sizeof(message));
363 get_remote_memory()->stack_areas().push_back(message.stack_region);
366 XBT_INFO("Ignoring an STACK_REGION message because we don't need to introspect memory.");
370 case MessageType::REGISTER_SYMBOL: {
371 consumed = sizeof(s_mc_message_register_symbol_t);
372 #if SIMGRID_HAVE_STATEFUL_MC
373 s_mc_message_register_symbol_t message;
374 xbt_assert(size >= static_cast<ssize_t>(sizeof(message)), "Broken message");
375 memcpy(&message, buffer, sizeof(message));
376 xbt_assert(not message.callback, "Support for client-side function proposition is not implemented.");
377 XBT_DEBUG("Received symbol: %s", message.name.data());
379 LivenessChecker::automaton_register_symbol(*get_remote_memory(), message.name.data(), remote((int*)message.data));
381 xbt_die("Please don't use liveness properties when MC is compiled out.");
386 case MessageType::WAITING:
387 consumed = sizeof(s_mc_message_t);
388 if (size > consumed) {
389 XBT_DEBUG("%d reinject %d bytes after a %s message", getpid(), (int)(size - consumed),
390 to_c_str(base_message.type));
391 channel_.reinject(&buffer[consumed], size - consumed);
396 case MessageType::ASSERTION_FAILED:
397 consumed = sizeof(s_mc_message_t);
398 Exploration::get_instance()->report_assertion_failure();
402 xbt_die("Unexpected message from the application");
404 if (size > consumed) {
405 XBT_DEBUG("%d reinject %d bytes after a %s message", getpid(), (int)(size - consumed), to_c_str(base_message.type));
406 channel_.reinject(&buffer[consumed], size - consumed);
411 void CheckerSide::wait_for_requests()
413 XBT_DEBUG("Resume the application");
414 if (get_channel().send(MessageType::CONTINUE) != 0)
415 throw xbt::errno_error();
416 clear_memory_cache();
422 void CheckerSide::clear_memory_cache()
424 #if SIMGRID_HAVE_STATEFUL_MC
426 remote_memory_->clear_cache();
430 void CheckerSide::handle_dead_child(int status)
432 // From PTRACE_O_TRACEEXIT:
434 if (status >> 8 == (SIGTRAP | (PTRACE_EVENT_EXIT << 8))) {
435 unsigned long eventmsg;
436 xbt_assert(ptrace(PTRACE_GETEVENTMSG, pid_, 0, &eventmsg) != -1, "Could not get exit status");
437 status = static_cast<int>(eventmsg);
438 if (WIFSIGNALED(status)) {
440 Exploration::get_instance()->report_crash(status);
445 // We don't care about non-lethal signals, just reinject them:
446 if (WIFSTOPPED(status)) {
447 XBT_DEBUG("Stopped with signal %i", (int)WSTOPSIG(status));
450 ptrace(PTRACE_CONT, pid_, 0, WSTOPSIG(status));
452 ptrace(PT_CONTINUE, pid_, (caddr_t)1, WSTOPSIG(status));
454 xbt_assert(errno == 0, "Could not PTRACE_CONT: %s", strerror(errno));
457 else if (WIFSIGNALED(status)) {
459 Exploration::get_instance()->report_crash(status);
460 } else if (WIFEXITED(status)) {
461 XBT_DEBUG("Child process is over");
466 void CheckerSide::handle_waitpid()
468 XBT_DEBUG("%d checks for wait event. %s", getpid(),
469 child_checker_ == nullptr ? "Wait directly." : "Ask our proxy to wait for its child.");
471 if (child_checker_ == nullptr) { // Wait directly
474 while ((pid = waitpid(-1, &status, WNOHANG)) != 0) {
476 if (errno == ECHILD) { // No more children:
477 xbt_assert(not this->running(), "Inconsistent state");
480 xbt_die("Could not wait for pid: %s", strerror(errno));
484 if (pid == get_pid())
485 handle_dead_child(status);
488 } else { // Ask our proxy to wait for us
490 s_mc_message_int_t request = {};
491 request.type = MessageType::WAIT_CHILD;
492 request.value = pid_;
493 xbt_assert(child_checker_->get_channel().send(request) == 0,
494 "Could not ask my child to waitpid its child for me: %s", strerror(errno));
496 s_mc_message_int_t answer;
497 ssize_t answer_size = child_checker_->get_channel().receive(answer);
498 xbt_assert(answer_size != -1, "Could not receive message");
499 xbt_assert(answer.type == MessageType::WAIT_CHILD_REPLY,
500 "The received message is not the WAIT_CHILD_REPLY I was expecting but of type %s",
501 to_c_str(answer.type));
502 xbt_assert(answer_size == sizeof answer, "Broken message (size=%zd; expected %zu)", answer_size, sizeof answer);
503 handle_dead_child(answer.value);
506 } // namespace simgrid::mc