1 /* Copyright (c) 2013-2021. The SimGrid Team. All rights reserved. */
3 /* This program is free software; you can redistribute it and/or modify it
4 * under the terms of the license (GNU LGPL) which comes with this package. */
6 #include <simgrid/Exception.hpp>
8 #include "src/plugins/vm/VmLiveMigration.hpp"
9 #include "src/instr/instr_private.hpp"
10 #include "src/plugins/vm/VirtualMachineImpl.hpp"
11 #include "src/plugins/vm/VmHostExt.hpp"
13 XBT_LOG_NEW_DEFAULT_SUBCATEGORY(vm_live_migration, s4u, "S4U virtual machines live migration");
17 xbt::Extension<s4u::Host, VmMigrationExt> VmMigrationExt::EXTENSION_ID;
19 void VmMigrationExt::ensureVmMigrationExtInstalled()
21 if (not EXTENSION_ID.valid())
22 EXTENSION_ID = s4u::Host::extension_create<VmMigrationExt>();
25 void MigrationRx::operator()()
27 XBT_DEBUG("mig: rx_start");
28 bool received_finalize = false;
30 std::string finalize_task_name =
31 std::string("__mig_stage3:") + vm_->get_cname() + "(" + src_pm_->get_cname() + "-" + dst_pm_->get_cname() + ")";
33 while (not received_finalize) {
34 auto payload = mbox->get_unique<std::string>();
36 if (finalize_task_name == *payload)
37 received_finalize = true;
40 // Here Stage 1, 2 and 3 have been performed.
41 // Hence complete the migration
43 /* Update the vm location */
44 /* precopy migration makes the VM temporally paused */
45 xbt_assert(vm_->get_state() == s4u::VirtualMachine::State::SUSPENDED);
47 /* Update the vm location and resume it */
51 // Now the VM is running on the new host (the migration is completed) (even if the SRC crash)
52 vm_->get_vm_impl()->end_migration();
53 XBT_DEBUG("VM(%s) moved from PM(%s) to PM(%s)", vm_->get_cname(), src_pm_->get_cname(), dst_pm_->get_cname());
55 if (TRACE_vm_is_enabled()) {
56 static long long int counter = 0;
57 std::string key = std::to_string(counter);
61 auto* msg = instr::Container::by_name(vm_->get_name());
62 instr::Container::get_root()->get_link("VM_LINK")->start_event(msg, "M", key);
64 // destroy existing container of this vm
65 instr::Container::by_name(vm_->get_name())->remove_from_parent();
67 // create new container on the new_host location
68 new instr::Container(vm_->get_name(), "VM", instr::Container::by_name(dst_pm_->get_name()));
71 msg = instr::Container::by_name(vm_->get_name());
72 instr::Container::get_root()->get_link("VM_LINK")->end_event(msg, "M", key);
74 // Inform the SRC that the migration has been correctly performed
75 auto* payload = new std::string("__mig_stage4:");
76 *payload = *payload + vm_->get_cname() + "(" + src_pm_->get_cname() + "-" + dst_pm_->get_cname() + ")";
78 mbox_ctl->put(payload, 0);
80 XBT_DEBUG("mig: rx_done");
83 static sg_size_t get_updated_size(double computed, double dp_rate, sg_size_t dp_cap)
85 auto updated_size = static_cast<sg_size_t>(computed * dp_rate);
86 XBT_DEBUG("updated_size %llu dp_rate %f", updated_size, dp_rate);
87 if (updated_size > dp_cap) {
88 updated_size = dp_cap;
94 sg_size_t MigrationTx::sendMigrationData(sg_size_t size, int stage, int stage2_round, double mig_speed, double timeout)
96 sg_size_t sent = size;
97 auto* msg = new std::string("__mig_stage");
98 *msg = *msg + std::to_string(stage) + ":" + vm_->get_cname() + "(" + src_pm_->get_cname() + "-" +
99 dst_pm_->get_cname() + ")";
101 double clock_sta = s4u::Engine::get_clock();
103 s4u::CommPtr comm = mbox->put_init(msg, size);
105 comm->set_rate(mig_speed);
107 comm->wait_for(timeout);
108 } catch (const Exception&) {
109 auto remaining = static_cast<sg_size_t>(comm->get_remaining());
110 XBT_VERB("timeout (%lf s) in sending_migration_data, remaining %llu bytes of %llu", timeout, remaining, size);
115 double clock_end = s4u::Engine::get_clock();
116 double duration = clock_end - clock_sta;
117 double actual_speed = static_cast<double>(size) / duration;
120 XBT_DEBUG("mig-stage%d.%d: sent %llu duration %f actual_speed %f (target %f)", stage, stage2_round, size, duration,
121 actual_speed, mig_speed);
123 XBT_DEBUG("mig-stage%d: sent %llu duration %f actual_speed %f (target %f)", stage, size, duration, actual_speed,
129 void MigrationTx::operator()()
131 XBT_DEBUG("mig: tx_start");
133 double host_speed = vm_->get_pm()->get_speed();
134 const sg_size_t ramsize = vm_->get_ramsize();
135 const double dp_rate =
136 host_speed != 0.0 ? (sg_vm_get_migration_speed(vm_) * sg_vm_get_dirty_page_intensity(vm_)) / host_speed : 1;
137 const sg_size_t dp_cap = sg_vm_get_working_set_memory(vm_);
138 const double mig_speed = sg_vm_get_migration_speed(vm_);
139 double max_downtime = sg_vm_get_max_downtime(vm_);
141 double mig_timeout = 10000000.0;
142 bool skip_stage2 = false;
144 size_t remaining_size = ramsize;
146 double clock_prev_send;
147 double clock_post_send;
151 /* check parameters */
153 XBT_WARN("migrate a VM, but ramsize is zero");
155 if (max_downtime <= 0) {
156 XBT_WARN("use the default max_downtime value 30ms");
160 /* Stage1: send all memory pages to the destination. */
161 XBT_DEBUG("mig-stage1: remaining_size %zu", remaining_size);
162 sg_vm_start_dirty_page_tracking(vm_);
164 double computed_during_stage1 = 0;
165 clock_prev_send = s4u::Engine::get_clock();
168 /* At stage 1, we do not need timeout. We have to send all the memory pages even though the duration of this
169 * transfer exceeds the timeout value. */
170 XBT_VERB("Stage 1: Gonna send %llu bytes", ramsize);
171 sg_size_t sent = sendMigrationData(ramsize, 1, 0, mig_speed, -1);
172 remaining_size -= sent;
173 computed_during_stage1 = sg_vm_lookup_computed_flops(vm_);
175 if (sent < ramsize) {
176 XBT_VERB("mig-stage1: timeout, force moving to stage 3");
178 } else if (sent > ramsize)
180 } catch (const Exception&) {
181 // hostfailure (if you want to know whether this is the SRC or the DST check directly in send_migration_data code)
182 // Stop the dirty page tracking and return (there is no memory space to release)
183 sg_vm_stop_dirty_page_tracking(vm_);
187 clock_post_send = s4u::Engine::get_clock();
188 mig_timeout -= (clock_post_send - clock_prev_send);
189 if (mig_timeout < 0) {
190 XBT_VERB("The duration of stage 1 exceeds the timeout value, skip stage 2");
194 /* estimate bandwidth */
195 bandwidth = ramsize / (clock_post_send - clock_prev_send);
196 threshold = bandwidth * max_downtime;
197 XBT_DEBUG("actual bandwidth %f (MB/s), threshold %zu", bandwidth / 1024 / 1024, threshold);
199 /* Stage2: send update pages iteratively until the size of remaining states becomes smaller than threshold value. */
200 if (not skip_stage2) {
201 int stage2_round = 0;
202 /* just after stage1, nothing has been updated. But, we have to send the data updated during stage1 */
203 sg_size_t updated_size = get_updated_size(computed_during_stage1, dp_rate, dp_cap);
204 remaining_size += updated_size;
205 XBT_DEBUG("mig-stage2.%d: remaining_size %zu (%s threshold %zu)", stage2_round, remaining_size,
206 (remaining_size < threshold) ? "<" : ">", threshold);
208 /* When the remaining size is below the threshold value, move to stage 3. */
209 while (threshold < remaining_size) {
210 XBT_DEBUG("mig-stage 2:%d updated_size %llu computed_during_stage1 %f dp_rate %f dp_cap %llu", stage2_round,
211 updated_size, computed_during_stage1, dp_rate, dp_cap);
214 clock_prev_send = s4u::Engine::get_clock();
216 XBT_DEBUG("Stage 2, gonna send %llu", updated_size);
217 sent = sendMigrationData(updated_size, 2, stage2_round, mig_speed, mig_timeout);
218 } catch (const Exception&) {
219 // hostfailure (if you want to know whether this is the SRC or the DST check directly in send_migration_data
221 // Stop the dirty page tracking and return (there is no memory space to release)
222 sg_vm_stop_dirty_page_tracking(vm_);
226 remaining_size -= sent;
227 double computed = sg_vm_lookup_computed_flops(vm_);
229 clock_post_send = s4u::Engine::get_clock();
231 if (sent == updated_size) {
232 bandwidth = updated_size / (clock_post_send - clock_prev_send);
233 threshold = bandwidth * max_downtime;
234 XBT_DEBUG("actual bandwidth %f, threshold %zu", bandwidth / 1024 / 1024, threshold);
236 mig_timeout -= (clock_post_send - clock_prev_send);
237 xbt_assert(mig_timeout > 0);
238 XBT_DEBUG("mig-stage2.%d: remaining_size %zu (%s threshold %zu)", stage2_round, remaining_size,
239 (remaining_size < threshold) ? "<" : ">", threshold);
240 updated_size = get_updated_size(computed, dp_rate, dp_cap);
241 remaining_size += updated_size;
243 /* When timeout happens, we move to stage 3. The size of memory pages
244 * updated before timeout must be added to the remaining size. */
245 XBT_VERB("mig-stage2.%d: timeout, force moving to stage 3. sent %llu / %llu, eta %lf", stage2_round, sent,
246 updated_size, (clock_post_send - clock_prev_send));
247 updated_size = get_updated_size(computed, dp_rate, dp_cap);
248 remaining_size += updated_size;
254 /* Stage3: stop the VM and copy the rest of states. */
255 XBT_DEBUG("mig-stage3: remaining_size %zu", remaining_size);
257 sg_vm_stop_dirty_page_tracking(vm_);
260 XBT_DEBUG("Stage 3: Gonna send %zu bytes", remaining_size);
261 sendMigrationData(remaining_size, 3, 0, mig_speed, -1);
262 } catch (const Exception&) {
263 // hostfailure (if you want to know whether this is the SRC or the DST check directly in send_migration_data code)
264 // Stop the dirty page tracking and return (there is no memory space to release)
269 // At that point the Migration is considered valid for the SRC node but remind that the DST side should relocate
270 // effectively the VM on the DST node.
271 XBT_DEBUG("mig: tx_done");
274 } // namespace simgrid
276 static void onVirtualMachineShutdown(simgrid::s4u::VirtualMachine const& vm)
278 if (vm.get_vm_impl()->is_migrating()) {
279 vm.extension<simgrid::vm::VmMigrationExt>()->rx_->kill();
280 vm.extension<simgrid::vm::VmMigrationExt>()->tx_->kill();
281 vm.extension<simgrid::vm::VmMigrationExt>()->issuer_->kill();
282 vm.get_vm_impl()->end_migration();
286 void sg_vm_live_migration_plugin_init()
288 sg_vm_dirty_page_tracking_init();
289 simgrid::vm::VmMigrationExt::ensureVmMigrationExtInstalled();
290 simgrid::s4u::VirtualMachine::on_shutdown.connect(&onVirtualMachineShutdown);
293 simgrid::s4u::VirtualMachine* sg_vm_create_migratable(simgrid::s4u::Host* pm, const char* name, int coreAmount,
294 int ramsize, int mig_netspeed, int dp_intensity)
296 simgrid::vm::VmHostExt::ensureVmExtInstalled();
298 /* For the moment, intensity_rate is the percentage against the migration bandwidth */
300 auto* vm = new simgrid::s4u::VirtualMachine(name, pm, coreAmount, static_cast<sg_size_t>(ramsize) * 1024 * 1024);
301 sg_vm_set_dirty_page_intensity(vm, dp_intensity / 100.0);
302 sg_vm_set_working_set_memory(vm, vm->get_ramsize() * 0.9); // assume working set memory is 90% of ramsize
303 sg_vm_set_migration_speed(vm, mig_netspeed * 1024 * 1024.0);
305 XBT_DEBUG("migspeed : %f intensity mem : %d", mig_netspeed * 1024 * 1024.0, dp_intensity);
310 int sg_vm_is_migrating(const simgrid::s4u::VirtualMachine* vm)
312 return vm->get_vm_impl()->is_migrating();
315 void sg_vm_migrate(simgrid::s4u::VirtualMachine* vm, simgrid::s4u::Host* dst_pm)
317 simgrid::s4u::Host* src_pm = vm->get_pm();
319 if (not src_pm->is_on())
320 throw simgrid::VmFailureException(
321 XBT_THROW_POINT, simgrid::xbt::string_printf("Cannot migrate VM '%s' from host '%s', which is offline.",
322 vm->get_cname(), src_pm->get_cname()));
323 if (not dst_pm->is_on())
324 throw simgrid::VmFailureException(
325 XBT_THROW_POINT, simgrid::xbt::string_printf("Cannot migrate VM '%s' to host '%s', which is offline.",
326 vm->get_cname(), dst_pm->get_cname()));
327 if (vm->get_state() != simgrid::s4u::VirtualMachine::State::RUNNING)
328 throw simgrid::VmFailureException(
330 simgrid::xbt::string_printf("Cannot migrate VM '%s' that is not running yet.", vm->get_cname()));
331 if (vm->get_vm_impl()->is_migrating())
332 throw simgrid::VmFailureException(
334 simgrid::xbt::string_printf("Cannot migrate VM '%s' that is already migrating.", vm->get_cname()));
336 vm->get_vm_impl()->start_migration();
337 simgrid::s4u::VirtualMachine::on_migration_start(*vm);
339 std::string rx_name =
340 std::string("__pr_mig_rx:") + vm->get_cname() + "(" + src_pm->get_cname() + "-" + dst_pm->get_cname() + ")";
341 std::string tx_name =
342 std::string("__pr_mig_tx:") + vm->get_cname() + "(" + src_pm->get_cname() + "-" + dst_pm->get_cname() + ")";
344 simgrid::s4u::ActorPtr rx =
345 simgrid::s4u::Actor::create(rx_name.c_str(), dst_pm, simgrid::vm::MigrationRx(vm, dst_pm));
346 simgrid::s4u::ActorPtr tx =
347 simgrid::s4u::Actor::create(tx_name.c_str(), src_pm, simgrid::vm::MigrationTx(vm, dst_pm));
349 vm->extension_set<simgrid::vm::VmMigrationExt>(new simgrid::vm::VmMigrationExt(simgrid::s4u::Actor::self(), rx, tx));
351 /* wait until the migration have finished or on error has occurred */
352 XBT_DEBUG("wait for reception of the final ACK (i.e. migration has been correctly performed");
353 simgrid::s4u::Mailbox* mbox_ctl = simgrid::s4u::Mailbox::by_name(
354 std::string("__mbox_mig_ctl:") + vm->get_cname() + "(" + src_pm->get_cname() + "-" + dst_pm->get_cname() + ")");
355 mbox_ctl->get_unique<std::string>();
359 vm->get_vm_impl()->end_migration();
360 simgrid::s4u::VirtualMachine::on_migration_end(*vm);