From 8952dc42246e5200e94a17224430664ddf97d526 Mon Sep 17 00:00:00 2001 From: SUTER Frederic Date: Sun, 2 Jan 2022 14:14:26 +0100 Subject: [PATCH] convert the last two simdag examples. Simdag can die --- MANIFEST.in | 17 +- examples/cpp/CMakeLists.txt | 9 +- examples/cpp/dag-comm/s4u-dag-comm.tesh | 4 +- examples/cpp/dag-failure/s4u-dag-failure.tesh | 4 +- .../cpp/dag-from-dax/s4u-dag-from-dax.cpp | 79 +++++ .../cpp/dag-from-dax/s4u-dag-from-dax.tesh | 37 +++ .../dag-from-dax}/simple_dax_with_cycle.xml | 0 .../daxload => cpp/dag-from-dax}/smalldax.xml | 0 examples/cpp/dag-io/s4u-dag-io.tesh | 2 +- .../dag-scheduling}/Montage_25.xml | 0 .../dag-scheduling}/expected_output.jed | 0 .../cpp/dag-scheduling/s4u-dag-scheduling.cpp | 292 ++++++++++++++++++ .../dag-scheduling/s4u-dag-scheduling.tesh | 35 +++ examples/cpp/dag-simple/s4u-dag-simple.tesh | 4 +- examples/deprecated/simdag/CMakeLists.txt | 19 -- .../deprecated/simdag/daxload/sd_daxload.c | 113 ------- .../deprecated/simdag/daxload/sd_daxload.tesh | 136 -------- .../simdag/scheduling/sd_scheduling.c | 257 --------------- .../simdag/scheduling/sd_scheduling.tesh | 39 --- include/simgrid/s4u/Activity.hpp | 8 +- include/simgrid/s4u/Engine.hpp | 1 + src/simdag/sd_daxloader.cpp | 251 +++++---------- src/simdag/simdag_private.hpp | 2 - tools/cmake/DefinePackages.cmake | 1 - 24 files changed, 549 insertions(+), 761 deletions(-) create mode 100644 examples/cpp/dag-from-dax/s4u-dag-from-dax.cpp create mode 100644 examples/cpp/dag-from-dax/s4u-dag-from-dax.tesh rename examples/{deprecated/simdag/daxload => cpp/dag-from-dax}/simple_dax_with_cycle.xml (100%) rename examples/{deprecated/simdag/daxload => cpp/dag-from-dax}/smalldax.xml (100%) rename examples/{deprecated/simdag/scheduling => cpp/dag-scheduling}/Montage_25.xml (100%) rename examples/{deprecated/simdag/scheduling => cpp/dag-scheduling}/expected_output.jed (100%) create mode 100644 examples/cpp/dag-scheduling/s4u-dag-scheduling.cpp create mode 100644 examples/cpp/dag-scheduling/s4u-dag-scheduling.tesh delete mode 100644 examples/deprecated/simdag/CMakeLists.txt delete mode 100644 examples/deprecated/simdag/daxload/sd_daxload.c delete mode 100644 examples/deprecated/simdag/daxload/sd_daxload.tesh delete mode 100644 examples/deprecated/simdag/scheduling/sd_scheduling.c delete mode 100644 examples/deprecated/simdag/scheduling/sd_scheduling.tesh diff --git a/MANIFEST.in b/MANIFEST.in index 57b0fb51fe..e18ca85b9b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -206,12 +206,20 @@ include examples/cpp/dag-comm/s4u-dag-comm.cpp include examples/cpp/dag-comm/s4u-dag-comm.tesh include examples/cpp/dag-failure/s4u-dag-failure.cpp include examples/cpp/dag-failure/s4u-dag-failure.tesh +include examples/cpp/dag-from-dax/s4u-dag-from-dax.cpp +include examples/cpp/dag-from-dax/s4u-dag-from-dax.tesh +include examples/cpp/dag-from-dax/simple_dax_with_cycle.xml +include examples/cpp/dag-from-dax/smalldax.xml include examples/cpp/dag-from-dot/dag.dot include examples/cpp/dag-from-dot/dag_with_cycle.dot include examples/cpp/dag-from-dot/s4u-dag-from-dot.cpp include examples/cpp/dag-from-dot/s4u-dag-from-dot.tesh include examples/cpp/dag-io/s4u-dag-io.cpp include examples/cpp/dag-io/s4u-dag-io.tesh +include examples/cpp/dag-scheduling/Montage_25.xml +include examples/cpp/dag-scheduling/expected_output.jed +include examples/cpp/dag-scheduling/s4u-dag-scheduling.cpp +include examples/cpp/dag-scheduling/s4u-dag-scheduling.tesh include examples/cpp/dag-simple/s4u-dag-simple.cpp include examples/cpp/dag-simple/s4u-dag-simple.tesh include examples/cpp/dht-chord/s4u-dht-chord-node.cpp @@ -492,14 +500,6 @@ include examples/deprecated/msg/mc/centralized_mutex.tesh include examples/deprecated/msg/mc/deploy_bugged2_liveness.xml include examples/deprecated/msg/mc/deploy_centralized_mutex.xml include examples/deprecated/msg/mc/promela_bugged2_liveness -include examples/deprecated/simdag/daxload/sd_daxload.c -include examples/deprecated/simdag/daxload/sd_daxload.tesh -include examples/deprecated/simdag/daxload/simple_dax_with_cycle.xml -include examples/deprecated/simdag/daxload/smalldax.xml -include examples/deprecated/simdag/scheduling/Montage_25.xml -include examples/deprecated/simdag/scheduling/expected_output.jed -include examples/deprecated/simdag/scheduling/sd_scheduling.c -include examples/deprecated/simdag/scheduling/sd_scheduling.tesh include examples/python/actor-create/actor-create.py include examples/python/actor-create/actor-create.tesh include examples/python/actor-daemon/actor-daemon.py @@ -1955,7 +1955,6 @@ include examples/c/CMakeLists.txt include examples/cpp/CMakeLists.txt include examples/deprecated/java/CMakeLists.txt include examples/deprecated/msg/mc/CMakeLists.txt -include examples/deprecated/simdag/CMakeLists.txt include examples/platforms/CMakeLists.txt include examples/platforms/bypassRoute.xml include examples/platforms/bypassZoneRoute.xml diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 0c1929a5a1..d0fa3df3c5 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -98,7 +98,7 @@ foreach (example actor-create actor-daemon actor-exiting actor-join actor-kill comm-pingpong comm-ready comm-serialize comm-suspend comm-wait comm-waitany comm-waitall comm-waituntil comm-dependent comm-host2host comm-failure comm-throttling cloud-capping cloud-migration cloud-simple - dag-comm dag-from-dot dag-failure dag-io dag-simple + dag-comm dag-from-dax dag-from-dot dag-failure dag-io dag-scheduling dag-simple dht-chord dht-kademlia energy-exec energy-boot energy-link energy-vm energy-exec-ptask energy-wifi engine-filtering engine-run-partial @@ -232,6 +232,7 @@ set(xml_files ${xml_files} ${CMAKE_CURRENT_SOURCE_DIR}/actor-create/s4u-a ${CMAKE_CURRENT_SOURCE_DIR}/comm-waitall/s4u-comm-waitall_d.xml ${CMAKE_CURRENT_SOURCE_DIR}/comm-ready/s4u-comm-ready_d.xml ${CMAKE_CURRENT_SOURCE_DIR}/comm-waituntil/s4u-comm-waituntil_d.xml + ${CMAKE_CURRENT_SOURCE_DIR}/dag-scheduling/Montage_25.xml ${CMAKE_CURRENT_SOURCE_DIR}/dht-chord/s4u-dht-chord_d.xml ${CMAKE_CURRENT_SOURCE_DIR}/dht-kademlia/s4u-dht-kademlia_d.xml ${CMAKE_CURRENT_SOURCE_DIR}/energy-boot/platform_boot.xml @@ -251,10 +252,12 @@ set(xml_files ${xml_files} ${CMAKE_CURRENT_SOURCE_DIR}/actor-create/s4u-a set(bin_files ${bin_files} ${CMAKE_CURRENT_SOURCE_DIR}/dht-kademlia/generate.py ${CMAKE_CURRENT_SOURCE_DIR}/mc-bugged1-liveness/s4u-mc-bugged1-liveness-stack-cleaner ${CMAKE_CURRENT_SOURCE_DIR}/mc-bugged1-liveness/promela_bugged1_liveness PARENT_SCOPE) -set(txt_files ${txt_files} ${CMAKE_CURRENT_SOURCE_DIR}/dag-from-dot/dag.dot +set(txt_files ${txt_files} ${CMAKE_CURRENT_SOURCE_DIR}/dag-from-dax/simple_dax_with_cycle.xml + ${CMAKE_CURRENT_SOURCE_DIR}/dag-from-dax/smalldax.xml + ${CMAKE_CURRENT_SOURCE_DIR}/dag-from-dot/dag.dot ${CMAKE_CURRENT_SOURCE_DIR}/dag-from-dot/dag_with_cycle.dot + ${CMAKE_CURRENT_SOURCE_DIR}/dag-scheduling/expected_output.jed ${CMAKE_CURRENT_SOURCE_DIR}/replay-comm/s4u-replay-comm-split-p0.txt ${CMAKE_CURRENT_SOURCE_DIR}/replay-comm/s4u-replay-comm-split-p1.txt ${CMAKE_CURRENT_SOURCE_DIR}/replay-comm/s4u-replay-comm.txt ${CMAKE_CURRENT_SOURCE_DIR}/replay-io/s4u-replay-io.txt PARENT_SCOPE) - diff --git a/examples/cpp/dag-comm/s4u-dag-comm.tesh b/examples/cpp/dag-comm/s4u-dag-comm.tesh index 8c30e6f2c5..04824f8c7d 100644 --- a/examples/cpp/dag-comm/s4u-dag-comm.tesh +++ b/examples/cpp/dag-comm/s4u-dag-comm.tesh @@ -8,11 +8,11 @@ $ ${bindir:=.}/s4u-dag-comm ${platfdir}/two_hosts.xml --log=s4u_activity.t:verbo > [ 0.000000] (0:maestro@) Activity 'transfert' vetoed. Dependencies: NOT solved; Ressources: NOT assigned > [ 0.000000] (0:maestro@) Activity 'child' vetoed. Dependencies: NOT solved; Ressources: assigned > [ 0.000000] (0:maestro@) Activity 'transfert' vetoed. Dependencies: NOT solved; Ressources: assigned +> [ 1.000000] (0:maestro@) Activity 'parent' is complete (start time: 0.000000, finish time: 1.000000) > [ 1.000000] (0:maestro@) Remove a dependency from 'parent' on 'transfert' > [ 1.000000] (0:maestro@) 'transfert' is assigned to a resource and all dependencies are solved. Let's start -> [ 1.000000] (0:maestro@) Activity 'parent' is complete (start time: 0.000000, finish time: 1.000000) +> [ 2.083775] (0:maestro@) Activity 'transfert' is complete > [ 2.083775] (0:maestro@) Remove a dependency from 'transfert' on 'child' > [ 2.083775] (0:maestro@) 'child' is assigned to a resource and all dependencies are solved. Let's start -> [ 2.083775] (0:maestro@) Activity 'transfert' is complete > [ 3.083775] (0:maestro@) Activity 'child' is complete (start time: 2.083775, finish time: 3.083775) > [ 3.083775] (0:maestro@) Simulation time 3.08378 diff --git a/examples/cpp/dag-failure/s4u-dag-failure.tesh b/examples/cpp/dag-failure/s4u-dag-failure.tesh index b47537d300..9fcb888643 100644 --- a/examples/cpp/dag-failure/s4u-dag-failure.tesh +++ b/examples/cpp/dag-failure/s4u-dag-failure.tesh @@ -13,9 +13,9 @@ $ ${bindir:=.}/s4u-dag-failure ${platfdir}/faulty_host.xml --log=s4u_activity.t: > [ 10.000000] (0:maestro@) let's unschedule Activity 'Poor task' and reschedule it on the 'Safe Host' > [ 10.000000] (0:maestro@) 'Poor task' is assigned to a resource and all dependencies are solved. Let's start > [ 10.000000] (0:maestro@) Run the simulation again +> [ 50.000000] (0:maestro@) Activity 'Poor task' is complete (start time: 10.000000, finish time: 50.000000) > [ 50.000000] (0:maestro@) Remove a dependency from 'Poor task' on 'Child' > [ 50.000000] (0:maestro@) 'Child' is assigned to a resource and all dependencies are solved. Let's start -> [ 50.000000] (0:maestro@) Activity 'Poor task' is complete (start time: 10.000000, finish time: 50.000000) > [ 90.000000] (0:maestro@) Activity 'Child' is complete (start time: 50.000000, finish time: 90.000000) > [ 90.000000] (0:maestro@) Second test: parallel Exec activity > [ 90.000000] (0:maestro@) Schedule Activity 'Poor parallel task' on 'Safe Host' and 'Faulty Host' @@ -25,8 +25,8 @@ $ ${bindir:=.}/s4u-dag-failure ${platfdir}/faulty_host.xml --log=s4u_activity.t: > [100.000000] (0:maestro@) let's unschedule Activity 'Poor parallel task' and reschedule it only on the 'Safe Host' > [100.000000] (0:maestro@) 'Poor parallel task' is assigned to a resource and all dependencies are solved. Let's start > [100.000000] (0:maestro@) Run the simulation again +> [180.000000] (0:maestro@) Activity 'Poor parallel task' is complete (start time: 100.000000, finish time: 180.000000) > [180.000000] (0:maestro@) Remove a dependency from 'Poor parallel task' on 'Child' > [180.000000] (0:maestro@) 'Child' is assigned to a resource and all dependencies are solved. Let's start -> [180.000000] (0:maestro@) Activity 'Poor parallel task' is complete (start time: 100.000000, finish time: 180.000000) > [220.000000] (0:maestro@) Activity 'Child' is complete (start time: 180.000000, finish time: 220.000000) diff --git a/examples/cpp/dag-from-dax/s4u-dag-from-dax.cpp b/examples/cpp/dag-from-dax/s4u-dag-from-dax.cpp new file mode 100644 index 0000000000..54c27ce6aa --- /dev/null +++ b/examples/cpp/dag-from-dax/s4u-dag-from-dax.cpp @@ -0,0 +1,79 @@ +/* simple test trying to load a DAX file. */ + +/* Copyright (c) 2009-2021. The SimGrid Team. + * All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#include "simgrid/s4u.hpp" + +#include +#include + +XBT_LOG_NEW_DEFAULT_CATEGORY(dag_from_dax, "Logging specific to this example"); + +int main(int argc, char** argv) +{ + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + + std::vector dag = simgrid::s4u::create_DAG_from_DAX(argv[2]); + + if (dag.empty()) { + XBT_ERROR("A problem occurred during DAX parsing (cycle or syntax). Do not continue this test"); + exit(2); + } + + XBT_INFO("--------- Display all activities of the loaded DAG -----------"); + for (const auto& a : dag) { + std::string type = "an Exec"; + std::string task = "flops to execute"; + if (dynamic_cast(a.get()) != nullptr) { + type = "a Comm"; + task = "bytes to transfer"; + } + XBT_INFO("'%s' is %s: %.0f %s. Dependencies: %s; Ressources: %s", a->get_cname(), type.c_str(), a->get_remaining(), + task.c_str(), (a->dependencies_solved() ? "solved" : "NOT solved"), + (a->is_assigned() ? "assigned" : "NOT assigned")); + } + + XBT_INFO("------------------- Schedule tasks ---------------------------"); + auto hosts = e.get_all_hosts(); + auto count = e.get_host_count(); + int cursor = 0; + // Schedule end first + static_cast(dag.back().get())->set_host(hosts[0]); + + for (const auto& a : dag) { + auto* exec = dynamic_cast(a.get()); + if (exec != nullptr && exec->get_name() != "end") { + exec->set_host(hosts[cursor % count]); + cursor++; + } + auto* comm = dynamic_cast(a.get()); + if (comm != nullptr) { + auto pred = dynamic_cast((*comm->get_dependencies().begin()).get()); + auto succ = dynamic_cast(comm->get_successors().front().get()); + comm->set_source(pred->get_host())->set_destination(succ->get_host()); + } + } + + XBT_INFO("------------------- Run the schedule -------------------------"); + e.run(); + + XBT_INFO("-------------- Summary of executed schedule ------------------"); + for (const auto& a : dag) { + const auto* exec = dynamic_cast(a.get()); + if (exec != nullptr) { + XBT_INFO("[%f->%f] '%s' executed on %s", exec->get_start_time(), exec->get_finish_time(), exec->get_cname(), + exec->get_host()->get_cname()); + } + const auto* comm = dynamic_cast(a.get()); + if (comm != nullptr) { + XBT_INFO("[%f->%f] '%s' transferred from %s to %s", comm->get_start_time(), comm->get_finish_time(), + comm->get_cname(), comm->get_source()->get_cname(), comm->get_destination()->get_cname()); + } + } + return 0; +} diff --git a/examples/cpp/dag-from-dax/s4u-dag-from-dax.tesh b/examples/cpp/dag-from-dax/s4u-dag-from-dax.tesh new file mode 100644 index 0000000000..34705f95a7 --- /dev/null +++ b/examples/cpp/dag-from-dax/s4u-dag-from-dax.tesh @@ -0,0 +1,37 @@ +#!/usr/bin/env tesh +p Test the DAX loader on a small DAX instance + +$ ${bindir:=.}/s4u-dag-from-dax --log=no_loc ${platfdir}/cluster_backbone.xml ${srcdir:=.}/smalldax.xml +> [0.000000] [sd_daxparse/WARNING] Ignore file o1 size redefinition from 1000000 to 304 +> [0.000000] [sd_daxparse/WARNING] Ignore file o2 size redefinition from 1000000 to 304 +> [0.000000] [dag_from_dax/INFO] --------- Display all activities of the loaded DAG ----------- +> [0.000000] [dag_from_dax/INFO] 'root' is an Exec: 0 flops to execute. Dependencies: solved; Ressources: NOT assigned +> [0.000000] [dag_from_dax/INFO] '1@task1' is an Exec: 42000000000 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dax/INFO] '2@task2' is an Exec: 42000000000 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dax/INFO] '3@task1' is an Exec: 42000000000 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dax/INFO] 'root_i1_1@task1' is a Comm: 1000000 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dax/INFO] 'root_i2_2@task2' is a Comm: 1000000 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dax/INFO] '1@task1_o1_3@task1' is a Comm: 1000000 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dax/INFO] '2@task2_o2_3@task1' is a Comm: 1000000 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dax/INFO] '3@task1_o3_end' is a Comm: 4167312 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dax/INFO] 'end' is an Exec: 0 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dax/INFO] ------------------- Schedule tasks --------------------------- +> [0.000000] [dag_from_dax/INFO] ------------------- Run the schedule ------------------------- +> [84.090777] [dag_from_dax/INFO] -------------- Summary of executed schedule ------------------ +> [84.090777] [dag_from_dax/INFO] [0.000000->0.000000] 'root' executed on node-0.simgrid.org +> [84.090777] [dag_from_dax/INFO] [0.024301->42.024301] '1@task1' executed on node-1.simgrid.org +> [84.090777] [dag_from_dax/INFO] [0.024301->42.024301] '2@task2' executed on node-10.simgrid.org +> [84.090777] [dag_from_dax/INFO] [42.048602->84.048602] '3@task1' executed on node-11.simgrid.org +> [84.090777] [dag_from_dax/INFO] [0.000000->0.024301] 'root_i1_1@task1' transferred from node-0.simgrid.org to node-1.simgrid.org +> [84.090777] [dag_from_dax/INFO] [0.000000->0.024301] 'root_i2_2@task2' transferred from node-0.simgrid.org to node-10.simgrid.org +> [84.090777] [dag_from_dax/INFO] [42.024301->42.048602] '1@task1_o1_3@task1' transferred from node-1.simgrid.org to node-11.simgrid.org +> [84.090777] [dag_from_dax/INFO] [42.024301->42.048602] '2@task2_o2_3@task1' transferred from node-10.simgrid.org to node-11.simgrid.org +> [84.090777] [dag_from_dax/INFO] [84.048602->84.090777] '3@task1_o3_end' transferred from node-11.simgrid.org to node-0.simgrid.org +> [84.090777] [dag_from_dax/INFO] [84.090777->84.090777] 'end' executed on node-0.simgrid.org + +p Test the DAX loader with a DAX comprising a cycle. + +! expect return 2 +$ ${bindir:=.}/s4u-dag-from-dax --log=no_loc ${platfdir}/cluster_backbone.xml ${srcdir:=.}/simple_dax_with_cycle.xml +> [0.000000] [sd_daxparse/ERROR] The DAX described in simple_dax_with_cycle.xml is not a DAG. It contains a cycle. +> [0.000000] [dag_from_dax/ERROR] A problem occurred during DAX parsing (cycle or syntax). Do not continue this test diff --git a/examples/deprecated/simdag/daxload/simple_dax_with_cycle.xml b/examples/cpp/dag-from-dax/simple_dax_with_cycle.xml similarity index 100% rename from examples/deprecated/simdag/daxload/simple_dax_with_cycle.xml rename to examples/cpp/dag-from-dax/simple_dax_with_cycle.xml diff --git a/examples/deprecated/simdag/daxload/smalldax.xml b/examples/cpp/dag-from-dax/smalldax.xml similarity index 100% rename from examples/deprecated/simdag/daxload/smalldax.xml rename to examples/cpp/dag-from-dax/smalldax.xml diff --git a/examples/cpp/dag-io/s4u-dag-io.tesh b/examples/cpp/dag-io/s4u-dag-io.tesh index 2dcac17518..186126ba96 100644 --- a/examples/cpp/dag-io/s4u-dag-io.tesh +++ b/examples/cpp/dag-io/s4u-dag-io.tesh @@ -5,9 +5,9 @@ $ ${bindir:=.}/s4u-dag-io ${platfdir}/hosts_with_disks.xml --log=s4u_activity.t: > [ 0.000000] (0:maestro@) Activity 'write' vetoed. Dependencies: NOT solved; Ressources: assigned > [ 0.000000] (0:maestro@) Activity 'read' vetoed. Dependencies: NOT solved; Ressources: assigned > [ 0.000000] (0:maestro@) Activity 'child' vetoed. Dependencies: NOT solved; Ressources: assigned +> [ 1.000000] (0:maestro@) Activity 'parent' is complete (start time: 0.000000, finish time: 1.000000) > [ 1.000000] (0:maestro@) Remove a dependency from 'parent' on 'write' > [ 1.000000] (0:maestro@) 'write' is assigned to a resource and all dependencies are solved. Let's start -> [ 1.000000] (0:maestro@) Activity 'parent' is complete (start time: 0.000000, finish time: 1.000000) > [ 26.000000] (0:maestro@) Remove a dependency from 'write' on 'read' > [ 26.000000] (0:maestro@) 'read' is assigned to a resource and all dependencies are solved. Let's start > [ 36.000000] (0:maestro@) Remove a dependency from 'read' on 'child' diff --git a/examples/deprecated/simdag/scheduling/Montage_25.xml b/examples/cpp/dag-scheduling/Montage_25.xml similarity index 100% rename from examples/deprecated/simdag/scheduling/Montage_25.xml rename to examples/cpp/dag-scheduling/Montage_25.xml diff --git a/examples/deprecated/simdag/scheduling/expected_output.jed b/examples/cpp/dag-scheduling/expected_output.jed similarity index 100% rename from examples/deprecated/simdag/scheduling/expected_output.jed rename to examples/cpp/dag-scheduling/expected_output.jed diff --git a/examples/cpp/dag-scheduling/s4u-dag-scheduling.cpp b/examples/cpp/dag-scheduling/s4u-dag-scheduling.cpp new file mode 100644 index 0000000000..a3cca2534e --- /dev/null +++ b/examples/cpp/dag-scheduling/s4u-dag-scheduling.cpp @@ -0,0 +1,292 @@ +/* Copyright (c) 2009-2021. The SimGrid Team. + * All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +/* simple test to schedule a DAX file with the Min-Min algorithm. */ +#include "simgrid/s4u.hpp" +#include +#include + +#if SIMGRID_HAVE_JEDULE +#include "simgrid/jedule/jedule_sd_binding.h" +#endif + +XBT_LOG_NEW_DEFAULT_CATEGORY(dag_scheduling, "Logging specific to this example"); + +typedef struct _HostAttribute* HostAttribute; +struct _HostAttribute { + /* Earliest time at which a host is ready to execute a task */ + double available_at; + simgrid::s4u::Exec* last_scheduled_task; +}; + +static double sg_host_get_available_at(const simgrid::s4u::Host* host) +{ + return static_cast(host->get_data())->available_at; +} + +static void sg_host_set_available_at(simgrid::s4u::Host* host, double time) +{ + auto* attr = static_cast(host->get_data()); + attr->available_at = time; + host->set_data(attr); +} + +static simgrid::s4u::Exec* sg_host_get_last_scheduled_task(const simgrid::s4u::Host* host) +{ + return static_cast(host->get_data())->last_scheduled_task; +} + +static void sg_host_set_last_scheduled_task(simgrid::s4u::Host* host, simgrid::s4u::ExecPtr task) +{ + auto* attr = static_cast(host->get_data()); + attr->last_scheduled_task = task.get(); + host->set_data(attr); +} + +static bool dependency_exists(simgrid::s4u::Exec* src, simgrid::s4u::Exec* dst) +{ + const auto& dependencies = src->get_dependencies(); + const auto& successors = src->get_successors(); + return (std::find(successors.begin(), successors.end(), dst) != successors.end() || + dependencies.find(dst) != dependencies.end()); +} + +static std::vector get_ready_tasks(const std::vector dax) +{ + std::vector ready_tasks; + std::map candidate_execs; + + for (auto& a : dax) { + // Only loot at activity that have their dependencies solved but are not assigned + if (a->dependencies_solved() && not a->is_assigned()) { + // if it is an exec, it's ready + auto* exec = dynamic_cast(a.get()); + if (exec != nullptr) + ready_tasks.push_back(exec); + // if it a comm, we consider its successor as a candidate. If a candidate solves all its dependencies, + // i.e., get all its input data, it's ready + auto* comm = dynamic_cast(a.get()); + if (comm != nullptr) { + auto* next_exec = static_cast(comm->get_successors().front().get()); + candidate_execs[next_exec]++; + if (next_exec->get_dependencies().size() == candidate_execs[next_exec]) + ready_tasks.push_back(next_exec); + } + } + } + XBT_DEBUG("There are %zu ready tasks", ready_tasks.size()); + return ready_tasks; +} + +static double finish_on_at(const simgrid::s4u::ExecPtr task, const simgrid::s4u::Host* host) +{ + double result; + + const auto& parents = task->get_dependencies(); + + if (not parents.empty()) { + double data_available = 0.; + double last_data_available; + /* compute last_data_available */ + last_data_available = -1.0; + for (const auto& parent : parents) { + /* normal case */ + auto* comm = dynamic_cast(parent.get()); + if (comm != nullptr) { + auto source = comm->get_source(); + XBT_DEBUG("transfer from %s to %s", source->get_cname(), host->get_cname()); + /* Estimate the redistribution time from this parent */ + double redist_time; + if (comm->get_remaining() <= 1e-6) { + redist_time = 0; + } else { + redist_time = sg_host_get_route_latency(source, host) + + comm->get_remaining() / sg_host_get_route_bandwidth(source, host); + } + // We use the user data field to store the finish time of the predecessor of the comm, i.e., its potential start + // time + data_available = *(static_cast(comm->get_data())) + redist_time; + } + + auto* exec = dynamic_cast(parent.get()); + /* no transfer, control dependency */ + if (exec != nullptr) { + data_available = exec->get_finish_time(); + } + + if (last_data_available < data_available) + last_data_available = data_available; + } + + result = fmax(sg_host_get_available_at(host), last_data_available) + task->get_remaining() / host->get_speed(); + } else + result = sg_host_get_available_at(host) + task->get_remaining() / host->get_speed(); + + return result; +} + +static simgrid::s4u::Host* get_best_host(const simgrid::s4u::ExecPtr exec) +{ + std::vector hosts = simgrid::s4u::Engine::get_instance()->get_all_hosts(); + auto best_host = hosts.front(); + double min_EFT = finish_on_at(exec, best_host); + + for (const auto& h : hosts) { + double EFT = finish_on_at(exec, h); + XBT_DEBUG("%s finishes on %s at %f", exec->get_cname(), h->get_cname(), EFT); + + if (EFT < min_EFT) { + min_EFT = EFT; + best_host = h; + } + } + return best_host; +} + +int main(int argc, char** argv) +{ + double min_finish_time = -1.0; + simgrid::s4u::Exec* selected_task = nullptr; + simgrid::s4u::Host* selected_host = nullptr; + char* tracefilename = nullptr; + + simgrid::s4u::Engine e(&argc, argv); + std::set vetoed; + e.track_vetoed_activities(&vetoed); + + simgrid::s4u::Activity::on_completion.connect([](simgrid::s4u::Activity& activity) { + // when an Exec completes, we need to set the potential start time of all its ouput comms + const auto* exec = dynamic_cast(&activity); + if (exec == nullptr) // Only Execs are concerned here + return; + for (const auto& succ : exec->get_successors()) { + auto* comm = dynamic_cast(succ.get()); + if (comm != nullptr) { + double* finish_time = new double(exec->get_finish_time()); + // We use the user data field to store the finish time of the predecessor of the comm, i.e., its potential start + // time + comm->set_data(finish_time); + } + } + }); + /* Check our arguments */ + xbt_assert(argc > 2, + "Usage: %s platform_file dax_file [jedule_file]\n" + "\tExample: %s simulacrum_7_hosts.xml Montage_25.xml Montage_25.jed", + argv[0], argv[0]); + + if (argc == 4) + tracefilename = xbt_strdup(argv[3]); + + e.load_platform(argv[1]); + + /* Allocating the host attribute */ + unsigned int total_nhosts = e.get_host_count(); + const auto hosts = e.get_all_hosts(); + + for (unsigned int i = 0; i < total_nhosts; i++) + hosts[i]->set_data(xbt_new0(struct _HostAttribute, 1)); + + /* load the DAX file */ + std::vector dax = simgrid::s4u::create_DAG_from_DAX(argv[2]); + + /* Schedule the root first */ + auto* root = static_cast(dax.front().get()); + auto host = get_best_host(root); + root->set_host(host); + // we can also set the source of all the output comms of the root node + for (const auto& succ : root->get_successors()) { + auto* comm = dynamic_cast(succ.get()); + if (comm != nullptr) + comm->set_source(host); + } + + e.run(); + + while (not vetoed.empty()) { + XBT_DEBUG("Start new scheduling round"); + /* Get the set of ready tasks */ + auto ready_tasks = get_ready_tasks(dax); + vetoed.clear(); + + if (ready_tasks.empty()) { + ready_tasks.clear(); + /* there is no ready task, let advance the simulation */ + e.run(); + continue; + } + /* For each ready task: + * get the host that minimizes the completion time. + * select the task that has the minimum completion time on its best host. + */ + for (auto task : ready_tasks) { + XBT_DEBUG("%s is ready", task->get_cname()); + host = get_best_host(task); + double finish_time = finish_on_at(task, host); + if (min_finish_time < 0 || finish_time < min_finish_time) { + min_finish_time = finish_time; + selected_task = task; + selected_host = host; + } + } + + XBT_INFO("Schedule %s on %s", selected_task->get_cname(), selected_host->get_cname()); + selected_task->set_host(selected_host); + // we can also set the destination of all the input comms of the selected task + for (const auto& pred : selected_task->get_dependencies()) { + auto* comm = dynamic_cast(pred.get()); + if (comm != nullptr) { + comm->set_destination(selected_host); + delete static_cast(comm->get_data()); + } + } + // we can also set the source of all the output comms of the selected task + for (const auto& succ : selected_task->get_successors()) { + auto* comm = dynamic_cast(succ.get()); + if (comm != nullptr) + comm->set_source(selected_host); + } + + /* + * tasks can be executed concurrently when they can by default. + * Yet schedulers take decisions assuming that tasks wait for resource availability to start. + * The solution (well crude hack is to keep track of the last task scheduled on a host and add a special type of + * dependency if needed to force the sequential execution meant by the scheduler. + * If the last scheduled task is already done, has failed or is a predecessor of the current task, no need for a + * new dependency + */ + + auto last_scheduled_task = sg_host_get_last_scheduled_task(selected_host); + if (last_scheduled_task && (last_scheduled_task->get_state() != simgrid::s4u::Activity::State::FINISHED) && + (last_scheduled_task->get_state() != simgrid::s4u::Activity::State::FAILED) && + not dependency_exists(sg_host_get_last_scheduled_task(selected_host), selected_task)) + last_scheduled_task->add_successor(selected_task); + + sg_host_set_last_scheduled_task(selected_host, selected_task); + sg_host_set_available_at(selected_host, min_finish_time); + + ready_tasks.clear(); + /* reset the min_finish_time for the next set of ready tasks */ + min_finish_time = -1.; + e.run(); + } + + XBT_INFO("Simulation Time: %f", simgrid_get_clock()); + XBT_INFO("------------------- Produce the trace file---------------------------"); + XBT_INFO("Producing a jedule output (if active) of the run into %s", + tracefilename ? tracefilename : "minmin_test.jed"); +#if SIMGRID_HAVE_JEDULE + jedule_sd_dump(tracefilename); +#endif + free(tracefilename); + + for (auto h : hosts) { + xbt_free(h->get_data()); + h->set_data(nullptr); + } + + return 0; +} diff --git a/examples/cpp/dag-scheduling/s4u-dag-scheduling.tesh b/examples/cpp/dag-scheduling/s4u-dag-scheduling.tesh new file mode 100644 index 0000000000..af398b3706 --- /dev/null +++ b/examples/cpp/dag-scheduling/s4u-dag-scheduling.tesh @@ -0,0 +1,35 @@ +#!/usr/bin/env tesh + +$ ${bindir:=.}/s4u-dag-scheduling --log=sd_daxparse.thresh:critical --log=no_loc ${platfdir}/simulacrum_7_hosts.xml ${srcdir:=.}/Montage_25.xml +> [0.000000] [dag_scheduling/INFO] Schedule ID00002@mProjectPP on Host 27 +> [0.001301] [dag_scheduling/INFO] Schedule ID00000@mProjectPP on Host 26 +> [0.001301] [dag_scheduling/INFO] Schedule ID00003@mProjectPP on Host 30 +> [0.001306] [dag_scheduling/INFO] Schedule ID00004@mProjectPP on Host 27 +> [0.001736] [dag_scheduling/INFO] Schedule ID00001@mProjectPP on Host 32 +> [14.582104] [dag_scheduling/INFO] Schedule ID00010@mDiffFit on Host 26 +> [17.042829] [dag_scheduling/INFO] Schedule ID00008@mDiffFit on Host 27 +> [29.547201] [dag_scheduling/INFO] Schedule ID00013@mDiffFit on Host 26 +> [32.629864] [dag_scheduling/INFO] Schedule ID00009@mDiffFit on Host 30 +> [32.631165] [dag_scheduling/INFO] Schedule ID00011@mDiffFit on Host 27 +> [32.632466] [dag_scheduling/INFO] Schedule ID00005@mDiffFit on Host 26 +> [32.632466] [dag_scheduling/INFO] Schedule ID00006@mDiffFit on Host 32 +> [32.632469] [dag_scheduling/INFO] Schedule ID00012@mDiffFit on Host 31 +> [32.632896] [dag_scheduling/INFO] Schedule ID00007@mDiffFit on Host 28 +> [60.434831] [dag_scheduling/INFO] Schedule ID00014@mConcatFit on Host 27 +> [61.228672] [dag_scheduling/INFO] Schedule ID00015@mBgModel on Host 27 +> [62.772102] [dag_scheduling/INFO] Schedule ID00016@mBackground on Host 27 +> [62.772102] [dag_scheduling/INFO] Schedule ID00017@mBackground on Host 26 +> [62.773403] [dag_scheduling/INFO] Schedule ID00020@mBackground on Host 30 +> [62.773403] [dag_scheduling/INFO] Schedule ID00018@mBackground on Host 27 +> [62.773403] [dag_scheduling/INFO] Schedule ID00019@mBackground on Host 32 +> [88.573854] [dag_scheduling/INFO] Schedule ID00021@mImgTbl on Host 27 +> [90.158236] [dag_scheduling/INFO] Schedule ID00022@mAdd on Host 27 +> [93.450123] [dag_scheduling/INFO] Schedule ID00023@mShrink on Host 27 +> [97.646883] [dag_scheduling/INFO] Schedule ID00024@mJPEG on Host 27 +> [98.135775] [dag_scheduling/INFO] Schedule end on Host 27 +> [98.135796] [dag_scheduling/INFO] Simulation Time: 98.135796 +> [98.135796] [dag_scheduling/INFO] ------------------- Produce the trace file--------------------------- +> [98.135796] [dag_scheduling/INFO] Producing a jedule output (if active) of the run into minmin_test.jed + +! output ignore +$ cmake -E remove -f ${srcdir:=.}/sd_scheduling.jed diff --git a/examples/cpp/dag-simple/s4u-dag-simple.tesh b/examples/cpp/dag-simple/s4u-dag-simple.tesh index c232dcc555..e41d10ee99 100644 --- a/examples/cpp/dag-simple/s4u-dag-simple.tesh +++ b/examples/cpp/dag-simple/s4u-dag-simple.tesh @@ -4,12 +4,12 @@ $ ${bindir:=.}/s4u-dag-simple ${platfdir}/small_platform.xml --log=s4u_activity. > [ 0.000000] (0:maestro@) 'parent 1' is assigned to a resource and all dependencies are solved. Let's start > [ 0.000000] (0:maestro@) 'parent 2' is assigned to a resource and all dependencies are solved. Let's start > [ 0.000000] (0:maestro@) Activity 'child' vetoed. Dependencies: NOT solved; Ressources: NOT assigned -> [ 2.000000] (0:maestro@) Remove a dependency from 'parent 1' on 'child' > [ 2.000000] (0:maestro@) Activity 'parent 1' is complete (start time: 0.000000, finish time: 2.000000) +> [ 2.000000] (0:maestro@) Remove a dependency from 'parent 1' on 'child' > [ 2.000000] (0:maestro@) Activity child not ready. +> [ 3.000000] (0:maestro@) Activity 'parent 2' is complete (start time: 0.000000, finish time: 3.000000) > [ 3.000000] (0:maestro@) Remove a dependency from 'parent 2' on 'child' > [ 3.000000] (0:maestro@) Activity 'child' vetoed. Dependencies: solved; Ressources: NOT assigned -> [ 3.000000] (0:maestro@) Activity 'parent 2' is complete (start time: 0.000000, finish time: 3.000000) > [ 3.000000] (0:maestro@) Activity child's dependencies are resolved. Let's assign it to Fafard. > [ 3.000000] (0:maestro@) 'child' is assigned to a resource and all dependencies are solved. Let's start > [ 4.000000] (0:maestro@) Activity 'child' is complete (start time: 3.000000, finish time: 4.000000) diff --git a/examples/deprecated/simdag/CMakeLists.txt b/examples/deprecated/simdag/CMakeLists.txt deleted file mode 100644 index 8d611f955d..0000000000 --- a/examples/deprecated/simdag/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -foreach(x daxload scheduling) - add_executable (sd_${x} EXCLUDE_FROM_ALL ${x}/sd_${x}.c) - target_link_libraries(sd_${x} simgrid) - set_target_properties(sd_${x} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${x}) - add_dependencies(tests sd_${x}) - set(examples_src ${examples_src} ${CMAKE_CURRENT_SOURCE_DIR}/${x}/sd_${x}.c) - set(tesh_files ${tesh_files} ${CMAKE_CURRENT_SOURCE_DIR}/${x}/sd_${x}.tesh) -endforeach() - -set(tesh_files ${tesh_files} PARENT_SCOPE) -set(examples_src ${examples_src} PARENT_SCOPE) -set(xml_files ${xml_files} ${CMAKE_CURRENT_SOURCE_DIR}/scheduling/Montage_25.xml - ${CMAKE_CURRENT_SOURCE_DIR}/daxload/simple_dax_with_cycle.xml - ${CMAKE_CURRENT_SOURCE_DIR}/daxload/smalldax.xml PARENT_SCOPE) -set(txt_files ${txt_files} ${CMAKE_CURRENT_SOURCE_DIR}/scheduling/expected_output.jed PARENT_SCOPE) - -foreach(x daxload scheduling) - ADD_TESH(simdag-${x} --setenv srcdir=${CMAKE_HOME_DIRECTORY}/examples/deprecated/simdag --cd ${CMAKE_BINARY_DIR}/examples/deprecated/simdag ${CMAKE_HOME_DIRECTORY}/examples/deprecated/simdag/${x}/sd_${x}.tesh) -endforeach() diff --git a/examples/deprecated/simdag/daxload/sd_daxload.c b/examples/deprecated/simdag/daxload/sd_daxload.c deleted file mode 100644 index 8588c90614..0000000000 --- a/examples/deprecated/simdag/daxload/sd_daxload.c +++ /dev/null @@ -1,113 +0,0 @@ -/* simple test trying to load a DAX file. */ - -/* Copyright (c) 2009-2021. The SimGrid Team. - * All rights reserved. */ - -/* This program is free software; you can redistribute it and/or modify it - * under the terms of the license (GNU LGPL) which comes with this package. */ - -#include "simgrid/simdag.h" - -#include -#include - -XBT_LOG_NEW_DEFAULT_CATEGORY(test, "Logging specific to this SimDag example"); - -static int name_compare_hosts(const void *n1, const void *n2) -{ - return strcmp(sg_host_get_name(*(const sg_host_t*)n1), sg_host_get_name(*(const sg_host_t*)n2)); -} - -int main(int argc, char **argv) -{ - xbt_dynar_t dax; - unsigned int cursor; - SD_task_t task; - - /* SD initialization */ - SD_init(&argc, argv); - - /* Check our arguments */ - xbt_assert(argc > 2, "Usage: %s platform_file dax_file [jedule_file]\n" - "\tExample: %s simulacrum_7_hosts.xml Montage_25.xml Montage_25.jed", argv[0], argv[0]); - - const char* last = strrchr(argv[2], '.'); - char * tracefilename = bprintf("%.*s.trace",(int) (last == NULL ? strlen(argv[2]):last - argv[2]), argv[2]); - if (argc == 4) - tracefilename = xbt_strdup(argv[3]); - - /* creation of the environment */ - SD_create_environment(argv[1]); - - /* load the DAX file */ - dax = SD_daxload(argv[2]); - if (!dax){ - XBT_ERROR("A problem occurred during DAX parsing (cycle or syntax). Do not continue this test"); - free(tracefilename); - - exit(255); - } - - /* Display all the tasks */ - XBT_INFO("------------------- Display all tasks of the loaded DAG ---------------------------"); - xbt_dynar_foreach(dax, cursor, task) { - SD_task_dump(task); - } - - FILE *dotout = fopen("dax.dot", "w"); - fprintf(dotout, "digraph A {\n"); - xbt_dynar_foreach(dax, cursor, task) { - SD_task_dotty(task, dotout); - } - fprintf(dotout, "}\n"); - fclose(dotout); - - /* Schedule them all on the first host */ - XBT_INFO("------------------- Schedule tasks ---------------------------"); - sg_host_t *host_list = sg_host_list(); - int hosts_count = sg_host_count(); - qsort(host_list, hosts_count, sizeof(sg_host_t), name_compare_hosts); - - xbt_dynar_foreach(dax, cursor, task) { - if (SD_task_get_kind(task) == SD_TASK_COMP_SEQ) { - if (!strcmp(SD_task_get_name(task), "end")) - SD_task_schedulel(task, 1, host_list[0]); - else - SD_task_schedulel(task, 1, host_list[cursor % hosts_count]); - } - } - xbt_free(host_list); - - XBT_INFO("------------------- Run the schedule ---------------------------"); - SD_simulate(-1); - XBT_INFO("------------------- Produce the trace file---------------------------"); - const char* basename = strrchr(tracefilename, '/'); - XBT_INFO("Producing the trace of the run into %s", basename ? basename + 1 : tracefilename); - FILE *out = fopen(tracefilename, "w"); - xbt_assert(out, "Cannot write to %s", tracefilename); - free(tracefilename); - - xbt_dynar_foreach(dax, cursor, task) { - int kind = SD_task_get_kind(task); - sg_host_t *wsl = SD_task_get_workstation_list(task); - switch (kind) { - case SD_TASK_COMP_SEQ: - fprintf(out, "[%f] %s compute %f # %s\n", SD_task_get_start_time(task), sg_host_get_name(wsl[0]), - SD_task_get_amount(task), SD_task_get_name(task)); - break; - case SD_TASK_COMM_E2E: - fprintf(out, "[%f] %s send %s %f # %s\n", SD_task_get_start_time(task), sg_host_get_name(wsl[0]), - sg_host_get_name(wsl[1]), SD_task_get_amount(task), SD_task_get_name(task)); - fprintf(out, "[%f] %s recv %s %f # %s\n", SD_task_get_finish_time(task), sg_host_get_name(wsl[1]), - sg_host_get_name(wsl[0]), SD_task_get_amount(task), SD_task_get_name(task)); - break; - default: - xbt_die("Task %s is of unknown kind %u", SD_task_get_name(task), SD_task_get_kind(task)); - } - SD_task_destroy(task); - } - fclose(out); - xbt_dynar_free_container(&dax); - - return 0; -} diff --git a/examples/deprecated/simdag/daxload/sd_daxload.tesh b/examples/deprecated/simdag/daxload/sd_daxload.tesh deleted file mode 100644 index 44fe501bfb..0000000000 --- a/examples/deprecated/simdag/daxload/sd_daxload.tesh +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env tesh -p Test the DAX loader on a small DAX instance - -! output sort -$ ${bindir:=.}/daxload/sd_daxload --log=no_loc ${srcdir:=.}/../../platforms/cluster_backbone.xml ${srcdir:=.}/daxload/smalldax.xml -> [0.000000] [xbt_cfg/INFO] Switching to the L07 model to handle parallel tasks. -> [0.000000] [sd_daxparse/WARNING] Ignore file o1 size redefinition from 1000000 to 304 -> [0.000000] [sd_daxparse/WARNING] Ignore file o2 size redefinition from 1000000 to 304 -> [0.000000] [test/INFO] ------------------- Display all tasks of the loaded DAG --------------------------- -> [0.000000] [sd_task/INFO] Displaying task root -> [0.000000] [sd_task/INFO] - state: schedulable not runnable -> [0.000000] [sd_task/INFO] - kind: sequential computation -> [0.000000] [sd_task/INFO] - amount: 0 -> [0.000000] [sd_task/INFO] - Dependencies to satisfy: 0 -> [0.000000] [sd_task/INFO] - post-dependencies: -> [0.000000] [sd_task/INFO] root_i1_1@task1 -> [0.000000] [sd_task/INFO] root_i2_2@task2 -> [0.000000] [sd_task/INFO] Displaying task 1@task1 -> [0.000000] [sd_task/INFO] - state: not scheduled not runnable -> [0.000000] [sd_task/INFO] - kind: sequential computation -> [0.000000] [sd_task/INFO] - amount: 42000000000 -> [0.000000] [sd_task/INFO] - Dependencies to satisfy: 1 -> [0.000000] [sd_task/INFO] - pre-dependencies: -> [0.000000] [sd_task/INFO] root_i1_1@task1 -> [0.000000] [sd_task/INFO] - post-dependencies: -> [0.000000] [sd_task/INFO] 3@task1 -> [0.000000] [sd_task/INFO] 1@task1_o1_3@task1 -> [0.000000] [sd_task/INFO] Displaying task 2@task2 -> [0.000000] [sd_task/INFO] - state: not scheduled not runnable -> [0.000000] [sd_task/INFO] - kind: sequential computation -> [0.000000] [sd_task/INFO] - amount: 42000000000 -> [0.000000] [sd_task/INFO] - Dependencies to satisfy: 1 -> [0.000000] [sd_task/INFO] - pre-dependencies: -> [0.000000] [sd_task/INFO] root_i2_2@task2 -> [0.000000] [sd_task/INFO] - post-dependencies: -> [0.000000] [sd_task/INFO] 3@task1 -> [0.000000] [sd_task/INFO] 2@task2_o2_3@task1 -> [0.000000] [sd_task/INFO] Displaying task 3@task1 -> [0.000000] [sd_task/INFO] - state: not scheduled not runnable -> [0.000000] [sd_task/INFO] - kind: sequential computation -> [0.000000] [sd_task/INFO] - amount: 42000000000 -> [0.000000] [sd_task/INFO] - Dependencies to satisfy: 4 -> [0.000000] [sd_task/INFO] - pre-dependencies: -> [0.000000] [sd_task/INFO] 2@task2 -> [0.000000] [sd_task/INFO] 1@task1 -> [0.000000] [sd_task/INFO] 2@task2_o2_3@task1 -> [0.000000] [sd_task/INFO] 1@task1_o1_3@task1 -> [0.000000] [sd_task/INFO] - post-dependencies: -> [0.000000] [sd_task/INFO] 3@task1_o3_end -> [0.000000] [sd_task/INFO] Displaying task root_i2_2@task2 -> [0.000000] [sd_task/INFO] - state: not scheduled not runnable -> [0.000000] [sd_task/INFO] - kind: end-to-end communication -> [0.000000] [sd_task/INFO] - amount: 1000000 -> [0.000000] [sd_task/INFO] - Dependencies to satisfy: 1 -> [0.000000] [sd_task/INFO] - pre-dependencies: -> [0.000000] [sd_task/INFO] root -> [0.000000] [sd_task/INFO] - post-dependencies: -> [0.000000] [sd_task/INFO] 2@task2 -> [0.000000] [sd_task/INFO] Displaying task 1@task1_o1_3@task1 -> [0.000000] [sd_task/INFO] - state: not scheduled not runnable -> [0.000000] [sd_task/INFO] - kind: end-to-end communication -> [0.000000] [sd_task/INFO] - amount: 1000000 -> [0.000000] [sd_task/INFO] - Dependencies to satisfy: 1 -> [0.000000] [sd_task/INFO] - pre-dependencies: -> [0.000000] [sd_task/INFO] 1@task1 -> [0.000000] [sd_task/INFO] - post-dependencies: -> [0.000000] [sd_task/INFO] 3@task1 -> [0.000000] [sd_task/INFO] Displaying task 2@task2_o2_3@task1 -> [0.000000] [sd_task/INFO] - state: not scheduled not runnable -> [0.000000] [sd_task/INFO] - kind: end-to-end communication -> [0.000000] [sd_task/INFO] - amount: 1000000 -> [0.000000] [sd_task/INFO] - Dependencies to satisfy: 1 -> [0.000000] [sd_task/INFO] - pre-dependencies: -> [0.000000] [sd_task/INFO] 2@task2 -> [0.000000] [sd_task/INFO] - post-dependencies: -> [0.000000] [sd_task/INFO] 3@task1 -> [0.000000] [sd_task/INFO] Displaying task 3@task1_o3_end -> [0.000000] [sd_task/INFO] - state: not scheduled not runnable -> [0.000000] [sd_task/INFO] - kind: end-to-end communication -> [0.000000] [sd_task/INFO] - amount: 4167312 -> [0.000000] [sd_task/INFO] - Dependencies to satisfy: 1 -> [0.000000] [sd_task/INFO] - pre-dependencies: -> [0.000000] [sd_task/INFO] 3@task1 -> [0.000000] [sd_task/INFO] - post-dependencies: -> [0.000000] [sd_task/INFO] end -> [0.000000] [sd_task/INFO] Displaying task root_i1_1@task1 -> [0.000000] [sd_task/INFO] - state: not scheduled not runnable -> [0.000000] [sd_task/INFO] - kind: end-to-end communication -> [0.000000] [sd_task/INFO] - amount: 1000000 -> [0.000000] [sd_task/INFO] - Dependencies to satisfy: 1 -> [0.000000] [sd_task/INFO] - pre-dependencies: -> [0.000000] [sd_task/INFO] root -> [0.000000] [sd_task/INFO] - post-dependencies: -> [0.000000] [sd_task/INFO] 1@task1 -> [0.000000] [sd_task/INFO] Displaying task end -> [0.000000] [sd_task/INFO] - state: not scheduled not runnable -> [0.000000] [sd_task/INFO] - kind: sequential computation -> [0.000000] [sd_task/INFO] - amount: 0 -> [0.000000] [sd_task/INFO] - Dependencies to satisfy: 1 -> [0.000000] [sd_task/INFO] - pre-dependencies: -> [0.000000] [sd_task/INFO] 3@task1_o3_end -> [0.000000] [test/INFO] ------------------- Schedule tasks --------------------------- -> [0.000000] [test/INFO] ------------------- Run the schedule --------------------------- -> [84.067138] [test/INFO] ------------------- Produce the trace file--------------------------- -> [84.067138] [test/INFO] Producing the trace of the run into smalldax.trace - -$ cat ${srcdir:=.}/daxload/smalldax.trace -> [0.000000] node-0.simgrid.org compute 0.000000 # root -> [0.016600] node-1.simgrid.org compute 42000000000.000000 # 1@task1 -> [0.016600] node-10.simgrid.org compute 42000000000.000000 # 2@task2 -> [42.033200] node-11.simgrid.org compute 42000000000.000000 # 3@task1 -> [0.000000] node-0.simgrid.org send node-1.simgrid.org 1000000.000000 # root_i1_1@task1 -> [0.016600] node-1.simgrid.org recv node-0.simgrid.org 1000000.000000 # root_i1_1@task1 -> [0.000000] node-0.simgrid.org send node-10.simgrid.org 1000000.000000 # root_i2_2@task2 -> [0.016600] node-10.simgrid.org recv node-0.simgrid.org 1000000.000000 # root_i2_2@task2 -> [42.016600] node-1.simgrid.org send node-11.simgrid.org 1000000.000000 # 1@task1_o1_3@task1 -> [42.033200] node-11.simgrid.org recv node-1.simgrid.org 1000000.000000 # 1@task1_o1_3@task1 -> [42.016600] node-10.simgrid.org send node-11.simgrid.org 1000000.000000 # 2@task2_o2_3@task1 -> [42.033200] node-11.simgrid.org recv node-10.simgrid.org 1000000.000000 # 2@task2_o2_3@task1 -> [84.033200] node-11.simgrid.org send node-0.simgrid.org 4167312.000000 # 3@task1_o3_end -> [84.067138] node-0.simgrid.org recv node-11.simgrid.org 4167312.000000 # 3@task1_o3_end -> [84.067138] node-0.simgrid.org compute 0.000000 # end - -! output ignore -$ cmake -E remove -f ${srcdir:=.}/dax.dot ${srcdir:=.}/daxload/smalldax.trace - -p Test the DAX loader with a DAX comprising a cycle. - -! expect return 255 -$ ${bindir:=.}/daxload/sd_daxload --log=no_loc ${srcdir:=.}/../../platforms/cluster_backbone.xml ${srcdir:=.}/daxload/simple_dax_with_cycle.xml -> [0.000000] [xbt_cfg/INFO] Switching to the L07 model to handle parallel tasks. -> [0.000000] [sd_daxparse/WARNING] the task root is not marked -> [0.000000] [sd_daxparse/WARNING] the task 1@task1 is in a cycle -> [0.000000] [sd_daxparse/WARNING] the task 3@task3 is in a cycle -> [0.000000] [sd_daxparse/ERROR] The DAX described in simple_dax_with_cycle.xml is not a DAG. It contains a cycle. -> [0.000000] [test/ERROR] A problem occurred during DAX parsing (cycle or syntax). Do not continue this test diff --git a/examples/deprecated/simdag/scheduling/sd_scheduling.c b/examples/deprecated/simdag/scheduling/sd_scheduling.c deleted file mode 100644 index 8c64253f3c..0000000000 --- a/examples/deprecated/simdag/scheduling/sd_scheduling.c +++ /dev/null @@ -1,257 +0,0 @@ -/* Copyright (c) 2009-2021. The SimGrid Team. - * All rights reserved. */ - -/* This program is free software; you can redistribute it and/or modify it - * under the terms of the license (GNU LGPL) which comes with this package. */ - -/* simple test to schedule a DAX file with the Min-Min algorithm. */ -#include "simgrid/simdag.h" -#include -#include - -#if SIMGRID_HAVE_JEDULE -#include "simgrid/jedule/jedule_sd_binding.h" -#endif - -XBT_LOG_NEW_DEFAULT_CATEGORY(test, "Logging specific to this SimDag example"); - -typedef struct _HostAttribute *HostAttribute; -struct _HostAttribute { - /* Earliest time at which a host is ready to execute a task */ - double available_at; - SD_task_t last_scheduled_task; -}; - -static double sg_host_get_available_at(const_sg_host_t host) -{ - const struct _HostAttribute* attr = (HostAttribute)sg_host_get_data(host); - return attr->available_at; -} - -static void sg_host_set_available_at(sg_host_t host, double time) -{ - HostAttribute attr = (HostAttribute)sg_host_get_data(host); - attr->available_at = time; - sg_host_set_data(host, attr); -} - -static SD_task_t sg_host_get_last_scheduled_task(const_sg_host_t host) -{ - const struct _HostAttribute* attr = (HostAttribute)sg_host_get_data(host); - return attr->last_scheduled_task; -} - -static void sg_host_set_last_scheduled_task(sg_host_t host, SD_task_t task){ - HostAttribute attr = (HostAttribute)sg_host_get_data(host); - attr->last_scheduled_task=task; - sg_host_set_data(host, attr); -} - -static xbt_dynar_t get_ready_tasks(const_xbt_dynar_t dax) -{ - unsigned int i; - xbt_dynar_t ready_tasks; - SD_task_t task; - - ready_tasks = xbt_dynar_new(sizeof(SD_task_t), NULL); - xbt_dynar_foreach(dax, i, task) { - if (SD_task_get_kind(task) == SD_TASK_COMP_SEQ && SD_task_get_state(task) == SD_SCHEDULABLE) { - xbt_dynar_push(ready_tasks, &task); - } - } - XBT_DEBUG("There are %lu ready tasks", xbt_dynar_length(ready_tasks)); - - return ready_tasks; -} - -static double finish_on_at(const_SD_task_t task, const_sg_host_t host) -{ - double result; - - xbt_dynar_t parents = SD_task_get_parents(task); - - if (!xbt_dynar_is_empty(parents)) { - unsigned int i; - double data_available = 0.; - double last_data_available; - /* compute last_data_available */ - SD_task_t parent; - last_data_available = -1.0; - xbt_dynar_foreach(parents, i, parent) { - /* normal case */ - if (SD_task_get_kind(parent) == SD_TASK_COMM_E2E) { - sg_host_t * parent_host= SD_task_get_workstation_list(parent); - /* Estimate the redistribution time from this parent */ - double redist_time; - if (SD_task_get_amount(parent) <= 1e-6){ - redist_time= 0; - } else { - redist_time = sg_host_get_route_latency(parent_host[0], host) + - SD_task_get_amount(parent) / sg_host_get_route_bandwidth(parent_host[0], host); - } - data_available = SD_task_get_start_time(parent) + redist_time; - } - - /* no transfer, control dependency */ - if (SD_task_get_kind(parent) == SD_TASK_COMP_SEQ) { - data_available = SD_task_get_finish_time(parent); - } - - if (last_data_available < data_available) - last_data_available = data_available; - } - - xbt_dynar_free_container(&parents); - - result = - fmax(sg_host_get_available_at(host), last_data_available) + SD_task_get_amount(task) / sg_host_get_speed(host); - } else { - xbt_dynar_free_container(&parents); - - result = sg_host_get_available_at(host) + SD_task_get_amount(task) / sg_host_get_speed(host); - } - return result; -} - -static sg_host_t SD_task_get_best_host(const_SD_task_t task) -{ - sg_host_t *hosts = sg_host_list(); - int nhosts = sg_host_count(); - sg_host_t best_host = hosts[0]; - double min_EFT = finish_on_at(task, hosts[0]); - - for (int i = 1; i < nhosts; i++) { - double EFT = finish_on_at(task, hosts[i]); - XBT_DEBUG("%s finishes on %s at %f", SD_task_get_name(task), sg_host_get_name(hosts[i]), EFT); - - if (EFT < min_EFT) { - min_EFT = EFT; - best_host = hosts[i]; - } - } - xbt_free(hosts); - return best_host; -} - -int main(int argc, char **argv) -{ - unsigned int cursor; - double min_finish_time = -1.0; - SD_task_t task; - SD_task_t selected_task = NULL; - xbt_dynar_t ready_tasks; - sg_host_t selected_host = NULL; - char * tracefilename = NULL; - - /* initialization of SD */ - SD_init(&argc, argv); - - /* Check our arguments */ - xbt_assert(argc > 2, "Usage: %s platform_file dax_file [jedule_file]\n" - "\tExample: %s simulacrum_7_hosts.xml Montage_25.xml Montage_25.jed", argv[0], argv[0]); - - if (argc == 4) - tracefilename = xbt_strdup(argv[3]); - - /* creation of the environment */ - SD_create_environment(argv[1]); - - /* Allocating the host attribute */ - unsigned int total_nhosts = sg_host_count(); - sg_host_t *hosts = sg_host_list(); - - for (cursor = 0; cursor < total_nhosts; cursor++) - sg_host_set_data(hosts[cursor], xbt_new0(struct _HostAttribute, 1)); - - /* load the DAX file */ - xbt_dynar_t dax = SD_daxload(argv[2]); - - xbt_dynar_foreach(dax, cursor, task) { - SD_task_watch(task, SD_DONE); - } - - /* Schedule the root first */ - xbt_dynar_get_cpy(dax, 0, &task); - sg_host_t host = SD_task_get_best_host(task); - SD_task_schedulel(task, 1, host); - xbt_dynar_t changed_tasks = xbt_dynar_new(sizeof(SD_task_t), NULL); - SD_simulate_with_update(-1.0, changed_tasks); - - while (!xbt_dynar_is_empty(changed_tasks)) { - /* Get the set of ready tasks */ - ready_tasks = get_ready_tasks(dax); - xbt_dynar_reset(changed_tasks); - - if (xbt_dynar_is_empty(ready_tasks)) { - xbt_dynar_free_container(&ready_tasks); - /* there is no ready task, let advance the simulation */ - SD_simulate_with_update(-1.0, changed_tasks); - continue; - } - /* For each ready task: - * get the host that minimizes the completion time. - * select the task that has the minimum completion time on its best host. - */ - xbt_dynar_foreach(ready_tasks, cursor, task) { - XBT_DEBUG("%s is ready", SD_task_get_name(task)); - host = SD_task_get_best_host(task); - double finish_time = finish_on_at(task, host); - if (min_finish_time < 0 || finish_time < min_finish_time) { - min_finish_time = finish_time; - selected_task = task; - selected_host = host; - } - } - - XBT_INFO("Schedule %s on %s", SD_task_get_name(selected_task), sg_host_get_name(selected_host)); - SD_task_schedulel(selected_task, 1, selected_host); - - /* - * SimDag allows tasks to be executed concurrently when they can by default. - * Yet schedulers take decisions assuming that tasks wait for resource availability to start. - * The solution (well crude hack is to keep track of the last task scheduled on a host and add a special type of - * dependency if needed to force the sequential execution meant by the scheduler. - * If the last scheduled task is already done, has failed or is a predecessor of the current task, no need for a - * new dependency - */ - - SD_task_t last_scheduled_task = sg_host_get_last_scheduled_task(selected_host); - if (last_scheduled_task && (SD_task_get_state(last_scheduled_task) != SD_DONE) && - (SD_task_get_state(last_scheduled_task) != SD_FAILED) && - !SD_task_dependency_exists(sg_host_get_last_scheduled_task(selected_host), selected_task)) - SD_task_dependency_add(last_scheduled_task, selected_task); - - sg_host_set_last_scheduled_task(selected_host, selected_task); - sg_host_set_available_at(selected_host, min_finish_time); - - xbt_dynar_free_container(&ready_tasks); - /* reset the min_finish_time for the next set of ready tasks */ - min_finish_time = -1.; - xbt_dynar_reset(changed_tasks); - SD_simulate_with_update(-1.0, changed_tasks); - } - - XBT_INFO("Simulation Time: %f", simgrid_get_clock()); - XBT_INFO("------------------- Produce the trace file---------------------------"); - XBT_INFO("Producing a jedule output (if active) of the run into %s", tracefilename?tracefilename:"minmin_test.jed"); -#if SIMGRID_HAVE_JEDULE - jedule_sd_dump(tracefilename); -#endif - free(tracefilename); - - xbt_dynar_free_container(&ready_tasks); - xbt_dynar_free(&changed_tasks); - - xbt_dynar_foreach(dax, cursor, task) { - SD_task_destroy(task); - } - xbt_dynar_free_container(&dax); - - for (cursor = 0; cursor < total_nhosts; cursor++) { - free(sg_host_get_data(hosts[cursor])); - sg_host_set_data(hosts[cursor], NULL); - } - - xbt_free(hosts); - return 0; -} diff --git a/examples/deprecated/simdag/scheduling/sd_scheduling.tesh b/examples/deprecated/simdag/scheduling/sd_scheduling.tesh deleted file mode 100644 index c2353d7ff0..0000000000 --- a/examples/deprecated/simdag/scheduling/sd_scheduling.tesh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env tesh - -p Simple test of simdag - -$ ${bindir:=.}/scheduling/sd_scheduling --log=sd_daxparse.thresh:critical ${srcdir:=.}/../../platforms/simulacrum_7_hosts.xml ${srcdir:=.}/scheduling/Montage_25.xml -> [0.000000] [xbt_cfg/INFO] Switching to the L07 model to handle parallel tasks. -> [0.000000] [test/INFO] Schedule ID00002@mProjectPP on Host 27 -> [0.000105] [test/INFO] Schedule ID00000@mProjectPP on Host 26 -> [0.000105] [test/INFO] Schedule ID00003@mProjectPP on Host 30 -> [0.000310] [test/INFO] Schedule ID00004@mProjectPP on Host 27 -> [0.000417] [test/INFO] Schedule ID00001@mProjectPP on Host 32 -> [14.576327] [test/INFO] Schedule ID00010@mDiffFit on Host 26 -> [17.041620] [test/INFO] Schedule ID00008@mDiffFit on Host 27 -> [29.541425] [test/INFO] Schedule ID00013@mDiffFit on Host 26 -> [32.620694] [test/INFO] Schedule ID00009@mDiffFit on Host 30 -> [32.620897] [test/INFO] Schedule ID00011@mDiffFit on Host 27 -> [32.620999] [test/INFO] Schedule ID00005@mDiffFit on Host 26 -> [32.620999] [test/INFO] Schedule ID00006@mDiffFit on Host 32 -> [32.621211] [test/INFO] Schedule ID00012@mDiffFit on Host 31 -> [32.621314] [test/INFO] Schedule ID00007@mDiffFit on Host 28 -> [60.411150] [test/INFO] Schedule ID00014@mConcatFit on Host 27 -> [61.202981] [test/INFO] Schedule ID00015@mBgModel on Host 27 -> [62.745210] [test/INFO] Schedule ID00016@mBackground on Host 27 -> [62.745210] [test/INFO] Schedule ID00017@mBackground on Host 26 -> [62.745316] [test/INFO] Schedule ID00020@mBackground on Host 30 -> [62.745423] [test/INFO] Schedule ID00018@mBackground on Host 27 -> [62.745423] [test/INFO] Schedule ID00019@mBackground on Host 32 -> [88.539165] [test/INFO] Schedule ID00021@mImgTbl on Host 27 -> [90.115640] [test/INFO] Schedule ID00022@mAdd on Host 27 -> [93.406326] [test/INFO] Schedule ID00023@mShrink on Host 27 -> [97.602942] [test/INFO] Schedule ID00024@mJPEG on Host 27 -> [98.091829] [test/INFO] Schedule end on Host 27 -> [98.091849] [test/INFO] Simulation Time: 98.091849 -> [98.091849] [test/INFO] ------------------- Produce the trace file--------------------------- -> [98.091849] [test/INFO] Producing a jedule output (if active) of the run into minmin_test.jed - - -! output ignore -$ cmake -E remove -f ${srcdir:=.}/scheduling/sd_scheduling.jed diff --git a/include/simgrid/s4u/Activity.hpp b/include/simgrid/s4u/Activity.hpp index 653533b9e8..2a8504c1eb 100644 --- a/include/simgrid/s4u/Activity.hpp +++ b/include/simgrid/s4u/Activity.hpp @@ -6,7 +6,6 @@ #ifndef SIMGRID_S4U_ACTIVITY_HPP #define SIMGRID_S4U_ACTIVITY_HPP -#include #include #include #include @@ -14,6 +13,8 @@ #include #include #include +#include +#include #include #include @@ -27,12 +28,13 @@ namespace s4u { * This class is the ancestor of every activities that an actor can undertake. * That is, activities are all the things that do take time to the actor in the simulated world. */ -class XBT_PUBLIC Activity { +class XBT_PUBLIC Activity : public xbt::Extendable { friend Comm; friend Exec; friend Io; #ifndef DOXYGEN friend std::vector create_DAG_from_dot(const std::string& filename); + friend std::vector create_DAG_from_DAX(const std::string& filename); #endif public: @@ -113,9 +115,9 @@ public: void complete(Activity::State state) { state_ = state; + on_completion(*this); if (state == State::FINISHED) release_dependencies(); - on_completion(*this); } static std::set* get_vetoed_activities() { return vetoed_activities_; } diff --git a/include/simgrid/s4u/Engine.hpp b/include/simgrid/s4u/Engine.hpp index e57c3b6264..31f6a4088c 100644 --- a/include/simgrid/s4u/Engine.hpp +++ b/include/simgrid/s4u/Engine.hpp @@ -219,6 +219,7 @@ private: }; std::vector create_DAG_from_dot(const std::string& filename); +std::vector create_DAG_from_DAX(const std::string& filename); #ifndef DOXYGEN /* Internal use only, no need to expose it */ template diff --git a/src/simdag/sd_daxloader.cpp b/src/simdag/sd_daxloader.cpp index 91ffa116e7..8cf36445d2 100644 --- a/src/simdag/sd_daxloader.cpp +++ b/src/simdag/sd_daxloader.cpp @@ -6,8 +6,8 @@ #include "simdag_private.hpp" #include "simgrid/s4u/Comm.hpp" +#include "simgrid/s4u/Engine.hpp" #include "simgrid/s4u/Exec.hpp" -#include "simgrid/simdag.h" #include "xbt/file.hpp" #include "xbt/log.h" #include "xbt/misc.h" @@ -21,111 +21,14 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(sd_daxparse, sd, "Parsing DAX files"); /* Ensure that transfer tasks have unique names even though a file is used several times */ -void uniq_transfer_task_name(SD_task_t task) +static void uniq_transfer_task_name(simgrid::s4u::Comm* comm) { - const_SD_task_t child = *(task->get_successors().begin()); - const_SD_task_t parent = *(task->get_dependencies().begin()); + const auto& child = comm->get_successors().front(); + const auto& parent = *(comm->get_dependencies().begin()); - std::string new_name = parent->get_name() + "_" + task->get_name() + "_" + child->get_name(); + std::string new_name = parent->get_name() + "_" + comm->get_name() + "_" + child->get_name(); - task->set_name(new_name); -} - -static bool children_are_marked(const_SD_task_t task) -{ - return std::none_of(task->get_successors().begin(), task->get_successors().end(), - [](const SD_task_t& elm) { return not elm->is_marked(); }); -} - -static bool parents_are_marked(const_SD_task_t task) -{ - return std::none_of(task->get_dependencies().begin(), task->get_dependencies().end(), - [](const SD_task_t& elm) { return not elm->is_marked(); }); -} - -bool acyclic_graph_detail(const_xbt_dynar_t dag) -{ - unsigned int count; - SD_task_t task = nullptr; - std::vector current; - xbt_dynar_foreach (dag, count, task) - if (task->get_kind() != SD_TASK_COMM_E2E && task->is_waited_by() == 0) - current.push_back(task); - - while (not current.empty()) { - std::vector next; - for (auto const& t : current) { - //Mark task - t->mark(); - for (auto const& input : t->get_predecessors()) { - if (input->get_kind() == SD_TASK_COMM_E2E || input->get_kind() == SD_TASK_COMM_PAR_MXN_1D_BLOCK) { - input->mark(); - // Inputs are communication, hence they can have only one predecessor - auto input_pred = *(input->get_dependencies().begin()); - if (children_are_marked(input_pred)) - next.push_back(input_pred); - } - } - for (auto const& pred : t->get_dependencies()) { - if (children_are_marked(pred)) - next.push_back(pred); - } - } - current.clear(); - current = next; - } - - bool all_marked = true; - //test if all tasks are marked - xbt_dynar_foreach(dag,count,task){ - if (task->get_kind() != SD_TASK_COMM_E2E && not task->is_marked()) { - XBT_WARN("the task %s is not marked", task->get_cname()); - all_marked = false; - break; - } - } - - if (not all_marked) { - XBT_VERB("there is at least one cycle in your task graph"); - xbt_dynar_foreach(dag,count,task){ - if (task->get_kind() != SD_TASK_COMM_E2E && task->has_unsolved_dependencies() == 0) { - task->mark(); - current.push_back(task); - } - } - //test if something has to be done for the next iteration - while (not current.empty()) { - std::vector next; - //test if the current iteration is done - for (auto const& t : current) { - t->mark(); - for (auto const& output : t->get_successors()) { - if (output->get_kind() == SD_TASK_COMM_E2E || output->get_kind() == SD_TASK_COMM_PAR_MXN_1D_BLOCK) { - output->mark(); - // outputs are communication, hence they can have only one successor - SD_task_t output_succ = output->get_successors().front(); - if (parents_are_marked(output_succ)) - next.push_back(output_succ); - } - } - for (SD_task_t const& succ : t->get_successors()) { - if (parents_are_marked(succ)) - next.push_back(succ); - } - } - current.clear(); - current = next; - } - - all_marked = true; - xbt_dynar_foreach(dag,count,task){ - if (task->get_kind() != SD_TASK_COMM_E2E && not task->is_marked()) { - XBT_WARN("the task %s is in a cycle", task->get_cname()); - all_marked = false; - } - } - } - return all_marked; + comm->set_name(new_name)->vetoable_start(); } bool check_for_cycle(const std::vector& dag) @@ -164,33 +67,35 @@ bool check_for_cycle(const std::vector& dag) static YY_BUFFER_STATE input_buffer; -static xbt_dynar_t result; -static std::map> jobs; -static std::map> files; -static SD_task_t current_job; +namespace simgrid { +namespace s4u { + +static std::vector result; +static std::map> jobs; +static std::map> files; +static ExecPtr current_job; /** @brief loads a DAX file describing a DAG * * See https://confluence.pegasus.isi.edu/display/pegasus/WorkflowGenerator for more details. */ -xbt_dynar_t SD_daxload(const char *filename) +std::vector create_DAG_from_DAX(const std::string& filename) { - SD_task_t file; - FILE* in_file = fopen(filename, "r"); - xbt_assert(in_file, "Unable to open \"%s\"\n", filename); + FILE* in_file = fopen(filename.c_str(), "r"); + xbt_assert(in_file, "Unable to open \"%s\"\n", filename.c_str()); input_buffer = dax__create_buffer(in_file, 10); dax__switch_to_buffer(input_buffer); dax_lineno = 1; - result = xbt_dynar_new(sizeof(SD_task_t), nullptr); - SD_task_t root_task = SD_task_create_comp_seq("root", nullptr, 0); - /* by design the root task is always SCHEDULABLE */ - root_task->set_state(SD_SCHEDULABLE); + auto root_task = Exec::init()->set_name("root")->set_flops_amount(0); + root_task->vetoable_start(); - xbt_dynar_push(result, &root_task); - SD_task_t end_task = SD_task_create_comp_seq("end", nullptr, 0); + result.push_back(root_task); - xbt_assert(dax_lex() == 0, "Parse error in %s: %s", filename, dax__parse_err_msg()); + auto end_task = Exec::init()->set_name("end")->set_flops_amount(0); + end_task->vetoable_start(); + + xbt_assert(dax_lex() == 0, "Parse error in %s: %s", filename.c_str(), dax__parse_err_msg()); dax__delete_buffer(input_buffer); fclose(in_file); dax_lex_destroy(); @@ -200,37 +105,38 @@ xbt_dynar_t SD_daxload(const char *filename) * Files not produced in the system are said to be produced by root task (top of DAG). * Files not consumed in the system are said to be consumed by end task (bottom of DAG). */ + CommPtr file; for (auto const& elm : files) { file = elm.second; - SD_task_t newfile; - if (file->has_unsolved_dependencies() == 0) { - for (SD_task_t const& it : file->get_successors()) { - newfile = SD_task_create_comm_e2e(file->get_cname(), nullptr, file->get_amount()); - root_task->dependency_add(newfile); - newfile->dependency_add(it); - xbt_dynar_push(result, &newfile); + CommPtr newfile; + if (file->dependencies_solved()) { + for (auto const& it : file->get_successors()) { + newfile = Comm::sendto_init()->set_name(file->get_name())->set_payload_size(file->get_remaining()); + root_task->add_successor(newfile); + newfile->add_successor(it); + result.push_back(newfile); } } if (file->is_waited_by() == 0) { - for (SD_task_t const& it : file->get_dependencies()) { - newfile = SD_task_create_comm_e2e(file->get_cname(), nullptr, file->get_amount()); - it->dependency_add(newfile); - newfile->dependency_add(end_task); - xbt_dynar_push(result, &newfile); + for (auto const& it : file->get_dependencies()) { + newfile = Comm::sendto_init()->set_name(file->get_name())->set_payload_size(file->get_remaining()); + it->add_successor(newfile); + newfile->add_successor(end_task); + result.push_back(newfile); } } - for (SD_task_t const& it : file->get_dependencies()) { - for (SD_task_t const& it2 : file->get_successors()) { + for (auto const& it : file->get_dependencies()) { + for (auto const& it2 : file->get_successors()) { if (it == it2) { XBT_WARN("File %s is produced and consumed by task %s." "This loop dependency will prevent the execution of the task.", file->get_cname(), it->get_cname()); } - newfile = SD_task_create_comm_e2e(file->get_cname(), nullptr, file->get_amount()); - it->dependency_add(newfile); - newfile->dependency_add(it2); - xbt_dynar_push(result, &newfile); + newfile = Comm::sendto_init()->set_name(file->get_name())->set_payload_size(file->get_remaining()); + it->add_successor(newfile); + newfile->add_successor(it2); + result.push_back(newfile); } } /* Free previous copy of the files */ @@ -238,37 +144,39 @@ xbt_dynar_t SD_daxload(const char *filename) } /* Push end task last */ - xbt_dynar_push(result, &end_task); + result.push_back(end_task); - unsigned int cpt; - xbt_dynar_foreach(result, cpt, file) { - if (file->get_kind() == SD_TASK_COMM_E2E) { - uniq_transfer_task_name(file); + for (const auto& a : result) { + auto* comm = dynamic_cast(a.get()); + if (comm != nullptr) { + uniq_transfer_task_name(comm); } else { /* If some tasks do not take files as input, connect them to the root * if they don't produce files, connect them to the end node. */ - if ((file != root_task) && (file != end_task)) { - if (file->has_unsolved_dependencies() == 0) - root_task->dependency_add(file); - if (file->is_waited_by() == 0) - file->dependency_add(end_task); + if ((a != root_task) && (a != end_task)) { + if (a->dependencies_solved()) + root_task->add_successor(a); + if (a->is_waited_by() == 0) + a->add_successor(end_task); } } } - if (not acyclic_graph_detail(result)) { + if (not check_for_cycle(result)) { XBT_ERROR("The DAX described in %s is not a DAG. It contains a cycle.", simgrid::xbt::Path(filename).get_base_name().c_str()); - xbt_dynar_foreach(result, cpt, file) - file->destroy(); - xbt_dynar_free_container(&result); - result = nullptr; + for (const auto& a : result) + a->destroy(); + result.clear(); } return result; } +} // namespace s4u +} // namespace simgrid + void STag_dax__adag() { try { @@ -287,9 +195,9 @@ void STag_dax__job() std::string name = std::string(A_dax__job_id) + "@" + A_dax__job_name; runtime *= 4200000000.; /* Assume that timings were done on a 4.2GFlops machine. I mean, why not? */ XBT_DEBUG("See ", A_dax__job_id, A_dax__job_runtime, runtime); - current_job = SD_task_create_comp_seq(name.c_str(), nullptr, runtime); - jobs.insert({A_dax__job_id, current_job}); - xbt_dynar_push(result, ¤t_job); + simgrid::s4u::current_job = simgrid::s4u::Exec::init()->set_name(name)->set_flops_amount(runtime)->vetoable_start(); + simgrid::s4u::jobs.insert({A_dax__job_id, simgrid::s4u::current_job}); + simgrid::s4u::result.push_back(simgrid::s4u::current_job); } catch (const std::invalid_argument&) { throw std::invalid_argument(std::string("Parse error: ") + A_dax__job_runtime + " is not a double"); } @@ -306,33 +214,32 @@ void STag_dax__uses() bool is_input = (A_dax__uses_link == A_dax__uses_link_input); XBT_DEBUG("See ",A_dax__uses_file,(is_input?"in":"out")); - auto it = files.find(A_dax__uses_file); - SD_task_t file; - if (it == files.end()) { - file = SD_task_create_comm_e2e(A_dax__uses_file, nullptr, size); - sd_global->initial_tasks.erase(file); - files[A_dax__uses_file] = file; + auto it = simgrid::s4u::files.find(A_dax__uses_file); + simgrid::s4u::CommPtr file; + if (it == simgrid::s4u::files.end()) { + file = simgrid::s4u::Comm::sendto_init()->set_name(A_dax__uses_file)->set_payload_size(size); + simgrid::s4u::files[A_dax__uses_file] = file.get(); } else { file = it->second; - if (file->get_amount() < size || file->get_amount() > size) { - XBT_WARN("Ignore file %s size redefinition from %.0f to %.0f", A_dax__uses_file, file->get_amount(), size); + if (file->get_remaining() < size || file->get_remaining() > size) { + XBT_WARN("Ignore file %s size redefinition from %.0f to %.0f", A_dax__uses_file, file->get_remaining(), size); } } if (is_input) { - file->dependency_add(current_job); + file->add_successor(simgrid::s4u::current_job); } else { - current_job->dependency_add(file); - if (file->has_unsolved_dependencies() > 1) { + simgrid::s4u::current_job->add_successor(file); + if (file->get_dependencies().size() > 1) { XBT_WARN("File %s created at more than one location...", file->get_cname()); } } } -static SD_task_t current_child; +static simgrid::s4u::ExecPtr current_child; void STag_dax__child() { - auto job = jobs.find(A_dax__child_ref); - if (job != jobs.end()) { + auto job = simgrid::s4u::jobs.find(A_dax__child_ref); + if (job != simgrid::s4u::jobs.end()) { current_child = job->second; } else { throw std::out_of_range(std::string("Parse error on line ") + std::to_string(dax_lineno) + @@ -347,10 +254,10 @@ void ETag_dax__child() void STag_dax__parent() { - auto job = jobs.find(A_dax__parent_ref); - if (job != jobs.end()) { + auto job = simgrid::s4u::jobs.find(A_dax__parent_ref); + if (job != simgrid::s4u::jobs.end()) { auto parent = job->second; - parent->dependency_add(current_child); + parent->add_successor(current_child); XBT_DEBUG("Control-flow dependency from %s to %s", current_child->get_cname(), parent->get_cname()); } else { throw std::out_of_range(std::string("Parse error on line ") + std::to_string(dax_lineno) + @@ -366,7 +273,7 @@ void ETag_dax__adag() void ETag_dax__job() { - current_job = nullptr; + simgrid::s4u::current_job = nullptr; XBT_DEBUG("See "); } diff --git a/src/simdag/simdag_private.hpp b/src/simdag/simdag_private.hpp index ef9e87da54..5a19542891 100644 --- a/src/simdag/simdag_private.hpp +++ b/src/simdag/simdag_private.hpp @@ -173,7 +173,5 @@ extern XBT_PRIVATE std::unique_ptr sd_global; /* SimDag private functions */ XBT_PRIVATE bool check_for_cycle(const std::vector& dag); -XBT_PRIVATE bool acyclic_graph_detail(const_xbt_dynar_t dag); -XBT_PRIVATE void uniq_transfer_task_name(SD_task_t task); XBT_PRIVATE const char *__get_state_name(e_SD_task_state_t state); #endif diff --git a/tools/cmake/DefinePackages.cmake b/tools/cmake/DefinePackages.cmake index 22263af796..9669d1c132 100644 --- a/tools/cmake/DefinePackages.cmake +++ b/tools/cmake/DefinePackages.cmake @@ -1057,7 +1057,6 @@ set(CMAKEFILES_TXT examples/python/CMakeLists.txt examples/deprecated/java/CMakeLists.txt examples/deprecated/msg/mc/CMakeLists.txt - examples/deprecated/simdag/CMakeLists.txt teshsuite/java/CMakeLists.txt teshsuite/kernel/CMakeLists.txt -- 2.20.1