From: SUTER Frederic Date: Thu, 23 Dec 2021 12:50:56 +0000 (+0100) Subject: first attempt to a DAG loader for s4u. To be improved and polished before release X-Git-Tag: v3.30~197 X-Git-Url: http://bilbo.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/commitdiff_plain/db66286ed61be507f467174663aa19a9dfa8612b first attempt to a DAG loader for s4u. To be improved and polished before release --- diff --git a/MANIFEST.in b/MANIFEST.in index 3b8df1dd53..70382b68d3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -206,6 +206,9 @@ include examples/cpp/dag-comm/s4u-dag-comm.cpp include examples/cpp/dag-comm/s4u-dag-comm.tesh include examples/cpp/dag-failure/s4u-dag-failure.cpp include examples/cpp/dag-failure/s4u-dag-failure.tesh +include examples/cpp/dag-from-dot/dag.dot +include examples/cpp/dag-from-dot/s4u_dag-frm-dot.cpp +include examples/cpp/dag-from-dot/s4u_dag-frm-dot.tesh include examples/cpp/dag-io/s4u-dag-io.cpp include examples/cpp/dag-io/s4u-dag-io.tesh include examples/cpp/dag-simple/s4u-dag-simple.cpp diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 4b19e362b0..db1d78aa0c 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -28,6 +28,21 @@ foreach (example ${MC_regular_tests}) set(_${example}_factories "^thread") endforeach() +if(HAVE_GRAPHVIZ) + add_executable (s4u_dag-from-dot EXCLUDE_FROM_ALL dag-from-dot/s4u_dag-from-dot.cpp) + target_link_libraries(s4u_dag-from-dot simgrid) + set_target_properties(s4u_dag-from-dot PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dag-from-dot) + add_dependencies(tests s4u_dag-from-dot) + + ADD_TESH(s4u-dag-from-dot --setenv srcdir=${CMAKE_HOME_DIRECTORY}/examples/cpp/dag-from-dot + --setenv platfdir=${CMAKE_HOME_DIRECTORY}/examples/platforms + --cd ${CMAKE_BINARY_DIR}/examples/cpp/dag-from-dot + ${CMAKE_HOME_DIRECTORY}/examples/cpp/dag-from-dot/s4u_dag-from-dot.tesh) + endif() + set(examples_src ${examples_src} ${CMAKE_CURRENT_SOURCE_DIR}/dag-from-dot/s4u_dag-frm-dot.cpp) + set(tesh_files ${tesh_files} ${CMAKE_CURRENT_SOURCE_DIR}/dag-from-dot/s4u_dag-frm-dot.tesh) + + if(SIMGRID_HAVE_MC) add_executable (s4u-mc-bugged1-liveness EXCLUDE_FROM_ALL mc-bugged1-liveness/s4u-mc-bugged1-liveness.cpp) target_link_libraries(s4u-mc-bugged1-liveness simgrid) @@ -277,7 +292,8 @@ set(xml_files ${xml_files} ${CMAKE_CURRENT_SOURCE_DIR}/actor-create/s4u-a set(bin_files ${bin_files} ${CMAKE_CURRENT_SOURCE_DIR}/dht-kademlia/generate.py ${CMAKE_CURRENT_SOURCE_DIR}/mc-bugged1-liveness/s4u-mc-bugged1-liveness-stack-cleaner ${CMAKE_CURRENT_SOURCE_DIR}/mc-bugged1-liveness/promela_bugged1_liveness PARENT_SCOPE) -set(txt_files ${txt_files} ${CMAKE_CURRENT_SOURCE_DIR}/replay-comm/s4u-replay-comm-split-p0.txt +set(txt_files ${txt_files} ${CMAKE_CURRENT_SOURCE_DIR}/dag-from-dot/dag.dot + ${CMAKE_CURRENT_SOURCE_DIR}/replay-comm/s4u-replay-comm-split-p0.txt ${CMAKE_CURRENT_SOURCE_DIR}/replay-comm/s4u-replay-comm-split-p1.txt ${CMAKE_CURRENT_SOURCE_DIR}/replay-comm/s4u-replay-comm.txt ${CMAKE_CURRENT_SOURCE_DIR}/replay-io/s4u-replay-io.txt PARENT_SCOPE) diff --git a/examples/cpp/dag-from-dot/dag.dot b/examples/cpp/dag-from-dot/dag.dot new file mode 100644 index 0000000000..32f4c49259 --- /dev/null +++ b/examples/cpp/dag-from-dot/dag.dot @@ -0,0 +1,24 @@ +digraph G { +end [size="10000000129"]; +0 [size="10000000129"]; +1 [size="10000000131"]; +2 [size="10000000121"]; +3 [size="10000000230"]; +4 [size="10000000004"]; +5 [size="10000000046"]; +6 [size="10000000091"]; +7 [size="10000000040"]; +8 [size="10000000250"]; +9 [size="10000000079"]; +0->1 [size="10001"]; +1->2 [size="10004"]; +2->3 [size="10001"]; +3->4 [size="-1"]; +4->5 [size="10029"]; +5->6 [size="0.0"]; +6->7 [size="10004"]; +7->8 [size="10000"]; +8->9 ; +7->end [size="10014000"]; +root->5 [size="10014000"]; +} diff --git a/examples/cpp/dag-from-dot/s4u_dag-from-dot.cpp b/examples/cpp/dag-from-dot/s4u_dag-from-dot.cpp new file mode 100644 index 0000000000..deb430ecf6 --- /dev/null +++ b/examples/cpp/dag-from-dot/s4u_dag-from-dot.cpp @@ -0,0 +1,71 @@ +/* simple test trying to load a DOT file. */ + +/* Copyright (c) 2010-2021. The SimGrid Team. + * All rights reserved. */ + +/* This program is free software; you can redistribute it and/or modify it + * under the terms of the license (GNU LGPL) which comes with this package. */ + +#include "simgrid/s4u.hpp" +#include + +XBT_LOG_NEW_DEFAULT_CATEGORY(dag_from_dot, "Logging specific to this example"); + +int main(int argc, char** argv) +{ + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + + std::vector dag = simgrid::s4u::create_DAG_from_dot(argv[2]); + + XBT_INFO("--------- Display all activities of the loaded DAG -----------"); + for (const auto& a : dag) { + std::string type = "an Exec"; + std::string task = "flops to execute"; + if (dynamic_cast(a.get()) != nullptr) { + type = "a Comm"; + task = "bytes to transfer"; + } + XBT_INFO("'%s' is %s: %.0f %s. Dependencies: %s; Ressources: %s", a->get_cname(), type.c_str(), a->get_remaining(), + task.c_str(), (a->dependencies_solved() ? "solved" : "NOT solved"), + (a->is_assigned() ? "assigned" : "NOT assigned")); + } + + XBT_INFO("------------------- Schedule tasks ---------------------------"); + auto hosts = e.get_all_hosts(); + auto count = e.get_host_count(); + int cursor = 0; + // Schedule end first + static_cast(dag.back().get())->set_host(hosts[0]); + + for (const auto& a : dag) { + auto* exec = dynamic_cast(a.get()); + if (exec != nullptr && exec->get_name() != "end") { + exec->set_host(hosts[cursor % count]); + cursor++; + } + auto* comm = dynamic_cast(a.get()); + if (comm != nullptr) { + auto pred = dynamic_cast(comm->get_parent().get()); + auto succ = dynamic_cast(comm->get_child().get()); + comm->set_from(pred->get_host())->set_to(succ->get_host()); + } + } + + XBT_INFO("------------------- Run the schedule -------------------------"); + e.run(); + + XBT_INFO("-------------- Summary of executed schedule ------------------"); + for (const auto& a : dag) { + auto* exec = dynamic_cast(a.get()); + if (exec != nullptr) { + XBT_INFO("[%f->%f] '%s' executed on %s", exec->get_start_time(), exec->get_finish_time(), exec->get_cname(), + exec->get_host()->get_cname()); + } + auto* comm = dynamic_cast(a.get()); + if (comm != nullptr) { + XBT_INFO("%s", comm->get_cname()); + } + } + return 0; +} diff --git a/examples/cpp/dag-from-dot/s4u_dag-from-dot.tesh b/examples/cpp/dag-from-dot/s4u_dag-from-dot.tesh new file mode 100644 index 0000000000..349a97a18d --- /dev/null +++ b/examples/cpp/dag-from-dot/s4u_dag-from-dot.tesh @@ -0,0 +1,47 @@ +#!/usr/bin/env tesh + +$ ${bindir:=.}/s4u_dag-from-dot --log=no_loc ${platfdir}/cluster_backbone.xml ${srcdir:=.}/dag.dot +> [0.000000] [dag_from_dot/INFO] --------- Display all activities of the loaded DAG ----------- +> [0.000000] [dag_from_dot/INFO] 'root' is an Exec: 0 flops to execute. Dependencies: solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '0' is an Exec: 10000000129 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '1' is an Exec: 10000000131 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '2' is an Exec: 10000000121 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '3' is an Exec: 10000000230 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '4' is an Exec: 10000000004 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '5' is an Exec: 10000000046 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '6' is an Exec: 10000000091 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '7' is an Exec: 10000000040 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '8' is an Exec: 10000000250 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '9' is an Exec: 10000000079 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '0->1' is a Comm: 10001 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '1->2' is a Comm: 10004 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '2->3' is a Comm: 10001 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '4->5' is a Comm: 10029 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '6->7' is a Comm: 10004 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '7->8' is a Comm: 10000 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] '7->end' is a Comm: 10014000 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] 'root->5' is a Comm: 10014000 bytes to transfer. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] 'end' is an Exec: 10000000129 flops to execute. Dependencies: NOT solved; Ressources: NOT assigned +> [0.000000] [dag_from_dot/INFO] ------------------- Schedule tasks --------------------------- +> [0.000000] [dag_from_dot/INFO] ------------------- Run the schedule ------------------------- +> [110.047415] [dag_from_dot/INFO] -------------- Summary of executed schedule ------------------ +> [110.047415] [dag_from_dot/INFO] [0.000000->0.000000] 'root' executed on node-0.simgrid.org +> [110.047415] [dag_from_dot/INFO] [0.000000->10.000000] '0' executed on node-1.simgrid.org +> [110.047415] [dag_from_dot/INFO] [10.007889->20.007889] '1' executed on node-10.simgrid.org +> [110.047415] [dag_from_dot/INFO] [20.015777->30.015777] '2' executed on node-11.simgrid.org +> [110.047415] [dag_from_dot/INFO] [30.023666->40.023666] '3' executed on node-12.simgrid.org +> [110.047415] [dag_from_dot/INFO] [40.023666->50.023666] '4' executed on node-13.simgrid.org +> [110.047415] [dag_from_dot/INFO] [50.031555->60.031555] '5' executed on node-14.simgrid.org +> [110.047415] [dag_from_dot/INFO] [60.031555->70.031555] '6' executed on node-15.simgrid.org +> [110.047415] [dag_from_dot/INFO] [70.039443->80.039443] '7' executed on node-16.simgrid.org +> [110.047415] [dag_from_dot/INFO] [80.047414->90.047415] '8' executed on node-17.simgrid.org +> [110.047415] [dag_from_dot/INFO] [90.047415->100.047415] '9' executed on node-18.simgrid.org +> [110.047415] [dag_from_dot/INFO] 0->1 +> [110.047415] [dag_from_dot/INFO] 1->2 +> [110.047415] [dag_from_dot/INFO] 2->3 +> [110.047415] [dag_from_dot/INFO] 4->5 +> [110.047415] [dag_from_dot/INFO] 6->7 +> [110.047415] [dag_from_dot/INFO] 7->8 +> [110.047415] [dag_from_dot/INFO] 7->end +> [110.047415] [dag_from_dot/INFO] root->5 +> [110.047415] [dag_from_dot/INFO] [100.047415->110.047415] 'end' executed on node-0.simgrid.org diff --git a/include/simgrid/s4u/Engine.hpp b/include/simgrid/s4u/Engine.hpp index 0703c0c201..20f6ee873b 100644 --- a/include/simgrid/s4u/Engine.hpp +++ b/include/simgrid/s4u/Engine.hpp @@ -218,6 +218,8 @@ private: void initialize(int* argc, char** argv); }; +std::vector create_DAG_from_dot(const std::string& filename); + #ifndef DOXYGEN /* Internal use only, no need to expose it */ template XBT_PRIVATE void get_filtered_netzones_recursive(const s4u::NetZone* current, std::vector* whereto) diff --git a/src/simdag/sd_dotloader.cpp b/src/simdag/sd_dotloader.cpp index 959c60d89e..f671c6320d 100644 --- a/src/simdag/sd_dotloader.cpp +++ b/src/simdag/sd_dotloader.cpp @@ -5,7 +5,10 @@ * under the terms of the license (GNU LGPL) which comes with this package. */ #include "simdag_private.hpp" +#include "simgrid/s4u/Activity.hpp" +#include "simgrid/s4u/Comm.hpp" #include "simgrid/s4u/Engine.hpp" +#include "simgrid/s4u/Exec.hpp" #include "simgrid/simdag.h" #include "src/internal_config.h" #include "xbt/file.hpp" @@ -19,6 +22,110 @@ XBT_LOG_NEW_DEFAULT_SUBCATEGORY(sd_dotparse, sd, "Parsing DOT files"); +namespace simgrid { +namespace s4u { + +std::vector create_DAG_from_dot(const std::string& filename) +{ + FILE* in_file = fopen(filename.c_str(), "r"); + xbt_assert(in_file != nullptr, "Failed to open file: %s", filename.c_str()); + + Agraph_t* dag_dot = agread(in_file, NIL(Agdisc_t*)); + + std::unordered_map activities; + std::vector dag; + + ActivityPtr root; + ActivityPtr end; + ActivityPtr act; + /* Create all the nodes */ + Agnode_t* node = nullptr; + for (node = agfstnode(dag_dot); node; node = agnxtnode(dag_dot, node)) { + char* name = agnameof(node); + double amount = atof(agget(node, (char*)"size")); + + if (activities.find(name) == activities.end()) { + XBT_DEBUG("See ", name, amount); + act = Exec::init()->set_name(name)->set_flops_amount(amount)->vetoable_start(); + activities.insert({std::string(name), act}); + if (strcmp(name, "root") && strcmp(name, "end")) + dag.push_back(act); + } else { + XBT_WARN("Exec '%s' is defined more than once", name); + } + } + /*Check if 'root' and 'end' nodes have been explicitly declared. If not, create them. */ + if (activities.find("root") == activities.end()) + root = Exec::init()->set_name("root")->set_flops_amount(0)->vetoable_start(); + else + root = activities.at("root"); + + if (activities.find("end") == activities.end()) + end = Exec::init()->set_name("end")->set_flops_amount(0)->vetoable_start(); + else + end = activities.at("end"); + + /* Create edges */ + std::vector edges; + for (node = agfstnode(dag_dot); node; node = agnxtnode(dag_dot, node)) { + edges.clear(); + for (Agedge_t* edge = agfstout(dag_dot, node); edge; edge = agnxtout(dag_dot, edge)) + edges.push_back(edge); + + /* Be sure edges are sorted */ + std::sort(edges.begin(), edges.end(), [](const Agedge_t* a, const Agedge_t* b) { return AGSEQ(a) < AGSEQ(b); }); + + for (Agedge_t* edge : edges) { + const char* src_name = agnameof(agtail(edge)); + const char* dst_name = agnameof(aghead(edge)); + double size = atof(agget(edge, (char*)"size")); + + ActivityPtr src = activities.at(src_name); + ActivityPtr dst = activities.at(dst_name); + if (size > 0) { + std::string name = std::string(src_name) + "->" + dst_name; + XBT_DEBUG("See ", name.c_str(), size); + if (activities.find(name) == activities.end()) { + act = Comm::sendto_init()->set_name(name)->set_payload_size(size)->vetoable_start(); + src->add_successor(act); + act->add_successor(dst); + activities.insert({name, act}); + dag.push_back(act); + } else { + XBT_WARN("Comm '%s' is defined more than once", name.c_str()); + } + } else { + src->add_successor(dst); + } + } + } + + XBT_DEBUG("All activities have been created, put %s at the beginning and %s at the end", root->get_cname(), + end->get_cname()); + dag.insert(dag.begin(), root); + dag.push_back(end); + + /* Connect entry tasks to 'root', and exit tasks to 'end'*/ + for (const auto& a : dag) { + if (a->dependencies_solved() && a != root) { + XBT_DEBUG("Activity '%s' has no dependencies. Add dependency from 'root'", a->get_cname()); + root->add_successor(a); + } + + if (a->is_waited_by() == 0 && a != end) { + XBT_DEBUG("Activity '%s' has no successors. Add dependency to 'end'", a->get_cname()); + a->add_successor(end); + } + } + agclose(dag_dot); + fclose(in_file); + + return dag; +} + +} // namespace s4u +} // namespace simgrid + xbt_dynar_t SD_dotload_generic(const char* filename, bool sequential, bool schedule); static void dot_task_p_free(void *task) { @@ -236,6 +343,16 @@ xbt_dynar_t SD_dotload_generic(const char* filename, bool sequential, bool sched return result; } #else +namespace simgrid { +namespace s4u { +std::vector create_DAG_from_dot(const std::string& filename) +{ + xbt_die("create_DAG_from_dot() is not usable because graphviz was not found.\n" + "Please install graphviz, graphviz-dev, and libgraphviz-dev (and erase CMakeCache.txt) before recompiling."); +} +// namespace s4u +// namespace simgrid + xbt_dynar_t SD_dotload(const char *filename) { xbt_die("SD_dotload_generic() is not usable because graphviz was not found.\n" "Please install graphviz, graphviz-dev, and libgraphviz-dev (and erase CMakeCache.txt) before recompiling.");