From 319324b9226c85a78ccc2faea87be6de86cf6056 Mon Sep 17 00:00:00 2001 From: Adrien Date: Mon, 13 Mar 2023 21:12:47 +0000 Subject: [PATCH] Add wfformat json DAG loader and DAG doc --- docs/source/Tutorial_DAG.rst | 418 +++++++++++ docs/source/index.rst | 1 + docs/source/tuto_dag/dag_lab1.cpp | 48 ++ docs/source/tuto_dag/dag_lab2-1.cpp | 37 + docs/source/tuto_dag/dag_lab2-2.cpp | 19 + docs/source/tuto_dag/dag_lab2-3.cpp | 37 + docs/source/tuto_dag/img/dag.svg | 955 ++++++++++++++++++++++++ docs/source/tuto_dag/img/dag1.svg | 196 +++++ docs/source/tuto_dag/img/dag2.svg | 398 ++++++++++ docs/source/tuto_dag/simple_dax.xml | 20 + docs/source/tuto_dag/simple_dot.dot | 10 + docs/source/tuto_dag/simple_json.json | 42 ++ docs/source/tuto_dag/small_platform.xml | 194 +++++ include/simgrid/s4u/Activity.hpp | 1 + include/simgrid/s4u/Engine.hpp | 1 + src/dag/loaders.cpp | 72 ++ 16 files changed, 2449 insertions(+) create mode 100644 docs/source/Tutorial_DAG.rst create mode 100644 docs/source/tuto_dag/dag_lab1.cpp create mode 100644 docs/source/tuto_dag/dag_lab2-1.cpp create mode 100644 docs/source/tuto_dag/dag_lab2-2.cpp create mode 100644 docs/source/tuto_dag/dag_lab2-3.cpp create mode 100644 docs/source/tuto_dag/img/dag.svg create mode 100644 docs/source/tuto_dag/img/dag1.svg create mode 100644 docs/source/tuto_dag/img/dag2.svg create mode 100644 docs/source/tuto_dag/simple_dax.xml create mode 100644 docs/source/tuto_dag/simple_dot.dot create mode 100644 docs/source/tuto_dag/simple_json.json create mode 100644 docs/source/tuto_dag/small_platform.xml diff --git a/docs/source/Tutorial_DAG.rst b/docs/source/Tutorial_DAG.rst new file mode 100644 index 0000000000..e807cc4f29 --- /dev/null +++ b/docs/source/Tutorial_DAG.rst @@ -0,0 +1,418 @@ +.. _simdag: + +Simulating DAG +============== + +This tutorial presents the basics to understand how DAG are represented in Simgrid and how to simulate their workflow. + +Definition of a DAG +------------------- + +Directed Acyclic Graph: + +.. math:: + + \mathcal{G} = (\mathcal{V},\mathcal{E}) + +Set of vertices representing :ref:`Activities `: + +.. math:: + + \mathcal{V} = {v_i | i = 1, ..., V} + +Set of edges representing precedence constraints between :ref:`Activities `: + +.. math:: + + \mathcal{E} = {e_i,j | (i,j) \in {1, ..., V} x {1, ..., V}} + +.. image:: /tuto_dag/img/dag.svg + :align: center + +Representing Vertices/Activities +................................ + +There is two types of :ref:`Activities ` that can represent Vertices: :ref:`Exec ` and :ref:`Comm `. +Thoses activities must be initiated and configured to properly describe your worflow. + +An Exec represents the execution of an amount of flop on a :ref:`Host ` of your platform. + +.. code-block:: cpp + + ExecPtr exec = Exec::init(); + exec->set_flops_amount(int); + exec->set_host(Host*); + exec->start(); + +A Comm represents a data transfer between two :ref:`Hosts ` of your platform. + +.. code-block:: cpp + + CommPtr comm = Comm::sendto_init(); + comm->set_source(Host*); + comm->set_destination(Host*); + comm->start(); + +Representing Edges/Dependencies +............................... + +An activity will not start until all of its dependencies have been completed. +Activities may have any number of successors. +Dependencies between Activities are created using :cpp:func:`Activity::add_successor(ActivityPtr)`. + +.. code-block:: cpp + + exec->add_successor(comm); + +The Activity ``comm`` will not start until ``exec`` has been completed. + +Lab 1: Basics +--------------- + +The goal of this lab is to describe the following DAG: + +.. image:: /tuto_dag/img/dag1.svg + :align: center + +In this DAG we want ``c1`` to compute 1e9 flops, ``c2`` to compute 5e9 flops and ``c3`` to compute 2e9 flops. +There is also a data transfer of 5e8 bytes between ``c1`` and ``c3``. + +First of all, include the Simgrid library and define the log category. + +.. code-block:: cpp + + #include "simgrid/s4u.hpp" + + XBT_LOG_NEW_DEFAULT_CATEGORY(main, "Messages specific for this s4u tutorial"); + +Inside the ``main`` function create an instance of :ref:`Engine ` and load the platform. + +.. code-block:: cpp + + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + +Retrieve pointers to some hosts. + +.. code-block:: cpp + + simgrid::s4u::Host* tremblay = e.host_by_name("Tremblay"); + simgrid::s4u::Host* jupiter = e.host_by_name("Jupiter"); + +Initiate the activities. + +.. code-block:: cpp + + simgrid::s4u::ExecPtr c1 = simgrid::s4u::Exec::init(); + simgrid::s4u::ExecPtr c2 = simgrid::s4u::Exec::init(); + simgrid::s4u::ExecPtr c3 = simgrid::s4u::Exec::init(); + simgrid::s4u::CommPtr t1 = simgrid::s4u::Comm::sendto_init(); + +Give names to thoses activities. + +.. code-block:: cpp + + c1->set_name("c1"); + c2->set_name("c2"); + c3->set_name("c3"); + t1->set_name("t1"); + +Set the amount of work for each activity. + +.. code-block:: cpp + + c1->set_flops_amount(1e9); + c2->set_flops_amount(5e9); + c3->set_flops_amount(2e9); + t1->set_payload_size(5e8); + +Define the dependencies between the activities. + +.. code-block:: cpp + + c1->add_successor(t1); + t1->add_successor(c3); + c2->add_successor(c3); + +Set the location of each Exec activity and source and destination for the Comm activity. + +.. code-block:: cpp + + c1->set_host(tremblay); + c2->set_host(jupiter); + c3->set_host(jupiter); + t1->set_source(tremblay); + t1->set_destination(jupiter); + +Start the executions of Activities without dependencies. + +.. code-block:: cpp + + c1->start(); + c2->start(); + +Add a callback to monitor the activities. + +.. code-block:: cpp + + Activity::on_completion_cb([](simgrid::s4u::Activity const& activity) { + XBT_INFO("Activity '%s' is complete (start time: %f, finish time: %f)", activity.get_cname(), activity.get_start_time(), + activity.get_finish_time()); + }); + +Finally, run the simulation. + +.. code-block:: cpp + + e.run(); + +The execution of this code should give you the following output: + +.. code-block:: bash + + [10.194200] [main/INFO] Activity 'c1' is complete (start time: 0.000000, finish time: 10.194200) + [65.534235] [main/INFO] Activity 'c2' is complete (start time: 0.000000, finish time: 65.534235) + [85.283378] [main/INFO] Activity 't1' is complete (start time: 10.194200, finish time: 85.283378) + [111.497072] [main/INFO] Activity 'c3' is complete (start time: 85.283378, finish time: 111.497072) + +Lab 2: Import a DAG from a file +--------------- + +In this lab we present how to import a DAG into you Simgrid simulation, either using a DOT file, a JSON file, or a DAX file. + +The files presented in this lab describe the following DAG: + +.. image:: /tuto_dag/img/dag2.svg + :align: center + +From a DOT file +............... + +A DOT file describes a workflow in accordance with the graphviz format. + +The following DOT file describes the workflow presented at the beginning of this lab: + +.. code-block:: xml + + digraph G { + c1 [size="1e9"]; + c2 [size="5e9"]; + c3 [size="2e9"]; + + root->c1 [size="2e8"]; + root->c2 [size="1e8"]; + c1->c3 [size="5e8"]; + c2->c3 [size="-1"]; + c3->end [size="2e8"]; + } + +It can be imported as a vector of Activities into Simgrid using :cpp:func:`create_DAG_from_DOT(const std::string& filename)`. Then, you have to assign hosts to your Activities. + +.. code-block:: cpp + + #include "simgrid/s4u.hpp" + + XBT_LOG_NEW_DEFAULT_CATEGORY(main, "Messages specific for this s4u example"); + + int main(int argc, char* argv[]) { + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + + std::vector dag = simgrid::s4u::create_DAG_from_dot(argv[2]); + + simgrid::s4u::Host* tremblay = e.host_by_name("Tremblay"); + simgrid::s4u::Host* jupiter = e.host_by_name("Jupiter"); + simgrid::s4u::Host* fafard = e.host_by_name("Fafard"); + + dynamic_cast(dag[0].get())->set_host(fafard); + dynamic_cast(dag[1].get())->set_host(tremblay); + dynamic_cast(dag[2].get())->set_host(jupiter); + dynamic_cast(dag[3].get())->set_host(jupiter); + dynamic_cast(dag[8].get())->set_host(jupiter); + + for (const auto& a : dag) { + if (auto* comm = dynamic_cast(a.get())) { + auto pred = dynamic_cast((*comm->get_dependencies().begin()).get()); + auto succ = dynamic_cast(comm->get_successors().front().get()); + comm->set_source(pred->get_host())->set_destination(succ->get_host()); + } + } + + simgrid::s4u::Activity::on_completion_cb([](simgrid::s4u::Activity const& activity) { + XBT_INFO("Activity '%s' is complete (start time: %f, finish time: %f)", activity.get_cname(), activity.get_start_time(), + activity.get_finish_time()); + }); + + e.run(); + return 0; + } + +The execution of this code should give you the following output: + +.. code-block:: bash + + [0.000000] [main/INFO] Activity 'root' is complete (start time: 0.000000, finish time: 0.000000) + [33.394394] [main/INFO] Activity 'root->c2' is complete (start time: 0.000000, finish time: 33.394394) + [39.832311] [main/INFO] Activity 'root->c1' is complete (start time: 0.000000, finish time: 39.832311) + [50.026511] [main/INFO] Activity 'c1' is complete (start time: 39.832311, finish time: 50.026511) + [98.928629] [main/INFO] Activity 'c2' is complete (start time: 33.394394, finish time: 98.928629) + [125.115689] [main/INFO] Activity 'c1->c3' is complete (start time: 50.026511, finish time: 125.115689) + [151.329383] [main/INFO] Activity 'c3' is complete (start time: 125.115689, finish time: 151.329383) + [151.743605] [main/INFO] Activity 'c3->end' is complete (start time: 151.329383, finish time: 151.743605) + [151.743605] [main/INFO] Activity 'end' is complete (start time: 151.743605, finish time: 151.743605) + +From a JSON file +................ + +A JSON file describes a workflow in accordance with the `wfformat `_ . + +The following JSON file describes the workflow presented at the beginning of this lab: + +.. code-block:: JSON + + { + "name": "simple_json", + "schemaVersion": "1.0", + "workflow": { + "makespan": 0, + "executedAt": "2023-03-09T00:00:00-00:00", + "tasks": [ + { + "name": "c1", + "type": "compute", + "parents": [], + "runtime": 1e9, + "machine": "Tremblay" + }, + { + "name": "t1", + "type": "transfer", + "parents": ["c1"], + "bytesWritten": 5e8, + "machine": "Jupiter" + }, + { + "name": "c2", + "type": "compute", + "parents": [], + "runtime": 5e9, + "machine": "Jupiter" + }, + { + "name": "c3", + "type": "compute", + "parents": ["t1","c2"], + "runtime": 2e9, + "machine": "Jupiter" + } + ], + "machines": [ + {"nodeName": "Tremblay"}, + {"nodeName": "Jupiter"} + ] + } + } + +It can be imported as a vector of Activities into Simgrid using :cpp:func:`create_DAG_from_json(const std::string& filename)`. + +.. code-block:: cpp + + #include "simgrid/s4u.hpp" + + XBT_LOG_NEW_DEFAULT_CATEGORY(main, "Messages specific for this s4u example"); + + int main(int argc, char* argv[]) { + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + + std::vector dag = simgrid::s4u::create_DAG_from_json(argv[2]); + + simgrid::s4u::Activity::on_completion_cb([](simgrid::s4u::Activity const& activity) { + XBT_INFO("Activity '%s' is complete (start time: %f, finish time: %f)", activity.get_cname(), activity.get_start_time(), + activity.get_finish_time()); + }); + + e.run(); + return 0; + } + +The execution of this code should give you the following output: + +.. code-block:: bash + + [10.194200] [main/INFO] Activity 'c1' is complete (start time: 0.000000, finish time: 10.194200) + [65.534235] [main/INFO] Activity 'c2' is complete (start time: 0.000000, finish time: 65.534235) + [85.283378] [main/INFO] Activity 't1' is complete (start time: 10.194200, finish time: 85.283378) + [111.497072] [main/INFO] Activity 'c3' is complete (start time: 85.283378, finish time: 111.497072) + +From a DAX file [deprecated] +............................ + +A DAX file describes a workflow in accordance with the `Pegasus `_ format. + +The following DAX file describes the workflow presented at the beginning of this lab: + +.. code-block:: xml + + + + + + + + + + + + + + + + + + + + +It can be imported as a vector of Activities into Simgrid using :cpp:func:`create_DAG_from_DAX(std::string)`. + +.. code-block:: cpp + + #include "simgrid/s4u.hpp" + + XBT_LOG_NEW_DEFAULT_CATEGORY(main, "Messages specific for this s4u example"); + + int main(int argc, char* argv[]) { + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + + std::vector dag = simgrid::s4u::create_DAG_from_DAX(argv[2]); + + simgrid::s4u::Host* tremblay = e.host_by_name("Tremblay"); + simgrid::s4u::Host* jupiter = e.host_by_name("Jupiter"); + simgrid::s4u::Host* fafard = e.host_by_name("Fafard"); + + dynamic_cast(dag[0].get())->set_host(fafard); + dynamic_cast(dag[1].get())->set_host(tremblay); + dynamic_cast(dag[2].get())->set_host(jupiter); + dynamic_cast(dag[3].get())->set_host(jupiter); + dynamic_cast(dag[8].get())->set_host(jupiter); + + for (const auto& a : dag) { + if (auto* comm = dynamic_cast(a.get())) { + auto pred = dynamic_cast((*comm->get_dependencies().begin()).get()); + auto succ = dynamic_cast(comm->get_successors().front().get()); + comm->set_source(pred->get_host())->set_destination(succ->get_host()); + } + } + + simgrid::s4u::Activity::on_completion_cb([](simgrid::s4u::Activity const& activity) { + XBT_INFO("Activity '%s' is complete (start time: %f, finish time: %f)", activity.get_cname(), activity.get_start_time(), + activity.get_finish_time()); + }); + + e.run(); + return 0; + } + diff --git a/docs/source/index.rst b/docs/source/index.rst index 9c41acb382..38445ad0c3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -59,6 +59,7 @@ of every page. Bugs in the code should be reported Simulating distributed algorithms Simulating MPI applications Model-checking algorithms + Simulating DAG .. toctree:: :hidden: diff --git a/docs/source/tuto_dag/dag_lab1.cpp b/docs/source/tuto_dag/dag_lab1.cpp new file mode 100644 index 0000000000..1242ffa068 --- /dev/null +++ b/docs/source/tuto_dag/dag_lab1.cpp @@ -0,0 +1,48 @@ +#include "simgrid/s4u.hpp" + +XBT_LOG_NEW_DEFAULT_CATEGORY(main, "Messages specific for this s4u tutorial"); + +int main(int argc, char* argv[]) { + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + + simgrid::s4u::Host* tremblay = e.host_by_name("Tremblay"); + simgrid::s4u::Host* jupiter = e.host_by_name("Jupiter"); + + simgrid::s4u::ExecPtr c1 = simgrid::s4u::Exec::init(); + simgrid::s4u::ExecPtr c2 = simgrid::s4u::Exec::init(); + simgrid::s4u::ExecPtr c3 = simgrid::s4u::Exec::init(); + simgrid::s4u::CommPtr t1 = simgrid::s4u::Comm::sendto_init(); + + c1->set_name("c1"); + c2->set_name("c2"); + c3->set_name("c3"); + t1->set_name("t1"); + + c1->set_flops_amount(1e9); + c2->set_flops_amount(5e9); + c3->set_flops_amount(2e9); + t1->set_payload_size(5e8); + + c1->add_successor(t1); + t1->add_successor(c3); + c2->add_successor(c3); + + c1->set_host(tremblay); + c2->set_host(jupiter); + c3->set_host(jupiter); + t1->set_source(tremblay); + t1->set_destination(jupiter); + + c1->start(); + c2->start(); + + simgrid::s4u::Activity::on_completion_cb([](simgrid::s4u::Activity const& activity) { + XBT_INFO("Activity '%s' is complete (start time: %f, finish time: %f)", activity.get_cname(), activity.get_start_time(), + activity.get_finish_time()); + }); + + e.run(); + return 0; +} + diff --git a/docs/source/tuto_dag/dag_lab2-1.cpp b/docs/source/tuto_dag/dag_lab2-1.cpp new file mode 100644 index 0000000000..9ee5baa4b0 --- /dev/null +++ b/docs/source/tuto_dag/dag_lab2-1.cpp @@ -0,0 +1,37 @@ +#include "simgrid/s4u.hpp" + +XBT_LOG_NEW_DEFAULT_CATEGORY(main, "Messages specific for this s4u example"); + +int main(int argc, char* argv[]) { + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + + std::vector dag = simgrid::s4u::create_DAG_from_dot(argv[2]); + + simgrid::s4u::Host* tremblay = e.host_by_name("Tremblay"); + simgrid::s4u::Host* jupiter = e.host_by_name("Jupiter"); + simgrid::s4u::Host* fafard = e.host_by_name("Fafard"); + + dynamic_cast(dag[0].get())->set_host(fafard); + dynamic_cast(dag[1].get())->set_host(tremblay); + dynamic_cast(dag[2].get())->set_host(jupiter); + dynamic_cast(dag[3].get())->set_host(jupiter); + dynamic_cast(dag[8].get())->set_host(jupiter); + + for (const auto& a : dag) { + if (auto* comm = dynamic_cast(a.get())) { + auto pred = dynamic_cast((*comm->get_dependencies().begin()).get()); + auto succ = dynamic_cast(comm->get_successors().front().get()); + comm->set_source(pred->get_host())->set_destination(succ->get_host()); + } + } + + simgrid::s4u::Activity::on_completion_cb([](simgrid::s4u::Activity const& activity) { + XBT_INFO("Activity '%s' is complete (start time: %f, finish time: %f)", activity.get_cname(), activity.get_start_time(), + activity.get_finish_time()); + }); + + e.run(); + return 0; +} + diff --git a/docs/source/tuto_dag/dag_lab2-2.cpp b/docs/source/tuto_dag/dag_lab2-2.cpp new file mode 100644 index 0000000000..f392ecb99a --- /dev/null +++ b/docs/source/tuto_dag/dag_lab2-2.cpp @@ -0,0 +1,19 @@ +#include "simgrid/s4u.hpp" + +XBT_LOG_NEW_DEFAULT_CATEGORY(main, "Messages specific for this s4u example"); + +int main(int argc, char* argv[]) { + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + + std::vector dag = simgrid::s4u::create_DAG_from_json(argv[2]); + + simgrid::s4u::Activity::on_completion_cb([](simgrid::s4u::Activity const& activity) { + XBT_INFO("Activity '%s' is complete (start time: %f, finish time: %f)", activity.get_cname(), activity.get_start_time(), + activity.get_finish_time()); + }); + + e.run(); + return 0; +} + diff --git a/docs/source/tuto_dag/dag_lab2-3.cpp b/docs/source/tuto_dag/dag_lab2-3.cpp new file mode 100644 index 0000000000..e3db543c3c --- /dev/null +++ b/docs/source/tuto_dag/dag_lab2-3.cpp @@ -0,0 +1,37 @@ +#include "simgrid/s4u.hpp" + +XBT_LOG_NEW_DEFAULT_CATEGORY(main, "Messages specific for this s4u example"); + +int main(int argc, char* argv[]) { + simgrid::s4u::Engine e(&argc, argv); + e.load_platform(argv[1]); + + std::vector dag = simgrid::s4u::create_DAG_from_DAX(argv[2]); + + simgrid::s4u::Host* tremblay = e.host_by_name("Tremblay"); + simgrid::s4u::Host* jupiter = e.host_by_name("Jupiter"); + simgrid::s4u::Host* fafard = e.host_by_name("Fafard"); + + dynamic_cast(dag[0].get())->set_host(fafard); + dynamic_cast(dag[1].get())->set_host(tremblay); + dynamic_cast(dag[2].get())->set_host(jupiter); + dynamic_cast(dag[3].get())->set_host(jupiter); + dynamic_cast(dag[8].get())->set_host(jupiter); + + for (const auto& a : dag) { + if (auto* comm = dynamic_cast(a.get())) { + auto pred = dynamic_cast((*comm->get_dependencies().begin()).get()); + auto succ = dynamic_cast(comm->get_successors().front().get()); + comm->set_source(pred->get_host())->set_destination(succ->get_host()); + } + } + + simgrid::s4u::Activity::on_completion_cb([](simgrid::s4u::Activity const& activity) { + XBT_INFO("Activity '%s' is complete (start time: %f, finish time: %f)", activity.get_cname(), activity.get_start_time(), + activity.get_finish_time()); + }); + + e.run(); + return 0; +} + diff --git a/docs/source/tuto_dag/img/dag.svg b/docs/source/tuto_dag/img/dag.svg new file mode 100644 index 0000000000..58a2e07bdf --- /dev/null +++ b/docs/source/tuto_dag/img/dag.svg @@ -0,0 +1,955 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 12345678910111213141516171819202122232524 + + + + + + + + + diff --git a/docs/source/tuto_dag/img/dag1.svg b/docs/source/tuto_dag/img/dag1.svg new file mode 100644 index 0000000000..7a6835f5d5 --- /dev/null +++ b/docs/source/tuto_dag/img/dag1.svg @@ -0,0 +1,196 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + t1c1c2c3 + + + + + + + + + diff --git a/docs/source/tuto_dag/img/dag2.svg b/docs/source/tuto_dag/img/dag2.svg new file mode 100644 index 0000000000..76111394ba --- /dev/null +++ b/docs/source/tuto_dag/img/dag2.svg @@ -0,0 +1,398 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + t1c1c2c3 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/source/tuto_dag/simple_dax.xml b/docs/source/tuto_dag/simple_dax.xml new file mode 100644 index 0000000000..2572f7cbdb --- /dev/null +++ b/docs/source/tuto_dag/simple_dax.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + diff --git a/docs/source/tuto_dag/simple_dot.dot b/docs/source/tuto_dag/simple_dot.dot new file mode 100644 index 0000000000..603409fa01 --- /dev/null +++ b/docs/source/tuto_dag/simple_dot.dot @@ -0,0 +1,10 @@ +digraph G { + c1 [size="1e9"]; + c2 [size="5e9"]; + c3 [size="2e9"]; + root->c1 [size="2e8"]; + root->c2 [size="1e8"]; + c1->c3 [size="5e8"]; + c2->c3 [size="-1."]; + c3->end [size="2e8"]; +} diff --git a/docs/source/tuto_dag/simple_json.json b/docs/source/tuto_dag/simple_json.json new file mode 100644 index 0000000000..ea7785097f --- /dev/null +++ b/docs/source/tuto_dag/simple_json.json @@ -0,0 +1,42 @@ +{ + "name": "simple_json", + "schemaVersion": "1.0", + "workflow": { + "makespan": 0, + "executedAt": "2023-03-09T00:00:00-00:00", + "tasks": [ + { + "name": "c1", + "type": "compute", + "parents": [], + "runtime": 1e9, + "machine": "Tremblay" + }, + { + "name": "t1", + "type": "transfer", + "parents": ["c1"], + "bytesWritten": 5e8, + "machine": "Jupiter" + }, + { + "name": "c2", + "type": "compute", + "parents": [], + "runtime": 5e9, + "machine": "Jupiter" + }, + { + "name": "c3", + "type": "compute", + "parents": ["t1","c2"], + "runtime": 2e9, + "machine": "Jupiter" + } + ], + "machines": [ + {"nodeName": "Tremblay"}, + {"nodeName": "Jupiter"} + ] + } +} \ No newline at end of file diff --git a/docs/source/tuto_dag/small_platform.xml b/docs/source/tuto_dag/small_platform.xml new file mode 100644 index 0000000000..69bdcae955 --- /dev/null +++ b/docs/source/tuto_dag/small_platform.xml @@ -0,0 +1,194 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/include/simgrid/s4u/Activity.hpp b/include/simgrid/s4u/Activity.hpp index 0e0b72aabc..bc027130b5 100644 --- a/include/simgrid/s4u/Activity.hpp +++ b/include/simgrid/s4u/Activity.hpp @@ -39,6 +39,7 @@ class XBT_PUBLIC Activity : public xbt::Extendable { friend kernel::activity::ActivityImpl; friend std::vector create_DAG_from_dot(const std::string& filename); friend std::vector create_DAG_from_DAX(const std::string& filename); + friend std::vector create_DAG_from_json(const std::string& filename); #endif public: diff --git a/include/simgrid/s4u/Engine.hpp b/include/simgrid/s4u/Engine.hpp index c9fd4754f4..742bb360c7 100644 --- a/include/simgrid/s4u/Engine.hpp +++ b/include/simgrid/s4u/Engine.hpp @@ -266,6 +266,7 @@ private: std::vector create_DAG_from_dot(const std::string& filename); std::vector create_DAG_from_DAX(const std::string& filename); +std::vector create_DAG_from_json(const std::string& filename); #ifndef DOXYGEN /* Internal use only, no need to expose it */ template diff --git a/src/dag/loaders.cpp b/src/dag/loaders.cpp index de7b4687f8..e78e7169a0 100644 --- a/src/dag/loaders.cpp +++ b/src/dag/loaders.cpp @@ -7,6 +7,8 @@ #include "src/internal_config.h" #include #include +#include +#include #include #include #include @@ -19,6 +21,8 @@ #include "dax_dtd.h" #include "dax_dtd.c" +#include + #if HAVE_GRAPHVIZ #include #endif @@ -79,6 +83,74 @@ static std::map> jobs; static std::map> files; static ExecPtr current_job; +/** @brief loads a JSON file describing a DAG + * + * See https://github.com/wfcommons/wfformat for more details. + */ +std::vector create_DAG_from_json(const std::string& filename) +{ + std::ifstream f(filename); + auto data = nlohmann::json::parse(f); + std::vector dag = {}; + std::map> successors = {}; + std::map comms_destinations = {}; + ActivityPtr current; + + for (auto const& task: data["workflow"]["tasks"]) { + if (task["type"] == "compute") { + current = Exec::init()->set_name(task["name"])->set_flops_amount(task["runtime"]); + if (task.contains("machine")) + dynamic_cast(current.get())->set_host(simgrid::s4u::Engine::get_instance()->host_by_name(task["machine"])); + } + else if (task["type"] == "transfer"){ + current = Comm::sendto_init()->set_name(task["name"])->set_payload_size(task["bytesWritten"]); + if (task.contains("machine")) + comms_destinations[current] = simgrid::s4u::Engine::get_instance()->host_by_name(task["machine"]); + if (task["parents"].size() == 1) { + ActivityPtr parent_activity; + for (auto const& activity: dag) { + if (activity->get_name() == task["parents"][0]) { + parent_activity = activity; + break; + } + } + if (dynamic_cast(parent_activity.get()) != nullptr) + dynamic_cast(current.get())->set_source(dynamic_cast(parent_activity.get())->get_host()); + else if (dynamic_cast(parent_activity.get()) != nullptr) + dynamic_cast(current.get())->set_source(dynamic_cast(parent_activity.get())->get_destination()); + } + } + else + XBT_DEBUG("Task type \"%s\" not supported.", task["type"]); + + dag.push_back(current); + for (auto const& parent: task["parents"]) { + auto it = successors.find(parent); + if (it == successors.end()) + successors[parent] = {}; + successors[parent].push_back(current); + } + } + // Assign successors + for (auto const& [parent, successors_list] : successors) + for (auto const& activity: dag) + if (activity->get_name() == parent) { + for (auto const& successor: successors_list) + activity->add_successor(successor); + break; + } + // Assign destinations of Comms (if done before successors are assigned there is a bug) + for (auto const& [comm, destination]: comms_destinations) + dynamic_cast(comm.get())->set_destination(destination); + + // Start only Activities with dependencies solved + for (auto const& activity: dag) { + if (dynamic_cast(activity.get()) != nullptr and activity->dependencies_solved()) + activity->start(); + } + return dag; +} + /** @brief loads a DAX file describing a DAG * * See https://confluence.pegasus.isi.edu/display/pegasus/WorkflowGenerator for more details. -- 2.20.1