-/* Copyright (c) 2009-2022. The SimGrid Team.
- * All rights reserved. */
+/* Copyright (c) 2009-2023. The SimGrid Team. All rights reserved. */
/* This program is free software; you can redistribute it and/or modify it
* under the terms of the license (GNU LGPL) which comes with this package. */
#include "src/internal_config.h"
#include <algorithm>
#include <map>
+#include <fstream>
+#include <simgrid/s4u/Host.hpp>
#include <simgrid/s4u/Comm.hpp>
#include <simgrid/s4u/Engine.hpp>
#include <simgrid/s4u/Exec.hpp>
#include "dax_dtd.h"
#include "dax_dtd.c"
+#if SIMGRID_HAVE_JSON
+// Disable implicit conversions. See https://github.com/nlohmann/json#implicit-conversions
+#ifdef JSON_USE_IMPLICIT_CONVERSIONS
+#undef JSON_USE_IMPLICIT_CONVERSIONS
+#endif
+#define JSON_USE_IMPLICIT_CONVERSIONS 0
+#include <nlohmann/json.hpp>
+#include <sstream>
+#endif
+
#if HAVE_GRAPHVIZ
#include <graphviz/cgraph.h>
#endif
std::string new_name = parent->get_name() + "_" + comm->get_name() + "_" + child->get_name();
- comm->set_name(new_name)->vetoable_start();
+ comm->set_name(new_name)->start();
}
static bool check_for_cycle(const std::vector<simgrid::s4u::ActivityPtr>& dag)
{
std::vector<simgrid::s4u::ActivityPtr> current;
- for (const auto& a : dag)
- if (dynamic_cast<simgrid::s4u::Exec*>(a.get()) != nullptr && a->has_no_successor())
- current.push_back(a);
+ std::copy_if(begin(dag), end(dag), back_inserter(current), [](const auto& a) {
+ return dynamic_cast<simgrid::s4u::Exec*>(a.get()) != nullptr && a->has_no_successor();
+ });
while (not current.empty()) {
std::vector<simgrid::s4u::ActivityPtr> next;
static YY_BUFFER_STATE input_buffer;
-namespace simgrid {
-namespace s4u {
+namespace simgrid::s4u {
static std::vector<ActivityPtr> result;
static std::map<std::string, ExecPtr, std::less<>> jobs;
static std::map<std::string, Comm*, std::less<>> files;
static ExecPtr current_job;
+/** @brief loads a JSON file describing a DAG
+ *
+ * See https://github.com/wfcommons/wfformat for more details. We support wfformat 1.4.
+ */
+std::vector<ActivityPtr> create_DAG_from_json(const std::string& filename)
+{
+#if SIMGRID_HAVE_JSON
+ std::ifstream f(filename);
+ auto data = nlohmann::json::parse(f);
+ std::vector<ActivityPtr> dag = {};
+ std::map<std::string, std::vector<ActivityPtr>, std::less<>> successors = {};
+ std::map<ActivityPtr, Host*> comms_destinations = {};
+ ActivityPtr current;
+
+ for (auto const& task: data["workflow"]["tasks"]) {
+ if (task["type"] == "compute") {
+ current =
+ Exec::init()->set_name(task["name"].get<std::string>())->set_flops_amount(task["runtimeInSeconds"].get<double>());
+ if (task.contains("machine"))
+ dynamic_cast<Exec*>(current.get())
+ ->set_host(simgrid::s4u::Engine::get_instance()->host_by_name(task["machine"].get<std::string>()));
+ }
+ else if (task["type"] == "transfer"){
+ current = Comm::sendto_init()
+ ->set_name(task["name"].get<std::string>())
+ ->set_payload_size(task["writtenBytes"].get<double>());
+ if (task.contains("machine"))
+ comms_destinations[current] =
+ simgrid::s4u::Engine::get_instance()->host_by_name(task["machine"].get<std::string>());
+ if (task["parents"].size() == 1) {
+ ActivityPtr parent_activity;
+ for (auto const& activity: dag) {
+ if (activity->get_name() == task["parents"][0]) {
+ parent_activity = activity;
+ break;
+ }
+ }
+ if (dynamic_cast<Exec*>(parent_activity.get()) != nullptr)
+ dynamic_cast<Comm*>(current.get())->set_source(dynamic_cast<Exec*>(parent_activity.get())->get_host());
+ else if (dynamic_cast<Comm*>(parent_activity.get()) != nullptr)
+ dynamic_cast<Comm*>(current.get())->set_source(dynamic_cast<Comm*>(parent_activity.get())->get_destination());
+ }
+ } else if (XBT_LOG_ISENABLED(dag_parsing, xbt_log_priority_debug)) {
+ std::stringstream ss;
+ ss << task["type"];
+ XBT_DEBUG("Task type \"%s\" not supported.", ss.str().c_str());
+ }
+
+ dag.push_back(current);
+ for (auto const& parent : task["parents"])
+ successors[parent.get<std::string>()].push_back(current);
+ }
+ // Assign successors
+ for (auto const& [parent, successors_list] : successors)
+ for (auto const& activity: dag)
+ if (activity->get_name() == parent) {
+ for (auto const& successor: successors_list)
+ activity->add_successor(successor);
+ break;
+ }
+ // Assign destinations of Comms (if done before successors are assigned there is a bug)
+ for (auto const& [comm, destination]: comms_destinations)
+ dynamic_cast<Comm*>(comm.get())->set_destination(destination);
+
+ // Start only Activities with dependencies solved
+ for (auto const& activity: dag) {
+ if (dynamic_cast<Exec*>(activity.get()) != nullptr && activity->dependencies_solved())
+ activity->start();
+ }
+ return dag;
+#else
+ xbt_die("JSON support was not compiled in, probably because nlohmann/json was not found. Please install "
+ "nlohmann-json3-dev and recompile SimGrid to use this feature.");
+#endif
+}
/** @brief loads a DAX file describing a DAG
*
* See https://confluence.pegasus.isi.edu/display/pegasus/WorkflowGenerator for more details.
dax_lineno = 1;
auto root_task = Exec::init()->set_name("root")->set_flops_amount(0);
- root_task->vetoable_start();
+ root_task->start();
result.push_back(root_task);
auto end_task = Exec::init()->set_name("end")->set_flops_amount(0);
- end_task->vetoable_start();
+ end_task->start();
xbt_assert(dax_lex() == 0, "Parse error in %s: %s", filename.c_str(), dax__parse_err_msg());
dax__delete_buffer(input_buffer);
* Files not produced in the system are said to be produced by root task (top of DAG).
* Files not consumed in the system are said to be consumed by end task (bottom of DAG).
*/
- CommPtr file;
-
- for (auto const& elm : files) {
- file = elm.second;
+ for (auto const& [_, elm] : files) {
+ CommPtr file = elm;
CommPtr newfile;
if (file->dependencies_solved()) {
for (auto const& it : file->get_successors()) {
FILE* in_file = fopen(filename.c_str(), "r");
xbt_assert(in_file != nullptr, "Failed to open file: %s", filename.c_str());
- Agraph_t* dag_dot = agread(in_file, NIL(Agdisc_t*));
+ Agraph_t* dag_dot = agread(in_file, nullptr);
std::unordered_map<std::string, ActivityPtr> activities;
std::vector<ActivityPtr> dag;
if (activities.find(name) == activities.end()) {
XBT_DEBUG("See <Exec id = %s amount = %.0f>", name.c_str(), amount);
- act = Exec::init()->set_name(name)->set_flops_amount(amount)->vetoable_start();
- activities.insert({name, act});
+ act = Exec::init()->set_name(name)->set_flops_amount(amount)->start();
+ activities.try_emplace(name, act);
if (name != "root" && name != "end")
dag.push_back(act);
} else {
}
/*Check if 'root' and 'end' nodes have been explicitly declared. If not, create them. */
if (activities.find("root") == activities.end())
- root = Exec::init()->set_name("root")->set_flops_amount(0)->vetoable_start();
+ root = Exec::init()->set_name("root")->set_flops_amount(0)->start();
else
root = activities.at("root");
if (activities.find("end") == activities.end())
- end = Exec::init()->set_name("end")->set_flops_amount(0)->vetoable_start();
+ end = Exec::init()->set_name("end")->set_flops_amount(0)->start();
else
end = activities.at("end");
std::string name = std::string(src_name) + "->" + dst_name;
XBT_DEBUG("See <Comm id=%s amount = %.0f>", name.c_str(), size);
if (activities.find(name) == activities.end()) {
- act = Comm::sendto_init()->set_name(name)->set_payload_size(size)->vetoable_start();
+ act = Comm::sendto_init()->set_name(name)->set_payload_size(size)->start();
src->add_successor(act);
act->add_successor(dst);
- activities.insert({name, act});
+ activities.try_emplace(name, act);
dag.push_back(act);
} else {
XBT_WARN("Comm '%s' is defined more than once", name.c_str());
"Please install graphviz, graphviz-dev, and libgraphviz-dev (and erase CMakeCache.txt) before recompiling.");
}
#endif
-} // namespace s4u
-} // namespace simgrid
+} // namespace simgrid::s4u
void STag_dax__adag()
{
try {
- double version = std::stod(std::string(A_dax__adag_version));
+ double version = std::stod(A_dax__adag_version);
xbt_assert(version == 2.1, "Expected version 2.1 in <adag> tag, got %f. Fix the parser or your file", version);
} catch (const std::invalid_argument&) {
throw std::invalid_argument(std::string("Parse error: ") + A_dax__adag_version + " is not a double");
void STag_dax__job()
{
try {
- double runtime = std::stod(std::string(A_dax__job_runtime));
+ double runtime = std::stod(A_dax__job_runtime);
std::string name = std::string(A_dax__job_id) + "@" + A_dax__job_name;
runtime *= 4200000000.; /* Assume that timings were done on a 4.2GFlops machine. I mean, why not? */
XBT_DEBUG("See <job id=%s runtime=%s %.0f>", A_dax__job_id, A_dax__job_runtime, runtime);
- simgrid::s4u::current_job = simgrid::s4u::Exec::init()->set_name(name)->set_flops_amount(runtime)->vetoable_start();
- simgrid::s4u::jobs.insert({A_dax__job_id, simgrid::s4u::current_job});
+ simgrid::s4u::current_job = simgrid::s4u::Exec::init()->set_name(name)->set_flops_amount(runtime)->start();
+ simgrid::s4u::jobs.try_emplace(A_dax__job_id, simgrid::s4u::current_job);
simgrid::s4u::result.push_back(simgrid::s4u::current_job);
} catch (const std::invalid_argument&) {
throw std::invalid_argument(std::string("Parse error: ") + A_dax__job_runtime + " is not a double");
{
double size;
try {
- size = std::stod(std::string(A_dax__uses_size));
+ size = std::stod(A_dax__uses_size);
} catch (const std::invalid_argument&) {
throw std::invalid_argument(std::string("Parse error: ") + A_dax__uses_size + " is not a double");
}
if (job != simgrid::s4u::jobs.end()) {
current_child = job->second;
} else {
- throw std::out_of_range(std::string("Parse error on line ") + std::to_string(dax_lineno) +
+ throw std::out_of_range("Parse error on line " + std::to_string(dax_lineno) +
": Asked to add dependencies to the non-existent " + A_dax__child_ref + "task");
}
}
parent->add_successor(current_child);
XBT_DEBUG("Control-flow dependency from %s to %s", current_child->get_cname(), parent->get_cname());
} else {
- throw std::out_of_range(std::string("Parse error on line ") + std::to_string(dax_lineno) +
- ": Asked to add a dependency from " + current_child->get_name() + " to " +
- A_dax__parent_ref + ", but " + A_dax__parent_ref + " does not exist");
+ throw std::out_of_range("Parse error on line " + std::to_string(dax_lineno) + ": Asked to add a dependency from " +
+ current_child->get_name() + " to " + A_dax__parent_ref + ", but " + A_dax__parent_ref +
+ " does not exist");
}
}
