From 63d459f76e3df79eb8dd08f7053f6653d0e33509 Mon Sep 17 00:00:00 2001 From: Millian Poquet <millian.poquet@inria.fr> Date: Mon, 16 Mar 2020 18:29:05 +0100 Subject: [PATCH] [code] dump schedule as batsim jobs output format https://batsim.readthedocs.io/en/latest/output-jobs.html --- src/algo/conservative_bf.cpp | 22 +++++++++ src/algo/conservative_bf.hpp | 2 + src/schedule.cpp | 92 +++++++++++++++++++++++++++++++++++- src/schedule.hpp | 3 ++ 4 files changed, 118 insertions(+), 1 deletion(-) diff --git a/src/algo/conservative_bf.cpp b/src/algo/conservative_bf.cpp index 1a46278..0eaf863 100644 --- a/src/algo/conservative_bf.cpp +++ b/src/algo/conservative_bf.cpp @@ -2,12 +2,27 @@ #include <loguru.hpp> +#include "../pempek_assert.hpp" + using namespace std; ConservativeBackfilling::ConservativeBackfilling(Workload *workload, SchedulingDecision *decision, Queue *queue, ResourceSelector * selector, double rjms_delay, rapidjson::Document *variant_options) : ISchedulingAlgorithm(workload, decision, queue, selector, rjms_delay, variant_options) { + if (variant_options->HasMember("dump_previsional_schedules")) + { + PPK_ASSERT_ERROR((*variant_options)["dump_previsional_schedules"].IsBool(), + "Invalid options: 'dump_previsional_schedules' should be a boolean"); + _dump_provisional_schedules = (*variant_options)["dump_previsional_schedules"].GetBool(); + } + + if (variant_options->HasMember("dump_prefix")) + { + PPK_ASSERT_ERROR((*variant_options)["dump_prefix"].IsString(), + "Invalid options: 'dump_prefix' should be a string"); + _dump_prefix = (*variant_options)["dump_prefix"].GetString(); + } } ConservativeBackfilling::~ConservativeBackfilling() @@ -92,7 +107,11 @@ void ConservativeBackfilling::make_decisions(double date, const Job * job = (*job_it)->job; _schedule.remove_job_if_exists(job); +// if (_dump_provisional_schedules) +// _schedule.incremental_dump_as_batsim_jobs_file(_dump_prefix); Schedule::JobAlloc alloc = _schedule.add_job_first_fit(job, _selector); +// if (_dump_provisional_schedules) +// _schedule.incremental_dump_as_batsim_jobs_file(_dump_prefix); if (alloc.started_in_first_slice) { @@ -112,4 +131,7 @@ void ConservativeBackfilling::make_decisions(double date, double answer = _schedule.query_wait(new_job->nb_requested_resources, new_job->walltime, _selector); _decision->add_answer_estimate_waiting_time(job_id, answer, date); } + + if (_dump_provisional_schedules) + _schedule.incremental_dump_as_batsim_jobs_file(_dump_prefix); } diff --git a/src/algo/conservative_bf.hpp b/src/algo/conservative_bf.hpp index 3f2c5c4..717a991 100644 --- a/src/algo/conservative_bf.hpp +++ b/src/algo/conservative_bf.hpp @@ -24,4 +24,6 @@ public: private: Schedule _schedule; + bool _dump_provisional_schedules = false; + std::string _dump_prefix = "/tmp/dump"; }; diff --git a/src/schedule.cpp b/src/schedule.cpp index 8f7348e..c417394 100644 --- a/src/schedule.cpp +++ b/src/schedule.cpp @@ -896,7 +896,7 @@ void Schedule::write_svg_to_file(const string &filename) const void Schedule::output_to_svg(const string &filename_prefix) { - const int bufsize = 128; + const int bufsize = 4096; char *buf = new char[bufsize]; snprintf(buf, bufsize, "%s%06d.svg", filename_prefix.c_str(), _output_number); @@ -907,6 +907,96 @@ void Schedule::output_to_svg(const string &filename_prefix) delete[] buf; } +void Schedule::dump_to_batsim_jobs_file(const string &filename) const +{ + ofstream f(filename); + if (f.is_open()) + { + f << "job_id,submission_time,requested_number_of_resources,requested_time,starting_time,finish_time,allocated_resources\n"; + + PPK_ASSERT_ERROR(_profile.size() > 0); + + const int buf_size = 4096; + char *buf = new char[buf_size]; + + map<const Job *, Rational> jobs_starting_times; + set<const Job *> current_jobs; + for (auto mit : _profile.begin()->allocated_jobs) + { + const Job *allocated_job = mit.first; + current_jobs.insert(allocated_job); + jobs_starting_times[allocated_job] = _profile.begin()->begin; + } + + // Let's traverse the profile to find the beginning of each job + for (auto slice_it = _profile.begin(); slice_it != _profile.end(); ++slice_it) + { + const TimeSlice &slice = *slice_it; + set<const Job *> allocated_jobs; + for (auto mit : slice.allocated_jobs) + { + const Job *job = mit.first; + allocated_jobs.insert(job); + } + + set<const Job *> finished_jobs; + set_difference(current_jobs.begin(), current_jobs.end(), allocated_jobs.begin(), allocated_jobs.end(), + std::inserter(finished_jobs, finished_jobs.end())); + + for (const Job *job : finished_jobs) + { + // Find where the job has been allocated + PPK_ASSERT_ERROR(slice_it != _profile.begin()); + auto previous_slice_it = slice_it; + --previous_slice_it; + IntervalSet job_machines = previous_slice_it->allocated_jobs.at(job); + + snprintf(buf, buf_size, "%s,%g,%d,%g,%g,%g,%s\n", + job->id.c_str(), + job->submission_time, + job->nb_requested_resources, + (double)job->walltime, + (double)jobs_starting_times[job], + (double)slice_it->begin, + job_machines.to_string_hyphen(" ", "-").c_str()); + f << buf; + } + + set<const Job *> new_jobs; + set_difference(allocated_jobs.begin(), allocated_jobs.end(), current_jobs.begin(), current_jobs.end(), + std::inserter(new_jobs, new_jobs.end())); + + for (const Job *job : new_jobs) + { + jobs_starting_times[job] = slice.begin; + } + + // Update current_jobs + for (const Job *job : finished_jobs) + current_jobs.erase(job); + for (const Job *job : new_jobs) + current_jobs.insert(job); + } + + delete[] buf; + } + + f.close(); +} + +void Schedule::incremental_dump_as_batsim_jobs_file(const string &filename_prefix) +{ + const int bufsize = 4096; + char *buf = new char[bufsize]; + + snprintf(buf, bufsize, "%s%06d.csv", filename_prefix.c_str(), _output_number); + _output_number = (_output_number + 1) % 10000000; + + dump_to_batsim_jobs_file(buf); + + delete[] buf; +} + int Schedule::nb_machines() const { return _nb_machines; diff --git a/src/schedule.hpp b/src/schedule.hpp index d82e2ff..cf90c5f 100644 --- a/src/schedule.hpp +++ b/src/schedule.hpp @@ -101,6 +101,9 @@ public: void write_svg_to_file(const std::string & filename) const; void output_to_svg(const std::string & filename_prefix = "/tmp/schedule"); + void dump_to_batsim_jobs_file(const std::string & filename) const; + void incremental_dump_as_batsim_jobs_file(const std::string & filename_prefix = "/tmp/schedule"); + int nb_machines() const; private: -- GitLab