dump

8a6a59e4 · Millian Poquet · cdaa7fac · 8a6a59e4 · 8a6a59e4 · 8a6a59e4
Commit 8a6a59e4 authored 7 months ago by Millian Poquet
--- a/.gitignore
+++ b/.gitignore
+result*
+simu-out
+*analysis.html
--- a/README.md
+++ b/README.md
+# Repository content
+- `sched` contains schedulers compatible with Batsim 5 called External Decision Components (EDCs)
+- `input` contains Batsim inputs: a SimGrid platform and a Batsim workload
+- `flake.nix` contains reproducible debuggable Nix environments into which the simulations and the analysis can be run
+- `run-simus.sh` is a simple script to run several Batsim instances in sequence
+- `run-rmarkdown-notebook.R` is a simple script to render a [Rmarkdown notebook](https://rmarkdown.rstudio.com/)
+# Prerequisites
+- Nix installed: https://nixos.org/download/
+- Flakes enabled in your Nix setup: https://nixos.wiki/wiki/Flakes#Other_Distros.2C_without_Home-Manager
+- Enable the `batuto` binary cache if you do not want to recompile heavy deps: https://app.cachix.org/cache/battuto#pull
+# Run simulations and visualize results
+```sh
+nix develop .#simulation --command ./run-simus.sh
+```
+This should populate a `simu-out` directory, where each directory is a simulation output.
+# Analyze and visualize results
+```sh
+nix develop .#r-notebook --command Rscript run-rmarkdown-notebook.R ./simulation-output-analysis.Rmd
+```
+This should create/update a `simulation-output-analysis.html` file, that you can open with a web browser.
--- a/default.nix
+++ b/default.nix
+(import (fetchTarball {
+    url = "https://github.com/edolstra/flake-compat/archive/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9.tar.gz";
+    sha256 = "sha256:1prd9b1xx8c0sfwnyzkspplh30m613j42l1k789s521f4kv4c2z2";
+}) {
+  src =  ./.;
+}).defaultNix
--- a/flake.nix
+++ b/flake.nix
@@ -36,15 +36,27 @@
        py = pkgs.python3;
        pyPkgs = pkgs.python3Packages;
        kapack = nur-kapack.packages.${system};
-        batprotopkgs = batprotocol-flake.packages.${system};
+        battuto-rev = if (self ? shortRev) then self.shortRev else self.dirtyShortRev;
-        intervalsetpkgs = intervalset-flake.packages.${system};
-        batpkgs = batsim-flake.packages-release.${system};
+        # debuggable versions of Batsim and its important deps are used to show how to debug your EDC
-        battuto-rev = if (self ? rev) then self.rev else self.dirtyRev;
+        # avoid it on real simulation campaigns, as it severely slowdowns your simulations
+        batprotopkgs = batprotocol-flake.packages-debug.${system};
+        intervalsetpkgs = intervalset-flake.packages-debug.${system};
+        batpkgs = batsim-flake.packages-debug.${system};
+        cppMesonDevBase = nur-kapack.lib.${system}.cppMesonDevBase;
+        intervalset = intervalsetpkgs.intervalset;
+        batprotocol-cpp = batprotopkgs.batprotocol-cpp;
      in rec {
        packages = {
          batsim = batpkgs.batsim;
-          edc = pkgs.stdenv.mkDerivation {
+          edc = (cppMesonDevBase {
-            name = "battuto-edc";
+            inherit (pkgs) stdenv lib meson ninja pkg-config;
+            debug = true;
+            werror = false;
+            doCoverage = false;
+          }).overrideAttrs(attrs: rec {
+            pname = "battuto-edc";
            version = battuto-rev;
            src = pkgs.lib.sourceByRegex ./sched [
              "^meson\.build"
@@ -52,21 +64,24 @@
              "^.*\.h"
            ];
            buildInputs = [
-              batprotopkgs.batprotocol-cpp
+              batprotocol-cpp
-              intervalsetpkgs.intervalset
+              intervalset
              pkgs.nlohmann_json
            ];
-          };
+            passthru = rec {
+              DEBUG_SRC_DIRS = intervalset.DEBUG_SRC_DIRS ++ batprotocol-cpp.DEBUG_SRC_DIRS ++ [ "${src}" ];
+              GDB_DIR_ARGS = map (x: "--directory=" + x) DEBUG_SRC_DIRS;
+            };
+          });
        };
        devShells = rec {
          simulation = pkgs.mkShell {
            buildInputs = [
              packages.batsim
-              # TODO add sched
+              packages.edc
-              packages.python-scripts
            ];
            BATSIM_ROOT_PATH="${batsim-flake}";
-            # EDC_LIBRARY_PATH="${packages.easypower-sched-lib}/lib"; TODO
+            EDC_LIBRARY_PATH="${packages.edc}/lib";
          };
          r-notebook = pkgs.mkShell {
            buildInputs = [

--- a/input/cluster_energy_128.xml
+++ b/input/cluster_energy_128.xml
+<?xml version='1.0'?>
+<!DOCTYPE platform SYSTEM "http://simgrid.gforge.inria.fr/simgrid/simgrid.dtd">
+<platform version="4.1">
+<zone id="AS0" routing="Full">
+<cluster id="cluster_compute" prefix="host" suffix="" radical="0-127"
+             bw="125MBps" lat="50us" bb_bw="2.25GBps" bb_lat="500us"
+             speed="100.0Mf, 88.95899053627761Mf, 83.67952522255192Mf, 80.57142857142857Mf, 76.21621621621621Mf, 72.49357326478149Mf, 68.78048780487805Mf, 64.6788990825688Mf, 60.775862068965516Mf, 58.62785862785863Mf, 50.088809946714036Mf, 49.21465968586388Mf, 44.97607655502392Mf, 1e-9Mf, 0.1639344262295082f, 0.006599788806758183f">
+    <!-- real pstates: 0 to 12
+         off: pstate: 13
+              consumption: 9.75 W
+         shutdown: pstate: 14
+                   time: 6.1 s,
+                   consumption: 100.99672131147543 W
+         boot: pstate: 15
+               time: 151.52 s,
+               consumption: 125.1743848996832 W
+    -->
+    <prop id="wattage_per_state" value="95.0:190.738, 95.0:171.02, 95.0:165.62, 95.0:160.47, 95.0:155.729, 95.0:151.3, 95.0:146.92, 95.0:142.95, 95.0:138.928, 95.0:135.368, 95.0:132.519, 95.0:128.87, 95.0:125.88, 9.75:9.75, 100.99672131147543:100.99672131147543, 125.1743848996832:125.1743848996832" />
+    <prop id="wattage_off" value="9.75" />
+    <!-- OFF : ON->OFF (shutdown) : OFF->ON (booting) -->
+    <prop id="sleep_pstates" value="13:14:15" />
+</cluster>
+<cluster id="cluster_master" prefix="master_host" suffix="" radical="0-0"
+             bw="125MBps" lat="50us" bb_bw="2.25GBps" bb_lat="500us"
+             speed="100.0Mf, 88.95899053627761Mf, 83.67952522255192Mf, 80.57142857142857Mf, 76.21621621621621Mf, 72.49357326478149Mf, 68.78048780487805Mf, 64.6788990825688Mf, 60.775862068965516Mf, 58.62785862785863Mf, 50.088809946714036Mf, 49.21465968586388Mf, 44.97607655502392Mf, 1e-9Mf, 0.1639344262295082f, 0.006599788806758183f">
+    <!-- real pstates: 0 to 12
+         off: pstate: 13
+              consumption: 9.75 W
+         shutdown: pstate: 14
+                   time: 6.1 s,
+                   consumption: 100.99672131147543 W
+         boot: pstate: 15
+               time: 151.52 s,
+               consumption: 125.1743848996832 W
+    -->
+    <prop id="wattage_per_state" value="95.0:190.738, 95.0:171.02, 95.0:165.62, 95.0:160.47, 95.0:155.729, 95.0:151.3, 95.0:146.92, 95.0:142.95, 95.0:138.928, 95.0:135.368, 95.0:132.519, 95.0:128.87, 95.0:125.88, 9.75:9.75, 100.99672131147543:100.99672131147543, 125.1743848996832:125.1743848996832" />
+    <prop id="wattage_off" value="9.75" />
+    <prop id="role" value="master" />
+    <!-- OFF : ON->OFF (shutdown) : OFF->ON (booting) -->
+    <prop id="sleep_pstates" value="13:14:15" />
+</cluster>
+    <link id="backbone" bandwidth="1.25GBps" latency="500us" />
+    <zoneRoute src="cluster_compute" dst="cluster_master" gw_src="hostcluster_compute_router"
+        gw_dst="master_hostcluster_master_router">
+        <link_ctn id="backbone" />
+    </zoneRoute>
+</zone>
+</platform>
--- a/input/test_batsim_paper_workload_seed1.json
+++ b/input/test_batsim_paper_workload_seed1.json
--- a/run-rmarkdown-notebook.R
+++ b/run-rmarkdown-notebook.R
+#!/usr/bin/env Rscript
+args = commandArgs(trailingOnly=TRUE)
+if (length(args) == 1) {
+  library(rmarkdown)
+  rmarkdown::render(args[1])
+} else {
+  write("usage: run-rmarkdown-notebook <NOTEBOOK-FILE>", stderr())
+  invokeRestart("abort")
+}
--- a/run-simus.sh
+++ b/run-simus.sh
+#!/usr/bin/env sh
+set -eu
+PLATFORM=$(realpath input/cluster_energy_128.xml)
+WORKLOAD=$(realpath input/test_batsim_paper_workload_seed1.json)
+SIMUOUT=$(realpath ./simu-out)
+# clean previous runs
+rm -rf ${SIMUOUT}
+# run all simulations
+for sched in 'rejecter' 'exec1by1' 'fcfs' 'easy'
+do
+  batsim -p ${PLATFORM} -w ${WORKLOAD} --mmax-workload -l ${EDC_LIBRARY_PATH}/lib${sched}.so 0 '' -e ${SIMUOUT}/${sched}/
+done
--- a/sched/meson.build
+++ b/sched/meson.build
@@ -17,6 +17,11 @@ common = [
  'batsim_edc.h'
 ]
+rejecter = shared_library('rejecter', common + ['rejecter.cpp'],
+  dependencies: deps + [boost_dep, intervalset_dep],
+  install: true,
+)
 exec1by1 = shared_library('exec1by1', common + ['exec1by1.cpp'],
  dependencies: deps + [boost_dep, intervalset_dep],
  install: true,

--- a/simulation-output-analysis.Rmd
+++ b/simulation-output-analysis.Rmd
+---
+title: "Simulation output analysis"
+author: "Millian Poquet"
+date: "2024-10-23"
+params:
+  simulation_output: "./simu-out"
+output:
+  rmdformats::readthedown
+---
+# Initial load and variable setup
+```{r, echo = TRUE}
+library(tidyverse)
+library(viridis)
+simu_output = params$simulation_output
+```
+# Summary of each simulation
+```{r, echo = TRUE}
+schedule_rejecter = read_csv(paste(simu_output, "rejecter", "schedule.csv", sep="/"), show_col_types = FALSE) %>% mutate(sched='rejecter')
+schedule_exec1by1 = read_csv(paste(simu_output, "exec1by1", "schedule.csv", sep="/"), show_col_types = FALSE) %>% mutate(sched='exec1by1')
+schedule_fcfs = read_csv(paste(simu_output, "fcfs", "schedule.csv", sep="/"), show_col_types = FALSE) %>% mutate(sched='fcfs')
+schedule_easy = read_csv(paste(simu_output, "easy", "schedule.csv", sep="/"), show_col_types = FALSE) %>% mutate(sched='easy')
+all_schedule = bind_rows(schedule_rejecter, schedule_exec1by1, schedule_fcfs, schedule_easy) %>% select(
+  sched, consumed_joules, makespan, mean_waiting_time, nb_jobs_success, nb_jobs_rejected
+)
+knitr::kable(all_schedule)
+```
+# Gantt chart of jobs
+```{r, echo = TRUE}
+jobs_rejecter = read_csv(paste(simu_output, "rejecter", "jobs.csv", sep="/"), show_col_types = FALSE) %>% mutate(sched='rejecter')
+jobs_exec1by1 = read_csv(paste(simu_output, "exec1by1", "jobs.csv", sep="/"), show_col_types = FALSE) %>% mutate(sched='exec1by1')
+jobs_fcfs = read_csv(paste(simu_output, "fcfs", "jobs.csv", sep="/"), show_col_types = FALSE) %>% mutate(sched='fcfs')
+jobs_easy = read_csv(paste(simu_output, "easy", "jobs.csv", sep="/"), show_col_types = FALSE) %>% mutate(sched='easy')
+all_jobs = bind_rows(jobs_rejecter, jobs_exec1by1, jobs_fcfs, jobs_easy) %>% mutate(
+  job_color = as.factor(job_id %% 6)
+)
+all_jobs_sep = all_jobs %>%
+    separate_rows(allocated_resources, sep=" ") %>%
+    separate(allocated_resources, into = c("psetmin", "psetmax"), fill="right") %>%
+    mutate(psetmax = as.integer(psetmax), psetmin = as.integer(psetmin)) %>%
+    mutate(psetmax = ifelse(is.na(psetmax), psetmin, psetmax))
+p = all_jobs_sep %>%
+  ggplot(aes(xmin=starting_time, xmax=finish_time,
+             ymin=psetmin, ymax=psetmax+1)) +
+  geom_rect(aes(fill=job_color), show.legend=FALSE) +
+  facet_wrap(vars(sched), ncol=1) +
+  theme_bw() +
+  scale_fill_viridis(discrete=TRUE) +
+  labs(x='Time', y='Resources')
+p
+p + facet_wrap(vars(sched), ncol=1, scales='free_x')
+```
+# Waiting time diff between EASY and FCFS?
+```{r, echo = TRUE}
+all_jobs_sep %>%
+  filter(sched %in% c('easy', 'fcfs')) %>%
+  filter(waiting_time > 1) %>%
+  ggplot(aes(xmin=starting_time, xmax=finish_time,
+             ymin=psetmin, ymax=psetmax+1)) +
+  geom_rect(aes(fill=waiting_time)) +
+  facet_wrap(vars(sched), ncol=1) +
+  theme_bw() +
+  scale_fill_viridis(discrete=FALSE) +
+  labs(x='Time', y='Resources')
+```