script: campaign runner

flake.nix

@@ -117,6 +117,7 @@
             buildInputs = [
               packages.batsim
               packages.energusched
+              packages.python-scripts
             ];
             BATSIM_ROOT_PATH="${batsim-flake}";
             EDC_LIBRARY_PATH="${packages.energusched}/lib";

scripts-py/expe_energumen/m100_run_batsim_instances.py

+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import subprocess
+from concurrent.futures import ProcessPoolExecutor
+
+EDC_LIBRARY_PATH = None
+
+def write_instance_state(fname, state):
+    with open(fname, 'w') as f:
+        f.write(state + '\n')
+
+'''
+state
+- none: nothing has run
+- ok: simu has been executed successfully
+- fail: simu has been executed but returned non-zero
+- analyzed: simu has been executed successfully and simulation output has been analyzed
+'''
+
+def manage_batsim_instance(instance_hash, instance, output_dir, workloads_dir):
+    instance_dir = f'{output_dir}/{instance_hash}'
+    state_file = f'{instance_dir}/state'
+    if os.path.exists(instance_dir) and os.path.exists(state_file):
+        state = open(state_file).read().strip()
+        if state == 'analyzed':
+            print(f'instance {instance_hash}: {state} (already)', flush=True)
+            return state
+    else:
+        state = 'none'
+        os.makedirs(instance_dir, exist_ok=True)
+
+    if state != 'ok':
+        batsim_cmd_args = [
+            'batsim',
+            '-l', f'{EDC_LIBRARY_PATH}/lib{instance["algo_name"]}.so', '0', json.dumps(instance),
+            '-p', f'{instance["platform_filepath"]}', '--mmax-workload',
+            '-w', f'{workloads_dir}/wload_delay_{instance["start_dt_s"]}.json',
+            '-e', f'{instance_dir}/',
+        ]
+
+        with open(f'{instance_dir}/batsim.stdout', 'w') as batout:
+            with open(f'{instance_dir}/batsim.stderr', 'w') as baterr:
+                p = subprocess.run(batsim_cmd_args, stdout=batout, stderr=baterr)
+
+                if p.returncode == 0:
+                    state = 'ok'
+                else:
+                    state = 'fail'
+                write_instance_state(state_file, state)
+                print(f'instance {instance_hash}: {state}', flush=True)
+
+    if state != 'analyzed':
+        pass
+        # TODO: run instance analysis
+
+    return state
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("instances", type=str, help='path to the instances JSON file')
+    parser.add_argument("-w", "--workloads_dir", type=str, help='filepath to where all workloads files are')
+    parser.add_argument('--output_state_file', type=str, required=True, help='path where the state of the execution of each instance should be written')
+    parser.add_argument("-o", "--output_dir", type=str, required=True, help='path prefix where the simulation instances should be written')
+    parser.add_argument("--max_processes", type=int, default=64, help='the maximum number of processes to use to execute the instances')
+    args = parser.parse_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    global EDC_LIBRARY_PATH
+    EDC_LIBRARY_PATH = os.environ['EDC_LIBRARY_PATH']
+    assert os.path.exists(EDC_LIBRARY_PATH), 'EDC_LIBRARY_PATH should point towards an accessible filepath'
+
+    assert os.path.exists(args.workloads_dir), f"workloads_dir '{args.workloads_dir}' does not exist"
+
+    with open(args.instances) as f:
+        instances = json.load(f)
+
+    fut = dict()
+    state_dict = dict()
+    with ProcessPoolExecutor(max_workers=args.max_processes) as executor:
+        for instance_hash, instance in instances.items():
+            fut[instance_hash] = executor.submit(manage_batsim_instance, instance_hash, instance, args.output_dir, args.workloads_dir)
+
+        for instance_hash, f in fut.items():
+            state = f.result()
+            if state is None:
+                state = 'error'
+            state_dict[instance_hash] = state
+
+    with open(f'{args.output_state_file}', 'w') as f:
+        json.dump(state_dict, f, sort_keys=True, indent=2)
+

scripts-py/pyproject.toml

@@ -21,3 +21,4 @@ m100-generate-expe-workload-params = "expe_energumen.m100_generate_expe_workload
 m100-generate-expe-workloads = "expe_energumen.m100_generate_expe_workloads:main"
 m100-generate-expe-params = "expe_energumen.m100_generate_expe_params:main"
 m100-find-max-power = "expe_energumen.m100_find_max_power:main"
+m100-run-batsim-instances = "expe_energumen.m100_run_batsim_instances:main"