script: gen wload params & wloads

scripts-py/expe_energumen/m100_generate_expe_workload_params.py

+#!/usr/bin/env python3
+import argparse
+import datetime
+import json
+import random
+import sys
+
+EPOCH_M100 = datetime.datetime(year=2022, month=1, day=1)
+
+def main():
+    datetime_parser = lambda f: datetime.datetime.strptime(f, '%Y-%m-%d %H:%M:%S')
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--begin", type=datetime_parser, default='2022-01-08 00:00:00', help='the begin datetime of the replay period to consider')
+    parser.add_argument("--end", type=datetime_parser, default='2022-09-20 00:00:00', help='the end datetime of the replay period')
+    parser.add_argument("--nb_workloads", type=int, default=30, help='the number of workloads to generate')
+    parser.add_argument("--replay_duration", type=int, default=60*60*24, help='the duration in seconds of the window in which jobs are submitted in each workload')
+    parser.add_argument("--seed", type=int, default=0, help='the random number generator seed')
+    parser.add_argument("-o", "--output_file", type=str, help="if set, write output to this file instead of stdout")
+    args = parser.parse_args()
+
+    assert isinstance(args.seed, int)
+    random.seed(args.seed)
+
+    epoch_offset_seconds = int((args.begin - EPOCH_M100).total_seconds())
+    period_range_seconds = int((args.end - args.begin).total_seconds())
+
+    offset_start_points_seconds = sorted([random.randint(0, period_range_seconds) for _ in range(args.nb_workloads)])
+    workload_params = [{
+        'start_dt': str(args.begin + datetime.timedelta(seconds=start_point)),
+        'start_dt_s': epoch_offset_seconds + start_point,
+        'end_dt': str(args.begin + datetime.timedelta(seconds=start_point + args.replay_duration)),
+        'end_dt_s': epoch_offset_seconds + start_point + args.replay_duration,
+    } for start_point in offset_start_points_seconds]
+
+    f = sys.stdout
+    if args.output_file is not None:
+        f = open(args.output_file, 'w')
+    print(json.dumps(workload_params, indent=2), file=f)

scripts-py/expe_energumen/m100_generate_expe_workloads.py

+#!/usr/bin/env python3
+import argparse
+import datetime
+import json
+import random
+import subprocess
+import sys
+
+EPOCH_M100 = datetime.datetime(year=2022, month=1, day=1)
+
+def main():
+    datetime_parser = lambda f: datetime.datetime.strptime(f, '%Y-%m-%d %H:%M:%S')
+    parser = argparse.ArgumentParser()
+    parser.add_argument("workload_params_filepath", type=str, help='filepath to the workload parameters')
+    parser.add_argument("input_jobs", help='path to the CSV file that contains the jobs information')
+    parser.add_argument("input_power_timeseries_prefix", help="filepath prefix to the location of the parquet files that contain node power consumption time series")
+    parser.add_argument("-o", "--output_dir", required=True, help="filepath where all the workload-related files should be generated")
+    args = parser.parse_args()
+
+    with open(args.workload_params_filepath) as f:
+        params = json.load(f)
+
+    for index, param in enumerate(params):
+        process_args = [
+            'm100-generate-batsim-workload',
+            args.input_jobs,
+            args.input_power_timeseries_prefix,
+            '-o', args.output_dir,
+            '-p', 'delay',
+            '--begin', param['start_dt'],
+            '--end', param['end_dt'],
+        ]
+        print(f"{datetime.datetime.now()} starting generation of workload {index+1}/{len(params)} (start_dt={param['start_dt']})")
+        subprocess.run(process_args, check=True)
+
+    print(f"{datetime.datetime.now()} all workloads have been generated")

scripts-py/pyproject.toml

@@ -17,3 +17,5 @@ m100-join-usable-jobs-info = "expe_energumen.m100_join_usable_jobs_info:join"
 m100-generate-sg-platform = "expe_energumen.m100_generate_simgrid_platform:main"
 m100-generate-batsim-workload = "expe_energumen.m100_generate_batsim_workload:main"
 m100-compute-gantt-power-consumption = "expe_energumen.m100_compute_gantt_power_consumption:main"
+m100-generate-expe-workload-params = "expe_energumen.m100_generate_expe_workload_params:main"
+m100-generate-expe-workloads = "expe_energumen.m100_generate_expe_workloads:main"