From 6d77b8963ac5960f3f75a9e5974dd5f350b882af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20Madon?= <mael.madon@irit.fr> Date: Sun, 30 Jan 2022 16:43:22 +0100 Subject: [PATCH] campaign1 ready to be launched with 2 expe for testing --- .gitignore | 2 +- campaign1.py | 17 +++++------------ instance1.py | 37 ++++++++++++++++++++++++------------- passive_submit.sh | 6 ++++++ scripts/install_nix.sh | 2 +- 5 files changed, 37 insertions(+), 27 deletions(-) create mode 100755 passive_submit.sh mode change 100644 => 100755 scripts/install_nix.sh diff --git a/.gitignore b/.gitignore index 696250c..9c5d5e1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ sched_input/*.json workload/* !workload/empty_workload.json *.ipynb_checkpoints -*/__pycache__/ +*__pycache__/ out/* cache_nix diff --git a/campaign1.py b/campaign1.py index 73e267e..d227a5b 100644 --- a/campaign1.py +++ b/campaign1.py @@ -5,7 +5,6 @@ import concurrent.futures from instance1 import start_instance # Prepare the start date samples - begin_trace = 1356994806 # according to original SWF header jun1_unix_time, nov30_unix_time = mktime(strptime( 'Sun Jun 1 00:00:00 2014')), mktime(strptime('Sun Nov 30 23:59:59 2014')) @@ -17,23 +16,17 @@ random.seed(1997) nb_expe = 2 -def print_two(a, b): - print(a, b) - sleep(3) - print("hello") - return 0 - - start_dates = [random.randint(jun1, nov30 - 72 * 3600) for _ in range(nb_expe)] with concurrent.futures.ProcessPoolExecutor() as executor: - - instances = [executor.submit(start_instance, i, start_dates[i]) - for i in range(nb_expe)] + instances = [] + for i in range(nb_expe): + print(f"Submit expe {i}") + instances.append(executor.submit(start_instance, i, start_dates[i], True, True)) for instance in concurrent.futures.as_completed(instances): - print(instance.result()) + print(f"Expe {instance.result()} terminated") # For every start date # Call the one_instance file with this date diff --git a/instance1.py b/instance1.py index 0b0a4bc..5123840 100755 --- a/instance1.py +++ b/instance1.py @@ -28,8 +28,9 @@ def prepare_input_data(expe_num, start_date): job_walltime_factor=8) -def run_expe(expe_num, user_category, window_size): - """Run batmen with given behavior and demand response window""" +def run_expe(expe_num, user_category, window_size, clean_log): + """Run batmen with given behavior and demand response window. + Expe_num should be a small integer (eg < 100)""" # Useful vars and output folder if window_size==0.5: w_size = '05' @@ -62,24 +63,34 @@ def run_expe(expe_num, user_category, window_size): json.dump(data, user_description_file) # Generate and run robin instance + socket_batsim = f"tcp://localhost:280{expe_num:02d}" + socket_batsched = f"tcp://*:280{expe_num:02d}" batcmd = gen_batsim_cmd( - pf, wl, EXPE_DIR, "--energy --enable-compute-sharing --enable-dynamic-jobs --acknowledge-dynamic-jobs --enable-profile-reuse") - schedcmd = f"batsched -v bin_packing_energy --queue_order=desc_size --variant_options_filepath={uf}" + pf, wl, EXPE_DIR, f"--socket-endpoint={socket_batsim} --energy --enable-compute-sharing --enable-dynamic-jobs --acknowledge-dynamic-jobs --enable-profile-reuse") + schedcmd = f"batsched --socket-endpoint={socket_batsched} -v bin_packing_energy --queue_order=desc_size --variant_options_filepath={uf}" instance = RobinInstance(output_dir=EXPE_DIR, batcmd=batcmd, schedcmd=schedcmd, - simulation_timeout=30, ready_timeout=5, - success_timeout=10, failure_timeout=0 + simulation_timeout=604800, ready_timeout=10, + success_timeout=3600, failure_timeout=5 ) instance.to_file(EXPE_FILE) - print(f"before run {EXPE_FILE}") + print(f"Run robin {EXPE_FILE}") ret = run_robin(EXPE_FILE) - print(f"after run {EXPE_FILE}") + print(f"Robin {EXPE_FILE} finished") + # Remove the log files that can quickly become heavy... + if clean_log: + os.remove(f"{EXPE_DIR}/log/batsim.log") + os.remove(f"{EXPE_DIR}/log/sched.err.log") + os.remove(f"{EXPE_DIR}/log/sched.out.log") -def start_instance(expe_num, start_date): + + +def start_instance(expe_num, start_date, prepare_workload=True, clean_log=False): # Prepare workload - #prepare_input_data(expe_num, start_date) + if prepare_workload: + prepare_input_data(expe_num, start_date) # Create expe folder if not os.path.exists(f"{ROOT_DIR}/out/expe{expe_num}"): @@ -88,18 +99,18 @@ def start_instance(expe_num, start_date): # Run with Rigid behavior (the demand response window has no influence here) run_expe(expe_num=expe_num, user_category="replay_user_rigid", - window_size=1) + window_size=1, clean_log=clean_log) # 4*4 = 16 expe for behavior in ["dm_user_reconfig","dm_user_degrad", "dm_user_renonce","dm_user_delay"]: for window_size in [0.5, 1, 2, 4]: - run_expe(expe_num, behavior, window_size) + run_expe(expe_num, behavior, window_size, clean_log=clean_log) ###### Output data treatment ###### # Produce the utilisation viz? - return 0 + return expe_num def main(): parser = argparse.ArgumentParser( diff --git a/passive_submit.sh b/passive_submit.sh new file mode 100755 index 0000000..68278c3 --- /dev/null +++ b/passive_submit.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd ~/demand-response-user + +source scripts/install_nix.sh +nix-shell --pure -A exp_env --run "python3 campaign1.py" + diff --git a/scripts/install_nix.sh b/scripts/install_nix.sh old mode 100644 new mode 100755 index fa86af0..3199a8b --- a/scripts/install_nix.sh +++ b/scripts/install_nix.sh @@ -3,5 +3,5 @@ cd ~/demand-response-user sudo-g5k sudo su root --command "echo 1 > /proc/sys/kernel/unprivileged_userns_clone" curl https://nixos.org/releases/nix/nix-2.6.0/install | sh -. ${HOME}/.nix-profile/etc/profile.d/nix.sh +source ${HOME}/.nix-profile/etc/profile.d/nix.sh nix-store --import < cache_nix -- GitLab