instance.py

#!/usr/bin/env python3

import time
import os
import subprocess
import argparse
import json

import scripts.swf_to_batsim_split_by_user as split_user
from scripts.util import *


def prepare_input_data(expe_num, start_date):
    """Cut the original trace to extract 72h starting from this start date"""
    end_date = start_date + 72*3600
    to_keep = f"submit_time >= {start_date} and submit_time <= {end_date}"

    if not os.path.exists(f'{WL_DIR}/expe{expe_num}'):
        os.makedirs(f'{WL_DIR}/expe{expe_num}')
    split_user.generate_workload(
        input_swf=f'{WL_DIR}/MC_selection_article.swf',
        output_folder=f'{WL_DIR}/expe{expe_num}',
        keep_only=to_keep,
        job_grain=10,
        job_walltime_factor=8)


def run_expe(expe_num, user_category, window_size, clean_log):
    """Run batmen with given behavior and demand response window.
    Expe_num should be a small integer (eg < 100)"""
    # Useful vars and output folder
    EXPE_DIR = f"{ROOT_DIR}/out/expe{expe_num}/{user_category}_window{window_size}"
    create_dir_rec_if_needed(EXPE_DIR)
    create_dir_rec_if_needed(f"{EXPE_DIR}/cmd")
    EXPE_FILE = f"{EXPE_DIR}/cmd/robinfile.yaml"
    wl_folder = f'{WL_DIR}/expe{expe_num}'
    pf = f"{ROOT_DIR}/platform/average_metacentrum.xml"
    wl = f"{WL_DIR}/empty_workload.json"
    uf = f"{EXPE_DIR}/cmd/user_description_file.json"

    # Demand response window, from 16 to (16 + window_size) on day2
    dm_window = [(24+16)*3600, (int) ((24+16+window_size)*3600)]

    # User description file
    def user_description(user):
        return {
            "name": user,
            "category": user_category,
            "param": {"input_json": f"{wl_folder}/{user}.json"}
        }
    user_names = [user_file.split('.')[0] for user_file in os.listdir(wl_folder)]
    data = {}
    data["dm_window"] = dm_window
    data["log_user_stats"] = True
    data["log_folder"] = EXPE_DIR
    data["users"] = [user_description(user) for user in user_names]
    with open(uf, 'w') as user_description_file:
        json.dump(data, user_description_file)

    # Generate and run robin instance
    socket_batsim = f"tcp://localhost:280{expe_num:02d}"
    socket_batsched = f"tcp://*:280{expe_num:02d}"
    batcmd = gen_batsim_cmd(
        pf, wl, EXPE_DIR, f"--socket-endpoint={socket_batsim} --energy --enable-compute-sharing --enable-dynamic-jobs --acknowledge-dynamic-jobs --enable-profile-reuse")
    schedcmd = f"batsched --socket-endpoint={socket_batsched} -v bin_packing_energy --queue_order=desc_size --variant_options_filepath={uf}"
    instance = RobinInstance(output_dir=EXPE_DIR,
                            batcmd=batcmd,
                            schedcmd=schedcmd,
                            simulation_timeout=604800, ready_timeout=10,
                            success_timeout=3600, failure_timeout=5
                            )
    instance.to_file(EXPE_FILE)
    print(f"Run robin {EXPE_FILE}")
    ret = run_robin(EXPE_FILE)
    print(f"Robin {EXPE_FILE} finished")

    # Remove the log files that can quickly become heavy...
    if clean_log:
        os.remove(f"{EXPE_DIR}/log/batsim.log")
        os.remove(f"{EXPE_DIR}/log/sched.err.log")
        os.remove(f"{EXPE_DIR}/log/sched.out.log")



def start_instance(expe_num, start_date, prepare_workload=True, clean_log=False):
    # Prepare workload
    if prepare_workload:
        prepare_input_data(expe_num, start_date)

    # Create expe folder
    create_dir_rec_if_needed(f"{ROOT_DIR}/out/expe{expe_num}")

    # Run with Rigid behavior (the demand response window has no influence here)
    run_expe(expe_num=expe_num, 
             user_category="replay_user_rigid",
             window_size=1, clean_log=clean_log)

    # 4*2 = 8 expe
    for behavior in ["dm_user_reconfig","dm_user_degrad",
                     "dm_user_renounce","dm_user_delay"]:
        for window_size in [1, 4]:
            run_expe(expe_num, behavior, window_size, clean_log=clean_log)


    ###### Output data treatment ######
    # Produce the utilisation viz?
    return expe_num

def main():
    parser = argparse.ArgumentParser(
        description='One expe instance. To launch for example with `oarsub -l walltime=2 "./1_one_instance arg1 arg2 arg3"`')
    parser.add_argument('expe_num', type=int, help='The expe ID')
    parser.add_argument('start_date', type=int, 
                        help='Start of the 3-day window (in seconds since the start of the original trace)')
    args = parser.parse_args()
    
    start_instance(args.expe_num, args.start_date)



if __name__ == "__main__":
    main()