Skip to content
Snippets Groups Projects
0_mojitos.py 3.46 KiB
# python3 0_mojitos.py --dir "Log/Flower_campaign/Flower_instance_fedAvg_cifar10_epoch1"

import os
import json
import pandas as pd
from pathlib import Path
import glob
import argparse

# Command-line argument parsing
parser = argparse.ArgumentParser(description="Specify log directory for processing Mojitos files")
parser.add_argument('--dir', required=True, help="Path to the directory containing log files")
args = parser.parse_args()

# Directory from command line argument
processing_dir = args.dir

# Correct the directory name if necessary
config_pattern = os.path.join(processing_dir, "Expetator", "config_instance_*.json")
config_files = glob.glob(config_pattern)
print(config_files)

if config_files:
    meta_file = config_files[0]  # Since there is only one, take the first match
    print("Found config file:", meta_file)
else:
    print("No config file found")
    exit(1)

with open(meta_file, "r") as file:
    metadata = json.load(file)

server_ip = metadata["server"]["ip"]
clients_info = metadata["clients"]
clients_ips = {client["ip"]: client["name"] for client in clients_info}

# Function to find the single "_mojitos" directory
def find_single_mojitos_dir(folder_path):
    for root, dirs, files in os.walk(folder_path):
        for dir_name in dirs:
            if dir_name.endswith("mojitos"):
                return os.path.join(root, dir_name)
    return None  # Return None if no mojitos folder is found

# Find the single "_mojitos" directory
mojitos_dir = find_single_mojitos_dir(processing_dir)

if mojitos_dir is None:
    print("No mojitos directory found")
    exit(1)

# Defined pattern to find all files in the mojitos directory
mojitos_file_pattern = os.path.join(mojitos_dir, "*-*.grid5000.fr_flower_*")
mojitos_file_list = glob.glob(mojitos_file_pattern)

# Dictionary to store all the individual frames
mojitos_individual_frames = {}
# Create output directory for processed files
output_dir = os.path.join(mojitos_dir, "processed_files")
os.makedirs(output_dir, exist_ok=True)

for file_path in sorted(mojitos_file_list):
    filename = os.path.basename(file_path)
    identifier, last_number = filename.split("_flower_")
    identifier = identifier.split('.')[0]
    last_number = last_number.strip()

    df = pd.read_csv(file_path, sep="\s+")

    energy_columns = [
        col for col in df.columns 
        if col.startswith("package") or col.startswith("dram")
    ]
    
    power_columns = [
        col for col in df.columns
        if col.startswith("gpu") and col.endswith("_power")
    ]

    df["sum_package_dram"] = df[energy_columns].sum(axis=1) if energy_columns else 0
    df["sum_gpu"] = df[power_columns].sum(axis=1) * 100 if power_columns else 0  # mW * 0.1s → µJ (mojit sap 0.1)

    # Luôn tạo cột sum_package_dram_gpu
    df["sum_package_dram_gpu"] = df["sum_package_dram"] + df["sum_gpu"]

    mojitos_individual_frames[filename] = df

    output_file = os.path.join(output_dir, f"processed_{filename}")
    df.to_csv(output_file, index=False)


print("Processed files in: ", output_dir)
# # Merging all dataframes into one
# merged_df = pd.concat(mojitos_individual_frames.values(), ignore_index=True)

# # Save the merged dataframe into a CSV file
# test_camp = os.path.basename(os.path.dirname(processing_dir))
# test_case = os.path.basename(processing_dir)

# output_merged_file = os.path.join(output_dir, f"merged_mojitos_{test_camp}_{test_case}.csv")
# merged_df.to_csv(output_merged_file, index=False)

# print(f"Merged file saved to: {output_merged_file}")