-
huongdm1896 authoredhuongdm1896 authored
0_mojitos.py 3.46 KiB
# python3 0_mojitos.py --dir "Log/Flower_campaign/Flower_instance_fedAvg_cifar10_epoch1"
import os
import json
import pandas as pd
from pathlib import Path
import glob
import argparse
# Command-line argument parsing
parser = argparse.ArgumentParser(description="Specify log directory for processing Mojitos files")
parser.add_argument('--dir', required=True, help="Path to the directory containing log files")
args = parser.parse_args()
# Directory from command line argument
processing_dir = args.dir
# Correct the directory name if necessary
config_pattern = os.path.join(processing_dir, "Expetator", "config_instance_*.json")
config_files = glob.glob(config_pattern)
print(config_files)
if config_files:
meta_file = config_files[0] # Since there is only one, take the first match
print("Found config file:", meta_file)
else:
print("No config file found")
exit(1)
with open(meta_file, "r") as file:
metadata = json.load(file)
server_ip = metadata["server"]["ip"]
clients_info = metadata["clients"]
clients_ips = {client["ip"]: client["name"] for client in clients_info}
# Function to find the single "_mojitos" directory
def find_single_mojitos_dir(folder_path):
for root, dirs, files in os.walk(folder_path):
for dir_name in dirs:
if dir_name.endswith("mojitos"):
return os.path.join(root, dir_name)
return None # Return None if no mojitos folder is found
# Find the single "_mojitos" directory
mojitos_dir = find_single_mojitos_dir(processing_dir)
if mojitos_dir is None:
print("No mojitos directory found")
exit(1)
# Defined pattern to find all files in the mojitos directory
mojitos_file_pattern = os.path.join(mojitos_dir, "*-*.grid5000.fr_flower_*")
mojitos_file_list = glob.glob(mojitos_file_pattern)
# Dictionary to store all the individual frames
mojitos_individual_frames = {}
# Create output directory for processed files
output_dir = os.path.join(mojitos_dir, "processed_files")
os.makedirs(output_dir, exist_ok=True)
for file_path in sorted(mojitos_file_list):
filename = os.path.basename(file_path)
identifier, last_number = filename.split("_flower_")
identifier = identifier.split('.')[0]
last_number = last_number.strip()
df = pd.read_csv(file_path, sep="\s+")
energy_columns = [
col for col in df.columns
if col.startswith("package") or col.startswith("dram")
]
power_columns = [
col for col in df.columns
if col.startswith("gpu") and col.endswith("_power")
]
df["sum_package_dram"] = df[energy_columns].sum(axis=1) if energy_columns else 0
df["sum_gpu"] = df[power_columns].sum(axis=1) * 100 if power_columns else 0 # mW * 0.1s → µJ (mojit sap 0.1)
# Luôn tạo cột sum_package_dram_gpu
df["sum_package_dram_gpu"] = df["sum_package_dram"] + df["sum_gpu"]
mojitos_individual_frames[filename] = df
output_file = os.path.join(output_dir, f"processed_{filename}")
df.to_csv(output_file, index=False)
print("Processed files in: ", output_dir)
# # Merging all dataframes into one
# merged_df = pd.concat(mojitos_individual_frames.values(), ignore_index=True)
# # Save the merged dataframe into a CSV file
# test_camp = os.path.basename(os.path.dirname(processing_dir))
# test_case = os.path.basename(processing_dir)
# output_merged_file = os.path.join(output_dir, f"merged_mojitos_{test_camp}_{test_case}.csv")
# merged_df.to_csv(output_merged_file, index=False)
# print(f"Merged file saved to: {output_merged_file}")