Skip to content
Snippets Groups Projects
Select Git revision
  • main default protected
1 result
Blame Permalink
5_combine_energy_fl.py 3.55 KiB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87 






# cmd python3 5_combine_energy_fl.py -a Data_analysis/Output_level_2/flower_fedAvg2Clients_accuracy_time.csv -e Data_analysis/Output_level_2/mean_energy_Flower_TetHoliday_Flower_instance_fedAvg2Clients_cifar10.csv -m Log/Flower_TetHoliday/Flower_instance_fedAvg2Clients_cifar10/Expetator_gros-24.nancy.grid5000.fr_1738035839
# output: combine_{extracted_name}.csv -> combine_Flower_TetHoliday_Flower_instance_fedAvg2Clients_cifar10.csv

import pandas as pd
import os
from datetime import datetime
import argparse

parser = argparse.ArgumentParser(description="Specify pattern with parsed values")
parser.add_argument('-a','--accuracy', required=True, help="Accuracy file path")
parser.add_argument('-e','--energy', required=True, help="Energy file path")
parser.add_argument('-m','--meta', required=True, help="Meta data file path")
args = parser.parse_args()
 
current_dir = os.getcwd()
parent_dir = os.path.abspath(os.path.join(current_dir))

# Load data
accuracies_file = os.path.abspath(os.path.join(current_dir, args.accuracy))
energy_file = os.path.abspath(os.path.join(current_dir, args.energy))
exp_file = os.path.abspath(os.path.join(current_dir, args.meta))
# accuracies_file = "/home/hdomai/Documents/RepoLab/Data/Data_analysis/Output_level_2/fedAvg_accuracies.csv"
# energy_file = "/home/hdomai/Documents/RepoLab/Data/Data_analysis/Output_level_2/mean_E_fedAvg_file_number.csv"
# exp_file = "/home/hdomai/Documents/RepoLab/Data/Log/Flower_TetHoliday/Flower_instance_fedAvg_cifar10/Expetator_gros-24.nancy.grid5000.fr_1738029020"

# Get the name of output file
basename = os.path.basename(energy_file)
extracted = os.path.splitext(basename)[0].replace("mean_energy_", "")

df_acc = pd.read_csv(accuracies_file)
df_energy = pd.read_csv(energy_file)
df_exp = pd.read_csv(exp_file, sep='\s+')

# # Print columns to debug
# print("Columns in df_exp:", df_exp.columns)
# exit()
# Step 1: Extract timestamp from file_path in accuracies
def extract_timestamp(file_path):
    date_str = file_path.split("_")[-2] + "_" + file_path.split("_")[-1].split(".")[0]
    return int(datetime.strptime(date_str, "%Y%m%d_%H%M%S").timestamp())

df_acc['timestamp'] = df_acc['filepath'].apply(extract_timestamp)

# Step 2: Find the closest file_number from energy for each timestamp
df_energy['file_number'] = df_energy['file_number'].astype(int)

def find_closest_file_number(acc_timestamp, energy_timestamps):
    closest_timestamp = min(energy_timestamps, key=lambda x: abs(x - acc_timestamp))
    return closest_timestamp

df_acc['mapped_file_number'] = df_acc['timestamp'].apply(
    lambda x: find_closest_file_number(x, df_energy['file_number'].values)
)

# Step 3: Combine accuracies and energy data
df_combined = pd.merge(
    df_acc,
    df_energy,
    left_on="mapped_file_number",
    right_on="file_number",
    how="inner"
)

# Step 4: Add the fmax column by merging with file mojitos records
df_combined = pd.merge(
    df_combined,
    df_exp[["startTime", "fmax"]],
    left_on="file_number",
    right_on="startTime",
    how="left"
)

df_combined = df_combined[['file_number','fmax', 'accuracy','max_acc', 'in_round', 'time', 'mean_power', 'mean_mojitos', 'std_power_combined', 'std_mojitos_combined', 'sum_mojitos', 'n_data_mojitos', 'n_data_power', 'startTime','filepath',]]


print(df_combined)

# Construct the full path
output_dir = os.path.join(parent_dir, "Data_analysis", "Output_level_3")
output_file = os.path.join(output_dir, f"combine_{extracted}.csv")

# Ensure the directory exists
os.makedirs(output_dir, exist_ok=True)

df_combined.to_csv(output_file, index=False)
print(f"Combined summary saved to {output_file}")