diff --git a/expetator/bundle.py b/expetator/bundle.py index e28376757a446aa43a2b02b3b86e4242a82b7f3f..4ee276cde34768295156a1d0a168c0417ee85c09 100644 --- a/expetator/bundle.py +++ b/expetator/bundle.py @@ -19,7 +19,7 @@ def init_bundle(bundlename): def save_bundle(bundle_name, bundle_data, target_directory): os.makedirs(target_directory, exist_ok=True) - bundle_data.to_csv(target_directory+'/'+bundle_name,sep=' ', index=False) + bundle_data.drop('basename', axis=1).to_csv(target_directory+'/'+bundle_name,sep=' ', index=False) diff --git a/expetator/clean_csv.py b/expetator/clean_csv.py new file mode 100644 index 0000000000000000000000000000000000000000..e2b9bac7d82e052a41eaaf07c449be487713c77a --- /dev/null +++ b/expetator/clean_csv.py @@ -0,0 +1,35 @@ +#! /usr/bin/python3 + +import sys +import pandas as pd + +def clean_csv(filename, nb): + + df = pd.read_csv(filename, sep=' ', ) + + # remove the last incomplete experiment + last = (len(df)//nb)*nb + df = df[:last] + + # remove complete experiment is the monitoring failed + for bloc_id in range(len(df)//nb-1,-1, -1): + for idx in range(nb): + tmp = df.iloc[bloc_id*nb+idx] + if min(tmp[10:]) <= -1: + df = df.drop( range(bloc_id*nb, (bloc_id+1)*nb) ) + break + + + df.to_csv(filename[:-4]+"_cleaned.csv", sep=' ', index=False) + +def main(): + if len(sys.argv) != 3: + print("usage %s filename expe_size\nwhere expe_size is the number of experiment for one experimental set") + + else: + clean_csv(sys.argv[1], int(sys.argv[2])) + +if __name__ == '__main__': + main() + + diff --git a/expetator/monitors/kwollect.py b/expetator/monitors/kwollect.py index feadeab7741fdd397b96de158f9f86588be227d2..c42dc3237f3e0361b925d5ed350e3c2b74b926b4 100644 --- a/expetator/monitors/kwollect.py +++ b/expetator/monitors/kwollect.py @@ -5,11 +5,12 @@ import os import json import platform -def get_g5k_target_metric(): - cluster_name = platform.node().split('-')[0] +def get_g5k_target_metric(cluster_name=None): + if cluster_name is None: + cluster_name = platform.node().split('-')[0] if cluster_name in ['grisou', 'graoully', 'grimoire', - 'grisou', 'gros', 'gruss', 'paravance']: + 'gros', 'gruss', 'paravance']: return 'pdu_outlet_power_watt' if cluster_name in ['troll', 'yeti', 'gemini', 'neowise', 'orion', 'pyxis', 'sagittaire', 'taurus']: diff --git a/expetator/remove_watermark.py b/expetator/remove_watermark.py index 3a485fff7921e6b57b3cdfb7492e9971a15135e3..f25dd256cef55051ed0b0c06de043366d431572c 100644 --- a/expetator/remove_watermark.py +++ b/expetator/remove_watermark.py @@ -23,6 +23,7 @@ def remove_watermark(target_file, target_dir): watermark.remove_wt_name(bundle_data) if target_file.endswith('.zip'): target_file = target_file[:-4] + bundle.save_bundle(target_file, bundle_data, target_dir) if not moj_cleaned is None: diff --git a/setup.py b/setup.py index 79a16e490e7189c993c98cc752c78bfefc2c0481..9b00c900172d671bfcad508cd8a192dff4b991cc 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="expetator", - version="0.3.13", + version="0.3.14", author="Georges Da Costa", author_email="georges.da-costa@irit.fr", description="A framework for monitoring HPC applications using DVFS", @@ -23,7 +23,7 @@ setuptools.setup( 'monitors/*.bz2', 'monitors/*.diff', 'leverages/*.[ch]', 'leverages/*.sh', 'leverages/*_mak']}, include_package_data=True, - install_requires=['execo'], + install_requires=['execo', 'requests'], entry_points={ 'console_scripts': [ 'remove_watermark = expetator.remove_watermark:main',