Adds tools, corrects several bugs

expetator/bundle.py

@@ -19,7 +19,7 @@ def init_bundle(bundlename):
 
 def save_bundle(bundle_name, bundle_data, target_directory):
     os.makedirs(target_directory, exist_ok=True)
-    bundle_data.to_csv(target_directory+'/'+bundle_name,sep=' ', index=False)
+    bundle_data.drop('basename', axis=1).to_csv(target_directory+'/'+bundle_name,sep=' ', index=False)
 
 
 

expetator/clean_csv.py

+#! /usr/bin/python3
+
+import sys
+import pandas as pd
+
+def clean_csv(filename, nb):
+
+    df = pd.read_csv(filename, sep=' ', )
+
+    # remove the last incomplete experiment
+    last = (len(df)//nb)*nb
+    df = df[:last]
+
+    # remove complete experiment is the monitoring failed
+    for bloc_id in range(len(df)//nb-1,-1, -1):
+        for idx in range(nb):
+            tmp = df.iloc[bloc_id*nb+idx]
+            if min(tmp[10:]) <= -1:
+                df = df.drop( range(bloc_id*nb, (bloc_id+1)*nb) )
+                break
+                
+
+    df.to_csv(filename[:-4]+"_cleaned.csv", sep=' ', index=False)
+
+def main():
+    if len(sys.argv) != 3:
+        print("usage %s filename expe_size\nwhere expe_size is the number of experiment for one experimental set")
+    
+    else:
+        clean_csv(sys.argv[1], int(sys.argv[2]))
+
+if __name__ == '__main__':
+    main()
+
+    

expetator/monitors/kwollect.py

@@ -5,11 +5,12 @@ import os
 import json
 
 import platform 
-def get_g5k_target_metric():
-    cluster_name = platform.node().split('-')[0]
+def get_g5k_target_metric(cluster_name=None):
+    if cluster_name is None:
+        cluster_name = platform.node().split('-')[0]
 
     if cluster_name in ['grisou', 'graoully', 'grimoire',
-                        'grisou', 'gros', 'gruss', 'paravance']:
+                        'gros', 'gruss', 'paravance']:
         return 'pdu_outlet_power_watt'
     if cluster_name in ['troll', 'yeti', 'gemini', 'neowise',
                           'orion', 'pyxis', 'sagittaire', 'taurus']:

expetator/remove_watermark.py

@@ -23,6 +23,7 @@ def remove_watermark(target_file, target_dir):
     watermark.remove_wt_name(bundle_data)
     if target_file.endswith('.zip'):
         target_file = target_file[:-4]
+
     bundle.save_bundle(target_file, bundle_data, target_dir)
 
     if not moj_cleaned is None:

setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="expetator",
-    version="0.3.13",
+    version="0.3.14",
     author="Georges Da Costa",
     author_email="georges.da-costa@irit.fr",
     description="A framework for monitoring HPC applications using DVFS",
@@ -23,7 +23,7 @@ setuptools.setup(
                        'monitors/*.bz2', 'monitors/*.diff',
                        'leverages/*.[ch]', 'leverages/*.sh', 'leverages/*_mak']},
     include_package_data=True,
-    install_requires=['execo'],
+    install_requires=['execo', 'requests'],
     entry_points={
         'console_scripts': [
             'remove_watermark = expetator.remove_watermark:main',