Updates watermark detection

expetator/demo_watermark.py

+#! /usr/bin/python3
+
+import expetator.monitoring_csv as monitoring_csv
+import expetator.monitoring_list as monitoring_list
+import expetator.watermark as watermark
+import sys
+import pandas as pd
+import matplotlib.pyplot as plt
+import json
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3 or not sys.argv[1] in ['csv', 'list']:
+        print("""Usage :
+%s csv file.csv
+
+or
+
+%s list file.list
+""")
+        sys.exit(0)
+
+    filename = sys.argv[2]
+    if sys.argv[1] == 'csv':
+        
+        a = monitoring_csv._read_csv(filename)
+        watermark.demo_watermark_detection(a, 10)
+        plt.show(block=True)
+
+
+    if sys.argv[1] == 'list':
+
+        with open(filename) as f_id:
+            data = json.load(f_id)
+
+            for name, timestamps, power in data:
+
+                df = pd.DataFrame([timestamps, power]).transpose()
+                df.columns = ["#timestamp", "Values"]
+
+                watermark.demo_watermark_detection(df, 10)
+                plt.show(block=True)
+
+
+
+        
+                

expetator/experiment.py

@@ -31,7 +31,7 @@ class Executor:
                 self.nbcores = len(content)
             self.nbhosts = len(self.hostnames)
             self.mpi_options = '--map-by node --mca orte_rsh_agent oarsh'
-            if self.hostnames.split('-')[0] in ['grvingt', 'grimani', 'grele', 'troll', 'yeti', 'dahu']:
+            if self.hostnames[0].split('-')[0] in ['grvingt', 'grimani', 'grele', 'troll', 'yeti', 'dahu']:
                 self.mpi_options = '-mca mtl psm2 -mca pml ^ucx,ofi -mca btl ^ofi,openib ' + self.mpi_options
             self.sudo = 'sudo-g5k'
             self.ssh = 'oarsh'

expetator/monitoring_csv.py

@@ -3,14 +3,23 @@ import pandas as pd
 import matplotlib.pyplot as plt
 import sys
 
+
+def _read_csv(filename):
+    df = pd.read_csv(filename, sep=' ', skipinitialspace=True)
+    if df.columns[-1].startswith('Unnamed'):
+        df.drop(columns=df.columns[-1:], axis=1, inplace=True)
+    return df
+
+
+
 def read_host_csv(prefix, hostname, startTime, basename, fullname, archive_fid=None):
     fullpath= '%s_%s/%s_%s_%s' % (basename, prefix, hostname, fullname, startTime)
     if archive_fid is None:
         with open(fullpath) as file_id:
-            data = pd.read_csv(fullpath,sep=' ', skipinitialspace=True)
+            data = _read_csv(fullpath)
     else:
         with archive_fid.open(fullpath) as file_id:
-            data = pd.read_csv(file_id,sep=' ')
+            data = _read_csv(file_id)
     data = data.dropna(axis='columns')
     return data
 
@@ -45,7 +54,7 @@ def write_bundle_csv(prefix, bundle, data, target_directory):
 def show_csv(filename, norm=False):
     filename = sys.argv[-1]
 
-    a = pd.read_csv(filename, sep = " ")
+    a = _read_csv(filename)
     a['#timestamp'] = a['#timestamp']-a['#timestamp'][0]
     if norm:
         tmp = (a/a.max())

expetator/watermark.py

@@ -74,7 +74,7 @@ def get_shift(dataframe, frequency=10, duration=30, backward=False):
         
             #print(delta, res)
 
-    if current < .7:
+    if current < .6:
         res = pos_in_data, 0
     
     return res
@@ -102,7 +102,7 @@ def remove_wt_name(dataframe, target='fullname', signature='wt-30-'):
 
 ## Tool for virtualisation
 
-def demo_watermark_detection(focus, freq):
+def demo_watermark_detection(focus, freq=10):
     df = focus.loc[:, focus.columns != '#timestamp']
     norm_focus=(df-df.min())/(df.max()-df.min())
     norm_focus['#timestamp'] = focus['#timestamp']

setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="expetator",
-    version="0.3.16",
+    version="0.3.18",
     author="Georges Da Costa",
     author_email="georges.da-costa@irit.fr",
     description="A framework for monitoring HPC applications using DVFS",