From 79c20ad391e7ee312b5931b2791836024b1d9e3b Mon Sep 17 00:00:00 2001
From: Georges Da Costa <dacosta@irit.fr>
Date: Tue, 26 Jul 2022 21:34:45 +0200
Subject: [PATCH] Updates watermark detection
---
expetator/demo_watermark.py | 46 +++++++++++++++++++++++++++++++++++++
expetator/experiment.py | 2 +-
expetator/monitoring_csv.py | 15 +++++++++---
expetator/watermark.py | 4 ++--
setup.py | 2 +-
5 files changed, 62 insertions(+), 7 deletions(-)
create mode 100755 expetator/demo_watermark.py
diff --git a/expetator/demo_watermark.py b/expetator/demo_watermark.py
new file mode 100755
index 0000000..920a72c
--- /dev/null
+++ b/expetator/demo_watermark.py
@@ -0,0 +1,46 @@
+#! /usr/bin/python3
+
+import expetator.monitoring_csv as monitoring_csv
+import expetator.monitoring_list as monitoring_list
+import expetator.watermark as watermark
+import sys
+import pandas as pd
+import matplotlib.pyplot as plt
+import json
+
+if __name__ == '__main__':
+ if len(sys.argv) != 3 or not sys.argv[1] in ['csv', 'list']:
+ print("""Usage :
+%s csv file.csv
+
+or
+
+%s list file.list
+""")
+ sys.exit(0)
+
+ filename = sys.argv[2]
+ if sys.argv[1] == 'csv':
+
+ a = monitoring_csv._read_csv(filename)
+ watermark.demo_watermark_detection(a, 10)
+ plt.show(block=True)
+
+
+ if sys.argv[1] == 'list':
+
+ with open(filename) as f_id:
+ data = json.load(f_id)
+
+ for name, timestamps, power in data:
+
+ df = pd.DataFrame([timestamps, power]).transpose()
+ df.columns = ["#timestamp", "Values"]
+
+ watermark.demo_watermark_detection(df, 10)
+ plt.show(block=True)
+
+
+
+
+
diff --git a/expetator/experiment.py b/expetator/experiment.py
index ba91606..e1e0128 100755
--- a/expetator/experiment.py
+++ b/expetator/experiment.py
@@ -31,7 +31,7 @@ class Executor:
self.nbcores = len(content)
self.nbhosts = len(self.hostnames)
self.mpi_options = '--map-by node --mca orte_rsh_agent oarsh'
- if self.hostnames.split('-')[0] in ['grvingt', 'grimani', 'grele', 'troll', 'yeti', 'dahu']:
+ if self.hostnames[0].split('-')[0] in ['grvingt', 'grimani', 'grele', 'troll', 'yeti', 'dahu']:
self.mpi_options = '-mca mtl psm2 -mca pml ^ucx,ofi -mca btl ^ofi,openib ' + self.mpi_options
self.sudo = 'sudo-g5k'
self.ssh = 'oarsh'
diff --git a/expetator/monitoring_csv.py b/expetator/monitoring_csv.py
index 30e757e..8eb8b9a 100644
--- a/expetator/monitoring_csv.py
+++ b/expetator/monitoring_csv.py
@@ -3,14 +3,23 @@ import pandas as pd
import matplotlib.pyplot as plt
import sys
+
+def _read_csv(filename):
+ df = pd.read_csv(filename, sep=' ', skipinitialspace=True)
+ if df.columns[-1].startswith('Unnamed'):
+ df.drop(columns=df.columns[-1:], axis=1, inplace=True)
+ return df
+
+
+
def read_host_csv(prefix, hostname, startTime, basename, fullname, archive_fid=None):
fullpath= '%s_%s/%s_%s_%s' % (basename, prefix, hostname, fullname, startTime)
if archive_fid is None:
with open(fullpath) as file_id:
- data = pd.read_csv(fullpath,sep=' ', skipinitialspace=True)
+ data = _read_csv(fullpath)
else:
with archive_fid.open(fullpath) as file_id:
- data = pd.read_csv(file_id,sep=' ')
+ data = _read_csv(file_id)
data = data.dropna(axis='columns')
return data
@@ -45,7 +54,7 @@ def write_bundle_csv(prefix, bundle, data, target_directory):
def show_csv(filename, norm=False):
filename = sys.argv[-1]
- a = pd.read_csv(filename, sep = " ")
+ a = _read_csv(filename)
a['#timestamp'] = a['#timestamp']-a['#timestamp'][0]
if norm:
tmp = (a/a.max())
diff --git a/expetator/watermark.py b/expetator/watermark.py
index e622310..8f35a46 100644
--- a/expetator/watermark.py
+++ b/expetator/watermark.py
@@ -74,7 +74,7 @@ def get_shift(dataframe, frequency=10, duration=30, backward=False):
#print(delta, res)
- if current < .7:
+ if current < .6:
res = pos_in_data, 0
return res
@@ -102,7 +102,7 @@ def remove_wt_name(dataframe, target='fullname', signature='wt-30-'):
## Tool for virtualisation
-def demo_watermark_detection(focus, freq):
+def demo_watermark_detection(focus, freq=10):
df = focus.loc[:, focus.columns != '#timestamp']
norm_focus=(df-df.min())/(df.max()-df.min())
norm_focus['#timestamp'] = focus['#timestamp']
diff --git a/setup.py b/setup.py
index 2e302be..58c77d1 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
setuptools.setup(
name="expetator",
- version="0.3.16",
+ version="0.3.18",
author="Georges Da Costa",
author_email="georges.da-costa@irit.fr",
description="A framework for monitoring HPC applications using DVFS",
--
GitLab