Skip to content
Snippets Groups Projects
Commit c8c63a6f authored by danilo.carastandossantos's avatar danilo.carastandossantos
Browse files

consolidated prediction artifacts

parent 17fd6eeb
Branches
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 445 additions and 67 deletions
.gitignore 0 → 100644
+ 1
− 0
View file @ c8c63a6f
artifact-overview.pdf
artifact-overview.typ
+ 43
− 51
View file @ c8c63a6f
......@@ -377,125 +377,118 @@ The experimental workflow consists of three parts, (i) preprocessing of the orig
==== Step 1
#tododanilo[in the script source-code: change output filename of step 1
from a_0_filter12_singlenode.csv and from a_0_filter12_multinode.csv
to 22-0X_filter12_singlenode.csv and 22-0X_filter12_multinode.csv]
#fullbox(footer:[Memory: 128 Go. Time (sequential): 18:00:00])[
#fullbox(footer:[#emph-overhead[Memory: 128 Go. Time (sequential): 18:00:00]])[
```python
./scripts-py/expe_energumen/m100_pred_preprocess_1.py \
-j ../m100-data/22-01_jobs.parquet \
-p m100-data/22-01_power_total.parquet
-j ./m100-data/22-01_jobs.parquet \
-p ./m100-data/22-01_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_1.py \
-j ../m100-data/22-02_jobs.parquet \
-p m100-data/22-02_power_total.parquet
-j ./m100-data/22-02_jobs.parquet \
-p ./m100-data/22-02_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_1.py \
-j ../m100-data/22-03_jobs.parquet \
-p m100-data/22-03_power_total.parquet
-j ./m100-data/22-03_jobs.parquet \
-p ./m100-data/22-03_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_1.py \
-j ../m100-data/22-04_jobs.parquet \
-p m100-data/22-04_power_total.parquet
-j ./m100-data/22-04_jobs.parquet \
-p ./m100-data/22-04_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_1.py \
-j ../m100-data/22-05_jobs.parquet \
-p m100-data/22-05_power_total.parquet
-j ./m100-data/22-05_jobs.parquet \
-p ./m100-data/22-05_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_1.py \
-j ../m100-data/22-06_jobs.parquet \
-p m100-data/22-06_power_total.parquet
-j ./m100-data/22-06_jobs.parquet \
-p ./m100-data/22-06_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_1.py \
-j ../m100-data/22-07_jobs.parquet \
-p m100-data/22-07_power_total.parquet
-j ./m100-data/22-07_jobs.parquet \
-p ./m100-data/22-07_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_1.py \
-j ../m100-data/22-08_jobs.parquet \
-p m100-data/22-08_power_total.parquet
-j ./m100-data/22-08_jobs.parquet \
-p ./m100-data/22-08_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_1.py \
-j ../m100-data/22-09_jobs.parquet \
-p m100-data/22-09_power_total.parquet
-j ./m100-data/22-09_jobs.parquet \
-p ./m100-data/22-09_power_total.parquet
```
]
=== Step 2
#tododanilo[in the script source-code: change output filename of step 2
from a_0_filter123_singlenode.csv and from a_0_filter123_multinode.csv
to 22-0X_filter123_singlenode.csv and 22-0X_filter123_multinode.csv]
#fullbox(footer:[Memory: 128 Go. Time (sequential): 66:00:00])[
#fullbox(footer:[#emph-overhead[Memory: 128 Go. Time (sequential): 66:00:00]])[
```python
./scripts-py/expe_energumen/m100_pred_preprocess_2.py \
-js ./m100-data/22-01_filter12_singlenode.csv \
-jm ./m100-data/22-01_filter12_multinode.csv
-p m100-data/22-01_power_total.parquet
-p ./m100-data/22-01_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_2.py \
-js ./m100-data/22-02_filter12_singlenode.csv \
-jm ./m100-data/22-02_filter12_multinode.csv
-p m100-data/22-02_power_total.parquet
-p ../m100-data/22-02_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_2.py \
-js ./m100-data/22-03_filter12_singlenode.csv \
-jm ./m100-data/22-03_filter12_multinode.csv
-p m100-data/22-03_power_total.parquet
-p ./m100-data/22-03_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_2.py \
-js ./m100-data/22-04_filter12_singlenode.csv \
-jm ./m100-data/22-04_filter12_multinode.csv
-p m100-data/22-04_power_total.parquet
-p ./m100-data/22-04_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_2.py \
-js ./m100-data/22-05_filter12_singlenode.csv \
-jm ./m100-data/22-05_filter12_multinode.csv
-p m100-data/22-05_power_total.parquet
-p ./m100-data/22-05_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_2.py \
-js ./m100-data/22-06_filter12_singlenode.csv \
-jm ./m100-data/22-06_filter12_multinode.csv
-p m100-data/22-06_power_total.parquet
-p ./m100-data/22-06_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_2.py \
-js ./m100-data/22-07_filter12_singlenode.csv \
-jm ./m100-data/22-07_filter12_multinode.csv
-p m100-data/22-07_power_total.parquet
-p ./m100-data/22-07_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_2.py \
-js ./m100-data/22-08_filter12_singlenode.csv \
-jm ./m100-data/22-08_filter12_multinode.csv
-p m100-data/22-08_power_total.parquet
-p ./m100-data/22-08_power_total.parquet
```
```python
./scripts-py/expe_energumen/m100_pred_preprocess_2.py \
-js ./m100-data/22-09_filter12_singlenode.csv \
-jm ./m100-data/22-09_filter12_multinode.csv
-p m100-data/22-09_power_total.parquet
-p ./m100-data/22-09_power_total.parquet
```
]
......@@ -508,52 +501,50 @@ find . -name '*filter123*' | tar -zcvf exadata_job_energy_profiles.tar.gz --file
=== Compute power metrics and add job information
#tododanilo[Script source-code: change -d (dir path) and pass the path to the necessary files]
#fullbox(footer: [Disk: 32 Go.])[
``` python
./scripts-py/expe_energumen/m100_pred_jobs_extract_power_metrics.py \
-d ./data/year_month=22-01
-d ./m100-data/22-01
```
``` python
./scripts-py/expe_energumen/m100_pred_jobs_extract_power_metrics.py \
-d ./data/year_month=22-02
-d ./m100-data/22-02
```
``` python
./scripts-py/expe_energumen/m100_pred_jobs_extract_power_metrics.py \
-d ./data/year_month=22-03
-d ./m100-data/22-03
```
``` python
./scripts-py/expe_energumen/m100_pred_jobs_extract_power_metrics.py \
-d ./data/year_month=22-04
-d ./m100-data/22-04
```
``` python
./scripts-py/expe_energumen/m100_pred_jobs_extract_power_metrics.py \
-d ./data/year_month=22-05
-d ./m100-data/22-05
```
``` python
./scripts-py/expe_energumen/m100_pred_jobs_extract_power_metrics.py \
-d ./data/year_month=22-06
-d ./m100-data/22-06
```
``` python
./scripts-py/expe_energumen/m100_pred_jobs_extract_power_metrics.py \
-d ./data/year_month=22-07
-d ./m100-data/22-07
```
``` python
./scripts-py/expe_energumen/m100_pred_jobs_extract_power_metrics.py \
-d ./data/year_month=22-08
-d ./m100-data/22-08
```
``` python
./scripts-py/expe_energumen/m100_pred_jobs_extract_power_metrics.py \
-d ./data/year_month=22-09
-d ./m100-data/22-09
```
]
......@@ -561,8 +552,6 @@ find . -name '*filter123*' | tar -zcvf exadata_job_energy_profiles.tar.gz --file
This will output the `filter123_all_jobs_aggmetrics.csv.gz` needed for the prediction script
#tododanilo[check if /m100-data/ path is correct and also the path of the output]
#fullbox(footer: [Disk: 82 Mo.])[
``` python
......@@ -572,7 +561,7 @@ This will output the `filter123_all_jobs_aggmetrics.csv.gz` needed for the predi
== Predicting Job mean and maximum power consumption
#fullbox(footer:[Memory: 128 Go. Time (sequential): 72:00:00])[
#fullbox(footer:[#emph-overhead[Memory: 128 Go. Time (sequential): 72:00:00]])[
```
mkdir ./m100-data/total_power_mean_predictions_users_allmethods_mean
mkdir ./m100-data/total_power_mean_predictions_users_allmethods_max
......@@ -617,7 +606,10 @@ Output from the previous section
- `m100-data/power_pred_users_allmethods_mean.tar.gz`, the jobs mean power predictions.
- `m100-data/power_pred_users_allmethods_max.tar.gz`, the jobs maximum power predictions.
#tododanilo[Add notebook that make plots]
=== Reproducing the paper's plots
Please refer to this #link("./notebooks/m100_process_prediction_results.ipynb")[Notebook] for
the scripts to reproduce the paper's plots, notably Figures 2 and 3.
== Job scheduling with power prediction <sec-sched>
......
notebooks/m100_process_prediction_results.ipynb 0 → 100644
+ 387
− 0
View file @ c8c63a6f
%% Cell type:markdown id: tags:
## Processing the mean power prediction results (script `run_prediction_per_user_allmethods_mean.py`)
%% Cell type:code id: tags:
``` python
import pandas as pd
import seaborn as sns
import os
RESULTS_PATH = "../m100-data/total_power_mean_predictions_users_allmethods_mean/"
PRED_COLS = ["hist_pred_total_power_mean",
"LinearRegression_total_power_mean_watts",
"RandomForestRegressor_total_power_mean_watts",
"LinearSVR_total_power_mean_watts",
"SGDRegressor_total_power_mean_watts"]
result_filenames = os.listdir(RESULTS_PATH)
df_all_results = pd.concat([pd.read_csv(RESULTS_PATH+filename, low_memory=False) for filename in result_filenames])
df_all_results = df_all_results.dropna(subset=PRED_COLS)
df_all_results
```
%% Cell type:code id: tags:
``` python
from sklearn.metrics import mean_absolute_error, mean_squared_error, mean_absolute_percentage_error
lst_users = df_all_results["user_id"].drop_duplicates().to_list()
#print(lst_users)
df_results_user_group = df_all_results.groupby("user_id")
lst_stats_per_user = []
for user in lst_users:
results_user = df_results_user_group.get_group(user)
hist_mape = mean_absolute_percentage_error(results_user["total_power_mean_watts"], results_user["hist_pred_total_power_mean"])
LR_mape = mean_absolute_percentage_error(results_user["total_power_mean_watts"], results_user["LinearRegression_total_power_mean_watts"])
RF_mape = mean_absolute_percentage_error(results_user["total_power_mean_watts"], results_user["RandomForestRegressor_total_power_mean_watts"])
LSVR_mape = mean_absolute_percentage_error(results_user["total_power_mean_watts"], results_user["LinearSVR_total_power_mean_watts"])
SGD_mape = mean_absolute_percentage_error(results_user["total_power_mean_watts"], results_user["SGDRegressor_total_power_mean_watts"])
res = {"user_id": user,
"hist_mape": hist_mape,
"LinearRegression_mape": LR_mape,
"RandomForestRegressor_mape": RF_mape,
"LinearSVR_mape": LSVR_mape,
"SGDRegressor_mape": SGD_mape}
lst_stats_per_user.append(res)
#break
df_stats_per_user = pd.DataFrame(lst_stats_per_user)
df_stats_per_user
```
%% Cell type:code id: tags:
``` python
COLS = ["hist_mape","LinearRegression_mape","RandomForestRegressor_mape","LinearSVR_mape","SGDRegressor_mape"]
df_stats_per_user[COLS].describe()
```
%% Cell type:code id: tags:
``` python
COLS = ["hist_mape","LinearRegression_mape","RandomForestRegressor_mape","LinearSVR_mape","SGDRegressor_mape"]
df_stats_per_user_pivot = pd.melt(df_stats_per_user, id_vars="user_id")
df_stats_per_user_pivot
```
%% Cell type:markdown id: tags:
### Figure 3 A
%% Cell type:code id: tags:
``` python
import matplotlib.pyplot as plt
TINY_SIZE = 2
SMALL_SIZE = 5
MEDIUM_SIZE = 20
BIGGER_SIZE = 50
FIG_WIDTH = 40
FIG_HEIGHT = 10
#plt.rc('font', size=16) # controls default text sizes
plt.rc('font', size=20) # controls default text sizes
plt.rc('axes', titlesize=MEDIUM_SIZE) # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
plt.rc('xtick', labelsize=MEDIUM_SIZE) # fontsize of the tick labels
plt.rc('ytick', labelsize=MEDIUM_SIZE) # fontsize of the tick labels
plt.rc('legend', fontsize=MEDIUM_SIZE) # legend fontsize
plt.rc('figure', titlesize=MEDIUM_SIZE) # fontsize of the figure title
#g = sns.boxplot(x="variable", y="value", data=df_stats_per_user_pivot, showfliers=False)
#plt.xticks(ticks=[0,1,2,3,4],labels=["History", "LinearRegression", "RandomForest", "LinearSVR", "SGDRegressor"],rotation=30)
g = sns.boxplot(y="variable", x="value", data=df_stats_per_user_pivot, showfliers=False)
plt.yticks(ticks=[0,1,2,3,4],labels=["History", "LinearRegression", "RandomForest", "LinearSVR", "SGDRegressor"],rotation=0)
g.set_ylabel("Prediction Method")
g.set_xlabel("Mean Absolute Percentage Error (MAPE) ")
```
%% Cell type:markdown id: tags:
## Processing the max power prediction results (script `run_prediction_per_user_allmethods_max.py`)
%% Cell type:code id: tags:
``` python
import pandas as pd
import seaborn as sns
import os
RESULTS_PATH = "./m100-data/total_power_mean_predictions_users_allmethods_max/"
PRED_COLS = ["hist_pred_total_power_max",
"LinearRegression_total_power_max_watts",
"RandomForestRegressor_total_power_max_watts",
"LinearSVR_total_power_max_watts",
"SGDRegressor_total_power_max_watts"]
result_filenames = os.listdir(RESULTS_PATH)
df_all_results = pd.concat([pd.read_csv(RESULTS_PATH+filename, low_memory=False) for filename in result_filenames])
df_all_results = df_all_results.dropna(subset=PRED_COLS)
df_all_results
```
%% Cell type:code id: tags:
``` python
from sklearn.metrics import mean_absolute_error, mean_squared_error, mean_absolute_percentage_error
lst_users = df_all_results["user_id"].drop_duplicates().to_list()
#print(lst_users)
df_results_user_group = df_all_results.groupby("user_id")
lst_stats_per_user = []
for user in lst_users:
results_user = df_results_user_group.get_group(user)
hist_mape = mean_absolute_percentage_error(results_user["total_power_max_watts"], results_user["hist_pred_total_power_max"])
LR_mape = mean_absolute_percentage_error(results_user["total_power_max_watts"], results_user["LinearRegression_total_power_max_watts"])
RF_mape = mean_absolute_percentage_error(results_user["total_power_max_watts"], results_user["RandomForestRegressor_total_power_max_watts"])
LSVR_mape = mean_absolute_percentage_error(results_user["total_power_max_watts"], results_user["LinearSVR_total_power_max_watts"])
SGD_mape = mean_absolute_percentage_error(results_user["total_power_max_watts"], results_user["SGDRegressor_total_power_max_watts"])
res = {"user_id": user,
"hist_mape": hist_mape,
"LinearRegression_mape": LR_mape,
"RandomForestRegressor_mape": RF_mape,
"LinearSVR_mape": LSVR_mape,
"SGDRegressor_mape": SGD_mape}
lst_stats_per_user.append(res)
#break
df_stats_per_user = pd.DataFrame(lst_stats_per_user)
df_stats_per_user
```
%% Cell type:code id: tags:
``` python
COLS = ["hist_mape","LinearRegression_mape","RandomForestRegressor_mape","LinearSVR_mape","SGDRegressor_mape"]
df_stats_per_user[COLS].describe()
```
%% Cell type:code id: tags:
``` python
COLS = ["hist_mape","LinearRegression_mape","RandomForestRegressor_mape","LinearSVR_mape","SGDRegressor_mape"]
df_stats_per_user_pivot = pd.melt(df_stats_per_user, id_vars="user_id")
df_stats_per_user_pivot
```
%% Cell type:markdown id: tags:
### Figure 3 B
%% Cell type:code id: tags:
``` python
import matplotlib.pyplot as plt
TINY_SIZE = 2
SMALL_SIZE = 5
MEDIUM_SIZE = 20
BIGGER_SIZE = 50
FIG_WIDTH = 40
FIG_HEIGHT = 10
plt.rc('font', size=20) # controls default text sizes
plt.rc('axes', titlesize=MEDIUM_SIZE) # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
plt.rc('xtick', labelsize=MEDIUM_SIZE) # fontsize of the tick labels
plt.rc('ytick', labelsize=MEDIUM_SIZE) # fontsize of the tick labels
plt.rc('legend', fontsize=MEDIUM_SIZE) # legend fontsize
plt.rc('figure', titlesize=MEDIUM_SIZE) # fontsize of the figure title
#g = sns.boxplot(x="variable", y="value", data=df_stats_per_user_pivot, showfliers=False)
#plt.xticks(ticks=[0,1,2,3,4],labels=["History", "LinearRegression", "RandomForest", "LinearSVR", "SGDRegressor"],rotation=30)
#g.set_xlabel("Prediction Method")
#g.set_ylabel("Mean Absolute Percentage Error (MAPE) ")
g = sns.boxplot(y="variable", x="value", data=df_stats_per_user_pivot, showfliers=False)
plt.yticks(ticks=[0,1,2,3,4],labels=["History", "LinearRegression", "RandomForest", "LinearSVR", "SGDRegressor"],rotation=0)
g.set_ylabel("Prediction Method")
g.set_xlabel("Mean Absolute Percentage Error (MAPE)")
```
%% Cell type:markdown id: tags:
## Getting the actual mean and max power distributions
%% Cell type:markdown id: tags:
### Mean (Figure 2 A)
%% Cell type:code id: tags:
``` python
import matplotlib.pyplot as plt
import seaborn as sns
TINY_SIZE = 2
SMALL_SIZE = 5
MEDIUM_SIZE = 20
BIGGER_SIZE = 50
FIG_WIDTH = 40
FIG_HEIGHT = 10
plt.clf()
plt.rc('figure', figsize=(8, 6))
plt.rc('font', size=MEDIUM_SIZE) # controls default text sizes
plt.rc('axes', titlesize=MEDIUM_SIZE) # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
plt.rc('xtick', labelsize=MEDIUM_SIZE) # fontsize of the tick labels
plt.rc('ytick', labelsize=MEDIUM_SIZE) # fontsize of the tick labels
plt.rc('legend', fontsize=MEDIUM_SIZE) # legend fontsize
plt.rc('figure', titlesize=MEDIUM_SIZE) # fontsize of the figure title
g = sns.histplot(x="total_power_mean_watts", data=df_all_results, bins=25, fill=False)
#g.ax.set_yscale('log')
g.set_xlabel("Total Power (watts)")
g.set_ylabel("Number of Jobs")
plt.xticks(ticks=[0,250,500,750,1000,1250,1500], rotation=30)
```
%% Cell type:markdown id: tags:
### Max (Figure 2 B)
%% Cell type:code id: tags:
``` python
import matplotlib.pyplot as plt
import seaborn as sns
TINY_SIZE = 2
SMALL_SIZE = 5
MEDIUM_SIZE = 20
BIGGER_SIZE = 50
FIG_WIDTH = 40
FIG_HEIGHT = 10
plt.clf()
plt.rc('figure', figsize=(8, 6))
plt.rc('font', size=MEDIUM_SIZE) # controls default text sizes
plt.rc('axes', titlesize=MEDIUM_SIZE) # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
plt.rc('xtick', labelsize=MEDIUM_SIZE) # fontsize of the tick labels
plt.rc('ytick', labelsize=MEDIUM_SIZE) # fontsize of the tick labels
plt.rc('legend', fontsize=MEDIUM_SIZE) # legend fontsize
plt.rc('figure', titlesize=MEDIUM_SIZE) # fontsize of the figure title
#g = sns.displot(x="total_power_max_watts", data=df_all_results)
g = sns.histplot(x="total_power_max_watts", data=df_all_results, bins=25, fill=False)
#g.ax.set_yscale('log')
g.set_xlabel("Total Power (watts)")
g.set_ylabel("Number of Jobs")
plt.xticks(ticks=[0,250,500,750,1000,1250,1500,1750,2000], rotation=30)
```
scripts-py/expe_energumen/m100_pred_jobs_extract_power_metrics.py
+ 5
− 5
View file @ c8c63a6f
......@@ -8,10 +8,10 @@ from scipy.stats import iqr
Read input data
"""
def read_data(rootdir):
df_jobs_single = pd.read_csv(rootdir+"/plugin=job_table/metric=job_info_marconi100/a_0_filter123_singlenode.csv")
df_jobs_multi = pd.read_csv(rootdir+"/plugin=job_table/metric=job_info_marconi100/a_0_filter123_multinode.csv")
df_power_single = pd.read_csv(rootdir+"/plugin=ipmi_pub/metric=total_power/a_0_filter123_singlenode.csv")
df_power_multi = pd.read_csv(rootdir+"/plugin=ipmi_pub/metric=total_power/a_0_filter123_multinode.csv")
df_jobs_single = pd.read_csv(rootdir+"_filter123_singlenode.csv")
df_jobs_multi = pd.read_csv(rootdir+"_filter123_multinode.csv")
df_power_single = pd.read_csv(rootdir+"_filter123_singlenode.csv")
df_power_multi = pd.read_csv(rootdir+"_filter123_multinode.csv")
df_jobs = pd.concat([df_jobs_single, df_jobs_multi]).reset_index(drop=True)
df_power = pd.concat([df_power_single, df_power_multi]).reset_index(drop=True)
df_power['node'] = pd.to_numeric(df_power['node'])
......@@ -37,7 +37,7 @@ def calculate_agg_metrics(df_jobs, df_power):
Save results
"""
def save_results(df_jobs_aggmetrics, rootdir):
df_jobs_aggmetrics.to_csv(rootdir+"/plugin=job_table/metric=job_info_marconi100/a_0_filter123_aggmetrics.csv")
df_jobs_aggmetrics.to_csv(rootdir+"_filter123_aggmetrics.csv")
"""
Run workflow
......
scripts-py/expe_energumen/m100_pred_merge_jobfiles.py
+ 1
− 1
View file @ c8c63a6f
......@@ -9,7 +9,7 @@ Read job files spread in the months folders
def read_jobifles(rootdir):
#DATASET_PATH = "/home/dancarastan/Documentos/exadata_job_energy_profiles/"
jobfiles_list = glob.glob(rootdir+"*"+"/plugin=job_table"+"/metric=job_info_marconi100"+"/a_0_filter123_aggmetrics.csv")
jobfiles_list = glob.glob(rootdir+"*"+"_filter123_aggmetrics.csv")
#print(len(jobfiles_list))
df_jobs = pd.concat([pd.read_csv(jobfile) for jobfile in jobfiles_list]).reset_index(drop=True)
......
scripts-py/expe_energumen/m100_pred_preprocess_1.py
+ 4
− 5
View file @ c8c63a6f
......@@ -105,11 +105,10 @@ def filter2_multi(df_jobs, df_power):
"""
Save intermediate results to csv
"""
def save_results(df_jobs_single, df_jobs_multi, jobfile, metricfile):
jobfile_out = jobfile.rstrip("a_0.parquet")
metric = metricfile.split("/")[-2]
df_jobs_single.to_csv(jobfile_out+metric+"_filter12_singlenode.csv", index=False)
df_jobs_multi.to_csv(jobfile_out+metric+"_filter12_multinode.csv", index=False)
def save_results(df_jobs_single, df_jobs_multi, jobfile, metricfile):
jobfile_out = jobfile.rstrip("jobs.parquet")
df_jobs_single.to_csv(jobfile_out+"_filter12_singlenode.csv", index=False)
df_jobs_multi.to_csv(jobfile_out+"_filter12_multinode.csv", index=False)
"""
Run workflow
......
scripts-py/expe_energumen/m100_pred_preprocess_2.py
+ 4
− 5
View file @ c8c63a6f
......@@ -183,11 +183,10 @@ def filter3_1_multi(df_jobs_multi, df_total_power):
Save results to csv
"""
def save_results(df_exclusive_jobs_single, df_exclusive_jobs_multi, df_total_power_exclusive_single, df_total_power_exclusive_multi, jobfile_single, metricfile):
metric = metricfile.split("/")[-2]
jobfile_out = jobfile_single.rstrip(metric+"_filter12_singlenode.csv")
metricfile_out = metricfile.rstrip("a_0.parquet")
df_exclusive_jobs_single.to_csv(jobfile_out+metric+"_filter123_singlenode.csv", index=False)
df_exclusive_jobs_multi.to_csv(jobfile_out+metric+"_filter123_multinode.csv", index=False)
jobfile_out = jobfile_single.rstrip("_filter12_singlenode.csv")
metricfile_out = metricfile.rstrip("power_total.parquet")
df_exclusive_jobs_single.to_csv(jobfile_out+"_filter123_singlenode.csv", index=False)
df_exclusive_jobs_multi.to_csv(jobfile_out+"_filter123_multinode.csv", index=False)
df_total_power_exclusive_single.to_csv(metricfile_out+"a_0_filter123_singlenode.csv", index=False)
df_total_power_exclusive_multi.to_csv(metricfile_out+"a_0_filter123_multinode.csv", index=False)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment