Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
D
Demand Response User
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
sepia-pub
Open Science
Demand Response User
Commits
d2c60cfb
Commit
d2c60cfb
authored
3 years ago
by
Maël Madon
Browse files
Options
Downloads
Patches
Plain Diff
start writing the script for first expe, to finish
parent
f53c201a
No related branches found
No related tags found
No related merge requests found
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
1_full_expe.py
+13
-0
13 additions, 0 deletions
1_full_expe.py
1_one_instance.py
+76
-0
76 additions, 0 deletions
1_one_instance.py
scripts/run_batsim_exp.py
+55
-0
55 additions, 0 deletions
scripts/run_batsim_exp.py
with
144 additions
and
0 deletions
1_full_expe.py
0 → 100644
+
13
−
0
View file @
d2c60cfb
from
random
import
*
from
time
import
*
# Prepare the start date samples
begin_trace
=
1356994806
# according to original SWF header
jun1_unix_time
,
nov30_unix_time
=
mktime
(
strptime
(
'
Sun Jun 1 00:00:00 2014
'
)),
mktime
(
strptime
(
'
Sun Nov 30 23:59:59 2014
'
))
jun1
,
nov30
=
(
int
)
(
jun1_unix_time
-
begin_trace
),
(
int
)
(
nov30_unix_time
-
begin_trace
)
start_date
=
randint
(
jun1
,
nov30
-
72
*
3600
)
# For every start date
# Call the one_instance file with this date
\ No newline at end of file
This diff is collapsed.
Click to expand it.
1_one_instance.py
0 → 100644
+
76
−
0
View file @
d2c60cfb
from
run_batsim_exp
import
*
import
swf_to_batsim_split_by_user
as
split_user
import
time
import
sys
import
os
import
subprocess
sys
.
path
.
insert
(
0
,
'
/scripts
'
)
#import swf_moulinette
###### Prepare input data ######
# Cut the original trace to extract 72h starting from this start date
def
prepare_input_data
(
expe_num
,
start_date
):
end_date
=
start_date
+
72
*
3600
to_keep
=
f
"
submit_time >=
{
start_date
}
and submit_time <=
{
end_date
}
"
if
not
os
.
path
.
exists
(
f
'
workload/expe
{
expe_num
}
.swf
'
):
os
.
makedirs
(
f
'
workload/expe
{
expe_num
}
.swf
'
)
split_user
.
generate_workload
(
input_swf
=
'
workload/MC_selection_article.swf
'
,
output_folder
=
f
'
workload/expe
{
expe_num
}
'
,
keep_only
=
to_keep
,
job_grain
=
10
,
job_walltime_factor
=
8
)
###### Run the expe ######
# Run batmen with Rigid behavior
EXPE_FILE
=
f
"
out/expe
{
expe_num
}
"
wl_folder
=
f
'
workload/expe
{
expe_num
}
'
pf
=
"
platform/average_metacentrum.xml
"
wl
=
"
workload/empty_workload.json
"
uf
=
"
sched_input/user_description_file.json
"
def
user_description
(
user
):
return
{
"
name
"
:
user
,
"
category
"
:
"
dm_user_degrad
"
,
"
param
"
:
{
"
input_json
"
:
f
"
{
wl_folder
}
/
{
user
}
.json
"
}
}
user_names
=
[
user_file
.
split
(
'
.
'
)[
0
]
for
user_file
in
os
.
listdir
(
wl_folder
)]
data
=
{}
data
[
"
dm_window
"
]
=
dm_window
data
[
"
log_user_stats
"
]
=
True
data
[
"
log_folder
"
]
=
EXPE_FILE
data
[
"
users
"
]
=
[
user_description
(
user
)
for
user
in
user_names
]
uf
=
"
sched_input/user_description_file.json
"
with
open
(
uf
,
'
w
'
)
as
user_description_file
:
json
.
dump
(
data
,
user_description_file
)
batcmd
=
gen_batsim_cmd
(
pf
,
wl
,
EXPE_FILE
,
"
--energy --enable-compute-sharing --enable-dynamic-jobs --acknowledge-dynamic-jobs --enable-profile-reuse
"
)
schedcmd
=
f
"
batsched -v bin_packing_energy --queue_order=desc_size --variant_options_filepath=
{
uf
}
"
instance
=
RobinInstance
(
output_dir
=
EXPE_FILE
,
batcmd
=
batcmd
,
schedcmd
=
schedcmd
,
simulation_timeout
=
30
,
ready_timeout
=
5
,
success_timeout
=
10
,
failure_timeout
=
0
)
instance
.
to_file
(
robin_filename
)
ret
=
run_robin
(
robin_filename
)
# For each user behavior in {Renonce, Delay, Degrad, Reconfig}
# For each dm_window duration in {.5, 1, 2, 4}
# Run batmen
###### Output data treatment ######
# Produce the utilisation viz
This diff is collapsed.
Click to expand it.
scripts/run_batsim_exp.py
0 → 100644
+
55
−
0
View file @
d2c60cfb
#!/usr/bin/env python3
import
os
import
os.path
import
subprocess
import
filecmp
from
collections
import
namedtuple
class
RobinInstance
(
object
):
def
__init__
(
self
,
output_dir
,
batcmd
,
schedcmd
,
simulation_timeout
,
ready_timeout
,
success_timeout
,
failure_timeout
):
self
.
output_dir
=
output_dir
self
.
batcmd
=
batcmd
self
.
schedcmd
=
schedcmd
self
.
simulation_timeout
=
simulation_timeout
self
.
ready_timeout
=
ready_timeout
self
.
success_timeout
=
success_timeout
self
.
failure_timeout
=
failure_timeout
def
to_yaml
(
self
):
# Manual dump to avoid dependencies
return
f
'''
output-dir:
'
{
self
.
output_dir
}
'
batcmd:
"
{
self
.
batcmd
}
"
schedcmd:
"
{
self
.
schedcmd
}
"
simulation-timeout:
{
self
.
simulation_timeout
}
ready-timeout:
{
self
.
ready_timeout
}
success-timeout:
{
self
.
success_timeout
}
failure-timeout:
{
self
.
failure_timeout
}
'''
def
to_file
(
self
,
filename
):
create_dir_rec_if_needed
(
os
.
path
.
dirname
(
filename
))
write_file
(
filename
,
self
.
to_yaml
())
def
gen_batsim_cmd
(
platform
,
workload
,
output_dir
,
more_flags
):
return
f
"
batsim -p
'
{
platform
}
'
-w
'
{
workload
}
'
-e
'
{
output_dir
}
/
'
{
more_flags
}
"
def
write_file
(
filename
,
content
):
file
=
open
(
filename
,
"
w
"
)
file
.
write
(
content
)
file
.
close
()
def
create_dir_rec_if_needed
(
dirname
):
if
not
os
.
path
.
exists
(
dirname
):
os
.
makedirs
(
dirname
)
def
run_robin
(
filename
):
return
subprocess
.
run
([
'
robin
'
,
filename
])
# def init_instance(test_name):
# output_dir = os.path.abspath(f'test-out/{test_name}')
# robin_filename = os.path.abspath(f'test-instances/{test_name}.yaml')
# schedconf_filename = f'{output_dir}/schedconf.json'
# create_dir_rec_if_needed(output_dir)
# return (output_dir, robin_filename, schedconf_filename)
\ No newline at end of file
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment