Skip to content
Snippets Groups Projects
Commit 6cb6c182 authored by AxelCarayon's avatar AxelCarayon
Browse files

refactoring

parent 2fa33f20
Branches
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
dockerModule.py 0 → 100644
+ 23
− 0
View file @ 6cb6c182
import subprocess
def getWorkir(dockerFile) -> str :
workdir = "/"
with open(dockerFile,"r") as file:
for line in file.read().splitlines():
if line.startswith("WORKDIR"):
workdir = line.split(" ")[1]
return workdir
def buildDockerImage(imageName) -> None:
print("Building the docker image ...")
try :
subprocess.run(f"docker build -t {imageName.lower()}experiment ./",shell=True).check_returncode()
except :
subprocess.run(f"sudo docker build -t {imageName.lower()}experiment ./",shell=True).check_returncode()
def runDockerImage(imageName,workDirectory) -> None:
print("binding docker image to the current directory and running it...")
try:
subprocess.run(f"docker run -it --mount type=bind,source=\"$PWD\",target={workDirectory} {imageName.lower()}experiment",shell=True).check_returncode()
except :
subprocess.run(f"sudo docker run -it --mount type=bind,source=\"$PWD\",target={workDirectory} {imageName.lower()}experiment",shell=True).check_returncode()
\ No newline at end of file
loadExperiment.py
+ 21
− 73
View file @ 6cb6c182
......@@ -4,57 +4,30 @@ import subprocess
import yaml
import hashlib
import warnings
from utils import typeEnterToContinue
from dockerModule import getWorkir, buildDockerImage, runDockerImage
INPUT_FOLDER = "inputs"
OUTPUT_FOLDER = "outputs"
repo = None
folder = None
commandsFile = None
instructionFile = None
inputFiles = []
outputFiles = []
dockerfileIsPresent = False
beforeHash = None
params = None
def init(repository,branch) -> None :
global repo, folder
folder = repository.split('/')[-1].split('.')[0]
if os.path.exists(folder) :
print(f"Folder ./{folder} already exists, do you want to delete it ? (y/n)")
answer = input()
if answer == 'y' :
os.system(f"rm -rf ./{folder}")
else :
print("Aborting")
exit(0)
git.Git("./").clone(repository)
repo = git.Repo(folder)
if not os.path.exists(folder) :
git.Git("./").clone(repository)
try :
repo.git.checkout(branch)
git.Repo(folder).git.checkout(branch)
except git.exc.GitCommandError :
raise Exception(f"Branch {branch} not found in the repository")
os.chdir(folder)
def getParameters() -> None :
global commandsFile, inputFiles, outputFiles, beforeHash, instructionFile, dockerfileIsPresent
global params
if not (os.path.exists('experimentResume.yaml')):
raise Exception("No exeperimentResume.yaml file found, the branch is not an exeperiment")
with open('experimentResume.yaml', 'r') as stream:
parameters = yaml.safe_load(stream)
commandsFile = parameters.get('commands')
outputFiles = parameters.get('outputs')
inputFiles = parameters.get('inputs')
beforeHash = parameters.get('checksums')
instructionFile = parameters.get('instructions')
dockerfileIsPresent = parameters.get('dockerfile')
params = yaml.safe_load(stream)
def runExperiment() -> None :
file = open(commandsFile, "r")
file = open(params.get('commands'), "r")
for line in file.read().splitlines():
print(f"running {line} ...")
process = subprocess.run(line,shell=True)
......@@ -62,14 +35,11 @@ def runExperiment() -> None :
print("done")
def checkForInstructions() -> None :
if (instructionFile != None) :
print("You can check the instructions for the experiment in the file : " + instructionFile)
if (params.get('instruction') != None) :
print("You can check the instructions for the experiment in the file : " + params.get('instruction'))
else :
warnings.warn("No instructions for the experiment found in the repository")
print("Run the exepriment and then press enter when it's done")
done = "nope"
while (done != "") :
done = input()
typeEnterToContinue("Run the exepriment and then press enter when it's done")
def genChecksum(file) -> str :
hash_md5 = hashlib.md5()
......@@ -80,57 +50,35 @@ def genChecksum(file) -> str :
def genChecksums() -> list[dict]:
checksums = []
for file in os.listdir(OUTPUT_FOLDER) :
outputFolder = params.get('outputFolder')
for file in os.listdir(outputFolder) :
if not file.endswith(".gitkeep"):
checksums.append({f"{OUTPUT_FOLDER}/{file}" : genChecksum(f'{OUTPUT_FOLDER}/{file}')})
checksums.append({f"{outputFolder}/{file}" : genChecksum(f'{outputFolder}/{file}')})
return checksums
def compareChecksums() -> bool:
changes = False
for (dict1, dict2) in zip(beforeHash, genChecksums()):
for (dict1, dict2) in zip(params.get('checksums'), genChecksums()):
for (key, value) in dict1.items():
if dict2.get(key) != value :
warnings.warn(f"{key} has changed")
changes = True
return changes
def buildDockerImage() -> None:
print("Building the docker image ...")
try :
subprocess.run(f"docker build -t experimentreproduction ./",shell=True).check_returncode()
except :
subprocess.run(f"sudo docker build -t experimentreproduction ./",shell=True).check_returncode()
def getWorkir() -> str :
workdir = "/"
with open("Dockerfile","r") as file:
for line in file.read().splitlines():
if line.startswith("WORKDIR"):
workdir = line.split(" ")[1]
return workdir
def runDockerImage() -> None:
print("binding docker image to the current directory and running it...")
try:
subprocess.run(f"docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} experimentreproduction",shell=True).check_returncode()
except :
subprocess.run(f"sudo docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} experimentreproduction",shell=True).check_returncode()
def run(repository, branch) -> None :
print("Initializing the experiment repository ...")
init(repository, branch)
print("Getting the experiment parameters ...")
getParameters()
print("Running the experiment ...")
if (dockerfileIsPresent) :
if (params.get('dockerfile')) :
print("Dockerimage was found ! Using it to run the experiment...")
buildDockerImage()
runDockerImage()
name = params.get("name")
buildDockerImage(name)
runDockerImage(name,getWorkir("Dockerfile"))
else:
if (commandsFile != None) :
if (params.get('commands') != None) :
runExperiment()
else :
checkForInstructions()
......
registerExperiment.py
+ 66
− 105
View file @ 6cb6c182
from multiprocessing.connection import answer_challenge
import os
from unicodedata import name
import git
import subprocess
import yaml
import hashlib
import warnings
from dockerModule import getWorkir, buildDockerImage, runDockerImage
from utils import yesOrNo
EXPERIMENT_RESUME = "experimentResume.yaml"
DOCKERFILE = "Dockerfile"
path = "./"
repository = None
inputFolder = None
inputFiles = []
paramsFolder = None
paramsFiles = []
commandsFile = None
instructionFile = None
experimentName = None
outputFolder = None
outputFiles = []
currentTag = None
tags = None
gitContent = {
'repository' : None,
'branchName' : None,
'currentTag' : None,
'tags' : None
}
experiment = {
'name' : None,
'inputFolder' : None,
'inputFiles' : [],
'paramsFolder' : None,
'paramsFiles' : [],
'commandsFile' : None,
'instructionFile' : None,
'outputFolder' : None,
'outputFiles' : [],
'dockerfile' : None,
'checksums' : []
}
def isGitRepo(path) -> bool:
try:
......@@ -39,58 +40,32 @@ def isGitRepo(path) -> bool:
except git.exc.InvalidGitRepositoryError:
return False
def dockerfileIsPresent() -> bool:
if fileExists(DOCKERFILE):
answer = input("A dockerfile was found ! It will be used to reproduce the experiment. Is that ok for you ? (y/n)")
if answer == "n":
if yesOrNo("A dockerfile was found ! It will be used to reproduce the experiment. Is that ok for you ?"):
return True
else :
raise Exception("""Remove the dockerfile and try again""")
return True
else :
return False
def buildDockerImage() -> None:
print("Building the docker image ...")
try :
subprocess.run(f"docker build -t {experimentName.lower()}experiment ./",shell=True).check_returncode()
except :
subprocess.run(f"sudo docker build -t {experimentName.lower()}experiment ./",shell=True).check_returncode()
def getWorkir() -> str :
workdir = "/"
with open(DOCKERFILE,"r") as file:
for line in file.read().splitlines():
if line.startswith("WORKDIR"):
workdir = line.split(" ")[1]
return workdir
def runDockerImage() -> None:
print("binding docker image to the current directory and running it...")
try:
subprocess.run(f"docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} {experimentName.lower()}experiment",shell=True).check_returncode()
except :
subprocess.run(f"sudo docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} {experimentName.lower()}experiment",shell=True).check_returncode()
#TODO : vérifier si la reproduction avec un Dockerfile marche dans l'autre sens
def init(pathInput) -> None :
global repository,path,experimentName,tags, currentTag
if isGitRepo(pathInput):
path += pathInput
path = pathInput
if not (pathInput[len(pathInput)-1] == "/"):
path+="/"
repository = git.Repo(path)
experimentName = repository.active_branch.name
gitContent['repository'] = git.Repo(path)
experiment["name"] = gitContent['repository'].active_branch.name
os.chdir(path)
else :
raise Exception(f"{pathInput} is not a git repository")
tags = repository.tags
currentTag = repository.git.describe('--tags')
gitContent['tags'] = gitContent['repository'].tags
gitContent['currentTag'] = gitContent['repository'].git.describe('--tags')
if not(currentVersionIsTagged()):
raise Exception("Current version is not tagged, you can only reproduce an experiment from a tagged version.")
def currentVersionIsTagged() -> bool:
return currentTag in tags
return gitContent['currentTag'] in gitContent['tags']
def fileExists(fileName) -> bool:
return os.path.exists(fileName)
......@@ -99,20 +74,18 @@ def folderExists(folderName) -> bool:
return os.path.isdir(folderName)
def askForInputFolder() -> None:
global inputFolder
answer = input("If you use input data, where are they stored ? Give the path from the root of the repository : ")
if answer == "":
warnings.warn("No input folder given, no input files will be registered")
else:
if not folderExists(answer):
raise Exception(f"{path}/{answer} folder does not exist")
raise Exception(f"{answer} folder does not exist")
else:
if not answer.endswith("/"):
answer+="/"
inputFolder = answer
experiment['inputFolder'] = answer
def askForOutputFolder() -> None:
global outputFolder
answer = input("Where are the outputs generated ? Give the path from the root of the repository : ")
if answer == "":
warnings.warn("No output folder given, no output files will be registered")
......@@ -122,29 +95,27 @@ def askForOutputFolder() -> None:
else:
if not answer.endswith("/"):
answer+="/"
outputFolder = answer
experiment['outputFolder'] = answer
def askForParamsFolder() -> None:
global paramsFolder
answer = input("In which folder do you store your parameters ? Give the path from the root of the repository : ")
if answer == "":
warnings.warn("No parameters folder given, no parameters will be registered")
else:
if not folderExists(answer):
raise Exception(f"{path}/{answer} folder does not exist")
raise Exception(f"{answer} folder does not exist")
else:
if not answer.endswith("/"):
answer+="/"
paramsFolder = answer
experiment['paramsFolder'] = answer
def askForCommandsFile() -> None:
global commandsFile
commandsFile = input("Enter the name of the commands file: ")
if not fileExists(commandsFile):
raise Exception(f"{commandsFile} file does not exist")
experiment['commandsFile'] = commandsFile
def askForInstructionFile() -> None :
global instructionFile
print("If you have an instruction file, enter its name, otherwise press enter")
instructionFile = input()
if instructionFile == "":
......@@ -152,16 +123,17 @@ def askForInstructionFile() -> None :
else:
if not fileExists(instructionFile):
raise Exception(f"{instructionFile} file does not exist")
experiment['insuctionFile'] = instructionFile
def registeringExperimentInputs(inputs) -> None:
with open(commandsFile, "w") as file:
with open(experiment['commandsFile'], "w") as file:
for input in inputs:
file.write(input+"\n")
def runExperiment() -> None:
print("Trying to run experiment")
file = open(commandsFile, "r")
file = open(experiment['commandsFile'], "r")
for line in file.read().splitlines():
print(f"running {line} ...")
process = subprocess.run(line,shell=True)
......@@ -169,26 +141,29 @@ def runExperiment() -> None:
print("done")
def scanInputFiles() -> None:
for file in os.listdir(inputFolder):
for file in os.listdir(experiment['inputFolder']):
if not file.endswith(".gitkeep"):
inputFiles.append(f"{inputFolder}{file}")
experiment['inputFiles'].append(f"{experiment['inputFolder']}{file}")
def scanOutputsGenerated() -> None:
for file in os.listdir(outputFolder):
for file in os.listdir(experiment['outputFolder']):
if not file.endswith(".gitkeep"):
outputFiles.append(f"{outputFolder}{file}")
experiment['outputFiles'].append(f"{experiment['outputFolder']}{file}")
def scanParameters() -> None:
for file in os.listdir(paramsFolder):
for file in os.listdir(experiment['paramsFolder']):
if not file.endswith(".gitkeep"):
paramsFiles.append(f"{paramsFolder}{file}")
experiment['paramsFiles'].append(f"{experiment['paramsFolder']}{file}")
def isNotAnOutputfile(file) -> bool: return file not in outputFiles
def isNotAnInputfile(file) -> bool: return file not in inputFiles
def isNotAParamFile(file) -> bool: return file not in paramsFiles
def isNotAnOutputfile(file) -> bool: return file not in experiment['outputFiles']
def isNotAnInputfile(file) -> bool: return file not in experiment['inputFiles']
def isNotAParamFile(file) -> bool: return file not in experiment['paramsFiles']
def checkGeneratedFiles() -> None :
editedFiles = [ item.a_path for item in repository.index.diff(None) ]+ [ item.a_path for item in repository.index.diff(repository.head.name) ] + repository.untracked_files
repository = gitContent['repository']
editedFiles = [ item.a_path for item in repository.index.diff(None) ] \
+[ item.a_path for item in repository.index.diff(repository.head.name) ] \
+ repository.untracked_files
outOfPlaceFiles = []
logFile = open("outOfPlaceFiles.log","w")
for file in editedFiles:
......@@ -209,31 +184,16 @@ def checkGeneratedFiles() -> None :
def writeInYaml() -> None:
if fileExists(EXPERIMENT_RESUME):
with open(EXPERIMENT_RESUME, "r") as yamlFile:
cur_yaml = yaml.safe_load(yamlFile)
cur_yaml.update({"name":experimentName})
cur_yaml.update({"commands":commandsFile})
cur_yaml.update({"inputs":inputFiles})
cur_yaml.update({"outputs":outputFiles})
cur_yaml.update({"params":paramsFiles})
cur_yaml.update({"instruction":instructionFile})
cur_yaml.update({"dockerfile":fileExists(DOCKERFILE)})
checksums = {"checksums":genChecksums()}
cur_yaml.update(checksums)
with open(EXPERIMENT_RESUME, 'w') as yamlFile:
yaml.safe_dump(cur_yaml, yamlFile)
else:
with open(EXPERIMENT_RESUME, "w") as yamlFile:
yaml.safe_dump({"name":experimentName, "commands":commandsFile, "inputs":inputFiles, "outputs":outputFiles, "params":paramsFiles, "instruction":instructionFile, "dockerfile":fileExists(DOCKERFILE), "checksums":genChecksums()}, yamlFile)
with open(EXPERIMENT_RESUME, "w") as file:
yaml.safe_dump(experiment, file)
def pushBranch(version=1) -> None:
print("Pushing branch...")
experimentName = experiment['name'] ; repository = gitContent['repository']
while f"{experimentName}Experiment{version}" in repository.remote().refs:
print(f"{experimentName}Experiment{version} already exists")
version += 1
newTag = f"{currentTag}-e{version}"
newTag = f"{gitContent['currentTag']}-e{version}"
print(f"creating {experimentName}Experiment{version} branch and pushing changes to it ...")
repository.git.checkout(b=f"{experimentName}Experiment{version}")
repository.git.add(all=True)
......@@ -253,7 +213,7 @@ def genChecksum(file) -> str :
def genChecksums() -> list[dict]:
checksums = []
for file in outputFiles:
for file in experiment['outputFiles']:
checksums.append({file : genChecksum(file)})
return checksums
......@@ -265,8 +225,9 @@ def askFolders() -> None :
def reproduceExperiment() -> None:
if dockerfileIsPresent() :
buildDockerImage()
runDockerImage()
buildDockerImage(experiment['name'])
runDockerImage(experiment['name'], getWorkir(DOCKERFILE))
experiment['dockerfile'] = DOCKERFILE
else:
userInput = input("Do you have a pre-recorded commands file? (y/n)")
if userInput == "y":
......@@ -279,16 +240,16 @@ def reproduceExperiment() -> None:
done = input("Run your experiment and then type 'done' when you are done : ")
def scanAfterExecution() -> None:
if inputFolder != None :
if experiment['inputFolder'] != None :
scanInputFiles()
if outputFolder != None :
if experiment['outputFolder'] != None :
scanOutputsGenerated()
if paramsFolder != None :
if experiment['paramsFolder'] != None :
scanParameters()
def run(folder) -> None :
init(folder)
repository.active_branch.checkout()
gitContent['repository'].active_branch.checkout()
askFolders()
reproduceExperiment()
scanAfterExecution()
......@@ -297,4 +258,4 @@ def run(folder) -> None :
print(f"Please check the {EXPERIMENT_RESUME} file, if everything is correct, press enter to continue, otherwise type \"abort\"")
if input() == "abort":
raise Exception("Aborted")
pushBranch()
pushBranch()
\ No newline at end of file
utils.py 0 → 100644
+ 11
− 0
View file @ 6cb6c182
def yesOrNo(question) -> bool:
while True:
answer = input(question + " (y/n): ")
if answer == 'y' or answer == 'n':
return answer == 'y'
def typeEnterToContinue(text) -> None:
answer = "not empty"
while (answer != "") :
answer = input(text)
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment