Skip to content
Snippets Groups Projects
Unverified Commit 6c741966 authored by AxelCarayon's avatar AxelCarayon Committed by GitHub
Browse files

Merge pull request #21 from AxelCarayon/ajoutDocker

Ajout docker
parents 609087d9 df6bc06f
No related branches found
No related tags found
No related merge requests found
...@@ -17,6 +17,7 @@ instructionFile = None ...@@ -17,6 +17,7 @@ instructionFile = None
inputFiles = [] inputFiles = []
outputFiles = [] outputFiles = []
dockerfileIsPresent = False
beforeHash = None beforeHash = None
def init(repository,branch) -> None : def init(repository,branch) -> None :
...@@ -39,7 +40,7 @@ def init(repository,branch) -> None : ...@@ -39,7 +40,7 @@ def init(repository,branch) -> None :
os.chdir(folder) os.chdir(folder)
def getParameters() -> None : def getParameters() -> None :
global commandsFile, inputFiles, outputFiles, beforeHash, instructionFile global commandsFile, inputFiles, outputFiles, beforeHash, instructionFile, dockerfileIsPresent
if not (os.path.exists('experimentResume.yaml')): if not (os.path.exists('experimentResume.yaml')):
raise Exception("No exeperimentResume.yaml file found, the branch is not an exeperiment") raise Exception("No exeperimentResume.yaml file found, the branch is not an exeperiment")
with open('experimentResume.yaml', 'r') as stream: with open('experimentResume.yaml', 'r') as stream:
...@@ -49,6 +50,7 @@ def getParameters() -> None : ...@@ -49,6 +50,7 @@ def getParameters() -> None :
inputFiles = parameters.get('inputs') inputFiles = parameters.get('inputs')
beforeHash = parameters.get('checksums') beforeHash = parameters.get('checksums')
instructionFile = parameters.get('instructions') instructionFile = parameters.get('instructions')
dockerfileIsPresent = parameters.get('dockerfile')
def runExperiment() -> None : def runExperiment() -> None :
...@@ -94,16 +96,44 @@ def compareChecksums() -> bool: ...@@ -94,16 +96,44 @@ def compareChecksums() -> bool:
return changes return changes
def buildDockerImage() -> None:
print("Building the docker image ...")
try :
subprocess.run(f"docker build -t experimentreproduction ./",shell=True).check_returncode()
except :
subprocess.run(f"sudo docker build -t experimentreproduction ./",shell=True).check_returncode()
def getWorkir() -> str :
workdir = "/"
with open("Dockerfile","r") as file:
for line in file.read().splitlines():
if line.startswith("WORKDIR"):
workdir = line.split(" ")[1]
return workdir
def runDockerImage() -> None:
print("binding docker image to the current directory and running it...")
try:
subprocess.run(f"docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} experimentreproduction",shell=True).check_returncode()
except :
subprocess.run(f"sudo docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} experimentreproduction",shell=True).check_returncode()
def run(repository, branch) -> None : def run(repository, branch) -> None :
print("Initializing the experiment repository ...") print("Initializing the experiment repository ...")
init(repository, branch) init(repository, branch)
print("Getting the experiment parameters ...") print("Getting the experiment parameters ...")
getParameters() getParameters()
print("Running the experiment ...") print("Running the experiment ...")
if (commandsFile != None) : if (dockerfileIsPresent) :
runExperiment() print("Dockerimage was found ! Using it to run the experiment...")
else : buildDockerImage()
checkForInstructions() runDockerImage()
else:
if (commandsFile != None) :
runExperiment()
else :
checkForInstructions()
print("Comparing checksums of the outputs ...") print("Comparing checksums of the outputs ...")
if (compareChecksums()) : if (compareChecksums()) :
print("The exepriment was reproduced with succes but some output files are differents.") print("The exepriment was reproduced with succes but some output files are differents.")
......
from multiprocessing.connection import answer_challenge
import os import os
from unicodedata import name
import git import git
import subprocess import subprocess
import yaml import yaml
...@@ -6,6 +8,7 @@ import hashlib ...@@ -6,6 +8,7 @@ import hashlib
import warnings import warnings
EXPERIMENT_RESUME = "experimentResume.yaml" EXPERIMENT_RESUME = "experimentResume.yaml"
DOCKERFILE = "Dockerfile"
path = "./" path = "./"
...@@ -36,13 +39,39 @@ def isGitRepo(path) -> bool: ...@@ -36,13 +39,39 @@ def isGitRepo(path) -> bool:
except git.exc.InvalidGitRepositoryError: except git.exc.InvalidGitRepositoryError:
return False return False
def checkForChanges() -> None:
changesNotAdded = [ item.a_path for item in repository.index.diff(None) ]
changesAdded = [ item.a_path for item in repository.index.diff(repository.head.name) ]
untrackedFiles = repository.untracked_files
if ((len(changesNotAdded) + len(changesAdded) + len(untrackedFiles)) > 0):
raise Exception("There are changes in the repository since the last commit, you can only register an experiment from a clean repository.") def dockerfileIsPresent() -> bool:
if fileExists(DOCKERFILE):
answer = input("A dockerfile was found ! It will be used to reproduce the experiment. Is that ok for you ? (y/n)")
if answer == "n":
raise Exception("""Remove the dockerfile and try again""")
return True
else :
return False
def buildDockerImage() -> None:
print("Building the docker image ...")
try :
subprocess.run(f"docker build -t {experimentName.lower()}experiment ./",shell=True).check_returncode()
except :
subprocess.run(f"sudo docker build -t {experimentName.lower()}experiment ./",shell=True).check_returncode()
def getWorkir() -> str :
workdir = "/"
with open(DOCKERFILE,"r") as file:
for line in file.read().splitlines():
if line.startswith("WORKDIR"):
workdir = line.split(" ")[1]
return workdir
def runDockerImage() -> None:
print("binding docker image to the current directory and running it...")
try:
subprocess.run(f"docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} {experimentName.lower()}experiment",shell=True).check_returncode()
except :
subprocess.run(f"sudo docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} {experimentName.lower()}experiment",shell=True).check_returncode()
#TODO : vérifier si la reproduction avec un Dockerfile marche dans l'autre sens
def init(pathInput) -> None : def init(pathInput) -> None :
global repository,path,experimentName,tags, currentTag global repository,path,experimentName,tags, currentTag
...@@ -55,7 +84,6 @@ def init(pathInput) -> None : ...@@ -55,7 +84,6 @@ def init(pathInput) -> None :
os.chdir(path) os.chdir(path)
else : else :
raise Exception(f"{pathInput} is not a git repository") raise Exception(f"{pathInput} is not a git repository")
checkForChanges()
tags = repository.tags tags = repository.tags
currentTag = repository.git.describe('--tags') currentTag = repository.git.describe('--tags')
if not(currentVersionIsTagged()): if not(currentVersionIsTagged()):
...@@ -70,7 +98,7 @@ def fileExists(fileName) -> bool: ...@@ -70,7 +98,7 @@ def fileExists(fileName) -> bool:
def folderExists(folderName) -> bool: def folderExists(folderName) -> bool:
return os.path.isdir(folderName) return os.path.isdir(folderName)
def searchForInputFolder() -> None: def askForInputFolder() -> None:
global inputFolder global inputFolder
answer = input("If you use input data, where are they stored ? Give the path from the root of the repository : ") answer = input("If you use input data, where are they stored ? Give the path from the root of the repository : ")
if answer == "": if answer == "":
...@@ -83,7 +111,7 @@ def searchForInputFolder() -> None: ...@@ -83,7 +111,7 @@ def searchForInputFolder() -> None:
answer+="/" answer+="/"
inputFolder = answer inputFolder = answer
def searchForOutputFolder() -> None: def askForOutputFolder() -> None:
global outputFolder global outputFolder
answer = input("Where are the outputs generated ? Give the path from the root of the repository : ") answer = input("Where are the outputs generated ? Give the path from the root of the repository : ")
if answer == "": if answer == "":
...@@ -96,7 +124,7 @@ def searchForOutputFolder() -> None: ...@@ -96,7 +124,7 @@ def searchForOutputFolder() -> None:
answer+="/" answer+="/"
outputFolder = answer outputFolder = answer
def searchForParamsFolder() -> None: def askForParamsFolder() -> None:
global paramsFolder global paramsFolder
answer = input("In which folder do you store your parameters ? Give the path from the root of the repository : ") answer = input("In which folder do you store your parameters ? Give the path from the root of the repository : ")
if answer == "": if answer == "":
...@@ -155,14 +183,20 @@ def scanParameters() -> None: ...@@ -155,14 +183,20 @@ def scanParameters() -> None:
if not file.endswith(".gitkeep"): if not file.endswith(".gitkeep"):
paramsFiles.append(f"{paramsFolder}{file}") paramsFiles.append(f"{paramsFolder}{file}")
def isNotAnOutputfile(file) -> bool: return file not in outputFiles
def isNotAnInputfile(file) -> bool: return file not in inputFiles
def isNotAParamFile(file) -> bool: return file not in paramsFiles
def checkGeneratedFiles() -> None : def checkGeneratedFiles() -> None :
editedFiles = [ item.a_path for item in repository.index.diff(None) ] + repository.untracked_files editedFiles = [ item.a_path for item in repository.index.diff(None) ]+ [ item.a_path for item in repository.index.diff(repository.head.name) ] + repository.untracked_files
outOfPlaceFiles = [] outOfPlaceFiles = []
logFile = open("outOfPlaceFiles.log","w") logFile = open("outOfPlaceFiles.log","w")
for file in editedFiles: for file in editedFiles:
if (outputFolder is not None and file.startswith(outputFolder)) and \ if file!=DOCKERFILE and \
(inputFolder is not None and file.startswith(inputFolder)) and \ file!=EXPERIMENT_RESUME and \
(paramsFolder is not None and file.startswith(paramsFolder)): isNotAnOutputfile(file) and \
isNotAnInputfile(file) and \
isNotAParamFile(file):
outOfPlaceFiles.append(file) outOfPlaceFiles.append(file)
logFile.write(f"{file}\n") logFile.write(f"{file}\n")
...@@ -183,9 +217,10 @@ def writeInYaml() -> None: ...@@ -183,9 +217,10 @@ def writeInYaml() -> None:
cur_yaml.update({"outputs":outputFiles}) cur_yaml.update({"outputs":outputFiles})
cur_yaml.update({"params":paramsFiles}) cur_yaml.update({"params":paramsFiles})
cur_yaml.update({"instruction":instructionFile}) cur_yaml.update({"instruction":instructionFile})
cur_yaml.update({"dockerfile":fileExists(DOCKERFILE)})
checksums = {"checksums":genChecksums()} checksums = {"checksums":genChecksums()}
cur_yaml.update(checksums) cur_yaml.update(checksums)
with open('experimentResume.yaml', 'w') as yamlFile: with open(EXPERIMENT_RESUME, 'w') as yamlFile:
yaml.safe_dump(cur_yaml, yamlFile) yaml.safe_dump(cur_yaml, yamlFile)
...@@ -219,30 +254,43 @@ def genChecksums() -> list[dict]: ...@@ -219,30 +254,43 @@ def genChecksums() -> list[dict]:
return checksums return checksums
def run(folder) -> None : def askFolders() -> None :
init(folder) askForInputFolder()
repository.active_branch.checkout() askForOutputFolder()
searchForInputFolder() askForParamsFolder()
searchForOutputFolder()
searchForParamsFolder() def reproduceExperiment() -> None:
userInput = input("Do you have a pre-recorded commands file? (y/n)") if dockerfileIsPresent() :
if userInput == "y": buildDockerImage()
askForCommandsFile() runDockerImage()
runExperiment()
else: else:
askForInstructionFile() userInput = input("Do you have a pre-recorded commands file? (y/n)")
done = "" if userInput == "y":
while(done != "done"): askForCommandsFile()
done = input("Run your experiment and then type 'done' when you are done : ") runExperiment()
else:
askForInstructionFile()
done = ""
while(done != "done"):
done = input("Run your experiment and then type 'done' when you are done : ")
def scanAfterExecution() -> None:
if inputFolder != None : if inputFolder != None :
scanInputFiles() scanInputFiles()
if outputFolder != None : if outputFolder != None :
scanOutputsGenerated() scanOutputsGenerated()
if paramsFolder != None : if paramsFolder != None :
scanParameters() scanParameters()
def run(folder) -> None :
init(folder)
repository.active_branch.checkout()
askFolders()
reproduceExperiment()
scanAfterExecution()
checkGeneratedFiles() checkGeneratedFiles()
writeInYaml() writeInYaml()
print("Please check the experimentResume.yaml, if everything is correct, press enter to continue, otherwise type \"abort\"") print(f"Please check the {EXPERIMENT_RESUME} file, if everything is correct, press enter to continue, otherwise type \"abort\"")
if input() == "abort": if input() == "abort":
raise Exception("Aborted") raise Exception("Aborted")
pushBranch() pushBranch()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment