diff --git a/loadExperiment.py b/loadExperiment.py index a54d51dae2a8bb227a5adf6e20d3099c901af511..fa03e699306cf13f93ba026221432474ddf21860 100644 --- a/loadExperiment.py +++ b/loadExperiment.py @@ -17,6 +17,7 @@ instructionFile = None inputFiles = [] outputFiles = [] +dockerfileIsPresent = False beforeHash = None def init(repository,branch) -> None : @@ -39,7 +40,7 @@ def init(repository,branch) -> None : os.chdir(folder) def getParameters() -> None : - global commandsFile, inputFiles, outputFiles, beforeHash, instructionFile + global commandsFile, inputFiles, outputFiles, beforeHash, instructionFile, dockerfileIsPresent if not (os.path.exists('experimentResume.yaml')): raise Exception("No exeperimentResume.yaml file found, the branch is not an exeperiment") with open('experimentResume.yaml', 'r') as stream: @@ -49,6 +50,7 @@ def getParameters() -> None : inputFiles = parameters.get('inputs') beforeHash = parameters.get('checksums') instructionFile = parameters.get('instructions') + dockerfileIsPresent = parameters.get('dockerfile') def runExperiment() -> None : @@ -94,16 +96,44 @@ def compareChecksums() -> bool: return changes +def buildDockerImage() -> None: + print("Building the docker image ...") + try : + subprocess.run(f"docker build -t experimentreproduction ./",shell=True).check_returncode() + except : + subprocess.run(f"sudo docker build -t experimentreproduction ./",shell=True).check_returncode() + +def getWorkir() -> str : + workdir = "/" + with open("Dockerfile","r") as file: + for line in file.read().splitlines(): + if line.startswith("WORKDIR"): + workdir = line.split(" ")[1] + return workdir + +def runDockerImage() -> None: + print("binding docker image to the current directory and running it...") + try: + subprocess.run(f"docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} experimentreproduction",shell=True).check_returncode() + except : + subprocess.run(f"sudo docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} experimentreproduction",shell=True).check_returncode() + + def run(repository, branch) -> None : print("Initializing the experiment repository ...") init(repository, branch) print("Getting the experiment parameters ...") getParameters() print("Running the experiment ...") - if (commandsFile != None) : - runExperiment() - else : - checkForInstructions() + if (dockerfileIsPresent) : + print("Dockerimage was found ! Using it to run the experiment...") + buildDockerImage() + runDockerImage() + else: + if (commandsFile != None) : + runExperiment() + else : + checkForInstructions() print("Comparing checksums of the outputs ...") if (compareChecksums()) : print("The exepriment was reproduced with succes but some output files are differents.") diff --git a/registerExperiment.py b/registerExperiment.py index b9daa5d4e3d05a97444cb29dbd7a039daee523d1..aa1b061659e216b40d08a6c9fbaec640f49e35ec 100644 --- a/registerExperiment.py +++ b/registerExperiment.py @@ -1,4 +1,6 @@ +from multiprocessing.connection import answer_challenge import os +from unicodedata import name import git import subprocess import yaml @@ -6,6 +8,7 @@ import hashlib import warnings EXPERIMENT_RESUME = "experimentResume.yaml" +DOCKERFILE = "Dockerfile" path = "./" @@ -36,13 +39,39 @@ def isGitRepo(path) -> bool: except git.exc.InvalidGitRepositoryError: return False -def checkForChanges() -> None: - changesNotAdded = [ item.a_path for item in repository.index.diff(None) ] - changesAdded = [ item.a_path for item in repository.index.diff(repository.head.name) ] - untrackedFiles = repository.untracked_files - if ((len(changesNotAdded) + len(changesAdded) + len(untrackedFiles)) > 0): - raise Exception("There are changes in the repository since the last commit, you can only register an experiment from a clean repository.") + +def dockerfileIsPresent() -> bool: + if fileExists(DOCKERFILE): + answer = input("A dockerfile was found ! It will be used to reproduce the experiment. Is that ok for you ? (y/n)") + if answer == "n": + raise Exception("""Remove the dockerfile and try again""") + return True + else : + return False + +def buildDockerImage() -> None: + print("Building the docker image ...") + try : + subprocess.run(f"docker build -t {experimentName.lower()}experiment ./",shell=True).check_returncode() + except : + subprocess.run(f"sudo docker build -t {experimentName.lower()}experiment ./",shell=True).check_returncode() + +def getWorkir() -> str : + workdir = "/" + with open(DOCKERFILE,"r") as file: + for line in file.read().splitlines(): + if line.startswith("WORKDIR"): + workdir = line.split(" ")[1] + return workdir + +def runDockerImage() -> None: + print("binding docker image to the current directory and running it...") + try: + subprocess.run(f"docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} {experimentName.lower()}experiment",shell=True).check_returncode() + except : + subprocess.run(f"sudo docker run -it --mount type=bind,source=\"$PWD\",target={getWorkir()} {experimentName.lower()}experiment",shell=True).check_returncode() + #TODO : vérifier si la reproduction avec un Dockerfile marche dans l'autre sens def init(pathInput) -> None : global repository,path,experimentName,tags, currentTag @@ -55,7 +84,6 @@ def init(pathInput) -> None : os.chdir(path) else : raise Exception(f"{pathInput} is not a git repository") - checkForChanges() tags = repository.tags currentTag = repository.git.describe('--tags') if not(currentVersionIsTagged()): @@ -70,7 +98,7 @@ def fileExists(fileName) -> bool: def folderExists(folderName) -> bool: return os.path.isdir(folderName) -def searchForInputFolder() -> None: +def askForInputFolder() -> None: global inputFolder answer = input("If you use input data, where are they stored ? Give the path from the root of the repository : ") if answer == "": @@ -83,7 +111,7 @@ def searchForInputFolder() -> None: answer+="/" inputFolder = answer -def searchForOutputFolder() -> None: +def askForOutputFolder() -> None: global outputFolder answer = input("Where are the outputs generated ? Give the path from the root of the repository : ") if answer == "": @@ -96,7 +124,7 @@ def searchForOutputFolder() -> None: answer+="/" outputFolder = answer -def searchForParamsFolder() -> None: +def askForParamsFolder() -> None: global paramsFolder answer = input("In which folder do you store your parameters ? Give the path from the root of the repository : ") if answer == "": @@ -155,14 +183,20 @@ def scanParameters() -> None: if not file.endswith(".gitkeep"): paramsFiles.append(f"{paramsFolder}{file}") +def isNotAnOutputfile(file) -> bool: return file not in outputFiles +def isNotAnInputfile(file) -> bool: return file not in inputFiles +def isNotAParamFile(file) -> bool: return file not in paramsFiles + def checkGeneratedFiles() -> None : - editedFiles = [ item.a_path for item in repository.index.diff(None) ] + repository.untracked_files + editedFiles = [ item.a_path for item in repository.index.diff(None) ]+ [ item.a_path for item in repository.index.diff(repository.head.name) ] + repository.untracked_files outOfPlaceFiles = [] logFile = open("outOfPlaceFiles.log","w") for file in editedFiles: - if (outputFolder is not None and file.startswith(outputFolder)) and \ - (inputFolder is not None and file.startswith(inputFolder)) and \ - (paramsFolder is not None and file.startswith(paramsFolder)): + if file!=DOCKERFILE and \ + file!=EXPERIMENT_RESUME and \ + isNotAnOutputfile(file) and \ + isNotAnInputfile(file) and \ + isNotAParamFile(file): outOfPlaceFiles.append(file) logFile.write(f"{file}\n") @@ -183,9 +217,10 @@ def writeInYaml() -> None: cur_yaml.update({"outputs":outputFiles}) cur_yaml.update({"params":paramsFiles}) cur_yaml.update({"instruction":instructionFile}) + cur_yaml.update({"dockerfile":fileExists(DOCKERFILE)}) checksums = {"checksums":genChecksums()} cur_yaml.update(checksums) - with open('experimentResume.yaml', 'w') as yamlFile: + with open(EXPERIMENT_RESUME, 'w') as yamlFile: yaml.safe_dump(cur_yaml, yamlFile) @@ -219,30 +254,43 @@ def genChecksums() -> list[dict]: return checksums -def run(folder) -> None : - init(folder) - repository.active_branch.checkout() - searchForInputFolder() - searchForOutputFolder() - searchForParamsFolder() - userInput = input("Do you have a pre-recorded commands file? (y/n)") - if userInput == "y": - askForCommandsFile() - runExperiment() +def askFolders() -> None : + askForInputFolder() + askForOutputFolder() + askForParamsFolder() + +def reproduceExperiment() -> None: + if dockerfileIsPresent() : + buildDockerImage() + runDockerImage() else: - askForInstructionFile() - done = "" - while(done != "done"): - done = input("Run your experiment and then type 'done' when you are done : ") + userInput = input("Do you have a pre-recorded commands file? (y/n)") + if userInput == "y": + askForCommandsFile() + runExperiment() + else: + askForInstructionFile() + done = "" + while(done != "done"): + done = input("Run your experiment and then type 'done' when you are done : ") + +def scanAfterExecution() -> None: if inputFolder != None : scanInputFiles() if outputFolder != None : scanOutputsGenerated() if paramsFolder != None : scanParameters() + +def run(folder) -> None : + init(folder) + repository.active_branch.checkout() + askFolders() + reproduceExperiment() + scanAfterExecution() checkGeneratedFiles() writeInYaml() - print("Please check the experimentResume.yaml, if everything is correct, press enter to continue, otherwise type \"abort\"") + print(f"Please check the {EXPERIMENT_RESUME} file, if everything is correct, press enter to continue, otherwise type \"abort\"") if input() == "abort": raise Exception("Aborted") pushBranch()