From 338df9b844c252176d26ddd80aeb04f7e99818a3 Mon Sep 17 00:00:00 2001 From: AxelCarayon <axel.carayon@gmail.com> Date: Tue, 14 Dec 2021 11:37:29 +0100 Subject: [PATCH] =?UTF-8?q?ajout=20module=20pour=20reproduire=20une=20exp?= =?UTF-8?q?=C3=A9rience?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- loadExperiment.py | 96 +++++++++++++++++++++++++++++++++++++++++++ registerExperiment.py | 1 + reprodExperiment.py | 10 +++-- 3 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 loadExperiment.py diff --git a/loadExperiment.py b/loadExperiment.py new file mode 100644 index 0000000..4d4dea1 --- /dev/null +++ b/loadExperiment.py @@ -0,0 +1,96 @@ +import git +import os +import subprocess +import yaml +import hashlib +import warnings + +INPUT_FOLDER = "inputs" +OUTPUT_FOLDER = "outputs" + +repo = None +folder = None + +commandsFile = None + +inputFiles = [] + +beforeHash = None + +def init(repository,branch) -> None : + global repo, folder + folder = repository.split('/')[-1].split('.')[0] + if os.path.exists(folder) : + print(f"Folder ./{folder} already exists, do you want to delete it ? (y/n)") + answer = input() + if answer == 'y' : + os.system(f"rm -rf ./{folder}") + else : + print("Aborting") + exit(0) + git.Git("./").clone(repository) + repo = git.Repo(folder) + try : + repo.git.checkout(branch) + except git.exc.GitCommandError : + raise Exception(f"Branch {branch} not found in the repository") + os.chdir(folder) + +def getParameters() -> None : + global commandsFile, inputFiles, outputFiles, beforeHash + if not (os.path.exists('experimentResume.yaml')): + raise Exception("No exeperimentResume.yaml file found, the branch is not an exeperiment") + with open('experimentResume.yaml', 'r') as stream: + parameters = yaml.safe_load(stream) + commandsFile = parameters.get('commands') + outputFiles = parameters.get('outputs') + inputFiles = parameters.get('inputs') + beforeHash = parameters.get('checksums') + +def runExperiment() -> None : + file = open(commandsFile, "r") + for line in file.read().splitlines(): + print(f"running {line} ...") + process = subprocess.run(line,shell=True) + process.check_returncode() + print("done") + + +def genChecksum(file) -> str : + hash_md5 = hashlib.md5() + with open(file, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + +def genChecksums() -> list[dict]: + checksums = [] + for file in os.listdir(OUTPUT_FOLDER) : + if not file.endswith(".gitkeep"): + checksums.append({file : genChecksum(f'outputs/{file}')}) + return checksums + + +def compareChecksums() -> bool: + changes = False + for (dict1, dict2) in zip(beforeHash, genChecksums()): + for (key, value) in dict1.items(): + if dict2.get(key) != value : + warnings.warn(f"{OUTPUT_FOLDER}/{key} has changed") + changes = True + return changes + + +def run(repository, branch) -> None : + print("Initializing the experiment repository ...") + init(repository, branch) + print("Getting the experiment parameters ...") + getParameters() + print("Running the experiment ...") + runExperiment() + print("Comparing checksums of the outputs ...") + if (compareChecksums()) : + print("The exepriment was reproduced with succes but some output files are differents.") + else : + print("The exepriment was reproduced with succes !") + \ No newline at end of file diff --git a/registerExperiment.py b/registerExperiment.py index 7dd0205..aacb0dc 100644 --- a/registerExperiment.py +++ b/registerExperiment.py @@ -26,6 +26,7 @@ def isGitRepo(path) -> bool: return True except git.exc.InvalidGitRepositoryError: return False + def init(pathInput) -> None : global repository,path,experimentName if isGitRepo(pathInput): diff --git a/reprodExperiment.py b/reprodExperiment.py index 1e33297..c3bcbef 100644 --- a/reprodExperiment.py +++ b/reprodExperiment.py @@ -1,15 +1,19 @@ import argparse import registerExperiment +import loadExperiment if (__name__ == "__main__"): parser = argparse.ArgumentParser(description="Reproduction tool") g = parser.add_mutually_exclusive_group() g.add_argument("-s", "--save", help="Register a new experiment from a local git repository") - g.add_argument("-l", "--load", help="Reproduce an experiment") + g.add_argument("-l", "--load", help="Reproduce an experiment from a distant git repository") + parser.add_argument("-b", "--branch", help="Branch to use for the experiment") args = parser.parse_args() if args.save: registerExperiment.run(args.save) if args.load: - #TODO - pass \ No newline at end of file + if not (args.branch): + print("Please specify a branch") + exit(1) + loadExperiment.run(args.load, args.branch) \ No newline at end of file -- GitLab