Skip to content
Snippets Groups Projects
Commit 338df9b8 authored by AxelCarayon's avatar AxelCarayon
Browse files

ajout module pour reproduire une expérience

parent 1708ae02
No related branches found
No related tags found
No related merge requests found
import git
import os
import subprocess
import yaml
import hashlib
import warnings
INPUT_FOLDER = "inputs"
OUTPUT_FOLDER = "outputs"
repo = None
folder = None
commandsFile = None
inputFiles = []
beforeHash = None
def init(repository,branch) -> None :
global repo, folder
folder = repository.split('/')[-1].split('.')[0]
if os.path.exists(folder) :
print(f"Folder ./{folder} already exists, do you want to delete it ? (y/n)")
answer = input()
if answer == 'y' :
os.system(f"rm -rf ./{folder}")
else :
print("Aborting")
exit(0)
git.Git("./").clone(repository)
repo = git.Repo(folder)
try :
repo.git.checkout(branch)
except git.exc.GitCommandError :
raise Exception(f"Branch {branch} not found in the repository")
os.chdir(folder)
def getParameters() -> None :
global commandsFile, inputFiles, outputFiles, beforeHash
if not (os.path.exists('experimentResume.yaml')):
raise Exception("No exeperimentResume.yaml file found, the branch is not an exeperiment")
with open('experimentResume.yaml', 'r') as stream:
parameters = yaml.safe_load(stream)
commandsFile = parameters.get('commands')
outputFiles = parameters.get('outputs')
inputFiles = parameters.get('inputs')
beforeHash = parameters.get('checksums')
def runExperiment() -> None :
file = open(commandsFile, "r")
for line in file.read().splitlines():
print(f"running {line} ...")
process = subprocess.run(line,shell=True)
process.check_returncode()
print("done")
def genChecksum(file) -> str :
hash_md5 = hashlib.md5()
with open(file, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
def genChecksums() -> list[dict]:
checksums = []
for file in os.listdir(OUTPUT_FOLDER) :
if not file.endswith(".gitkeep"):
checksums.append({file : genChecksum(f'outputs/{file}')})
return checksums
def compareChecksums() -> bool:
changes = False
for (dict1, dict2) in zip(beforeHash, genChecksums()):
for (key, value) in dict1.items():
if dict2.get(key) != value :
warnings.warn(f"{OUTPUT_FOLDER}/{key} has changed")
changes = True
return changes
def run(repository, branch) -> None :
print("Initializing the experiment repository ...")
init(repository, branch)
print("Getting the experiment parameters ...")
getParameters()
print("Running the experiment ...")
runExperiment()
print("Comparing checksums of the outputs ...")
if (compareChecksums()) :
print("The exepriment was reproduced with succes but some output files are differents.")
else :
print("The exepriment was reproduced with succes !")
\ No newline at end of file
...@@ -26,6 +26,7 @@ def isGitRepo(path) -> bool: ...@@ -26,6 +26,7 @@ def isGitRepo(path) -> bool:
return True return True
except git.exc.InvalidGitRepositoryError: except git.exc.InvalidGitRepositoryError:
return False return False
def init(pathInput) -> None : def init(pathInput) -> None :
global repository,path,experimentName global repository,path,experimentName
if isGitRepo(pathInput): if isGitRepo(pathInput):
......
import argparse import argparse
import registerExperiment import registerExperiment
import loadExperiment
if (__name__ == "__main__"): if (__name__ == "__main__"):
parser = argparse.ArgumentParser(description="Reproduction tool") parser = argparse.ArgumentParser(description="Reproduction tool")
g = parser.add_mutually_exclusive_group() g = parser.add_mutually_exclusive_group()
g.add_argument("-s", "--save", help="Register a new experiment from a local git repository") g.add_argument("-s", "--save", help="Register a new experiment from a local git repository")
g.add_argument("-l", "--load", help="Reproduce an experiment") g.add_argument("-l", "--load", help="Reproduce an experiment from a distant git repository")
parser.add_argument("-b", "--branch", help="Branch to use for the experiment")
args = parser.parse_args() args = parser.parse_args()
if args.save: if args.save:
registerExperiment.run(args.save) registerExperiment.run(args.save)
if args.load: if args.load:
#TODO if not (args.branch):
pass print("Please specify a branch")
\ No newline at end of file exit(1)
loadExperiment.run(args.load, args.branch)
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment