From 9766991666cf961c51664ca95215df553bed4b04 Mon Sep 17 00:00:00 2001
From: lrivier <laura.riviere@irit.fr>
Date: Mon, 30 Jan 2023 16:31:07 +0100
Subject: [PATCH] fix multi-process pb and change config
---
DISCUT/code/config_global_1.1.json | 8 ++++----
DISCUT/code/discut22_2.py | 18 +++++++++++++++---
2 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/DISCUT/code/config_global_1.1.json b/DISCUT/code/config_global_1.1.json
index b45806e..767319b 100644
--- a/DISCUT/code/config_global_1.1.json
+++ b/DISCUT/code/config_global_1.1.json
@@ -13,9 +13,9 @@
"syntactic_tool": "stanza",
"sentence_split": true,
"tokenization": true,
- "syntactic_parsing": false,
+ "syntactic_parsing": true,
"create_metadata": {
- "to_do": true,
+ "to_do": false,
"line": "paragraph",
"sent": "sent"
}
@@ -36,11 +36,11 @@
"output":{
"conll_file":{
"to_do": true,
- "metadata": true,
+ "metadata": false,
"with_gold_labels": false
},
"txt_file":{
- "to_do": false,
+ "to_do": true,
"metadata": true
}
}
diff --git a/DISCUT/code/discut22_2.py b/DISCUT/code/discut22_2.py
index 4c5caaa..f0d56f3 100644
--- a/DISCUT/code/discut22_2.py
+++ b/DISCUT/code/discut22_2.py
@@ -10,6 +10,8 @@
import argparse
from datetime import datetime
import os
+import subprocess
+from subprocess import Popen
import re
import json
import utils.syntactic_parsing as synt_pars
@@ -114,9 +116,18 @@ class Data:
fi_ner = self.ner if fi_ner == None else fi_ner
model = steps.model_path if model == None else model
self.pred_json = f"{self.resu}/{js_name}_pred.json"
- cmd = f"allennlp predict --use-dataset-reader --output-file {self.pred_json} {model} {fi_ner} &> {self.resu}/logs_predictions.txt"
+ cmd = f"allennlp predict --use-dataset-reader --output-file {self.pred_json} {model} {fi_ner}"
print(f"----> Making predictions: {cmd}.")
- os.system(cmd)
+
+ cp = subprocess.run(cmd, shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ print(f"----> done")
+ with open(f"{self.resu}/logs_predictions.txt", "w") as fe:
+ fe.write(cp.stderr)
+
+ #with open(f"{self.resu}/logs_predictions.txt", 'w') as fe: # too long !
+ # p = Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=fe, universal_newlines=True)
+ # p.wait()
+
logs.add_infos('predictions_cmd', cmd)
def pred_json_to_conll_w_metadata_w_gold(self, name=None): # here and 3 below..sorry..factorsation TBD
@@ -268,7 +279,7 @@ class Process:
if self.model == "tony":
arch = "french_tokens.tar.gz"
if not os.path.isfile(f"../model/tony/{arch}"):
- dl = "wget https://zenodo.org/record/4235850/files/french_tokens.tar.gz -P ../model --progress=bar"
+ dl = "wget https://zenodo.org/record/4235850/files/french_tokens.tar.gz -P ../model/tony --progress=bar"
os.system(dl)
self.model_path = f"../model/tony/{arch}"
else:
@@ -419,5 +430,6 @@ if __name__ == '__main__':
if steps.eval == True:
data.evaluation(steps, prod, name=steps.test_data, model=steps.model_ft_path)
+ logs.add_infos('end', get_stamp())
logs.print(stamp_time)
print(f"----> All logs saved in {logs.file_path}")
\ No newline at end of file
--
GitLab