fix bug data

Utils/PostpreprocesTXT.py

@@ -106,6 +106,7 @@ print(len(dfs['X']))
 rs = pd.merge(dfs, data3, on="X",how="inner").reindex(dfs.index)
 rs.drop('Y1', inplace=True, axis=1)
 rs.drop('Y2', inplace=True, axis=1)
+# rs.drop_duplicates()
 
 rs.to_csv("../Datasets/silver_dataset_links.csv", index=False)
 
@@ -117,15 +118,18 @@ dfg['Y'] = Zg
 rg = pd.merge(dfg, data3, on="X",how="inner").reindex(dfg.index)
 rg.drop('Y1', inplace=True, axis=1)
 rg.drop('Y2', inplace=True, axis=1)
+# rg.drop_duplicates()
 
 rg.to_csv("../Datasets/gold_dataset_links.csv", index=False)
 
 data1 = pd.read_csv('../Datasets/gold_dataset_links.csv')
 data2 = pd.read_csv('../Datasets/silver_dataset_links.csv')
 df = pd.merge(data1, data2,how='outer')
+df = df.drop_duplicates(subset=['X'])
+
 
 #
-df.to_csv("../Datasets/goldANDsilver_dataset_links.csv", index=False)
+df[:len(df)-1].to_csv("../Datasets/goldANDsilver_dataset_links.csv", index=False)
 
 #
 # import os