diff --git a/transformer-tests/collect_metrics.py b/transformer-tests/collect_metrics.py
new file mode 100644
index 0000000000000000000000000000000000000000..bd28c8d4b3e2d088e81a494993b5224ef4fb6183
--- /dev/null
+++ b/transformer-tests/collect_metrics.py
@@ -0,0 +1,126 @@
+import re
+import csv
+import json
+import numpy as np
+import pandas as pd
+from pathlib import Path
+from scipy.stats import spearmanr, pearsonr
+
+dataset = ['20ng', 'wiki']
+prompt_intr = ['p1_v6', 'p2_v6', 'p3_v6', 'p4_v6', 'p5_v6', 'p6_v6']
+version_rating = ['v1','v2','v3','v4']
+
+def load_gpt3_resp(file_path):
+    reader = open(file_path)
+    data = [line.replace("\n", "") for line in reader.readlines()]
+    counts = re.findall(r'\[.*?\]', data[0])
+    counts = counts[0].replace('[','').replace(']','').replace(',','').split()
+    counts = [int(num) for num in counts]
+
+    return counts
+
+
+def load_npmis(model, data):
+    with open('./results/'+model+'-'+data+'/npmis.txt') as file:
+        npmis = file.readlines()
+    npmis = [float(term.replace('\n', '')) for term in npmis]
+    return npmis
+
+def collect_metrics():
+    model = ['mallet', 'dvae', 'etm']
+    header = ['dataset', 'model', 'npmi']
+    gpt3_path = './transformer-tests/'
+
+    with open(Path(gpt3_path, 'intrusion_metrics.csv'), 'w', encoding='UTF8') as file:
+        header_intr = header + prompt_intr
+        writer = csv.writer(file)
+        writer.writerow(header_intr)
+        for d in dataset:
+            for m in model:
+
+            # for j in range(len(prompt)):
+                # gpt3_intr_mallet = load_gpt3_resp(Path(model[0], d, 'intr_' + prompt[j] + '_counts.txt'))
+                # gpt3_intr_dvae = load_gpt3_resp(Path(model[1], d, 'intr_' + prompt[j] + '_counts.txt'))
+                # gpt3_intr_etm = load_gpt3_resp(Path(model[2], d, 'intr_' + prompt[j] + '_counts.txt'))
+                # for i in range(len(gpt3_intr_mallet)):
+                #     row = [d, prompt[j], gpt3_intr_mallet[i], gpt3_intr_dvae[i], gpt3_intr_etm[i]]
+
+                npmi = load_npmis(m, d)
+                gpt3_intr = [load_gpt3_resp(Path(gpt3_path, m, d, 'intr_' + p + '_counts.txt')) for p in prompt_intr]
+                for i in range(len(gpt3_intr[0])):
+                    row = [d, m, npmi[i], gpt3_intr[0][i], gpt3_intr[1][i],
+                           gpt3_intr[2][i], gpt3_intr[3][i], gpt3_intr[4][i], gpt3_intr[5][i]]
+                    writer.writerow(row)
+
+
+    with open(Path(gpt3_path, 'rating_metrics.csv'), 'w', encoding='UTF8') as file:
+        header_rat = header + version_rating
+        writer = csv.writer(file)
+        writer.writerow(header_rat)
+        for d in dataset:
+            for m in model:
+                npmi = load_npmis(m, d)
+                gpt3_rating = [load_gpt3_resp(Path(gpt3_path, m, d, 'rating_p3_' + v + '_counts.txt')) for v in version_rating]
+                for i in range(len(gpt3_rating[0])):
+                    row = [d, m, npmi[i], gpt3_rating[0][i], gpt3_rating[1][i],
+                           gpt3_rating[2][i], gpt3_rating[3][i]]
+                    writer.writerow(row)
+
+
+
+collect_metrics()
+dfs_intr = pd.read_csv('./transformer-tests/intrusion_metrics.csv')
+
+with open('./transformer-tests/metrics.txt', 'w') as f:
+    for d in dataset:
+        data_intr = dfs_intr[dfs_intr['dataset'] == d]
+        npmi_intr = data_intr['npmi']
+        for p in prompt_intr:
+            p_val = data_intr[p]
+            acc = np.mean(p_val)
+            variance = np.var(p_val)
+            spear_rho, spear_p = spearmanr(npmi_intr, p_val)
+            pear_rho, pear_p = pearsonr(npmi_intr, p_val)
+            metrics = {
+                "task":'intrusion',
+                "dataset": d,
+                "prompt": p,
+                "mean": acc,
+                "var": variance,
+                "spear_rho": spear_rho,
+                "spear_p": spear_p,
+                "pear_rho": pear_rho,
+                "pear_p": pear_p
+            }
+            f.write(str(metrics) + '\n')
+
+            print(metrics)
+
+
+
+dfs_rating = pd.read_csv('./transformer-tests/rating_metrics.csv')
+
+with open('./transformer-tests/metrics.txt', 'a') as f:
+    for d in dataset:
+        data_rating = dfs_rating[dfs_rating['dataset'] == d]
+        npmi_rating = data_rating['npmi']
+        for v in version_rating:
+            p_val = data_rating[v]
+            acc = np.mean(p_val)
+            variance = np.var(p_val)
+            spear_rho, spear_p = spearmanr(npmi_rating, p_val)
+            pear_rho, pear_p = pearsonr(npmi_rating, p_val)
+            metrics = {
+                "task":'rating',
+                "dataset": d,
+                "version": v,
+                "mean": acc,
+                "var": variance,
+                "spear_rho": spear_rho,
+                "spear_p": spear_p,
+                "pear_rho": pear_rho,
+                "pear_p": pear_p
+            }
+            f.write(str(metrics) + '\n')
+
+            print(metrics)
\ No newline at end of file
diff --git a/transformer-tests/intrusion.py b/transformer-tests/intrusion.py
index d3dd064a779fdefc417210d6ad6da5f0e4c4cd67..c0646d557927e95276a0ba358f033d154a36f475 100644
--- a/transformer-tests/intrusion.py
+++ b/transformer-tests/intrusion.py
@@ -4,7 +4,6 @@ import random
 import numpy as np
 from gpt3 import gpt3
 from scipy.stats import spearmanr, pearsonr
-#from bloom import bloom
 
 class Intrusion():
     def __init__(self, topics, num_topics, path_save, file_path):
@@ -15,7 +14,7 @@ class Intrusion():
         self.num_topics = num_topics
         self.path_save = path_save
         self.file_path = file_path
-        self.prompt_name = 'intr_p1_v6'
+        self.prompt_name = 'intr_p2_v6'
 
 
     def create_intruder(self, num_terms=5, sample_top_topic_terms=False):
@@ -157,7 +156,7 @@ class Intrusion():
             # list_terms = str(list_of_terms[i]).replace("[", "").replace("]", "").replace("'", "")
 
             # prompt = 'What is the intruder term in the following terms? ' + list_terms
-            prompt = 'Show the least related term\nTerms: ' + list_terms + '\nAnswer: '
+            prompt = 'Select which term is the least related to all other terms\nTerms: ' + list_terms + '\nAnswer: '
 
             topic_intruder_prompt.append(
                 {
@@ -172,20 +171,22 @@ class Intrusion():
                 f.write(str(topic_intruder_prompt[i]) + '\n')
             f.close()
 
-        del list_of_terms
         return topic_intruder_prompt
 
 
     def run_gpt3(self, tip_dict):
         # run GPT-3 for all topics
+        response_list = []
         with open(self.path_save + self.prompt_name + '_gpt3.txt', 'w') as f:
             for i in range(len(tip_dict)):
                 prompt = tip_dict[i]['prompt']
                 response = str(gpt3(prompt)).replace('\n', '')
+                response_list.append(response)
                 f.write(response + '\n')
                 print(prompt, " ", response)
-                time.sleep(2)
+                time.sleep(3)
             f.close()
+        return response_list
 
 
     def load_npmis(self):
diff --git a/transformer-tests/intrusion_metrics.csv b/transformer-tests/intrusion_metrics.csv
new file mode 100644
index 0000000000000000000000000000000000000000..5a2d7a56288df4336a3424a4072576353d8e9008
--- /dev/null
+++ b/transformer-tests/intrusion_metrics.csv
@@ -0,0 +1,301 @@
+dataset,model,npmi,p1_v6,p2_v6,p3_v6,p4_v6,p5_v6,p6_v6
+20ng,mallet,0.1314275058044984,0,0,0,0,0,0
+20ng,mallet,0.12868722938942015,0,0,0,0,0,0
+20ng,mallet,0.35750602263515147,1,0,0,0,0,0
+20ng,mallet,0.11305819494914734,0,0,0,0,0,0
+20ng,mallet,0.11484206083907826,1,1,0,0,0,0
+20ng,mallet,0.11176888846533804,0,0,0,0,0,0
+20ng,mallet,0.12732699467385925,0,0,0,0,0,0
+20ng,mallet,0.2287394936063482,0,0,0,0,0,0
+20ng,mallet,0.44473939676352114,1,1,1,1,1,1
+20ng,mallet,0.24127019674935196,0,1,1,1,1,1
+20ng,mallet,0.32284272065971203,1,1,1,1,1,1
+20ng,mallet,0.43142111410155054,1,1,1,1,1,1
+20ng,mallet,0.2910513037578202,0,0,0,0,0,0
+20ng,mallet,0.18014990648358223,1,1,1,1,1,1
+20ng,mallet,0.32188953772530043,1,1,0,1,1,0
+20ng,mallet,0.6603008717784691,1,1,1,1,1,1
+20ng,mallet,0.30699249515929244,0,0,0,0,0,0
+20ng,mallet,0.13326988122411804,1,1,0,1,1,1
+20ng,mallet,0.6421268827929683,1,1,1,1,1,1
+20ng,mallet,0.30043303281639544,1,1,1,1,1,1
+20ng,mallet,0.28048604272718247,0,0,0,0,0,0
+20ng,mallet,0.16676331019961838,1,1,1,1,1,1
+20ng,mallet,0.3726503665794892,1,0,0,0,0,0
+20ng,mallet,0.3765487569290983,1,1,1,1,1,1
+20ng,mallet,0.23026012345157132,1,1,1,1,1,0
+20ng,mallet,0.13333804691738643,0,0,0,0,0,1
+20ng,mallet,0.20357481024945415,1,1,1,1,0,1
+20ng,mallet,0.35850791330099435,0,0,0,1,1,1
+20ng,mallet,0.38752138070365977,1,1,1,1,1,0
+20ng,mallet,0.0799560563666676,0,0,0,0,0,0
+20ng,mallet,0.17852991901351475,0,0,0,0,0,1
+20ng,mallet,0.2742371586926216,1,1,1,1,1,1
+20ng,mallet,0.3715196209666211,0,0,1,0,0,0
+20ng,mallet,0.42261689038903927,1,1,1,1,1,1
+20ng,mallet,0.38192610751506706,1,1,0,1,1,0
+20ng,mallet,0.12258062217387942,1,1,0,1,1,0
+20ng,mallet,0.15059222629790764,0,0,0,0,0,0
+20ng,mallet,0.2023676323552062,0,0,0,0,0,0
+20ng,mallet,0.7693309879198367,0,0,0,0,0,1
+20ng,mallet,0.43051248340180165,0,1,0,0,0,0
+20ng,mallet,0.20686042222320672,1,1,1,1,1,1
+20ng,mallet,0.25629312128444554,1,1,1,1,1,0
+20ng,mallet,0.2010377483499159,0,0,0,0,0,0
+20ng,mallet,0.14900240808594048,0,0,0,0,0,0
+20ng,mallet,0.10959274106419682,0,0,0,0,0,0
+20ng,mallet,0.21311907647038422,1,1,1,1,1,1
+20ng,mallet,0.1879680802758673,1,1,0,1,1,0
+20ng,mallet,0.20926676826294122,1,1,1,1,0,1
+20ng,mallet,0.15470010994497765,0,0,0,0,0,0
+20ng,mallet,0.1706810915779308,0,0,0,0,0,0
+20ng,dvae,0.24867697774022374,0,0,0,0,0,0
+20ng,dvae,0.3396118005925878,0,0,0,0,0,0
+20ng,dvae,0.29970309841727477,0,0,0,0,0,0
+20ng,dvae,0.23866633667824944,0,0,0,0,0,0
+20ng,dvae,0.3699640259380764,0,0,0,0,0,0
+20ng,dvae,0.42240324296068005,1,0,0,0,0,0
+20ng,dvae,0.8874487186560454,0,0,0,0,0,0
+20ng,dvae,0.128691716880522,0,0,0,0,0,0
+20ng,dvae,0.37589863667431506,0,0,0,0,0,0
+20ng,dvae,0.6238240724850099,1,0,0,0,0,0
+20ng,dvae,0.17956630920866357,0,1,1,1,1,1
+20ng,dvae,0.29258062003294744,0,0,1,0,0,0
+20ng,dvae,0.3877803368148381,1,1,1,1,1,1
+20ng,dvae,0.33284345912394303,1,0,0,0,0,1
+20ng,dvae,0.17428405447076523,0,1,1,1,1,1
+20ng,dvae,0.4708279864428822,0,0,0,0,0,0
+20ng,dvae,0.33234795901646436,1,1,0,1,1,0
+20ng,dvae,0.2659239621234882,0,1,0,1,1,1
+20ng,dvae,0.4472987543185158,1,1,0,0,0,0
+20ng,dvae,0.18977108703994489,0,0,0,0,0,0
+20ng,dvae,0.21801183330846072,0,0,0,0,0,0
+20ng,dvae,0.45164395443930255,0,0,1,0,0,1
+20ng,dvae,0.5162468786823781,0,0,0,1,1,0
+20ng,dvae,0.4219132846940917,1,1,1,1,1,1
+20ng,dvae,0.37711305638174364,0,0,0,0,0,0
+20ng,dvae,0.5037645757835705,0,0,0,0,0,0
+20ng,dvae,0.36761746934438233,0,0,0,0,0,0
+20ng,dvae,0.6534066162568825,1,1,0,0,0,0
+20ng,dvae,0.09283550832829766,0,0,0,0,0,0
+20ng,dvae,0.2860944139426903,0,1,0,1,1,0
+20ng,dvae,0.3716574301502446,1,1,0,0,0,0
+20ng,dvae,0.10686569980483274,1,1,0,1,1,0
+20ng,dvae,0.47924113528456774,0,0,0,0,1,0
+20ng,dvae,0.3303952745347445,1,1,0,0,0,0
+20ng,dvae,0.42508422599921253,0,0,0,0,0,0
+20ng,dvae,0.41984768456307325,0,0,0,0,0,0
+20ng,dvae,0.40716769678535675,0,0,0,0,0,0
+20ng,dvae,0.408620814860296,0,0,0,0,0,0
+20ng,dvae,0.11413587466631586,1,1,0,0,0,0
+20ng,dvae,0.12248643588268114,0,0,0,0,0,0
+20ng,dvae,0.23690636746040614,0,0,0,0,0,0
+20ng,dvae,0.4823283121857407,0,0,0,0,0,0
+20ng,dvae,0.186655726461628,1,1,0,1,1,0
+20ng,dvae,0.6961456071854104,1,0,0,1,1,1
+20ng,dvae,0.45522040363986493,0,0,0,0,0,0
+20ng,dvae,0.5989490902099036,1,1,0,1,1,0
+20ng,dvae,0.49535041831334065,1,1,0,1,1,1
+20ng,dvae,0.5920919493249797,1,0,0,0,0,0
+20ng,dvae,0.769231707047005,1,1,1,1,1,1
+20ng,dvae,0.4833977306079935,1,0,0,0,0,1
+20ng,etm,0.07848605495278796,0,0,1,0,0,1
+20ng,etm,0.21328584909712833,0,0,0,1,0,0
+20ng,etm,0.16799076962198772,0,0,0,0,0,0
+20ng,etm,0.12183382621090537,0,0,0,0,0,0
+20ng,etm,0.19279748866717403,1,1,1,1,1,1
+20ng,etm,0.09666872272011014,1,0,0,1,1,1
+20ng,etm,0.08767949691275889,0,0,0,0,0,0
+20ng,etm,0.11020999746869907,0,0,0,0,0,0
+20ng,etm,0.24171168500496862,0,0,0,0,0,0
+20ng,etm,0.20521845532961316,0,1,1,1,1,1
+20ng,etm,0.27272151636049047,1,1,0,1,1,0
+20ng,etm,0.23137416387552698,1,1,1,1,1,1
+20ng,etm,0.1458776787327446,1,1,1,1,1,1
+20ng,etm,0.15639292061352442,0,0,0,0,0,1
+20ng,etm,0.09666872272011014,0,0,0,0,0,0
+20ng,etm,0.08791069303364062,0,0,0,0,0,0
+20ng,etm,0.13429349867431195,1,1,1,1,1,1
+20ng,etm,0.10610405727202535,0,0,1,0,0,0
+20ng,etm,0.39672121139939837,0,0,0,0,0,0
+20ng,etm,0.11177041401400996,0,0,0,0,0,0
+20ng,etm,0.14383479078798841,1,1,1,1,1,1
+20ng,etm,0.08767949691275888,0,0,0,0,0,0
+20ng,etm,0.34523811142100447,1,1,1,1,1,1
+20ng,etm,0.3999433091980417,0,0,0,1,0,0
+20ng,etm,0.1782174324788786,1,1,0,0,0,1
+20ng,etm,0.23780285865870993,0,0,0,0,0,0
+20ng,etm,0.08767949691275892,0,0,0,0,0,0
+20ng,etm,0.1467177646091656,1,1,1,1,1,1
+20ng,etm,0.08767949691275892,1,0,1,1,1,1
+20ng,etm,0.41888563195253214,1,1,1,1,1,1
+20ng,etm,0.08571979438381021,0,0,0,0,0,0
+20ng,etm,0.17344625865203767,0,0,0,0,0,0
+20ng,etm,0.3053923176182822,0,0,0,0,0,0
+20ng,etm,0.16360972802369766,1,1,1,1,1,1
+20ng,etm,0.16394144519225215,1,1,1,1,1,1
+20ng,etm,0.1175482300339774,0,0,0,0,0,0
+20ng,etm,0.15265430531494578,0,0,0,0,0,0
+20ng,etm,0.09517116802114771,0,0,0,0,0,0
+20ng,etm,0.12178250895220842,0,0,0,0,0,0
+20ng,etm,0.11908460298883951,0,0,0,0,0,0
+20ng,etm,0.19835765148176882,1,1,1,1,1,1
+20ng,etm,0.2603634061718126,0,0,0,0,0,0
+20ng,etm,0.277454810526896,1,1,1,1,1,1
+20ng,etm,0.18905821329673844,1,1,0,0,0,0
+20ng,etm,0.20408578488080548,0,0,0,0,0,0
+20ng,etm,0.1963516393456708,1,1,1,1,1,1
+20ng,etm,0.37255350028093653,1,1,0,1,1,0
+20ng,etm,0.15490547257309728,0,0,1,0,0,0
+20ng,etm,0.06754457583744143,1,0,0,0,0,0
+20ng,etm,0.34019381396472376,1,1,1,1,1,1
+wiki,mallet,0.2689839733873824,0,0,0,0,0,0
+wiki,mallet,0.233509639067077,0,0,0,0,0,0
+wiki,mallet,0.2091672244613921,1,1,1,1,1,1
+wiki,mallet,0.24196489806173224,1,1,1,1,1,1
+wiki,mallet,0.21101726425596937,1,1,1,0,0,0
+wiki,mallet,0.14360179775585216,0,1,0,0,0,0
+wiki,mallet,0.3220178892577338,1,1,1,1,0,1
+wiki,mallet,0.2533548710417281,1,1,1,1,1,1
+wiki,mallet,0.20018029382644514,1,1,1,1,1,1
+wiki,mallet,0.39153268710299616,1,1,0,1,1,0
+wiki,mallet,0.3161962487259974,1,1,1,1,1,1
+wiki,mallet,0.1793602730550851,0,1,1,1,1,1
+wiki,mallet,0.2391022155379864,1,1,1,1,1,1
+wiki,mallet,0.2634998223646122,1,1,0,1,1,1
+wiki,mallet,0.24252844038457835,1,1,0,1,1,1
+wiki,mallet,0.2959826535218742,1,1,1,1,1,1
+wiki,mallet,0.4000450322417536,1,1,0,1,1,1
+wiki,mallet,0.264452320626896,1,1,0,0,0,0
+wiki,mallet,0.13067652448201164,1,1,1,1,1,1
+wiki,mallet,0.1558377693435571,0,1,0,0,0,0
+wiki,mallet,0.20791799494943936,0,1,0,1,1,0
+wiki,mallet,0.13918009957582778,1,1,1,1,1,1
+wiki,mallet,0.2574165964663429,0,0,0,0,0,0
+wiki,mallet,0.1738353653525663,1,1,1,1,1,1
+wiki,mallet,0.19054039886779456,1,1,1,1,1,1
+wiki,mallet,0.29871310582168886,1,1,1,1,1,1
+wiki,mallet,0.16920582648998816,1,1,1,1,1,1
+wiki,mallet,0.27540949931196845,1,0,0,0,0,0
+wiki,mallet,0.13486620395429627,1,1,0,1,0,0
+wiki,mallet,0.1523359110603592,1,0,0,0,0,0
+wiki,mallet,0.3339516490571772,1,1,1,1,1,1
+wiki,mallet,0.1759393397970967,1,1,0,0,0,1
+wiki,mallet,0.07472710712092726,0,0,0,0,0,0
+wiki,mallet,0.18055466664549447,1,1,1,1,1,1
+wiki,mallet,0.32633642438567984,1,1,0,1,1,0
+wiki,mallet,0.22545362784105932,0,0,0,0,0,0
+wiki,mallet,0.17323505371047257,0,0,0,0,0,0
+wiki,mallet,0.15773617968710743,0,0,0,0,0,0
+wiki,mallet,0.16755707256328778,0,0,0,0,0,0
+wiki,mallet,0.2572597350061184,0,1,0,0,0,0
+wiki,mallet,0.22425789198759724,1,1,1,1,1,1
+wiki,mallet,0.16858526377784225,1,1,1,1,1,1
+wiki,mallet,0.31286288749216973,0,0,0,0,0,1
+wiki,mallet,0.2652862471717779,1,0,0,1,1,1
+wiki,mallet,0.2644593428594425,0,0,1,1,1,0
+wiki,mallet,0.21055883414981424,0,0,0,0,0,0
+wiki,mallet,0.24193722515529006,1,1,0,1,1,1
+wiki,mallet,0.30243104215986205,1,1,1,1,1,1
+wiki,mallet,0.21706699924472342,1,1,1,1,1,1
+wiki,mallet,0.455008584010293,1,1,1,1,1,0
+wiki,dvae,0.7217955074040513,1,1,0,1,1,1
+wiki,dvae,0.8449497046071232,1,1,1,1,1,1
+wiki,dvae,0.2664973793660157,1,1,1,1,1,1
+wiki,dvae,0.41015537870684277,1,0,1,0,0,1
+wiki,dvae,0.44793629011265856,0,0,0,0,0,0
+wiki,dvae,0.5068716250918338,1,1,1,1,1,1
+wiki,dvae,0.5869756393954607,1,1,1,1,1,1
+wiki,dvae,0.40779636218595866,1,1,0,1,1,0
+wiki,dvae,0.32365262852428495,0,0,0,0,0,0
+wiki,dvae,0.3706530395260478,1,1,0,1,1,1
+wiki,dvae,0.6191137102459842,1,1,1,1,1,1
+wiki,dvae,0.5194978794532137,1,1,1,1,1,1
+wiki,dvae,0.24563058030745483,1,1,1,1,1,1
+wiki,dvae,0.626175312158884,1,1,0,1,1,1
+wiki,dvae,0.32396256683781666,1,1,0,1,0,0
+wiki,dvae,0.5739839565518935,1,1,1,1,1,1
+wiki,dvae,0.5284204235749705,1,1,1,1,1,1
+wiki,dvae,0.5312909044467004,1,1,1,1,1,1
+wiki,dvae,0.36389183117374196,1,1,1,1,1,1
+wiki,dvae,0.2464605956066343,0,0,0,0,0,0
+wiki,dvae,0.36255692884689855,1,1,0,1,1,1
+wiki,dvae,0.7123711703156873,1,1,1,1,1,1
+wiki,dvae,0.2751115356544055,0,0,0,0,0,0
+wiki,dvae,0.4693476885444469,1,1,1,1,1,1
+wiki,dvae,0.6086562465660189,1,1,1,1,1,1
+wiki,dvae,0.7296206567911837,1,1,1,1,1,1
+wiki,dvae,0.539164685906222,1,0,0,0,0,0
+wiki,dvae,0.4556988939400416,1,1,1,1,1,1
+wiki,dvae,0.5171653657991538,0,0,0,0,0,0
+wiki,dvae,0.35983835287120886,1,1,1,1,1,1
+wiki,dvae,0.4876362794776332,1,1,1,1,1,1
+wiki,dvae,0.6215960079286554,1,1,0,1,1,0
+wiki,dvae,0.6037290051569684,1,1,1,1,1,1
+wiki,dvae,0.5247944845051506,1,1,1,1,1,1
+wiki,dvae,0.46251880155546765,1,1,1,1,1,1
+wiki,dvae,0.47585938780808573,1,0,0,0,0,0
+wiki,dvae,0.22948938920078743,1,0,0,0,0,0
+wiki,dvae,0.5422161524901976,1,1,1,1,1,1
+wiki,dvae,0.47674696602922834,1,1,1,1,1,1
+wiki,dvae,0.462252664781505,1,1,1,1,1,1
+wiki,dvae,0.48974542792852566,1,1,1,1,1,1
+wiki,dvae,0.5157172252590485,1,1,1,1,1,1
+wiki,dvae,0.4766109503971607,0,0,0,0,0,0
+wiki,dvae,0.3748874274113518,1,1,0,1,1,1
+wiki,dvae,0.33051162921453836,1,1,0,1,1,0
+wiki,dvae,0.6729153603615396,0,0,0,0,0,0
+wiki,dvae,0.5285041126988406,1,0,0,0,0,0
+wiki,dvae,0.16229422961833365,1,1,1,1,1,1
+wiki,dvae,0.5807691921228599,1,1,1,1,1,1
+wiki,dvae,0.3852226832817468,1,1,1,1,1,1
+wiki,etm,0.0313438014365729,1,1,1,1,1,1
+wiki,etm,0.0153669444858178,0,0,0,0,0,0
+wiki,etm,0.1038583235647836,0,0,0,0,0,0
+wiki,etm,0.0031907604666791,1,1,1,1,1,1
+wiki,etm,0.09004764195663488,0,0,0,0,0,0
+wiki,etm,0.1020442099567863,0,0,1,0,0,0
+wiki,etm,0.0329307385670234,0,0,0,0,0,0
+wiki,etm,0.1963455412871243,0,0,0,0,0,0
+wiki,etm,0.11808804695266718,0,0,0,0,0,0
+wiki,etm,0.1181861855398127,0,1,1,1,1,1
+wiki,etm,0.1915341149672391,0,0,0,0,0,0
+wiki,etm,0.0146229273561867,0,0,0,0,0,0
+wiki,etm,0.06026206697464678,0,0,0,0,0,0
+wiki,etm,0.1304279441290738,0,0,0,0,0,0
+wiki,etm,0.2155043451835679,0,0,0,0,0,0
+wiki,etm,0.0195869916928982,0,0,0,0,0,0
+wiki,etm,0.1246862697547656,1,0,0,0,0,0
+wiki,etm,0.1372769597127376,0,0,0,0,0,0
+wiki,etm,0.0892123696568991,1,1,0,0,0,0
+wiki,etm,0.1597733231789676,1,1,0,0,0,0
+wiki,etm,0.0760781555234959,0,0,0,0,0,0
+wiki,etm,0.21675332559834665,0,0,0,0,0,0
+wiki,etm,0.19853561068392747,0,0,0,0,0,0
+wiki,etm,0.1348618453589524,0,0,0,0,0,0
+wiki,etm,0.0796914371880895,0,0,0,0,0,0
+wiki,etm,0.04320981999849475,1,0,1,0,0,1
+wiki,etm,0.1472156294768876,1,1,0,0,0,0
+wiki,etm,0.20602537824887898,0,1,1,0,0,1
+wiki,etm,0.15718236652349887,0,0,0,0,0,0
+wiki,etm,0.05120785588594204,0,0,0,1,1,1
+wiki,etm,0.03070653989583847,1,1,1,1,1,1
+wiki,etm,0.02122091899303859,0,0,0,0,0,0
+wiki,etm,0.09879154828384014,0,0,0,0,0,0
+wiki,etm,0.1123475455945978,1,1,0,0,0,0
+wiki,etm,0.03771398379587894,0,0,0,0,0,0
+wiki,etm,0.02035322469349827,0,0,0,0,0,0
+wiki,etm,0.16596275540234956,0,0,0,0,0,0
+wiki,etm,0.04964237219475874,0,0,0,0,0,0
+wiki,etm,0.09764010974984784,0,0,1,0,0,1
+wiki,etm,0.11736309724938788,0,0,0,0,0,0
+wiki,etm,0.098865376649869,0,0,0,0,0,0
+wiki,etm,0.12707757584985782,0,0,1,0,0,0
+wiki,etm,0.22095588024878768,1,1,1,1,1,1
+wiki,etm,0.25567923368578593,1,1,0,0,0,0
+wiki,etm,0.1218720838853789,0,1,0,0,0,0
+wiki,etm,0.09673317495876894,1,1,0,0,0,0
+wiki,etm,0.16072450969843788,0,0,0,0,0,0
+wiki,etm,0.08069220479349892,1,1,0,1,0,0
+wiki,etm,0.2401680099847689,0,0,0,0,0,0
+wiki,etm,0.0967331749039874,0,0,0,0,0,0
diff --git a/transformer-tests/metrics.txt b/transformer-tests/metrics.txt
new file mode 100644
index 0000000000000000000000000000000000000000..56756e456e053568a81e5bf3ffbb534a51ab3c5c
--- /dev/null
+++ b/transformer-tests/metrics.txt
@@ -0,0 +1,20 @@
+{'task': 'intrusion', 'dataset': '20ng', 'prompt': 'p1_v6', 'mean': 0.44, 'var': 0.24640000000000034, 'spear_rho': 0.20595229788917765, 'spear_p': 0.011456548250679422, 'pear_rho': 0.19401274319485065, 'pear_p': 0.01736281772663954}
+{'task': 'intrusion', 'dataset': '20ng', 'prompt': 'p2_v6', 'mean': 0.41333333333333333, 'var': 0.2424888888888887, 'spear_rho': 0.15289097550689398, 'spear_p': 0.061782013734402404, 'pear_rho': 0.09275498820681638, 'pear_p': 0.2589213039856191}
+{'task': 'intrusion', 'dataset': '20ng', 'prompt': 'p3_v6', 'mean': 0.30666666666666664, 'var': 0.2126222222222229, 'spear_rho': 0.0612703916531885, 'spear_p': 0.45637531030997347, 'pear_rho': 0.017889733672921668, 'pear_p': 0.8279838448169439}
+{'task': 'intrusion', 'dataset': '20ng', 'prompt': 'p4_v6', 'mean': 0.4, 'var': 0.23999999999999966, 'spear_rho': 0.15022464626313464, 'spear_p': 0.06651986084785377, 'pear_rho': 0.10099484660948387, 'pear_p': 0.21879874361895862}
+{'task': 'intrusion', 'dataset': '20ng', 'prompt': 'p5_v6', 'mean': 0.38, 'var': 0.23559999999999984, 'spear_rho': 0.1635158818392516, 'spear_p': 0.04556756871490956, 'pear_rho': 0.125844021716626, 'pear_p': 0.12491117691758058}
+{'task': 'intrusion', 'dataset': '20ng', 'prompt': 'p6_v6', 'mean': 0.3466666666666667, 'var': 0.2264888888888885, 'spear_rho': 0.09414309084210924, 'spear_p': 0.2518306470608761, 'pear_rho': 0.08726140169503305, 'pear_p': 0.2883192016594074}
+{'task': 'intrusion', 'dataset': 'wiki', 'prompt': 'p1_v6', 'mean': 0.6, 'var': 0.2399999999999998, 'spear_rho': 0.4833575100484224, 'spear_p': 3.7242168017583847e-10, 'pear_rho': 0.4527196510715345, 'pear_p': 6.00671893065629e-09}
+{'task': 'intrusion', 'dataset': 'wiki', 'prompt': 'p2_v6', 'mean': 0.5866666666666667, 'var': 0.24248888888888875, 'spear_rho': 0.39144987905779, 'spear_p': 7.297496346825347e-07, 'pear_rho': 0.3683953287606051, 'pear_p': 3.511993085754505e-06}
+{'task': 'intrusion', 'dataset': 'wiki', 'prompt': 'p3_v6', 'mean': 0.4266666666666667, 'var': 0.244622222222222, 'spear_rho': 0.30631262775434553, 'spear_p': 0.00013772569815159272, 'pear_rho': 0.30872339027182294, 'pear_p': 0.00012115923949052753}
+{'task': 'intrusion', 'dataset': 'wiki', 'prompt': 'p4_v6', 'mean': 0.5133333333333333, 'var': 0.24982222222222242, 'spear_rho': 0.47853474942211577, 'spear_p': 5.87642512120306e-10, 'pear_rho': 0.45053084145896294, 'pear_p': 7.251661580397554e-09}
+{'task': 'intrusion', 'dataset': 'wiki', 'prompt': 'p5_v6', 'mean': 0.4866666666666667, 'var': 0.24982222222222192, 'spear_rho': 0.4914722836839303, 'spear_p': 1.7010562192108877e-10, 'pear_rho': 0.4676837652123441, 'pear_p': 1.5984195458030146e-09}
+{'task': 'intrusion', 'dataset': 'wiki', 'prompt': 'p6_v6', 'mean': 0.48, 'var': 0.24960000000000007, 'spear_rho': 0.3966194541646198, 'spear_p': 5.045452582834669e-07, 'pear_rho': 0.382881650598807, 'pear_p': 1.327039804041509e-06}
+{'task': 'rating', 'dataset': '20ng', 'version': 'v1', 'mean': 1.8666666666666667, 'var': 0.8222222222222232, 'spear_rho': 0.07616232334429132, 'spear_p': 0.35427074270785164, 'pear_rho': 0.035054092213827506, 'pear_p': 0.6702079407070977}
+{'task': 'rating', 'dataset': '20ng', 'version': 'v2', 'mean': 1.8066666666666666, 'var': 0.8226222222222224, 'spear_rho': 0.05681368242063749, 'spear_p': 0.4898421896302837, 'pear_rho': 0.030604375089933872, 'pear_p': 0.7100588605739637}
+{'task': 'rating', 'dataset': '20ng', 'version': 'v3', 'mean': 1.8933333333333333, 'var': 0.8952888888888877, 'spear_rho': 0.027835779497273512, 'spear_p': 0.7352645540591634, 'pear_rho': -0.027969794962175873, 'pear_p': 0.7340376818463089}
+{'task': 'rating', 'dataset': '20ng', 'version': 'v4', 'mean': 1.8, 'var': 0.8666666666666665, 'spear_rho': 0.05716221668528551, 'spear_p': 0.4871786818978283, 'pear_rho': 0.012676575519977178, 'pear_p': 0.8776388257346328}
+{'task': 'rating', 'dataset': 'wiki', 'version': 'v1', 'mean': 2.2266666666666666, 'var': 0.9219555555555548, 'spear_rho': 0.7237455678867271, 'spear_p': 1.2876403780202825e-25, 'pear_rho': 0.6473953553083143, 'pear_p': 3.494095744754713e-19}
+{'task': 'rating', 'dataset': 'wiki', 'version': 'v2', 'mean': 2.1533333333333333, 'var': 0.929822222222223, 'spear_rho': 0.7144361265209603, 'spear_p': 1.01483625364132e-24, 'pear_rho': 0.6507444376293546, 'pear_p': 1.9939884597209992e-19}
+{'task': 'rating', 'dataset': 'wiki', 'version': 'v3', 'mean': 2.2533333333333334, 'var': 0.9224888888888897, 'spear_rho': 0.7139329131603146, 'spear_p': 1.131989644062808e-24, 'pear_rho': 0.6343912384934123, 'pear_p': 2.8913024599384382e-18}
+{'task': 'rating', 'dataset': 'wiki', 'version': 'v4', 'mean': 2.2066666666666666, 'var': 0.9372888888888883, 'spear_rho': 0.684405449309189, 'spear_p': 4.679174840200377e-22, 'pear_rho': 0.6042466262354305, 'pear_p': 2.6881240353952396e-16}
diff --git a/transformer-tests/rating.py b/transformer-tests/rating.py
index d7380c6a52cabd867c36393a66b4e1ec3a0f3aab..876c43df21bd1e501e639dcb28dc4b4a087d3368 100644
--- a/transformer-tests/rating.py
+++ b/transformer-tests/rating.py
@@ -12,7 +12,7 @@ class Rating():
         self.num_topics = num_topics
         self.path_save = path_save
         self.file_path = file_path
-        self.prompt_name = 'rating_p4_v4'
+        self.prompt_name = 'rating_p3_v1'
 
 
     def run_gpt3(self, prompts):
@@ -28,34 +28,29 @@ class Rating():
 
 
     def create_prompt(self):
+        '''
+        p1: Rate how related the following terms are to each other as 'very related', 'somewhat related' or 'not related'
+        p2: Rate how related the following terms are to each other in a range from 1 to 3
+        '''
 
-        #topics = self.topics
         prompts = []
         topics = [topic.split() for topic in self.topics]
         for i in range(len(self.topics)):
-            # list1 = "['file', 'window', 'problem', 'run', 'system', 'program', 'font', 'work', 'win', 'change']"
-            # list2 = "['chip', 'clipper', 'phone', 'key', 'encryption', 'government', 'system', 'write', 'nsa', 'communication']"
+            list1 = "['file', 'window', 'problem', 'run', 'system', 'program', 'font', 'work', 'win', 'change']"
+            list2 = "['chip', 'clipper', 'phone', 'key', 'encryption', 'government', 'system', 'write', 'nsa', 'communication']"
             list_ten_terms = str(topics[i][:10])
 
-            # list1 = '[file, window, problem, run, system, program, font, work, win, change]'
-            # list2 = '[chip, clipper, phone, key, encryption, government, system, write, nsa, communication]'
-            # list_ten_terms = list_ten_terms.replace("'", "")
-
-            # list1 = "['file', 'window', 'problem', 'run', 'system', 'program', 'font', 'work', 'win', 'change']"
-            # list2 = "['chip', 'clipper', 'phone', 'key', 'encryption', 'government', 'system', 'write', 'nsa', 'communication']"
+            # list1 = list1.replace("[", "").replace("]", "")
+            # list2 = list2.replace("[", "").replace("]", "")
             # list_ten_terms = list_ten_terms.replace("[", "").replace("]", "")
 
-            list1 = 'file, window, problem, run, system, program, font, work, win, change'
-            list2 = 'chip, clipper, phone, key, encryption, government, system, write, nsa, communication'
-            list_ten_terms = list_ten_terms.replace("'", "").replace("[", "").replace("]", "")
-
+            # list1 = list1.replace("'", "")
+            # list2 = list2.replace("'", "")
+            # list_ten_terms = list_ten_terms.replace("'", "")
 
-            # prompts.append("Rate how related the following terms are to each other " \
-            #          "as 'very related', 'somewhat related' or 'not related': " + list1 + "\n" \
-            #          "Answer: Very related\n\nRate how related the following terms are to each other " \
-            #          "as 'very related', 'somewhat related' or 'not related': " + list2 + "\n"
-            #          "Answer: Somewhat related\n\nRate how related the following terms are to each other " \
-            #          "as 'very related', 'somewhat related' or 'not related': " + list_ten_terms + "\nAnswer: ")
+            # list1 = 'file, window, problem, run, system, program, font, work, win, change'
+            # list2 = 'chip, clipper, phone, key, encryption, government, system, write, nsa, communication'
+            # list_ten_terms = list_ten_terms.replace("'", "").replace("[", "").replace("]", "")
 
             prompts.append("Rate how related the following terms are to each other " \
                            "as '3-very related', '2-somewhat related' or '1-not related': " + list1 + "\n" \
@@ -64,8 +59,7 @@ class Rating():
                            "Answer: 2\n\nRate how related the following terms are to each other " \
                            "as '3-very related', '2-somewhat related' or '1-not related': " + list_ten_terms + "\nAnswer: ")
 
-            # prompts.append("Rate how related the following terms are to each other " \
-            #                "in a range from 1 to 3: " + list_ten_terms)
+
 
         with open(self.path_save + self.prompt_name + '.txt', 'w') as f:
             for i in range(self.num_topics):
@@ -83,12 +77,16 @@ class Rating():
         return npmis
 
 
-    def calculate_metrics(self):
-
+    def load_responses(self):
         with open(self.path_save + self.prompt_name + '_gpt3.txt') as file:
             responses = file.readlines()
-        responses = [int(term.replace('\n', '').replace(' ', '')) for term in responses]
+        numeric_response = [int(term.replace('\n', '').replace(' ', '')) for term in responses]
+
+        return numeric_response
 
+
+    def calculate_metrics(self):
+        responses = self.load_responses()
         # calculate accuracy
         acc = np.mean(responses)
         variance = np.var(responses)
@@ -108,7 +106,7 @@ class Rating():
 
     def run_rating(self):
         print("Creation of prompts for GPT-3...")
-        #self.create_prompt()
-        print(self.calculate_metrics())
+        self.create_prompt()
+        self.calculate_metrics()
 
 
diff --git a/transformer-tests/rating_metrics.csv b/transformer-tests/rating_metrics.csv
new file mode 100644
index 0000000000000000000000000000000000000000..07856ef1c18be43ba1d54acb62ae3188d701e778
--- /dev/null
+++ b/transformer-tests/rating_metrics.csv
@@ -0,0 +1,301 @@
+dataset,model,npmi,v1,v2,v3,v4
+20ng,mallet,0.1314275058044984,1,1,1,1
+20ng,mallet,0.12868722938942015,1,1,1,1
+20ng,mallet,0.35750602263515147,3,3,3,3
+20ng,mallet,0.11305819494914734,1,1,1,1
+20ng,mallet,0.11484206083907826,1,1,1,1
+20ng,mallet,0.11176888846533804,1,1,1,1
+20ng,mallet,0.12732699467385925,2,2,2,1
+20ng,mallet,0.2287394936063482,1,1,1,1
+20ng,mallet,0.44473939676352114,3,3,3,3
+20ng,mallet,0.24127019674935196,3,3,3,3
+20ng,mallet,0.32284272065971203,3,3,3,3
+20ng,mallet,0.43142111410155054,3,3,3,3
+20ng,mallet,0.2910513037578202,3,3,3,1
+20ng,mallet,0.18014990648358223,3,3,3,2
+20ng,mallet,0.32188953772530043,1,1,1,1
+20ng,mallet,0.6603008717784691,3,3,3,3
+20ng,mallet,0.30699249515929244,3,2,3,2
+20ng,mallet,0.13326988122411804,3,3,3,3
+20ng,mallet,0.6421268827929683,1,1,1,1
+20ng,mallet,0.30043303281639544,3,3,3,3
+20ng,mallet,0.28048604272718247,3,3,3,3
+20ng,mallet,0.16676331019961838,2,2,2,2
+20ng,mallet,0.3726503665794892,3,3,3,3
+20ng,mallet,0.3765487569290983,3,3,3,3
+20ng,mallet,0.23026012345157132,1,1,1,1
+20ng,mallet,0.13333804691738643,2,1,3,2
+20ng,mallet,0.20357481024945415,1,1,1,1
+20ng,mallet,0.35850791330099435,2,2,2,2
+20ng,mallet,0.38752138070365977,3,3,3,3
+20ng,mallet,0.0799560563666676,1,1,1,1
+20ng,mallet,0.17852991901351475,2,2,2,2
+20ng,mallet,0.2742371586926216,3,3,3,3
+20ng,mallet,0.3715196209666211,2,2,2,2
+20ng,mallet,0.42261689038903927,3,3,3,3
+20ng,mallet,0.38192610751506706,3,3,3,3
+20ng,mallet,0.12258062217387942,3,3,3,3
+20ng,mallet,0.15059222629790764,1,1,1,1
+20ng,mallet,0.2023676323552062,2,1,2,1
+20ng,mallet,0.7693309879198367,1,1,1,1
+20ng,mallet,0.43051248340180165,1,1,1,1
+20ng,mallet,0.20686042222320672,3,3,3,3
+20ng,mallet,0.25629312128444554,3,3,3,3
+20ng,mallet,0.2010377483499159,1,1,1,1
+20ng,mallet,0.14900240808594048,1,1,1,1
+20ng,mallet,0.10959274106419682,2,2,2,2
+20ng,mallet,0.21311907647038422,1,1,1,1
+20ng,mallet,0.1879680802758673,3,2,1,1
+20ng,mallet,0.20926676826294122,3,3,3,3
+20ng,mallet,0.15470010994497765,3,3,3,2
+20ng,mallet,0.1706810915779308,2,1,2,2
+20ng,dvae,0.24867697774022374,1,1,1,1
+20ng,dvae,0.3396118005925878,1,1,1,1
+20ng,dvae,0.29970309841727477,1,1,1,1
+20ng,dvae,0.23866633667824944,1,1,1,1
+20ng,dvae,0.3699640259380764,1,1,1,1
+20ng,dvae,0.42240324296068005,3,2,3,3
+20ng,dvae,0.8874487186560454,1,1,1,1
+20ng,dvae,0.128691716880522,1,1,1,1
+20ng,dvae,0.37589863667431506,1,1,1,1
+20ng,dvae,0.6238240724850099,1,1,1,1
+20ng,dvae,0.17956630920866357,1,1,1,1
+20ng,dvae,0.29258062003294744,3,3,3,3
+20ng,dvae,0.3877803368148381,3,3,3,3
+20ng,dvae,0.33284345912394303,3,3,3,3
+20ng,dvae,0.17428405447076523,1,1,1,1
+20ng,dvae,0.4708279864428822,1,1,1,1
+20ng,dvae,0.33234795901646436,1,1,1,1
+20ng,dvae,0.2659239621234882,1,1,1,1
+20ng,dvae,0.4472987543185158,3,3,3,3
+20ng,dvae,0.18977108703994489,1,1,1,1
+20ng,dvae,0.21801183330846072,1,1,1,1
+20ng,dvae,0.45164395443930255,2,1,3,3
+20ng,dvae,0.5162468786823781,1,1,1,1
+20ng,dvae,0.4219132846940917,3,3,3,3
+20ng,dvae,0.37711305638174364,1,1,1,1
+20ng,dvae,0.5037645757835705,1,1,1,1
+20ng,dvae,0.36761746934438233,1,1,1,1
+20ng,dvae,0.6534066162568825,3,3,3,3
+20ng,dvae,0.09283550832829766,1,1,1,1
+20ng,dvae,0.2860944139426903,1,1,1,1
+20ng,dvae,0.3716574301502446,1,1,1,1
+20ng,dvae,0.10686569980483274,1,1,1,1
+20ng,dvae,0.47924113528456774,3,3,3,3
+20ng,dvae,0.3303952745347445,2,1,3,1
+20ng,dvae,0.42508422599921253,1,1,1,1
+20ng,dvae,0.41984768456307325,1,1,1,1
+20ng,dvae,0.40716769678535675,1,1,1,1
+20ng,dvae,0.408620814860296,1,1,1,1
+20ng,dvae,0.11413587466631586,1,1,1,1
+20ng,dvae,0.12248643588268114,1,1,1,1
+20ng,dvae,0.23690636746040614,1,1,1,1
+20ng,dvae,0.4823283121857407,1,1,1,1
+20ng,dvae,0.186655726461628,1,1,1,1
+20ng,dvae,0.6961456071854104,1,1,1,1
+20ng,dvae,0.45522040363986493,1,1,1,1
+20ng,dvae,0.5989490902099036,1,1,1,1
+20ng,dvae,0.49535041831334065,1,1,1,1
+20ng,dvae,0.5920919493249797,1,1,1,1
+20ng,dvae,0.769231707047005,3,3,1,1
+20ng,dvae,0.4833977306079935,3,3,3,3
+20ng,etm,0.07848605495278796,1,1,1,1
+20ng,etm,0.21328584909712833,1,1,1,1
+20ng,etm,0.16799076962198772,1,1,2,1
+20ng,etm,0.12183382621090537,2,2,2,2
+20ng,etm,0.19279748866717403,3,2,3,3
+20ng,etm,0.09666872272011014,2,2,3,3
+20ng,etm,0.08767949691275889,3,3,3,3
+20ng,etm,0.11020999746869907,1,1,1,1
+20ng,etm,0.24171168500496862,3,3,3,3
+20ng,etm,0.20521845532961316,3,3,3,3
+20ng,etm,0.27272151636049047,3,3,3,3
+20ng,etm,0.23137416387552698,1,1,1,1
+20ng,etm,0.1458776787327446,1,1,1,1
+20ng,etm,0.15639292061352442,3,3,3,3
+20ng,etm,0.09666872272011014,2,2,2,2
+20ng,etm,0.08791069303364062,3,3,3,3
+20ng,etm,0.13429349867431195,2,2,2,1
+20ng,etm,0.10610405727202535,1,1,1,1
+20ng,etm,0.39672121139939837,2,2,3,3
+20ng,etm,0.11177041401400996,1,1,3,3
+20ng,etm,0.14383479078798841,3,3,3,3
+20ng,etm,0.08767949691275888,3,3,3,3
+20ng,etm,0.34523811142100447,3,3,3,3
+20ng,etm,0.3999433091980417,3,3,3,3
+20ng,etm,0.1782174324788786,2,3,3,3
+20ng,etm,0.23780285865870993,3,3,3,3
+20ng,etm,0.08767949691275892,3,3,3,3
+20ng,etm,0.1467177646091656,2,2,1,1
+20ng,etm,0.08767949691275892,2,2,3,3
+20ng,etm,0.41888563195253214,3,3,3,3
+20ng,etm,0.08571979438381021,3,3,3,3
+20ng,etm,0.17344625865203767,3,3,3,3
+20ng,etm,0.3053923176182822,1,1,1,1
+20ng,etm,0.16360972802369766,2,2,1,1
+20ng,etm,0.16394144519225215,1,1,1,1
+20ng,etm,0.1175482300339774,1,1,1,1
+20ng,etm,0.15265430531494578,1,1,1,1
+20ng,etm,0.09517116802114771,3,3,3,2
+20ng,etm,0.12178250895220842,1,1,1,1
+20ng,etm,0.11908460298883951,1,1,1,1
+20ng,etm,0.19835765148176882,1,1,1,1
+20ng,etm,0.2603634061718126,1,1,1,1
+20ng,etm,0.277454810526896,3,3,3,3
+20ng,etm,0.18905821329673844,2,2,1,1
+20ng,etm,0.20408578488080548,2,2,3,3
+20ng,etm,0.1963516393456708,1,1,2,1
+20ng,etm,0.37255350028093653,2,1,1,1
+20ng,etm,0.15490547257309728,2,2,2,2
+20ng,etm,0.06754457583744143,1,1,1,1
+20ng,etm,0.34019381396472376,3,3,3,3
+wiki,mallet,0.2689839733873824,3,3,3,3
+wiki,mallet,0.233509639067077,3,3,3,3
+wiki,mallet,0.2091672244613921,3,3,3,3
+wiki,mallet,0.24196489806173224,3,3,3,3
+wiki,mallet,0.21101726425596937,3,2,3,3
+wiki,mallet,0.14360179775585216,3,3,3,1
+wiki,mallet,0.3220178892577338,3,3,3,3
+wiki,mallet,0.2533548710417281,3,3,3,3
+wiki,mallet,0.20018029382644514,3,3,3,3
+wiki,mallet,0.39153268710299616,3,3,3,3
+wiki,mallet,0.3161962487259974,3,3,3,3
+wiki,mallet,0.1793602730550851,3,3,3,3
+wiki,mallet,0.2391022155379864,3,3,3,3
+wiki,mallet,0.2634998223646122,3,3,3,3
+wiki,mallet,0.24252844038457835,3,3,3,3
+wiki,mallet,0.2959826535218742,3,3,3,3
+wiki,mallet,0.4000450322417536,3,3,3,3
+wiki,mallet,0.264452320626896,3,3,3,3
+wiki,mallet,0.13067652448201164,3,3,3,3
+wiki,mallet,0.1558377693435571,2,3,3,3
+wiki,mallet,0.20791799494943936,2,2,3,3
+wiki,mallet,0.13918009957582778,3,2,3,3
+wiki,mallet,0.2574165964663429,3,3,3,3
+wiki,mallet,0.1738353653525663,3,3,3,3
+wiki,mallet,0.19054039886779456,3,3,3,3
+wiki,mallet,0.29871310582168886,3,3,3,3
+wiki,mallet,0.16920582648998816,3,2,3,3
+wiki,mallet,0.27540949931196845,3,3,3,3
+wiki,mallet,0.13486620395429627,3,3,3,3
+wiki,mallet,0.1523359110603592,3,1,3,3
+wiki,mallet,0.3339516490571772,3,3,3,3
+wiki,mallet,0.1759393397970967,3,3,3,3
+wiki,mallet,0.07472710712092726,1,1,1,1
+wiki,mallet,0.18055466664549447,3,3,3,3
+wiki,mallet,0.32633642438567984,3,3,3,3
+wiki,mallet,0.22545362784105932,1,1,3,3
+wiki,mallet,0.17323505371047257,3,3,3,3
+wiki,mallet,0.15773617968710743,3,3,3,3
+wiki,mallet,0.16755707256328778,3,3,3,3
+wiki,mallet,0.2572597350061184,3,3,3,3
+wiki,mallet,0.22425789198759724,3,2,3,3
+wiki,mallet,0.16858526377784225,3,3,3,3
+wiki,mallet,0.31286288749216973,3,3,3,3
+wiki,mallet,0.2652862471717779,3,3,3,3
+wiki,mallet,0.2644593428594425,3,3,3,3
+wiki,mallet,0.21055883414981424,3,3,3,3
+wiki,mallet,0.24193722515529006,3,3,3,3
+wiki,mallet,0.30243104215986205,3,3,3,3
+wiki,mallet,0.21706699924472342,3,3,3,3
+wiki,mallet,0.455008584010293,3,3,3,3
+wiki,dvae,0.7217955074040513,3,3,3,3
+wiki,dvae,0.8449497046071232,3,3,3,3
+wiki,dvae,0.2664973793660157,3,3,3,3
+wiki,dvae,0.41015537870684277,3,3,3,3
+wiki,dvae,0.44793629011265856,3,3,3,3
+wiki,dvae,0.5068716250918338,3,3,3,3
+wiki,dvae,0.5869756393954607,3,3,3,3
+wiki,dvae,0.40779636218595866,3,3,3,3
+wiki,dvae,0.32365262852428495,1,1,2,2
+wiki,dvae,0.3706530395260478,2,2,3,3
+wiki,dvae,0.6191137102459842,3,3,3,3
+wiki,dvae,0.5194978794532137,3,3,3,3
+wiki,dvae,0.24563058030745483,2,1,2,2
+wiki,dvae,0.626175312158884,3,3,3,3
+wiki,dvae,0.32396256683781666,3,3,3,3
+wiki,dvae,0.5739839565518935,3,3,3,3
+wiki,dvae,0.5284204235749705,3,3,3,3
+wiki,dvae,0.5312909044467004,3,3,3,3
+wiki,dvae,0.36389183117374196,3,3,3,3
+wiki,dvae,0.2464605956066343,3,2,3,2
+wiki,dvae,0.36255692884689855,1,1,1,1
+wiki,dvae,0.7123711703156873,3,3,3,3
+wiki,dvae,0.2751115356544055,3,3,3,3
+wiki,dvae,0.4693476885444469,3,3,3,3
+wiki,dvae,0.6086562465660189,3,3,3,3
+wiki,dvae,0.7296206567911837,3,3,3,3
+wiki,dvae,0.539164685906222,3,3,3,3
+wiki,dvae,0.4556988939400416,3,3,3,3
+wiki,dvae,0.5171653657991538,3,3,3,3
+wiki,dvae,0.35983835287120886,3,1,3,1
+wiki,dvae,0.4876362794776332,3,3,3,3
+wiki,dvae,0.6215960079286554,3,3,3,3
+wiki,dvae,0.6037290051569684,3,3,3,3
+wiki,dvae,0.5247944845051506,3,3,3,1
+wiki,dvae,0.46251880155546765,3,3,3,3
+wiki,dvae,0.47585938780808573,3,3,3,3
+wiki,dvae,0.22948938920078743,1,1,1,1
+wiki,dvae,0.5422161524901976,3,3,3,3
+wiki,dvae,0.47674696602922834,3,3,3,3
+wiki,dvae,0.462252664781505,3,3,3,3
+wiki,dvae,0.48974542792852566,3,3,3,3
+wiki,dvae,0.5157172252590485,3,3,3,1
+wiki,dvae,0.4766109503971607,3,3,3,3
+wiki,dvae,0.3748874274113518,3,1,1,3
+wiki,dvae,0.33051162921453836,3,3,3,3
+wiki,dvae,0.6729153603615396,3,3,3,3
+wiki,dvae,0.5285041126988406,3,3,3,3
+wiki,dvae,0.16229422961833365,1,1,1,1
+wiki,dvae,0.5807691921228599,3,3,3,3
+wiki,dvae,0.3852226832817468,3,3,3,3
+wiki,etm,0.0313438014365729,1,1,1,1
+wiki,etm,0.0153669444858178,1,1,1,1
+wiki,etm,0.1038583235647836,1,1,1,1
+wiki,etm,0.0031907604666791,1,1,1,1
+wiki,etm,0.09004764195663488,1,1,1,1
+wiki,etm,0.1020442099567863,1,1,1,1
+wiki,etm,0.0329307385670234,1,1,1,1
+wiki,etm,0.1963455412871243,1,1,1,1
+wiki,etm,0.11808804695266718,1,1,1,1
+wiki,etm,0.1181861855398127,1,1,1,1
+wiki,etm,0.1915341149672391,1,1,1,1
+wiki,etm,0.0146229273561867,1,1,1,1
+wiki,etm,0.06026206697464678,1,1,1,1
+wiki,etm,0.1304279441290738,1,1,1,1
+wiki,etm,0.2155043451835679,1,1,1,1
+wiki,etm,0.0195869916928982,1,1,1,1
+wiki,etm,0.1246862697547656,1,1,1,1
+wiki,etm,0.1372769597127376,1,1,1,1
+wiki,etm,0.0892123696568991,1,1,1,1
+wiki,etm,0.1597733231789676,1,1,1,1
+wiki,etm,0.0760781555234959,1,1,1,1
+wiki,etm,0.21675332559834665,1,1,1,1
+wiki,etm,0.19853561068392747,1,1,1,1
+wiki,etm,0.1348618453589524,1,1,1,1
+wiki,etm,0.0796914371880895,1,1,1,1
+wiki,etm,0.04320981999849475,1,1,1,1
+wiki,etm,0.1472156294768876,1,1,1,1
+wiki,etm,0.20602537824887898,1,1,1,1
+wiki,etm,0.15718236652349887,1,1,1,1
+wiki,etm,0.05120785588594204,1,1,1,1
+wiki,etm,0.03070653989583847,1,1,1,1
+wiki,etm,0.02122091899303859,1,1,1,1
+wiki,etm,0.09879154828384014,1,1,1,1
+wiki,etm,0.1123475455945978,1,1,1,1
+wiki,etm,0.03771398379587894,1,1,1,1
+wiki,etm,0.02035322469349827,1,1,1,1
+wiki,etm,0.16596275540234956,1,1,1,1
+wiki,etm,0.04964237219475874,1,1,1,1
+wiki,etm,0.09764010974984784,1,1,1,1
+wiki,etm,0.11736309724938788,1,1,1,1
+wiki,etm,0.098865376649869,1,1,1,1
+wiki,etm,0.12707757584985782,1,1,1,1
+wiki,etm,0.22095588024878768,1,1,1,1
+wiki,etm,0.25567923368578593,1,1,1,1
+wiki,etm,0.1218720838853789,1,1,1,1
+wiki,etm,0.09673317495876894,1,1,1,1
+wiki,etm,0.16072450969843788,1,1,1,1
+wiki,etm,0.08069220479349892,1,1,1,1
+wiki,etm,0.2401680099847689,1,1,1,1
+wiki,etm,0.0967331749039874,1,1,1,1