diff --git a/transformer-tests/gpt3.py b/transformer-tests/gpt3.py
new file mode 100644
index 0000000000000000000000000000000000000000..0131f06e3c502e6c9406cafd5232851f90238675
--- /dev/null
+++ b/transformer-tests/gpt3.py
@@ -0,0 +1,21 @@
+import openai
+
+def read_apikey():
+ api_key_path = './transformer-tests/gpt3-api-key.txt'
+ with open(api_key_path, 'r') as file:
+ api_key = file.read().replace('\n', '')
+ return api_key
+
+
+def gpt3(message):
+ openai.api_key = read_apikey()
+ response = openai.Completion.create(
+ engine="text-davinci-003",
+ prompt=message,
+ temperature=0,
+ max_tokens=60,
+ top_p=1,
+ frequency_penalty=0,
+ presence_penalty=0
+ )
+ return response.choices[0].text
\ No newline at end of file
diff --git a/transformer-tests/intrusion.py b/transformer-tests/intrusion.py
new file mode 100644
index 0000000000000000000000000000000000000000..d3dd064a779fdefc417210d6ad6da5f0e4c4cd67
--- /dev/null
+++ b/transformer-tests/intrusion.py
@@ -0,0 +1,242 @@
+import os
+import time
+import random
+import numpy as np
+from gpt3 import gpt3
+from scipy.stats import spearmanr, pearsonr
+#from bloom import bloom
+
+class Intrusion():
+ def __init__(self, topics, num_topics, path_save, file_path):
+ super(Intrusion, self).__init__()
+
+ ## define hyperparameters
+ self.topics = topics
+ self.num_topics = num_topics
+ self.path_save = path_save
+ self.file_path = file_path
+ self.prompt_name = 'intr_p1_v6'
+
+
+ def create_intruder(self, num_terms=5, sample_top_topic_terms=False):
+ """
+ topics_list: format is a list of dicts [
+ {"topic_id": 1, "terms": ["a","b","c"]},
+ {"topic_id": 2, "terms": ["a","b","c"]}
+ ]
+
+ n: The number of topics to sample
+ """
+ # sample_top_terms = 10
+ # topic_intruder = []
+ # # Generate n random ints for the selection of topics we'll conduct intrusion on
+ # topic_idxs = random.sample(range(len(self.topics)), self.num_topics)
+ # selected_intruders = set()
+ #
+ # for topic_idx in topic_idxs:
+ # # select another topic from which to grab a term, exclude the current topic
+ # random_topic_idx = random.choice([idx for idx in range(0, len(self.topics))
+ # if (idx != topic_idx and idx not in selected_intruders)])
+ # print(random_topic_idx)
+ # selected_intruders.add(random_topic_idx)
+ #
+ # # assert that the new word is not in the top 5 words of the original topic
+ # correct_words = [word for word in self.topics[topic_idx]["terms"][:num_terms]]
+ # # collect top 10 words of the current topic
+ # top_topic_words = [word for word in self.topics[topic_idx]["terms"][:sample_top_terms]]
+ # print(top_topic_words)
+ # # collect top words of the 'intruder' topics that do NOT overlap with any of the top 10 words of the other topic
+ # if sample_top_terms:
+ # top_random_words = random.sample(
+ # [word for word in self.topics[random_topic_idx]["terms"][:sample_top_terms]
+ # if word not in top_topic_words], num_terms)
+ # else:
+ # top_random_words = [word for word in self.topics[random_topic_idx]["terms"][:num_terms]
+ # if word not in top_topic_words]
+ #
+ # print(top_random_words)
+ # # select the intruder word
+ # selected_intruder = random.choice(top_random_words)
+ #
+ # topic_intruder.append(
+ # {
+ # "topic_id": self.topics[topic_idx]["topic_id"],
+ # "intruder_id": self.topics[random_topic_idx]["topic_id"],
+ # "intruder_term": selected_intruder,
+ # "topic_terms": correct_words
+ # }
+ # )
+ # print('Done!')
+ # sorted_by_topic = sorted(topic_intruder, key=lambda d: d['topic_id'])
+ #
+ # # Write the topic intruders to a file
+ # if not os.path.isdir(self.path_save):
+ # os.system('mkdir -p ' + self.path_save)
+ #
+ # with open(self.path_save + 'intruders.txt', 'w') as f:
+ # for i in range(len(sorted_by_topic)):
+ # f.write(str(sorted_by_topic[i]) + '\n')
+
+ intruder_list = []
+ selected_intruders = set()
+ for topic_idx in range(self.num_topics):
+ # select another topic from which to grab a term, exclude the current topic
+ random_topic_idx = random.choice(
+ [idx for idx in range(0, self.num_topics) if (idx != topic_idx and idx not in selected_intruders)])
+ selected_intruders.add(random_topic_idx)
+ # take the top 5 words of the current topic and ONE of the top 5 terms from the top of the other topic
+ # assert that the new word is not in the top 50 words of the original topic
+ correct_words = [word for word in self.topics[topic_idx]["terms"][:num_terms]]
+
+ # This collects the top 50 words of the current topic
+ top_topic_words = [word for word in self.topics[topic_idx]["terms"][:50]]
+
+ # This collects the top words of the 'intruder' topics that do NOT overlap with any of the top
+ # 10 words of the other topic
+ if sample_top_topic_terms:
+ top_random_words = random.sample([word for word in self.topics[random_topic_idx]["terms"][:10] \
+ if word not in top_topic_words], num_terms)
+ else:
+ top_random_words = [word for word in self.topics[random_topic_idx]["terms"][:4] \
+ if word not in top_topic_words]
+
+ # EDGE-CASE - The top 50 words of the selected topic may overlap heavily with the
+ # 'intruder' topics's top words. In this case, narrow down the set of excluded terms
+ # for the current topic to just the top 10. If that doesn't work, then..... skip??
+ if not top_random_words:
+ top_topic_words = [word for word in self.topics[topic_idx]["terms"][:10]]
+ top_random_words = [word for word in self.topics[random_topic_idx]["terms"][:5] \
+ if word not in top_topic_words]
+
+ if not top_random_words:
+ print(f"Skipping word intrusion for topic {topic_idx} with intruder {random_topic_idx}")
+ continue
+ # select the intruder word
+ selected_intruder = random.choice(top_random_words)
+
+ # The last word in each list is the 'intruder', this should be randomized before showing
+ # [topics_list[topic_idx]["topic_id"]] + correct_words + [selected_intruder]
+ intruder_list.append(
+ {
+ "topic_id": self.topics[topic_idx]["topic_id"],
+ "intruder_id": self.topics[random_topic_idx]["topic_id"],
+ "intruder_term": selected_intruder,
+ "topic_terms": correct_words
+ }
+ )
+ # Write the topic intruders to a file
+ if not os.path.isdir(self.path_save):
+ os.system('mkdir -p ' + self.path_save)
+
+ with open(self.path_save + 'intruders.txt', 'w') as f:
+ for i in range(len(intruder_list)):
+ f.write(str(intruder_list[i]) + '\n')
+
+ return intruder_list
+
+
+ def create_prompt(self, topic_intruder):
+ ''' p1: Show the least related term
+ p2: Select which term is the least related to all other terms
+ p3: What is the intruder term in the following terms?
+ p4: Which word does not belong?
+ p5: Which one of the following words does not belong?
+ p6: Find the intruder term
+ '''
+
+ list_of_terms = []
+ topic_intruder_prompt = []
+ for i in range(len(topic_intruder)):
+ list_of_terms.append(topic_intruder[i]['topic_terms'] + topic_intruder[i]['intruder_term'].split())
+
+ for i in range(len(list_of_terms)):
+ random.shuffle(list_of_terms[i])
+ # list_terms = str(list_of_terms[i])
+ list_terms = str(list_of_terms[i]).replace("[", "").replace("]", "")
+ # list_terms = str(list_of_terms[i]).replace("'", "")
+ # list_terms = str(list_of_terms[i]).replace("[", "").replace("]", "").replace("'", "")
+
+ # prompt = 'What is the intruder term in the following terms? ' + list_terms
+ prompt = 'Show the least related term\nTerms: ' + list_terms + '\nAnswer: '
+
+ topic_intruder_prompt.append(
+ {
+ "topic_id": topic_intruder[i]["topic_id"],
+ "topic_terms": topic_intruder[i]['topic_terms'],
+ "intruder_term": topic_intruder[i]['intruder_term'],
+ "prompt": prompt
+ }
+ )
+ with open(self.path_save + self.prompt_name + '.txt', 'w') as f:
+ for i in range(len(topic_intruder_prompt)):
+ f.write(str(topic_intruder_prompt[i]) + '\n')
+ f.close()
+
+ del list_of_terms
+ return topic_intruder_prompt
+
+
+ def run_gpt3(self, tip_dict):
+ # run GPT-3 for all topics
+ with open(self.path_save + self.prompt_name + '_gpt3.txt', 'w') as f:
+ for i in range(len(tip_dict)):
+ prompt = tip_dict[i]['prompt']
+ response = str(gpt3(prompt)).replace('\n', '')
+ f.write(response + '\n')
+ print(prompt, " ", response)
+ time.sleep(2)
+ f.close()
+
+
+ def load_npmis(self):
+ with open(self.file_path + 'npmis.txt') as file:
+ npmis = file.readlines()
+ npmis = [float(term.replace('\n', '')) for term in npmis]
+ return npmis
+
+
+ def compare_intruder(self, tip_dict):
+ real_intruder = [tip_dict[i]['intruder_term'] for i in range(len(tip_dict))]
+ topic_terms = [tip_dict[i]['topic_terms'] for i in range(len(tip_dict))]
+
+ with open(self.path_save + self.prompt_name + '_gpt3.txt') as file:
+ gpt3_intruders = file.readlines()
+ gpt3_intruders = [term.replace('\n', '').lower() for term in gpt3_intruders]
+
+ compared_resp_topics = [[term for term in topic_terms[idx] if(term in gpt3_intruders[idx])] for idx in range(len(tip_dict))]
+ # put 1 if topic terms are not in gpt3 response
+ int_transform_topic = [1 if len(list) == 0 else 0 for list in compared_resp_topics]
+ # put 1 if intruder term in gpt3 response
+ compared_resp_intruder = [[1 if(real_intruder[idx] in gpt3_intruders[idx]) else 0] for idx in range(len(tip_dict))]
+ int_transform_intruder = sum(compared_resp_intruder, [])
+ counts = int_transform_intruder and int_transform_topic
+
+ # calculate accuracy
+ acc = np.mean(counts)
+ variance = np.var(counts)
+ spear_rho, spear_p = spearmanr(self.load_npmis(), counts)
+ pear_rho, pear_p = pearsonr(self.load_npmis(), counts)
+ correlations = {
+ "spear_rho": spear_rho,
+ "spear_p": spear_p,
+ "pear_rho": pear_rho,
+ "pear_p": pear_p
+ }
+
+ with open(self.path_save + self.prompt_name + '_counts.txt', 'w') as f:
+ f.write("Counts: {}\nAccuracy: {}\nVariance: {}\nCorrelations: {}".format(counts, acc, variance, correlations))
+ print("\nAccuracy: {}\nVariance: {}\nCorrelations: {}\n".format(acc, variance, correlations))
+
+ return counts
+
+
+ def run_intrusion(self):
+ print("Creation of intruder words...")
+ topic_intruder = self.create_intruder()
+ print("Creation of prompts for GPT-3...")
+ topic_intruder_prompt = self.create_prompt(topic_intruder)
+ print("Running GPT-3...")
+ self.run_gpt3(topic_intruder_prompt)
+ print("Comparing intruder terms...")
+ self.compare_intruder(topic_intruder_prompt)
+
diff --git a/transformer-tests/main.py b/transformer-tests/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f18fde767e413ca51cf2ab857dab7fe1ed67edc
--- /dev/null
+++ b/transformer-tests/main.py
@@ -0,0 +1,66 @@
+import random
+import configargparse
+from intrusion import Intrusion
+from rating import Rating
+
+
+def load_topics_rating(input_dir):
+ file_path = input_dir + 'topics.txt'
+ reader = open(file_path)
+ # create list of list
+ list_topics = [line.replace("\n", "") for line in reader.readlines()]
+
+ return list_topics
+
+def load_topics_intrusion(input_dir):
+ lines = load_topics_rating(input_dir)
+ topics = []
+ # create list of dictionaries
+ for idx, line in enumerate(lines):
+ topics.append(
+ {
+ "topic_id": idx,
+ "terms": line.split()
+ }
+ )
+ return topics
+
+
+if __name__ == "__main__":
+ parser = configargparse.ArgParser(
+ description="parse args",
+ config_file_parser_class=configargparse.YAMLConfigFileParser
+ )
+
+ # Data
+ parser.add("-c", "--config", is_config_file=True, default=None)
+ parser.add("--input_dir", required=False, default=None)
+ parser.add("--output_dir", required=False, default=None)
+ parser.add("--data", required=True, default=None, choices=["20ng", "wiki", "nyt"])
+
+ # Task-specific hyperparams
+ parser.add("--task", default="intrusion", choices=["intrusion", "rating"])
+
+ # Model-specific hyperparams
+ parser.add("--model", required=True, default=None, choices=["mallet", "dvae", "etm"])
+ parser.add("--num_topics", default=50, type=int)
+
+ args = parser.parse_args()
+ topic_file_path = './results/' + str(args.model) + '-' + str(args.data) +'/'
+ path_save = './transformer-tests/' + str(args.model) + '/' + str(args.data) + '/'
+ random.seed(5)
+ if args.task == "intrusion":
+ topics = load_topics_intrusion(topic_file_path)
+ task_model = Intrusion(topics, args.num_topics, path_save, topic_file_path)
+ task_model.run_intrusion()
+
+ elif args.task == "rating":
+ topics = load_topics_rating(topic_file_path)
+ task_model = Rating(topics, args.num_topics, path_save, topic_file_path)
+ task_model.run_rating()
+
+ else:
+ print("The task is not specified! Select the task to be rating or intrusion.")
+
+ print('Done!')
+
diff --git a/transformer-tests/rating.py b/transformer-tests/rating.py
new file mode 100644
index 0000000000000000000000000000000000000000..d7380c6a52cabd867c36393a66b4e1ec3a0f3aab
--- /dev/null
+++ b/transformer-tests/rating.py
@@ -0,0 +1,114 @@
+import time
+import numpy as np
+from gpt3 import gpt3
+from scipy.stats import spearmanr, pearsonr
+
+class Rating():
+ def __init__(self, topics, num_topics, path_save, file_path):
+ super(Rating, self).__init__()
+
+ ## define hyperparameters
+ self.topics = topics
+ self.num_topics = num_topics
+ self.path_save = path_save
+ self.file_path = file_path
+ self.prompt_name = 'rating_p4_v4'
+
+
+ def run_gpt3(self, prompts):
+ # run GPT-3 for all topics
+ with open(self.path_save + self.prompt_name + '_gpt3.txt', 'w') as f:
+ for i in range(len(prompts)):
+ prompt = prompts[i]
+ response = str(gpt3(prompt)).replace('\n', '')
+ f.write(response + '\n')
+ print(prompt, " ", response)
+ time.sleep(3)
+ f.close()
+
+
+ def create_prompt(self):
+
+ #topics = self.topics
+ prompts = []
+ topics = [topic.split() for topic in self.topics]
+ for i in range(len(self.topics)):
+ # list1 = "['file', 'window', 'problem', 'run', 'system', 'program', 'font', 'work', 'win', 'change']"
+ # list2 = "['chip', 'clipper', 'phone', 'key', 'encryption', 'government', 'system', 'write', 'nsa', 'communication']"
+ list_ten_terms = str(topics[i][:10])
+
+ # list1 = '[file, window, problem, run, system, program, font, work, win, change]'
+ # list2 = '[chip, clipper, phone, key, encryption, government, system, write, nsa, communication]'
+ # list_ten_terms = list_ten_terms.replace("'", "")
+
+ # list1 = "['file', 'window', 'problem', 'run', 'system', 'program', 'font', 'work', 'win', 'change']"
+ # list2 = "['chip', 'clipper', 'phone', 'key', 'encryption', 'government', 'system', 'write', 'nsa', 'communication']"
+ # list_ten_terms = list_ten_terms.replace("[", "").replace("]", "")
+
+ list1 = 'file, window, problem, run, system, program, font, work, win, change'
+ list2 = 'chip, clipper, phone, key, encryption, government, system, write, nsa, communication'
+ list_ten_terms = list_ten_terms.replace("'", "").replace("[", "").replace("]", "")
+
+
+ # prompts.append("Rate how related the following terms are to each other " \
+ # "as 'very related', 'somewhat related' or 'not related': " + list1 + "\n" \
+ # "Answer: Very related\n\nRate how related the following terms are to each other " \
+ # "as 'very related', 'somewhat related' or 'not related': " + list2 + "\n"
+ # "Answer: Somewhat related\n\nRate how related the following terms are to each other " \
+ # "as 'very related', 'somewhat related' or 'not related': " + list_ten_terms + "\nAnswer: ")
+
+ prompts.append("Rate how related the following terms are to each other " \
+ "as '3-very related', '2-somewhat related' or '1-not related': " + list1 + "\n" \
+ "Answer: 3\n\nRate how related the following terms are to each other " \
+ "as '3-very related', '2-somewhat related' or '1-not related': " + list2 + "\n"
+ "Answer: 2\n\nRate how related the following terms are to each other " \
+ "as '3-very related', '2-somewhat related' or '1-not related': " + list_ten_terms + "\nAnswer: ")
+
+ # prompts.append("Rate how related the following terms are to each other " \
+ # "in a range from 1 to 3: " + list_ten_terms)
+
+ with open(self.path_save + self.prompt_name + '.txt', 'w') as f:
+ for i in range(self.num_topics):
+ f.write(prompts[i] + '\n')
+ f.close()
+
+ print("Running GPT-3...")
+ self.run_gpt3(prompts)
+
+
+ def load_npmis(self):
+ with open(self.file_path + 'npmis.txt') as file:
+ npmis = file.readlines()
+ npmis = [float(term.replace('\n', '')) for term in npmis]
+ return npmis
+
+
+ def calculate_metrics(self):
+
+ with open(self.path_save + self.prompt_name + '_gpt3.txt') as file:
+ responses = file.readlines()
+ responses = [int(term.replace('\n', '').replace(' ', '')) for term in responses]
+
+ # calculate accuracy
+ acc = np.mean(responses)
+ variance = np.var(responses)
+ spear_rho, spear_p = spearmanr(self.load_npmis(), responses)
+ pear_rho, pear_p = pearsonr(self.load_npmis(), responses)
+ correlations = {
+ "spear_rho": spear_rho,
+ "spear_p": spear_p,
+ "pear_rho": pear_rho,
+ "pear_p": pear_p
+ }
+
+ with open(self.path_save + self.prompt_name + '_counts.txt', 'w') as f:
+ f.write("Counts: {}\nMean: {}\nVariance: {}\nCorrelations: {}".format(responses, acc, variance, correlations))
+ print("\nMean: {}\nVariance: {}\nCorrelations: {}\n".format(acc, variance, correlations))
+
+
+ def run_rating(self):
+ print("Creation of prompts for GPT-3...")
+ #self.create_prompt()
+ print(self.calculate_metrics())
+
+