check output

lib/predictCall.py

+import gensim
+from nltk import RegexpTokenizer
+from nltk.corpus import stopwords
+from os import listdir
+from os.path import isfile, join
+from gensim.models.doc2vec import Doc2Vec, TaggedDocument
+
 def predict():
-    print("hello")
+    old_model = gensim.models.doc2vec.Doc2Vec.load('testModel.model')
\ No newline at end of file
+
+    tokens = "computer gaming is the best thing".split()
+    new_vector = d2v_model.infer_vector(tokens)
+
+    print(old_model.docvecs.most_similar(new_vector))
+    print(old_model.most_similar([new_vector]))
+    print(old_model.docvecs.offset2doctag([new_vector]))
\ No newline at end of file

lib/trainCall.py

 from App import db
 from Models import *
+from multiprocessing.pool import Pool
+from nltk.tokenize import RegexpTokenizer
+import gensim
+from nltk import RegexpTokenizer
+from nltk.corpus import stopwords
 
-def load_paper():
+from os import listdir
+from os.path import isfile, join
+from gensim.models.doc2vec import Doc2Vec, TaggedDocument
 
 
-def train():
-    #print(Paper.query.count())
+labels = []
+tokenizer = RegexpTokenizer(r'\w+')
+stopword_set = set(stopwords.words('english'))
+
+def nltk_cleaning(data):
+    clean_data = []
+        
+    new_str = data.lower()
+    dlist = tokenizer.tokenize(new_str)
+    dlist = list(set(dlist).difference(stopword_set))
+    clean_data = dlist
+    
+    return clean_data
+
+def tokenize(article):
+    preprocessed = article[0].replace("\r","").replace("\n","")
+    data = nltk_cleaning(preprocessed)
+    return  data, article[1]
 
-    load_paper()
-    print("hello")
+def load_paper(labels):
+    result = []
\ No newline at end of file
+    articles = Paper.query.filter(Paper.Abstract!="").with_entities(Paper.Abstract,Paper.PaperKey).all()
+    with Pool() as pool:
+        result = pool.map(tokenize, articles)
+
+    for item in result:
+        labels.append(gensim.models.doc2vec.LabeledSentence(words = item[0], tags = [item[1]]))
+    return labels
+
+def train():
+    labels = []
+    print(Paper.query.count())
+    
+    dataset = load_paper(labels)
+    
+    model = gensim.models.Doc2Vec(size=300, min_count=0, alpha=0.025, min_alpha=0.025)
+    model.build_vocab(dataset)
+    for epoch in range(100):
+        print('iteration'+str(epoch+1))
+        model.train(labels, epochs=model.iter, total_examples=model.corpus_count)
+        model.alpha -= 0.002
+        model.min_alpha = model.alpha
+        model.save('testModel.model') 
\ No newline at end of file

main.py

-from lib import trainCall,predictCall
+from lib import TrainCall
 import argparse
 
 def main():
@@ -9,7 +9,7 @@ def main():
     args = parser.parse_args()
 
     if(args.run_opt == 2):
-        trainCall.train()
+        TrainCall.train()
     else:
         predictCall.predict()