127 lines
3.9 KiB
Python
127 lines
3.9 KiB
Python
from nltk.tokenize import RegexpTokenizer
|
|
from stop_words import get_stop_words
|
|
from nltk.stem.porter import PorterStemmer
|
|
from gensim import corpora, models
|
|
import gensim
|
|
from sklearn.cluster import KMeans
|
|
import dao
|
|
import csv
|
|
import helper
|
|
|
|
|
|
|
|
def lda_process(doc_set,f_name,n_topic,proj_id):
|
|
|
|
tokenizer = RegexpTokenizer(r'\w+')
|
|
|
|
# create English stop words list
|
|
en_stop = get_stop_words('en')
|
|
|
|
# Create p_stemmer of class PorterStemmer
|
|
p_stemmer = PorterStemmer()
|
|
|
|
# list for tokenized documents in loop
|
|
texts = []
|
|
x_id = []
|
|
issue_type = []
|
|
|
|
# loop through document list
|
|
for r in doc_set:
|
|
i = r[0]+'. '+r[1]
|
|
# clean and tokenize document string
|
|
raw = i.lower()
|
|
raw = helper.filter_code(raw)
|
|
raw = helper.filter_sign(raw)
|
|
tokens = tokenizer.tokenize(raw)
|
|
|
|
# remove num from stopped_tokens
|
|
numed_tokens = [i for i in tokens if not helper.filter_str(i) == '' ]
|
|
|
|
# remove stop words from tokens
|
|
stopped_tokens = [i for i in numed_tokens if not i in en_stop]
|
|
|
|
# stem tokens
|
|
stemmed_tokens = [p_stemmer.stem(i) for i in stopped_tokens]
|
|
|
|
# add tokens to list
|
|
texts.append(stemmed_tokens)
|
|
|
|
x_id.append(r[2])
|
|
issue_type.append(r[3])
|
|
|
|
# turn our tokenized documents into a id <-> term dictionary
|
|
dictionary = corpora.Dictionary(texts)
|
|
|
|
# filter word that frequency is more than len(doc_set)/4,and less than 3
|
|
dictionary.filter_extremes(3,0.95)
|
|
|
|
# convert tokenized documents into a document-term matrix
|
|
corpus = [dictionary.doc2bow(text) for text in texts]
|
|
|
|
# generate LDA model
|
|
ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=n_topic, id2word = dictionary, passes=20)
|
|
|
|
# save model
|
|
ldamodel.save(f_name)
|
|
|
|
X = []
|
|
# get probability of every topics for each doc
|
|
for bow in corpus:
|
|
doc_topic = ldamodel.get_document_topics(bow)
|
|
# doc_param = [0.0]*10
|
|
doc_param = [0.0 for i in range(n_topic)]
|
|
for doc_t in doc_topic:
|
|
doc_param[doc_t[0]] = doc_t[1]
|
|
X.append(doc_param)
|
|
|
|
# kmeans
|
|
y_pred = KMeans(n_clusters= 20).fit_predict(X)
|
|
|
|
# dao save in db
|
|
dao.save_kmeans_result(x_id,y_pred,proj_id)
|
|
|
|
# f_proj_id = file('proj_id.csv', 'r')
|
|
# reader = csv.reader(f_proj_id)
|
|
# for line in reader:
|
|
proj_id = '6013'
|
|
# proj_id = line[0]
|
|
path = 'result/'+proj_id+'/lda/'
|
|
helper.mkdir(path)
|
|
print("project: "+proj_id)
|
|
# cur_feature = dao.get_feature(proj_id)
|
|
# doc_feature = cur_feature.fetchall()
|
|
# lda_process(doc_feature,path+"feature_name_5.lda",5)
|
|
cur_bug = dao.get_feature_and_bug(proj_id)
|
|
doc_bug = cur_bug.fetchall()
|
|
print(len(doc_bug))
|
|
lda_process(doc_bug,path+"bug_feature_name_50.lda",50,proj_id)
|
|
|
|
# def do_process(proj_id):
|
|
# path = 'result/'+proj_id+'/lda/'
|
|
# helper.mkdir(path)
|
|
# print("project: "+proj_id)
|
|
# cur_feature = dao.get_feature(proj_id)
|
|
# doc_feature = cur_feature.fetchall()
|
|
# lda_process(doc_feature,path+"feature_name_5.lda",5)
|
|
# lda_process(doc_feature,path+"feature_name_10.lda",10)
|
|
# lda_process(doc_feature,path+"feature_name_15.lda",15)
|
|
# lda_process(doc_feature,path+"feature_name_20.lda",20)
|
|
# lda_process(doc_feature,path+"feature_name_100.lda",100)
|
|
# cur_bug = dao.get_bug(proj_id)
|
|
# doc_bug = cur_bug.fetchall()
|
|
# lda_process(doc_bug,path+"bug_name_5.lda",5)
|
|
# lda_process(doc_bug,path+"bug_name_10.lda",10)
|
|
# lda_process(doc_bug,path+"bug_name_15.lda",15)
|
|
# lda_process(doc_bug,path+"bug_name_20.lda",20)
|
|
# lda_process(doc_bug,path+"bug_name_100.lda",100)
|
|
# # lda_process(cur,feature_name_10,10)
|
|
# # lda_process(cur,feature_name_15,15)
|
|
# # lda_process(cur,feature_name_20,20)
|
|
# # lda_process(cur,feature_name_100,100)
|
|
#
|
|
#
|
|
# f_proj_id = file('proj_id.csv', 'r')
|
|
# reader = csv.reader(f_proj_id)
|
|
# for line in reader:
|
|
# do_process(line[0])
|