252 lines
7.9 KiB
Python
252 lines
7.9 KiB
Python
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
|
|
import cPickle as pickle
|
|
import helper
|
|
import pymysql
|
|
import dao
|
|
|
|
# path_oringan = 'result2'
|
|
# preprocess of data
|
|
def data_preprocess(project_name, methold_name = ''):
|
|
|
|
# project_name = "owncloud"
|
|
|
|
path = 'result/'+project_name+'/'
|
|
helper.mkdir(path)
|
|
# if len(methold_name):
|
|
# data_path = path + methold_name + '/data/'
|
|
# else:
|
|
# data_path = path + 'data/'
|
|
# helper.mkdir(data_path)
|
|
|
|
f_train = path +'train.pkl'
|
|
f_target = path + 'target.pkl'
|
|
f_id = path + 'id.pkl'
|
|
f_vect = path + 'vect.pkl'
|
|
f_train_data = path + 'train_data.pkl'
|
|
f_cur_all = path + 'cur_all.pkl'
|
|
print('=' * 80)
|
|
print("get data: ")
|
|
cur = dao.get_data(project_name)
|
|
fetchall = cur.fetchall()
|
|
with open(f_cur_all, 'w') as f: # open file with write-mode
|
|
pickle.dump(fetchall, f)
|
|
|
|
train_data = []
|
|
train_target = []
|
|
x_id = []
|
|
for r in fetchall:
|
|
str = helper.filter_str(r[1]+".\n"+r[2])
|
|
# str = helper.filter_code(str)
|
|
train_data.append(str)
|
|
train_target.append(r[0])
|
|
x_id.append(r[3])
|
|
print("data length is : ", len(train_target))
|
|
|
|
cur.close()
|
|
|
|
print('_' * 80)
|
|
print("processing data: TF-IDF")
|
|
|
|
# categories = ['bug','enhancement','feature','documentation','question','others']
|
|
categories = ['bug','enhancement']
|
|
y = [None]*len(train_target)
|
|
for i in range(len(train_target)):
|
|
y[i] = (categories.index(train_target[i]))
|
|
|
|
with open(f_train_data, 'w') as f: # open file with write-mode
|
|
pickle.dump(train_data, f)
|
|
# TF-IDF
|
|
vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
|
|
stop_words='english', tokenizer=helper.tokenize_help)
|
|
X = vectorizer.fit_transform(train_data)
|
|
|
|
# store data preprocess result in file
|
|
|
|
|
|
with open(f_train, 'w') as f: # open file with write-mode
|
|
pickle.dump(X, f) # serialize and save object
|
|
with open(f_target, 'w') as f: # open file with write-mode
|
|
pickle.dump(y, f) # serialize and save object
|
|
with open(f_id, 'w') as f: # open file with write-mode
|
|
pickle.dump(x_id, f) # serialize and save object
|
|
with open(f_vect, 'w') as f: # open file with write-mode
|
|
pickle.dump(vectorizer, f) # serialize and save object
|
|
|
|
print("done")
|
|
|
|
return X,y,x_id,vectorizer
|
|
|
|
def data_preprocess_from_file(project_name, methold_name = ''):
|
|
|
|
# project_name = "owncloud"
|
|
|
|
path = 'result/'+project_name+'/'
|
|
helper.mkdir(path)
|
|
if len(methold_name):
|
|
data_path = path + methold_name + '/data/'
|
|
else:
|
|
data_path = path + 'data/'
|
|
helper.mkdir(data_path)
|
|
|
|
|
|
f_train = path +'train.pkl'
|
|
f_target = path + 'target.pkl'
|
|
f_id = path + 'id.pkl'
|
|
f_vect = path + 'vect.pkl'
|
|
f_train_data = path + 'train_data.pkl'
|
|
print('=' * 80)
|
|
print("get data: ")
|
|
cur = dao.get_data(project_name)
|
|
train_data = []
|
|
train_target = []
|
|
x_id = []
|
|
for r in cur.fetchall():
|
|
str = helper.filter_str(repr(r[1])+".\n"+repr(r[2]))
|
|
# str = helper.filter_code(str)
|
|
train_data.append(str)
|
|
train_target.append(r[0])
|
|
x_id.append(r[3])
|
|
print("data length is : ", len(train_target))
|
|
|
|
cur.close()
|
|
|
|
print('_' * 80)
|
|
print("processing data: TF-IDF")
|
|
|
|
# categories = ['bug','enhancement','feature','documentation','question','others']
|
|
categories = ['bug','enhancement']
|
|
y = [None]*len(train_target)
|
|
for i in range(len(train_target)):
|
|
y[i] = (categories.index(train_target[i]))
|
|
|
|
with open(f_train_data, 'w') as f: # open file with write-mode
|
|
pickle.dump(train_data, f)
|
|
# TF-IDF
|
|
vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
|
|
stop_words='english', tokenizer=helper.tokenize_help)
|
|
X = vectorizer.fit_transform(train_data)
|
|
|
|
# store data preprocess result in file
|
|
|
|
|
|
with open(f_train, 'w') as f: # open file with write-mode
|
|
pickle.dump(X, f) # serialize and save object
|
|
with open(f_target, 'w') as f: # open file with write-mode
|
|
pickle.dump(y, f) # serialize and save object
|
|
with open(f_id, 'w') as f: # open file with write-mode
|
|
pickle.dump(x_id, f) # serialize and save object
|
|
with open(f_vect, 'w') as f: # open file with write-mode
|
|
pickle.dump(vectorizer, f) # serialize and save object
|
|
|
|
print("done")
|
|
|
|
return X,y,x_id,vectorizer
|
|
|
|
|
|
|
|
# def main():
|
|
#
|
|
# data_preprocess()
|
|
#
|
|
#
|
|
# # load data preprocess result from file
|
|
# print('get data: ')
|
|
# X = helper.get_pickle_record(f_train)
|
|
# y = helper.get_pickle_record(f_target)
|
|
# x_id = helper.get_pickle_record(f_id)
|
|
#
|
|
# y = np.array(y)
|
|
# x_id = np.array(x_id)
|
|
#
|
|
# print('done')
|
|
#
|
|
# results = []
|
|
# kf = KFold(len(y), n_folds=10)
|
|
#
|
|
# print("start training:")
|
|
#
|
|
# i = 0
|
|
#
|
|
# for train_index, test_index in kf:
|
|
# X_train, X_test = X[train_index], X[test_index]
|
|
# y_train, y_test = y[train_index], y[test_index]
|
|
# x_id_train, x_id_test = x_id[train_index], x_id[test_index]
|
|
# i = i+1
|
|
# print('turn '+ repr(i) +':')
|
|
# print('='*80)
|
|
# ###############################################################################
|
|
# # Benchmark classifiers
|
|
# def benchmark(clf):
|
|
# print('_' * 80)
|
|
# print("Training: ")
|
|
# print(clf)
|
|
# t0 = time()
|
|
# clf.fit(X_train, y_train)
|
|
# train_time = time() - t0
|
|
# print("train time: %0.3fs" % train_time)
|
|
#
|
|
# t0 = time()
|
|
# pred = clf.predict(X_test)
|
|
#
|
|
# test_time = time() - t0
|
|
# print("test time: %0.3fs" % test_time)
|
|
#
|
|
# score = metrics.accuracy_score(y_test, pred)
|
|
# print("accuracy: %0.3f" % score)
|
|
# # np.save(repr(clf.penalty)+"_y_test_"+repr(i),y_test)
|
|
# # np.save(repr(clf.penalty)+"_pred_"+repr(i),pred)
|
|
# np.save(path + "y_test_"+repr(i),y_test)
|
|
# np.save(path + "pred_"+repr(i),pred)
|
|
# np.save(path + "x_id_"+repr(i),x_id_test)
|
|
#
|
|
# probability = clf.predict_proba(X_test)
|
|
# np.save(path + "probability"+repr(i),probability)
|
|
#
|
|
# print()
|
|
# clf_descr = str(clf).split('(')[0]
|
|
#
|
|
# return clf_descr, score, train_time, test_time
|
|
#
|
|
#
|
|
# # for penalty in ["l2", "l1"]:
|
|
# # print('=' * 80)
|
|
# # print("%s penalty" % penalty.upper())
|
|
# # # Train Liblinear model (svm)
|
|
# # results.append(benchmark(svm.LinearSVC(loss='l2', penalty=penalty, dual=False, tol=1e-3)))
|
|
# results.append(benchmark(svm.SVC(kernel='linear',probability=True)))
|
|
#
|
|
#
|
|
# indices = np.arange(len(results))
|
|
#
|
|
# results = [[x[i] for x in results] for i in range(4)]
|
|
#
|
|
# clf_names, score, training_time, test_time = results
|
|
# training_time = np.array(training_time) / np.max(training_time)
|
|
# test_time = np.array(test_time) / np.max(test_time)
|
|
#
|
|
# score_all = np.array(score).sum()/len(score)
|
|
# print("accuracy for all: %0.3f" % score_all)
|
|
#
|
|
# # plot export
|
|
#
|
|
# plt.figure(figsize=(12, 8))
|
|
# plt.title("Score")
|
|
# plt.barh(indices, score, .2, label="score", color='r')
|
|
# plt.barh(indices + .3, training_time, .2, label="training time", color='g')
|
|
# plt.barh(indices + .6, test_time, .2, label="test time", color='b')
|
|
# plt.yticks(())
|
|
# plt.legend(loc='best')
|
|
# plt.subplots_adjust(left=.25)
|
|
# plt.subplots_adjust(top=.95)
|
|
# plt.subplots_adjust(bottom=.05)
|
|
#
|
|
# for i, c in zip(indices, clf_names):
|
|
# plt.text(-.3, i, c)
|
|
#
|
|
# plt.show()
|
|
#
|
|
#
|
|
#
|
|
# if __name__ == '__main__':
|
|
# main()
|