418 lines
20 KiB
Python
418 lines
20 KiB
Python
# import traceback
|
|
import nltk
|
|
import numpy as np
|
|
from sklearn import svm
|
|
from sklearn import metrics
|
|
from sklearn.cross_validation import KFold
|
|
from time import time
|
|
import csv
|
|
# from itertools import *
|
|
import dao
|
|
|
|
import sys
|
|
import helper
|
|
import classifier
|
|
|
|
# reload(sys)
|
|
# sys.setdefaultencoding("utf-8")
|
|
|
|
__author__ = 'mac'
|
|
import cPickle as pickle
|
|
|
|
|
|
def classifier_project_by_id(proj_id):
|
|
# project_name = repr(proj_id)
|
|
project_name = proj_id
|
|
methold_name = 'svm'
|
|
|
|
path = 'result/'+project_name+'/'
|
|
helper.mkdir(path)
|
|
path_analysis = path + methold_name + '/analysis/'
|
|
helper.mkdir(path_analysis)
|
|
path_data = path + methold_name + '/data/'
|
|
helper.mkdir(path_data)
|
|
csv_result = file(path_analysis + 'precision_method.csv', 'wb')
|
|
writer_result = csv.writer(csv_result)
|
|
|
|
feature_csv_result = file(path_analysis + 'feature_precision_method.csv', 'wb')
|
|
feature_writer_result = csv.writer(feature_csv_result)
|
|
|
|
csv_path = file(path_analysis + 'sentence_split.csv', 'wb')
|
|
writer = csv.writer(csv_path)
|
|
writer.writerow(['id','diff','y_test','pred','sentence_pred','sentence_pred_0','sentence_pred_1','sentence'])
|
|
|
|
|
|
csv_classifier = file(path_analysis + 'classifier_info.csv', 'wb')
|
|
writer_classifier = csv.writer(csv_classifier)
|
|
writer_classifier.writerow(['classifier infomation for project:',repr(project_name)])
|
|
|
|
threshold = []
|
|
break_count = 20 # cut the different, len(threshold)
|
|
change_break = 20 # cut the condition for changing class
|
|
|
|
for ind_c_th in range(1,break_count+1):
|
|
threshold.append(ind_c_th*1.0/break_count)
|
|
|
|
count_base = [0]*break_count
|
|
right_base = [0]*break_count
|
|
count_base2 = [0]*break_count
|
|
right_base2 = [0]*break_count
|
|
|
|
feature_count_base = [0]*break_count
|
|
feature_pred_base = [0]*break_count
|
|
feature_right_base = [0]*break_count
|
|
feature_count_base2 = [0]*break_count
|
|
feature_pred_base2 = [0]*break_count
|
|
feature_right_base2 = [0]*break_count
|
|
|
|
|
|
right_mine1 = [([0] * change_break) for i in range(break_count)]
|
|
right_mine2 = [([0] * change_break) for i in range(break_count)]
|
|
right_mine3 = [([0] * change_break) for i in range(break_count)]
|
|
right_mine4 = [([0] * change_break) for i in range(break_count)]
|
|
|
|
feature_right_mine1 = [([0] * change_break) for i in range(break_count)]
|
|
feature_right_mine2 = [([0] * change_break) for i in range(break_count)]
|
|
feature_right_mine3 = [([0] * change_break) for i in range(break_count)]
|
|
feature_right_mine4 = [([0] * change_break) for i in range(break_count)]
|
|
|
|
feature_count_mine1 = [([0] * change_break) for i in range(break_count)]
|
|
feature_count_mine2 = [([0] * change_break) for i in range(break_count)]
|
|
feature_count_mine3 = [([0] * change_break) for i in range(break_count)]
|
|
feature_count_mine4 = [([0] * change_break) for i in range(break_count)]
|
|
|
|
|
|
X,y,x_id_before,vect = classifier.data_preprocess(project_name,methold_name) # first time run, get tf-idf of train data
|
|
# X,y,x_id_before,vect = helper.get_tfidf_data(project_name) # fellow methold run, to get tf-idf store in disk
|
|
f_train_data = path + 'train_data.pkl'
|
|
train_data = helper.get_pickle_record(f_train_data)
|
|
|
|
# get tf-idf matirx of train data
|
|
# print('get data: ')
|
|
# f_train = path +'train.pkl'
|
|
# f_target = path + 'target.pkl'
|
|
# f_id = path + 'id.pkl'
|
|
# f_vect = path + 'vect.pkl'
|
|
# X = helper.get_pickle_record(f_train)
|
|
# y = helper.get_pickle_record(f_target)
|
|
# x_id = helper.get_pickle_record(f_id)
|
|
# vect = helper.get_pickle_record(f_vect)
|
|
# print('done')
|
|
|
|
y = np.array(y)
|
|
x_id = np.array(x_id_before)
|
|
|
|
results = []
|
|
kf = KFold(len(y), n_folds=10)
|
|
|
|
|
|
def change_flag(flag,ind_threshold,ind_change):
|
|
flag[ind_threshold][ind_change]=True
|
|
|
|
def data_process(change_flag_set,ind_threshold,status):
|
|
if status:
|
|
for ind_change in range(change_break):
|
|
if not change_flag_set[0][ind_threshold][ind_change]:
|
|
right_mine1[ind_threshold][ind_change] = right_mine1[ind_threshold][ind_change] + 1
|
|
if not change_flag_set[1][ind_threshold][ind_change]:
|
|
right_mine2[ind_threshold][ind_change] = right_mine2[ind_threshold][ind_change] + 1
|
|
if not change_flag_set[2][ind_threshold][ind_change]:
|
|
right_mine3[ind_threshold][ind_change] = right_mine3[ind_threshold][ind_change] + 1
|
|
if not change_flag_set[3][ind_threshold][ind_change]:
|
|
right_mine4[ind_threshold][ind_change] = right_mine4[ind_threshold][ind_change] + 1
|
|
if not status:
|
|
for ind_change in range(change_break):
|
|
if change_flag_set[0][ind_threshold][ind_change]:
|
|
right_mine1[ind_threshold][ind_change] = right_mine1[ind_threshold][ind_change] + 1
|
|
if change_flag_set[1][ind_threshold][ind_change]:
|
|
right_mine2[ind_threshold][ind_change] = right_mine2[ind_threshold][ind_change] + 1
|
|
if change_flag_set[2][ind_threshold][ind_change]:
|
|
right_mine3[ind_threshold][ind_change] = right_mine3[ind_threshold][ind_change] + 1
|
|
if change_flag_set[3][ind_threshold][ind_change]:
|
|
right_mine4[ind_threshold][ind_change] = right_mine4[ind_threshold][ind_change] + 1
|
|
|
|
def data_process2(change_flag_set,ind_threshold,status):
|
|
for ind_change in range(change_break):
|
|
if change_flag_set[0][ind_threshold][ind_change] and not status:
|
|
feature_right_mine1[ind_threshold][ind_change] = feature_right_mine1[ind_threshold][ind_change] + 1
|
|
if change_flag_set[1][ind_threshold][ind_change] and not status:
|
|
feature_right_mine2[ind_threshold][ind_change] = feature_right_mine2[ind_threshold][ind_change] + 1
|
|
if change_flag_set[2][ind_threshold][ind_change] and not status:
|
|
feature_right_mine3[ind_threshold][ind_change] = feature_right_mine3[ind_threshold][ind_change] + 1
|
|
if change_flag_set[3][ind_threshold][ind_change] and not status:
|
|
feature_right_mine4[ind_threshold][ind_change] = feature_right_mine4[ind_threshold][ind_change] + 1
|
|
|
|
|
|
turn_count = 0
|
|
print("start training:")
|
|
# ten-fold traning
|
|
for train_index, test_index in kf:
|
|
X_train, X_test = X[train_index], X[test_index]
|
|
y_train, y_test = y[train_index], y[test_index]
|
|
x_id_train, x_id_test = x_id[train_index], x_id[test_index]
|
|
turn_count = turn_count+1
|
|
print('turn '+ repr(turn_count) +':')
|
|
writer_classifier.writerow(['------------------------------'])
|
|
writer_classifier.writerow(['turn ', repr(turn_count) ,':'])
|
|
# print('='*80)
|
|
###############################################################################
|
|
# Benchmark classifiers
|
|
def benchmark(clf):
|
|
print('_' * 80)
|
|
print("Training: ")
|
|
print(clf)
|
|
t0 = time()
|
|
clf.fit(X_train, y_train)
|
|
train_time = time() - t0
|
|
print("train time: %0.3fs" % train_time)
|
|
writer_classifier.writerow(["train time:", train_time])
|
|
|
|
t0 = time()
|
|
pred = clf.predict(X_test)
|
|
|
|
test_time = time() - t0
|
|
print("test time: %0.3fs" % test_time)
|
|
writer_classifier.writerow(["test time:", test_time])
|
|
|
|
score = metrics.accuracy_score(y_test, pred)
|
|
print("accuracy: %0.3f" % score)
|
|
writer_classifier.writerow(["accuracy:", score])
|
|
|
|
probability = clf.predict_proba(X_test)
|
|
|
|
f_mechine = path_data + 'mechine_' + repr(turn_count)
|
|
with open(f_mechine, 'w') as f:
|
|
pickle.dump(clf, f)
|
|
|
|
np.save(path_data + "y_test_"+repr(turn_count),y_test)
|
|
np.save(path_data + "pred_"+repr(turn_count),pred)
|
|
np.save(path_data + "x_id_"+repr(turn_count),x_id_test)
|
|
np.save(path_data + "probability_"+repr(turn_count),probability)
|
|
|
|
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
|
|
|
|
for ind in range(len(pred)):
|
|
# reset flag for each test data
|
|
change_flag1 = [([False] * change_break) for i in range(break_count)]
|
|
change_flag2 = [([False] * change_break) for i in range(break_count)]
|
|
change_flag3 = [([False] * change_break) for i in range(break_count)]
|
|
change_flag4 = [([False] * change_break) for i in range(break_count)]
|
|
change_flag_set = [change_flag1,change_flag2,change_flag3,change_flag4]
|
|
|
|
# diff = np.sort(probability[ind:ind+1])[:1,-1:][0][0]-np.sort(probability[ind:ind+1],)[:1,-2:-1][0][0]
|
|
# get sentence info and split it
|
|
# issue = dao.get_info_by_id(x_id_test[ind])
|
|
# for temp in issue:
|
|
# issue_title = temp[0]
|
|
# issue_body = temp[1]
|
|
info = helper.get_info_by_id(x_id_test[ind],x_id_before,train_data)
|
|
# issue_title = issue[1]
|
|
# issue_body = issue[2]
|
|
# info = issue_title + '.\n' + issue_body
|
|
# info = ''.join(ifilterfalse(unicode.isdigit, info))
|
|
info = helper.filter_str(info)
|
|
info = helper.filter_code(info)
|
|
|
|
sentences = tokenizer.tokenize(info)
|
|
x_test = vect.transform(sentences)
|
|
x_pred = clf.predict(x_test)
|
|
x_prob = clf.predict_proba(x_test)
|
|
diff = probability[ind:ind+1,0][0]-probability[ind:ind+1,1][0]
|
|
|
|
# record split infor
|
|
if len(x_pred)>0:
|
|
for j in range(len(x_pred)):
|
|
if(helper.word_count(sentences[j])>3):
|
|
# data format : ['id','diff','y_test','pred','sentence_pred','sentence_pred_0','sentence_pred_1','sentence']
|
|
data = (x_id_test[ind],diff,y_test[ind],pred[ind],x_pred[j],x_prob[j:j+1,0][0],x_prob[j:j+1,1][0],sentences[j].encode('utf8'))
|
|
|
|
writer.writerow(data)
|
|
|
|
for ind_threshold in range(break_count):
|
|
if abs(diff) <= threshold[ind_threshold]:
|
|
count_base[ind_threshold] = count_base[ind_threshold] + 1
|
|
if int(y_test[ind]) == 1:
|
|
feature_count_base[ind_threshold] = feature_count_base[ind_threshold] +1
|
|
if int(pred[ind]) == 1:
|
|
feature_pred_base[ind_threshold] = feature_pred_base[ind_threshold] + 1
|
|
|
|
if y_test[ind] == pred[ind]:
|
|
right_base[ind_threshold] = right_base[ind_threshold] + 1
|
|
if int(y_test[ind]) == 1:
|
|
feature_right_base[ind_threshold] = feature_right_base[ind_threshold] + 1
|
|
if ind_threshold == 0:
|
|
low_threshold = -0.1
|
|
else:
|
|
low_threshold = threshold[ind_threshold]- 1.0/break_count
|
|
if abs(diff) <= threshold[ind_threshold] and abs(diff) > low_threshold:
|
|
count_base2[ind_threshold] = count_base2[ind_threshold] + 1
|
|
if int(y_test[ind]) == 1:
|
|
feature_count_base2[ind_threshold] = feature_count_base2[ind_threshold] +1
|
|
if int(pred[ind]) == 1:
|
|
feature_pred_base2[ind_threshold] = feature_pred_base2[ind_threshold] + 1
|
|
|
|
if y_test[ind] == pred[ind]:
|
|
right_base2[ind_threshold] = right_base2[ind_threshold] + 1
|
|
if int(y_test[ind]) == 1:
|
|
feature_right_base2[ind_threshold] = feature_right_base2[ind_threshold] + 1
|
|
|
|
# flag for diff
|
|
flag = 'zero'
|
|
if diff > 0:
|
|
flag = "+"
|
|
elif diff < 0:
|
|
flag = "-"
|
|
# print("id:"+repr(x_id[ind]))
|
|
if len(x_pred)>0:
|
|
for j in range(len(x_pred)):
|
|
if(helper.word_count(sentences[j])>3):
|
|
# recording which to change
|
|
for ind_change in range(change_break):
|
|
if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break:
|
|
change_flag(change_flag_set[0],ind_threshold,ind_change)
|
|
if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break and flag != '+':
|
|
change_flag(change_flag_set[1],ind_threshold,ind_change)
|
|
if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break and flag == '-':
|
|
change_flag(change_flag_set[2],ind_threshold,ind_change)
|
|
if pred[ind] == 1 and x_prob[j:j+1,0][0] > ind_change*1.0/change_break and flag == '+':
|
|
change_flag(change_flag_set[3],ind_threshold,ind_change)
|
|
|
|
status = (y_test[ind] == pred[ind])
|
|
data_process(change_flag_set,ind_threshold,status)
|
|
for ind_change in range(change_break):
|
|
if change_flag_set[0][ind_threshold][ind_change]:
|
|
feature_count_mine1[ind_threshold][ind_change] = feature_count_mine1[ind_threshold][ind_change] + 1
|
|
if change_flag_set[1][ind_threshold][ind_change]:
|
|
feature_count_mine2[ind_threshold][ind_change] = feature_count_mine2[ind_threshold][ind_change] + 1
|
|
if change_flag_set[2][ind_threshold][ind_change]:
|
|
feature_count_mine3[ind_threshold][ind_change] = feature_count_mine3[ind_threshold][ind_change] + 1
|
|
# if change_flag_set[3][ind_threshold][ind_change]:
|
|
# feature_count_mine4[ind_threshold][ind_change] = feature_count_mine4[ind_threshold][ind_change] + 1
|
|
|
|
data_process2(change_flag_set,ind_threshold,status)
|
|
|
|
# issue.close()
|
|
clf_descr = str(clf).split('(')[0]
|
|
|
|
return clf_descr, score, train_time, test_time
|
|
##############################################
|
|
results.append(benchmark(svm.SVC(kernel='linear',probability=True)))
|
|
|
|
results = [[x[i] for x in results] for i in range(4)]
|
|
clf_names, score, training_time, test_time = results
|
|
|
|
info_len = len(score)
|
|
training_time = np.array(training_time).sum() / info_len
|
|
test_time = np.array(test_time).sum() / info_len
|
|
score_all = np.array(score).sum()/ info_len
|
|
print("accuracy for all: %0.3f" % score_all)
|
|
writer_classifier.writerow(['------------------------------'])
|
|
writer_classifier.writerow(["traning time for all:", training_time])
|
|
writer_classifier.writerow(["test time for all:", test_time])
|
|
writer_classifier.writerow(["accuracy for all:", score_all])
|
|
|
|
# write result
|
|
writer_result.writerow(['base count for all:'])
|
|
writer_result.writerow([n for n in count_base])
|
|
writer_result.writerow(['base right count for all:'])
|
|
writer_result.writerow([n for n in right_base])
|
|
|
|
writer_result.writerow(['base count for threshold:'])
|
|
writer_result.writerow([n for n in count_base2])
|
|
writer_result.writerow(['base right count for threshold:'])
|
|
writer_result.writerow([n for n in right_base2])
|
|
|
|
|
|
feature_writer_result.writerow(['base count for all:'])
|
|
feature_writer_result.writerow([n for n in feature_count_base])
|
|
feature_writer_result.writerow(['base right count for all:'])
|
|
feature_writer_result.writerow([n for n in feature_right_base])
|
|
feature_writer_result.writerow(['base pred count for all:'])
|
|
feature_writer_result.writerow([n for n in feature_pred_base])
|
|
|
|
feature_writer_result.writerow(['base count for threshold:'])
|
|
feature_writer_result.writerow([n for n in feature_count_base2])
|
|
feature_writer_result.writerow(['base right count for threshold:'])
|
|
feature_writer_result.writerow([n for n in feature_right_base2])
|
|
feature_writer_result.writerow(['base pred count for threshold:'])
|
|
feature_writer_result.writerow([n for n in feature_pred_base2])
|
|
|
|
writer_result.writerow(['right count of mine method 1 for each threshold:'])
|
|
for i in range(break_count):
|
|
writer_result.writerow([l for l in right_mine1[i]])
|
|
writer_result.writerow(['right count of mine method 2 for each threshold:'])
|
|
for i in range(break_count):
|
|
writer_result.writerow([l for l in right_mine2[i]])
|
|
writer_result.writerow(['right count of mine method 3 for each threshold:'])
|
|
for i in range(break_count):
|
|
writer_result.writerow([l for l in right_mine3[i]])
|
|
writer_result.writerow(['right count of mine method 4 for each threshold:'])
|
|
for i in range(break_count):
|
|
writer_result.writerow([l for l in right_mine4[i]])
|
|
|
|
|
|
feature_writer_result.writerow(['right count of mine method 1 for each threshold:'])
|
|
for i in range(break_count):
|
|
feature_writer_result.writerow([l for l in feature_right_mine1[i]])
|
|
feature_writer_result.writerow(['right count of mine method 2 for each threshold:'])
|
|
for i in range(break_count):
|
|
feature_writer_result.writerow([l for l in feature_right_mine2[i]])
|
|
feature_writer_result.writerow(['right count of mine method 3 for each threshold:'])
|
|
for i in range(break_count):
|
|
feature_writer_result.writerow([l for l in feature_right_mine3[i]])
|
|
feature_writer_result.writerow(['right count of mine method 4 for each threshold:'])
|
|
for i in range(break_count):
|
|
feature_writer_result.writerow([l for l in feature_right_mine4[i]])
|
|
|
|
feature_writer_result.writerow(['change count of mine method 1 for each threshold:'])
|
|
for i in range(break_count):
|
|
feature_writer_result.writerow([l for l in feature_count_mine1[i]])
|
|
feature_writer_result.writerow(['change count of mine method 2 for each threshold:'])
|
|
for i in range(break_count):
|
|
feature_writer_result.writerow([l for l in feature_count_mine2[i]])
|
|
feature_writer_result.writerow(['change count of mine method 3 for each threshold:'])
|
|
for i in range(break_count):
|
|
feature_writer_result.writerow([l for l in feature_count_mine3[i]])
|
|
feature_writer_result.writerow(['change count of mine method 4 for each threshold:'])
|
|
for i in range(break_count):
|
|
feature_writer_result.writerow([l for l in feature_count_mine4[i]])
|
|
|
|
csv_result.close()
|
|
csv_path.close()
|
|
csv_classifier.close()
|
|
|
|
|
|
# classifier_project_by_id('6')
|
|
|
|
# break_id = 961
|
|
# projects = dao.get_project()
|
|
# flag_break = True
|
|
# csv_project = file('project_id.csv', 'wb')
|
|
# writer_project = csv.writer(csv_project)
|
|
# for project in projects:
|
|
# print('do classifier for project:'+ repr(project[0]))
|
|
# if project[0] == break_id:
|
|
# flag_break = True
|
|
#
|
|
# if project[1] > 500 and flag_break:
|
|
# try:
|
|
# classifier_project_by_id(project[0])
|
|
# writer_project.writerow(project)
|
|
# except:
|
|
# f=open("log.txt",'a')
|
|
# f.writelines("project:\t"+repr(project[0])+'\n')
|
|
# f.flush()
|
|
# f.close()
|
|
#
|
|
# csv_project.close()
|
|
# projects.close()
|
|
|
|
f_proj_id = file('proj_id.csv', 'r')
|
|
reader = csv.reader(f_proj_id)
|
|
for line in reader:
|
|
classifier_project_by_id(line[0])
|
|
|
|
print(line[0])
|
|
|
|
# classifier_project_by_id('546037')
|
|
dao.close() |