From da0a0934f03e5f5f8670fbafdb53e5c8aed50caa Mon Sep 17 00:00:00 2001 From: qiangge <316257774@qq.com> Date: Wed, 20 Jan 2016 22:06:48 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=E6=96=B0=E7=9A=84?= =?UTF-8?q?=E5=88=86=E7=B1=BB=E6=96=B9=E6=B3=95=EF=BC=8C=E6=A0=B9=E6=8D=AE?= =?UTF-8?q?=E5=8F=8D=E5=BA=94=E9=9C=80=E6=B1=82=E7=9A=84=E5=8F=A5=E5=AD=90?= =?UTF-8?q?=E6=89=80=E5=9C=A8=E7=9A=84=E4=BD=8D=E7=BD=AE=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E7=AD=9B=E9=80=89=20=E6=B7=BB=E5=8A=A0=E4=BA=86read=20me?= =?UTF-8?q?=E6=96=87=E4=BB=B6=EF=BC=8C=E6=96=B9=E4=BE=BF=E9=98=85=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + .idea/workspace.xml | 293 +++++++++++++++++++-------- classifier.py | 2 +- classifier.pyc | Bin 2035 -> 2036 bytes get_analysis_result.py | 43 ++-- read me.txt | 5 + sentence_classifier.py | 2 +- sentence_classifier2.py | 429 ++++++++++++++++++++++++++++++++++++++++ test.py | 0 9 files changed, 673 insertions(+), 102 deletions(-) create mode 100644 read me.txt create mode 100644 sentence_classifier2.py create mode 100644 test.py diff --git a/.gitignore b/.gitignore index d91d394..90728e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ final/ result/ +result*/ result_backup/ .idea \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 60f4bca..37260fb 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -15,10 +15,12 @@ - + + - + + + + + @@ -154,6 +187,8 @@ + + @@ -174,8 +209,6 @@ - - @@ -186,7 +219,7 @@ - + - - - + + + - - - - + + + + - - - + + + - + @@ -496,7 +529,7 @@ - + @@ -506,12 +539,12 @@ - + - + @@ -533,15 +566,92 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -638,13 +748,6 @@ - - - - - - - @@ -652,14 +755,6 @@ - - - - - - - - @@ -675,12 +770,20 @@ - + - + + + + + + + + + - + @@ -688,32 +791,58 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - \ No newline at end of file diff --git a/classifier.py b/classifier.py index 15e047b..8bc4749 100644 --- a/classifier.py +++ b/classifier.py @@ -31,7 +31,7 @@ def data_preprocess(project_name, methold_name = ''): train_target = [] x_id = [] for r in cur.fetchall(): - train_data.append(helper.filter_str(repr(r[1])+"\n"+repr(r[2]))) + train_data.append(helper.filter_str(repr(r[1])+".\n"+repr(r[2]))) train_target.append(r[0]) x_id.append(r[3]) print("data length is : ", len(train_target)) diff --git a/classifier.pyc b/classifier.pyc index 749fd686af0e53ef2063e71db5adffc5a093abca..a6ae919c669571e84f756f31a71febc184c882a0 100644 GIT binary patch delta 27 jcmey&|An8O`7= 4 and (j-1 <=1 or sentences_count-j <=1): + flag_t = True + if sentences_count >= 4 and (abs((j-1)*1.0/(sentences_count-j)) <= 1.0/3 or abs((sentences_count-j)*1.0/(j-1)) <= 1.0/3): + flag_t = True + if flag_t: + flag[ind_threshold][ind_change]=True + + def data_process(change_flag_set,ind_threshold,status): + if status: + for ind_change in range(change_break): + if not change_flag_set[0][ind_threshold][ind_change]: + right_mine1[ind_threshold][ind_change] = right_mine1[ind_threshold][ind_change] + 1 + if not change_flag_set[1][ind_threshold][ind_change]: + right_mine2[ind_threshold][ind_change] = right_mine2[ind_threshold][ind_change] + 1 + if not change_flag_set[2][ind_threshold][ind_change]: + right_mine3[ind_threshold][ind_change] = right_mine3[ind_threshold][ind_change] + 1 + if not change_flag_set[3][ind_threshold][ind_change]: + right_mine4[ind_threshold][ind_change] = right_mine4[ind_threshold][ind_change] + 1 + if not status: + for ind_change in range(change_break): + if change_flag_set[0][ind_threshold][ind_change]: + right_mine1[ind_threshold][ind_change] = right_mine1[ind_threshold][ind_change] + 1 + if change_flag_set[1][ind_threshold][ind_change]: + right_mine2[ind_threshold][ind_change] = right_mine2[ind_threshold][ind_change] + 1 + if change_flag_set[2][ind_threshold][ind_change]: + right_mine3[ind_threshold][ind_change] = right_mine3[ind_threshold][ind_change] + 1 + if change_flag_set[3][ind_threshold][ind_change]: + right_mine4[ind_threshold][ind_change] = right_mine4[ind_threshold][ind_change] + 1 + + def data_process2(change_flag_set,ind_threshold,status): + for ind_change in range(change_break): + if change_flag_set[0][ind_threshold][ind_change] and not status: + feature_right_mine1[ind_threshold][ind_change] = feature_right_mine1[ind_threshold][ind_change] + 1 + if change_flag_set[1][ind_threshold][ind_change] and not status: + feature_right_mine2[ind_threshold][ind_change] = feature_right_mine2[ind_threshold][ind_change] + 1 + if change_flag_set[2][ind_threshold][ind_change] and not status: + feature_right_mine3[ind_threshold][ind_change] = feature_right_mine3[ind_threshold][ind_change] + 1 + if change_flag_set[3][ind_threshold][ind_change] and not status: + feature_right_mine4[ind_threshold][ind_change] = feature_right_mine4[ind_threshold][ind_change] + 1 + + + turn_count = 0 + print("start training:") + # ten-fold traning + for train_index, test_index in kf: + X_train, X_test = X[train_index], X[test_index] + y_train, y_test = y[train_index], y[test_index] + x_id_train, x_id_test = x_id[train_index], x_id[test_index] + turn_count = turn_count+1 + print('turn '+ repr(turn_count) +':') + writer_classifier.writerow(['------------------------------']) + writer_classifier.writerow(['turn ', repr(turn_count) ,':']) + # print('='*80) + ############################################################################### + # Benchmark classifiers + def benchmark(clf): + print('_' * 80) + print("Training: ") + print(clf) + t0 = time() + clf.fit(X_train, y_train) + train_time = time() - t0 + print("train time: %0.3fs" % train_time) + writer_classifier.writerow(["train time:", train_time]) + + t0 = time() + pred = clf.predict(X_test) + + test_time = time() - t0 + print("test time: %0.3fs" % test_time) + writer_classifier.writerow(["test time:", test_time]) + + score = metrics.accuracy_score(y_test, pred) + print("accuracy: %0.3f" % score) + writer_classifier.writerow(["accuracy:", score]) + + probability = clf.predict_proba(X_test) + + f_mechine = path_data + 'mechine_' + repr(turn_count) + with open(f_mechine, 'w') as f: + pickle.dump(clf, f) + + + np.save(path_data + "y_test_"+repr(turn_count),y_test) + np.save(path_data + "pred_"+repr(turn_count),pred) + np.save(path_data + "x_id_"+repr(turn_count),x_id_test) + np.save(path_data + "probability_"+repr(turn_count),probability) + + tokenizer = nltk.data.load('tokenizers/punkt/english.pickle') + + for ind in range(len(pred)): + # reset flag for each test data + change_flag1 = [([False] * change_break) for i in range(break_count)] + change_flag2 = [([False] * change_break) for i in range(break_count)] + change_flag3 = [([False] * change_break) for i in range(break_count)] + change_flag4 = [([False] * change_break) for i in range(break_count)] + change_flag_set = [change_flag1,change_flag2,change_flag3,change_flag4] + + # diff = np.sort(probability[ind:ind+1])[:1,-1:][0][0]-np.sort(probability[ind:ind+1],)[:1,-2:-1][0][0] + # get sentence info and split it + # issue = dao.get_info_by_id(x_id_test[ind]) + # for temp in issue: + # issue_title = temp[0] + # issue_body = temp[1] + info = helper.get_info_by_id(x_id_test[ind],x_id_before,train_data) + # issue_title = issue[1] + # issue_body = issue[2] + # info = issue_title + '.\n' + issue_body + # info = ''.join(ifilterfalse(unicode.isdigit, info)) + info = helper.filter_str(info) + + sentences = tokenizer.tokenize(info) + x_test = vect.transform(sentences) + x_pred = clf.predict(x_test) + x_prob = clf.predict_proba(x_test) + diff = probability[ind:ind+1,0][0]-probability[ind:ind+1,1][0] + + # get num of sentences for dividing body (without title) + sentences_count = len(x_pred)-1 + # record split information + if len(x_pred)>0: + for j in range(len(x_pred)): + if(helper.word_count(sentences[j])>3): + # data format : ['id','diff','y_test','pred','sentence_pred','sentence_pred_0','sentence_pred_1','sentence'] + data = (x_id_test[ind],diff,y_test[ind],pred[ind],x_pred[j],x_prob[j:j+1,0][0],x_prob[j:j+1,1][0],sentences[j]) + + writer.writerow(data) + + for ind_threshold in range(break_count): + if abs(diff) <= threshold[ind_threshold]: + count_base[ind_threshold] = count_base[ind_threshold] + 1 + if int(y_test[ind]) == 1: + feature_count_base[ind_threshold] = feature_count_base[ind_threshold] +1 + if int(pred[ind]) == 1: + feature_pred_base[ind_threshold] = feature_pred_base[ind_threshold] + 1 + + if y_test[ind] == pred[ind]: + right_base[ind_threshold] = right_base[ind_threshold] + 1 + if int(y_test[ind]) == 1: + feature_right_base[ind_threshold] = feature_right_base[ind_threshold] + 1 + if ind_threshold == 0: + low_threshold = -0.1 + else: + low_threshold = threshold[ind_threshold]- 1.0/break_count + if abs(diff) <= threshold[ind_threshold] and abs(diff) > low_threshold: + count_base2[ind_threshold] = count_base2[ind_threshold] + 1 + if int(y_test[ind]) == 1: + feature_count_base2[ind_threshold] = feature_count_base2[ind_threshold] +1 + if int(pred[ind]) == 1: + feature_pred_base2[ind_threshold] = feature_pred_base2[ind_threshold] + 1 + + if y_test[ind] == pred[ind]: + right_base2[ind_threshold] = right_base2[ind_threshold] + 1 + if int(y_test[ind]) == 1: + feature_right_base2[ind_threshold] = feature_right_base2[ind_threshold] + 1 + + # flag for diff + flag = 'zero' + if diff > 0: + flag = "+" + elif diff < 0: + flag = "-" + # print("id:"+repr(x_id[ind])) + if len(x_pred)>0: + for j in range(len(x_pred)): + if(helper.word_count(sentences[j])>3): + # recording which to change + for ind_change in range(change_break): + if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break: + change_flag(change_flag_set[0],ind_threshold,ind_change,j,sentences_count) + if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break and flag != '+': + change_flag(change_flag_set[1],ind_threshold,ind_change,j,sentences_count) + if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break and flag == '-': + change_flag(change_flag_set[2],ind_threshold,ind_change,j,sentences_count) + # if pred[ind] == 1 and x_prob[j:j+1,0][0] > ind_change*1.0/change_break and flag == '+': + # change_flag(change_flag_set[3],ind_threshold,ind_change) + + status = (y_test[ind] == pred[ind]) + data_process(change_flag_set,ind_threshold,status) + for ind_change in range(change_break): + if change_flag_set[0][ind_threshold][ind_change]: + feature_count_mine1[ind_threshold][ind_change] = feature_count_mine1[ind_threshold][ind_change] + 1 + if change_flag_set[1][ind_threshold][ind_change]: + feature_count_mine2[ind_threshold][ind_change] = feature_count_mine2[ind_threshold][ind_change] + 1 + if change_flag_set[2][ind_threshold][ind_change]: + feature_count_mine3[ind_threshold][ind_change] = feature_count_mine3[ind_threshold][ind_change] + 1 + if change_flag_set[3][ind_threshold][ind_change]: + feature_count_mine4[ind_threshold][ind_change] = feature_count_mine4[ind_threshold][ind_change] + 1 + + data_process2(change_flag_set,ind_threshold,status) + + # issue.close() + clf_descr = str(clf).split('(')[0] + + return clf_descr, score, train_time, test_time +############################################## + results.append(benchmark(svm.SVC(kernel='linear',probability=True))) + + results = [[x[i] for x in results] for i in range(4)] + clf_names, score, training_time, test_time = results + + info_len = len(score) + training_time = np.array(training_time).sum() / info_len + test_time = np.array(test_time).sum() / info_len + score_all = np.array(score).sum()/ info_len + print("accuracy for all: %0.3f" % score_all) + writer_classifier.writerow(['------------------------------']) + writer_classifier.writerow(["traning time for all:", training_time]) + writer_classifier.writerow(["test time for all:", test_time]) + writer_classifier.writerow(["accuracy for all:", score_all]) + + # write result + writer_result.writerow(['base count for all:']) + writer_result.writerow([n for n in count_base]) + writer_result.writerow(['base right count for all:']) + writer_result.writerow([n for n in right_base]) + + writer_result.writerow(['base count for threshold:']) + writer_result.writerow([n for n in count_base2]) + writer_result.writerow(['base right count for threshold:']) + writer_result.writerow([n for n in right_base2]) + + + feature_writer_result.writerow(['base count for all:']) + feature_writer_result.writerow([n for n in feature_count_base]) + feature_writer_result.writerow(['base right count for all:']) + feature_writer_result.writerow([n for n in feature_right_base]) + feature_writer_result.writerow(['base pred count for all:']) + feature_writer_result.writerow([n for n in feature_pred_base]) + + feature_writer_result.writerow(['base count for threshold:']) + feature_writer_result.writerow([n for n in feature_count_base2]) + feature_writer_result.writerow(['base right count for threshold:']) + feature_writer_result.writerow([n for n in feature_right_base2]) + feature_writer_result.writerow(['base pred count for threshold:']) + feature_writer_result.writerow([n for n in feature_pred_base2]) + + writer_result.writerow(['right count of mine method 1 for each threshold:']) + for i in range(break_count): + writer_result.writerow([l for l in right_mine1[i]]) + writer_result.writerow(['right count of mine method 2 for each threshold:']) + for i in range(break_count): + writer_result.writerow([l for l in right_mine2[i]]) + writer_result.writerow(['right count of mine method 3 for each threshold:']) + for i in range(break_count): + writer_result.writerow([l for l in right_mine3[i]]) + writer_result.writerow(['right count of mine method 4 for each threshold:']) + for i in range(break_count): + writer_result.writerow([l for l in right_mine4[i]]) + + + feature_writer_result.writerow(['right count of mine method 1 for each threshold:']) + for i in range(break_count): + feature_writer_result.writerow([l for l in feature_right_mine1[i]]) + feature_writer_result.writerow(['right count of mine method 2 for each threshold:']) + for i in range(break_count): + feature_writer_result.writerow([l for l in feature_right_mine2[i]]) + feature_writer_result.writerow(['right count of mine method 3 for each threshold:']) + for i in range(break_count): + feature_writer_result.writerow([l for l in feature_right_mine3[i]]) + feature_writer_result.writerow(['right count of mine method 4 for each threshold:']) + for i in range(break_count): + feature_writer_result.writerow([l for l in feature_right_mine4[i]]) + + feature_writer_result.writerow(['change count of mine method 1 for each threshold:']) + for i in range(break_count): + feature_writer_result.writerow([l for l in feature_count_mine1[i]]) + feature_writer_result.writerow(['change count of mine method 2 for each threshold:']) + for i in range(break_count): + feature_writer_result.writerow([l for l in feature_count_mine2[i]]) + feature_writer_result.writerow(['change count of mine method 3 for each threshold:']) + for i in range(break_count): + feature_writer_result.writerow([l for l in feature_count_mine3[i]]) + feature_writer_result.writerow(['change count of mine method 4 for each threshold:']) + for i in range(break_count): + feature_writer_result.writerow([l for l in feature_count_mine4[i]]) + + csv_result.close() + csv_path.close() + csv_classifier.close() + + +# classifier_project_by_id('6') + +# break_id = 961 +# projects = dao.get_project() +# flag_break = True +# csv_project = file('project_id.csv', 'wb') +# writer_project = csv.writer(csv_project) +# for project in projects: +# print('do classifier for project:'+ repr(project[0])) +# if project[0] == break_id: +# flag_break = True +# +# if project[1] > 500 and flag_break: +# try: +# classifier_project_by_id(project[0]) +# writer_project.writerow(project) +# except: +# f=open("log.txt",'a') +# f.writelines("project:\t"+repr(project[0])+'\n') +# f.flush() +# f.close() +# +# csv_project.close() +# projects.close() + +f_proj_id = file('proj_id.csv', 'r') +reader = csv.reader(f_proj_id) +for line in reader: + classifier_project_by_id(line[0]) + + print(line[0]) +dao.close() \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..e69de29