srilm command

Yu

.gitignore

@@ -1,8 +0,0 @@
-final/
-result/
-result*/
-result_backup/
-.idea/
-test/
-final*/
-classifier_for_yu.py

.idea/.name

@@ -1 +0,0 @@
-word_process

.idea/misc.xml

@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectLevelVcsManager" settingsEditedManually="false">
-    <OptionsSetting value="true" id="Add" />
-    <OptionsSetting value="true" id="Remove" />
-    <OptionsSetting value="true" id="Checkout" />
-    <OptionsSetting value="true" id="Update" />
-    <OptionsSetting value="true" id="Status" />
-    <OptionsSetting value="true" id="Edit" />
-    <ConfirmationsSetting value="0" id="Add" />
-    <ConfirmationsSetting value="0" id="Remove" />
-  </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.10 virtualenv at E:\Program Files\Canopy" project-jdk-type="Python SDK" />
-</project>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/word_process.iml" filepath="$PROJECT_DIR$/.idea/word_process.iml" />
-    </modules>
-  </component>
-</project>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="" />
-  </component>
-</project>

.idea/word_process.iml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

FeatureAnalysis/ExtractCorpusFromCode.py

@@ -0,0 +1,68 @@
+'''
+Created on 2016/1/25
+
+@author: Fisher Yu
+'''
+
+proj_path = "D:/features/projects/"
+cloc_exe = "D:/features/cloc.exe --strip-comments=nc --original-dir "
+git_exe = "C:/Program Files (x86)/Git/bin/git.exe diff"
+
+import subprocess
+def StripCommentExtractor(proj_name):
+    #mdir = proj_path + proj_name
+    mdir = cloc_exe + "D:/features/di"
+    #print mdir   
+    cmd = subprocess.Popen(mdir, stderr=subprocess.PIPE)
+    if cmd.stderr:
+        error = cmd.stderr.readlines()
+        error = ''.join(error)
+        if "error" in error:
+            print error
+        else:
+            pass
+
+import os
+import difflib
+from unidiff import PatchSet
+def ExtractCommentFromDiff(proj_name):
+    path = "D:/features/di"
+    if not os.path.isdir(path):
+        print "not a file path, error"
+        return
+    for root, dirs, files in os.walk(path):
+        for sfile in files:
+            #print file
+            subf = sfile.split(".")
+            if subf[-1] != "nc":
+                #print sfile
+                source_file = os.path.join(root, sfile)
+                nc_file = source_file + ".nc"
+                if os.path.exists(nc_file):
+                    df = difflib.ndiff(open(source_file).readlines(), \
+                                         open(nc_file).readlines())
+                    diff = list(df)
+                    comments = []
+                    for line_diff in diff:
+                        if line_diff.startswith("-"):
+                            comments.append(line_diff)
+                            print line_diff
+                   
+                    
+                    '''
+                    git_cmd = "%s %s %s" % (git_exe, source_file, nc_file)
+                    print git_cmd
+                    cmd = subprocess.Popen(git_cmd, stdout=subprocess.PIPE)
+                    if cmd.stdout:
+                        diff_text = cmd.stdout.readlines()
+                        print diff_text
+                        #os.system('pause')
+                    else:
+                        print "no diff"
+                    '''            
+                #os.remove(dir)
+                #print dir+' remove'
+    
+            
+ExtractCommentFromDiff("")
+#StripCommentExtractor("11")

FeatureAnalysis/FeatureNLP.py

@@ -0,0 +1,61 @@
+'''
+Created on 2016/1/26
+
+@author: Fisher Yu
+'''
+
+from pynlpl.formats.sonar import Corpus
+from pynlpl.lm.lm import SimpleLanguageModel
+
+sentence = 'this is a foo bar sentences and i want to ngramize it.'
+t = 'this is a foo bar sentences and i want to ngramize it.'
+lm = SimpleLanguageModel()
+file = "C:\\Users\\Administrator\\Desktop\\kenlm-tools\\bible.en.txt"
+#lm.load("C:\\Users\\Administrator\\Desktop\\kenlm-tools\\bible.en.txt")
+lm.append(sentence)
+sc = lm.scoresentence(sentence)
+print sc
+
+
+'''
+import nltk
+from nltk.util import ngrams
+sentence = 'this is a foo bar sentences and i want to ngramize it'
+ts = 'foo i'
+n = 3
+tokenize = nltk.word_tokenize(sentence)
+n_model = ngrams(tokenize, n)
+for grams in n_model:
+    print grams
+'''
+
+
+'''
+import nltk
+
+print "... build"
+brown = nltk.corpus.brown
+corpus = [word.lower() for word in brown.words()]
+
+# Train on 95% f the corpus and test on the rest
+spl = 95*len(corpus)/100
+train = corpus[:100]
+test = corpus[100:120]
+
+# Remove rare words from the corpus
+fdist = nltk.FreqDist(w for w in train)
+vocabulary = set(map(lambda x: x[0], filter(lambda x: x[1] >= 5, fdist.iteritems())))
+
+train = map(lambda x: x if x in vocabulary else "*unknown*", train)
+test = map(lambda x: x if x in vocabulary else "*unknown*", test)
+
+print "... train"
+from nltk.model.ngram import NgramModel
+from nltk.probability import LidstoneProbDist
+
+estimator = lambda fdist, bins: LidstoneProbDist(fdist, 0.2) 
+lm = NgramModel(3, train, estimator=estimator)
+
+print "len(corpus) = %s, len(vocabulary) = %s, len(train) = %s, len(test) = %s" % ( len(corpus), len(vocabulary), len(train), len(test) )
+print "perplexity(test) =", lm.perplexity(test)
+'''

FeatureAnalysis/ngram-mit.py

@@ -0,0 +1,214 @@
+import math
+import random
+from collections import Counter, defaultdict
+
+class KneserNeyLM:
+
+    def __init__(self, highest_order, ngrams, start_pad_symbol='<s>',
+            end_pad_symbol='</s>'):
+        """
+        Constructor for KneserNeyLM.
+
+        Params:
+            highest_order [int] The order of the language model.
+            ngrams [list->tuple->string] Ngrams of the highest_order specified.
+                Ngrams at beginning / end of sentences should be padded.
+            start_pad_symbol [string] The symbol used to pad the beginning of
+                sentences.
+            end_pad_symbol [string] The symbol used to pad the beginning of
+                sentences.
+        """
+        self.highest_order = highest_order
+        self.start_pad_symbol = start_pad_symbol
+        self.end_pad_symbol = end_pad_symbol
+        self.lm = self.train(ngrams)
+
+    def train(self, ngrams):
+        """
+        Train the language model on the given ngrams.
+
+        Params:
+            ngrams [list->tuple->string] Ngrams of the highest_order specified.
+        """
+        kgram_counts = self._calc_adj_counts(Counter(ngrams))
+        probs = self._calc_probs(kgram_counts)
+        return probs
+
+    def highest_order_probs(self):
+        return self.lm[0]
+
+    def _calc_adj_counts(self, highest_order_counts):
+        """
+        Calculates the adjusted counts for all ngrams up to the highest order.
+
+        Params:
+            highest_order_counts [dict{tuple->string, int}] Counts of the highest
+                order ngrams.
+
+        Returns:
+            kgrams_counts [list->dict] List of dict from kgram to counts
+                where k is in descending order from highest_order to 0.
+        """
+        kgrams_counts = [highest_order_counts]
+        for i in range(1, self.highest_order):
+            last_order = kgrams_counts[-1]
+            new_order = defaultdict(int)
+            for ngram in last_order.keys():
+                suffix = ngram[1:]
+                new_order[suffix] += 1
+            kgrams_counts.append(new_order)
+        return kgrams_counts
+
+    def _calc_probs(self, orders):
+        """
+        Calculates interpolated probabilities of kgrams for all orders.
+        """
+        backoffs = []
+        for order in orders[:-1]:
+            backoff = self._calc_order_backoff_probs(order)
+            backoffs.append(backoff)
+        orders[-1] = self._calc_unigram_probs(orders[-1])
+        backoffs.append(defaultdict(int))
+        self._interpolate(orders, backoffs)
+        return orders
+
+    def _calc_unigram_probs(self, unigrams):
+        sum_vals = sum(v for v in unigrams.values())
+        unigrams = dict((k, math.log(v/sum_vals)) for k, v in unigrams.items())
+        return unigrams
+
+    def _calc_order_backoff_probs(self, order):
+        num_kgrams_with_count = Counter(
+            value for value in order.values() if value <= 4)
+        discounts = self._calc_discounts(num_kgrams_with_count)
+        prefix_sums = defaultdict(int)
+        backoffs = defaultdict(int)
+        for key in order.keys():
+            prefix = key[:-1]
+            count = order[key]
+            prefix_sums[prefix] += count
+            discount = self._get_discount(discounts, count)
+            order[key] -= discount
+            backoffs[prefix] += discount
+        for key in order.keys():
+            prefix = key[:-1]
+            print order[key], prefix_sums[prefix]
+            if order[key] != 0:
+                order[key] = math.log(order[key]/prefix_sums[prefix])
+        for prefix in backoffs.keys():
+            backoffs[prefix] = math.log(backoffs[prefix]/prefix_sums[prefix])
+        return backoffs
+
+    def _get_discount(self, discounts, count):
+        if count > 3:
+            return discounts[3]
+        return discounts[count]
+
+    def _calc_discounts(self, num_with_count):
+        """
+        Calculate the optimal discount values for kgrams with counts 1, 2, & 3+.
+        """
+        common = num_with_count[1]/(num_with_count[1] + 2 * num_with_count[2])
+        # Init discounts[0] to 0 so that discounts[i] is for counts of i
+        discounts = [0]
+        for i in range(1, 4):
+            if num_with_count[i] == 0:
+                discount = 0
+            else:
+                discount = (i - (i + 1) * common
+                        * num_with_count[i + 1] / num_with_count[i])
+            discounts.append(discount)
+        if any(d for d in discounts[1:] if d <= 0):
+            raise Exception(
+                '***Warning*** Non-positive discounts detected. '
+                'Your dataset is probably too small.')
+        return discounts
+
+    def _interpolate(self, orders, backoffs):
+        """
+        """
+        for last_order, order, backoff in zip(
+                reversed(orders), reversed(orders[:-1]), reversed(backoffs[:-1])):
+            for kgram in order.keys():
+                prefix, suffix = kgram[:-1], kgram[1:]
+                order[kgram] += last_order[suffix] + backoff[prefix]
+
+    def logprob(self, ngram):
+        for i, order in enumerate(self.lm):
+            if ngram[i:] in order:
+                return order[ngram[i:]]
+        return None
+
+    def score_sent(self, sent):
+        """
+        Return log prob of the sentence.
+
+        Params:
+            sent [tuple->string] The words in the unpadded sentence.
+        """
+        padded = (
+            (self.start_pad_symbol,) * (self.highest_order - 1) + sent +
+            (self.end_pad_symbol,))
+        sent_logprob = 0
+        for i in range(len(sent) - self.highest_order + 1):
+            ngram = sent[i:i+self.highest_order]
+            sent_logprob += self.logprob(ngram)
+        return sent_logprob
+
+    def generate_sentence(self, min_length=4):
+        """
+        Generate a sentence using the probabilities in the language model.
+
+        Params:
+            min_length [int] The mimimum number of words in the sentence.
+        """
+        sent = []
+        probs = self.highest_order_probs()
+        while len(sent) < min_length + self.highest_order:
+            sent = [self.start_pad_symbol] * (self.highest_order - 1)
+            # Append first to avoid case where start & end symbal are same
+            sent.append(self._generate_next_word(sent, probs))
+            while sent[-1] != self.end_pad_symbol:
+                sent.append(self._generate_next_word(sent, probs))
+        sent = ' '.join(sent[(self.highest_order - 1):-1])
+        return sent
+
+    def _get_context(self, sentence):
+        """
+        Extract context to predict next word from sentence.
+
+        Params:
+            sentence [tuple->string] The words currently in sentence.
+        """
+        return sentence[(len(sentence) - self.highest_order + 1):]
+
+    def _generate_next_word(self, sent, probs):
+        context = tuple(self._get_context(sent))
+        pos_ngrams = list(
+            (ngram, logprob) for ngram, logprob in probs.items()
+            if ngram[:-1] == context)
+        # Normalize to get conditional probability.
+        # Subtract max logprob from all logprobs to avoid underflow.
+        _, max_logprob = max(pos_ngrams, key=lambda x: x[1])
+        pos_ngrams = list(
+            (ngram, math.exp(prob - max_logprob)) for ngram, prob in pos_ngrams)
+        total_prob = sum(prob for ngram, prob in pos_ngrams)
+        pos_ngrams = list(
+            (ngram, prob/total_prob) for ngram, prob in pos_ngrams)
+        rand = random.random()
+        for ngram, prob in pos_ngrams:
+            rand -= prob
+            if rand < 0:
+                return ngram[-1]
+        return ngram[-1]
+
+from nltk.corpus import gutenberg
+from nltk.util import ngrams
+#from kneser_ney import KneserNeyLM
+
+gut_ngrams = (
+    ngram for sent in gutenberg.sents() for ngram in ngrams(sent, 3,
+    pad_left=True, pad_right=True, pad_symbol='<s>'))
+lm = KneserNeyLM(3, gut_ngrams, end_pad_symbol='<s>')
+
+#print(lm.score_sent(('This', 'is', 'a', 'sample', 'sentence', '.')))

analysis.py

@@ -1,249 +0,0 @@
-__author__ = 'mac'
-
-import numpy as np
-import pymysql
-import csv
-import helper
-path = ''
-
-
-def get_analysis(proj_id,method):
-    path = 'result/'+proj_id+'/'+method+'/data/'
-
-    # get variance/different of each results
-    different = []
-    y_test = []
-    pred = []
-    for i in range(1,11,1):
-        probability = np.load(path + 'probability_' + repr(i) + ".npy")
-        y_test_temp = np.load(path + 'y_test_' + repr(i) + ".npy")
-        pred_temp = np.load(path + 'pred_' + repr(i) + ".npy")
-        for j in range(len(probability)):
-            diff = probability[j:j+1,0][0]-probability[j:j+1,1][0]
-            different.append(diff)
-            y_test.append(y_test_temp[j])
-            pred.append(pred_temp[j])
-    rank_result = np.argsort([abs(i) for i in different])
-    acc = np.mean(helper.get_precision(proj_id,method))
-    hard_count = round(len(different)*(1-acc))
-    hard_right = 0
-    easy_right = 0
-    print(len(different))
-    for i, ind in enumerate(rank_result):
-        status = (y_test[ind] == pred[ind])
-        if status:
-            if i < hard_count:
-                hard_right = hard_right+1
-            else:
-                easy_right = easy_right+1
-    return hard_right,hard_count,easy_right,len(different)-hard_count,acc
-
-method = 'svm'
-f_proj_id = file('proj_id.csv', 'r')
-reader = csv.reader(f_proj_id)
-final_path = 'final2/'
-helper.mkdir(final_path)
-f_hard_tongji = file(final_path + 'hard_count_'+method+'.csv','w')
-writer_final = csv.writer(f_hard_tongji)
-writer_final.writerow(['proj_id','hard_right','hard_count','easy_right','easy_count','acc'])
-for line in reader:
-    temp = get_analysis(line[0],method)
-    data = (line[0],temp[0],temp[1],temp[2],temp[3],temp[4])
-    writer_final.writerow(data)
-f_hard_tongji.close()
-
-# get result of whose id = n
-def get_result_of_n(n):
-    print("-"*100)
-    print("get result of issue/pr:"+repr(n))
-    for i in range(1,10,1):
-        y_test = np.load(path + 'y_test_' + repr(i) + ".npy")
-        pred = np.load(path + 'pred_' + repr(i) + ".npy")
-        variance = np.load(path + 'variance' + repr(i) + ".npy")
-        different = np.load(path + 'different' + repr(i) + ".npy")
-        probability = np.load(path + 'probability' + repr(i) + ".npy")
-        x_id = np.load(path + 'x_id_' + repr(i) + ".npy")
-        for j in range(len(x_id)):
-            if x_id[j:j+1] == n:
-                print("x_id:"+repr(x_id[j:j+1].tolist())+"\ty_test:"+repr(y_test[j:j+1].tolist())+"\tpred:"+repr(pred[j:j+1].tolist())+"\tvariance:"+repr(variance[j:j+1].tolist())+"\tdifferent:"+repr(different[j:j+1].tolist()))
-                print(probability[j:j+1])
-
-# get_result_of_n(19150)
-
-# get title and description for selected issues and write in test.csv
-conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='123456', db='zlb_github')
-def get_title_description(issue_id,y_test,pred):
-    print("path:" + path + 'csv_hard.csv')
-    csvfile = file(path + 'csv_hard.csv', 'wb')
-    writer = csv.writer(csvfile)
-    writer.writerow(['id', 'y_test', 'pred', 'title', 'body'])
-
-    cur = conn.cursor()
-    for i in range(len(issue_id)):
-        sql = "select title,body from "\
-              # +table+" where id = " + str(issue_id[i])
-        cur.execute(sql)
-        r = cur.fetchone()
-        if r:
-            data = (issue_id[i], y_test[i], pred[i], r[0], r[1])
-            writer.writerow(data)
-    csvfile.close()
-
-# get result by set threshold of variance
-def get_result_by_variance(n):
-    print("-"*100)
-    print("get result by variance:"+repr(n))
-    for i in range(1,10,1):
-        y_test = np.load(path + 'y_test_' + repr(i) + ".npy")
-        pred = np.load(path + 'pred_' + repr(i) + ".npy")
-        variance = np.load(path + 'variance' + repr(i) + ".npy")
-        different = np.load(path + 'different' + repr(i) + ".npy")
-        probability = np.load(path + 'probability' + repr(i) + ".npy")
-        x_id = np.load(path + 'x_id_' + repr(i) + ".npy")
-        for j in range(len(x_id)):
-            if variance[j:j+1]<n:
-                print("x_id:"+repr(x_id[j:j+1].tolist())+"\ty_test:"+repr(y_test[j:j+1].tolist())+"\tpred:"+repr(pred[j:j+1].tolist())+"\tvariance:"+repr(variance[j:j+1].tolist())+"\tdifferent:"+repr(different[j:j+1].tolist()))
-                print(probability[j:j+1])
-
-# get result by set threshold of different of max and second
-def get_result_by_different(n, count=0, count_diff=0, count_sth=0, count_all=0):
-    path = ''
-    print("-"*100)
-    print("get result by diff:"+repr(n))
-    x_id_set = []
-    y_test_set = []
-    pred_set = []
-    for i in range(1,10,1):
-        y_test = np.load(path + 'y_test_' + repr(i) + ".npy")
-        pred = np.load(path + 'pred_' + repr(i) + ".npy")
-        variance = np.load(path + 'variance' + repr(i) + ".npy")
-        different = np.load(path + 'different' + repr(i) + ".npy")
-        probability = np.load(path + 'probability' + repr(i) + ".npy")
-        x_id = np.load(path + 'x_id_' + repr(i) + ".npy")
-        for j in range(len(x_id)):
-            count_all += 1
-            if pred[j:j+1] == 1:
-                count_sth += 1
-            if different[j:j+1]>=n:
-                count += 1
-                # print("x_id:"+repr(x_id[j:j+1].tolist())+"\ty_test:"+repr(y_test[j:j+1].tolist())+"\tpred:"+repr(pred[j:j+1].tolist())+"\tvariance:"+repr(variance[j:j+1].tolist())+"\tdifferent:"+repr(different[j:j+1].tolist()))
-                # print(probability[j:j+1])
-                if y_test[j:j+1] != pred[j:j+1]:
-                    print("x_id:"+repr(x_id[j:j+1].tolist())+"\ty_test:"+repr(y_test[j:j+1].tolist())+"\tpred:"+repr(pred[j:j+1].tolist())+"\tvariance:"+repr(variance[j:j+1].tolist())+"\tdifferent:"+repr(different[j:j+1].tolist()))
-                    print(probability[j:j+1])
-                    count_diff += 1
-                    x_id_set.append(x_id[j:j+1].tolist()[0])
-                    y_test_set.append(y_test[j:j+1].tolist()[0])
-                    pred_set.append(pred[j:j+1].tolist()[0])
-    print("different > n data count:"+repr(count))
-    print("different > n and wrong pred data count:"+repr(count_diff))
-    print("count sth:"+repr(count_sth))
-    print("all issue count:"+repr(count_all))
-    get_title_description(x_id_set,y_test_set,pred_set)
-
-# get result by set threshold of different of max and second
-def get_result_by_little_different(n, count=0, count_diff=0, count_sth=0, count_all=0):
-    print("-"*100)
-    print("get result by diff:"+repr(n))
-    x_id_set = []
-    y_test_set = []
-    pred_set = []
-    for i in range(1,10,1):
-        y_test = np.load(path + 'y_test_' + repr(i) + ".npy")
-        pred = np.load(path + 'pred_' + repr(i) + ".npy")
-        variance = np.load(path + 'variance' + repr(i) + ".npy")
-        different = np.load(path + 'different' + repr(i) + ".npy")
-        probability = np.load(path + 'probability' + repr(i) + ".npy")
-        x_id = np.load(path + 'x_id_' + repr(i) + ".npy")
-        for j in range(len(x_id)):
-            count_all += 1
-            if pred[j:j+1] == 1:
-                count_sth += 1
-            if different[j:j+1]<=n:
-                count += 1
-                print("x_id:"+repr(x_id[j:j+1].tolist())+"\ty_test:"+repr(y_test[j:j+1].tolist())+"\tpred:"+repr(pred[j:j+1].tolist())+"\tvariance:"+repr(variance[j:j+1].tolist())+"\tdifferent:"+repr(different[j:j+1].tolist()))
-                print(probability[j:j+1])
-                count_diff += 1
-                x_id_set.append(x_id[j:j+1].tolist()[0])
-                y_test_set.append(y_test[j:j+1].tolist()[0])
-                pred_set.append(pred[j:j+1].tolist()[0])
-    print("different < n data count:"+repr(count))
-    print("different < n and wrong pred data count:"+repr(count_diff))
-    print("count sth:"+repr(count_sth))
-    print("all issue count:"+repr(count_all))
-    get_title_description(x_id_set,y_test_set,pred_set)
-
-def get_all_pred_analysis(count_all=0):
-    x_id_set = []
-    y_test_set = []
-    pred_set = []
-    variance_set = []
-    different_set = []
-    probability_set = []
-    for i in range(1,10,1):
-        y_test = np.load(path + 'y_test_' + repr(i) + ".npy")
-        pred = np.load(path + 'pred_' + repr(i) + ".npy")
-        variance = np.load(path + 'variance' + repr(i) + ".npy")
-        different = np.load(path + 'different' + repr(i) + ".npy")
-        probability = np.load(path + 'probability' + repr(i) + ".npy")
-        x_id = np.load(path + 'x_id_' + repr(i) + ".npy")
-        for j in range(len(x_id)):
-            count_all += 1
-            x_id_set.append(x_id[j:j+1].tolist()[0])
-            y_test_set.append(y_test[j:j+1].tolist()[0])
-            pred_set.append(pred[j:j+1].tolist()[0])
-            variance_set.append(variance[j:j+1].tolist()[0])
-            different_set.append(different[j:j+1].tolist()[0])
-            probability_set.append(probability[j:j+1].tolist()[0])
-    print("all issue count:"+repr(count_all))
-    print("path:" + path + 'csv_analysis.csv')
-    csvfile = file(path + 'csv_analysis.csv', 'wb')
-    writer = csv.writer(csvfile)
-    # writer.writerow(['id', 'y_test', 'pred', 'variance', 'different', 'probability', 'title', 'body'])
-    writer.writerow(['id', 'y_test', 'pred', 'variance', 'different', 'probability'])
-
-    cur = conn.cursor()
-    for i in range(len(x_id_set)):
-        sql = "select title,body from "\
-              # +table+" where id = " + str(x_id_set[i])
-        cur.execute(sql)
-        r = cur.fetchone()
-        if r:
-            data = (x_id_set[i], y_test_set[i], pred_set[i], variance_set[i], different_set[i], probability_set[i])
-            writer.writerow(data)
-    csvfile.close()
-
-# def get_pred_rate(break = 10):
-
-
-
-# get_result_by_different(0.6)
-# get_result_by_little_different(0.1)
-# get_all_pred_analysis()
-
-# y_test = np.load(path + 'y_test_' + repr(1) + ".npy")
-# pred = np.load(path + 'pred_' + repr(1) + ".npy")
-# variance = np.load(path + 'variance' + repr(1) + ".npy")
-# different = np.load(path + 'different' + repr(1) + ".npy")
-# probability = np.load(path + 'probability' + repr(1) + ".npy")
-# x_id = np.load(path + 'x_id_' + repr(1) + ".npy")
-# count_diff = 0
-# count_all = 0
-# for j in range(len(y_test)):
-#     if y_test[j:j+1] != pred[j:j+1]:
-#         print("x_id:"+repr(x_id[j:j+1].tolist())+"\ty_test:"+repr(y_test[j:j+1].tolist())+"\tpred:"+repr(pred[j:j+1].tolist())+"\tvariance:"+repr(variance[j:j+1].tolist())+"\tdifferent:"+repr(different[j:j+1].tolist()))
-#         print(probability[j:j+1])
-#         count_diff += 1
-# print("total:"+repr(count_diff))
-# count_diff = 0
-# print("="*20)
-# for j in range(len(y_test)):
-#     if different[j:j+1]<0.3:
-#         count_all += 1
-#         print("y_test:"+repr(y_test[j:j+1].tolist())+"\tpred:"+repr(pred[j:j+1].tolist())+"\tvariance:"+repr(variance[j:j+1].tolist())+"\tdifferent:"+repr(different[j:j+1].tolist()))
-#         if y_test[j:j+1] != pred[j:j+1]:
-#             count_diff += 1
-# print("all:"+repr(count_all))
-# print("different:"+repr(count_diff))
-#
-

classifier.py

@@ -1,251 +0,0 @@
-from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
-import cPickle as pickle
-import helper
-import pymysql
-import dao
-
-# path_oringan = 'result2'
-# preprocess of data
-def data_preprocess(project_name, methold_name = ''):
-
-    # project_name = "owncloud"
-
-    path = 'result/'+project_name+'/'
-    helper.mkdir(path)
-    # if len(methold_name):
-    #     data_path = path + methold_name + '/data/'
-    # else:
-    #     data_path = path + 'data/'
-    # helper.mkdir(data_path)
-
-    f_train = path +'train.pkl'
-    f_target = path + 'target.pkl'
-    f_id = path + 'id.pkl'
-    f_vect = path + 'vect.pkl'
-    f_train_data = path + 'train_data.pkl'
-    f_cur_all = path + 'cur_all.pkl'
-    print('=' * 80)
-    print("get data: ")
-    cur = dao.get_data(project_name)
-    fetchall = cur.fetchall()
-    with open(f_cur_all, 'w') as f:                     # open file with write-mode
-        pickle.dump(fetchall, f)
-
-    train_data = []
-    train_target = []
-    x_id = []
-    for r in fetchall:
-        str = helper.filter_str(r[1]+".\n"+r[2])
-        # str = helper.filter_code(str)
-        train_data.append(str)
-        train_target.append(r[0])
-        x_id.append(r[3])
-    print("data length is : ", len(train_target))
-
-    cur.close()
-
-    print('_' * 80)
-    print("processing data: TF-IDF")
-
-    # categories = ['bug','enhancement','feature','documentation','question','others']
-    categories = ['bug','enhancement']
-    y = [None]*len(train_target)
-    for i in range(len(train_target)):
-        y[i] = (categories.index(train_target[i]))
-
-    with open(f_train_data, 'w') as f:                     # open file with write-mode
-        pickle.dump(train_data, f)
-    # TF-IDF
-    vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
-                                 stop_words='english', tokenizer=helper.tokenize_help)
-    X = vectorizer.fit_transform(train_data)
-
-    # store data preprocess result in file
-
-
-    with open(f_train, 'w') as f:                     # open file with write-mode
-        pickle.dump(X, f)   # serialize and save object
-    with open(f_target, 'w') as f:                     # open file with write-mode
-        pickle.dump(y, f)   # serialize and save object
-    with open(f_id, 'w') as f:                     # open file with write-mode
-        pickle.dump(x_id, f)   # serialize and save object
-    with open(f_vect, 'w') as f:                     # open file with write-mode
-        pickle.dump(vectorizer, f)   # serialize and save object
-
-    print("done")
-
-    return X,y,x_id,vectorizer
-
-def data_preprocess_from_file(project_name, methold_name = ''):
-
-    # project_name = "owncloud"
-
-    path = 'result/'+project_name+'/'
-    helper.mkdir(path)
-    if len(methold_name):
-        data_path = path + methold_name + '/data/'
-    else:
-        data_path = path + 'data/'
-    helper.mkdir(data_path)
-
-
-    f_train = path +'train.pkl'
-    f_target = path + 'target.pkl'
-    f_id = path + 'id.pkl'
-    f_vect = path + 'vect.pkl'
-    f_train_data = path + 'train_data.pkl'
-    print('=' * 80)
-    print("get data: ")
-    cur = dao.get_data(project_name)
-    train_data = []
-    train_target = []
-    x_id = []
-    for r in cur.fetchall():
-        str = helper.filter_str(repr(r[1])+".\n"+repr(r[2]))
-        # str = helper.filter_code(str)
-        train_data.append(str)
-        train_target.append(r[0])
-        x_id.append(r[3])
-    print("data length is : ", len(train_target))
-
-    cur.close()
-
-    print('_' * 80)
-    print("processing data: TF-IDF")
-
-    # categories = ['bug','enhancement','feature','documentation','question','others']
-    categories = ['bug','enhancement']
-    y = [None]*len(train_target)
-    for i in range(len(train_target)):
-        y[i] = (categories.index(train_target[i]))
-
-    with open(f_train_data, 'w') as f:                     # open file with write-mode
-        pickle.dump(train_data, f)
-    # TF-IDF
-    vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
-                                 stop_words='english', tokenizer=helper.tokenize_help)
-    X = vectorizer.fit_transform(train_data)
-
-    # store data preprocess result in file
-
-
-    with open(f_train, 'w') as f:                     # open file with write-mode
-        pickle.dump(X, f)   # serialize and save object
-    with open(f_target, 'w') as f:                     # open file with write-mode
-        pickle.dump(y, f)   # serialize and save object
-    with open(f_id, 'w') as f:                     # open file with write-mode
-        pickle.dump(x_id, f)   # serialize and save object
-    with open(f_vect, 'w') as f:                     # open file with write-mode
-        pickle.dump(vectorizer, f)   # serialize and save object
-
-    print("done")
-
-    return X,y,x_id,vectorizer
-
-
-
-# def main():
-#
-#     data_preprocess()
-#
-#
-#     # load data preprocess result from file
-#     print('get data: ')
-#     X = helper.get_pickle_record(f_train)
-#     y = helper.get_pickle_record(f_target)
-#     x_id = helper.get_pickle_record(f_id)
-#
-#     y = np.array(y)
-#     x_id = np.array(x_id)
-#
-#     print('done')
-#
-#     results = []
-#     kf = KFold(len(y), n_folds=10)
-#
-#     print("start training:")
-#
-#     i = 0
-#
-#     for train_index, test_index in kf:
-#         X_train, X_test = X[train_index], X[test_index]
-#         y_train, y_test = y[train_index], y[test_index]
-#         x_id_train, x_id_test = x_id[train_index], x_id[test_index]
-#         i = i+1
-#         print('turn '+ repr(i) +':')
-#         print('='*80)
-#         ###############################################################################
-#         # Benchmark classifiers
-#         def benchmark(clf):
-#             print('_' * 80)
-#             print("Training: ")
-#             print(clf)
-#             t0 = time()
-#             clf.fit(X_train, y_train)
-#             train_time = time() - t0
-#             print("train time: %0.3fs" % train_time)
-#
-#             t0 = time()
-#             pred = clf.predict(X_test)
-#
-#             test_time = time() - t0
-#             print("test time:  %0.3fs" % test_time)
-#
-#             score = metrics.accuracy_score(y_test, pred)
-#             print("accuracy:   %0.3f" % score)
-#             # np.save(repr(clf.penalty)+"_y_test_"+repr(i),y_test)
-#             # np.save(repr(clf.penalty)+"_pred_"+repr(i),pred)
-#             np.save(path + "y_test_"+repr(i),y_test)
-#             np.save(path + "pred_"+repr(i),pred)
-#             np.save(path + "x_id_"+repr(i),x_id_test)
-#
-#             probability = clf.predict_proba(X_test)
-#             np.save(path + "probability"+repr(i),probability)
-#
-#             print()
-#             clf_descr = str(clf).split('(')[0]
-#
-#             return clf_descr, score, train_time, test_time
-#
-#
-#         # for penalty in ["l2", "l1"]:
-#         #     print('=' * 80)
-#         #     print("%s penalty" % penalty.upper())
-#         #     # Train Liblinear model (svm)
-#         #     results.append(benchmark(svm.LinearSVC(loss='l2', penalty=penalty, dual=False, tol=1e-3)))
-#         results.append(benchmark(svm.SVC(kernel='linear',probability=True)))
-#
-#
-#     indices = np.arange(len(results))
-#
-#     results = [[x[i] for x in results] for i in range(4)]
-#
-#     clf_names, score, training_time, test_time = results
-#     training_time = np.array(training_time) / np.max(training_time)
-#     test_time = np.array(test_time) / np.max(test_time)
-#
-#     score_all = np.array(score).sum()/len(score)
-#     print("accuracy for all:   %0.3f" % score_all)
-#
-#     # plot export
-#
-#     plt.figure(figsize=(12, 8))
-#     plt.title("Score")
-#     plt.barh(indices, score, .2, label="score", color='r')
-#     plt.barh(indices + .3, training_time, .2, label="training time", color='g')
-#     plt.barh(indices + .6, test_time, .2, label="test time", color='b')
-#     plt.yticks(())
-#     plt.legend(loc='best')
-#     plt.subplots_adjust(left=.25)
-#     plt.subplots_adjust(top=.95)
-#     plt.subplots_adjust(bottom=.05)
-#
-#     for i, c in zip(indices, clf_names):
-#         plt.text(-.3, i, c)
-#
-#     plt.show()
-#
-#
-#
-# if __name__ == '__main__':
-#     main()

command-srilm.txt

@@ -0,0 +1,11 @@
+Steps:
+1) sentence spliting into lines;
+2) not sure whether need to do stop-words removal and stemming; 
+3) run: ngram-count -text train.data -order X -lm train.lm -interpolate -kndiscount (or -wbdiscount);
+4) run: ngram -ppl test.data -order X -lm train.lm -debug 1 > ppl_output.txt
+
+------------
+Notes:
+-kndiscount: modified Kneser-Ney discounting
+-wbdiscount: Witten-Bell discounting, good in short text;
+ Jelinek-Mercer smoothing

dao.py

@@ -1,100 +0,0 @@
-__author__ = 'mac'
-import pymysql
-
-# build connection
-conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='123456', db='zlb_github', charset='utf8')
-
-# get issue info by id
-def get_info_by_id(id):
-    cur = conn.cursor()
-    sql = 'select title, body from issues where id = '+repr(id)
-    cur.execute(sql)
-    return cur
-
-
-# get data from database, and issue_type indicate the type of issues
-def get_data(proj_id):
-    cur = conn.cursor()
-    sql = 'select issue_type, title, body, id from issues where project_id = '+repr(proj_id)+\
-          ' and issue_type is not NULL order by rand()'
-    cur.execute(sql)
-    return cur
-
-def get_feature(proj_id):
-    cur = conn.cursor()
-    sql = 'select title, body, id from issues where project_id = '+repr(proj_id)+\
-          ' and issue_type = "enhancement" '
-    cur.execute(sql)
-    return cur
-
-def get_feature_and_bug(proj_id):
-    cur = conn.cursor()
-    sql = 'select title, body, number, issue_type from issues where project_id = '+repr(proj_id)+ \
-            ' and issue_type is not NULL'
-    cur.execute(sql)
-    return cur
-
-def get_bug(proj_id):
-    cur = conn.cursor()
-    sql = 'select title, body, id from issues where project_id = '+repr(proj_id)+\
-          ' and issue_type = "bug"'
-    cur.execute(sql)
-    return cur
-
-def insert_data(proj_id,x_id,different,pred,y_test):
-    table_name = 'project_'+proj_id
-    cur = conn.cursor()
-    for i in range(len(x_id)):
-        sql = 'update '+table_name+ ' set pred_0 = '+ repr(different[i][0]) + \
-                                    ', pred_1 = '+ repr(different[i][1]) +\
-                                    ', pred = ' + repr(pred[i]) + \
-                                    ', y_test = '+ repr(y_test[i])+\
-                                    '  where id = '+ repr(x_id[i])
-        # data = (different[i][0],different[i][1],pred[i],y_test[i],x_id[i])
-        cur.execute(sql)
-        print(i)
-        if (i+1)%1000 == 0:
-            conn.commit()
-    conn.commit()
-
-def close():
-    conn.close()
-
-def get_project():
-    cur = conn.cursor()
-    sql = 'select project_id,COUNT(*) as num from issues ' \
-          'where issue_type is not NULL GROUP BY project_id ORDER BY num desc'
-    cur.execute(sql)
-    return cur
-
-def get_proj_name_by_id(proj_id):
-    cur = conn.cursor()
-    sql = "select CONCAT(user_name,"+repr('\\')+",repo_name) from project where project_id = "+proj_id
-    cur.execute(sql)
-    return cur
-
-def get_all_issue_by_proj_id(proj_id):
-    cur = conn.cursor()
-    sql = "select title, body, id from issues_for_yu where project_id = "+repr(proj_id)+\
-            ' and issue_type is NULL'
-    cur.execute(sql)
-    return cur
-
-def result_handle(pred,test_id):
-    cur = conn.cursor()
-    categories = ['bug','enhancement']      
-    for i in range(len(pred)):
-        issue_type = categories[pred[i]]
-        sql = "update issues_for_yu set issue_type = " + repr(issue_type) + " where id = "+repr(test_id[i])
-        cur.execute(sql)
-        if (i+1)%10000 == 0 or i ==len(pred)-1:
-            conn.commit()  
-    conn.commit()
-
-def save_kmeans_result(x_id,y_pred,proj_id):
-    cur = conn.cursor()
-    for i in range(len(x_id)):
-        # sql = 'update issues set kmeans = ' + repr(y_pred[i]) + ' where number = ' + repr(x_id[i]) + ' and project_id = ' + proj_id
-        sql = 'update numpy_sentences set kmeans = ' + repr(y_pred[i]) + ' where number = ' + repr(x_id[i])
-        cur.execute(sql)
-    conn.commit()

get_analysis_result.py

@@ -1,203 +0,0 @@
-import helper
-import csv
-import dao
-
-__author__ = 'mac'
-
-
-def analysis_process(project_id,method):
-    project_name = project_id
-    path = 'result/'+project_name+'/'+method+'/'
-    path_analysis = path + 'analysis/'
-    # path_data = path + 'data/'
-
-    csv_result = file(path_analysis + 'precision_method.csv', 'rb')
-    result_reader = csv.reader(csv_result)
-    csv_analysis = file(path_analysis + 'analysis.csv', 'wb')
-    writer_analysis = csv.writer(csv_analysis)
-    # writer_classifier.writerow(['classifier infomation for project:',repr(project_name)])
-
-
-    break_count = 20 # cut the different, len(threshold)
-    change_break = 20 # cut the condition for changing class
-
-
-    count_base = [0]*break_count
-    right_base = [0]*break_count
-    count_base2 = [0]*break_count
-    right_base2 = [0]*break_count
-
-    right_mine1 = [([0] * change_break) for i in range(break_count)]
-    right_mine2 = [([0] * change_break) for i in range(break_count)]
-    right_mine3 = [([0] * change_break) for i in range(break_count)]
-    right_mine4 = [([0] * change_break) for i in range(break_count)]
-
-    result_set = [count_base,right_base,count_base2,right_base2,right_mine1,right_mine2,right_mine3,right_mine4]
-
-    status = [-1,0]
-
-    # get data from line according to status
-    def get_data_from_line(line_info,status):
-        if status[0] == 0:
-            result_set[0] = [each for each in line_info]
-        elif status[0] == 1:
-            result_set[1] = [each for each in line_info]
-        elif status[0] == 2:
-            result_set[2] = [each for each in line_info]
-        elif status[0] == 3:
-            result_set[3] = [each for each in line_info]
-        elif status[0] == 4:
-            result_set[4][status[1]] = [each for each in line_info]
-        elif status[0] == 5:
-            result_set[5][status[1]] = [each for each in line_info]
-        elif status[0] == 6:
-            result_set[6][status[1]] = [each for each in line_info]
-        elif status[0] == 7:
-            result_set[7][status[1]] = [each for each in line_info]
-
-
-
-    def handle_line(line, status):
-        if line[0] == "base count for all:":
-            status[0]=0
-            status[1]=0
-        elif line[0]=="base right count for all:":
-            status[0]=1
-            status[1]=0
-        elif line[0]=="base count for threshold:":
-            status[0]=2
-            status[1]=0
-        elif line[0]=="base right count for threshold:":
-            status[0]=3
-            status[1]=0
-        elif line[0]=="right count of mine method 1 for each threshold:":
-            status[0]=4
-            status[1]=0
-        elif line[0]=="right count of mine method 2 for each threshold:":
-            status[0]=5
-            status[1]=0
-        elif line[0]=="right count of mine method 3 for each threshold:":
-            status[0]=6
-            status[1]=0
-        elif line[0]=="right count of mine method 4 for each threshold:":
-            status[0]=7
-            status[1]=0
-        else:
-            get_data_from_line(line,status)
-            status[1]=status[1]+1
-
-    for line in result_reader:
-        handle_line(line,status)
-
-    # print(result_set[4])
-    right_max = []
-    right_max_index = []
-    right_max_methold_index = []
-    final_threshold = 0
-    for ind_threshold in range(break_count):
-        count_base_temp = result_set[2][ind_threshold]
-        right_base_temp = result_set[3][ind_threshold]
-        index_for_methold = []
-        right_methold_1 = max([int(i) for i in result_set[4][ind_threshold]])
-        index_for_methold.append([int(i) for i in result_set[4][ind_threshold]].index(right_methold_1))
-        right_methold_2 = max([int(i) for i in result_set[5][ind_threshold]])
-        index_for_methold.append([int(i) for i in result_set[5][ind_threshold]].index(right_methold_2))
-        right_methold_3 = max([int(i) for i in result_set[6][ind_threshold]])
-        index_for_methold.append([int(i) for i in result_set[6][ind_threshold]].index(right_methold_3))
-        right_methold_4 = max([int(i) for i in result_set[7][ind_threshold]])
-        index_for_methold.append([int(i) for i in result_set[7][ind_threshold]].index(right_methold_4))
-        # get max right count
-        right_methold_max = max([right_methold_1,right_methold_2,right_methold_3,right_methold_4])
-        # record max right count
-        right_max.append(right_methold_max)
-        # record methold index of max right count
-        right_max_methold_index.append([right_methold_1,right_methold_2,right_methold_3,right_methold_4].index(right_methold_max))
-        # record index of methold for max right count
-        right_max_index.append(index_for_methold[right_max_methold_index[ind_threshold]])
-
-        if right_methold_max > int(right_base_temp):
-            final_threshold = ind_threshold
-
-    diff = [0]*break_count
-    temp = 0
-    final_threshold_temp = 0
-    max_temp = 0
-    hard_count = helper.read_hard_count(project_id,method)[0]
-    for ind_threshold in range(final_threshold+1):
-        temp = temp +int(right_max[ind_threshold]) - int(result_set[3][ind_threshold])
-        diff[ind_threshold] = temp
-
-        # if int(result_set[2][ind_threshold]) == 0 or (int(right_max[ind_threshold]) - int(result_set[3][ind_threshold]))*1.0/int(result_set[2][ind_threshold]) >= 0.05:
-        #     final_threshold_temp = ind_threshold
-        if int(result_set[0][ind_threshold]) < int(hard_count):
-            final_threshold_temp = ind_threshold + 1
-
-
-    count_improve = max(diff[0:final_threshold_temp+1])
-    final_threshold = final_threshold_temp
-
-    writer_analysis.writerow(['precision improve:', count_improve*1.0/int(result_set[0][-1])])
-    writer_analysis.writerow(['count improve:', count_improve])
-    writer_analysis.writerow(['final threshold:', final_threshold])
-    writer_analysis.writerow(['final method right count:'])
-    writer_analysis.writerow([i for i in right_max])
-    writer_analysis.writerow(['final methold index for right count:'])
-    writer_analysis.writerow([i for i in right_max_methold_index])
-    writer_analysis.writerow(['final index of methold for right count:'])
-    writer_analysis.writerow([i for i in right_max_index])
-    if int(result_set[0][final_threshold]) == 0:
-        r1 = 0
-    else:
-        r1 = count_improve*1.0/int(result_set[0][final_threshold])
-    return count_improve*1.0/int(result_set[0][-1]),r1,\
-           result_set[0][-1],result_set[0][final_threshold]
-
-def handle_project(line,method):
-    # method = 'svm2'
-    final_path = 'final2/'
-    helper.mkdir(final_path)
-    f_all_precision = file(final_path + 'precision_'+method+'.csv','a')
-    writer_final = csv.writer(f_all_precision)
-    precision_improve = []
-    precision_machine = []
-    p_id = []
-
-    print(line[0])
-    p_id.append(line[0])
-    temp = analysis_process(line[0],method)
-    precision_improve.append(temp)
-    precision_machine.append(helper.get_precision(line[0],method))
-    proj_infos = dao.get_proj_name_by_id(line[0])
-
-
-    for proj_info in proj_infos:
-        proj_name = proj_info[0]
-    # nb = helper.get_result_by_classifier('nb-result',line[0])
-    # rf = helper.get_result_by_classifier('rf-result',line[0])
-    # lrl1 = helper.get_result_by_classifier('lrl1-result',line[0])
-    # lrl2 = helper.get_result_by_classifier('lrl2-result',line[0])
-    # et = helper.get_result_by_classifier('nb-result',line[0])
-    # et_1000 = helper.get_result_by_classifier('nb-result',line[0])
-    # adaboost = helper.get_result_by_classifier('nb-result',line[0])
-    # data = (line[0],line[1],proj_name,analysis_process(line[0]),helper.get_precision(line[0]),et,et_1000,rf,nb,lrl1,lrl2,adaboost)
-    data = (line[0],line[1],proj_name,temp[0],temp[1],helper.get_precision(line[0],method),temp[2],temp[3])
-    writer_final.writerow(data)
-    # classifier_process(line[0])
-    f_all_precision.close()
-
-f_proj_id = file('proj_id.csv', 'r')
-reader = csv.reader(f_proj_id)
-# line = ['6','500']
-method = 'svm'
-final_path = 'final2/'
-helper.mkdir(final_path)
-f_all_precision = file(final_path + 'precision_'+method+'.csv','a')
-writer_final = csv.writer(f_all_precision)
-writer_final.writerow(['proj_id','issue_count','proj_name','improve_prec','improve_prec_part','mechine_prec','num_sample','num_hard'])
-f_all_precision.close()
-for line in reader:
-    writer_final = csv.writer(f_all_precision)
-
-    handle_project(line,method)
-
-f_proj_id.close()

get_feature_analysis_result.py

@@ -1,258 +0,0 @@
-import helper
-import csv
-import dao
-
-__author__ = 'mac'
-
-
-def analysis_process(project_id, methold):
-    project_name = project_id
-    path = 'result2/' + project_name + '/'
-    path_analysis = path + methold + '/analysis/'
-    # path_data = path + 'data/'
-
-    csv_result = file(path_analysis + 'feature_precision_method.csv', 'rb')
-    result_reader = csv.reader(csv_result)
-    csv_analysis = file(path_analysis + 'feature_analysis.csv', 'wb')
-    writer_analysis = csv.writer(csv_analysis)
-
-    csv_analysis_before = file(path_analysis + 'analysis.csv', 'rb')
-    analysis_reader = csv.reader(csv_analysis_before)
-    # writer_classifier.writerow(['classifier infomation for project:',repr(project_name)])
-
-
-    break_count = 20  # cut the different, len(threshold)
-    change_break = 20  # cut the condition for changing class
-
-    feature_count_base = [0] * break_count
-    feature_pred_base = [0] * break_count
-    feature_right_base = [0] * break_count
-    feature_count_base2 = [0] * break_count
-    feature_pred_base2 = [0] * break_count
-    feature_right_base2 = [0] * break_count
-
-    feature_right_mine1 = [([0] * change_break) for i in range(break_count)]
-    feature_right_mine2 = [([0] * change_break) for i in range(break_count)]
-    feature_right_mine3 = [([0] * change_break) for i in range(break_count)]
-    feature_right_mine4 = [([0] * change_break) for i in range(break_count)]
-
-    feature_count_mine1 = [([0] * change_break) for i in range(break_count)]
-    feature_count_mine2 = [([0] * change_break) for i in range(break_count)]
-    feature_count_mine3 = [([0] * change_break) for i in range(break_count)]
-    feature_count_mine4 = [([0] * change_break) for i in range(break_count)]
-
-    result_set = [feature_count_base, feature_pred_base, feature_right_base, feature_count_base2, feature_pred_base2,
-                  feature_right_base2, feature_right_mine1, feature_right_mine2, feature_right_mine3,
-                  feature_right_mine4,
-                  feature_count_mine1, feature_count_mine2, feature_count_mine3, feature_count_mine4]
-
-    status = [-1, 0]
-    flag = 0
-    for line in analysis_reader:
-        if line[0] == 'final threshold:':
-            final_threshold = int(line[1])
-            continue
-        if line[0] == 'final methold index for right count:':
-            flag = 1
-            continue
-        if line[0] == 'final index of methold for right count:':
-            flag = 2
-            continue
-        if flag == 1:
-            right_max_methold_index = [int(i) for i in line]
-            flag = 0
-            continue
-        if flag == 2:
-            right_max_index = [int(i) for i in line]
-            flag = 0
-            continue
-
-    # get data from line according to status
-    def get_data_from_line(line_info, status):
-        if status[0] < 6:
-            result_set[status[0]] = [int(each) for each in line_info]
-        elif status[0] >= 6:
-            result_set[status[0]][status[1]] = [int(each) for each in line_info]
-
-    def handle_line(line, status):
-        if line[0] == "base count for all:":
-            status[0] = 0
-            status[1] = 0
-        elif line[0] == "base right count for all:":
-            status[0] = 1
-            status[1] = 0
-        elif line[0] == "base pred count for all:":
-            status[0] = 2
-            status[1] = 0
-        elif line[0] == "base count for threshold:":
-            status[0] = 3
-            status[1] = 0
-        elif line[0] == "base right count for threshold:":
-            status[0] = 4
-            status[1] = 0
-        elif line[0] == "base pred count for threshold:":
-            status[0] = 5
-            status[1] = 0
-        elif line[0] == "right count of mine method 1 for each threshold:":
-            status[0] = 6
-            status[1] = 0
-        elif line[0] == "right count of mine method 2 for each threshold:":
-            status[0] = 7
-            status[1] = 0
-        elif line[0] == "right count of mine method 3 for each threshold:":
-            status[0] = 8
-            status[1] = 0
-        elif line[0] == "right count of mine method 4 for each threshold:":
-            status[0] = 9
-            status[1] = 0
-        elif line[0] == "change count of mine method 1 for each threshold:":
-            status[0] = 10
-            status[1] = 0
-        elif line[0] == "change count of mine method 2 for each threshold:":
-            status[0] = 11
-            status[1] = 0
-        elif line[0] == "change count of mine method 3 for each threshold:":
-            status[0] = 12
-            status[1] = 0
-        elif line[0] == "change count of mine method 4 for each threshold:":
-            status[0] = 13
-            status[1] = 0
-        else:
-            get_data_from_line(line, status)
-            status[1] = status[1] + 1
-
-    # read data from csv
-    for line in result_reader:
-        handle_line(line, status)
-
-    right_count = 0
-    change_count = 0
-
-    # change record according to get_analysis_result.py
-
-    for ind_threshold in range(int(final_threshold) + 1):
-        x_index = int(right_max_methold_index[ind_threshold])
-        y_index = int(right_max_index[ind_threshold])
-        feature_right_methold = [result_set[6], result_set[7], result_set[8], result_set[9]]
-        feature_count_methold = [result_set[10], result_set[11], result_set[12], result_set[13]]
-        right_count = right_count + int(feature_right_methold[x_index][ind_threshold][y_index])
-        change_count = change_count + int(feature_count_methold[x_index][ind_threshold][y_index])
-
-    part_improved_prf = helper.get_prec_recall_f1(right_count + result_set[1][final_threshold],
-                                                  result_set[0][final_threshold],
-                                                  change_count + result_set[2][final_threshold])
-    part_prf = helper.get_prec_recall_f1(result_set[1][final_threshold], result_set[0][final_threshold],
-                                         result_set[2][final_threshold])
-    improved_prf = helper.get_prec_recall_f1(right_count + result_set[1][-1], result_set[0][-1],
-                                             change_count + result_set[2][-1])
-    prf = helper.get_prec_recall_f1(result_set[1][-1], result_set[0][-1], result_set[2][-1])
-    print(final_threshold)
-    return [part_improved_prf, part_prf, improved_prf, prf, change_count]
-
-    # get best f1 score
-
-    # def get_f1_part(right_count, change_count,ind_threshold):
-    #     f1_part = [[None]]*4
-    #     for j in range(4):
-    #         for i in range(change_break):
-    #             f1_part[j].append(helper.get_prec_recall_f1(right_count[j][ind_threshold][i] + result_set[1][ind_threshold],
-    #                                 result_set[0][ind_threshold],change_count[j][ind_threshold][i] + result_set[2][ind_threshold])[2])
-    #     temp = [max(f1_part[0]),max(f1_part[1]),max(f1_part[2]),max(f1_part[3])]
-    #     max_f1 = max(temp)
-    #     x_index = temp.index(max_f1)
-    #     y_index = f1_part[x_index].index(temp)
-    #     return x_index,y_index,temp
-    #
-    # def get_f1(right_count, change_count,ind_threshold):
-    #     f1 = [[],[],[],[]]
-    #     for j in range(4):
-    #         for i in range(change_break):
-    #             f1[j].append(helper.get_prec_recall_f1(right_count[j][ind_threshold][i] + result_set[1][-1], result_set[0][-1],
-    #                                 change_count[j][ind_threshold][i] + result_set[2][-1])[2])
-    #     temp = [max(f1[0]),max(f1[1]),max(f1[2]),max(f1[3])]
-    #     max_f1 = max(temp)
-    #     x_index = temp.index(max_f1)
-    #     y_index = f1[x_index].index(max_f1)
-    #     return x_index,y_index
-    #
-    # def get_f1_2(right_count, change_count):
-    #     f1 = []
-    #     for i in range(change_break):
-    #         f1.append(helper.get_prec_recall_f1(right_count[i] + result_set[1][-1], result_set[0][-1],
-    #                             change_count[i] + result_set[2][-1])[2])
-    #     return f1.index(max(f1))
-    #
-    # right_method = [0]*break_count
-    # change_method = [0]*break_count
-    # x_index = [0]*break_count
-    # y_index = [0]*break_count
-    #
-    # diff_right = [0]*break_count
-    # temp_right = 0
-    # diff_change = [0]*break_count
-    # temp_change = 0
-    #
-    # for ind_threshold in range(break_count):
-    #     x_index[ind_threshold],y_index[ind_threshold] = get_f1(result_set[6:10],result_set[10:14],ind_threshold)
-    #     right_method[ind_threshold] = result_set[x_index[ind_threshold]+6][ind_threshold][y_index[ind_threshold]]
-    #     change_method[ind_threshold] = result_set[x_index[ind_threshold]+10][ind_threshold][y_index[ind_threshold]]
-    #
-    #     temp_right = temp_right + right_method[ind_threshold]
-    #     diff_right[ind_threshold] = temp_right
-    #     temp_change = temp_change + change_method[ind_threshold]
-    #     diff_change[ind_threshold] = temp_change
-    #
-    # final_threshold = get_f1_2(diff_right,diff_change)
-    # right_count = diff_right[final_threshold]
-    # change_count = diff_change[final_threshold]
-    # part_improved_prf = helper.get_prec_recall_f1(right_count + result_set[1][final_threshold],
-    #                                               result_set[0][final_threshold],
-    #                                               change_count + result_set[2][final_threshold])
-    # part_prf = helper.get_prec_recall_f1(result_set[1][final_threshold], result_set[0][final_threshold],
-    #                                      result_set[2][final_threshold])
-    # improved_prf = helper.get_prec_recall_f1(right_count + result_set[1][-1], result_set[0][-1],
-    #                                          change_count + result_set[2][-1])
-    # prf = helper.get_prec_recall_f1(result_set[1][-1], result_set[0][-1], result_set[2][-1])
-    # return [part_improved_prf, part_prf, improved_prf, prf]
-
-
-method = 'svm'
-f_proj_id = file('proj_id.csv', 'r')
-reader = csv.reader(f_proj_id)
-final_path = 'final2/'
-helper.mkdir(final_path)
-f_all_precision = file(final_path + 'feature_precision_' + method + '.csv', 'w')
-writer_final = csv.writer(f_all_precision)
-precision_improve = []
-precision_machine = []
-p_id = []
-writer_final.writerow(
-    ['proj_id', 'issue_count', 'proj_name','change_count', 'improve_prec_all', 'mechine_prec_all', 'improve_prec', 'mechine_prec',
-     'improve_recall_all', 'mechine_recall_all', 'improve_recall', 'mechine_recall',
-     'improve_f1_all', 'mechine_f1_all', 'improve_f1', 'mechine_f1'])
-
-# line = ['6', '500']
-for line in reader:
-# classifier_process(line[0])
-    print(line[0])
-    p_id.append(line[0])
-    temp = analysis_process(line[0], method)
-    precision_improve.append(temp)
-    # precision_machine.append(helper.get_precision(line[0]))
-    proj_infos = dao.get_proj_name_by_id(line[0])
-    for proj_info in proj_infos:
-        proj_name = proj_info[0]
-# nb = helper.get_result_by_classifier('nb-result',line[0])
-# rf = helper.get_result_by_classifier('rf-result',line[0])
-# lrl1 = helper.get_result_by_classifier('lrl1-result',line[0])
-# lrl2 = helper.get_result_by_classifier('lrl2-result',line[0])
-# et = helper.get_result_by_classifier('nb-result',line[0])
-# et_1000 = helper.get_result_by_classifier('nb-result',line[0])
-# adaboost = helper.get_result_by_classifier('nb-result',line[0])
-    data = (line[0], line[1], proj_name,temp[4], temp[2][0] - temp[3][0], temp[3][0], temp[0][0] - temp[1][0], temp[1][0],
-            temp[2][1] - temp[3][1], temp[3][1], temp[0][1] - temp[1][1], temp[1][1],
-            temp[2][2] - temp[3][2], temp[3][2], temp[0][2] - temp[1][2], temp[1][2])
-    writer_final.writerow(data)
-
-f_all_precision.close()
-f_proj_id.close()

helper.py

@@ -1,170 +0,0 @@
-# -*- coding: utf-8 -*
-import nltk
-
-__author__ = 'mac'
-
-# shell mkdir
-def mkdir(path):
-    import os
-    path=path.strip()
-    path=path.rstrip("\\")
-    isExists=os.path.exists(path)
-
-    if not isExists:
-        print path+': create successfull'
-        os.makedirs(path)
-        return True
-    else:
-        print path+': path already exist'
-        return False
-
-def strQ2B(ustring):
-    rstring = ""
-    for uchar in ustring:
-        inside_code=ord(uchar)
-        if inside_code == 12288:
-            inside_code = 32
-        elif (inside_code >= 65281 and inside_code <= 65374):
-            inside_code -= 65248
-
-        rstring += unichr(inside_code)
-    return rstring
-
-# filter numbers in string
-def filter_str(deal_str):
-    from itertools import ifilterfalse
-    # import re
-    if deal_str.__class__ == unicode:
-        deal_result = ''.join(ifilterfalse(unicode.isdigit, deal_str))
-        deal_result = strQ2B(deal_result)
-    elif deal_str.__class__ == str:
-        deal_result = ''.join(ifilterfalse(str.isdigit, deal_str))
-        deal_result = strQ2B(deal_result)
-    return deal_result
-
-def filter_sign(deal_str):
-    import re
-    deal_result = re.sub("[\s+\/_$%^*\-(+\"\']+|[:：+——！，。？、~@#￥%……&*（）]+".decode("utf8"), " ".decode("utf8"),deal_str)
-    deal_result = ' '.join(deal_result.split())
-    return deal_result
-
-# filter code in string
-def filter_code(deal_str):
-    import re
-    deal_result = re.sub(r'`{3,10}.*?`{3,10}', ' .', deal_str, 100, re.S)
-    return deal_result
-
-# count words of sentence
-def word_count(sentence):
-    import string
-    strip = string.whitespace + string.punctuation + string.digits + "\"'"
-    len_count = 0
-    for word in sentence.split():
-        word = word.strip(strip)
-        if len(word) >= 2:
-            len_count = len_count + 1
-    return len_count
-
-
-# get record of pickle
-def get_pickle_record(path):
-    import cPickle as pickle
-    with open(path, 'r') as f:
-        return  pickle.load(f)   # read file and build object
-
-# stemming
-from nltk import word_tokenize
-from nltk.stem import WordNetLemmatizer, PorterStemmer
-
-
-class LemmaTokenizer(object):
-    def __init__(self):
-        self.wnl = WordNetLemmatizer()
-    def __call__(self, doc):
-        return [self.wnl.lemmatize(t) for t in word_tokenize(doc)]
-
-stemmer = PorterStemmer()
-def stem_tokens(tokens, stemmer):
-    stemmed = []
-    for item in tokens:
-        stemmed.append(stemmer.stem(item))
-    return stemmed
-
-def tokenize_help(text):
-    import string
-    tokens = nltk.word_tokenize(text)
-    tokens = [i for i in tokens if i not in string.punctuation]
-    tokens = [i for i in tokens if len(i) > 2]
-    stems = stem_tokens(tokens, stemmer)
-    return stems
-
-def get_result_by_classifier(clf,proj_id):
-    path = 'result/'+proj_id+'/'+clf+'/'
-    path_analysis = path + 'analysis/'
-    f = open(path_analysis+clf+'.txt', 'r')
-    result = []
-    for line in f.readlines():
-        count,train_time,test_time,prec = line.strip('\n').split(',')
-        if count != 'avg':
-            result.append(prec)
-    return [float(i) for i in result]
-
-def get_precision(proj_id,method):
-    import csv
-    project_name = proj_id
-    path = 'result/'+project_name+'/'+method+'/'
-    path_analysis = path + 'analysis/'
-
-    csv_classifier = file(path_analysis + 'classifier_info.csv', 'rb')
-    classifier_reader = csv.reader(csv_classifier)
-
-    prec = []
-    for l in classifier_reader:
-        if l[0] == 'accuracy:':
-            prec.append(l[1])
-    return [float(i) for i in prec]
-
-def get_tfidf_data(project_name):
-    path = 'result/'+project_name+'/'
-
-    f_train = path +'train.pkl'
-    f_target = path + 'target.pkl'
-    f_id = path + 'id.pkl'
-    f_vect = path + 'vect.pkl'
-
-    # load data preprocess result from file
-    print('get data: ')
-    X = get_pickle_record(f_train)
-    y = get_pickle_record(f_target)
-    x_id = get_pickle_record(f_id)
-    vect = get_pickle_record(f_vect)
-
-    print('done')
-
-    return X,y,x_id,vect
-
-def get_info_by_id(id,x_id,train_data):
-    return train_data[x_id.index(id)]
-
-def get_prec_recall_f1(right_count,all_count,pred_count):
-    prec = 0
-    recall = 0
-    f1 = 0
-    if pred_count != 0:
-        prec = right_count*1.0/pred_count
-    if all_count != 0:
-        recall = right_count*1.0/all_count
-    if prec+recall != 0:
-        f1 = 2*prec*recall/(prec+recall)
-    return prec,recall,f1
-
-def read_hard_count(proj_id,methold):
-    import csv
-    path = 'final2/'
-
-    csv_classifier = file(path + 'hard_count_'+methold+'.csv', 'rb')
-    classifier_reader = csv.reader(csv_classifier)
-    for l in classifier_reader:
-        if l[0] == proj_id:
-            return(l[1],l[2],l[3],l[4])
-

insert_result_to_db.py

@@ -1,29 +0,0 @@
-__author__ = 'qiangge'
-import numpy as np
-import pymysql
-import csv
-import helper
-import dao
-
-def get_analysis(proj_id,method):
-    path = 'result/'+proj_id+'/'+method+'/data/'
-
-    # get variance/different of each results
-    different = []
-    y_test = []
-    pred = []
-    x_id = []
-    for i in range(1,11,1):
-        probability = np.load(path + 'probability_' + repr(i) + ".npy")
-        y_test_temp = np.load(path + 'y_test_' + repr(i) + ".npy")
-        pred_temp = np.load(path + 'pred_' + repr(i) + ".npy")
-        x_id_temp = np.load(path + 'x_id_' + repr(i) + ".npy")
-        for j in range(len(probability)):
-            # diff = probability[j:j+1,0][0]-probability[j:j+1,1][0]
-            different.append(probability[j])
-            y_test.append(y_test_temp[j])
-            pred.append(pred_temp[j])
-            x_id.append(x_id_temp[j])
-    dao.insert_data(proj_id,x_id,different,pred,y_test)
-
-get_analysis('524804','svm')

kmeans.py

@@ -1,87 +0,0 @@
-from sklearn.cluster import KMeans
-from sklearn.feature_extraction.text import TfidfVectorizer
-import helper
-import cPickle as pickle
-import dao
-
-
-
-
-def data_preprocess(project_name, methold_name = ''):
-
-    # project_name = "owncloud"
-
-    path = 'result/'+project_name+'/'
-    helper.mkdir(path)
-    # if len(methold_name):
-    #     data_path = path + methold_name + '/data/'
-    # else:
-    #     data_path = path + 'data/'
-    # helper.mkdir(data_path)
-
-    f_train = path +'train.pkl'
-    f_target = path + 'target.pkl'
-    f_id = path + 'id.pkl'
-    f_vect = path + 'vect.pkl'
-    f_train_data = path + 'train_data.pkl'
-    f_cur_all = path + 'cur_all.pkl'
-    print('=' * 80)
-    print("get data: ")
-    cur = dao.get_data(project_name)
-    fetchall = cur.fetchall()
-    with open(f_cur_all, 'w') as f:                     # open file with write-mode
-        pickle.dump(fetchall, f)
-
-    train_data = []
-    train_target = []
-    x_id = []
-    for r in fetchall:
-        str = helper.filter_str(r[1]+".\n"+r[2])
-        # str = helper.filter_code(str)
-        train_data.append(str)
-        train_target.append(r[0])
-        x_id.append(r[3])
-    print("data length is : ", len(train_target))
-
-    cur.close()
-
-    print('_' * 80)
-    print("processing data: TF-IDF")
-
-    # categories = ['bug','enhancement','feature','documentation','question','others']
-    categories = ['bug','enhancement']
-    y = [None]*len(train_target)
-    for i in range(len(train_target)):
-        y[i] = (categories.index(train_target[i]))
-
-    with open(f_train_data, 'w') as f:                     # open file with write-mode
-        pickle.dump(train_data, f)
-    # TF-IDF
-    vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
-                                 stop_words='english', tokenizer=helper.tokenize_help)
-    X = vectorizer.fit_transform(train_data)
-
-    # store data preprocess result in file
-
-
-    with open(f_train, 'w') as f:                     # open file with write-mode
-        pickle.dump(X, f)   # serialize and save object
-    with open(f_target, 'w') as f:                     # open file with write-mode
-        pickle.dump(y, f)   # serialize and save object
-    with open(f_id, 'w') as f:                     # open file with write-mode
-        pickle.dump(x_id, f)   # serialize and save object
-    with open(f_vect, 'w') as f:                     # open file with write-mode
-        pickle.dump(vectorizer, f)   # serialize and save object
-
-    print("done")
-
-    return X,y,x_id,vectorizer
-
-
-project_name = "11450"
-X,y,x_id,vect = helper.get_tfidf_data(project_name)
- # kmeans
-y_pred = KMeans(n_clusters= 10).fit_predict(X)
-
-# dao save in db
-dao.save_kmeans_result(x_id,y_pred,project_name)

lda_example.py

@@ -1,126 +0,0 @@
-from nltk.tokenize import RegexpTokenizer
-from stop_words import get_stop_words
-from nltk.stem.porter import PorterStemmer
-from gensim import corpora, models
-import gensim
-from sklearn.cluster import KMeans
-import dao
-import csv
-import helper
-
-
-
-def lda_process(doc_set,f_name,n_topic,proj_id):
-
-    tokenizer = RegexpTokenizer(r'\w+')
-
-    # create English stop words list
-    en_stop = get_stop_words('en')
-
-    # Create p_stemmer of class PorterStemmer
-    p_stemmer = PorterStemmer()
-
-    # list for tokenized documents in loop
-    texts = []
-    x_id = []
-    issue_type = []
-
-    # loop through document list
-    for r in doc_set:
-        i = r[0]+'. '+r[1]
-        # clean and tokenize document string
-        raw = i.lower()
-        raw = helper.filter_code(raw)
-        raw = helper.filter_sign(raw)
-        tokens = tokenizer.tokenize(raw)
-
-        # remove num from stopped_tokens
-        numed_tokens = [i for i in tokens if not helper.filter_str(i) == '' ]
-
-        # remove stop words from tokens
-        stopped_tokens = [i for i in numed_tokens if not i in en_stop]
-
-        # stem tokens
-        stemmed_tokens = [p_stemmer.stem(i) for i in stopped_tokens]
-
-        # add tokens to list
-        texts.append(stemmed_tokens)
-
-        x_id.append(r[2])
-        issue_type.append(r[3])
-
-    # turn our tokenized documents into a id <-> term dictionary
-    dictionary = corpora.Dictionary(texts)
-
-    # filter word that frequency is more than len(doc_set)/4,and less than 3
-    dictionary.filter_extremes(3,0.95)
-
-    # convert tokenized documents into a document-term matrix
-    corpus = [dictionary.doc2bow(text) for text in texts]
-
-    # generate LDA model
-    ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=n_topic, id2word = dictionary, passes=20)
-
-    # save model
-    ldamodel.save(f_name)
-
-    X = []
-    # get probability of every topics for each doc
-    for bow in corpus:
-        doc_topic = ldamodel.get_document_topics(bow)
-        # doc_param = [0.0]*10
-        doc_param = [0.0 for i in range(n_topic)]
-        for doc_t in doc_topic:
-            doc_param[doc_t[0]] = doc_t[1]
-        X.append(doc_param)
-
-    # kmeans
-    y_pred = KMeans(n_clusters= 20).fit_predict(X)
-
-    # dao save in db
-    dao.save_kmeans_result(x_id,y_pred,proj_id)
-
-# f_proj_id = file('proj_id.csv', 'r')
-# reader = csv.reader(f_proj_id)
-# for line in reader:
-proj_id = '6013'
-    # proj_id = line[0]
-path = 'result/'+proj_id+'/lda/'
-helper.mkdir(path)
-print("project: "+proj_id)
-# cur_feature = dao.get_feature(proj_id)
-# doc_feature = cur_feature.fetchall()
-# lda_process(doc_feature,path+"feature_name_5.lda",5)
-cur_bug = dao.get_feature_and_bug(proj_id)
-doc_bug = cur_bug.fetchall()
-print(len(doc_bug))
-lda_process(doc_bug,path+"bug_feature_name_50.lda",50,proj_id)
-
-# def do_process(proj_id):
-#     path = 'result/'+proj_id+'/lda/'
-#     helper.mkdir(path)
-#     print("project: "+proj_id)
-#     cur_feature = dao.get_feature(proj_id)
-#     doc_feature = cur_feature.fetchall()
-#     lda_process(doc_feature,path+"feature_name_5.lda",5)
-#     lda_process(doc_feature,path+"feature_name_10.lda",10)
-#     lda_process(doc_feature,path+"feature_name_15.lda",15)
-#     lda_process(doc_feature,path+"feature_name_20.lda",20)
-#     lda_process(doc_feature,path+"feature_name_100.lda",100)
- #     cur_bug = dao.get_bug(proj_id)
-#     doc_bug = cur_bug.fetchall()
-#     lda_process(doc_bug,path+"bug_name_5.lda",5)
-#     lda_process(doc_bug,path+"bug_name_10.lda",10)
-#     lda_process(doc_bug,path+"bug_name_15.lda",15)
-#     lda_process(doc_bug,path+"bug_name_20.lda",20)
-#     lda_process(doc_bug,path+"bug_name_100.lda",100)
-#     # lda_process(cur,feature_name_10,10)
-#     # lda_process(cur,feature_name_15,15)
-#     # lda_process(cur,feature_name_20,20)
-#     # lda_process(cur,feature_name_100,100)
-#
-#
-# f_proj_id = file('proj_id.csv', 'r')
-# reader = csv.reader(f_proj_id)
-# for line in reader:
-#     do_process(line[0])

proj_id.csv

@@ -1,111 +0,0 @@
-524804,8193
-24444,6766
-937,6293
-11450,5389
-64830,4261
-9793,4183
-4095,3814
-28763,3652
-27329,3546
-27319,2885
-2812,2519
-6013,2511
-31723,2348
-20096,2089
-86752,2041
-37,1997
-6785,1947
-3245727,1767
-3701929,1696
-7212135,1643
-7332776,1593
-8540,1554
-9316,1515
-35606,1473
-26697,1446
-4524570,1442
-3227,1442
-321869,1397
-2398573,1359
-3275,1276
-9550562,1273
-2122,1269
-11372864,1205
-6815,1202
-1823,1202
-2989453,1186
-2254769,1183
-340,1170
-3459504,1106
-10998645,1065
-37731,1046
-58572,1037
-43061,1030
-14915,995
-4708601,989
-540,984
-5219,976
-634,948
-4234273,891
-10200435,890
-3777,886
-13813,863
-5768596,862
-6946390,861
-417939,860
-4276991,854
-5342282,849
-2302110,841
-7294393,822
-7930999,820
-19550,810
-901621,807
-1394906,793
-1665150,787
-6653643,784
-12558,764
-1165,763
-1953978,762
-1996840,748
-6806,747
-10313229,746
-9315494,744
-525974,743
-6053661,718
-1531853,711
-2831,709
-7709841,691
-866,688
-3719,671
-9289602,669
-17635,663
-8052452,662
-49876,661
-7289481,648
-849289,645
-2265342,644
-23203,630
-853,623
-5911126,621
-4804053,610
-1964605,609
-8017495,605
-174642,601
-29670,586
-15762,578
-1018,553
-10322399,544
-183,539
-6,538
-10502,534
-5167,529
-3601879,526
-2911219,523
-7058,522
-5626,520
-864,515
-1043143,512
-101854,510
-39021,507
-3218707,504
-546037,501

read me.txt

@@ -1,10 +0,0 @@
-classifier.py:对数据进行预处理，包括stemming，去除数字，特殊字符，TFIDF计算
-sentence_classifier.py:分类程序，包括直接使用svm，对hard部分的特殊处理，结果统计与持久化
-dao:数据库访问方法
-helper:一些复杂方法的实现（包括特殊符号/数字筛选，代码识别，文件数据读取等）
-get_analysis_result/get_feature_analysis_result:对分类结果的分析
-lda_example:lda主题模型+kmeans
-read_lda:读取lda模型
-kmeans:用tfidf做kmeans
-
-proj_id.csv:实验用的项目id和已经分好类别的issue数量（bug,feature）

read_lda.py

@@ -1,20 +0,0 @@
-
-
-
-proj_id = '11450'
-path = "result/"+proj_id+'/lda/'
-from gensim import corpora, models
-import gensim
-lmodel = gensim.models.ldamodel.LdaModel.load(path+'bug_feature_name_10.lda')
-
-# print(lmodel)
-print(lmodel.show_topic(0,20))
-print(lmodel.show_topic(1,20))
-print(lmodel.show_topic(2,20))
-print(lmodel.show_topic(3,20))
-print(lmodel.show_topic(4,20))
-print(lmodel.show_topic(5,20))
-print(lmodel.show_topic(6,20))
-print(lmodel.show_topic(7,20))
-print(lmodel.show_topic(8,20))
-print(lmodel.show_topic(9,20))

sentence_classifier.py

@@ -1,418 +0,0 @@
-# import traceback
-import nltk
-import numpy as np
-from sklearn import svm
-from sklearn import metrics
-from sklearn.cross_validation import KFold
-from time import time
-import csv
-# from itertools import *
-import dao
-
-import sys
-import helper
-import classifier
-
-# reload(sys)
-# sys.setdefaultencoding("utf-8")
-
-__author__ = 'mac'
-import cPickle as pickle
-
-
-def classifier_project_by_id(proj_id):
-    # project_name = repr(proj_id)
-    project_name = proj_id
-    methold_name = 'svm'
-
-    path = 'result/'+project_name+'/'
-    helper.mkdir(path)
-    path_analysis = path + methold_name + '/analysis/'
-    helper.mkdir(path_analysis)
-    path_data = path + methold_name + '/data/'
-    helper.mkdir(path_data)
-    csv_result = file(path_analysis + 'precision_method.csv', 'wb')
-    writer_result = csv.writer(csv_result)
-
-    feature_csv_result = file(path_analysis + 'feature_precision_method.csv', 'wb')
-    feature_writer_result = csv.writer(feature_csv_result)
-
-    csv_path = file(path_analysis + 'sentence_split.csv', 'wb')
-    writer = csv.writer(csv_path)
-    writer.writerow(['id','diff','y_test','pred','sentence_pred','sentence_pred_0','sentence_pred_1','sentence'])
-
-
-    csv_classifier = file(path_analysis + 'classifier_info.csv', 'wb')
-    writer_classifier = csv.writer(csv_classifier)
-    writer_classifier.writerow(['classifier infomation for project:',repr(project_name)])
-
-    threshold = []
-    break_count = 20 # cut the different, len(threshold)
-    change_break = 20 # cut the condition for changing class
-
-    for ind_c_th in range(1,break_count+1):
-        threshold.append(ind_c_th*1.0/break_count)
-
-    count_base = [0]*break_count
-    right_base = [0]*break_count
-    count_base2 = [0]*break_count
-    right_base2 = [0]*break_count
-
-    feature_count_base = [0]*break_count
-    feature_pred_base = [0]*break_count
-    feature_right_base = [0]*break_count
-    feature_count_base2 = [0]*break_count
-    feature_pred_base2 = [0]*break_count
-    feature_right_base2 = [0]*break_count
-
-
-    right_mine1 = [([0] * change_break) for i in range(break_count)]
-    right_mine2 = [([0] * change_break) for i in range(break_count)]
-    right_mine3 = [([0] * change_break) for i in range(break_count)]
-    right_mine4 = [([0] * change_break) for i in range(break_count)]
-
-    feature_right_mine1 = [([0] * change_break) for i in range(break_count)]
-    feature_right_mine2 = [([0] * change_break) for i in range(break_count)]
-    feature_right_mine3 = [([0] * change_break) for i in range(break_count)]
-    feature_right_mine4 = [([0] * change_break) for i in range(break_count)]
-
-    feature_count_mine1 = [([0] * change_break) for i in range(break_count)]
-    feature_count_mine2 = [([0] * change_break) for i in range(break_count)]
-    feature_count_mine3 = [([0] * change_break) for i in range(break_count)]
-    feature_count_mine4 = [([0] * change_break) for i in range(break_count)]
-
-
-    X,y,x_id_before,vect = classifier.data_preprocess(project_name,methold_name) # first time run, get tf-idf of train data
-    # X,y,x_id_before,vect = helper.get_tfidf_data(project_name) # fellow methold run, to get tf-idf store in disk
-    f_train_data = path + 'train_data.pkl'
-    train_data = helper.get_pickle_record(f_train_data)
-
-    # get tf-idf matirx of train data
-    # print('get data: ')
-    # f_train = path +'train.pkl'
-    # f_target = path + 'target.pkl'
-    # f_id = path + 'id.pkl'
-    # f_vect = path + 'vect.pkl'
-    # X = helper.get_pickle_record(f_train)
-    # y = helper.get_pickle_record(f_target)
-    # x_id = helper.get_pickle_record(f_id)
-    # vect = helper.get_pickle_record(f_vect)
-    # print('done')
-
-    y = np.array(y)
-    x_id = np.array(x_id_before)
-
-    results = []
-    kf = KFold(len(y), n_folds=10)
-
-
-    def change_flag(flag,ind_threshold,ind_change):
-        flag[ind_threshold][ind_change]=True
-
-    def data_process(change_flag_set,ind_threshold,status):
-        if status:
-            for ind_change in range(change_break):
-                if not change_flag_set[0][ind_threshold][ind_change]:
-                    right_mine1[ind_threshold][ind_change] = right_mine1[ind_threshold][ind_change] + 1
-                if not change_flag_set[1][ind_threshold][ind_change]:
-                    right_mine2[ind_threshold][ind_change] = right_mine2[ind_threshold][ind_change] + 1
-                if not change_flag_set[2][ind_threshold][ind_change]:
-                    right_mine3[ind_threshold][ind_change] = right_mine3[ind_threshold][ind_change] + 1
-                if not change_flag_set[3][ind_threshold][ind_change]:
-                    right_mine4[ind_threshold][ind_change] = right_mine4[ind_threshold][ind_change] + 1
-        if not status:
-            for ind_change in range(change_break):
-                if change_flag_set[0][ind_threshold][ind_change]:
-                    right_mine1[ind_threshold][ind_change] = right_mine1[ind_threshold][ind_change] + 1
-                if change_flag_set[1][ind_threshold][ind_change]:
-                    right_mine2[ind_threshold][ind_change] = right_mine2[ind_threshold][ind_change] + 1
-                if change_flag_set[2][ind_threshold][ind_change]:
-                    right_mine3[ind_threshold][ind_change] = right_mine3[ind_threshold][ind_change] + 1
-                if change_flag_set[3][ind_threshold][ind_change]:
-                    right_mine4[ind_threshold][ind_change] = right_mine4[ind_threshold][ind_change] + 1
-
-    def data_process2(change_flag_set,ind_threshold,status):
-        for ind_change in range(change_break):
-            if change_flag_set[0][ind_threshold][ind_change] and not status:
-                feature_right_mine1[ind_threshold][ind_change] = feature_right_mine1[ind_threshold][ind_change] + 1
-            if change_flag_set[1][ind_threshold][ind_change] and not status:
-                feature_right_mine2[ind_threshold][ind_change] = feature_right_mine2[ind_threshold][ind_change] + 1
-            if change_flag_set[2][ind_threshold][ind_change] and not status:
-                feature_right_mine3[ind_threshold][ind_change] = feature_right_mine3[ind_threshold][ind_change] + 1
-            if change_flag_set[3][ind_threshold][ind_change] and not status:
-                feature_right_mine4[ind_threshold][ind_change] = feature_right_mine4[ind_threshold][ind_change] + 1
-
-
-    turn_count = 0
-    print("start training:")
-    # ten-fold traning
-    for train_index, test_index in kf:
-        X_train, X_test = X[train_index], X[test_index]
-        y_train, y_test = y[train_index], y[test_index]
-        x_id_train, x_id_test = x_id[train_index], x_id[test_index]
-        turn_count = turn_count+1
-        print('turn '+ repr(turn_count) +':')
-        writer_classifier.writerow(['------------------------------'])
-        writer_classifier.writerow(['turn ', repr(turn_count) ,':'])
-        # print('='*80)
-        ###############################################################################
-        # Benchmark classifiers
-        def benchmark(clf):
-            print('_' * 80)
-            print("Training: ")
-            print(clf)
-            t0 = time()
-            clf.fit(X_train, y_train)
-            train_time = time() - t0
-            print("train time: %0.3fs" % train_time)
-            writer_classifier.writerow(["train time:", train_time])
-
-            t0 = time()
-            pred = clf.predict(X_test)
-
-            test_time = time() - t0
-            print("test time:  %0.3fs" % test_time)
-            writer_classifier.writerow(["test time:", test_time])
-
-            score = metrics.accuracy_score(y_test, pred)
-            print("accuracy:   %0.3f" % score)
-            writer_classifier.writerow(["accuracy:", score])
-
-            probability = clf.predict_proba(X_test)
-
-            f_mechine = path_data + 'mechine_' + repr(turn_count)
-            with open(f_mechine, 'w') as f:
-                pickle.dump(clf, f)
-
-            np.save(path_data + "y_test_"+repr(turn_count),y_test)
-            np.save(path_data + "pred_"+repr(turn_count),pred)
-            np.save(path_data + "x_id_"+repr(turn_count),x_id_test)
-            np.save(path_data + "probability_"+repr(turn_count),probability)
-
-            tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
-
-            for ind in range(len(pred)):
-                # reset flag for each test data
-                change_flag1 = [([False] * change_break) for i in range(break_count)]
-                change_flag2 = [([False] * change_break) for i in range(break_count)]
-                change_flag3 = [([False] * change_break) for i in range(break_count)]
-                change_flag4 = [([False] * change_break) for i in range(break_count)]
-                change_flag_set = [change_flag1,change_flag2,change_flag3,change_flag4]
-
-                # diff = np.sort(probability[ind:ind+1])[:1,-1:][0][0]-np.sort(probability[ind:ind+1],)[:1,-2:-1][0][0]
-                # get sentence info and split it
-                # issue = dao.get_info_by_id(x_id_test[ind])
-                # for temp in issue:
-                #     issue_title = temp[0]
-                #     issue_body = temp[1]
-                info = helper.get_info_by_id(x_id_test[ind],x_id_before,train_data)
-                # issue_title = issue[1]
-                # issue_body = issue[2]
-                # info = issue_title + '.\n' + issue_body
-                # info = ''.join(ifilterfalse(unicode.isdigit, info))
-                info = helper.filter_str(info)
-                info = helper.filter_code(info)
-
-                sentences = tokenizer.tokenize(info)
-                x_test = vect.transform(sentences)
-                x_pred = clf.predict(x_test)
-                x_prob = clf.predict_proba(x_test)
-                diff = probability[ind:ind+1,0][0]-probability[ind:ind+1,1][0]
-
-                # record split infor
-                if len(x_pred)>0:
-                    for j in range(len(x_pred)):
-                        if(helper.word_count(sentences[j])>3):
-                            # data format : ['id','diff','y_test','pred','sentence_pred','sentence_pred_0','sentence_pred_1','sentence']
-                            data = (x_id_test[ind],diff,y_test[ind],pred[ind],x_pred[j],x_prob[j:j+1,0][0],x_prob[j:j+1,1][0],sentences[j].encode('utf8'))
-
-                            writer.writerow(data)
-
-                for ind_threshold in range(break_count):
-                    if abs(diff) <= threshold[ind_threshold]:
-                        count_base[ind_threshold] = count_base[ind_threshold] + 1
-                        if int(y_test[ind]) == 1:
-                            feature_count_base[ind_threshold] = feature_count_base[ind_threshold] +1
-                        if int(pred[ind]) == 1:
-                            feature_pred_base[ind_threshold] = feature_pred_base[ind_threshold] + 1
-
-                        if y_test[ind] == pred[ind]:
-                            right_base[ind_threshold] = right_base[ind_threshold] + 1
-                            if int(y_test[ind]) == 1:
-                                feature_right_base[ind_threshold] = feature_right_base[ind_threshold] + 1
-                    if ind_threshold == 0:
-                        low_threshold = -0.1
-                    else:
-                        low_threshold = threshold[ind_threshold]- 1.0/break_count
-                    if abs(diff) <= threshold[ind_threshold] and abs(diff) > low_threshold:
-                        count_base2[ind_threshold] = count_base2[ind_threshold] + 1
-                        if int(y_test[ind]) == 1:
-                            feature_count_base2[ind_threshold] = feature_count_base2[ind_threshold] +1
-                        if int(pred[ind]) == 1:
-                            feature_pred_base2[ind_threshold] = feature_pred_base2[ind_threshold] + 1
-
-                        if y_test[ind] == pred[ind]:
-                            right_base2[ind_threshold] = right_base2[ind_threshold] + 1
-                            if int(y_test[ind]) == 1:
-                                feature_right_base2[ind_threshold] = feature_right_base2[ind_threshold] + 1
-
-                        # flag for diff
-                        flag = 'zero'
-                        if diff > 0:
-                            flag = "+"
-                        elif diff < 0:
-                            flag = "-"
-                        # print("id:"+repr(x_id[ind]))
-                        if len(x_pred)>0:
-                            for j in range(len(x_pred)):
-                                if(helper.word_count(sentences[j])>3):
-                                    # recording which to change
-                                    for ind_change in range(change_break):
-                                        if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break:
-                                            change_flag(change_flag_set[0],ind_threshold,ind_change)
-                                        if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break and flag != '+':
-                                            change_flag(change_flag_set[1],ind_threshold,ind_change)
-                                        if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break and flag == '-':
-                                            change_flag(change_flag_set[2],ind_threshold,ind_change)
-                                        if pred[ind] == 1 and x_prob[j:j+1,0][0] > ind_change*1.0/change_break and flag == '+':
-                                            change_flag(change_flag_set[3],ind_threshold,ind_change)
-
-                        status = (y_test[ind] == pred[ind])
-                        data_process(change_flag_set,ind_threshold,status)
-                        for ind_change in range(change_break):
-                            if change_flag_set[0][ind_threshold][ind_change]:
-                                feature_count_mine1[ind_threshold][ind_change] = feature_count_mine1[ind_threshold][ind_change] + 1
-                            if change_flag_set[1][ind_threshold][ind_change]:
-                                feature_count_mine2[ind_threshold][ind_change] = feature_count_mine2[ind_threshold][ind_change] + 1
-                            if change_flag_set[2][ind_threshold][ind_change]:
-                                feature_count_mine3[ind_threshold][ind_change] = feature_count_mine3[ind_threshold][ind_change] + 1
-                            # if change_flag_set[3][ind_threshold][ind_change]:
-                            #     feature_count_mine4[ind_threshold][ind_change] = feature_count_mine4[ind_threshold][ind_change] + 1
-
-                        data_process2(change_flag_set,ind_threshold,status)
-
-                # issue.close()
-            clf_descr = str(clf).split('(')[0]
-
-            return clf_descr, score, train_time, test_time
-##############################################
-        results.append(benchmark(svm.SVC(kernel='linear',probability=True)))
-
-    results = [[x[i] for x in results] for i in range(4)]
-    clf_names, score, training_time, test_time = results
-
-    info_len = len(score)
-    training_time = np.array(training_time).sum() / info_len
-    test_time = np.array(test_time).sum() / info_len
-    score_all = np.array(score).sum()/ info_len
-    print("accuracy for all:   %0.3f" % score_all)
-    writer_classifier.writerow(['------------------------------'])
-    writer_classifier.writerow(["traning time for all:", training_time])
-    writer_classifier.writerow(["test time for all:", test_time])
-    writer_classifier.writerow(["accuracy for all:", score_all])
-
-    # write result
-    writer_result.writerow(['base count for all:'])
-    writer_result.writerow([n for n in count_base])
-    writer_result.writerow(['base right count for all:'])
-    writer_result.writerow([n for n in right_base])
-
-    writer_result.writerow(['base count for threshold:'])
-    writer_result.writerow([n for n in count_base2])
-    writer_result.writerow(['base right count for threshold:'])
-    writer_result.writerow([n for n in right_base2])
-
-
-    feature_writer_result.writerow(['base count for all:'])
-    feature_writer_result.writerow([n for n in feature_count_base])
-    feature_writer_result.writerow(['base right count for all:'])
-    feature_writer_result.writerow([n for n in feature_right_base])
-    feature_writer_result.writerow(['base pred count for all:'])
-    feature_writer_result.writerow([n for n in feature_pred_base])
-
-    feature_writer_result.writerow(['base count for threshold:'])
-    feature_writer_result.writerow([n for n in feature_count_base2])
-    feature_writer_result.writerow(['base right count for threshold:'])
-    feature_writer_result.writerow([n for n in feature_right_base2])
-    feature_writer_result.writerow(['base pred count for threshold:'])
-    feature_writer_result.writerow([n for n in feature_pred_base2])
-
-    writer_result.writerow(['right count of mine method 1 for each threshold:'])
-    for i in range(break_count):
-        writer_result.writerow([l for l in right_mine1[i]])
-    writer_result.writerow(['right count of mine method 2 for each threshold:'])
-    for i in range(break_count):
-        writer_result.writerow([l for l in right_mine2[i]])
-    writer_result.writerow(['right count of mine method 3 for each threshold:'])
-    for i in range(break_count):
-        writer_result.writerow([l for l in right_mine3[i]])
-    writer_result.writerow(['right count of mine method 4 for each threshold:'])
-    for i in range(break_count):
-        writer_result.writerow([l for l in right_mine4[i]])
-
-
-    feature_writer_result.writerow(['right count of mine method 1 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_right_mine1[i]])
-    feature_writer_result.writerow(['right count of mine method 2 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_right_mine2[i]])
-    feature_writer_result.writerow(['right count of mine method 3 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_right_mine3[i]])
-    feature_writer_result.writerow(['right count of mine method 4 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_right_mine4[i]])
-
-    feature_writer_result.writerow(['change count of mine method 1 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_count_mine1[i]])
-    feature_writer_result.writerow(['change count of mine method 2 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_count_mine2[i]])
-    feature_writer_result.writerow(['change count of mine method 3 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_count_mine3[i]])
-    feature_writer_result.writerow(['change count of mine method 4 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_count_mine4[i]])
-
-    csv_result.close()
-    csv_path.close()
-    csv_classifier.close()
-
-
-# classifier_project_by_id('6')
-
-# break_id = 961
-# projects = dao.get_project()
-# flag_break = True
-# csv_project = file('project_id.csv', 'wb')
-# writer_project = csv.writer(csv_project)
-# for project in projects:
-#     print('do classifier for project:'+ repr(project[0]))
-#     if project[0] == break_id:
-#         flag_break = True
-#
-#     if project[1] > 500 and flag_break:
-#         try:
-#             classifier_project_by_id(project[0])
-#             writer_project.writerow(project)
-#         except:
-#             f=open("log.txt",'a')
-#             f.writelines("project:\t"+repr(project[0])+'\n')
-#             f.flush()
-#             f.close()
-#
-# csv_project.close()
-# projects.close()
-
-f_proj_id = file('proj_id.csv', 'r')
-reader = csv.reader(f_proj_id)
-for line in reader:
-    classifier_project_by_id(line[0])
-
-    print(line[0])
-
-# classifier_project_by_id('546037')
-dao.close()

sentence_classifier2.py

@@ -1,438 +0,0 @@
-# import traceback
-import nltk
-import numpy as np
-from sklearn import svm
-from sklearn import metrics
-from sklearn.cross_validation import KFold
-from time import time
-import csv
-# from itertools import *
-import dao
-
-import sys
-import helper
-import classifier
-
-reload(sys)
-sys.setdefaultencoding("utf-8")
-
-__author__ = 'mac'
-import cPickle as pickle
-
-
-def classifier_project_by_id(proj_id):
-    # project_name = repr(proj_id)
-    project_name = proj_id
-    methold_name = 'svm2'
-
-    path = 'result/'+project_name+'/'
-    helper.mkdir(path)
-    path_analysis = path + methold_name + '/analysis/'
-    helper.mkdir(path_analysis)
-    path_data = path + methold_name + '/data/'
-    helper.mkdir(path_data)
-    csv_result = file(path_analysis + 'precision_method.csv', 'wb')
-    writer_result = csv.writer(csv_result)
-
-    feature_csv_result = file(path_analysis + 'feature_precision_method.csv', 'wb')
-    feature_writer_result = csv.writer(feature_csv_result)
-
-    csv_path = file(path_analysis + 'sentence_split.csv', 'wb')
-    writer = csv.writer(csv_path)
-    writer.writerow(['id','diff','y_test','pred','sentence_pred','sentence_pred_0','sentence_pred_1','sentence'])
-
-
-    csv_classifier = file(path_analysis + 'classifier_info.csv', 'wb')
-    writer_classifier = csv.writer(csv_classifier)
-    writer_classifier.writerow(['classifier infomation for project:',repr(project_name)])
-
-    threshold = []
-    break_count = 20 # cut the different, len(threshold)
-    change_break = 20 # cut the condition for changing class
-
-    for ind_c_th in range(1,break_count+1):
-        threshold.append(ind_c_th*1.0/break_count)
-
-    count_base = [0]*break_count
-    right_base = [0]*break_count
-    count_base2 = [0]*break_count
-    right_base2 = [0]*break_count
-
-    feature_count_base = [0]*break_count
-    feature_pred_base = [0]*break_count
-    feature_right_base = [0]*break_count
-    feature_count_base2 = [0]*break_count
-    feature_pred_base2 = [0]*break_count
-    feature_right_base2 = [0]*break_count
-
-
-    right_mine1 = [([0] * change_break) for i in range(break_count)]
-    right_mine2 = [([0] * change_break) for i in range(break_count)]
-    right_mine3 = [([0] * change_break) for i in range(break_count)]
-    right_mine4 = [([0] * change_break) for i in range(break_count)]
-
-    feature_right_mine1 = [([0] * change_break) for i in range(break_count)]
-    feature_right_mine2 = [([0] * change_break) for i in range(break_count)]
-    feature_right_mine3 = [([0] * change_break) for i in range(break_count)]
-    feature_right_mine4 = [([0] * change_break) for i in range(break_count)]
-
-    feature_count_mine1 = [([0] * change_break) for i in range(break_count)]
-    feature_count_mine2 = [([0] * change_break) for i in range(break_count)]
-    feature_count_mine3 = [([0] * change_break) for i in range(break_count)]
-    feature_count_mine4 = [([0] * change_break) for i in range(break_count)]
-
-
-    # X,y,x_id_before,vect = classifier.data_preprocess(project_name,methold_name) # first time run, get tf-idf of train data
-    X,y,x_id_before,vect = helper.get_tfidf_data(project_name) # fellow methold run, to get tf-idf store in disk
-    f_train_data = path + 'train_data.pkl'
-
-    # path2 = 'result_remove_code/'+project_name+'/'
-    # f_train_data = path2 + 'train_data.pkl'
-
-    train_data = helper.get_pickle_record(f_train_data)
-
-
-    # get tf-idf matirx of train data
-    # print('get data: ')
-    # f_train = path +'train.pkl'
-    # f_target = path + 'target.pkl'
-    # f_id = path + 'id.pkl'
-    # f_vect = path + 'vect.pkl'
-    # X = helper.get_pickle_record(f_train)
-    # y = helper.get_pickle_record(f_target)
-    # x_id = helper.get_pickle_record(f_id)
-    # vect = helper.get_pickle_record(f_vect)
-    # print('done')
-
-    y = np.array(y)
-    x_id = np.array(x_id_before)
-
-    results = []
-    kf = KFold(len(y), n_folds=10)
-
-
-    def change_flag(flag,ind_threshold,ind_change,j,sentences_count):
-        flag_t = False
-        if j == 0 or j == 1 or j == sentences_count:
-            flag[ind_threshold][ind_change]=True
-        else:
-            # todo: according to possision to decide whether to change flag
-            if sentences_count >= 4 and (j-1 <=1 or sentences_count-j <=1):
-                flag_t = True
-            if sentences_count >= 4 and (abs((j-1)*1.0/(sentences_count-j)) <= 1.0/3 or abs((sentences_count-j)*1.0/(j-1)) <= 1.0/3):
-                flag_t = True
-        if flag_t:
-            flag[ind_threshold][ind_change]=True
-
-    def data_process(change_flag_set,ind_threshold,status):
-        if status:
-            for ind_change in range(change_break):
-                if not change_flag_set[0][ind_threshold][ind_change]:
-                    right_mine1[ind_threshold][ind_change] = right_mine1[ind_threshold][ind_change] + 1
-                if not change_flag_set[1][ind_threshold][ind_change]:
-                    right_mine2[ind_threshold][ind_change] = right_mine2[ind_threshold][ind_change] + 1
-                if not change_flag_set[2][ind_threshold][ind_change]:
-                    right_mine3[ind_threshold][ind_change] = right_mine3[ind_threshold][ind_change] + 1
-                if not change_flag_set[3][ind_threshold][ind_change]:
-                    right_mine4[ind_threshold][ind_change] = right_mine4[ind_threshold][ind_change] + 1
-        if not status:
-            for ind_change in range(change_break):
-                if change_flag_set[0][ind_threshold][ind_change]:
-                    right_mine1[ind_threshold][ind_change] = right_mine1[ind_threshold][ind_change] + 1
-                if change_flag_set[1][ind_threshold][ind_change]:
-                    right_mine2[ind_threshold][ind_change] = right_mine2[ind_threshold][ind_change] + 1
-                if change_flag_set[2][ind_threshold][ind_change]:
-                    right_mine3[ind_threshold][ind_change] = right_mine3[ind_threshold][ind_change] + 1
-                if change_flag_set[3][ind_threshold][ind_change]:
-                    right_mine4[ind_threshold][ind_change] = right_mine4[ind_threshold][ind_change] + 1
-
-    def data_process2(change_flag_set,ind_threshold,status):
-        for ind_change in range(change_break):
-            if change_flag_set[0][ind_threshold][ind_change] and not status:
-                feature_right_mine1[ind_threshold][ind_change] = feature_right_mine1[ind_threshold][ind_change] + 1
-            if change_flag_set[1][ind_threshold][ind_change] and not status:
-                feature_right_mine2[ind_threshold][ind_change] = feature_right_mine2[ind_threshold][ind_change] + 1
-            if change_flag_set[2][ind_threshold][ind_change] and not status:
-                feature_right_mine3[ind_threshold][ind_change] = feature_right_mine3[ind_threshold][ind_change] + 1
-            if change_flag_set[3][ind_threshold][ind_change] and not status:
-                feature_right_mine4[ind_threshold][ind_change] = feature_right_mine4[ind_threshold][ind_change] + 1
-
-
-    turn_count = 0
-    print("start training:")
-    # ten-fold traning
-    for train_index, test_index in kf:
-        X_train, X_test = X[train_index], X[test_index]
-        y_train, y_test = y[train_index], y[test_index]
-        x_id_train, x_id_test = x_id[train_index], x_id[test_index]
-        turn_count = turn_count+1
-        print('turn '+ repr(turn_count) +':')
-        writer_classifier.writerow(['------------------------------'])
-        writer_classifier.writerow(['turn ', repr(turn_count) ,':'])
-        # print('='*80)
-        ###############################################################################
-        # Benchmark classifiers
-        def benchmark(clf):
-            print('_' * 80)
-            print("Training: ")
-            print(clf)
-            t0 = time()
-            clf.fit(X_train, y_train)
-            train_time = time() - t0
-            print("train time: %0.3fs" % train_time)
-            writer_classifier.writerow(["train time:", train_time])
-
-            t0 = time()
-            pred = clf.predict(X_test)
-
-            test_time = time() - t0
-            print("test time:  %0.3fs" % test_time)
-            writer_classifier.writerow(["test time:", test_time])
-
-            score = metrics.accuracy_score(y_test, pred)
-            print("accuracy:   %0.3f" % score)
-            writer_classifier.writerow(["accuracy:", score])
-
-            probability = clf.predict_proba(X_test)
-
-            f_mechine = path_data + 'mechine_' + repr(turn_count)
-            with open(f_mechine, 'w') as f:
-                pickle.dump(clf, f)
-
-
-            np.save(path_data + "y_test_"+repr(turn_count),y_test)
-            np.save(path_data + "pred_"+repr(turn_count),pred)
-            np.save(path_data + "x_id_"+repr(turn_count),x_id_test)
-            np.save(path_data + "probability_"+repr(turn_count),probability)
-
-            tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
-
-            for ind in range(len(pred)):
-                # reset flag for each test data
-                change_flag1 = [([False] * change_break) for i in range(break_count)]
-                change_flag2 = [([False] * change_break) for i in range(break_count)]
-                change_flag3 = [([False] * change_break) for i in range(break_count)]
-                change_flag4 = [([False] * change_break) for i in range(break_count)]
-                change_flag_set = [change_flag1,change_flag2,change_flag3,change_flag4]
-
-                # diff = np.sort(probability[ind:ind+1])[:1,-1:][0][0]-np.sort(probability[ind:ind+1],)[:1,-2:-1][0][0]
-                # get sentence info and split it
-                # issue = dao.get_info_by_id(x_id_test[ind])
-                # for temp in issue:
-                #     issue_title = temp[0]
-                #     issue_body = temp[1]
-                info = helper.get_info_by_id(x_id_test[ind],x_id_before,train_data)
-                # issue_title = issue[1]
-                # issue_body = issue[2]
-                # info = issue_title + '.\n' + issue_body
-                # info = ''.join(ifilterfalse(unicode.isdigit, info))
-                info = helper.filter_str(info)
-                info = helper.filter_code(info)
-
-                sentences = tokenizer.tokenize(info)
-                x_test = vect.transform(sentences)
-                x_pred = clf.predict(x_test)
-                x_prob = clf.predict_proba(x_test)
-                diff = probability[ind:ind+1,0][0]-probability[ind:ind+1,1][0]
-
-                # get num of sentences for dividing body (without title)
-                sentences_count = len(x_pred)-1
-                # record split information
-                if len(x_pred)>0:
-                    for j in range(len(x_pred)):
-                        if(helper.word_count(sentences[j])>3):
-                            # data format : ['id','diff','y_test','pred','sentence_pred','sentence_pred_0','sentence_pred_1','sentence']
-                            data = (x_id_test[ind],diff,y_test[ind],pred[ind],x_pred[j],x_prob[j:j+1,0][0],x_prob[j:j+1,1][0],sentences[j])
-
-                            writer.writerow(data)
-
-                for ind_threshold in range(break_count):
-                    if abs(diff) <= threshold[ind_threshold]:
-                        count_base[ind_threshold] = count_base[ind_threshold] + 1
-                        if int(y_test[ind]) == 1:
-                            feature_count_base[ind_threshold] = feature_count_base[ind_threshold] +1
-                        if int(pred[ind]) == 1:
-                            feature_pred_base[ind_threshold] = feature_pred_base[ind_threshold] + 1
-
-                        if y_test[ind] == pred[ind]:
-                            right_base[ind_threshold] = right_base[ind_threshold] + 1
-                            if int(y_test[ind]) == 1:
-                                feature_right_base[ind_threshold] = feature_right_base[ind_threshold] + 1
-                    if ind_threshold == 0:
-                        low_threshold = -0.1
-                    else:
-                        low_threshold = threshold[ind_threshold]- 1.0/break_count
-                    if abs(diff) <= threshold[ind_threshold] and abs(diff) > low_threshold:
-                        count_base2[ind_threshold] = count_base2[ind_threshold] + 1
-                        if int(y_test[ind]) == 1:
-                            feature_count_base2[ind_threshold] = feature_count_base2[ind_threshold] +1
-                        if int(pred[ind]) == 1:
-                            feature_pred_base2[ind_threshold] = feature_pred_base2[ind_threshold] + 1
-
-                        if y_test[ind] == pred[ind]:
-                            right_base2[ind_threshold] = right_base2[ind_threshold] + 1
-                            if int(y_test[ind]) == 1:
-                                feature_right_base2[ind_threshold] = feature_right_base2[ind_threshold] + 1
-
-                        # flag for diff
-                        flag = 'zero'
-                        if diff > 0:
-                            flag = "+"
-                        elif diff < 0:
-                            flag = "-"
-                        # print("id:"+repr(x_id[ind]))
-                        if len(x_pred)>0:
-                            for j in range(len(x_pred)):
-                                if(helper.word_count(sentences[j])>3):
-                                    # recording which to change
-                                    for ind_change in range(change_break):
-                                        if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break:
-                                            change_flag(change_flag_set[0],ind_threshold,ind_change,j,sentences_count)
-                                        if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break and flag != '+':
-                                            change_flag(change_flag_set[1],ind_threshold,ind_change,j,sentences_count)
-                                        if pred[ind] == 0 and x_prob[j:j+1,1][0] > ind_change*1.0/change_break and flag == '-':
-                                            change_flag(change_flag_set[2],ind_threshold,ind_change,j,sentences_count)
-                                        # if pred[ind] == 1 and x_prob[j:j+1,0][0] > ind_change*1.0/change_break and flag == '+':
-                                        #     change_flag(change_flag_set[3],ind_threshold,ind_change)
-
-                        status = (y_test[ind] == pred[ind])
-                        data_process(change_flag_set,ind_threshold,status)
-                        for ind_change in range(change_break):
-                            if change_flag_set[0][ind_threshold][ind_change]:
-                                feature_count_mine1[ind_threshold][ind_change] = feature_count_mine1[ind_threshold][ind_change] + 1
-                            if change_flag_set[1][ind_threshold][ind_change]:
-                                feature_count_mine2[ind_threshold][ind_change] = feature_count_mine2[ind_threshold][ind_change] + 1
-                            if change_flag_set[2][ind_threshold][ind_change]:
-                                feature_count_mine3[ind_threshold][ind_change] = feature_count_mine3[ind_threshold][ind_change] + 1
-                            if change_flag_set[3][ind_threshold][ind_change]:
-                                feature_count_mine4[ind_threshold][ind_change] = feature_count_mine4[ind_threshold][ind_change] + 1
-
-                        data_process2(change_flag_set,ind_threshold,status)
-
-                # issue.close()
-            clf_descr = str(clf).split('(')[0]
-
-            return clf_descr, score, train_time, test_time
-##############################################
-        results.append(benchmark(svm.SVC(kernel='linear',probability=True)))
-
-    results = [[x[i] for x in results] for i in range(4)]
-    clf_names, score, training_time, test_time = results
-
-    info_len = len(score)
-    training_time = np.array(training_time).sum() / info_len
-    test_time = np.array(test_time).sum() / info_len
-    score_all = np.array(score).sum()/ info_len
-    print("accuracy for all:   %0.3f" % score_all)
-    writer_classifier.writerow(['------------------------------'])
-    writer_classifier.writerow(["traning time for all:", training_time])
-    writer_classifier.writerow(["test time for all:", test_time])
-    writer_classifier.writerow(["accuracy for all:", score_all])
-
-    # write result
-    writer_result.writerow(['base count for all:'])
-    writer_result.writerow([n for n in count_base])
-    writer_result.writerow(['base right count for all:'])
-    writer_result.writerow([n for n in right_base])
-
-    writer_result.writerow(['base count for threshold:'])
-    writer_result.writerow([n for n in count_base2])
-    writer_result.writerow(['base right count for threshold:'])
-    writer_result.writerow([n for n in right_base2])
-
-
-    feature_writer_result.writerow(['base count for all:'])
-    feature_writer_result.writerow([n for n in feature_count_base])
-    feature_writer_result.writerow(['base right count for all:'])
-    feature_writer_result.writerow([n for n in feature_right_base])
-    feature_writer_result.writerow(['base pred count for all:'])
-    feature_writer_result.writerow([n for n in feature_pred_base])
-
-    feature_writer_result.writerow(['base count for threshold:'])
-    feature_writer_result.writerow([n for n in feature_count_base2])
-    feature_writer_result.writerow(['base right count for threshold:'])
-    feature_writer_result.writerow([n for n in feature_right_base2])
-    feature_writer_result.writerow(['base pred count for threshold:'])
-    feature_writer_result.writerow([n for n in feature_pred_base2])
-
-    writer_result.writerow(['right count of mine method 1 for each threshold:'])
-    for i in range(break_count):
-        writer_result.writerow([l for l in right_mine1[i]])
-    writer_result.writerow(['right count of mine method 2 for each threshold:'])
-    for i in range(break_count):
-        writer_result.writerow([l for l in right_mine2[i]])
-    writer_result.writerow(['right count of mine method 3 for each threshold:'])
-    for i in range(break_count):
-        writer_result.writerow([l for l in right_mine3[i]])
-    writer_result.writerow(['right count of mine method 4 for each threshold:'])
-    for i in range(break_count):
-        writer_result.writerow([l for l in right_mine4[i]])
-
-
-    feature_writer_result.writerow(['right count of mine method 1 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_right_mine1[i]])
-    feature_writer_result.writerow(['right count of mine method 2 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_right_mine2[i]])
-    feature_writer_result.writerow(['right count of mine method 3 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_right_mine3[i]])
-    feature_writer_result.writerow(['right count of mine method 4 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_right_mine4[i]])
-
-    feature_writer_result.writerow(['change count of mine method 1 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_count_mine1[i]])
-    feature_writer_result.writerow(['change count of mine method 2 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_count_mine2[i]])
-    feature_writer_result.writerow(['change count of mine method 3 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_count_mine3[i]])
-    feature_writer_result.writerow(['change count of mine method 4 for each threshold:'])
-    for i in range(break_count):
-        feature_writer_result.writerow([l for l in feature_count_mine4[i]])
-
-    csv_result.close()
-    csv_path.close()
-    csv_classifier.close()
-
-
-# classifier_project_by_id('6')
-
-# break_id = 961
-# projects = dao.get_project()
-# flag_break = True
-# csv_project = file('project_id.csv', 'wb')
-# writer_project = csv.writer(csv_project)
-# for project in projects:
-#     print('do classifier for project:'+ repr(project[0]))
-#     if project[0] == break_id:
-#         flag_break = True
-#
-#     if project[1] > 500 and flag_break:
-#         try:
-#             classifier_project_by_id(project[0])
-#             writer_project.writerow(project)
-#         except:
-#             f=open("log.txt",'a')
-#             f.writelines("project:\t"+repr(project[0])+'\n')
-#             f.flush()
-#             f.close()
-#
-# csv_project.close()
-# projects.close()
-
-f_proj_id = file('proj_id.csv', 'r')
-reader = csv.reader(f_proj_id)
-run_flag = True
-for line in reader:
-    if line[0] == '17635':
-        run_flag = True
-    if run_flag:
-        classifier_project_by_id(line[0])
-
-    print(line[0])
-dao.close()

test.py

@@ -1,41 +0,0 @@
-# -*- coding: utf-8 -*
-import re
-
-import dao
-import helper
-import nltk
-a = dao.get_info_by_id(14044309).fetchone()
-str = a[0]+".\n"+a[1]
-# str = '```asdf```asdf'
-# print(str)
-print('='*80)
-# # p = re.compile(r'`{3,}.*?`{3,}')
-# # print(p.findall(str))
-# # print p.sub(r'',str)
-#
-# import re
-#
-# def repl(m):
-#     return ' '
-#
-# s = str
-# # s = '1\n```\n2\n```'
-# s = '1\n```\n2\n```\n3```\n4```\n'
-print
-
-a= '2'
-print('start')
-print(repr(helper.filter_str(a)))
-print('end')
-a = 'DHE-RSA-AES-SHA\r\n\r\nHowever, I would like to avoid using SHA, and when I include the following directive in the SSLCipherSuite :!SHA the Android-ownCloud stops working.\r\n\r\nThis is a paid app and the new ciphers have been implemented in the Windows '
-tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
-sentences = tokenizer.tokenize(a)
-print(sentences)
-
-
-import re
-temp = "1/ 2/3/ 4Q：5.6.7 ！！\r\n？？ 8\r\n 。?9.1-0。11 ,"
-temp = temp.decode("utf8")
-string = re.sub("[\s+\/_$%^*\-(+\"\']+|[:：+——！，。？、~@#￥%……&*（）]+".decode("utf8"), " ".decode("utf8"),temp)
-string = ' '.join(string.split())
-print string