git_issue/helper.py

# -*- coding: utf-8 -*
import nltk

__author__ = 'mac'

# shell mkdir
def mkdir(path):
    import os
    path=path.strip()
    path=path.rstrip("\\")
    isExists=os.path.exists(path)

    if not isExists:
        print path+': create successfull'
        os.makedirs(path)
        return True
    else:
        print path+': path already exist'
        return False

def strQ2B(ustring):
    rstring = ""
    for uchar in ustring:
        inside_code=ord(uchar)
        if inside_code == 12288:
            inside_code = 32
        elif (inside_code >= 65281 and inside_code <= 65374):
            inside_code -= 65248

        rstring += unichr(inside_code)
    return rstring

# filter numbers in string
def filter_str(deal_str):
    from itertools import ifilterfalse
    # import re
    if deal_str.__class__ == unicode:
        deal_result = ''.join(ifilterfalse(unicode.isdigit, deal_str))
        deal_result = strQ2B(deal_result)
    elif deal_str.__class__ == str:
        deal_result = ''.join(ifilterfalse(str.isdigit, deal_str))
        deal_result = strQ2B(deal_result)
    return deal_result

def filter_sign(deal_str):
    import re
    deal_result = re.sub("[\s+\/_$%^*\-(+\"\']+|[:：+——！，。？、~@#￥%……&*（）]+".decode("utf8"), " ".decode("utf8"),deal_str)
    deal_result = ' '.join(deal_result.split())
    return deal_result

# filter code in string
def filter_code(deal_str):
    import re
    deal_result = re.sub(r'`{3,10}.*?`{3,10}', ' .', deal_str, 100, re.S)
    return deal_result

# count words of sentence
def word_count(sentence):
    import string
    strip = string.whitespace + string.punctuation + string.digits + "\"'"
    len_count = 0
    for word in sentence.split():
        word = word.strip(strip)
        if len(word) >= 2:
            len_count = len_count + 1
    return len_count


# get record of pickle
def get_pickle_record(path):
    import cPickle as pickle
    with open(path, 'r') as f:
        return  pickle.load(f)   # read file and build object

# stemming
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer, PorterStemmer


class LemmaTokenizer(object):
    def __init__(self):
        self.wnl = WordNetLemmatizer()
    def __call__(self, doc):
        return [self.wnl.lemmatize(t) for t in word_tokenize(doc)]

stemmer = PorterStemmer()
def stem_tokens(tokens, stemmer):
    stemmed = []
    for item in tokens:
        stemmed.append(stemmer.stem(item))
    return stemmed

def tokenize_help(text):
    import string
    tokens = nltk.word_tokenize(text)
    tokens = [i for i in tokens if i not in string.punctuation]
    tokens = [i for i in tokens if len(i) > 2]
    stems = stem_tokens(tokens, stemmer)
    return stems

def get_result_by_classifier(clf,proj_id):
    path = 'result/'+proj_id+'/'+clf+'/'
    path_analysis = path + 'analysis/'
    f = open(path_analysis+clf+'.txt', 'r')
    result = []
    for line in f.readlines():
        count,train_time,test_time,prec = line.strip('\n').split(',')
        if count != 'avg':
            result.append(prec)
    return [float(i) for i in result]

def get_precision(proj_id,method):
    import csv
    project_name = proj_id
    path = 'result/'+project_name+'/'+method+'/'
    path_analysis = path + 'analysis/'

    csv_classifier = file(path_analysis + 'classifier_info.csv', 'rb')
    classifier_reader = csv.reader(csv_classifier)

    prec = []
    for l in classifier_reader:
        if l[0] == 'accuracy:':
            prec.append(l[1])
    return [float(i) for i in prec]

def get_tfidf_data(project_name):
    path = 'result/'+project_name+'/'

    f_train = path +'train.pkl'
    f_target = path + 'target.pkl'
    f_id = path + 'id.pkl'
    f_vect = path + 'vect.pkl'

    # load data preprocess result from file
    print('get data: ')
    X = get_pickle_record(f_train)
    y = get_pickle_record(f_target)
    x_id = get_pickle_record(f_id)
    vect = get_pickle_record(f_vect)

    print('done')

    return X,y,x_id,vect

def get_info_by_id(id,x_id,train_data):
    return train_data[x_id.index(id)]

def get_prec_recall_f1(right_count,all_count,pred_count):
    prec = 0
    recall = 0
    f1 = 0
    if pred_count != 0:
        prec = right_count*1.0/pred_count
    if all_count != 0:
        recall = right_count*1.0/all_count
    if prec+recall != 0:
        f1 = 2*prec*recall/(prec+recall)
    return prec,recall,f1

def read_hard_count(proj_id,methold):
    import csv
    path = 'final2/'

    csv_classifier = file(path + 'hard_count_'+methold+'.csv', 'rb')
    classifier_reader = csv.reader(csv_classifier)
    for l in classifier_reader:
        if l[0] == proj_id:
            return(l[1],l[2],l[3],l[4])