diff --git a/shrdlurn/analyzer.py b/shrdlurn/analyzer.py new file mode 100644 index 0000000..07e6b8d --- /dev/null +++ b/shrdlurn/analyzer.py @@ -0,0 +1,115 @@ +import csv +import numpy as np +import pandas as pd +import matplotlib +import matplotlib.pyplot as plt +import json +%matplotlib inline + +plotsetting_default = {'color': 'r', 'linewidth': 2} +plotsetting = {'color': 'r', 'linewidth': 2} + +def plot_cumavg(x, y, xlabel='query#', ylabel='recall', title=None): + y_cum = np.cumsum(y).tolist() + #print accepts_np[:,1] + y_cumavg = [cum / float(count+1) for count,cum in enumerate(y_cum)] + #N = 500; + #y_cumavg = np.convolve(np.array(y), np.ones((N,))/N, mode='same').tolist() + + #print accept_rate + + #plt.scatter(means_baseline[0:], means[0:], s=colors, alpha=0.8, c='r') + plt.plot(x, y_cumavg, alpha=0.5, **plotsetting) + + plt.xlabel(xlabel, fontsize=12) + plt.ylabel(ylabel, fontsize=12) + # plt.xlim(0, 0.65) + plt.ylim(0, max(y_cumavg)*1.02) + + xp = np.linspace(0, 0.65, 300) + + #plt.gca().set_aspect('equal', adjustable='box') + plottitle = title if title is not None else '%s_vs_%s.pdf' % (xlabel, ylabel) + plt.savefig(plottitle , bbox_inches="tight") + +def print_avg(x, name = 'unnamed'): + print 'Average of %s is %f' % (name, reduce(lambda a,b: a+b, x) / float(len(x))); + +with open('../state/lastExec', 'rb') as lastExec: + lastExecInd = lastExec.readline().strip() +print lastExecInd + +rows = []; +execInd = lastExecInd; +with open('../state/execs/%s.exec/plotInfo.json' % execInd, 'rb') as jsonfile: + json_lines = jsonfile.readlines() + +rows = [json.loads(l) for l in json_lines] +# accepts = [[r['queryCount'], 1 if r['stats.rank']>=0 else 0] for r in rows if r['stats.type'] == 'accept']; + +def percent_induced_in_accepted(status = 'Core'): + filtered_rows = [r for r in rows if r['stats.type'] == 'accept'] + query_counts = [r['queryCount'] for r in filtered_rows] + is_induced = [1 if r['stats.status'] == status else 0 for r in filtered_rows] + print_avg(is_induced, 'percent_induced_accepted') + plotsetting['label'] = status.lower(); + plot_cumavg(query_counts, is_induced, xlabel='query #', ylabel='induced-accepted'); +# plt.figure() +# plotsetting['color'] = 'r'; percent_induced_in_accepted(status = 'Nothing'); +# plotsetting['color'] = 'b'; percent_induced_in_accepted(status = 'Induced'); +# plotsetting['color'] = 'k'; percent_induced_in_accepted(status = 'Core'); +# plt.legend(frameon=False) +# plt.savefig('accepted_stats.pdf' , bbox_inches="tight") + + +def precent_status(status = 'Core'): + filtered_rows = [r for r in rows if r['stats.type'] == 'q'] + query_counts = [r['queryCount'] for r in filtered_rows] + is_status = [1 if r['stats.status'] == status else 0 for r in filtered_rows] + print_avg(is_status, 'percent of status ' + status) + plotsetting['label'] = status.lower(); + plot_cumavg(query_counts, is_status, xlabel='query #', ylabel='%'); +plt.figure() +plotsetting['color'] = 'r'; precent_status(status = 'Nothing'); +plotsetting['color'] = 'b'; precent_status(status = 'Induced'); +plotsetting['color'] = 'k'; precent_status(status = 'Core'); +plt.legend(frameon=False) +plt.savefig('parse_status.pdf' , bbox_inches="tight") + +def simple_acc(): + filtered_rows = [r for r in rows if r['stats.type'] == 'accept' or r['stats.type'] == 'q'] + query_counts = [r['queryCount'] for r in filtered_rows] + accepts = [1 if r['stats.type'] != 'q' and r['stats.rank']==0 else 0 for r in filtered_rows] + plot_cumavg(query_counts, accepts, xlabel='query#', ylabel='accuracy'); + print_avg(accepts, 'accuracy') +plt.figure() +simple_acc() + +def simple_recall(): + filtered_rows = [r for r in rows if r['stats.type'] == 'accept'] + query_counts = [r['queryCount'] for r in filtered_rows] + recall = [1 if r['stats.rank']>=0 else 0 for r in filtered_rows] + plot_cumavg(query_counts, recall, xlabel='query#', ylabel='recall'); + print_avg(recall, 'recall') +plotsetting['color'] = 'r' +simple_recall() + +def average_stat(stat = 'stats.size', type = 'q'): + filtered_rows = [r for r in rows if r['stats.type'] == type] + query_counts = [r['queryCount'] for r in filtered_rows] + stats = [r[stat] for r in filtered_rows] + plot_cumavg(query_counts, stats, xlabel='query#', ylabel=stat); + print_avg(stats, stat) +plt.figure() +plotsetting['color'] = 'b' +average_stat(stat = 'stats.size') + + +def average_stat(stat = 'stats.rank'): + filtered_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.rank'] >= 0] + query_counts = [r['queryCount'] for r in filtered_rows] + stats = [r[stat] for r in filtered_rows] + plot_cumavg(query_counts, stats, xlabel='query#', ylabel=stat); + print_avg(stats, stat) +plt.figure() +average_stat(stat = 'stats.rank')