mirror of https://github.com/percyliang/sempre
425 lines
17 KiB
Plaintext
425 lines
17 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "raw",
|
|
"metadata": {},
|
|
"source": [
|
|
"A line looks like this\n",
|
|
" \"time\": \"2017-01-21T05:31:57.474\",\n",
|
|
" \"id\": \"AMT_A1HKYY6XI2OHO1\",\n",
|
|
" \"log\": \"(:q \\\"repeat 10 [ Ebony Wall; select front]\\\")\",\n",
|
|
" \"stats.type\": \"q\",\n",
|
|
" \"stats.size\": 1,\n",
|
|
" \"stats.status\": \"Induced\",\n",
|
|
" \"queryCount\": 2707"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import csv\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd \n",
|
|
"import matplotlib\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"import json\n",
|
|
"import os\n",
|
|
"import subprocess\n",
|
|
"from collections import OrderedDict\n",
|
|
"%matplotlib inline \n",
|
|
"\n",
|
|
"with open('../state/lastExec', 'rb') as lastExec:\n",
|
|
" lastExecInd = lastExec.readline().strip()\n",
|
|
"print lastExecInd\n",
|
|
" \n",
|
|
"rows = []; \n",
|
|
"execInd = lastExecInd;\n",
|
|
"execPath = '../state/execs/%s.exec/' % execInd\n",
|
|
"#print 'analyzing: ' + execPath\n",
|
|
"def printOptions():\n",
|
|
" with open(os.path.join(execPath,'options.map')) as optionsfile:\n",
|
|
" opts = filter(lambda l: 'file' in l or 'logFiles' in l or 'reqParams' in l, optionsfile.readlines());\n",
|
|
" for opt in opts: print opt.strip()\n",
|
|
" # egrep 'file|Simulator'\n",
|
|
"printOptions()\n",
|
|
"\n",
|
|
"with open('../state/execs/%s.exec/plotInfo.json' % execInd, 'rb') as jsonfile:\n",
|
|
" json_lines = jsonfile.readlines()\n",
|
|
"\n",
|
|
"rawrows = [json.loads(l) for l in json_lines]\n",
|
|
"rows = [r for r in rawrows if r.has_key('stats.type')]\n",
|
|
"\n",
|
|
"print '%d queries in plotInfo' % len(json_lines)\n",
|
|
"\n",
|
|
"def print_defstats():\n",
|
|
" filtered_rows = [r for r in rows if r['stats.type'] == 'def' and r['stats.num_rules'] >= 1]\n",
|
|
" total_failed = np.sum([r['stats.num_failed'] for r in filtered_rows])\n",
|
|
" total_body = np.sum([r['stats.num_body'] for r in filtered_rows])\n",
|
|
"\n",
|
|
" stats = {\\\n",
|
|
" 'total_def_queries': len(filtered_rows), \\\n",
|
|
" 'total_rules': np.sum([r['stats.num_rules'] for r in filtered_rows]), \\\n",
|
|
" 'total_failed': total_failed, \\\n",
|
|
" 'total_body': total_body\n",
|
|
" }\n",
|
|
" print ''\n",
|
|
" print stats\n",
|
|
" print 'failpercent: %.4f' % (float(total_failed)/float(total_body))\n",
|
|
"print_defstats()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"scrolled": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"\n",
|
|
"def print_stats():\n",
|
|
" filtered_rows = [r for r in rows if r['stats.type'] == 'accept']\n",
|
|
" induced_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status']=='Induced']\n",
|
|
" core_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status']=='Core']\n",
|
|
" none_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status']=='Nothing']\n",
|
|
"\n",
|
|
" stats = {\\\n",
|
|
" 'accepted': len(filtered_rows), \\\n",
|
|
" 'induced': len(induced_rows), \\\n",
|
|
" 'inducedp':len(induced_rows)/float(len(filtered_rows)),\\\n",
|
|
" 'core': len(core_rows), \\\n",
|
|
" 'corep':len(core_rows)/float(len(filtered_rows)),\\\n",
|
|
" 'none': len(none_rows), \\\n",
|
|
" 'nonep':len(none_rows)/float(len(filtered_rows))\\\n",
|
|
" }\n",
|
|
" print '\\nCount of accepted query / percentage'\n",
|
|
" print ' total:{accepted}\\n induced:{induced}({inducedp:.4f})\\n core:{core}({corep:.4f})\\n none:{none}({nonep:.4f}),'.format(**stats)\n",
|
|
" #print 'check %f' % (stats['inducedp']+stats['corep']+stats['nonep'])\n",
|
|
" \n",
|
|
" statscorrect = {\n",
|
|
" 'accepted0': np.mean([1 if r['stats.rank']==0 and r['stats.status']!='Nothing' else 0 for r in filtered_rows]),\\\n",
|
|
" 'accepted1': np.mean([1 if r['stats.rank']>=0 and r['stats.status']!='Nothing' else 0 for r in filtered_rows]),\\\n",
|
|
" 'induced0': np.mean([1 if r['stats.rank']==0 and r['stats.status']=='Induced' else 0 for r in filtered_rows]),\\\n",
|
|
" 'induced1': np.mean([1 if r['stats.rank']>=0 and r['stats.status']=='Induced' else 0 for r in filtered_rows]),\\\n",
|
|
" 'core0': np.mean([1 if r['stats.rank']==0 and r['stats.status']=='Core' else 0 for r in filtered_rows]),\\\n",
|
|
" 'core1': np.mean([1 if r['stats.rank']>=0 and r['stats.status']=='Core' else 0 for r in filtered_rows])\\\n",
|
|
" }\n",
|
|
" print '\\nranked 1 accepted / found accepted'\n",
|
|
" print ' total:{accepted0:.4f}/{accepted1:.4f}\\n induced:{induced0:.4f}/{induced1:.4f}\\n core:{core0:.4f}/{core1:.4f}'.format(**statscorrect)\n",
|
|
"\n",
|
|
" token_types = set();\n",
|
|
" rows_types = [r for r in rows if r['stats.type'] == 'q'];\n",
|
|
" for r in rows_types:\n",
|
|
" token_types |= set(r['q'].split(' '))\n",
|
|
" \n",
|
|
" print '\\nThere are %d token types of %d query types' % (len(token_types), len(rows_types))\n",
|
|
"print_stats()\n",
|
|
"\n",
|
|
"def percent_core():\n",
|
|
" allq = [r for r in rows if r['stats.type'] == 'accept'];\n",
|
|
" is_status = [100 if r['stats.status'] == 'Induced' else 0 for r in allq]\n",
|
|
" print 'percent_induced %f' % np.mean(is_status)\n",
|
|
" print 'percent_induced_last10k %f' % np.mean(is_status[len(is_status)-10000:])\n",
|
|
"percent_core()\n",
|
|
"\n",
|
|
"# print filtered_rows[0]\n",
|
|
"\n",
|
|
"\n",
|
|
"\n",
|
|
"def percent_error():\n",
|
|
" allerror = [r for r in rawrows if r.has_key('stats.error')];\n",
|
|
" alluerror = [r for r in rawrows if r.has_key('stats.uncaught_error')];\n",
|
|
" print 'errors %d (%d uncaught)' % (len(allerror), len(alluerror))\n",
|
|
"percent_error()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"def plot_reset():\n",
|
|
" global p\n",
|
|
" p = {'color': 'r', 'linewidth': 2, 'alpha':0.5}\n",
|
|
" #, 'marker':'*', 'markersize':0.3}\n",
|
|
"plot_reset()\n",
|
|
"def savefig(filename = 'fig.pdf'):\n",
|
|
" plt.savefig(os.path.join(execPath,filename) , bbox_inches=\"tight\")\n",
|
|
"\n",
|
|
"def plot_cumavg(x, y, xlabel='query#', ylabel='recall', title=None):\n",
|
|
" y_cum = np.cumsum(y).tolist()\n",
|
|
" #print accepts_np[:,1]\n",
|
|
" y_cumavg = [cum / float(count+1) for count,cum in enumerate(y_cum)]\n",
|
|
" #N = 500;\n",
|
|
" #y_cumavg = np.convolve(np.array(y), np.ones((N,))/N, mode='same').tolist()\n",
|
|
" #print accept_rate \n",
|
|
" #plt.scatter(means_baseline[0:], means[0:], s=colors, alpha=0.8, c='r')\n",
|
|
" plt.plot(x, y_cumavg, **p)\n",
|
|
" \n",
|
|
" plt.xlabel(xlabel, fontsize=12)\n",
|
|
" plt.ylabel(ylabel, fontsize=12)\n",
|
|
" # plt.xlim(0, 0.65)\n",
|
|
" plt.ylim(0, max(y_cumavg)*1.02)\n",
|
|
" plt.xlim(0, len(rows)*1.02)\n",
|
|
" xp = np.linspace(0, 0.65, 300)\n",
|
|
" \n",
|
|
" #plt.gca().set_aspect('equal', adjustable='box')\n",
|
|
" plottitle = title if title is not None else '%s_vs_%s.pdf' % (xlabel, ylabel)\n",
|
|
" # plt.savefig(os.path.join(execPath,plottitle) , bbox_inches=\"tight\")\n",
|
|
" \n",
|
|
"def print_avg(x, name = 'unnamed'):\n",
|
|
" print 'avg(%s): %f' % (name, reduce(lambda a,b: a+b, x) / float(len(x)));\n",
|
|
"def average_stat(stat = 'stats.size', type = 'accept'):\n",
|
|
" filtered_rows = [r for r in rows if r['stats.type'] == type]\n",
|
|
" query_counts = [r['queryCount'] for r in filtered_rows]\n",
|
|
" stats = [r[stat] for r in filtered_rows]\n",
|
|
" plot_cumavg(query_counts, stats, xlabel='query#', ylabel=stat.replace('stats.','').replace('size','# parses'));\n",
|
|
" print_avg(stats, stat)\n",
|
|
"plt.figure()\n",
|
|
"p['color'] = 'b'\n",
|
|
"average_stat(stat = 'stats.size')\n",
|
|
"#savefig('ambiguity.pdf')\n",
|
|
"\n",
|
|
"\n",
|
|
"def precent_status(filtered_rows, status = 'Core'):\n",
|
|
" query_counts = [r['queryCount'] for r in filtered_rows]\n",
|
|
" is_status = [100 if r['stats.status'] == status else 0 for r in filtered_rows]\n",
|
|
" print_avg(is_status, 'percent of status ' + status)\n",
|
|
" plot_cumavg(query_counts, is_status, xlabel='query #', ylabel='percent');\n",
|
|
"\n",
|
|
"\n",
|
|
"def plotCoreInducedNone(filtered_rows):\n",
|
|
" p['color'] = 'g'; p['label'] = 'induced';\n",
|
|
" precent_status(filtered_rows,status = 'Induced');\n",
|
|
" p['color'] = 'b'; p['label'] = 'core';\n",
|
|
" precent_status(filtered_rows, status = 'Core');\n",
|
|
" plt.legend(frameon=False)\n",
|
|
" \n",
|
|
"plt.figure()\n",
|
|
"plotCoreInducedNone([r for r in rows if r['stats.type'] == 'accept'])\n",
|
|
"# savefig('parse_status_accepted.pdf')\n",
|
|
"\n",
|
|
"def plotCoreInducedNone(filtered_rows):\n",
|
|
" p['color'] = 'r'; p['label'] = 'none';\n",
|
|
" precent_status(filtered_rows, status = 'Nothing');\n",
|
|
" p['color'] = 'g'; p['label'] = 'induced';\n",
|
|
" precent_status(filtered_rows,status = 'Induced');\n",
|
|
" p['color'] = 'b'; p['label'] = 'core';\n",
|
|
" precent_status(filtered_rows, status = 'Core');\n",
|
|
" plt.legend(frameon=False)\n",
|
|
"plt.figure()\n",
|
|
"plotCoreInducedNone([r for r in rows if r['stats.type'] == 'q' and r.has_key('stats.status')])\n",
|
|
"plt.ylim(0, 70)\n",
|
|
"savefig('parse_status_q.pdf')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def top_users(counts, line):\n",
|
|
" id = 'sessionId'\n",
|
|
" if line[id] in counts:\n",
|
|
" counts[line[id]] = counts[line[id]] + 1\n",
|
|
" else:\n",
|
|
" counts[line[id]] = 1\n",
|
|
" return counts\n",
|
|
"\n",
|
|
"accept_all = [r for r in rows if r['stats.type']=='accept']\n",
|
|
"accept_nothing = [r for r in rows if r['stats.type']=='accept' and r['stats.status']=='Nothing']\n",
|
|
"\n",
|
|
"for r in accept_nothing[:5]:\n",
|
|
" print '{q}'.format(**r)\n",
|
|
" \n",
|
|
"sorted(reduce(top_users, rows, {}).items(), key=lambda x: -x[1]);"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def precent_status_user(filtered_rows):\n",
|
|
" query_counts = [r['queryCount'] for r in filtered_rows]\n",
|
|
" is_status = [100 if r['stats.status'] == 'Induced' else 0 for r in filtered_rows]\n",
|
|
" plot_cumavg(query_counts, is_status, xlabel='query #', ylabel='precent induced');\n",
|
|
"\n",
|
|
"rows_to_count = [r for r in rows if r['stats.type'] == 'accept']\n",
|
|
"ranked_users = sorted(reduce(top_users, rows_to_count[5000:], {}).items(), key=lambda x: -x[1])\n",
|
|
"\n",
|
|
"plt.figure()\n",
|
|
"topnum = 5;\n",
|
|
"plot_reset()\n",
|
|
"p['alpha'] = 1;\n",
|
|
"p['linewidth'] = 5;\n",
|
|
"p['label'] = 'all'; \n",
|
|
"p['color'] = 'k';\n",
|
|
"precent_status_user([r for r in rows if r['stats.type'] == 'accept']); \n",
|
|
"\n",
|
|
"colors = ['c', 'r', 'm', 'y', 'b', 'g']\n",
|
|
"plot_reset()\n",
|
|
"p['alpha'] = 1;\n",
|
|
"p['linewidth'] = 5;\n",
|
|
"p['alpha'] = 0.5;\n",
|
|
"p['marker'] = 'o';\n",
|
|
"p['markersize'] = 1;\n",
|
|
"\n",
|
|
"for g in enumerate(ranked_users[0:5]):\n",
|
|
" # (0, (u'AMT_A1HKYY6XI2OHO1', 2830))\n",
|
|
" p['label'] = '#%d' % (g[0]+1);\n",
|
|
" #plotsetting['alpha'] = 1-float(g[0])/topnum;\n",
|
|
" p['color'] = colors[g[0]];\n",
|
|
" # print plotsetting['color']\n",
|
|
" precent_status_user([r for r in rows if r['stats.type'] == 'accept' and r['sessionId'] == g[1][0]])\n",
|
|
" print g\n",
|
|
"\n",
|
|
"plt.ylim(-0.1, 100)\n",
|
|
"plt.legend(frameon=False, loc='lower right')\n",
|
|
"savefig('parse_status_topuser.pdf');"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def expressivity(status = 'Core'):\n",
|
|
" filtered_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status'] == status]\n",
|
|
" query_counts = [r['queryCount'] for r in filtered_rows]\n",
|
|
" len_formula = [r['stats.len_formula'] for r in filtered_rows]\n",
|
|
" len_utterance = [r['stats.len_utterance'] for r in filtered_rows]\n",
|
|
" form_per_q = [float(ls[0])/ls[1] for ls in zip(len_formula, len_utterance)]\n",
|
|
" plot_cumavg(query_counts, form_per_q, xlabel='query#', ylabel='\"expressiveness\"');\n",
|
|
"\n",
|
|
" #plot_cumavg(query_counts, len_utterance, xlabel='query#', ylabel='length');\n",
|
|
" print_avg(form_per_q, 'formula length')\n",
|
|
"plt.figure()\n",
|
|
"plot_reset()\n",
|
|
"p['color'] = 'b'; p['label'] = 'core'; expressivity('Core')\n",
|
|
"p['color'] = 'g'; p['label'] = 'induced'; expressivity('Induced')\n",
|
|
"plt.legend(frameon=False, loc='upper left')\n",
|
|
"# savefig('expressiveness.pdf')\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def expressivity_by_users(filtered_rows):\n",
|
|
" query_counts = [r['queryCount'] for r in filtered_rows]\n",
|
|
" len_formula = [r['stats.len_formula'] for r in filtered_rows]\n",
|
|
" len_utterance = [r['stats.len_utterance'] for r in filtered_rows]\n",
|
|
" form_per_q = [float(ls[0])/ls[1] for ls in zip(len_formula, len_utterance)]\n",
|
|
" plot_cumavg(query_counts, form_per_q, xlabel='query#', ylabel='len(z) / len(x)');\n",
|
|
"\n",
|
|
"plot_reset()\n",
|
|
"plt.figure()\n",
|
|
"p['alpha'] = 1;\n",
|
|
"p['linewidth'] = 5;\n",
|
|
"p['label'] = 'all'; \n",
|
|
"p['color'] = 'k';\n",
|
|
"expressivity_by_users([r for r in rows if r['stats.type'] == 'accept' ])\n",
|
|
" \n",
|
|
"colors = ['c', 'r', 'm', 'y', 'b']\n",
|
|
"plot_reset()\n",
|
|
"p['alpha'] = 1;\n",
|
|
"p['linewidth'] = 5;\n",
|
|
"p['alpha'] = 0.5;\n",
|
|
"p['marker'] = 'o';\n",
|
|
"p['markersize'] = 1;\n",
|
|
"rows_to_count = [r for r in rows if r['stats.type'] == 'accept']\n",
|
|
"ranked_users = sorted(reduce(top_users, rows_to_count, {}).items(), key=lambda x: -x[1])\n",
|
|
"for g in enumerate(ranked_users[0:5]):\n",
|
|
" print g\n",
|
|
" p['label'] = '#%d' % (g[0]+1);\n",
|
|
" p['color'] = colors[g[0]];\n",
|
|
" expressivity_by_users([r for r in rows if r['stats.type'] == 'accept' \\\n",
|
|
" and r['sessionId'] == g[1][0]])\n",
|
|
"\n",
|
|
"plt.ylim([0,30])\n",
|
|
"plt.legend(frameon=False, loc='lower right')\n",
|
|
"savefig('expressiveness_by_user.pdf')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def query_reformulation_by_user():\n",
|
|
" q_all = [r for r in rows if r['stats.type']=='q']\n",
|
|
" q_nothing = [r for r in rows if r['stats.type']=='q' and r.has_key('stats.status') and r['stats.status']=='Nothing']\n",
|
|
" print 'q_nothing: %d / %d totallines: %d' % (len(q_nothing), len(q_all), len(json_lines))\n",
|
|
" ranked_users = sorted(reduce(top_users, q_all, {}).items(), key=lambda x: -x[1])\n",
|
|
"\n",
|
|
" num_user = 3;\n",
|
|
" num_query = 100;\n",
|
|
" for g in (ranked_users[0:num_user]):\n",
|
|
" print g\n",
|
|
" print '********************'\n",
|
|
" rows_userg = [r for r in rows if r['stats.type']=='q' and r['sessionId']==g[0] and r.has_key('stats.status') ]\n",
|
|
" print_count = 0\n",
|
|
" prev_nothing = False\n",
|
|
" for r in rows_userg[-num_query:]:\n",
|
|
" if prev_nothing or r['stats.status']=='Nothing':\n",
|
|
" print_count = print_count + 1\n",
|
|
" if print_count>100: break\n",
|
|
" print r['stats.status'] + ':\\t' + r['q'].replace('(:q \"','').replace('\")','')\n",
|
|
" prev_nothing = True if r['stats.status']=='Nothing' else False\n",
|
|
"\n",
|
|
"query_reformulation_by_user()"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 2",
|
|
"language": "python",
|
|
"name": "python2"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 2
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython2",
|
|
"version": "2.7.9"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
}
|