mirror of https://github.com/percyliang/sempre
more data analysis
This commit is contained in:
Sida Wang
parent
8d3d170bd9
commit
8d5d7c486f
|
|
@ -39,16 +39,17 @@ After `npm install` and taking care of dependencies, run `npm start local` to st
|
|||
|
||||
./interactive/run @mode=voxelurn -server -interactive
|
||||
|
||||
|
||||
2. Feed the server all the query logs
|
||||
|
||||
./interactive/run @mode=simulator @server=local @sandbox=none @task=freebig -maxQueries 103876
|
||||
|
||||
decrease maxQuery for shorter plots
|
||||
This currently takes just under 30 minutes. Decrease maxQuery for a quicker experiment. This generate `plotInfo.json` in `../state/execs/${lastExec}.exec/` where `lastExec` is `cat ../state/lastExec`.
|
||||
|
||||
3. Analyze the results and produce some numbers and graphs
|
||||
3. Taking `../state/execs/${lastExec}.exec/plotInfo.json` as input, we can analyze the data and produce some plots using the following ipython notebook
|
||||
|
||||
./interactive/run @mode=voxelurn
|
||||
ipython notebook analyze_data.ipynb
|
||||
|
||||
which prints out basic statistics and generates the plots used in our paper. The plots are saved at `../state/execs/${lastExec}.exec/`
|
||||
|
||||
## Client server (optional)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,420 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "raw",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"A line looks like this\n",
|
||||
" \"time\": \"2017-01-21T05:31:57.474\",\n",
|
||||
" \"id\": \"AMT_A1HKYY6XI2OHO1\",\n",
|
||||
" \"log\": \"(:q \\\"repeat 10 [ Ebony Wall; select front]\\\")\",\n",
|
||||
" \"stats.type\": \"q\",\n",
|
||||
" \"stats.size\": 1,\n",
|
||||
" \"stats.status\": \"Induced\",\n",
|
||||
" \"queryCount\": 2707"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import csv\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd \n",
|
||||
"import matplotlib\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"import subprocess\n",
|
||||
"from collections import OrderedDict\n",
|
||||
"%matplotlib inline \n",
|
||||
"\n",
|
||||
"with open('../state/lastExec', 'rb') as lastExec:\n",
|
||||
" lastExecInd = lastExec.readline().strip()\n",
|
||||
"print lastExecInd\n",
|
||||
" \n",
|
||||
"rows = []; \n",
|
||||
"execInd = lastExecInd;\n",
|
||||
"execPath = '../state/execs/%s.exec/' % execInd\n",
|
||||
"#print 'analyzing: ' + execPath\n",
|
||||
"def printOptions():\n",
|
||||
" with open(os.path.join(execPath,'options.map')) as optionsfile:\n",
|
||||
" opts = filter(lambda l: 'file' in l or 'logFiles' in l or 'reqParams' in l, optionsfile.readlines());\n",
|
||||
" for opt in opts: print opt.strip()\n",
|
||||
" # egrep 'file|Simulator'\n",
|
||||
"printOptions()\n",
|
||||
"\n",
|
||||
"with open('../state/execs/%s.exec/plotInfo.json' % execInd, 'rb') as jsonfile:\n",
|
||||
" json_lines = jsonfile.readlines()\n",
|
||||
"\n",
|
||||
"rawrows = [json.loads(l) for l in json_lines]\n",
|
||||
"rows = [r for r in rawrows if r.has_key('stats.type')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"scrolled": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print '%d queries in plotInfo' % len(json_lines)\n",
|
||||
"\n",
|
||||
"def print_stats():\n",
|
||||
" filtered_rows = [r for r in rows if r['stats.type'] == 'accept']\n",
|
||||
" induced_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status']=='Induced']\n",
|
||||
" core_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status']=='Core']\n",
|
||||
" none_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status']=='Nothing']\n",
|
||||
"\n",
|
||||
" stats = {\\\n",
|
||||
" 'accepted': len(filtered_rows), \\\n",
|
||||
" 'induced': len(induced_rows), \\\n",
|
||||
" 'inducedp':len(induced_rows)/float(len(filtered_rows)),\\\n",
|
||||
" 'core': len(core_rows), \\\n",
|
||||
" 'corep':len(core_rows)/float(len(filtered_rows)),\\\n",
|
||||
" 'none': len(none_rows), \\\n",
|
||||
" 'nonep':len(none_rows)/float(len(filtered_rows))\\\n",
|
||||
" }\n",
|
||||
" print '\\nCount of accepted query / percentage'\n",
|
||||
" print ' total:{accepted}\\n induced:{induced}({inducedp:.4f})\\n core:{core}({corep:.4f})\\n none:{none}({nonep:.4f}),'.format(**stats)\n",
|
||||
" #print 'check %f' % (stats['inducedp']+stats['corep']+stats['nonep'])\n",
|
||||
" \n",
|
||||
" statscorrect = {\n",
|
||||
" 'accepted0': np.mean([1 if r['stats.rank']==0 and r['stats.status']!='Nothing' else 0 for r in filtered_rows]),\\\n",
|
||||
" 'accepted1': np.mean([1 if r['stats.rank']>=0 and r['stats.status']!='Nothing' else 0 for r in filtered_rows]),\\\n",
|
||||
" 'induced0': np.mean([1 if r['stats.rank']==0 and r['stats.status']=='Induced' else 0 for r in filtered_rows]),\\\n",
|
||||
" 'induced1': np.mean([1 if r['stats.rank']>=0 and r['stats.status']=='Induced' else 0 for r in filtered_rows]),\\\n",
|
||||
" 'core0': np.mean([1 if r['stats.rank']==0 and r['stats.status']=='Core' else 0 for r in filtered_rows]),\\\n",
|
||||
" 'core1': np.mean([1 if r['stats.rank']>=0 and r['stats.status']=='Core' else 0 for r in filtered_rows])\\\n",
|
||||
" }\n",
|
||||
" print '\\nranked 1 accepted / found accepted'\n",
|
||||
" print ' total:{accepted0:.4f}/{accepted1:.4f}\\n induced:{induced0:.4f}/{induced1:.4f}\\n core:{core0:.4f}/{core1:.4f}'.format(**statscorrect)\n",
|
||||
"\n",
|
||||
" token_types = set();\n",
|
||||
" rows_types = [r for r in rows if r['stats.type'] == 'q'];\n",
|
||||
" for r in rows_types:\n",
|
||||
" token_types |= set(r['q'].split(' '))\n",
|
||||
" \n",
|
||||
" print '\\nThere are %d token types of %d query types' % (len(token_types), len(rows_types))\n",
|
||||
"print_stats()\n",
|
||||
"\n",
|
||||
"def percent_core():\n",
|
||||
" allq = [r for r in rows if r['stats.type'] == 'accept'];\n",
|
||||
" is_status = [100 if r['stats.status'] == 'Induced' else 0 for r in allq]\n",
|
||||
" print 'percent_induced %f' % np.mean(is_status)\n",
|
||||
" print 'percent_induced_last10k %f' % np.mean(is_status[len(is_status)-10000:])\n",
|
||||
"percent_core()\n",
|
||||
"\n",
|
||||
"# print filtered_rows[0]\n",
|
||||
"\n",
|
||||
"def print_defstats():\n",
|
||||
" filtered_rows = [r for r in rows if r['stats.type'] == 'def' and r['stats.num_rules'] >= 1]\n",
|
||||
" total_failed = np.sum([r['stats.num_failed'] for r in filtered_rows])\n",
|
||||
" total_body = np.sum([r['stats.num_body'] for r in filtered_rows])\n",
|
||||
"\n",
|
||||
" stats = {\\\n",
|
||||
" 'total_def_queries': len(filtered_rows), \\\n",
|
||||
" 'total_rules': np.sum([r['stats.num_rules'] for r in filtered_rows]), \\\n",
|
||||
" 'total_failed': total_failed, \\\n",
|
||||
" 'total_body': total_body\n",
|
||||
" }\n",
|
||||
" print ''\n",
|
||||
" print stats\n",
|
||||
" print 'failpercent: %.4f' % (float(total_failed)/float(total_body))\n",
|
||||
"print_defstats()\n",
|
||||
"\n",
|
||||
"def percent_error():\n",
|
||||
" allerror = [r for r in rawrows if r.has_key('stats.error')];\n",
|
||||
" alluerror = [r for r in rawrows if r.has_key('stats.uncaught_error')];\n",
|
||||
" print 'errors %d (%d uncaught)' % (len(allerror), len(alluerror))\n",
|
||||
"percent_error()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"def plot_reset():\n",
|
||||
" global p\n",
|
||||
" p = {'color': 'r', 'linewidth': 2, 'alpha':0.5}\n",
|
||||
" #, 'marker':'*', 'markersize':0.3}\n",
|
||||
"plot_reset()\n",
|
||||
"def savefig(filename = 'fig.pdf'):\n",
|
||||
" plt.savefig(os.path.join(execPath,filename) , bbox_inches=\"tight\")\n",
|
||||
"\n",
|
||||
"def plot_cumavg(x, y, xlabel='query#', ylabel='recall', title=None):\n",
|
||||
" y_cum = np.cumsum(y).tolist()\n",
|
||||
" #print accepts_np[:,1]\n",
|
||||
" y_cumavg = [cum / float(count+1) for count,cum in enumerate(y_cum)]\n",
|
||||
" #N = 500;\n",
|
||||
" #y_cumavg = np.convolve(np.array(y), np.ones((N,))/N, mode='same').tolist()\n",
|
||||
" #print accept_rate \n",
|
||||
" #plt.scatter(means_baseline[0:], means[0:], s=colors, alpha=0.8, c='r')\n",
|
||||
" plt.plot(x, y_cumavg, **p)\n",
|
||||
" \n",
|
||||
" plt.xlabel(xlabel, fontsize=12)\n",
|
||||
" plt.ylabel(ylabel, fontsize=12)\n",
|
||||
" # plt.xlim(0, 0.65)\n",
|
||||
" plt.ylim(0, max(y_cumavg)*1.02)\n",
|
||||
" plt.xlim(0, len(rows)*1.02)\n",
|
||||
" xp = np.linspace(0, 0.65, 300)\n",
|
||||
" \n",
|
||||
" #plt.gca().set_aspect('equal', adjustable='box')\n",
|
||||
" plottitle = title if title is not None else '%s_vs_%s.pdf' % (xlabel, ylabel)\n",
|
||||
" # plt.savefig(os.path.join(execPath,plottitle) , bbox_inches=\"tight\")\n",
|
||||
" \n",
|
||||
"def print_avg(x, name = 'unnamed'):\n",
|
||||
" print 'avg(%s): %f' % (name, reduce(lambda a,b: a+b, x) / float(len(x)));\n",
|
||||
"def average_stat(stat = 'stats.size', type = 'accept'):\n",
|
||||
" filtered_rows = [r for r in rows if r['stats.type'] == type]\n",
|
||||
" query_counts = [r['queryCount'] for r in filtered_rows]\n",
|
||||
" stats = [r[stat] for r in filtered_rows]\n",
|
||||
" plot_cumavg(query_counts, stats, xlabel='query#', ylabel=stat.replace('stats.','').replace('size','# parses'));\n",
|
||||
" print_avg(stats, stat)\n",
|
||||
"plt.figure()\n",
|
||||
"p['color'] = 'b'\n",
|
||||
"average_stat(stat = 'stats.size')\n",
|
||||
"#savefig('ambiguity.pdf')\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def precent_status(filtered_rows, status = 'Core'):\n",
|
||||
" query_counts = [r['queryCount'] for r in filtered_rows]\n",
|
||||
" is_status = [100 if r['stats.status'] == status else 0 for r in filtered_rows]\n",
|
||||
" print_avg(is_status, 'percent of status ' + status)\n",
|
||||
" plot_cumavg(query_counts, is_status, xlabel='query #', ylabel='percent');\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def plotCoreInducedNone(filtered_rows):\n",
|
||||
" p['color'] = 'g'; p['label'] = 'induced';\n",
|
||||
" precent_status(filtered_rows,status = 'Induced');\n",
|
||||
" p['color'] = 'b'; p['label'] = 'core';\n",
|
||||
" precent_status(filtered_rows, status = 'Core');\n",
|
||||
" plt.legend(frameon=False)\n",
|
||||
" \n",
|
||||
"plt.figure()\n",
|
||||
"plotCoreInducedNone([r for r in rows if r['stats.type'] == 'accept'])\n",
|
||||
"# savefig('parse_status_accepted.pdf')\n",
|
||||
"\n",
|
||||
"def plotCoreInducedNone(filtered_rows):\n",
|
||||
" p['color'] = 'r'; p['label'] = 'none';\n",
|
||||
" precent_status(filtered_rows, status = 'Nothing');\n",
|
||||
" p['color'] = 'g'; p['label'] = 'induced';\n",
|
||||
" precent_status(filtered_rows,status = 'Induced');\n",
|
||||
" p['color'] = 'b'; p['label'] = 'core';\n",
|
||||
" precent_status(filtered_rows, status = 'Core');\n",
|
||||
" plt.legend(frameon=False)\n",
|
||||
"plt.figure()\n",
|
||||
"plotCoreInducedNone([r for r in rows if r['stats.type'] == 'q' and r.has_key('stats.status')])\n",
|
||||
"plt.ylim(0, 70)\n",
|
||||
"savefig('parse_status_q.pdf')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def top_users(counts, line):\n",
|
||||
" id = 'sessionId'\n",
|
||||
" if line[id] in counts:\n",
|
||||
" counts[line[id]] = counts[line[id]] + 1\n",
|
||||
" else:\n",
|
||||
" counts[line[id]] = 1\n",
|
||||
" return counts\n",
|
||||
"\n",
|
||||
"accept_all = [r for r in rows if r['stats.type']=='accept']\n",
|
||||
"accept_nothing = [r for r in rows if r['stats.type']=='accept' and r['stats.status']=='Nothing']\n",
|
||||
"\n",
|
||||
"for r in accept_nothing[:5]:\n",
|
||||
" print '{q}'.format(**r)\n",
|
||||
" \n",
|
||||
"sorted(reduce(top_users, rows, {}).items(), key=lambda x: -x[1]);"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def precent_status_user(filtered_rows):\n",
|
||||
" query_counts = [r['queryCount'] for r in filtered_rows]\n",
|
||||
" is_status = [100 if r['stats.status'] == 'Induced' else 0 for r in filtered_rows]\n",
|
||||
" plot_cumavg(query_counts, is_status, xlabel='query #', ylabel='precent induced');\n",
|
||||
"\n",
|
||||
"rows_to_count = [r for r in rows if r['stats.type'] == 'accept']\n",
|
||||
"ranked_users = sorted(reduce(top_users, rows_to_count[5000:], {}).items(), key=lambda x: -x[1])\n",
|
||||
"\n",
|
||||
"plt.figure()\n",
|
||||
"topnum = 5;\n",
|
||||
"plot_reset()\n",
|
||||
"p['alpha'] = 1;\n",
|
||||
"p['linewidth'] = 5;\n",
|
||||
"p['label'] = 'all'; \n",
|
||||
"p['color'] = 'k';\n",
|
||||
"precent_status_user([r for r in rows if r['stats.type'] == 'accept']); \n",
|
||||
"\n",
|
||||
"colors = ['c', 'r', 'm', 'y', 'b', 'g']\n",
|
||||
"plot_reset()\n",
|
||||
"p['alpha'] = 1;\n",
|
||||
"p['linewidth'] = 5;\n",
|
||||
"p['alpha'] = 0.5;\n",
|
||||
"p['marker'] = 'o';\n",
|
||||
"p['markersize'] = 1;\n",
|
||||
"\n",
|
||||
"for g in enumerate(ranked_users[0:5]):\n",
|
||||
" # (0, (u'AMT_A1HKYY6XI2OHO1', 2830))\n",
|
||||
" p['label'] = '#%d' % (g[0]+1);\n",
|
||||
" #plotsetting['alpha'] = 1-float(g[0])/topnum;\n",
|
||||
" p['color'] = colors[g[0]];\n",
|
||||
" # print plotsetting['color']\n",
|
||||
" precent_status_user([r for r in rows if r['stats.type'] == 'accept' and r['sessionId'] == g[1][0]])\n",
|
||||
" print g\n",
|
||||
"\n",
|
||||
"plt.ylim(-0.1, 100)\n",
|
||||
"plt.legend(frameon=False, loc='lower right')\n",
|
||||
"savefig('parse_status_topuser.pdf');"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def expressivity(status = 'Core'):\n",
|
||||
" filtered_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status'] == status]\n",
|
||||
" query_counts = [r['queryCount'] for r in filtered_rows]\n",
|
||||
" len_formula = [r['stats.len_formula'] for r in filtered_rows]\n",
|
||||
" len_utterance = [r['stats.len_utterance'] for r in filtered_rows]\n",
|
||||
" form_per_q = [float(ls[0])/ls[1] for ls in zip(len_formula, len_utterance)]\n",
|
||||
" plot_cumavg(query_counts, form_per_q, xlabel='query#', ylabel='\"expressiveness\"');\n",
|
||||
"\n",
|
||||
" #plot_cumavg(query_counts, len_utterance, xlabel='query#', ylabel='length');\n",
|
||||
" print_avg(form_per_q, 'formula length')\n",
|
||||
"plt.figure()\n",
|
||||
"plot_reset()\n",
|
||||
"p['color'] = 'b'; p['label'] = 'core'; expressivity('Core')\n",
|
||||
"p['color'] = 'g'; p['label'] = 'induced'; expressivity('Induced')\n",
|
||||
"plt.legend(frameon=False, loc='upper left')\n",
|
||||
"# savefig('expressiveness.pdf')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def expressivity_by_users(filtered_rows):\n",
|
||||
" query_counts = [r['queryCount'] for r in filtered_rows]\n",
|
||||
" len_formula = [r['stats.len_formula'] for r in filtered_rows]\n",
|
||||
" len_utterance = [r['stats.len_utterance'] for r in filtered_rows]\n",
|
||||
" form_per_q = [float(ls[0])/ls[1] for ls in zip(len_formula, len_utterance)]\n",
|
||||
" plot_cumavg(query_counts, form_per_q, xlabel='query#', ylabel='len(z) / len(x)');\n",
|
||||
"\n",
|
||||
"plot_reset()\n",
|
||||
"plt.figure()\n",
|
||||
"p['alpha'] = 1;\n",
|
||||
"p['linewidth'] = 5;\n",
|
||||
"p['label'] = 'all'; \n",
|
||||
"p['color'] = 'k';\n",
|
||||
"expressivity_by_users([r for r in rows if r['stats.type'] == 'accept' ])\n",
|
||||
" \n",
|
||||
"colors = ['c', 'r', 'm', 'y', 'b']\n",
|
||||
"plot_reset()\n",
|
||||
"p['alpha'] = 1;\n",
|
||||
"p['linewidth'] = 5;\n",
|
||||
"p['alpha'] = 0.5;\n",
|
||||
"p['marker'] = 'o';\n",
|
||||
"p['markersize'] = 1;\n",
|
||||
"rows_to_count = [r for r in rows if r['stats.type'] == 'accept']\n",
|
||||
"ranked_users = sorted(reduce(top_users, rows_to_count, {}).items(), key=lambda x: -x[1])\n",
|
||||
"for g in enumerate(ranked_users[0:5]):\n",
|
||||
" print g\n",
|
||||
" p['label'] = '#%d' % (g[0]+1);\n",
|
||||
" p['color'] = colors[g[0]];\n",
|
||||
" expressivity_by_users([r for r in rows if r['stats.type'] == 'accept' \\\n",
|
||||
" and r['sessionId'] == g[1][0]])\n",
|
||||
"\n",
|
||||
"plt.ylim([0,30])\n",
|
||||
"plt.legend(frameon=False, loc='lower right')\n",
|
||||
"savefig('expressiveness_by_user.pdf')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def query_reformulation_by_user():\n",
|
||||
" q_all = [r for r in rows if r['stats.type']=='q']\n",
|
||||
" q_nothing = [r for r in rows if r['stats.type']=='q' and r.has_key('stats.status') and r['stats.status']=='Nothing']\n",
|
||||
" print 'q_nothing: %d / %d totallines: %d' % (len(q_nothing), len(q_all), len(json_lines))\n",
|
||||
" ranked_users = sorted(reduce(top_users, q_all, {}).items(), key=lambda x: -x[1])\n",
|
||||
"\n",
|
||||
" num_user = 3;\n",
|
||||
" num_query = 100;\n",
|
||||
" for g in (ranked_users[0:num_user]):\n",
|
||||
" print g\n",
|
||||
" print '********************'\n",
|
||||
" rows_userg = [r for r in rows if r['stats.type']=='q' and r['sessionId']==g[0]]\n",
|
||||
" print_count = 0\n",
|
||||
" prev_nothing = False\n",
|
||||
" for r in rows_userg[-num_query:]:\n",
|
||||
" if prev_nothing or r['stats.status']=='Nothing':\n",
|
||||
" print_count = print_count + 1\n",
|
||||
" if print_count>100: break\n",
|
||||
" print r['stats.status'] + ':\\t' + r['q'].replace('(:q \"','').replace('\")','')\n",
|
||||
" prev_nothing = True if r['stats.status']=='Nothing' else False\n",
|
||||
"\n",
|
||||
"query_reformulation_by_user()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 2",
|
||||
"language": "python",
|
||||
"name": "python2"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 2
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython2",
|
||||
"version": "2.7.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
|
|
@ -1,130 +0,0 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "raw",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"A line looks like this\n",
|
||||
" \"time\": \"2017-01-21T05:31:57.474\",\n",
|
||||
" \"id\": \"AMT_A1HKYY6XI2OHO1\",\n",
|
||||
" \"log\": \"(:q \\\"repeat 10 [ Ebony Wall; select front]\\\")\",\n",
|
||||
" \"stats.type\": \"q\",\n",
|
||||
" \"stats.size\": 1,\n",
|
||||
" \"stats.status\": \"Induced\",\n",
|
||||
" \"queryCount\": 2707"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 85,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"scrolled": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"995\n",
|
||||
"log.file\tstate/execs/995.exec/log\n",
|
||||
"Simulator.reqParams\tgrammar=1&cite=1&learn=1&logging=0\n",
|
||||
"Simulator.logFiles\t./interactive/queries/freebuildbig-0206.def.json.gz\n",
|
||||
"2000 lines in plotInfo\n",
|
||||
"[]\n",
|
||||
"errors:66, stats.uncaught_error:0\n",
|
||||
"{'total_rules': 2320, 'total_failed': 447, 'total_def_queries': 1911, 'total_body': 10973}\n",
|
||||
"failpercent: 0.0407\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import csv\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd \n",
|
||||
"import matplotlib\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"import subprocess\n",
|
||||
"from collections import OrderedDict\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"with open('../state/lastExec', 'rb') as lastExec:\n",
|
||||
" lastExecInd = lastExec.readline().strip()\n",
|
||||
"print lastExecInd\n",
|
||||
"\n",
|
||||
"rows = []; \n",
|
||||
"execInd = lastExecInd;\n",
|
||||
"execPath = '../state/execs/%s.exec/' % execInd\n",
|
||||
"#print 'analyzing: ' + execPath\n",
|
||||
"def printOptions():\n",
|
||||
" with open(os.path.join(execPath,'options.map')) as optionsfile:\n",
|
||||
" opts = filter(lambda l: 'file' in l or 'logFiles' in l or 'reqParams' in l, optionsfile.readlines());\n",
|
||||
" for opt in opts: print opt.strip()\n",
|
||||
" # egrep 'file|Simulator'\n",
|
||||
"printOptions()\n",
|
||||
"\n",
|
||||
"with open('../state/execs/%s.exec/plotInfo.json' % execInd, 'rb') as jsonfile:\n",
|
||||
" json_lines = jsonfile.readlines()\n",
|
||||
"print '%d lines in plotInfo' % len(json_lines)\n",
|
||||
"rows = [json.loads(l) for l in json_lines]\n",
|
||||
"\n",
|
||||
"typed_rows = [r for r in rows if r.has_key('stats.type')]\n",
|
||||
"error_rows = [r for r in rows if not r.has_key('stats.type')]\n",
|
||||
"print [r['stats.uncaught_error'] for r in error_rows]\n",
|
||||
"normal_error = [r for r in typed_rows if r.has_key('stats.error')]\n",
|
||||
"print 'errors:%d, stats.uncaught_error:%d' % (len(normal_error),len(error_rows))\n",
|
||||
"\n",
|
||||
"filtered_rows = [r for r in typed_rows if r['stats.type'] == 'def' and r['stats.num_rules'] >= 1]\n",
|
||||
"\n",
|
||||
"# print filtered_rows[0]\n",
|
||||
"\n",
|
||||
"def print_stats():\n",
|
||||
" total_failed = np.sum([r['stats.num_failed'] for r in filtered_rows])\n",
|
||||
" total_body = np.sum([r['stats.num_body'] for r in filtered_rows])\n",
|
||||
"\n",
|
||||
" stats = {\\\n",
|
||||
" 'total_def_queries': len(filtered_rows), \\\n",
|
||||
" 'total_rules': np.sum([r['stats.num_rules'] for r in filtered_rows]), \\\n",
|
||||
" 'total_failed': total_failed, \\\n",
|
||||
" 'total_body': total_body\n",
|
||||
" }\n",
|
||||
" print stats\n",
|
||||
" print 'failpercent: %.4f' % (float(total_failed)/float(total_body))\n",
|
||||
"\n",
|
||||
"print_stats()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 2",
|
||||
"language": "python",
|
||||
"name": "python2"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 2
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython2",
|
||||
"version": "2.7.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
import csv
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib
|
||||
import matplotlib.pyplot as plt
|
||||
import json
|
||||
%matplotlib inline
|
||||
|
||||
plotsetting_default = {'color': 'r', 'linewidth': 2}
|
||||
plotsetting = {'color': 'r', 'linewidth': 2}
|
||||
|
||||
def plot_cumavg(x, y, xlabel='query#', ylabel='recall', title=None):
|
||||
y_cum = np.cumsum(y).tolist()
|
||||
#print accepts_np[:,1]
|
||||
y_cumavg = [cum / float(count+1) for count,cum in enumerate(y_cum)]
|
||||
#N = 500;
|
||||
#y_cumavg = np.convolve(np.array(y), np.ones((N,))/N, mode='same').tolist()
|
||||
|
||||
#print accept_rate
|
||||
|
||||
#plt.scatter(means_baseline[0:], means[0:], s=colors, alpha=0.8, c='r')
|
||||
plt.plot(x, y_cumavg, alpha=0.5, **plotsetting)
|
||||
|
||||
plt.xlabel(xlabel, fontsize=12)
|
||||
plt.ylabel(ylabel, fontsize=12)
|
||||
# plt.xlim(0, 0.65)
|
||||
plt.ylim(0, max(y_cumavg)*1.02)
|
||||
|
||||
xp = np.linspace(0, 0.65, 300)
|
||||
|
||||
#plt.gca().set_aspect('equal', adjustable='box')
|
||||
plottitle = title if title is not None else '%s_vs_%s.pdf' % (xlabel, ylabel)
|
||||
plt.savefig(plottitle , bbox_inches="tight")
|
||||
|
||||
def print_avg(x, name = 'unnamed'):
|
||||
print 'Average of %s is %f' % (name, reduce(lambda a,b: a+b, x) / float(len(x)));
|
||||
|
||||
with open('../state/lastExec', 'rb') as lastExec:
|
||||
lastExecInd = lastExec.readline().strip()
|
||||
print lastExecInd
|
||||
|
||||
rows = [];
|
||||
execInd = lastExecInd;
|
||||
with open('../state/execs/%s.exec/plotInfo.json' % execInd, 'rb') as jsonfile:
|
||||
json_lines = jsonfile.readlines()
|
||||
|
||||
rows = [json.loads(l) for l in json_lines]
|
||||
# accepts = [[r['queryCount'], 1 if r['stats.rank']>=0 else 0] for r in rows if r['stats.type'] == 'accept'];
|
||||
|
||||
def percent_induced_in_accepted(status = 'Core'):
|
||||
filtered_rows = [r for r in rows if r['stats.type'] == 'accept']
|
||||
query_counts = [r['queryCount'] for r in filtered_rows]
|
||||
is_induced = [1 if r['stats.status'] == status else 0 for r in filtered_rows]
|
||||
print_avg(is_induced, 'percent_induced_accepted')
|
||||
plotsetting['label'] = status.lower();
|
||||
plot_cumavg(query_counts, is_induced, xlabel='query #', ylabel='induced-accepted');
|
||||
# plt.figure()
|
||||
# plotsetting['color'] = 'r'; percent_induced_in_accepted(status = 'Nothing');
|
||||
# plotsetting['color'] = 'b'; percent_induced_in_accepted(status = 'Induced');
|
||||
# plotsetting['color'] = 'k'; percent_induced_in_accepted(status = 'Core');
|
||||
# plt.legend(frameon=False)
|
||||
# plt.savefig('accepted_stats.pdf' , bbox_inches="tight")
|
||||
|
||||
|
||||
def precent_status(status = 'Core'):
|
||||
filtered_rows = [r for r in rows if r['stats.type'] == 'q']
|
||||
query_counts = [r['queryCount'] for r in filtered_rows]
|
||||
is_status = [1 if r['stats.status'] == status else 0 for r in filtered_rows]
|
||||
print_avg(is_status, 'percent of status ' + status)
|
||||
plotsetting['label'] = status.lower();
|
||||
plot_cumavg(query_counts, is_status, xlabel='query #', ylabel='%');
|
||||
plt.figure()
|
||||
plotsetting['color'] = 'r'; precent_status(status = 'Nothing');
|
||||
plotsetting['color'] = 'b'; precent_status(status = 'Induced');
|
||||
plotsetting['color'] = 'k'; precent_status(status = 'Core');
|
||||
plt.legend(frameon=False)
|
||||
plt.savefig('parse_status.pdf' , bbox_inches="tight")
|
||||
|
||||
def simple_acc():
|
||||
filtered_rows = [r for r in rows if r['stats.type'] == 'accept' or r['stats.type'] == 'q']
|
||||
query_counts = [r['queryCount'] for r in filtered_rows]
|
||||
accepts = [1 if r['stats.type'] != 'q' and r['stats.rank']==0 else 0 for r in filtered_rows]
|
||||
plot_cumavg(query_counts, accepts, xlabel='query#', ylabel='accuracy');
|
||||
print_avg(accepts, 'accuracy')
|
||||
plt.figure()
|
||||
simple_acc()
|
||||
|
||||
def simple_recall():
|
||||
filtered_rows = [r for r in rows if r['stats.type'] == 'accept']
|
||||
query_counts = [r['queryCount'] for r in filtered_rows]
|
||||
recall = [1 if r['stats.rank']>=0 else 0 for r in filtered_rows]
|
||||
plot_cumavg(query_counts, recall, xlabel='query#', ylabel='recall');
|
||||
print_avg(recall, 'recall')
|
||||
plotsetting['color'] = 'r'
|
||||
simple_recall()
|
||||
|
||||
def average_stat(stat = 'stats.size', type = 'q'):
|
||||
filtered_rows = [r for r in rows if r['stats.type'] == type]
|
||||
query_counts = [r['queryCount'] for r in filtered_rows]
|
||||
stats = [r[stat] for r in filtered_rows]
|
||||
plot_cumavg(query_counts, stats, xlabel='query#', ylabel=stat);
|
||||
print_avg(stats, stat)
|
||||
plt.figure()
|
||||
plotsetting['color'] = 'b'
|
||||
average_stat(stat = 'stats.size')
|
||||
|
||||
|
||||
def average_stat(stat = 'stats.rank'):
|
||||
filtered_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.rank'] >= 0]
|
||||
query_counts = [r['queryCount'] for r in filtered_rows]
|
||||
stats = [r[stat] for r in filtered_rows]
|
||||
plot_cumavg(query_counts, stats, xlabel='query#', ylabel=stat);
|
||||
print_avg(stats, stat)
|
||||
plt.figure()
|
||||
average_stat(stat = 'stats.rank')
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -85,9 +85,5 @@ public class SimulationAnalyzer {
|
|||
|
||||
acceptEval.logStats("accept");
|
||||
acceptEval.putOutput("accept");
|
||||
|
||||
// LogInfo.logs("Printing plotInfo to %s",
|
||||
// Execution.getFile("plotInfo.json"));
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue