data process, faster parsing, reproduce results

This commit is contained in:
Sida Wang 2017-04-07 22:41:34 -07:00
parent ff76ec27b6
commit 41abd8b8e9
24 changed files with 128 additions and 97 deletions

2
.gitignore vendored
View File

@ -40,7 +40,7 @@ rdf
# interactive outputs
int-output*
int-backup
_OUTPATH_
interactive/.ipynb_checkpoints
# Community server logs
community-server/data
community-server/data-backup

View File

@ -9,6 +9,7 @@ In this setting, the system begin with the dependency-based action language (`da
## Running the Voxelurn server
0. Setup SEMPRE dependencies and compile
./pull-dependencies core
@ -22,19 +23,19 @@ In this setting, the system begin with the dependency-based action language (`da
2. Feed the server existing definitions, which should take less than 2 minutes.
./interactive/run @mode=simulator @server=local @sandbox=none @task=freebigdef -maxQueries 2496
./interactive/run @mode=simulator @server=local @sandbox=none @task=freebuilddef -maxQueries 2496
try `add green monster` now.
try `add dancer` now.
### Interacting with the server
There are 3 ways to interactive with the server
There are 3 ways to interact and try your own commands
* Hit `Ctrl-D` on the server terminal, and type `add red top`, or `add green monster`
* Hit `Ctrl-D` on the terminal running the server, and type `add red top`, or `add green monster`
* On a browser, type `http://localhost:8410/sempre?q=(:q add green monster)`
* The visual way is to use our client at `https://github.com/sidaw/shrdlurn`, which has a more detailed [README.md](https://github.com/sidaw/shrdlurn/blob/master/README.md) for the client. Try `[add dancer; front 5] 3 times` after you set that up.
* The visual way is to use our client at `https://github.com/sidaw/shrdlurn`, which has a more detailed [README.md](https://github.com/sidaw/shrdlurn/blob/master/README.md). Try `[add dancer; front 5] 3 times` after you run the client. A live version is at [voxelurn.com](http://www.voxelurn.com).
## Experiments in ACL2017
@ -44,7 +45,7 @@ There are 3 ways to interactive with the server
2. Feed the server all the query logs
./interactive/run @mode=simulator @server=local @sandbox=none @task=freebig -maxQueries 103876
./interactive/run @mode=simulator @server=local @sandbox=none @task=freebuild -maxQueries 103876
This currently takes just under 30 minutes. Decrease maxQuery for a quicker experiment. This generate `plotInfo.json` in `../state/execs/${lastExec}.exec/` where `lastExec` is `cat ../state/lastExec`.

View File

@ -52,7 +52,25 @@
" json_lines = jsonfile.readlines()\n",
"\n",
"rawrows = [json.loads(l) for l in json_lines]\n",
"rows = [r for r in rawrows if r.has_key('stats.type')]"
"rows = [r for r in rawrows if r.has_key('stats.type')]\n",
"\n",
"print '%d queries in plotInfo' % len(json_lines)\n",
"\n",
"def print_defstats():\n",
" filtered_rows = [r for r in rows if r['stats.type'] == 'def' and r['stats.num_rules'] >= 1]\n",
" total_failed = np.sum([r['stats.num_failed'] for r in filtered_rows])\n",
" total_body = np.sum([r['stats.num_body'] for r in filtered_rows])\n",
"\n",
" stats = {\\\n",
" 'total_def_queries': len(filtered_rows), \\\n",
" 'total_rules': np.sum([r['stats.num_rules'] for r in filtered_rows]), \\\n",
" 'total_failed': total_failed, \\\n",
" 'total_body': total_body\n",
" }\n",
" print ''\n",
" print stats\n",
" print 'failpercent: %.4f' % (float(total_failed)/float(total_body))\n",
"print_defstats()"
]
},
{
@ -64,7 +82,7 @@
},
"outputs": [],
"source": [
"print '%d queries in plotInfo' % len(json_lines)\n",
"\n",
"\n",
"def print_stats():\n",
" filtered_rows = [r for r in rows if r['stats.type'] == 'accept']\n",
@ -113,21 +131,7 @@
"\n",
"# print filtered_rows[0]\n",
"\n",
"def print_defstats():\n",
" filtered_rows = [r for r in rows if r['stats.type'] == 'def' and r['stats.num_rules'] >= 1]\n",
" total_failed = np.sum([r['stats.num_failed'] for r in filtered_rows])\n",
" total_body = np.sum([r['stats.num_body'] for r in filtered_rows])\n",
"\n",
" stats = {\\\n",
" 'total_def_queries': len(filtered_rows), \\\n",
" 'total_rules': np.sum([r['stats.num_rules'] for r in filtered_rows]), \\\n",
" 'total_failed': total_failed, \\\n",
" 'total_body': total_body\n",
" }\n",
" print ''\n",
" print stats\n",
" print 'failpercent: %.4f' % (float(total_failed)/float(total_body))\n",
"print_defstats()\n",
"\n",
"def percent_error():\n",
" allerror = [r for r in rawrows if r.has_key('stats.error')];\n",
@ -382,7 +386,7 @@
" for g in (ranked_users[0:num_user]):\n",
" print g\n",
" print '********************'\n",
" rows_userg = [r for r in rows if r['stats.type']=='q' and r['sessionId']==g[0]]\n",
" rows_userg = [r for r in rows if r['stats.type']=='q' and r['sessionId']==g[0] and r.has_key('stats.status') ]\n",
" print_count = 0\n",
" prev_nothing = False\n",
" for r in rows_userg[-num_query:]:\n",

View File

@ -1,11 +1,23 @@
# Query logs
# README
## Processed
* freebuild.def.json.gz contains just the 2495 definition queries in freebuild.json.gz
gzcat freebuild.json.gz | grep '(:def'
* freebuild.json.gz is the main data file. Obtained by:
awk '{print "{\"qid\":" NR "," substr($0,2)}' freebuildbig-0206 > freebuild.id.json
jq -c '{"qid":.qid, "q":.q, "sessionId": .sessionId[:10], "time":.time}' freebuild.id.json > freebuild.json
## Raw queries
* freebuildbig-0206.def is the raw query log, without context
* qualifier3-0201: 30 turkers, 1 rejection
* qualifiers: turkers had to build a fixed target
* freebuild[12]: qualified turkers can build whatever they want
* freebuildbig-0206 is the main data file used to report results
* freebuildbig-0206.def just the 2495 definition attempts

Binary file not shown.

Binary file not shown.

View File

@ -123,17 +123,14 @@ addMode('simulator', 'run the simulator', lambda { |e| l(
}),
letDefault(:task, 'sidaw'),
sel(:task, {
'freebig' => o('logFiles', "./#{$path}/queries/freebuildbig-0206.json.gz"),
'freebigdef' => o('logFiles', "./#{$path}/queries/freebuildbig-0206.def.json.gz"),
'freebigcontext' => o('logFiles', "./#{$path}/queries/freebuildbig-0206.context.json.gz"),
'qual1' => o('logFiles', "./#{$path}/queries/qualifier1-0118.json.gz"),
'qual2' => o('logFiles', "./#{$path}/queries/qualifier2-0129.json.gz"),
'qual3' => o('logFiles', "./#{$path}/queries/qualifier3-0201.json.gz"), # both 2 and 3
'free1' => o('logFiles', "./#{$path}/queries/freebuild1-0121.json.gz"),
'free2' => o('logFiles', "./#{$path}/queries/freebuild2-0127.json.gz"),
'sidaw' => o('logFiles', "./#{$path}/queries/sidaw.json"),
'freebuild' => o('logFiles', "./#{$path}/queries/freebuild.json.gz"),
'freebuilddef' => o('logFiles', "./#{$path}/queries/freebuild.def.json.gz"),
'qual1' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier1-0118.json.gz"),
'qual2' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier2-0129.json.gz"),
'qual3' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier3-0201.json.gz"), # both 2 and 3
'free1' => o('logFiles', "./#{$path}/queries/rawqueries/freebuild1-0121.json.gz"),
'free2' => o('logFiles', "./#{$path}/queries/rawqueries/freebuild2-0127.json.gz"),
}),
nil)})
@ -161,16 +158,16 @@ addMode('voxelurn', 'interactive semantic parsing in a VoxelWorld', lambda { |e|
o('LanguageAnalyzer.lowerCaseTokens', true),
o('Parser.pruneErrorValues', true),
o('Parser', 'interactive.BeamFloatingParser'),
o('Parser', 'interactive.InteractiveBeamParser'),
o('Parser.callSetEvaluation', false),
o('Parser.coarsePrune', 'true'),
o('Parser.coarsePrune', true),
o('Parser.beamSize', 50),
o('BeamFloatingParser.maxNewTreesPerSpan', 5001),
o('InteractiveBeamParser.maxNewTreesPerSpan', 5001),
o('ParserState.customExpectedCounts', 'None'),
selo(0, 'BeamFloatingParser.floatStrategy', 'Never', 'NoParse', 'Always'),
o('BeamFloatingParser.trackedCats', 'Number', 'Numbers', 'Color', 'Direction', 'Set', 'Sets', 'Action', 'Actions'),
selo(0, 'InteractiveBeamParser.floatStrategy', 'Never', 'NoParse', 'Always'),
o('InteractiveBeamParser.trackedCats', 'Number', 'Numbers', 'Color', 'Direction', 'Set', 'Sets', 'Action', 'Actions'),
o('Derivation.derivComparator', 'AnchorPriorityScoreComparator'),
o('Params.l1Reg', 'nonlazy'),
@ -207,6 +204,7 @@ addMode('voxelurn', 'interactive semantic parsing in a VoxelWorld', lambda { |e|
o('Derivation.showTypes', false),
o('Derivation.showValues', false),
o('Derivation.showRules', false),
o('Derivation.anchoredBonus', 1.0),
o('NumberFn.allowedRange', 0, 100),
o('SimpleLexicon.inPaths', "./#{$path}/csscolors.lexicon"),
@ -214,10 +212,6 @@ addMode('voxelurn', 'interactive semantic parsing in a VoxelWorld', lambda { |e|
lambda { |e| system 'mkdir -p ./int-output/'; nil},
lambda { |e| system 'mkdir -p ./int-output/log/'; nil},
lambda { |e| system 'mkdir -p ./int-output/citation/'; nil},
lambda { |e| system 'mkdir -p server.exec';
l(o('execDir', 'server.exec/'),
o('overwriteExecDir'))
},
nil) })
############################################################

View File

@ -13,7 +13,7 @@
# typed lambdas
# Action a, ItemSet i, ValueSet v, Rel r, Direction d, Color c, Number n
# This grammar should have Rule.defaultIsFloating false
# Rule.defaultIsFloating should be false
# utilities
(rule $Action (base $Number $Number) (lambda x (lambda y (: @base (var x) (var y)))) (anchored 1))

View File

@ -30,6 +30,8 @@ public class Derivation implements SemanticFn.Callable, HasScore {
public boolean showExecutions = false;
@Option(gloss = "Pick the comparator used to sort derivations")
public String derivComparator = "ScoredDerivationComparator";
@Option(gloss = "bonus score for being all anchored")
public double anchoredBonus = 0.0;
}
public static Options opts = new Options();
@ -264,7 +266,7 @@ public class Derivation implements SemanticFn.Callable, HasScore {
public void addFeatures(FeatureVector fv) { this.localFeatureVector.add(fv); }
public double localScore(Params params) {
return localFeatureVector.dotProduct(params);
return localFeatureVector.dotProduct(params) + (this.allAnchored()? opts.anchoredBonus : 0.0);
}
// SHOULD NOT BE USED except during test time if the memory is desperately needed.

View File

@ -5,6 +5,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@ -38,16 +39,11 @@ import fig.basic.StopWatchSet;
import fig.exec.Execution;
/**
* A simple bottom-up chart-based parser that keeps the |BeamFloatingSize| top
* derivations for each chart cell (cat, start, end). Also supports fast
* indexing of lexicalized rules using a trie.
*
* Note that this code does not rely on the Grammar being binarized, which makes
* it more complex.
* A modified version of the BeamParser, with consideration for use in the interactive setting
*
* @author Percy Liang, sidaw
*/
public class BeamFloatingParser extends Parser {
public class InteractiveBeamParser extends Parser {
public static class Options {
@Option
public int maxNewTreesPerSpan = Integer.MAX_VALUE;
@ -64,13 +60,17 @@ public class BeamFloatingParser extends Parser {
public static Options opts = new Options();
Trie trie; // For non-cat-unary rules
public BeamFloatingParser(Spec spec) {
// so that duplicated rules are never added
Set<Rule> allRules;
List<Rule> interactiveCatUnaryRules;
public InteractiveBeamParser(Spec spec) {
super(spec);
if (opts.trackedCats != null) {
opts.trackedCats = opts.trackedCats.stream().map(s -> "$" + s).collect(Collectors.toList());
LogInfo.logs("Mapped trackedCats to: %s", opts.trackedCats);
}
interactiveCatUnaryRules = new LinkedList<>(super.catUnaryRules);
allRules = new LinkedHashSet<>(super.catUnaryRules);
// Index the non-cat-unary rules
trie = new Trie();
for (Rule rule : grammar.getRules()) {
@ -80,12 +80,31 @@ public class BeamFloatingParser extends Parser {
this.chartFillOut = IOUtils.openOutAppendEasy(Execution.getFile("chartfill"));
}
@Override
public synchronized void addRule(Rule rule) {
if (allRules.contains(rule))
return;
allRules.add(rule);
if (!rule.isCatUnary()) {
trie.add(rule);
} else {
interactiveCatUnaryRules.add(rule);
}
}
@Override
public List<Rule> getCatUnaryRules() {
return interactiveCatUnaryRules;
}
// for grammar induction, just need the formula, do not execute
public BeamFloatingParserState justParse(Params params, Example ex, boolean computeExpectedCounts) {
public InteractiveBeamParserState parseWithoutExecuting(Params params, Example ex, boolean computeExpectedCounts) {
// Parse
StopWatch watch = new StopWatch();
watch.start();
BeamFloatingParserState state = new BeamFloatingParserState(this, params, ex);
InteractiveBeamParserState state = new InteractiveBeamParserState(this, params, ex);
state.infer();
watch.stop();
state.parseTime = watch.getCurrTimeLong();
@ -96,25 +115,19 @@ public class BeamFloatingParser extends Parser {
return state;
}
@Override
public synchronized void addRule(Rule rule) {
if (!rule.isCatUnary()) {
trie.add(rule);
}
}
@Override
public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) {
BeamFloatingParserState coarseState = null;
InteractiveBeamParserState coarseState = null;
if (Parser.opts.coarsePrune) {
LogInfo.begin_track("Parser.coarsePrune");
coarseState = new BeamFloatingParserState(this, params, ex, computeExpectedCounts,
BeamFloatingParserState.Mode.bool, null);
coarseState = new InteractiveBeamParserState(this, params, ex, computeExpectedCounts,
InteractiveBeamParserState.Mode.bool, null);
coarseState.infer();
coarseState.keepTopDownReachable();
LogInfo.end_track();
}
return new BeamFloatingParserState(this, params, ex, computeExpectedCounts, BeamFloatingParserState.Mode.full,
return new InteractiveBeamParserState(this, params, ex, computeExpectedCounts, InteractiveBeamParserState.Mode.full,
coarseState);
}
}
@ -125,8 +138,9 @@ public class BeamFloatingParser extends Parser {
*
* @author Percy Liang
* @author Roy Frostig
* @author sidaw
*/
class BeamFloatingParserState extends ChartParserState {
class InteractiveBeamParserState extends ChartParserState {
public final Mode mode;
// Modes:
@ -137,13 +151,13 @@ class BeamFloatingParserState extends ChartParserState {
bool, full
}
private final BeamFloatingParser parser;
private final BeamFloatingParserState coarseState; // Used to prune
private final InteractiveBeamParser parser;
private final InteractiveBeamParserState coarseState; // Used to prune
private final boolean execute;
public List<Derivation> chartList;
public BeamFloatingParserState(BeamFloatingParser parser, Params params, Example ex) {
public InteractiveBeamParserState(InteractiveBeamParser parser, Params params, Example ex) {
super(parser, params, ex, false);
this.parser = parser;
this.mode = Mode.full;
@ -151,8 +165,8 @@ class BeamFloatingParserState extends ChartParserState {
this.execute = false;
}
public BeamFloatingParserState(BeamFloatingParser parser, Params params, Example ex, boolean computeExpectedCounts,
Mode mode, BeamFloatingParserState coarseState) {
public InteractiveBeamParserState(InteractiveBeamParser parser, Params params, Example ex, boolean computeExpectedCounts,
Mode mode, InteractiveBeamParserState coarseState) {
super(parser, params, ex, computeExpectedCounts);
this.parser = parser;
this.mode = mode;
@ -198,9 +212,9 @@ class BeamFloatingParserState extends ChartParserState {
this.chartList = this.collectChart();
boolean parseFloat = false;
if (BeamFloatingParser.opts.floatStrategy == BeamFloatingParser.FloatStrategy.Always)
if (InteractiveBeamParser.opts.floatStrategy == InteractiveBeamParser.FloatStrategy.Always)
parseFloat = true;
else if (BeamFloatingParser.opts.floatStrategy == BeamFloatingParser.FloatStrategy.NoParse)
else if (InteractiveBeamParser.opts.floatStrategy == InteractiveBeamParser.FloatStrategy.NoParse)
parseFloat = predDerivations.size() == 0;
else
parseFloat = false;
@ -397,7 +411,7 @@ class BeamFloatingParserState extends ChartParserState {
children.remove(children.size() - 1);
if (mode != Mode.full)
break; // Only need one hypothesis
if (numNew.value >= BeamFloatingParser.opts.maxNewTreesPerSpan)
if (numNew.value >= InteractiveBeamParser.opts.maxNewTreesPerSpan)
return;
}
}

View File

@ -153,7 +153,7 @@ public class InteractiveMaster extends Master {
response.ex = ex;
// Parse!
((BeamFloatingParser)builder.parser).justParse(builder.params, ex, false);
((InteractiveBeamParser)builder.parser).parseWithoutExecuting(builder.params, ex, false);
int rank = -1;
Derivation match = null;
@ -212,6 +212,8 @@ public class InteractiveMaster extends Master {
for (Rule rule : inducedRules) {
InteractiveUtils.addRuleInteractive(rule, builder.parser);
}
stats.put("total_rules", ((InteractiveBeamParser)builder.parser).allRules.size());
stats.put("total_unicat", ((InteractiveBeamParser)builder.parser).interactiveCatUnaryRules.size());
}
// TODO : should not have to parse again, I guess just set the formula
// or something
@ -275,7 +277,7 @@ public class InteractiveMaster extends Master {
if (isNonsense(exHead))
throw BadInteractionException.nonSenseDefinition(head);
BeamFloatingParserState state = ((BeamFloatingParser)parser).justParse(params, exHead, false);
InteractiveBeamParserState state = ((InteractiveBeamParser)parser).parseWithoutExecuting(params, exHead, false);
if (GrammarInducer.getParseStatus(exHead) == GrammarInducer.ParseStatus.Core)
throw BadInteractionException.headIsCore(head);

View File

@ -227,7 +227,8 @@ public class InteractiveServer {
future.cancel(true);
executor.shutdown();
long endTime = System.nanoTime();
response.stats.put("time", (endTime - startTime) / 1.0e9);
response.stats.put("walltime", (endTime - startTime) / 1.0e9);
response.stats.put("count", queryCounter.get());
}
return response;
}
@ -243,10 +244,10 @@ public class InteractiveServer {
LocalDateTime queryTime = LocalDateTime.now();
synchronized (queryLogLock) { // write the query log
Map<String, Object> jsonMap = new LinkedHashMap<>();
jsonMap.put("count", queryNumber);
jsonMap.put("q", query);
jsonMap.put("remote", remoteHost);
jsonMap.put("time", queryTime.toString());
jsonMap.put("query", queryNumber);
// jsonMap.put("remote", remoteHost);
// jsonMap.put("time", queryTime.toString());
jsonMap.put("sessionId", sessionId);
reqParams.remove("q");
jsonMap.putAll(reqParams);

View File

@ -92,7 +92,7 @@ public final class InteractiveUtils {
ex.preprocess();
LogInfo.logs("Parsing body: %s", ex.utterance);
((BeamFloatingParser)parser).justParse(params, ex, false);
((InteractiveBeamParser)parser).parseWithoutExecuting(params, ex, false);
boolean found = false;
Formula targetFormula = Formulas.fromLispTree(LispTree.proto.parseFromString(formula));
@ -161,10 +161,10 @@ public final class InteractiveUtils {
public static synchronized void addRuleInteractive(Rule rule, Parser parser) {
LogInfo.logs("addRuleInteractive: %s", rule);
parser.grammar.addRule(rule);
if (parser instanceof BeamParser || parser instanceof BeamFloatingParser) {
if (parser instanceof InteractiveBeamParser) {
parser.addRule(rule);
} else {
throw new RuntimeException("interactively adding rule not supported for paser " + parser.getClass().toString());
}
}

View File

@ -31,6 +31,7 @@ public class SimulationAnalyzer {
Map<String, Object> line = new LinkedHashMap<String, Object>(query);
LogInfo.logs("stats: %s", stats);
if (stats == null) {
LogInfo.logs("No stats");
LogInfo.log(query);
LogInfo.log(jsonResponse);
return;
@ -54,7 +55,7 @@ public class SimulationAnalyzer {
qEval.add("def.num_failed", (Integer) stats.get("num_failed"));
qEval.add("def.num_body", (Integer) stats.get("num_body"));
qEval.add("def.num_rules", (Integer) stats.get("num_rules"));
qEval.add("def.time", (Double) stats.get("time"));
qEval.add("def.time", (Integer) stats.get("count"));
}
if (stats.get("type").equals("q") && !stats.containsKey("error")) {

View File

@ -8,7 +8,7 @@ import java.util.function.Predicate;
import fig.basic.*;
import edu.stanford.nlp.sempre.*;
import edu.stanford.nlp.sempre.Parser.Spec;
import edu.stanford.nlp.sempre.interactive.BeamFloatingParser;
import edu.stanford.nlp.sempre.interactive.InteractiveBeamParser;
import edu.stanford.nlp.sempre.interactive.DALExecutor;
import org.testng.Assert;
@ -80,7 +80,7 @@ public class FloatingParsingTest {
ex.preprocess();
Spec defSpec = defaultSpec();
Parser parser = new BeamFloatingParser(defSpec);
Parser parser = new InteractiveBeamParser(defSpec);
ParserState state = parser.parse(new Params(), ex, false);
LogInfo.end_track();

View File

@ -27,7 +27,7 @@ import edu.stanford.nlp.sempre.Rule;
import edu.stanford.nlp.sempre.Session;
import edu.stanford.nlp.sempre.SimpleLexicon;
import edu.stanford.nlp.sempre.ValueEvaluator;
import edu.stanford.nlp.sempre.interactive.BeamFloatingParser;
import edu.stanford.nlp.sempre.interactive.InteractiveBeamParser;
import edu.stanford.nlp.sempre.interactive.DALExecutor;
import edu.stanford.nlp.sempre.interactive.DefinitionAligner;
import edu.stanford.nlp.sempre.interactive.GrammarInducer;
@ -94,7 +94,7 @@ public class GrammarInducerTest {
List<Rule> allRules;
public ParseTester() {
parser = new BeamFloatingParser(defaultSpec());
parser = new InteractiveBeamParser(defaultSpec());
params = new Params();
allRules = new ArrayList<>();
}