diff --git a/.gitignore b/.gitignore index 436b2f5..dc1e65a 100644 --- a/.gitignore +++ b/.gitignore @@ -40,7 +40,7 @@ rdf # interactive outputs int-output* int-backup -_OUTPATH_ +interactive/.ipynb_checkpoints # Community server logs community-server/data community-server/data-backup diff --git a/interactive/README.md b/interactive/README.md index 376c8bd..28c6e0d 100644 --- a/interactive/README.md +++ b/interactive/README.md @@ -9,6 +9,7 @@ In this setting, the system begin with the dependency-based action language (`da ## Running the Voxelurn server + 0. Setup SEMPRE dependencies and compile ./pull-dependencies core @@ -22,35 +23,35 @@ In this setting, the system begin with the dependency-based action language (`da 2. Feed the server existing definitions, which should take less than 2 minutes. - ./interactive/run @mode=simulator @server=local @sandbox=none @task=freebigdef -maxQueries 2496 + ./interactive/run @mode=simulator @server=local @sandbox=none @task=freebuilddef -maxQueries 2496 - try `add green monster` now. + try `add dancer` now. ### Interacting with the server -There are 3 ways to interactive with the server +There are 3 ways to interact and try your own commands -* Hit `Ctrl-D` on the server terminal, and type `add red top`, or `add green monster` +* Hit `Ctrl-D` on the terminal running the server, and type `add red top`, or `add green monster` * On a browser, type `http://localhost:8410/sempre?q=(:q add green monster)` -* The visual way is to use our client at `https://github.com/sidaw/shrdlurn`, which has a more detailed [README.md](https://github.com/sidaw/shrdlurn/blob/master/README.md) for the client. Try `[add dancer; front 5] 3 times` after you set that up. +* The visual way is to use our client at `https://github.com/sidaw/shrdlurn`, which has a more detailed [README.md](https://github.com/sidaw/shrdlurn/blob/master/README.md). Try `[add dancer; front 5] 3 times` after you run the client. A live version is at [voxelurn.com](http://www.voxelurn.com). ## Experiments in ACL2017 1. Start the server - ./interactive/run @mode=voxelurn -server -interactive + ./interactive/run @mode=voxelurn -server -interactive 2. Feed the server all the query logs - ./interactive/run @mode=simulator @server=local @sandbox=none @task=freebig -maxQueries 103876 + ./interactive/run @mode=simulator @server=local @sandbox=none @task=freebuild -maxQueries 103876 This currently takes just under 30 minutes. Decrease maxQuery for a quicker experiment. This generate `plotInfo.json` in `../state/execs/${lastExec}.exec/` where `lastExec` is `cat ../state/lastExec`. 3. Taking `../state/execs/${lastExec}.exec/plotInfo.json` as input, we can analyze the data and produce some plots using the following ipython notebook - ipython notebook analyze_data.ipynb + ipython notebook analyze_data.ipynb which prints out basic statistics and generates the plots used in our paper. The plots are saved at `../state/execs/${lastExec}.exec/` diff --git a/interactive/analyze_data.ipynb b/interactive/analyze_data.ipynb index 8ca85a2..dd77e22 100644 --- a/interactive/analyze_data.ipynb +++ b/interactive/analyze_data.ipynb @@ -52,7 +52,25 @@ " json_lines = jsonfile.readlines()\n", "\n", "rawrows = [json.loads(l) for l in json_lines]\n", - "rows = [r for r in rawrows if r.has_key('stats.type')]" + "rows = [r for r in rawrows if r.has_key('stats.type')]\n", + "\n", + "print '%d queries in plotInfo' % len(json_lines)\n", + "\n", + "def print_defstats():\n", + " filtered_rows = [r for r in rows if r['stats.type'] == 'def' and r['stats.num_rules'] >= 1]\n", + " total_failed = np.sum([r['stats.num_failed'] for r in filtered_rows])\n", + " total_body = np.sum([r['stats.num_body'] for r in filtered_rows])\n", + "\n", + " stats = {\\\n", + " 'total_def_queries': len(filtered_rows), \\\n", + " 'total_rules': np.sum([r['stats.num_rules'] for r in filtered_rows]), \\\n", + " 'total_failed': total_failed, \\\n", + " 'total_body': total_body\n", + " }\n", + " print ''\n", + " print stats\n", + " print 'failpercent: %.4f' % (float(total_failed)/float(total_body))\n", + "print_defstats()" ] }, { @@ -64,7 +82,7 @@ }, "outputs": [], "source": [ - "print '%d queries in plotInfo' % len(json_lines)\n", + "\n", "\n", "def print_stats():\n", " filtered_rows = [r for r in rows if r['stats.type'] == 'accept']\n", @@ -113,21 +131,7 @@ "\n", "# print filtered_rows[0]\n", "\n", - "def print_defstats():\n", - " filtered_rows = [r for r in rows if r['stats.type'] == 'def' and r['stats.num_rules'] >= 1]\n", - " total_failed = np.sum([r['stats.num_failed'] for r in filtered_rows])\n", - " total_body = np.sum([r['stats.num_body'] for r in filtered_rows])\n", "\n", - " stats = {\\\n", - " 'total_def_queries': len(filtered_rows), \\\n", - " 'total_rules': np.sum([r['stats.num_rules'] for r in filtered_rows]), \\\n", - " 'total_failed': total_failed, \\\n", - " 'total_body': total_body\n", - " }\n", - " print ''\n", - " print stats\n", - " print 'failpercent: %.4f' % (float(total_failed)/float(total_body))\n", - "print_defstats()\n", "\n", "def percent_error():\n", " allerror = [r for r in rawrows if r.has_key('stats.error')];\n", @@ -382,7 +386,7 @@ " for g in (ranked_users[0:num_user]):\n", " print g\n", " print '********************'\n", - " rows_userg = [r for r in rows if r['stats.type']=='q' and r['sessionId']==g[0]]\n", + " rows_userg = [r for r in rows if r['stats.type']=='q' and r['sessionId']==g[0] and r.has_key('stats.status') ]\n", " print_count = 0\n", " prev_nothing = False\n", " for r in rows_userg[-num_query:]:\n", diff --git a/interactive/queries/README.md b/interactive/queries/README.md index 1a33b40..11740bf 100644 --- a/interactive/queries/README.md +++ b/interactive/queries/README.md @@ -1,11 +1,23 @@ -# Query logs +# README + +## Processed + +* freebuild.def.json.gz contains just the 2495 definition queries in freebuild.json.gz + + gzcat freebuild.json.gz | grep '(:def' + +* freebuild.json.gz is the main data file. Obtained by: + + awk '{print "{\"qid\":" NR "," substr($0,2)}' freebuildbig-0206 > freebuild.id.json + jq -c '{"qid":.qid, "q":.q, "sessionId": .sessionId[:10], "time":.time}' freebuild.id.json > freebuild.json + + +## Raw queries + +* freebuildbig-0206.def is the raw query log, without context * qualifier3-0201: 30 turkers, 1 rejection * qualifiers: turkers had to build a fixed target * freebuild[12]: qualified turkers can build whatever they want - -* freebuildbig-0206 is the main data file used to report results - -* freebuildbig-0206.def just the 2495 definition attempts diff --git a/interactive/queries/freebuild.def.json.gz b/interactive/queries/freebuild.def.json.gz new file mode 100644 index 0000000..4ca8c0d Binary files /dev/null and b/interactive/queries/freebuild.def.json.gz differ diff --git a/interactive/queries/freebuild.json.gz b/interactive/queries/freebuild.json.gz new file mode 100644 index 0000000..5e8021b Binary files /dev/null and b/interactive/queries/freebuild.json.gz differ diff --git a/interactive/queries/freebuildbig-0206.def.json.gz b/interactive/queries/freebuildbig-0206.def.json.gz deleted file mode 100644 index 2317b8d..0000000 Binary files a/interactive/queries/freebuildbig-0206.def.json.gz and /dev/null differ diff --git a/interactive/queries/freebuild1-0121.json.gz b/interactive/queries/rawqueries/freebuild1-0121.json.gz similarity index 100% rename from interactive/queries/freebuild1-0121.json.gz rename to interactive/queries/rawqueries/freebuild1-0121.json.gz diff --git a/interactive/queries/freebuild2-0126.json.gz b/interactive/queries/rawqueries/freebuild2-0126.json.gz similarity index 100% rename from interactive/queries/freebuild2-0126.json.gz rename to interactive/queries/rawqueries/freebuild2-0126.json.gz diff --git a/interactive/queries/freebuild2-0127.json.gz b/interactive/queries/rawqueries/freebuild2-0127.json.gz similarity index 100% rename from interactive/queries/freebuild2-0127.json.gz rename to interactive/queries/rawqueries/freebuild2-0127.json.gz diff --git a/interactive/queries/freebuildbig-0206.json.gz b/interactive/queries/rawqueries/freebuildbig-0206.json.gz similarity index 100% rename from interactive/queries/freebuildbig-0206.json.gz rename to interactive/queries/rawqueries/freebuildbig-0206.json.gz diff --git a/interactive/queries/qualifier1-0118.json.gz b/interactive/queries/rawqueries/qualifier1-0118.json.gz similarity index 100% rename from interactive/queries/qualifier1-0118.json.gz rename to interactive/queries/rawqueries/qualifier1-0118.json.gz diff --git a/interactive/queries/qualifier2-0129.json.gz b/interactive/queries/rawqueries/qualifier2-0129.json.gz similarity index 100% rename from interactive/queries/qualifier2-0129.json.gz rename to interactive/queries/rawqueries/qualifier2-0129.json.gz diff --git a/interactive/queries/qualifier3-0201.json.gz b/interactive/queries/rawqueries/qualifier3-0201.json.gz similarity index 100% rename from interactive/queries/qualifier3-0201.json.gz rename to interactive/queries/rawqueries/qualifier3-0201.json.gz diff --git a/interactive/run b/interactive/run index f499693..d4ad8bf 100755 --- a/interactive/run +++ b/interactive/run @@ -123,17 +123,14 @@ addMode('simulator', 'run the simulator', lambda { |e| l( }), letDefault(:task, 'sidaw'), sel(:task, { - 'freebig' => o('logFiles', "./#{$path}/queries/freebuildbig-0206.json.gz"), - 'freebigdef' => o('logFiles', "./#{$path}/queries/freebuildbig-0206.def.json.gz"), - 'freebigcontext' => o('logFiles', "./#{$path}/queries/freebuildbig-0206.context.json.gz"), - - 'qual1' => o('logFiles', "./#{$path}/queries/qualifier1-0118.json.gz"), - 'qual2' => o('logFiles', "./#{$path}/queries/qualifier2-0129.json.gz"), - 'qual3' => o('logFiles', "./#{$path}/queries/qualifier3-0201.json.gz"), # both 2 and 3 - 'free1' => o('logFiles', "./#{$path}/queries/freebuild1-0121.json.gz"), - 'free2' => o('logFiles', "./#{$path}/queries/freebuild2-0127.json.gz"), - 'sidaw' => o('logFiles', "./#{$path}/queries/sidaw.json"), + 'freebuild' => o('logFiles', "./#{$path}/queries/freebuild.json.gz"), + 'freebuilddef' => o('logFiles', "./#{$path}/queries/freebuild.def.json.gz"), + 'qual1' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier1-0118.json.gz"), + 'qual2' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier2-0129.json.gz"), + 'qual3' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier3-0201.json.gz"), # both 2 and 3 + 'free1' => o('logFiles', "./#{$path}/queries/rawqueries/freebuild1-0121.json.gz"), + 'free2' => o('logFiles', "./#{$path}/queries/rawqueries/freebuild2-0127.json.gz"), }), nil)}) @@ -161,16 +158,16 @@ addMode('voxelurn', 'interactive semantic parsing in a VoxelWorld', lambda { |e| o('LanguageAnalyzer.lowerCaseTokens', true), o('Parser.pruneErrorValues', true), - o('Parser', 'interactive.BeamFloatingParser'), + o('Parser', 'interactive.InteractiveBeamParser'), o('Parser.callSetEvaluation', false), - o('Parser.coarsePrune', 'true'), + o('Parser.coarsePrune', true), o('Parser.beamSize', 50), - o('BeamFloatingParser.maxNewTreesPerSpan', 5001), + o('InteractiveBeamParser.maxNewTreesPerSpan', 5001), o('ParserState.customExpectedCounts', 'None'), - selo(0, 'BeamFloatingParser.floatStrategy', 'Never', 'NoParse', 'Always'), - o('BeamFloatingParser.trackedCats', 'Number', 'Numbers', 'Color', 'Direction', 'Set', 'Sets', 'Action', 'Actions'), + selo(0, 'InteractiveBeamParser.floatStrategy', 'Never', 'NoParse', 'Always'), + o('InteractiveBeamParser.trackedCats', 'Number', 'Numbers', 'Color', 'Direction', 'Set', 'Sets', 'Action', 'Actions'), o('Derivation.derivComparator', 'AnchorPriorityScoreComparator'), o('Params.l1Reg', 'nonlazy'), @@ -207,6 +204,7 @@ addMode('voxelurn', 'interactive semantic parsing in a VoxelWorld', lambda { |e| o('Derivation.showTypes', false), o('Derivation.showValues', false), o('Derivation.showRules', false), + o('Derivation.anchoredBonus', 1.0), o('NumberFn.allowedRange', 0, 100), o('SimpleLexicon.inPaths', "./#{$path}/csscolors.lexicon"), @@ -214,10 +212,6 @@ addMode('voxelurn', 'interactive semantic parsing in a VoxelWorld', lambda { |e| lambda { |e| system 'mkdir -p ./int-output/'; nil}, lambda { |e| system 'mkdir -p ./int-output/log/'; nil}, lambda { |e| system 'mkdir -p ./int-output/citation/'; nil}, - lambda { |e| system 'mkdir -p server.exec'; - l(o('execDir', 'server.exec/'), - o('overwriteExecDir')) - }, nil) }) ############################################################ diff --git a/interactive/voxelurn.grammar b/interactive/voxelurn.grammar index 3bd32a3..d098a7b 100644 --- a/interactive/voxelurn.grammar +++ b/interactive/voxelurn.grammar @@ -13,7 +13,7 @@ # typed lambdas # Action a, ItemSet i, ValueSet v, Rel r, Direction d, Color c, Number n -# This grammar should have Rule.defaultIsFloating false +# Rule.defaultIsFloating should be false # utilities (rule $Action (base $Number $Number) (lambda x (lambda y (: @base (var x) (var y)))) (anchored 1)) diff --git a/src/edu/stanford/nlp/sempre/Derivation.java b/src/edu/stanford/nlp/sempre/Derivation.java index 2b7b60e..931b8b5 100644 --- a/src/edu/stanford/nlp/sempre/Derivation.java +++ b/src/edu/stanford/nlp/sempre/Derivation.java @@ -30,6 +30,8 @@ public class Derivation implements SemanticFn.Callable, HasScore { public boolean showExecutions = false; @Option(gloss = "Pick the comparator used to sort derivations") public String derivComparator = "ScoredDerivationComparator"; + @Option(gloss = "bonus score for being all anchored") + public double anchoredBonus = 0.0; } public static Options opts = new Options(); @@ -264,7 +266,7 @@ public class Derivation implements SemanticFn.Callable, HasScore { public void addFeatures(FeatureVector fv) { this.localFeatureVector.add(fv); } public double localScore(Params params) { - return localFeatureVector.dotProduct(params); + return localFeatureVector.dotProduct(params) + (this.allAnchored()? opts.anchoredBonus : 0.0); } // SHOULD NOT BE USED except during test time if the memory is desperately needed. diff --git a/src/edu/stanford/nlp/sempre/interactive/BeamFloatingParser.java b/src/edu/stanford/nlp/sempre/interactive/InteractiveBeamParser.java similarity index 89% rename from src/edu/stanford/nlp/sempre/interactive/BeamFloatingParser.java rename to src/edu/stanford/nlp/sempre/interactive/InteractiveBeamParser.java index 3ad213b..24c5e7b 100644 --- a/src/edu/stanford/nlp/sempre/interactive/BeamFloatingParser.java +++ b/src/edu/stanford/nlp/sempre/interactive/InteractiveBeamParser.java @@ -5,6 +5,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -38,16 +39,11 @@ import fig.basic.StopWatchSet; import fig.exec.Execution; /** - * A simple bottom-up chart-based parser that keeps the |BeamFloatingSize| top - * derivations for each chart cell (cat, start, end). Also supports fast - * indexing of lexicalized rules using a trie. - * - * Note that this code does not rely on the Grammar being binarized, which makes - * it more complex. + * A modified version of the BeamParser, with consideration for use in the interactive setting * * @author Percy Liang, sidaw */ -public class BeamFloatingParser extends Parser { +public class InteractiveBeamParser extends Parser { public static class Options { @Option public int maxNewTreesPerSpan = Integer.MAX_VALUE; @@ -64,13 +60,17 @@ public class BeamFloatingParser extends Parser { public static Options opts = new Options(); Trie trie; // For non-cat-unary rules - - public BeamFloatingParser(Spec spec) { + // so that duplicated rules are never added + Set allRules; + List interactiveCatUnaryRules; + public InteractiveBeamParser(Spec spec) { super(spec); if (opts.trackedCats != null) { opts.trackedCats = opts.trackedCats.stream().map(s -> "$" + s).collect(Collectors.toList()); LogInfo.logs("Mapped trackedCats to: %s", opts.trackedCats); } + interactiveCatUnaryRules = new LinkedList<>(super.catUnaryRules); + allRules = new LinkedHashSet<>(super.catUnaryRules); // Index the non-cat-unary rules trie = new Trie(); for (Rule rule : grammar.getRules()) { @@ -80,12 +80,31 @@ public class BeamFloatingParser extends Parser { this.chartFillOut = IOUtils.openOutAppendEasy(Execution.getFile("chartfill")); } + @Override + public synchronized void addRule(Rule rule) { + if (allRules.contains(rule)) + return; + + allRules.add(rule); + + if (!rule.isCatUnary()) { + trie.add(rule); + } else { + interactiveCatUnaryRules.add(rule); + } + } + + @Override + public List getCatUnaryRules() { + return interactiveCatUnaryRules; + } + // for grammar induction, just need the formula, do not execute - public BeamFloatingParserState justParse(Params params, Example ex, boolean computeExpectedCounts) { + public InteractiveBeamParserState parseWithoutExecuting(Params params, Example ex, boolean computeExpectedCounts) { // Parse StopWatch watch = new StopWatch(); watch.start(); - BeamFloatingParserState state = new BeamFloatingParserState(this, params, ex); + InteractiveBeamParserState state = new InteractiveBeamParserState(this, params, ex); state.infer(); watch.stop(); state.parseTime = watch.getCurrTimeLong(); @@ -96,25 +115,19 @@ public class BeamFloatingParser extends Parser { return state; } - @Override - public synchronized void addRule(Rule rule) { - if (!rule.isCatUnary()) { - trie.add(rule); - } - } @Override public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) { - BeamFloatingParserState coarseState = null; + InteractiveBeamParserState coarseState = null; if (Parser.opts.coarsePrune) { LogInfo.begin_track("Parser.coarsePrune"); - coarseState = new BeamFloatingParserState(this, params, ex, computeExpectedCounts, - BeamFloatingParserState.Mode.bool, null); + coarseState = new InteractiveBeamParserState(this, params, ex, computeExpectedCounts, + InteractiveBeamParserState.Mode.bool, null); coarseState.infer(); coarseState.keepTopDownReachable(); LogInfo.end_track(); } - return new BeamFloatingParserState(this, params, ex, computeExpectedCounts, BeamFloatingParserState.Mode.full, + return new InteractiveBeamParserState(this, params, ex, computeExpectedCounts, InteractiveBeamParserState.Mode.full, coarseState); } } @@ -125,8 +138,9 @@ public class BeamFloatingParser extends Parser { * * @author Percy Liang * @author Roy Frostig + * @author sidaw */ -class BeamFloatingParserState extends ChartParserState { +class InteractiveBeamParserState extends ChartParserState { public final Mode mode; // Modes: @@ -137,13 +151,13 @@ class BeamFloatingParserState extends ChartParserState { bool, full } - private final BeamFloatingParser parser; - private final BeamFloatingParserState coarseState; // Used to prune + private final InteractiveBeamParser parser; + private final InteractiveBeamParserState coarseState; // Used to prune private final boolean execute; public List chartList; - public BeamFloatingParserState(BeamFloatingParser parser, Params params, Example ex) { + public InteractiveBeamParserState(InteractiveBeamParser parser, Params params, Example ex) { super(parser, params, ex, false); this.parser = parser; this.mode = Mode.full; @@ -151,8 +165,8 @@ class BeamFloatingParserState extends ChartParserState { this.execute = false; } - public BeamFloatingParserState(BeamFloatingParser parser, Params params, Example ex, boolean computeExpectedCounts, - Mode mode, BeamFloatingParserState coarseState) { + public InteractiveBeamParserState(InteractiveBeamParser parser, Params params, Example ex, boolean computeExpectedCounts, + Mode mode, InteractiveBeamParserState coarseState) { super(parser, params, ex, computeExpectedCounts); this.parser = parser; this.mode = mode; @@ -198,9 +212,9 @@ class BeamFloatingParserState extends ChartParserState { this.chartList = this.collectChart(); boolean parseFloat = false; - if (BeamFloatingParser.opts.floatStrategy == BeamFloatingParser.FloatStrategy.Always) + if (InteractiveBeamParser.opts.floatStrategy == InteractiveBeamParser.FloatStrategy.Always) parseFloat = true; - else if (BeamFloatingParser.opts.floatStrategy == BeamFloatingParser.FloatStrategy.NoParse) + else if (InteractiveBeamParser.opts.floatStrategy == InteractiveBeamParser.FloatStrategy.NoParse) parseFloat = predDerivations.size() == 0; else parseFloat = false; @@ -397,7 +411,7 @@ class BeamFloatingParserState extends ChartParserState { children.remove(children.size() - 1); if (mode != Mode.full) break; // Only need one hypothesis - if (numNew.value >= BeamFloatingParser.opts.maxNewTreesPerSpan) + if (numNew.value >= InteractiveBeamParser.opts.maxNewTreesPerSpan) return; } } diff --git a/src/edu/stanford/nlp/sempre/interactive/InteractiveMaster.java b/src/edu/stanford/nlp/sempre/interactive/InteractiveMaster.java index 6bc9fd7..0454a4a 100644 --- a/src/edu/stanford/nlp/sempre/interactive/InteractiveMaster.java +++ b/src/edu/stanford/nlp/sempre/interactive/InteractiveMaster.java @@ -153,7 +153,7 @@ public class InteractiveMaster extends Master { response.ex = ex; // Parse! - ((BeamFloatingParser)builder.parser).justParse(builder.params, ex, false); + ((InteractiveBeamParser)builder.parser).parseWithoutExecuting(builder.params, ex, false); int rank = -1; Derivation match = null; @@ -212,6 +212,8 @@ public class InteractiveMaster extends Master { for (Rule rule : inducedRules) { InteractiveUtils.addRuleInteractive(rule, builder.parser); } + stats.put("total_rules", ((InteractiveBeamParser)builder.parser).allRules.size()); + stats.put("total_unicat", ((InteractiveBeamParser)builder.parser).interactiveCatUnaryRules.size()); } // TODO : should not have to parse again, I guess just set the formula // or something @@ -275,7 +277,7 @@ public class InteractiveMaster extends Master { if (isNonsense(exHead)) throw BadInteractionException.nonSenseDefinition(head); - BeamFloatingParserState state = ((BeamFloatingParser)parser).justParse(params, exHead, false); + InteractiveBeamParserState state = ((InteractiveBeamParser)parser).parseWithoutExecuting(params, exHead, false); if (GrammarInducer.getParseStatus(exHead) == GrammarInducer.ParseStatus.Core) throw BadInteractionException.headIsCore(head); diff --git a/src/edu/stanford/nlp/sempre/interactive/InteractiveServer.java b/src/edu/stanford/nlp/sempre/interactive/InteractiveServer.java index 29f0127..266c76f 100644 --- a/src/edu/stanford/nlp/sempre/interactive/InteractiveServer.java +++ b/src/edu/stanford/nlp/sempre/interactive/InteractiveServer.java @@ -227,7 +227,8 @@ public class InteractiveServer { future.cancel(true); executor.shutdown(); long endTime = System.nanoTime(); - response.stats.put("time", (endTime - startTime) / 1.0e9); + response.stats.put("walltime", (endTime - startTime) / 1.0e9); + response.stats.put("count", queryCounter.get()); } return response; } @@ -243,10 +244,10 @@ public class InteractiveServer { LocalDateTime queryTime = LocalDateTime.now(); synchronized (queryLogLock) { // write the query log Map jsonMap = new LinkedHashMap<>(); + jsonMap.put("count", queryNumber); jsonMap.put("q", query); - jsonMap.put("remote", remoteHost); - jsonMap.put("time", queryTime.toString()); - jsonMap.put("query", queryNumber); + // jsonMap.put("remote", remoteHost); + // jsonMap.put("time", queryTime.toString()); jsonMap.put("sessionId", sessionId); reqParams.remove("q"); jsonMap.putAll(reqParams); diff --git a/src/edu/stanford/nlp/sempre/interactive/InteractiveUtils.java b/src/edu/stanford/nlp/sempre/interactive/InteractiveUtils.java index 9b315f2..440acbb 100644 --- a/src/edu/stanford/nlp/sempre/interactive/InteractiveUtils.java +++ b/src/edu/stanford/nlp/sempre/interactive/InteractiveUtils.java @@ -92,7 +92,7 @@ public final class InteractiveUtils { ex.preprocess(); LogInfo.logs("Parsing body: %s", ex.utterance); - ((BeamFloatingParser)parser).justParse(params, ex, false); + ((InteractiveBeamParser)parser).parseWithoutExecuting(params, ex, false); boolean found = false; Formula targetFormula = Formulas.fromLispTree(LispTree.proto.parseFromString(formula)); @@ -161,10 +161,10 @@ public final class InteractiveUtils { public static synchronized void addRuleInteractive(Rule rule, Parser parser) { LogInfo.logs("addRuleInteractive: %s", rule); - parser.grammar.addRule(rule); - - if (parser instanceof BeamParser || parser instanceof BeamFloatingParser) { + if (parser instanceof InteractiveBeamParser) { parser.addRule(rule); + } else { + throw new RuntimeException("interactively adding rule not supported for paser " + parser.getClass().toString()); } } diff --git a/src/edu/stanford/nlp/sempre/interactive/SimulationAnalyzer.java b/src/edu/stanford/nlp/sempre/interactive/SimulationAnalyzer.java index 2555303..804bd13 100644 --- a/src/edu/stanford/nlp/sempre/interactive/SimulationAnalyzer.java +++ b/src/edu/stanford/nlp/sempre/interactive/SimulationAnalyzer.java @@ -31,6 +31,7 @@ public class SimulationAnalyzer { Map line = new LinkedHashMap(query); LogInfo.logs("stats: %s", stats); if (stats == null) { + LogInfo.logs("No stats"); LogInfo.log(query); LogInfo.log(jsonResponse); return; @@ -54,7 +55,7 @@ public class SimulationAnalyzer { qEval.add("def.num_failed", (Integer) stats.get("num_failed")); qEval.add("def.num_body", (Integer) stats.get("num_body")); qEval.add("def.num_rules", (Integer) stats.get("num_rules")); - qEval.add("def.time", (Double) stats.get("time")); + qEval.add("def.time", (Integer) stats.get("count")); } if (stats.get("type").equals("q") && !stats.containsKey("error")) { diff --git a/src/edu/stanford/nlp/sempre/interactive/test/FloatingParsingTest.java b/src/edu/stanford/nlp/sempre/interactive/test/FloatingParsingTest.java index c012931..05ba586 100644 --- a/src/edu/stanford/nlp/sempre/interactive/test/FloatingParsingTest.java +++ b/src/edu/stanford/nlp/sempre/interactive/test/FloatingParsingTest.java @@ -8,7 +8,7 @@ import java.util.function.Predicate; import fig.basic.*; import edu.stanford.nlp.sempre.*; import edu.stanford.nlp.sempre.Parser.Spec; -import edu.stanford.nlp.sempre.interactive.BeamFloatingParser; +import edu.stanford.nlp.sempre.interactive.InteractiveBeamParser; import edu.stanford.nlp.sempre.interactive.DALExecutor; import org.testng.Assert; @@ -80,7 +80,7 @@ public class FloatingParsingTest { ex.preprocess(); Spec defSpec = defaultSpec(); - Parser parser = new BeamFloatingParser(defSpec); + Parser parser = new InteractiveBeamParser(defSpec); ParserState state = parser.parse(new Params(), ex, false); LogInfo.end_track(); diff --git a/src/edu/stanford/nlp/sempre/interactive/test/GrammarInducerTest.java b/src/edu/stanford/nlp/sempre/interactive/test/GrammarInducerTest.java index d16581c..15d529c 100644 --- a/src/edu/stanford/nlp/sempre/interactive/test/GrammarInducerTest.java +++ b/src/edu/stanford/nlp/sempre/interactive/test/GrammarInducerTest.java @@ -27,7 +27,7 @@ import edu.stanford.nlp.sempre.Rule; import edu.stanford.nlp.sempre.Session; import edu.stanford.nlp.sempre.SimpleLexicon; import edu.stanford.nlp.sempre.ValueEvaluator; -import edu.stanford.nlp.sempre.interactive.BeamFloatingParser; +import edu.stanford.nlp.sempre.interactive.InteractiveBeamParser; import edu.stanford.nlp.sempre.interactive.DALExecutor; import edu.stanford.nlp.sempre.interactive.DefinitionAligner; import edu.stanford.nlp.sempre.interactive.GrammarInducer; @@ -94,7 +94,7 @@ public class GrammarInducerTest { List allRules; public ParseTester() { - parser = new BeamFloatingParser(defaultSpec()); + parser = new InteractiveBeamParser(defaultSpec()); params = new Params(); allRules = new ArrayList<>(); }