From 7a10a749dcd62af4b6ed77211cc36c7aed4d7c6a Mon Sep 17 00:00:00 2001 From: Sida Wang Date: Sat, 7 Jan 2017 13:06:03 -0800 Subject: [PATCH] grammar induction tests working! --- run | 5 +- .../nlp/sempre/BeamFloatingParser.java | 13 +- .../stanford/nlp/sempre/InteractiveUtils.java | 28 ++- .../actions/GrammarInducerTest.java | 170 +++++++++++++----- 4 files changed, 148 insertions(+), 68 deletions(-) diff --git a/run b/run index d570632..35ad9f1 100755 --- a/run +++ b/run @@ -984,7 +984,7 @@ addMode('interactive', 'interactive semantic parsing in blocks', lambda { |e| l( 'edu.stanford.nlp.sempre.Main', figOpts, - o('Executor', 'interactive.actions.ActionExecutor'), + o('Executor', 'tions.ActionExecutor'), o('LanguageAnalyzer', 'interactive.actions.ActionLanguageAnalyzer'), o('ActionExecutor.convertNumberValues', true), o('ActionExecutor.printStackTrace', true), @@ -1003,7 +1003,8 @@ addMode('interactive', 'interactive semantic parsing in blocks', lambda { |e| l( o('Parser', 'BeamFloatingParser'), o('Parser.trackedCats', 'Number', 'Numbers', 'Color', 'Direction', 'Set', 'Sets', 'Action', 'Actions'), o('FloatingParser.defaultIsFloating', true), - o('BeamFloatingParser.alwaysFloat', false), + selo(0, 'BeamFloatingParser.floatStrategy', 'BeamFloatingParser.FloatStrategy.Never', + 'BeamFloatingParser.FloatStrategy.NoParse', 'BeamFloatingParser.FloatStrategy.Always'), o('ParserState.customExpectedCounts', 'UNIFORM'), o('Params.l1Reg', 'lazy'), diff --git a/src/edu/stanford/nlp/sempre/BeamFloatingParser.java b/src/edu/stanford/nlp/sempre/BeamFloatingParser.java index dcf6593..62325c5 100644 --- a/src/edu/stanford/nlp/sempre/BeamFloatingParser.java +++ b/src/edu/stanford/nlp/sempre/BeamFloatingParser.java @@ -25,8 +25,9 @@ import org.testng.collections.Sets; public class BeamFloatingParser extends Parser { public static class Options { @Option public int maxNewTreesPerSpan = Integer.MAX_VALUE; - @Option public boolean alwaysFloat = true; + @Option public FloatStrategy floatStrategy = FloatStrategy.Never; } + public enum FloatStrategy {Always, Never, NoParse}; public static Options opts = new Options(); Trie trie; // For non-cat-unary rules @@ -132,8 +133,16 @@ class BeamFloatingParserState extends ChartParserState { this.chartList = this.collectChart(); + boolean parseFloat; + if (BeamFloatingParser.opts.floatStrategy != BeamFloatingParser.FloatStrategy.Always) + parseFloat = true; + else if (BeamFloatingParser.opts.floatStrategy == BeamFloatingParser.FloatStrategy.NoParse) + parseFloat = predDerivations.size() == 0; + else + parseFloat = false; + /* If Beam Parser failed to find derivations, try a floating parser */ - if (BeamFloatingParser.opts.alwaysFloat || predDerivations.size() == 0) { + if (parseFloat) { /* For every base span of the chart, add the derivations from nothing rules */ List nothingRules = new ArrayList(); for (Rule rule : parser.grammar.rules) diff --git a/src/edu/stanford/nlp/sempre/InteractiveUtils.java b/src/edu/stanford/nlp/sempre/InteractiveUtils.java index 4c0c294..7d2d4e5 100644 --- a/src/edu/stanford/nlp/sempre/InteractiveUtils.java +++ b/src/edu/stanford/nlp/sempre/InteractiveUtils.java @@ -34,9 +34,8 @@ public final class InteractiveUtils { public static GrammarInducer getInducer(String head, String jsonDef, String sessionId, Parser parser, Params params) { return getInducer(head, jsonDef, sessionId, parser, params, ActionFormula.Mode.block); } - // parse the definition, match with the chart of origEx, and add new rules to grammar - public static GrammarInducer getInducer(String head, String jsonDef, String sessionId, Parser parser, Params params, ActionFormula.Mode mode) { - + + public static List derivsfromJson(String jsonDef, Parser parser, Params params) { @SuppressWarnings("unchecked") List body = Json.readValueHard(jsonDef, List.class); // string together the body definition @@ -54,7 +53,7 @@ public final class InteractiveUtils { } Example.Builder b = new Example.Builder(); - b.setId("session:" + sessionId); + // b.setId("session:" + sessionId); b.setUtterance(utt); Example ex = b.createExample(); ex.preprocess(); @@ -70,20 +69,17 @@ public final class InteractiveUtils { allDerivs.add(stripDerivation(d)); } } - if (!found) LogInfo.errors("Matching formula not found: %s", formula); + if (!found && !formula.equals("?")) LogInfo.errors("matching formula not found: %s", formula); - // just some hacks to make testing easier, use top derivation when we formula is not given + // just making testing easier, use top derivation when we formula is not given if (!found && (formula.equals("?") || formula==null) && ex.predDerivations.size() > 0) allDerivs.add(stripDerivation(ex.predDerivations.get(0))); } - - Derivation bodyDeriv; -// if (allDerivs.size() > 1) -// bodyDeriv = combineList(allDerivs, ActionFormula.Mode.block); -// else -// bodyDeriv = allDerivs.get(0); -// - bodyDeriv = combineList(allDerivs, mode); + return allDerivs; + } + // parse the definition, match with the chart of origEx, and add new rules to grammar + public static GrammarInducer getInducer(String head, String jsonDef, String sessionId, Parser parser, Params params, ActionFormula.Mode mode) { + Derivation bodyDeriv = combine(derivsfromJson(jsonDef, parser, params), mode); Example.Builder b = new Example.Builder(); b.setId("session:" + sessionId); @@ -99,7 +95,7 @@ public final class InteractiveUtils { LogInfo.logs("anchored: %s", state.chartList); GrammarInducer grammarInducer = new GrammarInducer(exHead, bodyDeriv, state.chartList); -// updates the value that the derivation is modified. +// updates the value that the derivation is modified. // for (Derivation d : grammarInducer.getHead().predDerivations) { // d.value = null; // cuz ensureExecuted checks. // d.ensureExecuted(parser.executor, exHead.context); @@ -138,7 +134,7 @@ public final class InteractiveUtils { b.init(LispTree.proto.parseFromString("(a block)")); return new Rule("$Action", Lists.newArrayList("$Action", "$Action"), b); } - static Derivation combineList(List children, ActionFormula.Mode mode) { + public static Derivation combine(List children, ActionFormula.Mode mode) { Formula f = new ActionFormula(mode, children.stream().map(d -> d.formula).collect(Collectors.toList())); Derivation res = new Derivation.Builder() diff --git a/src/edu/stanford/nlp/sempre/interactive/actions/GrammarInducerTest.java b/src/edu/stanford/nlp/sempre/interactive/actions/GrammarInducerTest.java index 2489022..782489d 100644 --- a/src/edu/stanford/nlp/sempre/interactive/actions/GrammarInducerTest.java +++ b/src/edu/stanford/nlp/sempre/interactive/actions/GrammarInducerTest.java @@ -12,6 +12,8 @@ import edu.stanford.nlp.sempre.*; import edu.stanford.nlp.sempre.Parser.Spec; import org.testng.Assert; +import org.testng.asserts.SoftAssert; +import org.testng.asserts.Assertion; import org.testng.annotations.Test; import org.testng.collections.Lists; @@ -20,12 +22,15 @@ import org.testng.collections.Lists; * @author Sida Wang */ public class GrammarInducerTest { + static Assertion A = new Assertion(); + // static Assertion A = new SoftAssert(); private static Spec defaultSpec() { FloatingParser.opts.defaultIsFloating = true; ActionExecutor.opts.convertNumberValues = true; ActionExecutor.opts.printStackTrace = true; ActionExecutor.opts.FlatWorldType = "BlocksWorld"; + LanguageAnalyzer.opts.languageAnalyzer = "interactive.actions.ActionLanguageAnalyzer"; Grammar.opts.inPaths = Lists.newArrayList("./shrdlurn/blocksworld.grammar"); Grammar.opts.useApplyFn = "interactive.ApplyFn"; Grammar.opts.binarizeRules = false; @@ -40,57 +45,126 @@ public class GrammarInducerTest { return new Parser.Spec(grammar, extractor, executor, valueEvaluator); } - protected static void induceHelper(String head, String body) { - Spec defSpec = defaultSpec(); - Parser parser = new BeamFloatingParser(defSpec); + protected String d(String... defs) { + List> defList = new ArrayList<>(); + for (String def : defs) { + defList.add(Lists.newArrayList(def, "?")); + } + return Json.writeValueAsStringHard(defList); + } + + class ParseTester { + Parser parser; + Params params; + List allRules; + public ParseTester() { + parser = new BeamFloatingParser(defaultSpec()); + params = new Params(); + allRules = new ArrayList<>(); + } + public void def(String head, String def) { + List induced = InteractiveUtils.getInducer(head, def, "testsession", parser, new Params()).getRules(); + allRules.addAll(induced); + induced.forEach(r -> InteractiveUtils.addRuleInteractive(r, parser)); + LogInfo.logs("Defining %s := %s, added %s", head, def, induced); + } + public boolean found(String head, String def) { + Example.Builder b = new Example.Builder(); + b.setUtterance(head); + Example exHead = b.createExample(); + exHead.preprocess(); + + // LogInfo.logs("Parsing definition: %s", ex.utterance); + parser.parse(params, exHead, false); + + Derivation defDeriv = InteractiveUtils.combine(InteractiveUtils.derivsfromJson(def, parser, params), ActionFormula.Mode.block); + boolean found = false; + int ind = 0; + for (Derivation d : exHead.predDerivations) { + // LogInfo.logs("considering: %s", d.formula.toString()); + if (d.formula.equals(defDeriv.formula)) { + found = true; + LogInfo.logs("found %s at %d", d.formula, ind); + } + ind++; + } + printAllRules(); + return found; + } - List induced = InteractiveUtils.getInducer(head, body, "test", parser, new Params()).getRules(); - for (Rule rule : induced) - LogInfo.logs("Induced: %s", rule); + public void printAllRules() { + LogInfo.begin_track("Rules induced"); + allRules.forEach(r -> LogInfo.log(r)); + LogInfo.end_track(); + } } - - protected static void induce(String head, String jsonDef) { - induceHelper(head, jsonDef); - } - - - protected static void test(String head, String def, String head2, String def2) { - Spec defSpec = defaultSpec(); - Parser parser = new BeamFloatingParser(defSpec); + // tests simple substitutions + @Test public void simpleTest() { + LogInfo.begin_track("test simple substitutions"); + ParseTester T = new ParseTester(); - List induced = InteractiveUtils.getInducer(head, def, "testsession", parser, new Params()).getRules(); - induced.forEach(r -> InteractiveUtils.addRuleInteractive(r, parser)); - } - - @Test public void basicTest() { - LogInfo.begin_track("test Grammar"); - induce("add red twice","[[\"add red top\",\"(: add red top)\"],[\"add red top\",\"(: add red top)\"]]"); - induce("add red thrice", "[[\"[add red top] 3 times\",\"(:loop (number 3) (: add red top))\"]]"); - induce("add a red block to the left", "[[\"add red left\",\"(: add red left)\"]]"); - induce("remove card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]"); - induce("delete card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]"); - induce("remove the leftmost red block", "[[\"remove very left of has color red\",\"?\"]]"); - induce("add red to both sides", "[[\"add red left\",\"(: add red left)\"],[\"add red right\",\"(: add red right)\"]]"); - induce("add a yellow block next to brown", - "[[\"select brown\",\"(:foreach (color brown) (: select))\"],[\"add yellow right\",\"(: add yellow right)\"]]"); - induce("select all but yellow", - "[[\"select not has color yellow\",\"?\"]]"); - LogInfo.end_track(); - } - - @Test public void learnSets() { - LogInfo.begin_track("test Grammar"); - induce("remove those red blocks", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]"); - induce("select card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]"); - induce("remove the leftmost red block", "[[\"remove very left of has color red\",\"?\"]]"); - induce("add a red column of size 3", - "[[\"repeat 3 [ add red; select top of this]\",\"(:loop (number 3) (:s (: add red here) (:for (call adj top this) (: select))))\"]]"); - LogInfo.end_track(); - } - @Test public void learnLoop() { - LogInfo.begin_track("test Grammar"); - induce("add a red column of size 3", - "[[\"repeat 3 [ add red; select top of this]\",\"(:loop (number 3) (:s (: add red here) (:for (call adj top this) (: select))))\"]]"); + T.def("add red twice", d("add red top","add red top")); + A.assertTrue(T.found("add blue twice", d("add blue top","add blue top"))); + + T.def("add red 3 times", d("repeat 3 [add red top]")); + A.assertTrue(T.found("add yellow 5 times", d("repeat 5 [add yellow top]"))); + + T.def("add a red block to the left", d("add red left")); + A.assertTrue(T.found("add a yellow block to the right", d("add yellow right"))); + + T.def("remove the leftmost red block", d("remove very left of has color red")); + A.assertTrue(T.found("remove the leftmost yellow block", d("remove very left of has color yellow"))); + + T.def("add red to both sides", d("add red left", "add red right")); + T.def("add red to both sides", d("add red back", "add red front")); + A.assertTrue(T.found("add green to both sides", d("add green left", "add green right"))); + A.assertTrue(T.found("add green to both sides", d("add green back", "add green front"))); + A.assertFalse(T.found("add green to both sides", d("add green left", "add green back"))); + + T.def("select all but yellow", d("select not has color yellow")); + A.assertTrue(T.found("select all but yellow", d("select not has color yellow"))); + + T.def("add a yellow block on top of red blocks", d("select has color red", "add yellow top")); + A.assertTrue(T.found("add a green block on top of blue blocks", d("select has color blue", "add green top"))); + + T.def("add a yellow block on top of red blocks", d("select has color red; add yellow top")); + A.assertTrue(T.found("add a green block on top of blue blocks", d("select has color blue ; add green top"))); + + //T.printAllRules(); + //A.assertAll(); + LogInfo.end_track(); } +// +// @Test public void basicTest() { +// LogInfo.begin_track("test Grammar"); +// induce("add red twice","[[\"add red top\",\"(: add red top)\"],[\"add red top\",\"(: add red top)\"]]"); +// induce("add red thrice", "[[\"[add red top] 3 times\",\"(:loop (number 3) (: add red top))\"]]"); +// induce("add a red block to the left", "[[\"add red left\",\"(: add red left)\"]]"); +// induce("remove card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]"); +// induce("delete card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]"); +// induce("remove the leftmost red block", "[[\"remove very left of has color red\",\"?\"]]"); +// induce("add red to both sides", "[[\"add red left\",\"(: add red left)\"],[\"add red right\",\"(: add red right)\"]]"); +// induce("add a yellow block next to brown", +// "[[\"select brown\",\"(:foreach (color brown) (: select))\"],[\"add yellow right\",\"(: add yellow right)\"]]"); +// induce("select all but yellow", +// "[[\"select not has color yellow\",\"?\"]]"); +// LogInfo.end_track(); +// } +// +// @Test public void learnSets() { +// LogInfo.begin_track("test Grammar"); +// induce("remove those red blocks", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]"); +// induce("select card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]"); +// induce("remove the leftmost red block", "[[\"remove very left of has color red\",\"?\"]]"); +// induce("add a red column of size 3", +// "[[\"repeat 3 [ add red; select top of this]\",\"(:loop (number 3) (:s (: add red here) (:for (call adj top this) (: select))))\"]]"); +// LogInfo.end_track(); +// } +// @Test public void learnLoop() { +// LogInfo.begin_track("test Grammar"); +// induce("add a red column of size 3", +// "[[\"repeat 3 [ add red; select top of this]\",\"(:loop (number 3) (:s (: add red here) (:for (call adj top this) (: select))))\"]]"); +// LogInfo.end_track(); +// } }