mirror of https://github.com/percyliang/sempre
grammar induction tests working!
This commit is contained in:
Sida Wang
parent
ef60c1c0dc
commit
7a10a749dc
5
run
5
run
|
|
@ -984,7 +984,7 @@ addMode('interactive', 'interactive semantic parsing in blocks', lambda { |e| l(
|
|||
'edu.stanford.nlp.sempre.Main',
|
||||
figOpts,
|
||||
|
||||
o('Executor', 'interactive.actions.ActionExecutor'),
|
||||
o('Executor', 'tions.ActionExecutor'),
|
||||
o('LanguageAnalyzer', 'interactive.actions.ActionLanguageAnalyzer'),
|
||||
o('ActionExecutor.convertNumberValues', true),
|
||||
o('ActionExecutor.printStackTrace', true),
|
||||
|
|
@ -1003,7 +1003,8 @@ addMode('interactive', 'interactive semantic parsing in blocks', lambda { |e| l(
|
|||
o('Parser', 'BeamFloatingParser'),
|
||||
o('Parser.trackedCats', 'Number', 'Numbers', 'Color', 'Direction', 'Set', 'Sets', 'Action', 'Actions'),
|
||||
o('FloatingParser.defaultIsFloating', true),
|
||||
o('BeamFloatingParser.alwaysFloat', false),
|
||||
selo(0, 'BeamFloatingParser.floatStrategy', 'BeamFloatingParser.FloatStrategy.Never',
|
||||
'BeamFloatingParser.FloatStrategy.NoParse', 'BeamFloatingParser.FloatStrategy.Always'),
|
||||
|
||||
o('ParserState.customExpectedCounts', 'UNIFORM'),
|
||||
o('Params.l1Reg', 'lazy'),
|
||||
|
|
|
|||
|
|
@ -25,8 +25,9 @@ import org.testng.collections.Sets;
|
|||
public class BeamFloatingParser extends Parser {
|
||||
public static class Options {
|
||||
@Option public int maxNewTreesPerSpan = Integer.MAX_VALUE;
|
||||
@Option public boolean alwaysFloat = true;
|
||||
@Option public FloatStrategy floatStrategy = FloatStrategy.Never;
|
||||
}
|
||||
public enum FloatStrategy {Always, Never, NoParse};
|
||||
public static Options opts = new Options();
|
||||
|
||||
Trie trie; // For non-cat-unary rules
|
||||
|
|
@ -132,8 +133,16 @@ class BeamFloatingParserState extends ChartParserState {
|
|||
|
||||
this.chartList = this.collectChart();
|
||||
|
||||
boolean parseFloat;
|
||||
if (BeamFloatingParser.opts.floatStrategy != BeamFloatingParser.FloatStrategy.Always)
|
||||
parseFloat = true;
|
||||
else if (BeamFloatingParser.opts.floatStrategy == BeamFloatingParser.FloatStrategy.NoParse)
|
||||
parseFloat = predDerivations.size() == 0;
|
||||
else
|
||||
parseFloat = false;
|
||||
|
||||
/* If Beam Parser failed to find derivations, try a floating parser */
|
||||
if (BeamFloatingParser.opts.alwaysFloat || predDerivations.size() == 0) {
|
||||
if (parseFloat) {
|
||||
/* For every base span of the chart, add the derivations from nothing rules */
|
||||
List<Rule> nothingRules = new ArrayList<Rule>();
|
||||
for (Rule rule : parser.grammar.rules)
|
||||
|
|
|
|||
|
|
@ -34,9 +34,8 @@ public final class InteractiveUtils {
|
|||
public static GrammarInducer getInducer(String head, String jsonDef, String sessionId, Parser parser, Params params) {
|
||||
return getInducer(head, jsonDef, sessionId, parser, params, ActionFormula.Mode.block);
|
||||
}
|
||||
// parse the definition, match with the chart of origEx, and add new rules to grammar
|
||||
public static GrammarInducer getInducer(String head, String jsonDef, String sessionId, Parser parser, Params params, ActionFormula.Mode mode) {
|
||||
|
||||
|
||||
public static List<Derivation> derivsfromJson(String jsonDef, Parser parser, Params params) {
|
||||
@SuppressWarnings("unchecked")
|
||||
List<Object> body = Json.readValueHard(jsonDef, List.class);
|
||||
// string together the body definition
|
||||
|
|
@ -54,7 +53,7 @@ public final class InteractiveUtils {
|
|||
}
|
||||
|
||||
Example.Builder b = new Example.Builder();
|
||||
b.setId("session:" + sessionId);
|
||||
// b.setId("session:" + sessionId);
|
||||
b.setUtterance(utt);
|
||||
Example ex = b.createExample();
|
||||
ex.preprocess();
|
||||
|
|
@ -70,20 +69,17 @@ public final class InteractiveUtils {
|
|||
allDerivs.add(stripDerivation(d));
|
||||
}
|
||||
}
|
||||
if (!found) LogInfo.errors("Matching formula not found: %s", formula);
|
||||
if (!found && !formula.equals("?")) LogInfo.errors("matching formula not found: %s", formula);
|
||||
|
||||
// just some hacks to make testing easier, use top derivation when we formula is not given
|
||||
// just making testing easier, use top derivation when we formula is not given
|
||||
if (!found && (formula.equals("?") || formula==null) && ex.predDerivations.size() > 0)
|
||||
allDerivs.add(stripDerivation(ex.predDerivations.get(0)));
|
||||
}
|
||||
|
||||
Derivation bodyDeriv;
|
||||
// if (allDerivs.size() > 1)
|
||||
// bodyDeriv = combineList(allDerivs, ActionFormula.Mode.block);
|
||||
// else
|
||||
// bodyDeriv = allDerivs.get(0);
|
||||
//
|
||||
bodyDeriv = combineList(allDerivs, mode);
|
||||
return allDerivs;
|
||||
}
|
||||
// parse the definition, match with the chart of origEx, and add new rules to grammar
|
||||
public static GrammarInducer getInducer(String head, String jsonDef, String sessionId, Parser parser, Params params, ActionFormula.Mode mode) {
|
||||
Derivation bodyDeriv = combine(derivsfromJson(jsonDef, parser, params), mode);
|
||||
|
||||
Example.Builder b = new Example.Builder();
|
||||
b.setId("session:" + sessionId);
|
||||
|
|
@ -99,7 +95,7 @@ public final class InteractiveUtils {
|
|||
LogInfo.logs("anchored: %s", state.chartList);
|
||||
|
||||
GrammarInducer grammarInducer = new GrammarInducer(exHead, bodyDeriv, state.chartList);
|
||||
// updates the value that the derivation is modified.
|
||||
// updates the value that the derivation is modified.
|
||||
// for (Derivation d : grammarInducer.getHead().predDerivations) {
|
||||
// d.value = null; // cuz ensureExecuted checks.
|
||||
// d.ensureExecuted(parser.executor, exHead.context);
|
||||
|
|
@ -138,7 +134,7 @@ public final class InteractiveUtils {
|
|||
b.init(LispTree.proto.parseFromString("(a block)"));
|
||||
return new Rule("$Action", Lists.newArrayList("$Action", "$Action"), b);
|
||||
}
|
||||
static Derivation combineList(List<Derivation> children, ActionFormula.Mode mode) {
|
||||
public static Derivation combine(List<Derivation> children, ActionFormula.Mode mode) {
|
||||
Formula f = new ActionFormula(mode,
|
||||
children.stream().map(d -> d.formula).collect(Collectors.toList()));
|
||||
Derivation res = new Derivation.Builder()
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ import edu.stanford.nlp.sempre.*;
|
|||
import edu.stanford.nlp.sempre.Parser.Spec;
|
||||
|
||||
import org.testng.Assert;
|
||||
import org.testng.asserts.SoftAssert;
|
||||
import org.testng.asserts.Assertion;
|
||||
import org.testng.annotations.Test;
|
||||
import org.testng.collections.Lists;
|
||||
|
||||
|
|
@ -20,12 +22,15 @@ import org.testng.collections.Lists;
|
|||
* @author Sida Wang
|
||||
*/
|
||||
public class GrammarInducerTest {
|
||||
static Assertion A = new Assertion();
|
||||
// static Assertion A = new SoftAssert();
|
||||
|
||||
private static Spec defaultSpec() {
|
||||
FloatingParser.opts.defaultIsFloating = true;
|
||||
ActionExecutor.opts.convertNumberValues = true;
|
||||
ActionExecutor.opts.printStackTrace = true;
|
||||
ActionExecutor.opts.FlatWorldType = "BlocksWorld";
|
||||
LanguageAnalyzer.opts.languageAnalyzer = "interactive.actions.ActionLanguageAnalyzer";
|
||||
Grammar.opts.inPaths = Lists.newArrayList("./shrdlurn/blocksworld.grammar");
|
||||
Grammar.opts.useApplyFn = "interactive.ApplyFn";
|
||||
Grammar.opts.binarizeRules = false;
|
||||
|
|
@ -40,57 +45,126 @@ public class GrammarInducerTest {
|
|||
return new Parser.Spec(grammar, extractor, executor, valueEvaluator);
|
||||
}
|
||||
|
||||
protected static void induceHelper(String head, String body) {
|
||||
Spec defSpec = defaultSpec();
|
||||
Parser parser = new BeamFloatingParser(defSpec);
|
||||
protected String d(String... defs) {
|
||||
List<List<String>> defList = new ArrayList<>();
|
||||
for (String def : defs) {
|
||||
defList.add(Lists.newArrayList(def, "?"));
|
||||
}
|
||||
return Json.writeValueAsStringHard(defList);
|
||||
}
|
||||
|
||||
class ParseTester {
|
||||
Parser parser;
|
||||
Params params;
|
||||
List<Rule> allRules;
|
||||
public ParseTester() {
|
||||
parser = new BeamFloatingParser(defaultSpec());
|
||||
params = new Params();
|
||||
allRules = new ArrayList<>();
|
||||
}
|
||||
public void def(String head, String def) {
|
||||
List<Rule> induced = InteractiveUtils.getInducer(head, def, "testsession", parser, new Params()).getRules();
|
||||
allRules.addAll(induced);
|
||||
induced.forEach(r -> InteractiveUtils.addRuleInteractive(r, parser));
|
||||
LogInfo.logs("Defining %s := %s, added %s", head, def, induced);
|
||||
}
|
||||
public boolean found(String head, String def) {
|
||||
Example.Builder b = new Example.Builder();
|
||||
b.setUtterance(head);
|
||||
Example exHead = b.createExample();
|
||||
exHead.preprocess();
|
||||
|
||||
// LogInfo.logs("Parsing definition: %s", ex.utterance);
|
||||
parser.parse(params, exHead, false);
|
||||
|
||||
Derivation defDeriv = InteractiveUtils.combine(InteractiveUtils.derivsfromJson(def, parser, params), ActionFormula.Mode.block);
|
||||
boolean found = false;
|
||||
int ind = 0;
|
||||
for (Derivation d : exHead.predDerivations) {
|
||||
// LogInfo.logs("considering: %s", d.formula.toString());
|
||||
if (d.formula.equals(defDeriv.formula)) {
|
||||
found = true;
|
||||
LogInfo.logs("found %s at %d", d.formula, ind);
|
||||
}
|
||||
ind++;
|
||||
}
|
||||
printAllRules();
|
||||
return found;
|
||||
}
|
||||
|
||||
List<Rule> induced = InteractiveUtils.getInducer(head, body, "test", parser, new Params()).getRules();
|
||||
for (Rule rule : induced)
|
||||
LogInfo.logs("Induced: %s", rule);
|
||||
public void printAllRules() {
|
||||
LogInfo.begin_track("Rules induced");
|
||||
allRules.forEach(r -> LogInfo.log(r));
|
||||
LogInfo.end_track();
|
||||
}
|
||||
}
|
||||
|
||||
protected static void induce(String head, String jsonDef) {
|
||||
induceHelper(head, jsonDef);
|
||||
}
|
||||
|
||||
|
||||
protected static void test(String head, String def, String head2, String def2) {
|
||||
Spec defSpec = defaultSpec();
|
||||
Parser parser = new BeamFloatingParser(defSpec);
|
||||
// tests simple substitutions
|
||||
@Test public void simpleTest() {
|
||||
LogInfo.begin_track("test simple substitutions");
|
||||
ParseTester T = new ParseTester();
|
||||
|
||||
List<Rule> induced = InteractiveUtils.getInducer(head, def, "testsession", parser, new Params()).getRules();
|
||||
induced.forEach(r -> InteractiveUtils.addRuleInteractive(r, parser));
|
||||
}
|
||||
|
||||
@Test public void basicTest() {
|
||||
LogInfo.begin_track("test Grammar");
|
||||
induce("add red twice","[[\"add red top\",\"(: add red top)\"],[\"add red top\",\"(: add red top)\"]]");
|
||||
induce("add red thrice", "[[\"[add red top] 3 times\",\"(:loop (number 3) (: add red top))\"]]");
|
||||
induce("add a red block to the left", "[[\"add red left\",\"(: add red left)\"]]");
|
||||
induce("remove card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
induce("delete card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
induce("remove the leftmost red block", "[[\"remove very left of has color red\",\"?\"]]");
|
||||
induce("add red to both sides", "[[\"add red left\",\"(: add red left)\"],[\"add red right\",\"(: add red right)\"]]");
|
||||
induce("add a yellow block next to brown",
|
||||
"[[\"select brown\",\"(:foreach (color brown) (: select))\"],[\"add yellow right\",\"(: add yellow right)\"]]");
|
||||
induce("select all but yellow",
|
||||
"[[\"select not has color yellow\",\"?\"]]");
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
@Test public void learnSets() {
|
||||
LogInfo.begin_track("test Grammar");
|
||||
induce("remove those red blocks", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
induce("select card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
induce("remove the leftmost red block", "[[\"remove very left of has color red\",\"?\"]]");
|
||||
induce("add a red column of size 3",
|
||||
"[[\"repeat 3 [ add red; select top of this]\",\"(:loop (number 3) (:s (: add red here) (:for (call adj top this) (: select))))\"]]");
|
||||
LogInfo.end_track();
|
||||
}
|
||||
@Test public void learnLoop() {
|
||||
LogInfo.begin_track("test Grammar");
|
||||
induce("add a red column of size 3",
|
||||
"[[\"repeat 3 [ add red; select top of this]\",\"(:loop (number 3) (:s (: add red here) (:for (call adj top this) (: select))))\"]]");
|
||||
T.def("add red twice", d("add red top","add red top"));
|
||||
A.assertTrue(T.found("add blue twice", d("add blue top","add blue top")));
|
||||
|
||||
T.def("add red 3 times", d("repeat 3 [add red top]"));
|
||||
A.assertTrue(T.found("add yellow 5 times", d("repeat 5 [add yellow top]")));
|
||||
|
||||
T.def("add a red block to the left", d("add red left"));
|
||||
A.assertTrue(T.found("add a yellow block to the right", d("add yellow right")));
|
||||
|
||||
T.def("remove the leftmost red block", d("remove very left of has color red"));
|
||||
A.assertTrue(T.found("remove the leftmost yellow block", d("remove very left of has color yellow")));
|
||||
|
||||
T.def("add red to both sides", d("add red left", "add red right"));
|
||||
T.def("add red to both sides", d("add red back", "add red front"));
|
||||
A.assertTrue(T.found("add green to both sides", d("add green left", "add green right")));
|
||||
A.assertTrue(T.found("add green to both sides", d("add green back", "add green front")));
|
||||
A.assertFalse(T.found("add green to both sides", d("add green left", "add green back")));
|
||||
|
||||
T.def("select all but yellow", d("select not has color yellow"));
|
||||
A.assertTrue(T.found("select all but yellow", d("select not has color yellow")));
|
||||
|
||||
T.def("add a yellow block on top of red blocks", d("select has color red", "add yellow top"));
|
||||
A.assertTrue(T.found("add a green block on top of blue blocks", d("select has color blue", "add green top")));
|
||||
|
||||
T.def("add a yellow block on top of red blocks", d("select has color red; add yellow top"));
|
||||
A.assertTrue(T.found("add a green block on top of blue blocks", d("select has color blue ; add green top")));
|
||||
|
||||
//T.printAllRules();
|
||||
//A.assertAll();
|
||||
|
||||
LogInfo.end_track();
|
||||
}
|
||||
//
|
||||
// @Test public void basicTest() {
|
||||
// LogInfo.begin_track("test Grammar");
|
||||
// induce("add red twice","[[\"add red top\",\"(: add red top)\"],[\"add red top\",\"(: add red top)\"]]");
|
||||
// induce("add red thrice", "[[\"[add red top] 3 times\",\"(:loop (number 3) (: add red top))\"]]");
|
||||
// induce("add a red block to the left", "[[\"add red left\",\"(: add red left)\"]]");
|
||||
// induce("remove card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
// induce("delete card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
// induce("remove the leftmost red block", "[[\"remove very left of has color red\",\"?\"]]");
|
||||
// induce("add red to both sides", "[[\"add red left\",\"(: add red left)\"],[\"add red right\",\"(: add red right)\"]]");
|
||||
// induce("add a yellow block next to brown",
|
||||
// "[[\"select brown\",\"(:foreach (color brown) (: select))\"],[\"add yellow right\",\"(: add yellow right)\"]]");
|
||||
// induce("select all but yellow",
|
||||
// "[[\"select not has color yellow\",\"?\"]]");
|
||||
// LogInfo.end_track();
|
||||
// }
|
||||
//
|
||||
// @Test public void learnSets() {
|
||||
// LogInfo.begin_track("test Grammar");
|
||||
// induce("remove those red blocks", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
// induce("select card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
// induce("remove the leftmost red block", "[[\"remove very left of has color red\",\"?\"]]");
|
||||
// induce("add a red column of size 3",
|
||||
// "[[\"repeat 3 [ add red; select top of this]\",\"(:loop (number 3) (:s (: add red here) (:for (call adj top this) (: select))))\"]]");
|
||||
// LogInfo.end_track();
|
||||
// }
|
||||
// @Test public void learnLoop() {
|
||||
// LogInfo.begin_track("test Grammar");
|
||||
// induce("add a red column of size 3",
|
||||
// "[[\"repeat 3 [ add red; select top of this]\",\"(:loop (number 3) (:s (: add red here) (:for (call adj top this) (: select))))\"]]");
|
||||
// LogInfo.end_track();
|
||||
// }
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue