mirror of https://github.com/percyliang/sempre
grammar induction test
This commit is contained in:
Sida Wang
parent
7a10a749dc
commit
3d68330bbe
1
run
1
run
|
|
@ -1034,6 +1034,7 @@ o('Learner.maxTrainIters', 1),
|
|||
o('server', true),
|
||||
o('Server.numThreads', 8),
|
||||
o('Server.maxCandidates', 50),
|
||||
o('Derivation.showTypes', false),
|
||||
nil) })
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@
|
|||
|
||||
####### domain specific floating rules
|
||||
# red means has color red
|
||||
(rule $Sets ($Color) (lambda c (color (var c))))
|
||||
# (rule $Sets ($Color) (lambda c (color (var c))))
|
||||
|
||||
# various actions can be performed
|
||||
# (rule $Action ($Color) (lambda c (: @add (var c) here)))
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@
|
|||
# Floating rules
|
||||
#############################
|
||||
|
||||
(rule $Action ($Numbers $Actions) (lambda n (lambda a (:loop (var n) (var a)))) (floating 1)) # do add red top 3 times
|
||||
# (rule $Action ($Numbers $Actions) (lambda n (lambda a (:loop (var n) (var a)))) (floating 1)) # do add red top 3 times
|
||||
# (rule $Action ($Sets $Actions) (lambda s (lambda a (:foreach (var s) (var a)))) (floating 1))
|
||||
# (rule $Action ($Actions $Actions) (lambda a1 (lambda a2 (:s (var a1) (var a2)))) (floating 1)) # add red top then remove them
|
||||
|
||||
|
|
@ -99,12 +99,12 @@
|
|||
# (rule $ValueSets ($Rel $Sets) (lambda r (lambda s
|
||||
# ((reverse (var r)) (var s)))) (floating 1))
|
||||
|
||||
(rule $Action ($Sets) (lambda s (: @select (var s))) (floating 1))
|
||||
# (rule $Action ($Sets) (lambda s (: @select (var s))) (floating 1))
|
||||
|
||||
# (rule $Set ($Rel $ValueSets) (lambda r (lambda s
|
||||
# ((var r) (var s)))) (floating 1))
|
||||
# (rule $Action ($Rel $ValueSets) (lambda r (lambda v (: @update (var r) (var v)))) (floating 1))
|
||||
# (rule $Action (nothing) (ConstantFn (: @remove)) (floating 1))
|
||||
|
||||
(rule $FROOT ($Action) (IdentityFn) (floating 1))
|
||||
(rule $FROOT ($Actions) (IdentityFn) (floating 1))
|
||||
# (rule $FROOT ($Action) (IdentityFn) (floating 1))
|
||||
# (rule $FROOT ($Actions) (IdentityFn) (floating 1))
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
public boolean showRules = false;
|
||||
@Option(gloss = "When printing derivations, to show canonical utterance")
|
||||
public boolean showUtterance = false;
|
||||
@Option(gloss = "When printing derivations, show the category")
|
||||
public boolean showCat = false;
|
||||
@Option(gloss = "When executing, show formulae (for debugging)")
|
||||
public boolean showExecutions = false;
|
||||
@Option(gloss = "changes ScoredDerivationComparator to use pragmatic_score instead")
|
||||
|
|
@ -347,6 +349,9 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
if (opts.showUtterance && canonicalUtterance != null) {
|
||||
tree.addChild(LispTree.proto.newList("canonicalUtterance", canonicalUtterance));
|
||||
}
|
||||
if (opts.showCat && cat != null) {
|
||||
tree.addChild(LispTree.proto.newList("cat", cat));
|
||||
}
|
||||
return tree;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -135,6 +135,13 @@ public final class InteractiveUtils {
|
|||
return new Rule("$Action", Lists.newArrayList("$Action", "$Action"), b);
|
||||
}
|
||||
public static Derivation combine(List<Derivation> children, ActionFormula.Mode mode) {
|
||||
// stop double blocking
|
||||
if (children.size() == 1) {
|
||||
ActionFormula.Mode cmode = ((ActionFormula)children.get(0).formula).mode;
|
||||
if (cmode == mode) {
|
||||
return children.get(0);
|
||||
}
|
||||
}
|
||||
Formula f = new ActionFormula(mode,
|
||||
children.stream().map(d -> d.formula).collect(Collectors.toList()));
|
||||
Derivation res = new Derivation.Builder()
|
||||
|
|
|
|||
|
|
@ -8,7 +8,10 @@ import java.util.Map;
|
|||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.testng.collections.Sets;
|
||||
|
||||
import com.beust.jcommander.internal.Lists;
|
||||
import com.google.common.base.Function;
|
||||
|
||||
import edu.stanford.nlp.sempre.ConstantFn;
|
||||
import edu.stanford.nlp.sempre.Derivation;
|
||||
|
|
@ -57,23 +60,28 @@ public class GrammarInducer {
|
|||
id = origEx.id;
|
||||
head = origEx;
|
||||
head.predDerivations = Lists.newArrayList(def);
|
||||
LogInfo.logs("chartList.size = %d", chartList.size());
|
||||
|
||||
tokens = origEx.getTokens();
|
||||
int numTokens = origEx.numTokens();
|
||||
|
||||
addMatches(def, makeChartMap(chartList));
|
||||
List<Derivation> bestPacking = bestPackingDP(this.matches, numTokens);
|
||||
LogInfo.logs("BestPacking: %s", bestPacking);
|
||||
|
||||
HashMap<String, String> formulaToCat = new HashMap<>();
|
||||
bestPacking.forEach(d -> formulaToCat.put(catFormulaKey(d), varName(d)));
|
||||
|
||||
LogInfo.logs("chartList.size = %d", chartList.size());
|
||||
LogInfo.logs("BestPacking: %s", bestPacking);
|
||||
LogInfo.logs("Matches: %s", this.matches);
|
||||
LogInfo.logs("formulaToCat: %s", formulaToCat);
|
||||
|
||||
buildFormula(def, formulaToCat);
|
||||
def.grammarInfo.start = 0;
|
||||
def.grammarInfo.end = tokens.size();
|
||||
|
||||
inducedRules = new ArrayList<>(induceRules(bestPacking, def));
|
||||
|
||||
|
||||
}
|
||||
|
||||
private Map<String, List<Derivation>> makeChartMap(List<Derivation> chartList) {
|
||||
|
|
@ -90,16 +98,18 @@ public class GrammarInducer {
|
|||
// this is used to test for matches, same cat, same formula
|
||||
// maybe cat needs to be more flexible
|
||||
private String catFormulaKey(Derivation d) {
|
||||
// return d.formula.toString();
|
||||
return getNormalCat(d) + "::" + d.formula.toString();
|
||||
}
|
||||
private String varName(Derivation anchored) {
|
||||
return getNormalCat(anchored) + anchored.start + "_" + anchored.end;
|
||||
}
|
||||
private String getNormalCat(Derivation def) {
|
||||
return def.getCat();
|
||||
// if (cat.endsWith("s"))
|
||||
// return cat.substring(0, cat.length()-1);
|
||||
// else return cat;
|
||||
// return def.cat;
|
||||
String cat = def.getCat();
|
||||
if (cat.endsWith("s"))
|
||||
return cat.substring(0, cat.length()-1);
|
||||
else return cat;
|
||||
}
|
||||
|
||||
//label the derivation tree with what it matches in chartList
|
||||
|
|
@ -176,7 +186,7 @@ public class GrammarInducer {
|
|||
}
|
||||
|
||||
private List<Rule> induceRules(List<Derivation> packings, Derivation defDeriv) {
|
||||
LogInfo.dbgs("packings: %s", packings);
|
||||
|
||||
List<Rule> inducedRules = new ArrayList<>();
|
||||
List<String> RHS = getRHS(defDeriv, packings);
|
||||
SemanticFn sem = getSemantics(defDeriv, packings);
|
||||
|
|
@ -195,9 +205,9 @@ public class GrammarInducer {
|
|||
|
||||
// populate grammarInfo.formula, replacing everything that can be replaced
|
||||
private void buildFormula(Derivation deriv, Map<String, String> replaceMap){
|
||||
//LogInfo.log("building " + deriv.grammarInfo.formula);
|
||||
if (deriv.grammarInfo.formula != null) return;
|
||||
// LogInfo.logs("BUILDING %s at (%d,%d) %s", deriv, deriv.start, deriv.end, catFormulaKey(deriv));
|
||||
if (replaceMap.containsKey(catFormulaKey(deriv))) {
|
||||
// LogInfo.logs("Found match %s, %s, %s", catFormulaKey(deriv), replaceMap, deriv);
|
||||
deriv.grammarInfo.formula = new VariableFormula(replaceMap.get(catFormulaKey(deriv)));
|
||||
return;
|
||||
}
|
||||
|
|
@ -219,7 +229,10 @@ public class GrammarInducer {
|
|||
for (Derivation arg : args) {
|
||||
if (!(f instanceof LambdaFormula))
|
||||
throw new RuntimeException("Expected LambdaFormula, but got " + f);
|
||||
f = Formulas.lambdaApply((LambdaFormula)f, arg.grammarInfo.formula);
|
||||
Formula after = renameBoundVars(f, new HashSet<>());
|
||||
// LogInfo.logs("renameBoundVar %s === %s", after, f);
|
||||
|
||||
f = Formulas.lambdaApply((LambdaFormula)after, arg.grammarInfo.formula);
|
||||
}
|
||||
deriv.grammarInfo.formula = f;
|
||||
} else if (rule.sem instanceof IdentityFn) {
|
||||
|
|
@ -230,10 +243,33 @@ public class GrammarInducer {
|
|||
} else {
|
||||
deriv.grammarInfo.formula = deriv.formula;
|
||||
}
|
||||
|
||||
//LogInfo.log("built " + deriv.grammarInfo.formula);
|
||||
|
||||
// LogInfo.logs("BUILT %s for %s", deriv.grammarInfo.formula, deriv.formula);
|
||||
// LogInfo.log("built " + deriv.grammarInfo.formula);
|
||||
}
|
||||
|
||||
private String newName(String s) {return s.endsWith("_")? s : s + "_";}
|
||||
private Formula renameBoundVars(Formula formula, Set<String> boundvars) {
|
||||
if (formula instanceof LambdaFormula) {
|
||||
LambdaFormula f = (LambdaFormula)formula;
|
||||
boundvars.add(f.var);
|
||||
return new LambdaFormula(newName(f.var), renameBoundVars(f.body, boundvars));
|
||||
} else {
|
||||
Formula after = formula.map(
|
||||
new Function<Formula, Formula>() {
|
||||
public Formula apply(Formula formula) {
|
||||
if (formula instanceof VariableFormula) { // Replace variable
|
||||
String name = ((VariableFormula) formula).name;
|
||||
if (boundvars.contains(name)) return new VariableFormula(newName(name));
|
||||
else return formula;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
return after;
|
||||
}
|
||||
}
|
||||
|
||||
private SemanticFn getSemantics(final Derivation def, List<Derivation> packings) {
|
||||
Formula baseFormula = def.grammarInfo.formula;
|
||||
if (packings.size() == 0) {
|
||||
|
|
@ -260,7 +296,7 @@ public class GrammarInducer {
|
|||
List<String> rhs = new ArrayList<>(tokens);
|
||||
for (Derivation deriv : packings) {
|
||||
// LogInfo.logs("got (%d,%d):%s:%s", deriv.start, deriv.end, deriv.formula, deriv.cat);
|
||||
rhs.set(deriv.start, deriv.cat);
|
||||
rhs.set(deriv.start, getNormalCat(deriv));
|
||||
for (int i = deriv.start + 1; i < deriv.end; i++) {
|
||||
rhs.set(i, null);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,14 +22,19 @@ import org.testng.collections.Lists;
|
|||
* @author Sida Wang
|
||||
*/
|
||||
public class GrammarInducerTest {
|
||||
static Assertion A = new Assertion();
|
||||
// static Assertion A = new SoftAssert();
|
||||
Assertion hard = new Assertion();
|
||||
Assertion soft = new SoftAssert();
|
||||
|
||||
private static Spec defaultSpec() {
|
||||
FloatingParser.opts.defaultIsFloating = true;
|
||||
ActionExecutor.opts.convertNumberValues = true;
|
||||
ActionExecutor.opts.printStackTrace = true;
|
||||
ActionExecutor.opts.FlatWorldType = "BlocksWorld";
|
||||
|
||||
Derivation.opts.showTypes = false;
|
||||
Derivation.opts.showRules = false;
|
||||
Derivation.opts.showCat = true;
|
||||
|
||||
LanguageAnalyzer.opts.languageAnalyzer = "interactive.actions.ActionLanguageAnalyzer";
|
||||
Grammar.opts.inPaths = Lists.newArrayList("./shrdlurn/blocksworld.grammar");
|
||||
Grammar.opts.useApplyFn = "interactive.ApplyFn";
|
||||
|
|
@ -68,7 +73,7 @@ public class GrammarInducerTest {
|
|||
induced.forEach(r -> InteractiveUtils.addRuleInteractive(r, parser));
|
||||
LogInfo.logs("Defining %s := %s, added %s", head, def, induced);
|
||||
}
|
||||
public boolean found(String head, String def) {
|
||||
public boolean match(String head, String def) {
|
||||
Example.Builder b = new Example.Builder();
|
||||
b.setUtterance(head);
|
||||
Example exHead = b.createExample();
|
||||
|
|
@ -89,6 +94,10 @@ public class GrammarInducerTest {
|
|||
ind++;
|
||||
}
|
||||
printAllRules();
|
||||
if (!found) {
|
||||
LogInfo.logs("Did not find %s", defDeriv.formula);
|
||||
LogInfo.log(exHead.predDerivations);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
|
|
@ -102,56 +111,129 @@ public class GrammarInducerTest {
|
|||
@Test public void simpleTest() {
|
||||
LogInfo.begin_track("test simple substitutions");
|
||||
ParseTester T = new ParseTester();
|
||||
Assertion A = hard;
|
||||
|
||||
T.def("add red twice", d("add red top","add red top"));
|
||||
A.assertTrue(T.found("add blue twice", d("add blue top","add blue top")));
|
||||
A.assertTrue(T.match("add blue twice", d("add blue top","add blue top")));
|
||||
|
||||
T.def("add red 3 times", d("repeat 3 [add red top]"));
|
||||
A.assertTrue(T.found("add yellow 5 times", d("repeat 5 [add yellow top]")));
|
||||
T.def("add red 3 times", d("repeat 3 [add red]"));
|
||||
A.assertTrue(T.match("add yellow 5 times", d("repeat 5 [add yellow]")));
|
||||
|
||||
T.def("add red 3 ^ 2 times", d("repeat 3 {[repeat 3 [add red]]}"));
|
||||
A.assertTrue(T.match("add blue 2 ^ 2 times", d("repeat 2 {[repeat 2 [add blue]]}")));
|
||||
|
||||
T.def("add a red block to the left", d("add red left"));
|
||||
A.assertTrue(T.found("add a yellow block to the right", d("add yellow right")));
|
||||
A.assertTrue(T.match("add a yellow block to the right", d("add yellow right")));
|
||||
|
||||
T.def("remove the leftmost red block", d("remove very left of has color red"));
|
||||
A.assertTrue(T.found("remove the leftmost yellow block", d("remove very left of has color yellow")));
|
||||
A.assertTrue(T.match("remove the leftmost yellow block", d("remove very left of has color yellow")));
|
||||
|
||||
T.def("add red to both sides", d("add red left", "add red right"));
|
||||
T.def("add red to both sides", d("add red back", "add red front"));
|
||||
A.assertTrue(T.found("add green to both sides", d("add green left", "add green right")));
|
||||
A.assertTrue(T.found("add green to both sides", d("add green back", "add green front")));
|
||||
A.assertFalse(T.found("add green to both sides", d("add green left", "add green back")));
|
||||
A.assertTrue(T.match("add green to both sides", d("add green left", "add green right")));
|
||||
A.assertTrue(T.match("add green to both sides", d("add green back", "add green front")));
|
||||
A.assertFalse(T.match("add green to both sides", d("add green left", "add green back")));
|
||||
|
||||
T.def("select all but yellow", d("select not has color yellow"));
|
||||
A.assertTrue(T.found("select all but yellow", d("select not has color yellow")));
|
||||
A.assertTrue(T.match("select all but yellow", d("select not has color yellow")));
|
||||
|
||||
T.def("add a yellow block on top of red blocks", d("select has color red", "add yellow top"));
|
||||
A.assertTrue(T.found("add a green block on top of blue blocks", d("select has color blue", "add green top")));
|
||||
A.assertTrue(T.match("add a green block on top of blue blocks", d("select has color blue", "add green top")));
|
||||
|
||||
T.def("add a yellow block on top of red blocks", d("select has color red; add yellow top"));
|
||||
A.assertTrue(T.found("add a green block on top of blue blocks", d("select has color blue ; add green top")));
|
||||
A.assertTrue(T.match("add a green block on top of blue blocks", d("select has color blue ; add green top")));
|
||||
|
||||
//T.printAllRules();
|
||||
//A.assertAll();
|
||||
|
||||
LogInfo.end_track();
|
||||
}
|
||||
//
|
||||
// @Test public void basicTest() {
|
||||
// LogInfo.begin_track("test Grammar");
|
||||
// induce("add red twice","[[\"add red top\",\"(: add red top)\"],[\"add red top\",\"(: add red top)\"]]");
|
||||
// induce("add red thrice", "[[\"[add red top] 3 times\",\"(:loop (number 3) (: add red top))\"]]");
|
||||
// induce("add a red block to the left", "[[\"add red left\",\"(: add red left)\"]]");
|
||||
// induce("remove card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
// induce("delete card", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
// induce("remove the leftmost red block", "[[\"remove very left of has color red\",\"?\"]]");
|
||||
// induce("add red to both sides", "[[\"add red left\",\"(: add red left)\"],[\"add red right\",\"(: add red right)\"]]");
|
||||
// induce("add a yellow block next to brown",
|
||||
// "[[\"select brown\",\"(:foreach (color brown) (: select))\"],[\"add yellow right\",\"(: add yellow right)\"]]");
|
||||
// induce("select all but yellow",
|
||||
// "[[\"select not has color yellow\",\"?\"]]");
|
||||
// LogInfo.end_track();
|
||||
// }
|
||||
//
|
||||
|
||||
@Test public void actionTest() {
|
||||
LogInfo.begin_track("test some action substitutions");
|
||||
ParseTester T = new ParseTester();
|
||||
Assertion A = soft;
|
||||
|
||||
T.def("add red twice", d("repeat 2 [add red]"));
|
||||
A.assertTrue(T.match("add blue top twice", d("repeat 2 [add blue top]")));
|
||||
|
||||
T.def("add red 3 times", d("repeat 3 [add red]"));
|
||||
A.assertTrue(T.match("add yellow top 5 times", d("repeat 5 [add yellow top]")));
|
||||
|
||||
T.def("add red twice", d("add red; add red top"));
|
||||
A.assertTrue(T.match("add yellow twice", d("add yellow; add yellow top")));
|
||||
|
||||
T.def("add red then add yellow left", d("add red; add yellow left"));
|
||||
A.assertTrue(T.match("remove red then add brown top", d("remove red; add brown top")));
|
||||
|
||||
//T.printAllRules();
|
||||
//A.assertAll();
|
||||
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
@Test public void learnCatTest() {
|
||||
LogInfo.begin_track("test the learning via alignment");
|
||||
ParseTester T = new ParseTester();
|
||||
Assertion A = soft;
|
||||
|
||||
T.def("add cardinal", d("add red"));
|
||||
A.assertTrue(T.match("select has color cardinal", d("select has color red")));
|
||||
|
||||
T.def("move up", d("move top"));
|
||||
A.assertTrue(T.match("select up", d("select top")));
|
||||
A.assertTrue(T.match("add red up", d("add red top")));
|
||||
|
||||
T.def("select the highest of has color red", d("select very top of has color red"));
|
||||
A.assertTrue(T.match("remove the highest of all", d("remove very top of all")));
|
||||
|
||||
T.def("select the top most of has color red", d("select very top of has color red"));
|
||||
A.assertTrue(T.match("select the bot most of all", d("select the very bot most of all")));
|
||||
|
||||
//T.printAllRules();
|
||||
//A.assertAll();
|
||||
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
@Test public void cubeTest() {
|
||||
LogInfo.begin_track("cubeTest");
|
||||
ParseTester T = new ParseTester();
|
||||
Assertion A = hard;
|
||||
|
||||
T.def("red stick of size 3", d("{repeat 3 [add red; select top]}"));
|
||||
T.def("red plate of size 3", d("{repeat 3 [red stick of size 3; select left]}"));
|
||||
T.def("red cube of size 3", d("{repeat 3 [red plate of size 3; select back]}"));
|
||||
|
||||
A.assertTrue(T.match("blue stick of size 4", d("{repeat 4 [add blue; select top]}")));
|
||||
A.assertTrue(T.match("blue plate of size 4", d("{repeat 4 [blue stick of size 4; select left]}")));
|
||||
A.assertTrue(T.match("blue cube of size 5", d("{repeat 5 [blue plate of size 5; select back]}")));
|
||||
|
||||
//T.printAllRules();
|
||||
//A.assertAll();
|
||||
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
@Test public void rectTest() {
|
||||
LogInfo.begin_track("rectTest");
|
||||
ParseTester T = new ParseTester();
|
||||
Assertion A = hard;
|
||||
|
||||
T.def("red stick of size 4", d("{repeat 4 [add red; select top]}"));
|
||||
T.def("red plate of size 3 by 4", d("{repeat 3 [red stick of size 4; select left]}"));
|
||||
T.def("red cube of size 2 by 3 by 4", d("{repeat 2 [red plate of size 3 by 4; select back]}"));
|
||||
|
||||
A.assertTrue(T.match("red plate of size 1 by 2", d("{repeat 1 [red stick of size 2; select left]}")));
|
||||
A.assertTrue(T.match("red cube of size 1 by 2 by 3", d("{repeat 1 [red plate of size 2 by 3; select back]}")));
|
||||
|
||||
//T.printAllRules();
|
||||
//A.assertAll();
|
||||
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
|
||||
// @Test public void learnSets() {
|
||||
// LogInfo.begin_track("test Grammar");
|
||||
// induce("remove those red blocks", "[[\"remove has color red\",\"(:foreach (color red) (: remove))\"]]");
|
||||
|
|
|
|||
Loading…
Reference in New Issue