natas_hw
/
sempre
mirror of https://github.com/percyliang/sempre
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

server updates

This commit is contained in:
Sida Wang 2017-01-03 16:10:26 -08:00
parent 9ba813dd75
commit f3a8f2be0e
15 changed files with 103 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -3,7 +3,7 @@ fig
sfig
refdb
virtuoso-opensource
shrdlurn_output
classes
libsempre
@ -13,6 +13,7 @@ module-classes.txt
state
out
test-output
int-output*
.project
.classpath

12
run
View File

@ -1015,12 +1015,12 @@ o('FeatureExtractor.featureDomains', 'rule', 'bff', 'stats'),
o('RicherStacksWorldFeatureComputer.ngramN', 2),
o('RicherStacksWorldFeatureComputer.parameterizeCats', 'Number','Color'),
lambda { |e| system 'mkdir -p ./shrdlurn_output/examples/'},
lambda { |e| system 'mkdir -p ./shrdlurn_output/logs/'},
lambda { |e| system 'mkdir -p ./shrdlurn_output/grammar/'},
o('Master.newExamplesPath', './shrdlurn_output/examples/'),
o('Master.logPath', './shrdlurn_output/logs/'),
o('Master.newGrammarPath', './shrdlurn_output/grammar/'),
lambda { |e| system 'mkdir -p ./int-output/examples/'},
lambda { |e| system 'mkdir -p ./int-output/logs/'},
lambda { |e| system 'mkdir -p ./int-output/grammar/'},
o('Master.newExamplesPath', './int-output/examples/'),
o('Master.logPath', './int-output/logs/'),
o('Master.newGrammarPath', './int-output/grammar/'),
o('Master.onlineLearnExamples', true),
o('Master.independentSessions', false),

4
shrdlurn/flatworld.grammar
View File

@ -69,8 +69,8 @@
############### arithmetics
(for @op (+ -)
(rule $Numbers ($Numbers @op $Number) (lambda n1 (lambda op (lambda n2
(@op (var n1) (var n2))))) (anchored 1))
(rule $Numbers ($Numbers @op $Number) (lambda n1 (lambda n2
(@op (var n1) (var n2)))) (anchored 1))
)
(rule $Number ([ $Numbers ]) (IdentityFn) (anchored 1))

1
src/edu/stanford/nlp/sempre/Derivation.java
View File

@ -337,7 +337,6 @@ public class Derivation implements SemanticFn.Callable, HasScore {
tree.addChild(values.size() + " values");
}
}
}
if (type != null && opts.showTypes)
tree.addChild(LispTree.proto.newList("type", type.toLispTree()));

12
src/edu/stanford/nlp/sempre/Example.java
View File

@ -265,12 +265,6 @@ public class Example {
tree.addChild(LispTree.proto.newList("id", id));
if (context != null)
tree.addChild(context.toLispTree());
// interactive stuff
tree.addChild(LispTree.proto.newList("timeStamp", LocalDateTime.now().toString()));
tree.addChild(LispTree.proto.newList("NBestInd", Integer.toString(NBestInd)));
if (definition != null)
tree.addChild(LispTree.proto.newList("definition", definition));
if (utterance != null)
tree.addChild(LispTree.proto.newList("utterance", utterance));
@ -299,6 +293,12 @@ public class Example {
tree.addChild(list);
}
// interactive stuff
if (definition != null)
tree.addChild(LispTree.proto.newList("definition", definition));
tree.addChild(LispTree.proto.newList("timeStamp", LocalDateTime.now().toString()));
tree.addChild(LispTree.proto.newList("NBestInd", Integer.toString(NBestInd)));
return tree;
}

85
src/edu/stanford/nlp/sempre/InteractiveUtils.java
View File

@ -1,5 +1,7 @@
package edu.stanford.nlp.sempre;
import java.io.PrintWriter;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@ -11,6 +13,8 @@ import com.google.common.collect.ImmutableList;
import edu.stanford.nlp.sempre.interactive.BlockFn;
import edu.stanford.nlp.sempre.interactive.GrammarInducer;
import edu.stanford.nlp.sempre.interactive.actions.ActionFormula;
import fig.basic.IOUtils;
import fig.basic.LispTree;
import fig.basic.LogInfo;
/**
@ -27,9 +31,15 @@ public final class InteractiveUtils {
return deriv;
}
public static GrammarInducer getInducer(String head, String jsonDef, String sessionId, Parser parser, Params params) {
return getInducer(head, jsonDef, sessionId, parser, params, ActionFormula.Mode.block);
}
// parse the definition, match with the chart of origEx, and add new rules to grammar
public static GrammarInducer getInducer(String head, List<Object> body, String sessionId, Parser parser, Params params) {
public static GrammarInducer getInducer(String head, String jsonDef, String sessionId, Parser parser, Params params, ActionFormula.Mode mode) {
logRawDef(head, jsonDef, sessionId);
@SuppressWarnings("unchecked")
List<Object> body = Json.readValueHard(jsonDef, List.class);
// string together the body definition
List<Derivation> allDerivs = new ArrayList<>();
@ -39,59 +49,80 @@ public final class InteractiveUtils {
String utt = pair.get(0);
String formula = pair.get(1);
if (formula.equals("()")) {
LogInfo.error("Got empty formula");
continue;
}
Example.Builder b = new Example.Builder();
b.setId("session:" + sessionId);
b.setUtterance(utt);
Example ex = b.createExample();
ex.preprocess();
LogInfo.logs("Parsing definition: %s", ex.utterance);
// LogInfo.logs("Parsing definition: %s", ex.utterance);
parser.parse(params, ex, false);
boolean found = false;
for (Derivation d : ex.predDerivations) {
LogInfo.logs("considering: %s", d.formula.toString());
// LogInfo.logs("considering: %s", d.formula.toString());
if (d.formula.toString().equals(formula)) {
found = true;
allDerivs.add(stripDerivation(d));
}
}
if (!found) LogInfo.errors("Definition fails, matching formula not found: %s", formula);
if (!found) LogInfo.errors("Matching formula not found: %s", formula);
// just some hacks to make testing easier, use top derivation when we formula is not given
if (!found && (formula.equals("?") || formula==null) && ex.predDerivations.size() > 0)
allDerivs.add(stripDerivation(ex.predDerivations.get(0)));
}
Derivation bodyDeriv;
if (allDerivs.size() > 1)
bodyDeriv = combineList(allDerivs, ActionFormula.Mode.block);
else
bodyDeriv = allDerivs.get(0);
// if (allDerivs.size() > 1)
// bodyDeriv = combineList(allDerivs, ActionFormula.Mode.block);
// else
// bodyDeriv = allDerivs.get(0);
//
bodyDeriv = combineList(allDerivs, mode);
Example.Builder b = new Example.Builder();
b.setId("session:" + sessionId);
b.setUtterance(head);
Example exHead = b.createExample();
exHead.preprocess();
LogInfo.logs("Parsing head: %s", exHead.utterance);
LogInfo.begin_track("Definition");
LogInfo.logs("mode: %s", mode);
LogInfo.logs("head: %s", exHead.utterance);
BeamFloatingParserState state = (BeamFloatingParserState)parser.parse(params, exHead, false);
LogInfo.logs("target deriv: %s", bodyDeriv.toLispTree());
LogInfo.logs("anchored elements: %s", state.chartList);
LogInfo.logs("defderiv: %s", bodyDeriv.toLispTree());
LogInfo.logs("anchored: %s", state.chartList);
GrammarInducer grammarInducer = new GrammarInducer(exHead, bodyDeriv, state.chartList);
// updates the value that the derivation is modified.
// for (Derivation d : grammarInducer.getHead().predDerivations) {
// d.value = null; // cuz ensureExecuted checks.
// d.ensureExecuted(parser.executor, exHead.context);
// }
// parser.parse(params, exHead, false);
LogInfo.end_track();;
return grammarInducer;
// PrintWriter out = IOUtils.openOutAppendHard(Paths.get(opts.newGrammarPath, sessionId + ".definition").toString());
// // deftree.addChild(oldEx.utterance);
// LispTree deftree = LispTree.proto.newList("definition", body.toString());
// Example oldEx = new Example.Builder()
// .setId(exHead.id)
// .setUtterance(exHead.utterance)
// .setTargetFormula(bodyDeriv.formula)
// .createExample();
// LispTree treewithdef = oldEx.toLispTree(false).addChild(deftree);
// out.println(treewithdef.toString());
// out.flush();
// out.close();
}
public static void logRawDef(String utt, String jsonDef, String sessionId) {
PrintWriter out = IOUtils.openOutAppendHard(Paths.get(Master.opts.newGrammarPath, sessionId + ".def").toString());
// deftree.addChild(oldEx.utterance);
LispTree deftree = LispTree.proto.newList(":def", utt);
deftree.addChild(jsonDef);
Example oldEx = new Example.Builder()
.setId(sessionId)
.setUtterance(utt)
.createExample();
LispTree treewithdef = oldEx.toLispTree(false).addChild(deftree);
out.println(treewithdef.toString());
out.flush();
out.close();
}
public static synchronized void addRuleInteractive(Rule rule, Parser parser) {
@ -103,15 +134,15 @@ public final class InteractiveUtils {
}
}
static Rule blockRule() {
return new Rule("$Action", Lists.newArrayList("$Action", "$Action"), new BlockFn());
static Rule blockRule(ActionFormula.Mode mode) {
return new Rule("$Action", Lists.newArrayList("$Action", "$Action"), new BlockFn(mode));
}
static Derivation combineList(List<Derivation> children, ActionFormula.Mode mode) {
Formula f = new ActionFormula(mode,
children.stream().map(d -> d.formula).collect(Collectors.toList()));
Derivation res = new Derivation.Builder()
.formula(f)
.withCallable(new SemanticFn.CallInfo("$Action", -1, -1, blockRule(), ImmutableList.copyOf(children)))
.withCallable(new SemanticFn.CallInfo("$Action", -1, -1, blockRule(mode), ImmutableList.copyOf(children)))
.createDerivation();
return res;
}

13
src/edu/stanford/nlp/sempre/Master.java
View File

@ -8,6 +8,7 @@ import com.google.common.collect.Lists;
import edu.stanford.nlp.sempre.interactive.ApplyFn;
import edu.stanford.nlp.sempre.interactive.GrammarInducer;
import edu.stanford.nlp.sempre.interactive.PrefixTrie;
import edu.stanford.nlp.sempre.interactive.actions.ActionFormula;
import fig.basic.*;
import fig.exec.Execution;
@ -441,13 +442,17 @@ public class Master {
} else {
session.context = new ContextValue(tree);
}
} else if (command.equals(":def")) {
} else if (command.equals(":def") || command.equals(":def_ret")) {
if (tree.children.size() == 3) {
String head = tree.children.get(1).value;
List<Object> body = Json.readValueHard(tree.children.get(2).value, List.class);
GrammarInducer inducer = InteractiveUtils.getInducer(head, body, session.id, builder.parser, builder.params);
ActionFormula.Mode blockmode = command.equals(":def")? ActionFormula.Mode.block : ActionFormula.Mode.blockr;
GrammarInducer inducer = InteractiveUtils.getInducer(head, tree.children.get(2).value, session.id, builder.parser, builder.params, blockmode);
List<Rule> inducedRules = inducer.getRules();
response.ex = inducer.getHead();
response.candidateIndex = response.ex.predDerivations.size() > 0 ? 0 : -1;
if (inducedRules.size() > 0) {
for (Rule rule : inducedRules) {
@ -462,7 +467,7 @@ public class Master {
out.close();
}
} else {
LogInfo.logs("Invalid format for uttdef");
LogInfo.logs("Invalid format for def");
}
} else if (command.equals(":accept")) {

14
src/edu/stanford/nlp/sempre/Parser.java
View File

@ -177,18 +177,8 @@ public abstract class Parser {
ex.predDerivations = state.predDerivations;
if (Master.opts.bePragmatic) {
// Compute probabilities
double[] probs = Derivation.getProbs(ex.predDerivations, 1);
for (int i = 0; i < ex.predDerivations.size(); i++) {
Derivation deriv = ex.predDerivations.get(i);
deriv.prob = probs[i];
}
params.pragmaticListener.inferPragmatically(ex);
Derivation.sortByPragmaticScore(ex.predDerivations);
} else {
Derivation.sortByScore(ex.predDerivations);
}
Derivation.sortByScore(ex.predDerivations);
// Evaluate
ex.evaluation = new Evaluation();
addToEvaluation(state, ex.evaluation);

2
src/edu/stanford/nlp/sempre/ParserState.java
View File

@ -230,7 +230,7 @@ public abstract class ParserState {
}
// Ensure that all the logical forms are executed and compatibilities are computed.
protected void ensureExecuted() {
public void ensureExecuted() {
LogInfo.begin_track("Parser.ensureExecuted");
// Execute predicted derivations to get value.
for (Derivation deriv : predDerivations) {

16
src/edu/stanford/nlp/sempre/interactive/BlockFn.java
View File

@ -1,17 +1,16 @@
package edu.stanford.nlp.sempre.interactive;
import java.util.*;
import java.util.List;
import java.util.stream.Collectors;
import org.testng.collections.Lists;
import com.google.common.collect.ImmutableList;
import edu.stanford.nlp.sempre.*;
import edu.stanford.nlp.sempre.SemanticFn.Callable;
import edu.stanford.nlp.sempre.Derivation;
import edu.stanford.nlp.sempre.DerivationStream;
import edu.stanford.nlp.sempre.Example;
import edu.stanford.nlp.sempre.Formula;
import edu.stanford.nlp.sempre.SemanticFn;
import edu.stanford.nlp.sempre.SingleDerivationStream;
import edu.stanford.nlp.sempre.interactive.actions.ActionFormula;
import fig.basic.LispTree;
import fig.basic.MapUtils;
import fig.basic.Option;
/**
@ -34,6 +33,7 @@ public class BlockFn extends SemanticFn {
}
public BlockFn() {}
public BlockFn(ActionFormula.Mode mode) { this.mode = mode;}
public DerivationStream call(final Example ex, final Callable c) {
return new SingleDerivationStream() {

1
src/edu/stanford/nlp/sempre/interactive/GrammarInducer.java
View File

@ -18,6 +18,7 @@ import edu.stanford.nlp.sempre.Formula;
import edu.stanford.nlp.sempre.Formulas;
import edu.stanford.nlp.sempre.IdentityFn;
import edu.stanford.nlp.sempre.LambdaFormula;
import edu.stanford.nlp.sempre.Master;
import edu.stanford.nlp.sempre.Rule;
import edu.stanford.nlp.sempre.SemanticFn;
import edu.stanford.nlp.sempre.VariableFormula;

4
src/edu/stanford/nlp/sempre/interactive/actions/ActionExecutor.java
View File

@ -243,8 +243,8 @@ public class ActionExecutor extends Executor {
if (formula instanceof ArithmeticFormula) {
ArithmeticFormula arithmeticFormula = (ArithmeticFormula)formula;
Double arg1 = (Double)processSetFormula(arithmeticFormula.child1, world);
Double arg2 = (Double)processSetFormula(arithmeticFormula.child2, world);
Integer arg1 = (Integer)processSetFormula(arithmeticFormula.child1, world);
Integer arg2 = (Integer)processSetFormula(arithmeticFormula.child2, world);
ArithmeticFormula.Mode mode = arithmeticFormula.mode;
if (mode == ArithmeticFormula.Mode.add)
return arg1 + arg2;

2
src/edu/stanford/nlp/sempre/interactive/actions/ActionLanguageAnalyzer.java
View File

@ -56,7 +56,7 @@ public class ActionLanguageAnalyzer extends LanguageAnalyzer {
else if (c == '>' || c == '<')
separate = (utterance.charAt(i + 1) != '=' && utterance.charAt(i + 1) != '=');
else
separate = (",?'\"[];{}".indexOf(c) != -1);
separate = (",?'\"[];{}+-".indexOf(c) != -1);
if (separate) buf.append(' ');
// Convert quotes

4
src/edu/stanford/nlp/sempre/interactive/actions/BeamFloatingFeatureComputer.java
View File

@ -87,7 +87,9 @@ public class BeamFloatingFeatureComputer implements FeatureComputer {
NotFormula notFormula = (NotFormula) f;
return "(not " + getSubtreeFeature(notFormula.child, depth-1, anchored) + ")";
} else {
throw new RuntimeException("Unhandled formula type: " + f.getClass().toString());
// throw new RuntimeException("Unhandled formula type: " + f.getClass().toString());
LogInfo.warning("Unhandled formula type: " + f.getClass().toString());
return f.toString();
}
}

5
src/edu/stanford/nlp/sempre/interactive/actions/GrammarInducerTest.java
View File

@ -40,7 +40,7 @@ public class GrammarInducerTest {
return new Parser.Spec(grammar, extractor, executor, valueEvaluator);
}
protected static void induceHelper(String head, List<Object> body) {
protected static void induceHelper(String head, String body) {
Spec defSpec = defaultSpec();
Parser parser = new BeamFloatingParser(defSpec);
@ -50,8 +50,7 @@ public class GrammarInducerTest {
}
protected static void induce(String head, String jsonDef) {
List<Object> body = Json.readValueHard(jsonDef, List.class);
induceHelper(head, body);
induceHelper(head, jsonDef);
}
@Test public void basicTest() {