diff --git a/run b/run index d0bc5f4..c1244d8 100755 --- a/run +++ b/run @@ -1012,8 +1012,8 @@ o('Parser.beamSize', 10), o('Params.initStepSize', 0.1), o('Params.adaptiveStepSize', true), -o('FeatureExtractor.featureComputers', 'interactive.actions.BeamFloatingFeatureComputer'), -o('FeatureExtractor.featureDomains', 'rule', 'bff', 'stats'), +o('FeatureExtractor.featureComputers', 'interactive.actions.ActionFeatureComputer'), +o('FeatureExtractor.featureDomains', 'rule', 'window', 'stats'), o('RicherStacksWorldFeatureComputer.ngramN', 2), o('RicherStacksWorldFeatureComputer.parameterizeCats', 'Number','Color'), diff --git a/src/edu/stanford/nlp/sempre/interactive/GrammarInducer.java b/src/edu/stanford/nlp/sempre/interactive/GrammarInducer.java index c9551bc..a15c9a2 100644 --- a/src/edu/stanford/nlp/sempre/interactive/GrammarInducer.java +++ b/src/edu/stanford/nlp/sempre/interactive/GrammarInducer.java @@ -175,6 +175,7 @@ public class GrammarInducer { bestpacking.add(bestDeriv); Packing newPack = new Packing(bestscore, bestpacking); bestEndsAtI.add(newPack); + LogInfo.logs("Adding Packing %d: %s", newPack.score, newPack.packing); } } diff --git a/src/edu/stanford/nlp/sempre/interactive/actions/ActionFeatureComputer.java b/src/edu/stanford/nlp/sempre/interactive/actions/ActionFeatureComputer.java new file mode 100644 index 0000000..10666ed --- /dev/null +++ b/src/edu/stanford/nlp/sempre/interactive/actions/ActionFeatureComputer.java @@ -0,0 +1,106 @@ +package edu.stanford.nlp.sempre.interactive.actions; + +import fig.basic.*; +import java.util.*; + +import com.beust.jcommander.internal.Lists; +import edu.stanford.nlp.sempre.*; + +/** + * Sida Wang + * Feature computer for the BeamFloatingParser + * TODOs: + * - control what categories to abstract out + * - efficiency improvement, right now use all members of the cross product + */ +public class ActionFeatureComputer implements FeatureComputer { + public static class Options { + @Option(gloss = "Verbosity") + public int verbose = 0; + + @Option(gloss = "the N in N-gram") + public int ngramN = 3; + + @Option(gloss = "size of the context window to consider") + public int windowSize = 2; + } + public static Options opts = new Options(); + + @Override public void extractLocal(Example ex, Derivation deriv) { + addStatsFeatures(ex, deriv); + addWindowFeatures(ex, deriv); + } + + + // function to abstract out ALL anchored stuff in the utterance. + private List abstractAnchors(Derivation deriv, List tokens) { + if (deriv.start == -1) + return tokens; + List newTokens = new ArrayList<>(); + int startInd = Math.max(0, deriv.start - opts.windowSize); + int endInd = Math.min(tokens.size(), deriv.end + opts.windowSize); + newTokens.addAll(tokens.subList(startInd, deriv.start)); + newTokens.add(deriv.cat); + newTokens.addAll(tokens.subList(deriv.end, endInd)); + return newTokens; + } + + + private List getAllNgrams(List tokens, int n, Derivation deriv) { + List ngrams = new ArrayList<>(); + List paddedTokens = new ArrayList<>(); + if (deriv.start == -1) // floating, just add everything + paddedTokens.addAll(tokens); + else { + paddedTokens.addAll(tokens.subList(Math.max(0, deriv.start-n+1), Math.min(tokens.size(), deriv.end+n-1))); + } + + for (int i=0; i current = new ArrayList<>(paddedTokens.subList(i, i+n)); + ngrams.add( current.toString() ); + } + return ngrams; + } + + private List getAllSkipGrams(List tokens, Derivation deriv) { + List ngrams = new ArrayList<>(); + List paddedTokens = new ArrayList<>(); + if (tokens.size() < 3) + return ngrams; + + if (deriv.start == -1) // floating, just add everything + paddedTokens.addAll(tokens); + else + paddedTokens.addAll(tokens.subList(Math.max(0, deriv.start-2), Math.min(tokens.size(), deriv.end+2))); + + for (int i=0; i abstractAnchors = abstractAnchors(deriv, ex.getTokens()); + deriv.addFeature("win", abstractAnchors.toString()); + } + } + + private void addStatsFeatures(Example ex, Derivation deriv) { + if (!FeatureExtractor.containsDomain("stats")) return; + if (deriv.rule != Rule.nullRule) { + String cat = deriv.rule.getLhs(); + if (deriv.rule.isAnchored()) + deriv.addFeature("stats", "numAnchored"); + // deriv.addFeature("stats", cat); + deriv.addFeature("stats", "anchored-" + cat + "-" + deriv.rule.isAnchored()); + deriv.addFeature("stats", "depth"); + + if (deriv.rule.getInfoTag("induced") == 1.0) + deriv.addFeature("stats", "numInduced"); + else + deriv.addFeature("stats", "numCore"); + } + } +}