mirror of https://github.com/percyliang/sempre
Refactoring cprune out of Derivation
This commit is contained in:
Panupong Pasupat
parent
3ad5477081
commit
7de8d83244
12
build.xml
12
build.xml
|
|
@ -25,7 +25,7 @@
|
|||
<antcall target="compile.released"/>
|
||||
</target>
|
||||
|
||||
<target name="compile.released" depends="init,core,cache,corenlp,freebase,tables,overnight"/>
|
||||
<target name="compile.released" depends="init,core,cache,corenlp,freebase,tables,cprune,overnight"/>
|
||||
|
||||
<!-- Compile core -->
|
||||
<target name="core" depends="init">
|
||||
|
|
@ -78,6 +78,16 @@
|
|||
<jar destfile="${libsempre}/sempre-tables.jar" basedir="${classes}/tables"/>
|
||||
</target>
|
||||
|
||||
<!-- Compile cprune -->
|
||||
<target name="cprune" depends="init,core,corenlp,tables">
|
||||
<echo message="Compiling ${ant.project.name}: cprune"/>
|
||||
<mkdir dir="${classes}/cprune"/>
|
||||
<javac srcdir="${src}" destdir="${classes}/cprune" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
|
||||
<include name="edu/stanford/nlp/sempre/cprune/"/>
|
||||
</javac>
|
||||
<jar destfile="${libsempre}/sempre-cprune.jar" basedir="${classes}/cprune"/>
|
||||
</target>
|
||||
|
||||
<!-- Compile overnight -->
|
||||
<target name="overnight" depends="init,core">
|
||||
<echo message="Compiling ${ant.project.name}: overnight"/>
|
||||
|
|
|
|||
38
run
38
run
|
|
@ -634,7 +634,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
|
|||
# Add @cldir=1 to use CodaLab's directory paths
|
||||
letDefault(:cldir, 0),
|
||||
# Usual header
|
||||
header('core,tables,corenlp'),
|
||||
header('core,tables,corenlp,cprune'),
|
||||
# Select class
|
||||
letDefault(:class, 'main'),
|
||||
sel(:class, {
|
||||
|
|
@ -674,35 +674,38 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
|
|||
}),
|
||||
# Parser
|
||||
letDefault(:parser, 'floatsize'),
|
||||
o('useSizeInsteadOfDepth'),
|
||||
sel(:parser, {
|
||||
'floatsize' => l(
|
||||
o('Builder.parser', 'FloatingParser'),
|
||||
o('useSizeInsteadOfDepth'),
|
||||
o('FloatingParser.maxDepth', 15),
|
||||
nil),
|
||||
'baseline' => l(
|
||||
o('Builder.parser', 'tables.baseline.TableBaselineParser'),
|
||||
nil),
|
||||
'baseline' => o('Builder.parser', 'tables.baseline.TableBaselineParser'),
|
||||
'serialized' => o('Builder.parser', 'tables.serialize.SerializedParser'),
|
||||
# ACL 2016
|
||||
'grow-dpd' => l(
|
||||
o('Builder.parser', 'tables.dpd.DPDParser'),
|
||||
o('useSizeInsteadOfDepth'),
|
||||
o('FloatingParser.maxDepth', 8),
|
||||
nil),
|
||||
'grow-float' => l(
|
||||
o('Builder.parser', 'FloatingParser'),
|
||||
o('useSizeInsteadOfDepth'),
|
||||
o('FloatingParser.maxDepth', 8),
|
||||
o('FloatingParser.betaReduce'), o('initialFloatingHasZeroDepth'),
|
||||
nil),
|
||||
'grow-mix' => l(
|
||||
o('Builder.parser', 'MixParser'),
|
||||
o('MixParser.parsers', 'FloatingParser', 'tables.serialize.SerializedParser:train-0xc'),
|
||||
o('useSizeInsteadOfDepth'),
|
||||
o('FloatingParser.maxDepth', 8),
|
||||
o('FloatingParser.betaReduce'), o('initialFloatingHasZeroDepth'),
|
||||
nil),
|
||||
# EMNLP 2017
|
||||
'cprune' => l(
|
||||
o('Builder.parser', 'cprune.CPruneFloatingParser'),
|
||||
o('FloatingParser.maxDepth', 15),
|
||||
o('beamSize', 100),
|
||||
o('maxNumNeighbors', 40),
|
||||
o('maxPredictedPatterns', 1000),
|
||||
nil),
|
||||
}),
|
||||
o('Parser.verbose', 0),
|
||||
letDefault(:pruning, 1),
|
||||
|
|
@ -731,7 +734,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
|
|||
letDefault(:normalize, 1),
|
||||
sel(:normalize,
|
||||
l(),
|
||||
l(o('genericDateValue'), o('numberCanStartAnywhere'), o('num2CanStartAnywhere'), o('NumberFn.alsoTestByIsolatedNER')),
|
||||
l(o('genericDateValue'), o('numberCanStartAnywhere'), o('num2CanStartAnywhere')),
|
||||
nil),
|
||||
letDefault(:anchor, 1),
|
||||
sel(:anchor, {
|
||||
|
|
@ -786,6 +789,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
|
|||
o('combineFromFloatingParser'),
|
||||
o('maxTrainIters', 3),
|
||||
o('showValues', false), o('showFirstValue'),
|
||||
o('customExpectedCounts', 'TOP'),
|
||||
nil),
|
||||
l(
|
||||
# for dumping derivations (@class=dump)
|
||||
|
|
@ -886,6 +890,11 @@ def tablesGrammarPaths
|
|||
o('Grammar.tags', *'scoped merge-and arithmetic comparison alternative neq yearrange part closedclass-generic scoped-2args-merge-and'.split),
|
||||
let(:anchor, 2),
|
||||
nil),
|
||||
# EMNLP 2017
|
||||
'extended' => l(
|
||||
o('Grammar.inPaths', "#{baseDir}extended.grammar"),
|
||||
o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge v-superlative'.split),
|
||||
nil),
|
||||
}),
|
||||
nil)
|
||||
}
|
||||
|
|
@ -895,13 +904,17 @@ def tablesDataPaths
|
|||
lambda { |e|
|
||||
baseDir = ['lib/data/WikiTableQuestions/data/', 'WikiTableQuestions/data/'][e[:cldir]]
|
||||
csvDir = ['lib/data/WikiTableQuestions/', 'WikiTableQuestions/'][e[:cldir]]
|
||||
nnDir = ['lib/data/nn_0/', 'nn_0/'][e[:cldir]]
|
||||
datasets = {
|
||||
'none' => l(),
|
||||
'train' => o('Dataset.inPaths', "train,#{baseDir}training.examples"),
|
||||
# Pristine test test
|
||||
'test' => o('Dataset.inPaths',
|
||||
"train,#{baseDir}training.examples",
|
||||
"test,#{baseDir}pristine-unseen-tables.examples"),
|
||||
'test' => l(
|
||||
o('Dataset.inPaths',
|
||||
"train,#{baseDir}training.examples",
|
||||
"test,#{baseDir}pristine-unseen-tables.examples"),
|
||||
o('neighborFilePath', "#{nnDir}/exact_nearest_neighbors.all"),
|
||||
nil),
|
||||
# @data=annotated can be used with @class=check only
|
||||
'annotated' => o('Dataset.inPaths', "train,#{baseDir}annotated-all.examples"),
|
||||
'before300' => o('Dataset.inPaths', "train,#{baseDir}training-before300.examples"),
|
||||
|
|
@ -913,6 +926,7 @@ def tablesDataPaths
|
|||
"train,#{baseDir}random-split-#{x}-train.examples",
|
||||
"dev,#{baseDir}random-split-#{x}-dev.examples",
|
||||
nil),
|
||||
o('neighborFilePath', "#{nnDir}/exact_nearest_neighbors.seed-#{x}.train"),
|
||||
nil)
|
||||
end
|
||||
# That's it!
|
||||
|
|
|
|||
|
|
@ -84,14 +84,11 @@ public class FloatingParser extends Parser {
|
|||
@Option(gloss = "DEBUG: Print amount of time spent on each rule")
|
||||
public boolean summarizeRuleTime = false;
|
||||
@Option(gloss = "Stop the parser if it has used more than this amount of time (in seconds)")
|
||||
public int maxFloatingParsingTime = 600;
|
||||
public int maxFloatingParsingTime = Integer.MAX_VALUE;
|
||||
}
|
||||
|
||||
public static Options opts = new Options();
|
||||
|
||||
protected List<Rule> orderedFloatingRules;
|
||||
public List<Rule> getOrderedFloatingRules() { return orderedFloatingRules; }
|
||||
|
||||
public FloatingParser(Spec spec) {
|
||||
super(spec);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ public class Grammar {
|
|||
// All the rules in the grammar. Each parser can read these and transform
|
||||
// them however the parser wishes.
|
||||
// This contains binarized rules
|
||||
ArrayList<Rule> rules = new ArrayList<>();
|
||||
protected ArrayList<Rule> rules = new ArrayList<>();
|
||||
public List<Rule> getRules() { return rules; }
|
||||
|
||||
Map<String, LispTree> macros = new HashMap<>(); // Map from macro name to its replacement value
|
||||
|
|
@ -260,7 +260,7 @@ public class Grammar {
|
|||
return cat;
|
||||
}
|
||||
|
||||
private void interpretRule(LispTree tree) {
|
||||
protected void interpretRule(LispTree tree) {
|
||||
if (tree.children.size() < 4)
|
||||
throw new RuntimeException("Invalid rule: " + tree);
|
||||
|
||||
|
|
@ -360,7 +360,7 @@ public class Grammar {
|
|||
|
||||
// Generate intermediate categories for binarization.
|
||||
public static final String INTERMEDIATE_PREFIX = "$Intermediate";
|
||||
private int freshCatIndex = 0;
|
||||
protected int freshCatIndex = 0;
|
||||
private String generateFreshCat() {
|
||||
freshCatIndex++;
|
||||
return INTERMEDIATE_PREFIX + freshCatIndex;
|
||||
|
|
@ -368,6 +368,9 @@ public class Grammar {
|
|||
public static boolean isIntermediate(String cat) {
|
||||
return cat.startsWith(INTERMEDIATE_PREFIX);
|
||||
}
|
||||
public int getFreshCatIndex() {
|
||||
return freshCatIndex;
|
||||
}
|
||||
|
||||
// Create multiple versions of this rule if there are optional RHS.
|
||||
// Restriction: must be able to split the RHS into two halves, each of
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ public abstract class Parser {
|
|||
|
||||
@Option(gloss = "Dump all features (for debugging)")
|
||||
public boolean dumpAllFeatures = false;
|
||||
|
||||
|
||||
@Option(gloss = "Call SetEvaluation during parsing")
|
||||
public boolean callSetEvaluation = true;
|
||||
}
|
||||
|
|
@ -100,8 +100,8 @@ public abstract class Parser {
|
|||
this.valueEvaluator = spec.valueEvaluator;
|
||||
|
||||
computeCatUnaryRules();
|
||||
LogInfo.logs("Parser: %d catUnaryRules (sorted), %d nonCatUnaryRules (in trie)",
|
||||
catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size());
|
||||
LogInfo.logs("%s: %d catUnaryRules (sorted), %d nonCatUnaryRules (in trie)",
|
||||
this.getClass().getSimpleName(), catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size());
|
||||
}
|
||||
|
||||
// If grammar changes, then we might need to update aspects of the parser.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
package edu.stanford.nlp.sempre.cprune;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.List;
|
||||
|
||||
public class CPruneDerivInfo {
|
||||
|
||||
public Map<String, Symbol> treeSymbols;
|
||||
public Map<String, Symbol> ruleSymbols;
|
||||
public List<String> customRuleStrings;
|
||||
public boolean containsCrossReference;
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
package edu.stanford.nlp.sempre.cprune;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import edu.stanford.nlp.sempre.*;
|
||||
import fig.basic.LogInfo;
|
||||
|
||||
public class CPruneFloatingParser extends FloatingParser {
|
||||
|
||||
FloatingParser exploreParser;
|
||||
|
||||
public CPruneFloatingParser(Spec spec) {
|
||||
super(spec);
|
||||
exploreParser = new FloatingParser(spec);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onBeginDataGroup(int iter, int numIters, String group) {
|
||||
if (CollaborativePruner.neighbors == null) {
|
||||
CollaborativePruner.customGrammar.init(grammar);
|
||||
CollaborativePruner.loadNeighbors();
|
||||
}
|
||||
CollaborativePruner.stats.reset(iter + "." + group);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) {
|
||||
return new CPruneFloatingParserState(this, params, ex, computeExpectedCounts);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class CPruneFloatingParserState extends ParserState {
|
||||
|
||||
public CPruneFloatingParserState(Parser parser, Params params, Example ex, boolean computeExpectedCounts) {
|
||||
super(parser, params, ex, computeExpectedCounts);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void infer() {
|
||||
LogInfo.begin_track("CPruneFloatingParser.infer()");
|
||||
boolean exploitSucceeds = exploit();
|
||||
if (computeExpectedCounts) {
|
||||
LogInfo.begin_track("Summary of Collaborative Pruning");
|
||||
LogInfo.logs("Exploit succeeds: " + exploitSucceeds);
|
||||
LogInfo.logs("Exploit success rate: " + CollaborativePruner.stats.successfulExploit + "/" + CollaborativePruner.stats.totalExploit);
|
||||
LogInfo.end_track();
|
||||
}
|
||||
// Explore only on the first training iteration
|
||||
if (CollaborativePruner.stats.iter.equals("0.train") && computeExpectedCounts && !exploitSucceeds
|
||||
&& (CollaborativePruner.stats.totalExplore <= CollaborativePruner.opts.maxExplorationIters)) {
|
||||
explore();
|
||||
LogInfo.logs("Consistent pattern: " + CollaborativePruner.getConsistentPattern(ex));
|
||||
LogInfo.logs("Explore success rate: " + CollaborativePruner.stats.successfulExplore + "/" + CollaborativePruner.stats.totalExplore);
|
||||
}
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
public void explore() {
|
||||
LogInfo.begin_track("Explore");
|
||||
CollaborativePruner.initialize(ex, CollaborativePruner.Mode.EXPLORE);
|
||||
ParserState exploreParserState = ((CPruneFloatingParser) parser).exploreParser.newParserState(params, ex, computeExpectedCounts);
|
||||
exploreParserState.infer();
|
||||
predDerivations.clear();
|
||||
predDerivations.addAll(exploreParserState.predDerivations);
|
||||
expectedCounts = exploreParserState.expectedCounts;
|
||||
CollaborativePruner.stats.totalExplore += 1;
|
||||
if (CollaborativePruner.foundConsistentDerivation)
|
||||
CollaborativePruner.stats.successfulExplore += 1;
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
public boolean exploit() {
|
||||
LogInfo.begin_track("Exploit");
|
||||
CollaborativePruner.initialize(ex, CollaborativePruner.Mode.EXPLOIT);
|
||||
Parser exploitParser = new FloatingParser(
|
||||
new Parser.Spec(new MiniGrammar(CollaborativePruner.predictedRules), parser.extractor, parser.executor, parser.valueEvaluator));
|
||||
ParserState exploitParserState = exploitParser.newParserState(params, ex, computeExpectedCounts);
|
||||
exploitParserState.infer();
|
||||
predDerivations.clear();
|
||||
predDerivations.addAll(exploitParserState.predDerivations);
|
||||
expectedCounts = exploitParserState.expectedCounts;
|
||||
boolean succeeds = CollaborativePruner.foundConsistentDerivation;
|
||||
CollaborativePruner.stats.totalExploit += 1;
|
||||
if (succeeds)
|
||||
CollaborativePruner.stats.successfulExploit += 1;
|
||||
LogInfo.end_track();
|
||||
return succeeds;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Helper classes
|
||||
// ============================================================
|
||||
|
||||
class MiniGrammar extends Grammar {
|
||||
|
||||
public MiniGrammar(List<Rule> rules) {
|
||||
this.rules.addAll(rules);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package edu.stanford.nlp.sempre.cprune;
|
||||
|
||||
/**
|
||||
* Stores various statistic.
|
||||
*/
|
||||
public class CPruneStats {
|
||||
public String iter;
|
||||
public int totalExplore = 0;
|
||||
public int successfulExplore = 0;
|
||||
public int totalExploit = 0;
|
||||
public int successfulExploit = 0;
|
||||
|
||||
public void reset(String iter) {
|
||||
this.iter = iter;
|
||||
this.totalExplore = 0;
|
||||
this.successfulExplore = 0;
|
||||
this.totalExploit = 0;
|
||||
this.successfulExploit = 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
package edu.stanford.nlp.sempre.cprune;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
|
||||
import fig.basic.*;
|
||||
import edu.stanford.nlp.sempre.*;
|
||||
import fig.basic.LogInfo;
|
||||
|
||||
/**
|
||||
* Static class for collaborative pruning.
|
||||
*/
|
||||
public class CollaborativePruner {
|
||||
public static class Options {
|
||||
@Option(gloss = "K = Maximum number of nearest-neighbor examples to consider (-1 to use all examples so far)")
|
||||
public int maxNumNeighbors = -1;
|
||||
@Option(gloss = "Load cached neighbors from this file")
|
||||
public String neighborFilePath = null;
|
||||
@Option(gloss = "Maximum number of matching patterns (default = use all patterns)")
|
||||
public int maxPredictedPatterns = Integer.MAX_VALUE;
|
||||
@Option(gloss = "Maximum number of derivations per example")
|
||||
public int maxDerivations = 5000;
|
||||
@Option(gloss = "Maximum number of exporation iterations")
|
||||
public int maxExplorationIters = Integer.MAX_VALUE;
|
||||
}
|
||||
|
||||
public static Options opts = new Options();
|
||||
|
||||
public enum Mode { EXPLORE, EXPLOIT, NONE }
|
||||
|
||||
public static Mode mode = Mode.NONE;
|
||||
public static CPruneStats stats = new CPruneStats();
|
||||
public static CustomGrammar customGrammar = new CustomGrammar();
|
||||
|
||||
// Static class; do not instantiate
|
||||
private CollaborativePruner() { throw new RuntimeException("Cannot instantiate CollaborativePruner"); }
|
||||
|
||||
// Global variables
|
||||
// Nearest neighbors
|
||||
static Map<String, List<String>> neighbors;
|
||||
// uid => pattern
|
||||
static Map<String, Pattern> consistentPattern = new HashMap<>();
|
||||
// patternString => customRuleString
|
||||
static Map<String, Set<String>> customRules = new HashMap<>();
|
||||
// set of patternStrings
|
||||
static Set<String> allConsistentPatterns = new HashSet<>();
|
||||
|
||||
// Example-level variables
|
||||
public static boolean foundConsistentDerivation = false;
|
||||
public static Map<String, Pattern> predictedPatterns;
|
||||
public static List<Rule> predictedRules;
|
||||
|
||||
/**
|
||||
* Read the cached neighbors file.
|
||||
* Line Format: ex_id [tab] neighbor_id1,neighbor_id2,...
|
||||
*/
|
||||
public static void loadNeighbors() {
|
||||
if (opts.neighborFilePath == null) {
|
||||
LogInfo.logs("neighborFilePath is null.");
|
||||
return;
|
||||
}
|
||||
LogInfo.logs("loading Neighbors " + opts.neighborFilePath);
|
||||
Map<String, List<String>> tmpMap = new HashMap<>();
|
||||
try {
|
||||
BufferedReader reader = IOUtils.openIn(opts.neighborFilePath);
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
String[] tokens = line.split("\t");
|
||||
String uid = tokens[0];
|
||||
String[] nids = tokens[1].split(",");
|
||||
tmpMap.put(uid, Arrays.asList(nids));
|
||||
}
|
||||
reader.close();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
neighbors = tmpMap;
|
||||
}
|
||||
|
||||
public static <K, V extends Comparable<? super V>> Map<K, V> sortByValue(Map<K, V> map) {
|
||||
List<Map.Entry<K, V>> list = new LinkedList<Map.Entry<K, V>>(map.entrySet());
|
||||
Collections.sort(list, new Comparator<Map.Entry<K, V>>() {
|
||||
public int compare(Map.Entry<K, V> o1, Map.Entry<K, V> o2) {
|
||||
return (o1.getValue()).compareTo(o2.getValue());
|
||||
}
|
||||
});
|
||||
|
||||
Map<K, V> result = new LinkedHashMap<K, V>();
|
||||
for (Map.Entry<K, V> entry : list) {
|
||||
result.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static void initialize(Example ex, Mode mode) {
|
||||
CollaborativePruner.mode = mode;
|
||||
predictedRules = null;
|
||||
predictedPatterns = null;
|
||||
foundConsistentDerivation = false;
|
||||
if (mode == Mode.EXPLOIT) {
|
||||
preprocessExample(ex);
|
||||
}
|
||||
}
|
||||
|
||||
static void preprocessExample(Example ex) {
|
||||
Map<String, Pattern> patternFreqMap = new HashMap<>();
|
||||
List<String> cachedNeighbors = neighbors.get(ex.id);
|
||||
int total = 0;
|
||||
|
||||
// Gather the neighbors
|
||||
if (opts.maxNumNeighbors > 0) {
|
||||
for (String nid : cachedNeighbors) {
|
||||
// Only get examples that have been previously processed
|
||||
if (!consistentPattern.containsKey(nid))
|
||||
continue;
|
||||
|
||||
String neighborPattern = consistentPattern.get(nid).pattern;
|
||||
if (!patternFreqMap.containsKey(neighborPattern))
|
||||
patternFreqMap.put(neighborPattern, new Pattern(neighborPattern, 0, 0));
|
||||
patternFreqMap.get(neighborPattern).frequency += 1;
|
||||
total++;
|
||||
if (total >= opts.maxNumNeighbors)
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
for (String patternString : allConsistentPatterns) {
|
||||
patternFreqMap.put(patternString, new Pattern(patternString, 0, 1));
|
||||
}
|
||||
}
|
||||
|
||||
// Gather the patterns
|
||||
patternFreqMap = sortByValue(patternFreqMap);
|
||||
int rank = 0;
|
||||
Set<String> predictedRulesStrings = new HashSet<>();
|
||||
predictedPatterns = new HashMap<>();
|
||||
LogInfo.begin_track("Predicted patterns");
|
||||
for (Map.Entry<String, Pattern> entry : patternFreqMap.entrySet()) {
|
||||
Pattern newPattern = entry.getValue();
|
||||
predictedPatterns.put(newPattern.pattern, newPattern);
|
||||
predictedRulesStrings.addAll(customRules.get(newPattern.pattern));
|
||||
LogInfo.logs((rank + 1) + ". " + newPattern.pattern + " (" + newPattern.frequency + ")");
|
||||
rank++;
|
||||
if (rank >= opts.maxPredictedPatterns)
|
||||
break;
|
||||
}
|
||||
|
||||
// Gather the rules
|
||||
predictedRules = customGrammar.getRules(predictedRulesStrings);
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
public static String getPatternString(Derivation deriv) {
|
||||
if (deriv.cat.equals("$TOKEN") || deriv.cat.equals("$PHRASE")
|
||||
|| deriv.cat.equals("$LEMMA_TOKEN") || deriv.cat.equals("$LEMMA_PHRASE")) {
|
||||
return deriv.cat;
|
||||
} else {
|
||||
return PatternInfo.convertToIndexedPattern(deriv);
|
||||
}
|
||||
}
|
||||
|
||||
public static void addRules(String patternString, Derivation deriv, Example ex) {
|
||||
if (!customRules.containsKey(patternString)) {
|
||||
customRules.put(patternString, new HashSet<String>());
|
||||
}
|
||||
Set<String> parsedCustomRules = customGrammar.addCustomRule(deriv, ex);
|
||||
customRules.get(patternString).addAll(parsedCustomRules);
|
||||
}
|
||||
|
||||
public static void updateConsistentPattern(ValueEvaluator evaluator, Example ex, Derivation deriv) {
|
||||
String uid = ex.id;
|
||||
if (ex.targetValue != null)
|
||||
deriv.compatibility = evaluator.getCompatibility(ex.targetValue, deriv.value);
|
||||
|
||||
if (deriv.isRootCat() && deriv.compatibility == 1) {
|
||||
foundConsistentDerivation = true;
|
||||
LogInfo.logs("Found consistent deriv: %s", deriv);
|
||||
|
||||
String patternString = getPatternString(deriv);
|
||||
Pattern newConsistentPattern = new Pattern(patternString, 0, 0);
|
||||
newConsistentPattern.score = deriv.getScore();
|
||||
|
||||
if (!consistentPattern.containsKey(uid)) {
|
||||
addRules(patternString, deriv, ex);
|
||||
consistentPattern.put(uid, newConsistentPattern);
|
||||
allConsistentPatterns.add(patternString);
|
||||
} else {
|
||||
Pattern oldConsistentPattern = consistentPattern.get(uid);
|
||||
if (newConsistentPattern.score > oldConsistentPattern.score) {
|
||||
addRules(patternString, deriv, ex);
|
||||
consistentPattern.put(uid, newConsistentPattern);
|
||||
allConsistentPatterns.add(patternString);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static Pattern getConsistentPattern(Example ex) {
|
||||
return consistentPattern.get(ex.id);
|
||||
}
|
||||
|
||||
public static Pattern getPattern(Example ex, Derivation deriv) {
|
||||
if (mode == Mode.EXPLORE)
|
||||
return null;
|
||||
if (!deriv.isRootCat())
|
||||
return null;
|
||||
return predictedPatterns.get(getPatternString(deriv));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,256 @@
|
|||
package edu.stanford.nlp.sempre.cprune;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import edu.stanford.nlp.sempre.*;
|
||||
import fig.basic.*;
|
||||
|
||||
public class CustomGrammar extends Grammar {
|
||||
public static class Options {
|
||||
@Option
|
||||
public boolean enableTemplateDecomposition = true;
|
||||
}
|
||||
|
||||
public static Options opts = new Options();
|
||||
|
||||
public static final Set<String> baseCategories = new HashSet<String>(Arrays.asList("$Unary", "$Binary", "$Entity", "$Property"));
|
||||
|
||||
ArrayList<Rule> baseRules = new ArrayList<>();
|
||||
|
||||
// symbolicFormulas => symbolicFormula ID
|
||||
Map<String, Integer> symbolicFormulas = new HashMap<>();
|
||||
// indexedSymbolicFormula => customRuleString
|
||||
Map<String, Set<String>> customRules = new HashMap<>();
|
||||
// customRuleString => Binarized rules
|
||||
Map<String, Set<Rule>> customBinarizedRules = new HashMap<>();
|
||||
|
||||
public void init(Grammar initGrammar) {
|
||||
baseCategories.add(Rule.tokenCat);
|
||||
baseCategories.add(Rule.phraseCat);
|
||||
baseCategories.add(Rule.lemmaTokenCat);
|
||||
baseCategories.add(Rule.lemmaPhraseCat);
|
||||
|
||||
baseRules = new ArrayList<>();
|
||||
for (Rule rule : initGrammar.getRules()) {
|
||||
if (baseCategories.contains(rule.lhs)) {
|
||||
baseRules.add(rule);
|
||||
}
|
||||
}
|
||||
this.freshCatIndex = initGrammar.getFreshCatIndex();
|
||||
}
|
||||
|
||||
public List<Rule> getRules(Collection<String> customRuleStrings) {
|
||||
Set<Rule> ruleSet = new LinkedHashSet<>();
|
||||
ruleSet.addAll(baseRules);
|
||||
for (String ruleString : customRuleStrings) {
|
||||
ruleSet.addAll(customBinarizedRules.get(ruleString));
|
||||
}
|
||||
return new ArrayList<Rule>(ruleSet);
|
||||
}
|
||||
|
||||
public Set<String> addCustomRule(Derivation deriv, Example ex) {
|
||||
String indexedSymbolicFormula = getIndexedSymbolicFormula(deriv);
|
||||
if (customRules.containsKey(indexedSymbolicFormula)) {
|
||||
return customRules.get(indexedSymbolicFormula);
|
||||
}
|
||||
|
||||
CPruneDerivInfo derivInfo = aggregateSymbols(deriv);
|
||||
Set<String> crossReferences = new HashSet<>();
|
||||
for (Symbol symbol : derivInfo.treeSymbols.values()) {
|
||||
if (symbol.frequency > 1) {
|
||||
crossReferences.add(symbol.formula);
|
||||
}
|
||||
}
|
||||
computeCustomRules(deriv, crossReferences);
|
||||
customRules.put(indexedSymbolicFormula, new HashSet<String>(derivInfo.customRuleStrings));
|
||||
|
||||
LogInfo.begin_track("Add custom rules for formula: " + indexedSymbolicFormula);
|
||||
for (String customRuleString : derivInfo.customRuleStrings) {
|
||||
if (customBinarizedRules.containsKey(customRuleString)) {
|
||||
LogInfo.log("Custom rule exists: " + customRuleString);
|
||||
continue;
|
||||
}
|
||||
|
||||
rules = new ArrayList<>();
|
||||
LispTree tree = LispTree.proto.parseFromString(customRuleString);
|
||||
interpretRule(tree);
|
||||
customBinarizedRules.put(customRuleString, new HashSet<Rule>(rules));
|
||||
|
||||
// Debug
|
||||
LogInfo.begin_track("Add custom rule: " + customRuleString);
|
||||
for (Rule rule : rules) {
|
||||
LogInfo.log(rule.toString());
|
||||
}
|
||||
LogInfo.end_track();
|
||||
}
|
||||
LogInfo.end_track();
|
||||
|
||||
// Debug
|
||||
System.out.println("consistent_lf\t" + ex.id + "\t" + deriv.formula.toString());
|
||||
|
||||
return customRules.get(indexedSymbolicFormula);
|
||||
}
|
||||
|
||||
public static String getIndexedSymbolicFormula(Derivation deriv) {
|
||||
return getIndexedSymbolicFormula(deriv, deriv.formula.toString());
|
||||
}
|
||||
|
||||
public static String getIndexedSymbolicFormula(Derivation deriv, String formula) {
|
||||
CPruneDerivInfo derivInfo = aggregateSymbols(deriv);
|
||||
int index = 1;
|
||||
List<Symbol> symbolList = new ArrayList<>(derivInfo.treeSymbols.values());
|
||||
for (Symbol symbol : symbolList)
|
||||
symbol.computeIndex(formula);
|
||||
Collections.sort(symbolList);
|
||||
for (Symbol symbol : symbolList) {
|
||||
if (formula.equals(symbol.formula))
|
||||
formula = symbol.category + "#" + index;
|
||||
formula = safeReplace(formula, symbol.formula, symbol.category + "#" + index);
|
||||
index += 1;
|
||||
}
|
||||
return formula;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Private methods
|
||||
// ============================================================
|
||||
|
||||
private static String safeReplace(String formula, String target, String replacement) {
|
||||
formula = formula.replace(target + ")", replacement + ")");
|
||||
formula = formula.replace(target + " ", replacement + " ");
|
||||
return formula;
|
||||
}
|
||||
|
||||
private static CPruneDerivInfo aggregateSymbols(Derivation deriv) {
|
||||
Map<String, Object> tempState = deriv.getTempState();
|
||||
if (tempState.containsKey("cprune")) {
|
||||
return (CPruneDerivInfo) tempState.get("cprune");
|
||||
}
|
||||
CPruneDerivInfo derivInfo = new CPruneDerivInfo();
|
||||
tempState.put("cprune", derivInfo);
|
||||
|
||||
Map<String, Symbol> treeSymbols = new LinkedHashMap<>();
|
||||
derivInfo.treeSymbols = treeSymbols;
|
||||
if (baseCategories.contains(deriv.cat)) {
|
||||
String formula = deriv.formula.toString();
|
||||
treeSymbols.put(formula, new Symbol(deriv.cat, formula, 1));
|
||||
} else {
|
||||
for (Derivation child : deriv.children) {
|
||||
CPruneDerivInfo childInfo = aggregateSymbols(child);
|
||||
for (Symbol symbol : childInfo.treeSymbols.values()) {
|
||||
if (derivInfo.treeSymbols.containsKey(symbol.formula)) {
|
||||
treeSymbols.get(symbol.formula).frequency += symbol.frequency;
|
||||
} else {
|
||||
treeSymbols.put(symbol.formula, symbol);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return derivInfo;
|
||||
}
|
||||
|
||||
private CPruneDerivInfo computeCustomRules(Derivation deriv, Set<String> crossReferences) {
|
||||
CPruneDerivInfo derivInfo = (CPruneDerivInfo) deriv.getTempState().get("cprune");
|
||||
Map<String, Symbol> ruleSymbols = new LinkedHashMap<>();
|
||||
derivInfo.ruleSymbols = ruleSymbols;
|
||||
derivInfo.customRuleStrings = new ArrayList<>();
|
||||
String formula = deriv.formula.toString();
|
||||
|
||||
if (baseCategories.contains(deriv.cat)) {
|
||||
// Leaf node induces no custom rule
|
||||
derivInfo.containsCrossReference = crossReferences.contains(formula);
|
||||
// Propagate the symbol of this derivation to the parent
|
||||
ruleSymbols.putAll(derivInfo.treeSymbols);
|
||||
} else {
|
||||
derivInfo.containsCrossReference = false;
|
||||
for (Derivation child : deriv.children) {
|
||||
CPruneDerivInfo childInfo = computeCustomRules(child, crossReferences);
|
||||
derivInfo.containsCrossReference = derivInfo.containsCrossReference || childInfo.containsCrossReference;
|
||||
}
|
||||
|
||||
for (Derivation child : deriv.children) {
|
||||
CPruneDerivInfo childInfo = (CPruneDerivInfo) child.getTempState().get("cprune");
|
||||
ruleSymbols.putAll(childInfo.ruleSymbols);
|
||||
derivInfo.customRuleStrings.addAll(childInfo.customRuleStrings);
|
||||
}
|
||||
|
||||
if (opts.enableTemplateDecomposition == false || derivInfo.containsCrossReference) {
|
||||
// If this node contains a cross reference
|
||||
if (deriv.isRootCat()) {
|
||||
// If this is the root node, then generate a custom rule
|
||||
derivInfo.customRuleStrings.add(getCustomRuleString(deriv, derivInfo));
|
||||
}
|
||||
} else {
|
||||
if (!deriv.cat.startsWith("$Intermediate")) {
|
||||
// Generate a custom rule for this node
|
||||
derivInfo.customRuleStrings.add(getCustomRuleString(deriv, derivInfo));
|
||||
|
||||
// Propagate this derivation as a category to the parent
|
||||
ruleSymbols.clear();
|
||||
ruleSymbols.put(formula, new Symbol(hash(deriv), deriv.formula.toString(), 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
return derivInfo;
|
||||
}
|
||||
|
||||
private String getCustomRuleString(Derivation deriv, CPruneDerivInfo derivInfo) {
|
||||
String formula = deriv.formula.toString();
|
||||
List<Symbol> rhsSymbols = new ArrayList<>(derivInfo.ruleSymbols.values());
|
||||
for (Symbol symbol : rhsSymbols)
|
||||
symbol.computeIndex(formula);
|
||||
Collections.sort(rhsSymbols);
|
||||
|
||||
String lhs = null;
|
||||
if (derivInfo.containsCrossReference)
|
||||
lhs = deriv.cat;
|
||||
else
|
||||
lhs = deriv.isRootCat() ? "$ROOT" : hash(deriv);
|
||||
|
||||
LinkedList<String> rhsList = new LinkedList<>();
|
||||
int index = 1;
|
||||
for (Symbol symbol : rhsSymbols) {
|
||||
if (formula.equals(symbol.formula)) {
|
||||
formula = "(IdentityFn)";
|
||||
} else {
|
||||
formula = safeReplace(formula, symbol.formula, "(var s" + index + ")");
|
||||
formula = "(lambda s" + index + " " + formula + ")";
|
||||
}
|
||||
rhsList.addFirst(symbol.category);
|
||||
index += 1;
|
||||
}
|
||||
String rhs = null;
|
||||
if (rhsList.size() > 0) {
|
||||
rhs = "(" + String.join(" ", rhsList) + ")";
|
||||
} else {
|
||||
rhs = "(nothing)";
|
||||
formula = "(ConstantFn " + formula + ")";
|
||||
}
|
||||
return "(rule " + lhs + " " + rhs + " " + formula + ")";
|
||||
}
|
||||
|
||||
private String hash(Derivation deriv) {
|
||||
if (baseCategories.contains(deriv.cat))
|
||||
return deriv.cat;
|
||||
|
||||
String formula = getSymbolicFormula(deriv);
|
||||
if (!symbolicFormulas.containsKey(formula)) {
|
||||
symbolicFormulas.put(formula, symbolicFormulas.size() + 1);
|
||||
String hashString = "$Formula" + symbolicFormulas.get(formula);
|
||||
LogInfo.log("Add symbolic formula: " + hashString + " = " + formula + " (" + deriv.cat + ")");
|
||||
}
|
||||
return "$Formula" + symbolicFormulas.get(formula);
|
||||
}
|
||||
|
||||
private static String getSymbolicFormula(Derivation deriv) {
|
||||
CPruneDerivInfo derivInfo = aggregateSymbols(deriv);
|
||||
String formula = deriv.formula.toString();
|
||||
for (Symbol symbol : derivInfo.treeSymbols.values()) {
|
||||
if (formula.equals(symbol.formula))
|
||||
formula = symbol.category;
|
||||
formula = safeReplace(formula, symbol.formula, symbol.category);
|
||||
}
|
||||
return formula;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package edu.stanford.nlp.sempre.cprune;
|
||||
|
||||
public class Pattern implements Comparable<Pattern> {
|
||||
public String pattern;
|
||||
public Integer rank;
|
||||
public Integer frequency;
|
||||
public Double score;
|
||||
|
||||
public Pattern(String pattern, Integer rank, Integer frequency) {
|
||||
this.pattern = pattern;
|
||||
this.rank = rank;
|
||||
this.frequency = frequency;
|
||||
}
|
||||
|
||||
public Double complexity() {
|
||||
return (double) (pattern.length() - pattern.replace("(@R", "***").replace("(", "").length());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "(" + pattern + ", " + rank + ", " + frequency + ")";
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(Pattern that) {
|
||||
if (this.frequency > that.frequency) {
|
||||
return -1;
|
||||
} else if (this.frequency < that.frequency) {
|
||||
return 1;
|
||||
} else {
|
||||
return this.complexity().compareTo(that.complexity());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
package edu.stanford.nlp.sempre.cprune;
|
||||
|
||||
import java.util.*;
|
||||
import edu.stanford.nlp.sempre.*;
|
||||
|
||||
public class PatternInfo {
|
||||
Set<String> baseCategories = new HashSet<String>(Arrays.asList("$Unary", "$Binary", "$Entity", "$Property"));
|
||||
|
||||
public static String removePropertyPredicates(String formula) {
|
||||
formula = RegexReplaceManager.replace(formula, "fb:cell\\.cell\\.[_0-9a-z]+", "@PPT");
|
||||
formula = formula.replace("(reverse @PPT)", "@PPT");
|
||||
|
||||
while (formula.contains("@PPT")) {
|
||||
int begin = formula.indexOf("(@PPT");
|
||||
if (begin == -1) {
|
||||
formula = formula.replace("@PPT", "");
|
||||
;
|
||||
break;
|
||||
}
|
||||
int count = 1;
|
||||
for (int i = begin + 1; i < formula.length(); i++) {
|
||||
if (formula.charAt(i) == '(') {
|
||||
count += 1;
|
||||
} else if (formula.charAt(i) == ')') {
|
||||
count -= 1;
|
||||
}
|
||||
if (count == 0) {
|
||||
int end = i;
|
||||
formula = formula.substring(0, begin) + formula.substring(begin + 6, end) + formula.substring(end + 1, formula.length());
|
||||
break;
|
||||
}
|
||||
if (i == formula.length() - 1) {
|
||||
System.out.println(formula);
|
||||
}
|
||||
}
|
||||
}
|
||||
return formula;
|
||||
}
|
||||
|
||||
public static String convertToPattern(Derivation deriv) {
|
||||
String formula = deriv.formula.toString();
|
||||
|
||||
// These can interfere with (number 1)
|
||||
formula = formula.replace("argmax (number 1) (number 1)", "argmax");
|
||||
formula = formula.replace("argmin (number 1) (number 1)", "argmin");
|
||||
|
||||
formula = removePropertyPredicates(formula);
|
||||
|
||||
formula = formula.replace("fb:type.object.type fb:type.row", "@type @row");
|
||||
|
||||
formula = RegexReplaceManager.replace(formula, "!fb:[._0-9a-z]+", "(reverse $0)").replace("reverse !fb", "reverse fb");
|
||||
formula = formula.replace("fb:row.row.index", "(reverse (lambda x ((reverse @index) (var x))))");
|
||||
formula = formula.replace("fb:row.row.next", "@next");
|
||||
formula = RegexReplaceManager.replace(formula, "\\(lambda [a-z]", "\\(lambda x");
|
||||
formula = RegexReplaceManager.replace(formula, "\\(var [a-z]\\)", "\\(var x\\)");
|
||||
|
||||
formula = RegexReplaceManager.replace(formula, "fb:row\\.row\\.[_0-9a-z]+", "\\$Binary");
|
||||
formula = RegexReplaceManager.replace(formula, "fb:cell\\.cell\\.[_0-9a-z]+", "\\$Property");
|
||||
formula = RegexReplaceManager.replace(formula, "fb:cell_[_0-9a-z]+\\.[_0-9a-z]+", "\\$Entity");
|
||||
formula = RegexReplaceManager.replace(formula, "\\(number [0-9]+[.]?[0-9]+\\)", "\\$Entity");
|
||||
formula = RegexReplaceManager.replace(formula, "\\(number [0-9]+\\)", "\\$Entity");
|
||||
formula = RegexReplaceManager.replace(formula, "\\(date [-]?[0-9]+ [-]?[0-9]+ [-]?[0-9]+\\)", "\\$Entity");
|
||||
|
||||
formula = RegexReplaceManager.replace(formula, "[ ]+", " ");
|
||||
formula = formula.replace("reverse", "@R");
|
||||
formula = RegexReplaceManager.replace(formula, "(<=|>=|>|<)", "@compare");
|
||||
return formula;
|
||||
}
|
||||
|
||||
public static String convertToIndexedPattern(Derivation deriv) {
|
||||
String formula = deriv.formula.toString();
|
||||
|
||||
// These can interfere with (number 1)
|
||||
formula = formula.replace("argmax (number 1) (number 1)", "argmax");
|
||||
formula = formula.replace("argmin (number 1) (number 1)", "argmin");
|
||||
|
||||
formula = removePropertyPredicates(formula);
|
||||
formula = CustomGrammar.getIndexedSymbolicFormula(deriv, formula);
|
||||
|
||||
formula = formula.replace("fb:type.object.type fb:type.row", "@type @row");
|
||||
|
||||
formula = RegexReplaceManager.replace(formula, "!fb:[._0-9a-z]+", "(reverse $0)").replace("reverse !fb", "reverse fb");
|
||||
formula = formula.replace("fb:row.row.index", "(reverse (lambda x ((reverse @index) (var x))))");
|
||||
formula = formula.replace("fb:row.row.next", "@next");
|
||||
formula = RegexReplaceManager.replace(formula, "\\(lambda [a-z]", "\\(lambda x");
|
||||
formula = RegexReplaceManager.replace(formula, "\\(var [a-z]\\)", "\\(var x\\)");
|
||||
formula = RegexReplaceManager.replace(formula, "[ ]+", " ");
|
||||
formula = formula.replace("reverse", "@R");
|
||||
formula = RegexReplaceManager.replace(formula, "(<=|>=|>|<)", "@compare");
|
||||
return formula;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package edu.stanford.nlp.sempre.cprune;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class RegexReplaceManager {
|
||||
static Map<String, java.util.regex.Pattern> dict = new HashMap<>();
|
||||
|
||||
public static String replace(String source, String regex, String replacement) {
|
||||
if (!dict.containsKey(regex)) {
|
||||
dict.put(regex, java.util.regex.Pattern.compile(regex));
|
||||
}
|
||||
java.util.regex.Pattern regexPattern = dict.get(regex);
|
||||
return regexPattern.matcher(source).replaceAll(replacement);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package edu.stanford.nlp.sempre.cprune;
|
||||
|
||||
public class Symbol implements Comparable<Symbol> {
|
||||
String category;
|
||||
String formula;
|
||||
Integer frequency;
|
||||
Integer index;
|
||||
|
||||
public Symbol(String category, String formula, int frequency) {
|
||||
this.category = category;
|
||||
this.formula = formula;
|
||||
this.frequency = frequency;
|
||||
}
|
||||
|
||||
public void computeIndex(String referenceString) {
|
||||
index = referenceString.indexOf(formula);
|
||||
if (index < 0) {
|
||||
index = Integer.MAX_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(Symbol that) {
|
||||
return index.compareTo(that.index);
|
||||
}
|
||||
}
|
||||
|
|
@ -238,6 +238,8 @@ public final class DenotationUtils {
|
|||
*/
|
||||
public static UnaryDenotation superlativeUnary(int rank, int count, List<Pair<Value, Value>> pairs,
|
||||
SuperlativeFormula.Mode mode, TypeProcessor processor) {
|
||||
if (rank <= 0 || count <= 0)
|
||||
LogInfo.fails("Invalid superlative (rank = %d, count = %d)", rank, count);
|
||||
if (pairs.isEmpty()) {
|
||||
if (LambdaDCSExecutor.opts.superlativesFailOnEmptyLists)
|
||||
throw new LambdaDCSException(Type.emptyList, "Cannot call %s on an empty list.", mode);
|
||||
|
|
@ -343,7 +345,7 @@ public final class DenotationUtils {
|
|||
public boolean isCompatible(Value v) {
|
||||
return v instanceof NumberValue;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean isSortable(Collection<Value> values) {
|
||||
return true;
|
||||
|
|
@ -387,19 +389,19 @@ public final class DenotationUtils {
|
|||
public boolean isCompatible(Value v) {
|
||||
return v instanceof DateValue;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean isSortable(Collection<Value> values) {
|
||||
DateValue firstDate = null;
|
||||
for (Value value : values) {
|
||||
DateValue date = (DateValue) value;
|
||||
DateValue date = (DateValue) value;
|
||||
if (firstDate == null) {
|
||||
firstDate = date;
|
||||
} else {
|
||||
if ((firstDate.year == -1) != (date.year == -1)) return false;
|
||||
if ((firstDate.month == -1) != (date.month == -1)) return false;
|
||||
if ((firstDate.day == -1) != (date.day == -1)) return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
@ -465,5 +467,5 @@ public final class DenotationUtils {
|
|||
throw new LambdaDCSException(Type.typeMismatch, "Cannot compare values");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,169 @@
|
|||
# (Extended) Generic Grammar
|
||||
# Use more generic compositional patterns.
|
||||
|
||||
################################################################
|
||||
# Macros
|
||||
|
||||
(def @R reverse)
|
||||
(def @type fb:type.object.type)
|
||||
(def @row fb:type.row)
|
||||
|
||||
(def @next fb:row.row.next)
|
||||
(def @!next !fb:row.row.next)
|
||||
(def @index fb:row.row.index)
|
||||
(def @!index !fb:row.row.index)
|
||||
|
||||
(def @p.num fb:cell.cell.number)
|
||||
(def @!p.num !fb:cell.cell.number)
|
||||
(def @p.date fb:cell.cell.date)
|
||||
(def @!p.date !fb:cell.cell.date)
|
||||
(def @p.second fb:cell.cell.second)
|
||||
(def @!p.second !fb:cell.cell.second)
|
||||
|
||||
################################################################
|
||||
# Lexicon
|
||||
|
||||
################################
|
||||
# Anchored Rules: Entity, Unary, Binary
|
||||
(rule $Entity ($PHRASE) (FuzzyMatchFn entity) (anchored 1))
|
||||
#(rule $Binary ($PHRASE) (FuzzyMatchFn binary) (anchored 1))
|
||||
(rule $Entity ($PHRASE) (NumberFn) (anchored 1))
|
||||
(rule $Entity ($PHRASE) (DateFn) (anchored 1))
|
||||
|
||||
################################
|
||||
# Create binary from thin air
|
||||
(rule $Binary (nothing) (FuzzyMatchFn any binary))
|
||||
(rule $Unary (nothing) (FuzzyMatchFn any unary))
|
||||
|
||||
################################
|
||||
# Property
|
||||
(for @property (@p.num @p.date)
|
||||
(rule $Property (nothing) (ConstantFn @property))
|
||||
)
|
||||
(when second
|
||||
(rule $Property (nothing) (ConstantFn @p.second))
|
||||
)
|
||||
|
||||
################################
|
||||
# Generic RowSet
|
||||
(rule $RowSet (nothing) (ConstantFn (@type @row)))
|
||||
|
||||
################################
|
||||
# Anchored ValueSet
|
||||
(rule $ValueSet ($Entity) (IdentityFn))
|
||||
|
||||
# [TAG] alternative: "X or Y" questions
|
||||
(when alternative
|
||||
(rule $ValueSet ($Entity $Entity)
|
||||
(lambda e1 (lambda e2 (or (var e1) (var e2))))
|
||||
)
|
||||
)
|
||||
|
||||
################################
|
||||
# Join
|
||||
|
||||
(rule $RowSet ($Binary $ValueSet) (lambda b (lambda v ((var b) (var v)))))
|
||||
|
||||
(rule $ValueSet ($Binary $RowSet) (lambda b (lambda r ((@R (var b)) (var r)))))
|
||||
|
||||
(rule $RowSet ($Binary $Property $ValueSet)
|
||||
(lambda b (lambda p (lambda v ((var b) ((var p) (var v))))))
|
||||
)
|
||||
|
||||
(rule $ValueSet ($Binary $Property $RowSet)
|
||||
(lambda b (lambda p (lambda r ((@R (var p)) ((@R (var b)) (var r))))))
|
||||
)
|
||||
|
||||
# [TAG] movement: "next" / "previous"
|
||||
(when movement
|
||||
(for @movement (@next @!next)
|
||||
(rule $RowSet ($RowSet) (lambda r (@movement (var r))))
|
||||
)
|
||||
)
|
||||
|
||||
# [TAG] comparison: "at least" / "more than"
|
||||
(when comparison
|
||||
(for @comparison (< > <= >=)
|
||||
(rule $RowSet ($Binary $Property $Entity)
|
||||
(lambda b (lambda p (lambda e ((var b) ((var p) (@comparison (var e)))))))
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# [TAG] != : "not zero" / "same"
|
||||
(when neq
|
||||
(rule $RowSet ($Binary $Entity) (lambda b (lambda e ((var b) (!= (var e))))))
|
||||
(rule $RowSet ($Binary $Property $Entity)
|
||||
(lambda b (lambda p (lambda e ((var b) ((var p) (!= (var e)))))))
|
||||
)
|
||||
)
|
||||
|
||||
################################
|
||||
# Aggregate
|
||||
|
||||
(when count
|
||||
(rule $SingleValue ($RowSet) (lambda r (count (var r))))
|
||||
)
|
||||
|
||||
(when aggregate
|
||||
(rule $SingleValue ($ValueSet) (lambda r (min (var r))))
|
||||
(rule $SingleValue ($ValueSet) (lambda r (max (var r))))
|
||||
(rule $SingleValue ($ValueSet) (lambda r (sum (var r))))
|
||||
(rule $SingleValue ($ValueSet) (lambda r (avg (var r))))
|
||||
)
|
||||
|
||||
################################
|
||||
# Superlative
|
||||
|
||||
(rule $FnOnRow ($Binary $Property)
|
||||
(lambda b (lambda p (lambda x ((@R (var p)) ((@R (var b)) (var x))))))
|
||||
)
|
||||
|
||||
(when superlative
|
||||
(for @argm (argmax argmin)
|
||||
(rule $RowSet ($RowSet) (lambda r (@argm 1 1 (var r) @index)))
|
||||
(rule $RowSet ($RowSet $FnOnRow) (lambda r (lambda f (@argm 1 1 (var r) (@R (var f))))))
|
||||
)
|
||||
)
|
||||
|
||||
################################
|
||||
# Merge
|
||||
|
||||
(when merge
|
||||
(rule $RowSet ($RowSet $RowSet)
|
||||
(lambda r1 (lambda r2 (and (var r1) (var r2))))
|
||||
)
|
||||
)
|
||||
|
||||
################################
|
||||
# Arithmatic
|
||||
|
||||
(rule $FnOnValue ($Binary $Binary $Property)
|
||||
(lambda b1 (lambda b2 (lambda p (lambda x ((@R (var p)) ((@R (var b2)) ((var b1) (var x))))))))
|
||||
)
|
||||
(rule $FnOnValue ($Binary)
|
||||
(lambda b (lambda x (count ((var b) (var x)))))
|
||||
)
|
||||
(rule $FnOnValue ($Binary $Property)
|
||||
(lambda b (lambda p (lambda x (count ((var b) ((var p) (var x)))))))
|
||||
)
|
||||
|
||||
(when arithmetic
|
||||
(rule $SingleValue ($FnOnValue $Entity $Entity)
|
||||
(lambda f (lambda e1 (lambda e2 (- ((var f) (var e1)) ((var f) (var e2))))))
|
||||
)
|
||||
)
|
||||
|
||||
################################
|
||||
# V-superlative
|
||||
|
||||
(when v-superlative
|
||||
(for @argm (argmax argmin)
|
||||
(rule $ValueSet ($ValueSet $FnOnValue) (lambda v (lambda f (@argm 1 1 (var v) (reverse (var f))))))
|
||||
)
|
||||
)
|
||||
|
||||
################################
|
||||
# ROOT
|
||||
(rule $ROOT ($ValueSet) (IdentityFn))
|
||||
(rule $ROOT ($SingleValue) (IdentityFn))
|
||||
Loading…
Reference in New Issue