From b2f0ff1aa8daf1c2e0573784f3add5e6ed4a56eb Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Mon, 10 Jul 2017 19:26:26 -0700 Subject: [PATCH 01/30] Write predicted output to a TSV file. --- run | 12 +++++++--- src/edu/stanford/nlp/sempre/BooleanValue.java | 3 +++ src/edu/stanford/nlp/sempre/DateValue.java | 1 + src/edu/stanford/nlp/sempre/ExampleUtils.java | 23 +++++++++++++++++++ src/edu/stanford/nlp/sempre/Learner.java | 11 ++++++--- src/edu/stanford/nlp/sempre/NameValue.java | 1 + src/edu/stanford/nlp/sempre/NumberValue.java | 1 + src/edu/stanford/nlp/sempre/StringValue.java | 1 + src/edu/stanford/nlp/sempre/UriValue.java | 3 +++ src/edu/stanford/nlp/sempre/Value.java | 3 +++ tables/README.md | 2 +- 11 files changed, 54 insertions(+), 7 deletions(-) diff --git a/run b/run index 2362606..f79ecfe 100755 --- a/run +++ b/run @@ -652,7 +652,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( }), # Fig parameters selo(:cldir, 'execDir', '_OUTPATH_', '.'), - o('overwriteExecDir'), o('addToView', 13), o('jarFiles', 'libsempre/*'), + o('overwriteExecDir'), o('addToView', 15), o('jarFiles', 'libsempre/*'), sel(:cldir, l(), '>/dev/null'), # Set environment for table execution o('executor', 'tables.lambdadcs.LambdaDCSExecutor'), @@ -660,6 +660,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( o('NumberFn.unitless'), o('NumberFn.alsoTestByConversion'), o('TypeInference.typeLookup', 'tables.TableTypeLookup'), o('JoinFn.specializedTypeCheck', false), o('JoinFn.typeInference', true), + o('Learner.outputPredValues'), # Value Evaluator letDefault(:eval, 'value'), sel(:eval, { @@ -741,19 +742,24 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( tablesDataPaths, # Verbosity o('FeatureVector.ignoreZeroWeight'), - o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10), o('logFeaturesLimit', 10), + o('logFeaturesLimit', 10), o('LambdaDCSException.noErrorMessage'), letDefault(:verbose, 0), sel(:verbose, - l(), l( + o('maxPrintedPredictions', 1), o('maxPrintedTrue', 1), + nil), + l( + o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10), o('putCellNameInCanonicalUtterance'), o('showUtterance'), nil), l( + o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10), o('putCellNameInCanonicalUtterance'), o('showUtterance'), o('summarizeRuleTime'), o('summarizeDenotations'), nil), l( + o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10), o('putCellNameInCanonicalUtterance'), o('showUtterance'), o('summarizeRuleTime'), o('summarizeDenotations'), o('showRules'), diff --git a/src/edu/stanford/nlp/sempre/BooleanValue.java b/src/edu/stanford/nlp/sempre/BooleanValue.java index ae478b8..497683d 100644 --- a/src/edu/stanford/nlp/sempre/BooleanValue.java +++ b/src/edu/stanford/nlp/sempre/BooleanValue.java @@ -19,6 +19,9 @@ public class BooleanValue extends Value { return tree; } + @Override public String sortString() { return "" + value; } + @Override public String pureString() { return "" + value; } + @Override public int hashCode() { return Boolean.valueOf(value).hashCode(); } @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/edu/stanford/nlp/sempre/DateValue.java b/src/edu/stanford/nlp/sempre/DateValue.java index 56d186d..3aa71f9 100644 --- a/src/edu/stanford/nlp/sempre/DateValue.java +++ b/src/edu/stanford/nlp/sempre/DateValue.java @@ -86,6 +86,7 @@ public class DateValue extends Value { + "-" + (month == -1 ? "xx" : String.format("%02d", month)) + "-" + (day == -1 ? "xx" : String.format("%02d", day)); } + @Override public String pureString() { return isoString(); } @Override public int hashCode() { int hash = 0x7ed55d16; diff --git a/src/edu/stanford/nlp/sempre/ExampleUtils.java b/src/edu/stanford/nlp/sempre/ExampleUtils.java index 9bc16dd..91a317b 100644 --- a/src/edu/stanford/nlp/sempre/ExampleUtils.java +++ b/src/edu/stanford/nlp/sempre/ExampleUtils.java @@ -119,6 +119,29 @@ public final class ExampleUtils { out.close(); } + public static void writePredictionTSV(int iter, String group, Example ex) { + String basePath = "preds-iter" + iter + "-" + group + ".tsv"; + String outPath = Execution.getFile(basePath); + if (outPath == null) return; + PrintWriter out = IOUtils.openOutAppendHard(outPath); + + List fields = new ArrayList<>(); + fields.add(ex.id); + + if (!ex.predDerivations.isEmpty()) { + Derivation deriv = ex.predDerivations.get(0); + if (deriv.value instanceof ListValue) { + List values = ((ListValue) deriv.value).values; + for (Value v : values) { + fields.add(v.pureString().replaceAll("\\s+", " ").trim()); + } + } + } + + out.println(String.join("\t", fields)); + out.close(); + } + //read lisptree and write json public static void main(String[] args) { Dataset dataset = new Dataset(); diff --git a/src/edu/stanford/nlp/sempre/Learner.java b/src/edu/stanford/nlp/sempre/Learner.java index 6b2ce83..dc45d03 100644 --- a/src/edu/stanford/nlp/sempre/Learner.java +++ b/src/edu/stanford/nlp/sempre/Learner.java @@ -27,6 +27,8 @@ public class Learner { @Option(gloss = "Write predDerivations to examples file (huge)") public boolean outputPredDerivations = false; + @Option(gloss = "Write predicted values to a TSV file") + public boolean outputPredValues = false; @Option(gloss = "Dump all features and compatibility scores") public boolean dumpFeaturesAndCompatibility = false; @@ -148,7 +150,7 @@ public class Learner { params.update(counts); LogInfo.end_track(); } - + public void onlineLearnExampleByFormula(Example ex, List formulas) { HashMap counts = new HashMap<>(); for (Derivation deriv : ex.predDerivations) @@ -222,8 +224,11 @@ public class Learner { addFeedback(ex); // Write out examples and predictions - if (opts.outputPredDerivations && Builder.opts.parser.equals("FloatingParser")) { - ExampleUtils.writeParaphraseSDF(iter, group, ex, opts.outputPredDerivations); + if (opts.outputPredDerivations) { + ExampleUtils.writeParaphraseSDF(iter, group, ex, true); + } + if (opts.outputPredValues) { + ExampleUtils.writePredictionTSV(iter, group, ex); } // To save memory diff --git a/src/edu/stanford/nlp/sempre/NameValue.java b/src/edu/stanford/nlp/sempre/NameValue.java index 3a82556..38d99f9 100644 --- a/src/edu/stanford/nlp/sempre/NameValue.java +++ b/src/edu/stanford/nlp/sempre/NameValue.java @@ -42,6 +42,7 @@ public class NameValue extends Value { } @Override public String sortString() { return id; } + @Override public String pureString() { return description == null ? id : description; } @Override public int hashCode() { return id.hashCode(); } @Override public boolean equals(Object o) { diff --git a/src/edu/stanford/nlp/sempre/NumberValue.java b/src/edu/stanford/nlp/sempre/NumberValue.java index 6408f1b..cd48725 100644 --- a/src/edu/stanford/nlp/sempre/NumberValue.java +++ b/src/edu/stanford/nlp/sempre/NumberValue.java @@ -40,6 +40,7 @@ public class NumberValue extends Value { } @Override public String sortString() { return "" + value; } + @Override public String pureString() { return "" + value; } @Override public int hashCode() { return Double.valueOf(value).hashCode(); } @Override public boolean equals(Object o) { diff --git a/src/edu/stanford/nlp/sempre/StringValue.java b/src/edu/stanford/nlp/sempre/StringValue.java index b67c426..91ebda6 100644 --- a/src/edu/stanford/nlp/sempre/StringValue.java +++ b/src/edu/stanford/nlp/sempre/StringValue.java @@ -20,6 +20,7 @@ public class StringValue extends Value { } @Override public String sortString() { return "\"" + value + "\""; } + @Override public String pureString() { return value; } @Override public int hashCode() { return value.hashCode(); } @Override public boolean equals(Object o) { diff --git a/src/edu/stanford/nlp/sempre/UriValue.java b/src/edu/stanford/nlp/sempre/UriValue.java index baec2d6..e039898 100644 --- a/src/edu/stanford/nlp/sempre/UriValue.java +++ b/src/edu/stanford/nlp/sempre/UriValue.java @@ -20,6 +20,9 @@ public class UriValue extends Value { return tree; } + @Override public String sortString() { return "" + value; } + @Override public String pureString() { return "" + value; } + @Override public int hashCode() { return value.hashCode(); } @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/edu/stanford/nlp/sempre/Value.java b/src/edu/stanford/nlp/sempre/Value.java index a69aa2a..486d5ec 100644 --- a/src/edu/stanford/nlp/sempre/Value.java +++ b/src/edu/stanford/nlp/sempre/Value.java @@ -24,6 +24,9 @@ public abstract class Value { // (optional) String used for sorting Values. The default is to call toString() public String sortString() { return toString(); } + // (optional) String without the LispTree structure. The default is to call toString() + public String pureString() { return toString(); } + @JsonCreator public static Value fromString(String str) { return Values.fromLispTree(LispTree.proto.parseFromString(str)); diff --git a/tables/README.md b/tables/README.md index 2197eb8..700944a 100644 --- a/tables/README.md +++ b/tables/README.md @@ -33,7 +33,7 @@ Running the code ./run @mode=tables @data=u-1 @feat=all @train=1 -maxex train,100 dev,100 - The command should take less than an hour. + The command should take less than 30 minutes. * To train on the complete development set, remove `-maxex train,100 dev,100` From f291f52ef43a190a5b05c4601ca38b4a75d643fb Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Mon, 10 Jul 2017 20:30:39 -0700 Subject: [PATCH 02/30] Changed how table target values are canonicalized --- run | 1 + src/edu/stanford/nlp/sempre/Example.java | 2 +- .../nlp/sempre/TargetValuePreprocessor.java | 4 +- .../tables/StringNormalizationUtils.java | 17 +++- .../sempre/tables/TableValuePreprocessor.java | 85 ++++++++++++++++++- .../tables/alter/AggregatedTurkData.java | 2 +- 6 files changed, 101 insertions(+), 10 deletions(-) diff --git a/run b/run index f79ecfe..72df538 100755 --- a/run +++ b/run @@ -917,6 +917,7 @@ def tablesDataPaths # That's it! l( o('TableKnowledgeGraph.baseCSVDir', csvDir), + o('TableValuePreprocessor.taggedFiles', "#{csvDir}/tagged/data/"), sel(:data, datasets), nil) } diff --git a/src/edu/stanford/nlp/sempre/Example.java b/src/edu/stanford/nlp/sempre/Example.java index 22f878e..2f93413 100644 --- a/src/edu/stanford/nlp/sempre/Example.java +++ b/src/edu/stanford/nlp/sempre/Example.java @@ -198,7 +198,7 @@ public class Example { public void preprocess() { this.languageInfo = LanguageAnalyzer.getSingleton().analyze(this.utterance); - this.targetValue = TargetValuePreprocessor.getSingleton().preprocess(this.targetValue); + this.targetValue = TargetValuePreprocessor.getSingleton().preprocess(this.targetValue, this); } public void log() { diff --git a/src/edu/stanford/nlp/sempre/TargetValuePreprocessor.java b/src/edu/stanford/nlp/sempre/TargetValuePreprocessor.java index ac1c817..40d3938 100644 --- a/src/edu/stanford/nlp/sempre/TargetValuePreprocessor.java +++ b/src/edu/stanford/nlp/sempre/TargetValuePreprocessor.java @@ -27,10 +27,10 @@ public abstract class TargetValuePreprocessor { } public static void setSingleton(TargetValuePreprocessor processor) { singleton = processor; } - public abstract Value preprocess(Value value); + public abstract Value preprocess(Value value, Example ex); } class IdentityTargetValuePreprocessor extends TargetValuePreprocessor { - public Value preprocess(Value value) { return value; } + public Value preprocess(Value value, Example ex) { return value; } } diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java index 3093007..8d06fff 100644 --- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java @@ -200,22 +200,31 @@ public final class StringNormalizationUtils { return null; } - public static final DateTimeFormatter dateFormat = DateTimeFormat.forPattern("MMM d, yyyy"); + public static final DateTimeFormatter americanDateFormat = DateTimeFormat.forPattern("MMM d, yyyy"); + public static final Pattern suTimeDateFormat = Pattern.compile("([0-9X]{4})(?:-([0-9X]{2}))?(?:-([0-9X]{2}))?"); /** * Convert string to DateValue. */ public static DateValue parseDate(String s) { + Matcher matcher = suTimeDateFormat.matcher(s.toUpperCase()); + if (matcher.matches()) { + String yS = matcher.group(1), mS = matcher.group(2), dS = matcher.group(3); + int y = -1, m = -1, d = -1; + if (!(yS == null || yS.isEmpty() || yS.contains("X"))) y = Integer.parseInt(yS); + if (!(mS == null || mS.isEmpty() || mS.contains("X"))) m = Integer.parseInt(mS); + if (!(dS == null || dS.isEmpty() || dS.contains("X"))) d = Integer.parseInt(dS); + if (y == -1 && m == -1 && d == -1) return null; + return new DateValue(y, m, d); + } try { - DateTime date = dateFormat.parseDateTime(s); + DateTime date = americanDateFormat.parseDateTime(s); return new DateValue(date.getYear(), date.getMonthOfYear(), date.getDayOfMonth()); } catch (IllegalArgumentException e) { return null; } } - public static final Pattern suTimeDateFormat = Pattern.compile("([0-9X]{4})(?:-([0-9X]{2}))?(?:-([0-9X]{2}))?"); - public static DateValue parseDateWithLanguageAnalyzer(LanguageInfo languageInfo) { if (languageInfo.numTokens() == 0) return null; String nerSpan = languageInfo.getNormalizedNerSpan("DATE", 0, languageInfo.numTokens()); diff --git a/src/edu/stanford/nlp/sempre/tables/TableValuePreprocessor.java b/src/edu/stanford/nlp/sempre/tables/TableValuePreprocessor.java index 49b7eed..4e62bfc 100644 --- a/src/edu/stanford/nlp/sempre/tables/TableValuePreprocessor.java +++ b/src/edu/stanford/nlp/sempre/tables/TableValuePreprocessor.java @@ -1,5 +1,9 @@ package edu.stanford.nlp.sempre.tables; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; import java.util.*; import edu.stanford.nlp.sempre.*; @@ -8,15 +12,20 @@ import fig.basic.*; public class TableValuePreprocessor extends TargetValuePreprocessor { public static class Options { @Option(gloss = "Verbosity") public int verbose = 0; + @Option(gloss = "Read preprocessed values from these .tagged files") + public List taggedFiles = new ArrayList<>(); } public static Options opts = new Options(); @Override - public Value preprocess(Value value) { + public Value preprocess(Value value, Example ex) { + if (!opts.taggedFiles.isEmpty() && ex != null) { + return getFromTaggedFile(ex.id); + } if (value instanceof ListValue) { List values = new ArrayList<>(); for (Value entry : ((ListValue) value).values) { - values.add(preprocess(entry)); + values.add(preprocessSingle(entry)); } return new ListValue(values); } else { @@ -67,4 +76,76 @@ public class TableValuePreprocessor extends TargetValuePreprocessor { return new DescriptionValue(origString); } + // ============================================================ + // Get preprocessed value from tagged file + // ============================================================ + + Map idToValue = null; + + public Value getFromTaggedFile(String id) { + if (idToValue == null) readTaggedFiles(); + return idToValue.get(id); + } + + protected void readTaggedFiles() { + LogInfo.begin_track("Reading .tagged files"); + idToValue = new HashMap<>(); + for (String path : opts.taggedFiles) { + File file = new File(path); + if (file.isDirectory()) { + for (File subpath : file.listFiles()) + readTaggedFile(subpath.toString()); + } else { + readTaggedFile(path); + } + } + LogInfo.logs("Read %d entries", idToValue.size()); + LogInfo.end_track(); + } + + protected void readTaggedFile(String path) { + LogInfo.begin_track("Reading %s", path); + try (BufferedReader reader = new BufferedReader(new FileReader(path))) { + // Read header + String[] header = reader.readLine().split("\t", -1); + int exIdIndex = 0, targetCanonIndex = 0; + while (!"id".equals(header[exIdIndex])) + exIdIndex++; + while (!"targetCanon".equals(header[targetCanonIndex])) + targetCanonIndex++; + // Read each line + String line; + while ((line = reader.readLine()) != null) { + String[] fields = line.split("\t", -1); // Include trailing spaces + String[] rawValues = fields[targetCanonIndex].split("\\|"); + List values = new ArrayList<>(); + for (String rawValue : rawValues) { + values.add(simpleCanonicalize(rawValue)); + } + idToValue.put(fields[exIdIndex], new ListValue(values)); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + LogInfo.end_track(); + } + + /** + * Like canonicalize, but assume that the string is already well-formed: + * - A number should look like a float + * - A date should be in the ISO format + * - Otherwise, the value is treated as a string. + */ + protected Value simpleCanonicalize(String origString) { + Value answer; + // Try converting to a number. + answer = StringNormalizationUtils.parseNumberStrict(origString); + if (answer != null) return answer; + // Try converting to a date. + answer = StringNormalizationUtils.parseDate(origString); + if (answer != null) return answer; + // Just treat as a description string + return new DescriptionValue(origString); + } + } diff --git a/src/edu/stanford/nlp/sempre/tables/alter/AggregatedTurkData.java b/src/edu/stanford/nlp/sempre/tables/alter/AggregatedTurkData.java index 0ebbf3e..e6ff578 100644 --- a/src/edu/stanford/nlp/sempre/tables/alter/AggregatedTurkData.java +++ b/src/edu/stanford/nlp/sempre/tables/alter/AggregatedTurkData.java @@ -93,7 +93,7 @@ public class AggregatedTurkData { List values = new ArrayList<>(); for (String x : response.split("\\|")) values.add(new DescriptionValue(StringNormalizationUtils.unescapeTSV(x))); - return TargetValuePreprocessor.getSingleton().preprocess(new ListValue(values)); + return TargetValuePreprocessor.getSingleton().preprocess(new ListValue(values), null); } } From af0957b445473a68b0e9a09fdde595ad295439c9 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Sun, 27 Aug 2017 13:49:56 -0700 Subject: [PATCH 03/30] Changed how values are evaluated --- .../tables/StringNormalizationUtils.java | 29 +++++++++++++++++++ .../sempre/tables/TableValueEvaluator.java | 8 +++++ 2 files changed, 37 insertions(+) diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java index 8d06fff..b5ad2fb 100644 --- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java @@ -352,6 +352,35 @@ public final class StringNormalizationUtils { return string.replaceAll("\\s+", " ").trim(); } + /** + * Normalization scheme in the official Python evaluator. + */ + public static String officialEvaluatorNormalize(String string) { + // Remove diacritics + string = Normalizer.normalize(string, Normalizer.Form.NFD).replaceAll("[\u0300-\u036F]", ""); + // Normalize quotes and dashes + string = string + .replaceAll("[‘’´`]", "'") + .replaceAll("[“”]", "\"") + .replaceAll("[‐‑‒–—−]", "-"); + String oldString; + do { + oldString = string; + // Remove citations + string = string.trim().replaceAll("((? Date: Sun, 27 Aug 2017 13:50:55 -0700 Subject: [PATCH 04/30] Updated the Wikipedia scraping scripts --- .../download-wikipedia-pages.py | 2 +- tables/wikipedia-scripts/find-good-tables.py | 95 +++++-- .../wikipedia-scripts/get-wikipedia-pages.py | 2 +- tables/wikipedia-scripts/table-to-csv.py | 243 +++++++----------- tables/wikipedia-scripts/weblib/table.py | 18 +- 5 files changed, 175 insertions(+), 185 deletions(-) diff --git a/tables/wikipedia-scripts/download-wikipedia-pages.py b/tables/wikipedia-scripts/download-wikipedia-pages.py index 516a2b4..0d3e4eb 100755 --- a/tables/wikipedia-scripts/download-wikipedia-pages.py +++ b/tables/wikipedia-scripts/download-wikipedia-pages.py @@ -28,7 +28,7 @@ def download(data, i, outdir): hashcode = data['hashcode'] = CACHE.get_hashcode(url) data['url'] = url result = CACHE.get_page(url) - if result is not None: + if result is not None and not os.path.exists(os.path.join(outdir, str(data['id']) + '.html')): os.symlink(os.path.join('..', 'web.cache', hashcode), os.path.join(outdir, str(data['id']) + '.html')) with open(os.path.join(outdir, str(data['id']) + '.json'), 'w') as fout: diff --git a/tables/wikipedia-scripts/find-good-tables.py b/tables/wikipedia-scripts/find-good-tables.py index 4f9d29a..8cf1c1c 100755 --- a/tables/wikipedia-scripts/find-good-tables.py +++ b/tables/wikipedia-scripts/find-good-tables.py @@ -49,10 +49,10 @@ class TableStat(object): self.num_min = min(table.num_rows, table.num_cols) self.num_max = max(table.num_rows, table.num_cols) self.num_cells = table.num_cells - self.num_empty_cells = len([x for x in table.cells if not x]) - self.num_long = len([x for x in table.cells if len(x) >= 40]) + self.num_empty_cells = len([x for x in table.cells if not x[1]]) + self.num_long = len([x for x in table.cells if len(x[1]) >= 40]) self.num_short_headers = len([x for x in table.rows[0] if len(x) <= 3]) - self.num_numeric_cells = len([x for x in table.cells if re.search('[0-9]', x)]) + self.num_numeric_cells = len([x for x in table.cells if re.search('[0-9]', x[1])]) self.num_repetitive_cols = sum([self.is_repetitive(col) for col in table.cols], 0) self.num_similar_colpairs = sum([self.are_similar(col1, col2) for (col1, col2) in combinations(table.cols, 2)], 0) @@ -61,11 +61,11 @@ class TableStat(object): self.get_scores() def is_repetitive(self, col): - return len(set(get_repeatable_token(x) for x in col)) < 0.5 * len(col) + return len(set(get_repeatable_token(x[1]) for x in col)) < 0.5 * len(col) def are_similar(self, col1, col2): - col1 = set(get_alpha(x) for x in col1) - set(['']) - col2 = set(get_alpha(x) for x in col2) - set(['']) + col1 = set(get_alpha(x[1]) for x in col1) - set(['']) + col2 = set(get_alpha(x[1]) for x in col2) - set(['']) return len(col1 & col2) >= 3 def __str__(self): @@ -119,6 +119,39 @@ def check_table(table, criterion): and stat.num_short_headers < stat.num_cols * 0.3): return stat +def dump(i, page_id, table_id, stat, args): + with open(os.path.join(args.source_dir, '%d.json' % page_id), 'r', 'utf8') as fin: + meta = json.load(fin) + meta['tableIndex'] = table_id + if args.custom_filenames: + while True: + try: + i = int(raw_input('filename for "%s": ' % meta['title'])) + break + except: + pass + with open(os.path.join(args.json_dir, '%d.json' % i), 'w', 'utf8') as fout: + json.dump(meta, fout) + if args.copy: + shutil.copy(os.path.join(args.source_dir, '%d.html' % page_id), + os.path.join(args.page_dir, '%d.html' % i)) + else: + os.symlink(os.path.relpath(os.path.join(args.source_dir, '%d.html' % page_id), + args.page_dir), + os.path.join(args.page_dir, '%d.html' % i)) + print '>' * 30, 'Written', i, page_id, table_id, '<' * 30 + print stat + +def read_used_pages(json_files, args): + used_pages = set() + for filename in json_files: + with open(os.path.join(args.json_dir, filename)) as fin: + data = json.load(fin) + used_pages.add(data['id']) + print 'Found {} used pages'.format(len(used_pages)) + return used_pages + + def main(): parser = argparse.ArgumentParser() parser.add_argument('-n', '--amount', type=int, default=20, @@ -130,26 +163,41 @@ def main(): help="criterion for finding tables (higher = more strict)") parser.add_argument('-d', '--dry-run', action='store_true', help="do not copy files") + parser.add_argument('-C', '--copy', action='store_true', + help='copy the html file instead of making a symlink') parser.add_argument('-i', '--fixed-id', type=int, help="use this fixed page id") parser.add_argument('--custom-filenames', action='store_true', help="set custom filename for each file") + parser.add_argument('-q', '--quick', action='store_true', + help='dump the good tables immediately after they are found') + parser.add_argument('-r', '--resume-quick', action='store_true', + help='continue the quick mode') parser.add_argument('outprefix') args = parser.parse_args() - json_dir = os.path.join(args.outprefix + '-json') - page_dir = os.path.join(args.outprefix + '-page') - if os.path.exists(json_dir) or os.path.exists(page_dir): + quick_index, quick_used_pages = 0, set() + args.json_dir = os.path.join(args.outprefix + '-json') + args.page_dir = os.path.join(args.outprefix + '-page') + if args.resume_quick: + # Read the highest number in json_dir + json_files = os.listdir(args.json_dir) + print 'Found {} JSON files'.format(len(json_files)) + quick_index = max(int(x.replace('.json', '')) for x in json_files) + 1 + quick_used_pages = read_used_pages(json_files, args) + elif os.path.exists(args.json_dir) or os.path.exists(args.page_dir): if raw_input('Path exists. Clobber? ')[0:].lower() != 'y': exit(1) elif not args.dry_run: - os.makedirs(json_dir) - os.makedirs(page_dir) + os.makedirs(args.json_dir) + os.makedirs(args.page_dir) if args.fixed_id is not None: filenames = [args.fixed_id] else: filenames = get_filenames(args.source_dir) + if args.quick: + filenames = sorted(set(filenames) - set(quick_used_pages)) print >> sys.stderr, 'Got %d candidates' % len(filenames) random.shuffle(filenames) filenames = filenames[:args.limit] @@ -171,6 +219,14 @@ def main(): print '%2d' % sum(stat.scores), print 'SCORE %9d (ID %9d) table %2d' % (page_id, meta['id'], table_id), print stat, stat.scores + if args.quick: + for table_id, stat in good_table_ids: + dump(quick_index, page_id, table_id, stat, args) + quick_index += 1 + + if args.quick: + return + random.shuffle(good_tables) good_tables = good_tables[:args.amount] if args.dry_run: @@ -178,22 +234,7 @@ def main(): for i, (page_id, good_table_ids) in enumerate(good_tables): table_id, stat = random.choice(good_table_ids) - with open(os.path.join(args.source_dir, '%d.json' % page_id), 'r', 'utf8') as fin: - meta = json.load(fin) - meta['tableIndex'] = table_id - if args.custom_filenames: - while True: - try: - i = int(raw_input('filename for "%s": ' % meta['title'])) - break - except: - pass - with open(os.path.join(json_dir, '%d.json' % i), 'w', 'utf8') as fout: - json.dump(meta, fout) - shutil.copy(os.path.join(args.source_dir, '%d.html' % page_id), - os.path.join(page_dir, '%d.html' % i)) - print '>' * 30, 'Written', i, page_id, table_id, '<' * 30 - print stat + dump(i, page_id, table_id, stat, args) if __name__ == '__main__': main() diff --git a/tables/wikipedia-scripts/get-wikipedia-pages.py b/tables/wikipedia-scripts/get-wikipedia-pages.py index ed1d1f3..99743fc 100755 --- a/tables/wikipedia-scripts/get-wikipedia-pages.py +++ b/tables/wikipedia-scripts/get-wikipedia-pages.py @@ -12,7 +12,7 @@ html_parser = HTMLParser.HTMLParser() def is_sane(content, args): if content.startswith('#REDIRECT'): return False - if args.table and '{|' not in content: + if args.table and ('{|' not in content or 'wikitable' not in content): return False return True diff --git a/tables/wikipedia-scripts/table-to-csv.py b/tables/wikipedia-scripts/table-to-csv.py index afdfdb0..dbe0dbe 100755 --- a/tables/wikipedia-scripts/table-to-csv.py +++ b/tables/wikipedia-scripts/table-to-csv.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +"""Convert HTML table into CSV / TSV / pretty-printed table.""" import sys, os, re, argparse, json from codecs import open @@ -10,24 +11,40 @@ from itertools import izip_longest ################ Dump CSV def simple_normalize_text(text): - return text.replace('\\', '\\\\').replace('"', r'\"').replace('\n', r'\n').replace(u'\xa0', ' ').strip() + return text.replace('\\', '\\\\').replace('"', r'\"').replace('\n', r'\\n').replace(u'\xa0', ' ').strip() def dump_csv(rows, fout): for row in rows: - fout.write(','.join('"%s"' % simple_normalize_text(x) for x in row) + '\n') + fout.write(','.join('"%s"' % simple_normalize_text(x[1]) for x in row) + '\n') def tab_normalize_text(text): - return re.sub(r'\s+', ' ', text.replace('\n', r'\n'), re.U).strip() + return re.sub(r'\s+', ' ', text.replace('\\', '\\\\').replace('|', r'\p').replace('\n', r'\n'), re.U).strip() def dump_tsv(rows, fout): for row in rows: - fout.write('\t'.join('%s' % tab_normalize_text(x) for x in row) + '\n') + fout.write('\t'.join('%s' % tab_normalize_text(x[1]) for x in row) + '\n') + +def table_normalize_text(text): + return re.sub(r'\s+', ' ', text, re.U).strip() + +def dump_table(rows, fout): + widths = defaultdict(int) + for row in rows: + for i, cell in enumerate(row): + widths[i] = max(widths[i], len(table_normalize_text(cell[1])) + 1) + for row in rows: + fout.write('|') + for i, cell in enumerate(row): + # wow this is so hacky + fout.write((' %-' + str(widths[i]) + 's') % table_normalize_text(cell[1])) + fout.write('|') + fout.write('\n') ################ More table normalization def debug_print(stuff): for x in stuff: - print >> sys.stderr, [simple_normalize_text(y) for y in x] + print >> sys.stderr, [simple_normalize_text(y[1]) for y in x] def transpose(rows): cols = [] @@ -38,7 +55,7 @@ def transpose(rows): try: col.append(row[i]) except LookupError: - col.append(None) + col.append(('', '')) cols.append(col) return cols @@ -53,7 +70,7 @@ def anti_transpose(cols): if col[i] is not None: row.append(col[i]) else: - row.append('') + row.append(('', '')) rows.append(row) return rows @@ -65,7 +82,7 @@ def remove_empty_columns(orig_cols): """Remove columns with <= 1 non-empty cells.""" cols = [] for col in orig_cols: - non_empty = sum((bool(cell) for cell in col), 0) + non_empty = sum((bool(cell[1]) for cell in col), 0) if non_empty >= 2: cols.append(col) return cols @@ -77,9 +94,9 @@ def are_mergeable(col1, col2): merged = [] for i in xrange(len(col1)): c1, c2 = col1[i], col2[i] - if not c1: + if not c1[1]: merged.append(c2) - elif not c2 or c1 == c2: + elif not c2[1] or c1 == c2: merged.append(c1) else: return None @@ -96,161 +113,93 @@ def merge_similar_columns(orig_cols): i += 1 return orig_cols -#### Split columns +#### Merge header rows -REGEX_NEWLINE = re.compile(ur'^([^\n]*)\n([^\n]*)$', re.U | re.DOTALL) -REGEX_NEWLINE_PAREN = re.compile(ur'^(.*)\n(\(.*\))$', re.U | re.DOTALL) -REGEX_SPLIT_PAREN = re.compile(ur'^(.*) +(\(.*\))$', re.U | re.DOTALL) - -def split_multiline_columns(orig_cols): - """Split columns with newline in each cell.""" - i = 0 - while i < len(orig_cols): - for regex, threshold in ((REGEX_NEWLINE, 0.5 * len(orig_cols[i])), - (REGEX_NEWLINE_PAREN, 1), - (REGEX_SPLIT_PAREN, 2)): - matches = [regex.match(cell or '') for cell in orig_cols[i]] - num_matches = sum((bool(match) for match in matches[1:]), 0) - if num_matches >= threshold: - splitted = [((None, None) if cell is None - else (cell, '') if not match - else (match.group(1), match.group(2))) - for (cell, match) in zip(orig_cols[i], matches)] - orig_cols[i:i+1] = [[(None if x[0] is None else x[0].strip()) for x in splitted], - [(None if x[1] is None else x[1].strip()) for x in splitted]] - if not orig_cols[i+1][0]: - orig_cols[i+1][0] = (orig_cols[i][0] or '') + '#n' - break - i += 1 - return orig_cols - -REGEX_FROM_TO = re.compile(ur'^(.*)[-–—‒―](.*)$', re.U) - -def split_from_to_columns(orig_cols): - """Split columns with pattern '... - ...'.""" - i = 0 - while i < len(orig_cols): - matches = [REGEX_FROM_TO.match(cell or '') for cell in orig_cols[i]] - num_matches = sum((bool(match) for match in matches), 0) - if num_matches > 0.5 * len(orig_cols[i]): - splitted = [((None, None) if cell is None - else (cell, '') if not match - else (match.group(1), match.group(2))) - for (cell, match) in zip(orig_cols[i], matches)] - orig_cols[i:i+1] = [[(None if x[0] is None else x[0].strip()) for x in splitted], - [(None if x[1] is None else x[1].strip()) for x in splitted]] - if not orig_cols[i+1][0]: - orig_cols[i+1][0] = orig_cols[i][0] - orig_cols[i][0] += '#f' - orig_cols[i+1][0] += '#t' - i += 1 - return orig_cols - -#### Normalize by column - -REGEX_NUMBERING = re.compile(ur'^([0-9]+)[.)]$', re.U) -REGEX_REFERENCE = re.compile(ur'^(.*)[‡^†*]+$', re.U) -REGEX_PARENS = re.compile(ur'^\((.*)\)$', re.U) -REGEX_QUOTES = re.compile(ur'^"(.*)"$', re.U) - -def normalize_common_punctuations(orig_cols): - """Normalize some punctuations if they appear a lot in the same row.""" - cols = [] - for col in orig_cols: - for regex, threshold in ((REGEX_NUMBERING, 0.8 * len(col)), - (REGEX_REFERENCE, 0.2 * len(col)), - (REGEX_PARENS, 0.5 * len(col)), - (REGEX_QUOTES, 0.5 * len(col))): - matches = [regex.match(cell or '') for cell in col] - num_matches = sum((not cell or bool(match) - for (cell, match) in zip(col, matches)), 0) - if num_matches >= threshold: - col = [(cell if not match else match.group(1)) - for (cell, match) in zip(col, matches)] - cols.append(col) - return cols - -#### Normalize by cell - -REGEX_COMMA = re.compile(ur'([0-9]),([0-9][0-9][0-9])', re.U) - -def remove_commas_from_single_number(x): - if not x: - return x - while REGEX_COMMA.search(x): - x = REGEX_COMMA.sub(r'\1\2', x) - return x - -def remove_commas_from_numbers(orig_stuff): - stuff = [] - for slab in orig_stuff: - stuff.append([remove_commas_from_single_number(x) for x in slab]) - return stuff +def merge_header_rows(orig_rows): + """Merge all header rows together.""" + header_rows, body_rows = [], [] + still_header = True + for row in orig_rows: + if not still_header or any(cell[0] == 'td' for cell in row): + still_header = False + body_rows.append(row) + else: + header_rows.append(row) + if len(header_rows) < 2 or not body_rows: + return orig_rows + # Merge header rows with '\n' + header_cols = transpose(header_rows) + header_row = [] + for col in header_cols: + texts = [None] + for cell in col: + if cell[1] != texts[-1]: + texts.append(cell[1]) + header_row.append(('th', '\n'.join(texts[1:]))) + return [header_row] + body_rows ################ Main function def main(): parser = argparse.ArgumentParser() - parser.add_argument('-s', '--source-dir', default='wikidump.cache/output', - help="source directory") - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-j', '--json', - help="json metadata file specifying page and table id") - group.add_argument('-J', '--turk-json', + parser.add_argument('-j', '--turk-json', help="json metadata file from MTurk task") - group.add_argument('-p', '--page-id', type=int, - help="page index") - parser.add_argument('-t', '--table-id', type=int, default=0, - help="table index (only used for -p / --page-id)") parser.add_argument('-o', '--outfile', help="output filename (default = stdout)") - parser.add_argument('-n', '--normalize', action='count', - help='degree of normalization') parser.add_argument('--tsv', action='store_true', - help='output TSV instead of CSV') + help='also print out tsv') + parser.add_argument('--human', action='store_true', + help='also print out human-readable table') + parser.add_argument('--html', action='store_true', + help='also print out cleaned html for the table') + parser.add_argument('--keep-hidden', action='store_true', + help='keep hidden texts as is') args = parser.parse_args() + assert not args.tsv or args.outfile.endswith('.csv') - if args.json: - with open(args.json) as fin: - metadata = json.load(fin) - args.page_id = metadata['id'] - args.table_id = metadata['tableIndex'] - inhtml = os.path.join(args.source_dir, '%d.html' % args.page_id) - elif args.turk_json: - with open(args.turk_json) as fin: - metadata = json.load(fin) - args.page_id = metadata['id'] - args.table_id = metadata['tableIndex'] - # The following replacement is pretty hacky: - inhtml = args.turk_json.replace('-json', '-page').replace('.json', '.html') - else: - inhtml = os.path.join(args.source_dir, '%d.html' % args.page_id) + with open(args.turk_json) as fin: + metadata = json.load(fin) + + # Get the path to the HTML file + # This is kind of hacky + match = re.match(r'^(?:json|page)/(\d+)-(?:json|page)/(\d+).json$', args.turk_json) + batch_id, data_id = match.groups() + inhtml = 'page/{}-page/{}.html'.format(batch_id, data_id) with open(inhtml, 'r', 'utf8') as fin: - table = Table.get_wikitable(fin.read(), args.table_id, normalization=Table.NORM_DUPLICATE) + raw = fin.read() + table = Table.get_wikitable(raw, metadata['tableIndex'], + normalization=Table.NORM_DUPLICATE, + remove_hidden=(not args.keep_hidden)) + if args.html: + raw_table = Table.get_wikitable(raw, metadata['tableIndex'], + remove_hidden=False).table + rows = table.rows - if args.normalize >= 1: - # Remove redundant rows and columns - rows = remove_full_rowspans(rows) - cols = transpose(rows) - cols = remove_empty_columns(cols) - cols = merge_similar_columns(cols) - rows = anti_transpose(cols) - if args.normalize >= 2: - # Split cells / Normalize texts - cols = transpose(rows) - cols = split_multiline_columns(cols) - cols = normalize_common_punctuations(cols) - cols = split_from_to_columns(cols) - cols = remove_commas_from_numbers(cols) - rows = anti_transpose(cols) - #debug_print(transpose(rows)) - outputter = dump_tsv if args.tsv else dump_csv + # rows = list of columns; column = list of cells; cell = (tag, text) + # Remove redundant rows and columns + rows = remove_full_rowspans(rows) + cols = transpose(rows) + cols = remove_empty_columns(cols) + cols = merge_similar_columns(cols) + rows = anti_transpose(cols) + rows = merge_header_rows(rows) + # Dump if not args.outfile: - outputter(rows, sys.stdout) + dump_csv(rows, sys.stdout) else: + stem = re.sub('\.csv$', '', args.outfile) with open(args.outfile, 'w', 'utf8') as fout: - outputter(rows, fout) + dump_csv(rows, fout) + if args.tsv: + with open(stem + '.tsv', 'w', 'utf8') as fout: + dump_tsv(rows, fout) + if args.human: + with open(stem + '.table', 'w', 'utf8') as fout: + dump_table(rows, fout) + if args.html: + with open(stem + '.html', 'w', 'utf8') as fout: + print >> fout, unicode(raw_table) if __name__ == '__main__': main() diff --git a/tables/wikipedia-scripts/weblib/table.py b/tables/wikipedia-scripts/weblib/table.py index e39975c..bab1c9f 100644 --- a/tables/wikipedia-scripts/weblib/table.py +++ b/tables/wikipedia-scripts/weblib/table.py @@ -7,7 +7,9 @@ Get statistics about a table and convert it to CSV. import sys, os, re, json from codecs import open from collections import defaultdict -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup as BeautifulSoupOriginal +def BeautifulSoup(markup=""): + return BeautifulSoupOriginal(markup, 'html.parser') class Table(object): NORM_NONE = 0 @@ -21,14 +23,11 @@ class Table(object): self.table = table if remove_hidden: self.remove_hidden() - if normalization == Table.NORM_NONE: - self.get_cells() - elif normalization == Table.NORM_CORNER: + if normalization == Table.NORM_CORNER: self.normalize_table() - self.get_cells() elif normalization == Table.NORM_DUPLICATE: self.normalize_table(deep=True) - self.get_cells() + self.get_cells() @staticmethod def get_wikitable(raw_html, index=None, **kwargs): @@ -53,11 +52,12 @@ class Table(object): tag.extract() def get_cells(self): + """Each cell is (tag, text)""" self.rows, self.cells = [], [] for x in self.table.find_all('tr', recursive=False): row = [] for y in x.find_all(['th', 'td'], recursive=False): - row.append(y.text.strip()) + row.append((y.name, y.text.strip())) self.rows.append(row) self.cells.extend(row) self.num_rows = len(self.rows) @@ -82,7 +82,7 @@ class Table(object): def get_cloned_cell(self, cell, rowspan=1, deep=False): if deep: # Hacky but works - return BeautifulSoup(unicode(cell)).body.contents[0] + return BeautifulSoup(unicode(cell)).contents[0] tag = Table.SOUP.new_tag(cell.name) if rowspan > 1: tag['rowspan'] = rowspan @@ -152,4 +152,4 @@ def test(): print table.table if __name__ == '__main__': - test() + test_wiki() From f041f1db03f483d6279dbf2b14e7278d20f10331 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Wed, 3 May 2017 17:30:29 -0700 Subject: [PATCH 05/30] Added TableColumnAnalyzer --- run | 1 + .../tables/StringNormalizationUtils.java | 1 + .../tables/test/TableColumnAnalyzer.java | 164 ++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java diff --git a/run b/run index 72df538..c513830 100755 --- a/run +++ b/run @@ -649,6 +649,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( 'alter' => l('edu.stanford.nlp.sempre.tables.alter.BatchTableAlterer', let(:parser, 'serialized')), 'alter-ex' => l('edu.stanford.nlp.sempre.tables.alter.AlteredTablesExecutor', let(:parser, 'serialized')), 'filter' => 'edu.stanford.nlp.sempre.tables.serialize.DumpFilterer', + 'column' => 'edu.stanford.nlp.sempre.tables.test.TableColumnAnalyzer', }), # Fig parameters selo(:cldir, 'execDir', '_OUTPATH_', '.'), diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java index b5ad2fb..2efb648 100644 --- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java @@ -53,6 +53,7 @@ public final class StringNormalizationUtils { // Cell normalization // ============================================================ + public static final Pattern STRICT_DASH = Pattern.compile("\\s*[-‐‑⁃‒–—―]\\s*"); public static final Pattern DASH = Pattern.compile("\\s*[-‐‑⁃‒–—―/,:;]\\s*"); public static final Pattern COMMA = Pattern.compile("\\s*(,\\s|\\n|/)\\s*"); public static final Pattern SPACE = Pattern.compile("\\s+"); diff --git a/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java b/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java new file mode 100644 index 0000000..e78f2cd --- /dev/null +++ b/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java @@ -0,0 +1,164 @@ +package edu.stanford.nlp.sempre.tables.test; + +import java.io.*; +import java.util.*; + +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.tables.*; +import fig.basic.*; +import fig.exec.Execution; + +/** + * Analyze table columns and print out any hard-to-process column. + * + * @author ppasupat + */ +public class TableColumnAnalyzer implements Runnable { + public static class Options { + @Option(gloss = "Maximum number of tables to process (for debugging)") + public int maxNumTables = Integer.MAX_VALUE; + } + public static Options opts = new Options(); + + public static void main(String[] args) { + Execution.run(args, "TableColumnAnalyzerMain", new TableColumnAnalyzer(), Master.getOptionsParser()); + } + + PrintWriter out; + PrintWriter outCompact; + + @Override + public void run() { + out = IOUtils.openOutHard(Execution.getFile("column-stats.tsv")); + outCompact = IOUtils.openOutHard(Execution.getFile("column-compact.tsv")); + Map> tableIdToExIds = getTableIds(); + int tablesProcessed = 0; + for (Map.Entry> entry : tableIdToExIds.entrySet()) { + String tableId = entry.getKey(), + tableIdAbbrev = tableId.replaceAll("csv/(\\d+)-csv/(\\d+)\\.csv", "$1-$2"); + LogInfo.begin_track("Processing %s ...", tableId); + TableKnowledgeGraph graph = TableKnowledgeGraph.fromFilename(tableId); + out.printf("%s\tIDS\t%s\n", tableIdAbbrev, String.join(" ", entry.getValue())); + out.printf("%s\tCOLUMNS\t%d\n", tableIdAbbrev, graph.numColumns()); + for (int i = 0; i < graph.numColumns(); i++) { + analyzeColumn(graph, graph.columns.get(i), tableIdAbbrev + "\t" + i); + } + LogInfo.end_track(); + if (tablesProcessed++ >= opts.maxNumTables) break; + } + out.close(); + outCompact.close(); + } + + protected Map> getTableIds() { + Map> tableIdToExIds = new LinkedHashMap<>(); + LogInfo.begin_track_printAll("Collect table IDs"); + for (Pair pathPair : Dataset.opts.inPaths) { + String group = pathPair.getFirst(); + String path = pathPair.getSecond(); + Execution.putOutput("group", group); + LogInfo.begin_track("Reading %s", path); + Iterator trees = LispTree.proto.parseFromFile(path); + while (trees.hasNext()) { + LispTree tree = trees.next(); + if ("metadata".equals(tree.child(0).value)) continue; + String exId = null, tableId = null; + for (int i = 1; i < tree.children.size(); i++) { + LispTree arg = tree.child(i); + String label = arg.child(0).value; + if ("id".equals(label)) { + exId = arg.child(1).value; + } else if ("context".equals(label)) { + tableId = arg.child(1).child(2).value; + } + } + if (exId != null && tableId != null) { + List exIdsForTable = tableIdToExIds.get(tableId); + if (exIdsForTable == null) + tableIdToExIds.put(tableId, exIdsForTable = new ArrayList<>()); + exIdsForTable.add(exId); + } + } + LogInfo.end_track(); + } + LogInfo.end_track(); + LogInfo.logs("Got %d IDs", tableIdToExIds.size()); + return tableIdToExIds; + } + + protected void analyzeColumn(TableKnowledgeGraph graph, TableColumn column, String printPrefix) { + List escapedCells = new ArrayList<>(); + // Print the header + String h = column.originalString, escapedH = StringNormalizationUtils.escapeTSV(h); + out.printf("%s\t0\t%s\n", printPrefix, escapedH); + escapedCells.add(escapedH); + // Print the cells + Map typeCounts = new HashMap<>(); + for (int j = 0; j < column.children.size(); j++) { + TableCell cell = column.children.get(j); + String c = cell.properties.originalString, escapedC = StringNormalizationUtils.escapeTSV(c); + escapedCells.add(escapedC); + // Infer the type + List types = analyzeCell(c); + for (String type : types) + MapUtils.incr(typeCounts, type); + out.printf("%s\t%d\t%s\t%s\n", printPrefix, j + 1, String.join("|", types), escapedC); + } + // Analyze the common types + List commonTypes = new ArrayList<>(); + for (Map.Entry entry : typeCounts.entrySet()) { + if (entry.getValue() == column.children.size()) { + commonTypes.add(entry.getKey()); + } else if (entry.getValue() == column.children.size() - 1) { + commonTypes.add("ALMOST-" + entry.getKey()); + } + } + outCompact.printf("%s\t%s\t%s\n", String.join("|", commonTypes), printPrefix, String.join("\t", escapedCells)); + } + + // ============================================================ + // Cell analysis + // ============================================================ + + protected List analyzeCell(String c) { + List types = new ArrayList<>(); + LanguageInfo languageInfo = LanguageAnalyzer.getSingleton().analyze(c); + { + // Integer + NumberValue n = StringNormalizationUtils.parseNumberStrict(c); + if (n != null) { + // Number + types.add("num"); + // Integer + double value = n.value; + if (Math.abs(value - Math.round(value)) < 1e-9) { + types.add("int"); + if (c.matches("^[12]\\d\\d\\d$")) { + // Year? + types.add("year"); + } + } + } + } + { + // Ordinal + + } + { + // Integer-Integer + String[] splitted = StringNormalizationUtils.STRICT_DASH.split(c); + if (splitted.length == 2 && splitted[0].matches("^[0-9]+$") && splitted[1].matches("^[0-9]+$")) { + types.add("2ints"); + } + } + { + // Date + DateValue date = StringNormalizationUtils.parseDateWithLanguageAnalyzer(languageInfo); + if (date != null) { + types.add("date"); + } + } + return types; + } + +} From a5d99a4ac9cc02ccf054831904a757569be318bb Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Wed, 3 May 2017 18:13:31 -0700 Subject: [PATCH 06/30] Edited TableColumnAnalyzer --- .../tables/test/TableColumnAnalyzer.java | 74 ++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java b/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java index e78f2cd..af93f56 100644 --- a/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java +++ b/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java @@ -2,6 +2,7 @@ package edu.stanford.nlp.sempre.tables.test; import java.io.*; import java.util.*; +import java.util.regex.*; import edu.stanford.nlp.sempre.*; import edu.stanford.nlp.sempre.tables.*; @@ -17,6 +18,8 @@ public class TableColumnAnalyzer implements Runnable { public static class Options { @Option(gloss = "Maximum number of tables to process (for debugging)") public int maxNumTables = Integer.MAX_VALUE; + @Option(gloss = "Load Wikipedia article titles from this file") + public String wikiTitles = null; } public static Options opts = new Options(); @@ -120,6 +123,8 @@ public class TableColumnAnalyzer implements Runnable { // Cell analysis // ============================================================ + public static final Pattern ORDINAL = Pattern.compile("^(\\d+)(st|nd|rd|th)$"); + protected List analyzeCell(String c) { List types = new ArrayList<>(); LanguageInfo languageInfo = LanguageAnalyzer.getSingleton().analyze(c); @@ -142,7 +147,10 @@ public class TableColumnAnalyzer implements Runnable { } { // Ordinal - + Matcher m = ORDINAL.matcher(c); + if (m.matches()) { + types.add("ordinal"); + } } { // Integer-Integer @@ -156,9 +164,73 @@ public class TableColumnAnalyzer implements Runnable { DateValue date = StringNormalizationUtils.parseDateWithLanguageAnalyzer(languageInfo); if (date != null) { types.add("date"); + // Also more detailed date type + types.add("date-" + + (date.year != -1 ? "Y" : "") + + (date.month != -1 ? "M" : "") + + (date.day != -1 ? "D" : "")); } } + { + // Quoted text + if (c.matches("^[“”\"].*[“”\"]$")) { + types.add("quoted"); + } + } + if (opts.wikiTitles != null) { + // Wikipedia titles + WikipediaTitleLibrary library = WikipediaTitleLibrary.getSingleton(); + if (library.contains(c)) { + types.add("wiki"); + } + } + { + // POS and NER + types.add("POS=" + String.join("-", languageInfo.posTags)); + types.add("NER=" + String.join("-", languageInfo.nerTags)); + } return types; } + // ============================================================ + // Helper class: Wikipedia titles + // ============================================================ + + public static class WikipediaTitleLibrary { + + private static WikipediaTitleLibrary _singleton = null; + + public static WikipediaTitleLibrary getSingleton() { + if (_singleton == null) + _singleton = new WikipediaTitleLibrary(); + return _singleton; + } + + Set titles = new HashSet<>(); + + private WikipediaTitleLibrary() { + assert opts.wikiTitles != null; + LogInfo.begin_track("Reading Wikipedia article titles from %s ...", opts.wikiTitles); + try { + BufferedReader reader = IOUtils.openIn(opts.wikiTitles); + String line; + while ((line = reader.readLine()) != null) { + titles.add(line); + if (titles.size() <= 10) { + LogInfo.logs("Example title: %s", line); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + LogInfo.logs("Read %d titles", titles.size()); + LogInfo.end_track(); + } + + public boolean contains(String c) { + return titles.contains(c.toLowerCase().trim()); + } + } + + } From 58f8798cbed9d0b813d066d4c94037cfc9200ae8 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Wed, 3 May 2017 18:24:48 -0700 Subject: [PATCH 07/30] Added table id --- src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java b/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java index af93f56..a777b37 100644 --- a/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java +++ b/src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java @@ -37,6 +37,7 @@ public class TableColumnAnalyzer implements Runnable { Map> tableIdToExIds = getTableIds(); int tablesProcessed = 0; for (Map.Entry> entry : tableIdToExIds.entrySet()) { + Execution.putOutput("example", tablesProcessed); String tableId = entry.getKey(), tableIdAbbrev = tableId.replaceAll("csv/(\\d+)-csv/(\\d+)\\.csv", "$1-$2"); LogInfo.begin_track("Processing %s ...", tableId); From 4dacb8505026c012484cbc8547f693f26bcaa9dd Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Mon, 10 Jul 2017 16:37:05 -0700 Subject: [PATCH 08/30] Changed how strings are normalized --- .../nlp/sempre/tables/StringNormalizationUtils.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java index 2efb648..b32afb7 100644 --- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java @@ -317,8 +317,7 @@ public final class StringNormalizationUtils { .replaceAll("[‘’´`]", "'") .replaceAll("[“”«»]", "\"") .replaceAll("[•†‡]", "") - .replaceAll("[‐‑–—]", "-") - .replaceAll("[\\u2E00-\\uFFFF]", ""); // (Sorry Chinese people) + .replaceAll("[‐‑–—]", "-"); return string.replaceAll("\\s+", " ").trim(); } @@ -330,13 +329,8 @@ public final class StringNormalizationUtils { // Citation string = string.replaceAll("\\[(nb ?)?\\d+\\]", ""); string = string.replaceAll("\\*+$", ""); - // Year in parentheses - string = string.replaceAll("\\(\\d* ?-? ?\\d*\\)", ""); // Outside Quote string = string.replaceAll("^\"(.*)\"$", "$1"); - // Numbering - if (!string.matches("^[0-9.]+$")) - string = string.replaceAll("^\\d+\\.", ""); return string.replaceAll("\\s+", " ").trim(); } @@ -347,7 +341,6 @@ public final class StringNormalizationUtils { // Dashed / Parenthesized information string = simpleNormalize(string); string = string.replaceAll("\\[[^\\]]*\\]", ""); - string = string.replaceAll("[\\u007F-\\uFFFF]", ""); string = string.trim().replaceAll(" - .*$", ""); string = string.trim().replaceAll("\\([^)]*\\)$", ""); return string.replaceAll("\\s+", " ").trim(); From 3ad547708187b9b53fa628bc7825d929948ac43a Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Mon, 28 Aug 2017 23:17:19 -0700 Subject: [PATCH 09/30] Add git-hash information when using CodaLab --- tables/x | 1 + 1 file changed, 1 insertion(+) diff --git a/tables/x b/tables/x index bfa9d06..c561759 100755 --- a/tables/x +++ b/tables/x @@ -16,6 +16,7 @@ if [ "$operation" == "up" ]; then ln -s ../module-classes.txt xxx/ ln -s ../tables/grammars xxx/ ln -s ../run xxx/ + { hostname; readlink -f .; date; git log -1; git status; } > xxx/git-hash cl upload -L -n stuff xxx rm -rf xxx exit 0 From 7de8d8324461b62f405f36035de53258f9e69538 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Tue, 29 Aug 2017 02:20:33 -0700 Subject: [PATCH 10/30] Refactoring cprune out of Derivation --- build.xml | 12 +- run | 38 ++- .../stanford/nlp/sempre/FloatingParser.java | 5 +- src/edu/stanford/nlp/sempre/Grammar.java | 9 +- src/edu/stanford/nlp/sempre/Parser.java | 6 +- .../nlp/sempre/cprune/CPruneDerivInfo.java | 13 + .../sempre/cprune/CPruneFloatingParser.java | 102 +++++++ .../nlp/sempre/cprune/CPruneStats.java | 20 ++ .../sempre/cprune/CollaborativePruner.java | 208 ++++++++++++++ .../nlp/sempre/cprune/CustomGrammar.java | 256 ++++++++++++++++++ .../stanford/nlp/sempre/cprune/Pattern.java | 34 +++ .../nlp/sempre/cprune/PatternInfo.java | 92 +++++++ .../sempre/cprune/RegexReplaceManager.java | 15 + .../stanford/nlp/sempre/cprune/Symbol.java | 26 ++ .../tables/lambdadcs/DenotationUtils.java | 12 +- tables/grammars/extended.grammar | 169 ++++++++++++ 16 files changed, 989 insertions(+), 28 deletions(-) create mode 100644 src/edu/stanford/nlp/sempre/cprune/CPruneDerivInfo.java create mode 100644 src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java create mode 100644 src/edu/stanford/nlp/sempre/cprune/CPruneStats.java create mode 100644 src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java create mode 100644 src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java create mode 100644 src/edu/stanford/nlp/sempre/cprune/Pattern.java create mode 100644 src/edu/stanford/nlp/sempre/cprune/PatternInfo.java create mode 100644 src/edu/stanford/nlp/sempre/cprune/RegexReplaceManager.java create mode 100644 src/edu/stanford/nlp/sempre/cprune/Symbol.java create mode 100644 tables/grammars/extended.grammar diff --git a/build.xml b/build.xml index d033b05..ac72a0a 100644 --- a/build.xml +++ b/build.xml @@ -25,7 +25,7 @@ - + @@ -78,6 +78,16 @@ + + + + + + + + + + diff --git a/run b/run index c513830..9a98f0f 100755 --- a/run +++ b/run @@ -634,7 +634,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( # Add @cldir=1 to use CodaLab's directory paths letDefault(:cldir, 0), # Usual header - header('core,tables,corenlp'), + header('core,tables,corenlp,cprune'), # Select class letDefault(:class, 'main'), sel(:class, { @@ -674,35 +674,38 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( }), # Parser letDefault(:parser, 'floatsize'), + o('useSizeInsteadOfDepth'), sel(:parser, { 'floatsize' => l( o('Builder.parser', 'FloatingParser'), - o('useSizeInsteadOfDepth'), o('FloatingParser.maxDepth', 15), nil), - 'baseline' => l( - o('Builder.parser', 'tables.baseline.TableBaselineParser'), - nil), + 'baseline' => o('Builder.parser', 'tables.baseline.TableBaselineParser'), 'serialized' => o('Builder.parser', 'tables.serialize.SerializedParser'), # ACL 2016 'grow-dpd' => l( o('Builder.parser', 'tables.dpd.DPDParser'), - o('useSizeInsteadOfDepth'), o('FloatingParser.maxDepth', 8), nil), 'grow-float' => l( o('Builder.parser', 'FloatingParser'), - o('useSizeInsteadOfDepth'), o('FloatingParser.maxDepth', 8), o('FloatingParser.betaReduce'), o('initialFloatingHasZeroDepth'), nil), 'grow-mix' => l( o('Builder.parser', 'MixParser'), o('MixParser.parsers', 'FloatingParser', 'tables.serialize.SerializedParser:train-0xc'), - o('useSizeInsteadOfDepth'), o('FloatingParser.maxDepth', 8), o('FloatingParser.betaReduce'), o('initialFloatingHasZeroDepth'), nil), + # EMNLP 2017 + 'cprune' => l( + o('Builder.parser', 'cprune.CPruneFloatingParser'), + o('FloatingParser.maxDepth', 15), + o('beamSize', 100), + o('maxNumNeighbors', 40), + o('maxPredictedPatterns', 1000), + nil), }), o('Parser.verbose', 0), letDefault(:pruning, 1), @@ -731,7 +734,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( letDefault(:normalize, 1), sel(:normalize, l(), - l(o('genericDateValue'), o('numberCanStartAnywhere'), o('num2CanStartAnywhere'), o('NumberFn.alsoTestByIsolatedNER')), + l(o('genericDateValue'), o('numberCanStartAnywhere'), o('num2CanStartAnywhere')), nil), letDefault(:anchor, 1), sel(:anchor, { @@ -786,6 +789,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( o('combineFromFloatingParser'), o('maxTrainIters', 3), o('showValues', false), o('showFirstValue'), + o('customExpectedCounts', 'TOP'), nil), l( # for dumping derivations (@class=dump) @@ -886,6 +890,11 @@ def tablesGrammarPaths o('Grammar.tags', *'scoped merge-and arithmetic comparison alternative neq yearrange part closedclass-generic scoped-2args-merge-and'.split), let(:anchor, 2), nil), + # EMNLP 2017 + 'extended' => l( + o('Grammar.inPaths', "#{baseDir}extended.grammar"), + o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge v-superlative'.split), + nil), }), nil) } @@ -895,13 +904,17 @@ def tablesDataPaths lambda { |e| baseDir = ['lib/data/WikiTableQuestions/data/', 'WikiTableQuestions/data/'][e[:cldir]] csvDir = ['lib/data/WikiTableQuestions/', 'WikiTableQuestions/'][e[:cldir]] + nnDir = ['lib/data/nn_0/', 'nn_0/'][e[:cldir]] datasets = { 'none' => l(), 'train' => o('Dataset.inPaths', "train,#{baseDir}training.examples"), # Pristine test test - 'test' => o('Dataset.inPaths', - "train,#{baseDir}training.examples", - "test,#{baseDir}pristine-unseen-tables.examples"), + 'test' => l( + o('Dataset.inPaths', + "train,#{baseDir}training.examples", + "test,#{baseDir}pristine-unseen-tables.examples"), + o('neighborFilePath', "#{nnDir}/exact_nearest_neighbors.all"), + nil), # @data=annotated can be used with @class=check only 'annotated' => o('Dataset.inPaths', "train,#{baseDir}annotated-all.examples"), 'before300' => o('Dataset.inPaths', "train,#{baseDir}training-before300.examples"), @@ -913,6 +926,7 @@ def tablesDataPaths "train,#{baseDir}random-split-#{x}-train.examples", "dev,#{baseDir}random-split-#{x}-dev.examples", nil), + o('neighborFilePath', "#{nnDir}/exact_nearest_neighbors.seed-#{x}.train"), nil) end # That's it! diff --git a/src/edu/stanford/nlp/sempre/FloatingParser.java b/src/edu/stanford/nlp/sempre/FloatingParser.java index 4f433f2..827b190 100644 --- a/src/edu/stanford/nlp/sempre/FloatingParser.java +++ b/src/edu/stanford/nlp/sempre/FloatingParser.java @@ -84,14 +84,11 @@ public class FloatingParser extends Parser { @Option(gloss = "DEBUG: Print amount of time spent on each rule") public boolean summarizeRuleTime = false; @Option(gloss = "Stop the parser if it has used more than this amount of time (in seconds)") - public int maxFloatingParsingTime = 600; + public int maxFloatingParsingTime = Integer.MAX_VALUE; } public static Options opts = new Options(); - protected List orderedFloatingRules; - public List getOrderedFloatingRules() { return orderedFloatingRules; } - public FloatingParser(Spec spec) { super(spec); } diff --git a/src/edu/stanford/nlp/sempre/Grammar.java b/src/edu/stanford/nlp/sempre/Grammar.java index bb94b36..72cf0c8 100644 --- a/src/edu/stanford/nlp/sempre/Grammar.java +++ b/src/edu/stanford/nlp/sempre/Grammar.java @@ -44,7 +44,7 @@ public class Grammar { // All the rules in the grammar. Each parser can read these and transform // them however the parser wishes. // This contains binarized rules - ArrayList rules = new ArrayList<>(); + protected ArrayList rules = new ArrayList<>(); public List getRules() { return rules; } Map macros = new HashMap<>(); // Map from macro name to its replacement value @@ -260,7 +260,7 @@ public class Grammar { return cat; } - private void interpretRule(LispTree tree) { + protected void interpretRule(LispTree tree) { if (tree.children.size() < 4) throw new RuntimeException("Invalid rule: " + tree); @@ -360,7 +360,7 @@ public class Grammar { // Generate intermediate categories for binarization. public static final String INTERMEDIATE_PREFIX = "$Intermediate"; - private int freshCatIndex = 0; + protected int freshCatIndex = 0; private String generateFreshCat() { freshCatIndex++; return INTERMEDIATE_PREFIX + freshCatIndex; @@ -368,6 +368,9 @@ public class Grammar { public static boolean isIntermediate(String cat) { return cat.startsWith(INTERMEDIATE_PREFIX); } + public int getFreshCatIndex() { + return freshCatIndex; + } // Create multiple versions of this rule if there are optional RHS. // Restriction: must be able to split the RHS into two halves, each of diff --git a/src/edu/stanford/nlp/sempre/Parser.java b/src/edu/stanford/nlp/sempre/Parser.java index c3a7fa3..9afa080 100644 --- a/src/edu/stanford/nlp/sempre/Parser.java +++ b/src/edu/stanford/nlp/sempre/Parser.java @@ -56,7 +56,7 @@ public abstract class Parser { @Option(gloss = "Dump all features (for debugging)") public boolean dumpAllFeatures = false; - + @Option(gloss = "Call SetEvaluation during parsing") public boolean callSetEvaluation = true; } @@ -100,8 +100,8 @@ public abstract class Parser { this.valueEvaluator = spec.valueEvaluator; computeCatUnaryRules(); - LogInfo.logs("Parser: %d catUnaryRules (sorted), %d nonCatUnaryRules (in trie)", - catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size()); + LogInfo.logs("%s: %d catUnaryRules (sorted), %d nonCatUnaryRules (in trie)", + this.getClass().getSimpleName(), catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size()); } // If grammar changes, then we might need to update aspects of the parser. diff --git a/src/edu/stanford/nlp/sempre/cprune/CPruneDerivInfo.java b/src/edu/stanford/nlp/sempre/cprune/CPruneDerivInfo.java new file mode 100644 index 0000000..3c487bb --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cprune/CPruneDerivInfo.java @@ -0,0 +1,13 @@ +package edu.stanford.nlp.sempre.cprune; + +import java.util.Map; +import java.util.List; + +public class CPruneDerivInfo { + + public Map treeSymbols; + public Map ruleSymbols; + public List customRuleStrings; + public boolean containsCrossReference; + +} diff --git a/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java b/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java new file mode 100644 index 0000000..e580359 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java @@ -0,0 +1,102 @@ +package edu.stanford.nlp.sempre.cprune; + +import java.util.List; + +import edu.stanford.nlp.sempre.*; +import fig.basic.LogInfo; + +public class CPruneFloatingParser extends FloatingParser { + + FloatingParser exploreParser; + + public CPruneFloatingParser(Spec spec) { + super(spec); + exploreParser = new FloatingParser(spec); + } + + @Override + public void onBeginDataGroup(int iter, int numIters, String group) { + if (CollaborativePruner.neighbors == null) { + CollaborativePruner.customGrammar.init(grammar); + CollaborativePruner.loadNeighbors(); + } + CollaborativePruner.stats.reset(iter + "." + group); + } + + @Override + public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) { + return new CPruneFloatingParserState(this, params, ex, computeExpectedCounts); + } + +} + +class CPruneFloatingParserState extends ParserState { + + public CPruneFloatingParserState(Parser parser, Params params, Example ex, boolean computeExpectedCounts) { + super(parser, params, ex, computeExpectedCounts); + } + + @Override + public void infer() { + LogInfo.begin_track("CPruneFloatingParser.infer()"); + boolean exploitSucceeds = exploit(); + if (computeExpectedCounts) { + LogInfo.begin_track("Summary of Collaborative Pruning"); + LogInfo.logs("Exploit succeeds: " + exploitSucceeds); + LogInfo.logs("Exploit success rate: " + CollaborativePruner.stats.successfulExploit + "/" + CollaborativePruner.stats.totalExploit); + LogInfo.end_track(); + } + // Explore only on the first training iteration + if (CollaborativePruner.stats.iter.equals("0.train") && computeExpectedCounts && !exploitSucceeds + && (CollaborativePruner.stats.totalExplore <= CollaborativePruner.opts.maxExplorationIters)) { + explore(); + LogInfo.logs("Consistent pattern: " + CollaborativePruner.getConsistentPattern(ex)); + LogInfo.logs("Explore success rate: " + CollaborativePruner.stats.successfulExplore + "/" + CollaborativePruner.stats.totalExplore); + } + LogInfo.end_track(); + } + + public void explore() { + LogInfo.begin_track("Explore"); + CollaborativePruner.initialize(ex, CollaborativePruner.Mode.EXPLORE); + ParserState exploreParserState = ((CPruneFloatingParser) parser).exploreParser.newParserState(params, ex, computeExpectedCounts); + exploreParserState.infer(); + predDerivations.clear(); + predDerivations.addAll(exploreParserState.predDerivations); + expectedCounts = exploreParserState.expectedCounts; + CollaborativePruner.stats.totalExplore += 1; + if (CollaborativePruner.foundConsistentDerivation) + CollaborativePruner.stats.successfulExplore += 1; + LogInfo.end_track(); + } + + public boolean exploit() { + LogInfo.begin_track("Exploit"); + CollaborativePruner.initialize(ex, CollaborativePruner.Mode.EXPLOIT); + Parser exploitParser = new FloatingParser( + new Parser.Spec(new MiniGrammar(CollaborativePruner.predictedRules), parser.extractor, parser.executor, parser.valueEvaluator)); + ParserState exploitParserState = exploitParser.newParserState(params, ex, computeExpectedCounts); + exploitParserState.infer(); + predDerivations.clear(); + predDerivations.addAll(exploitParserState.predDerivations); + expectedCounts = exploitParserState.expectedCounts; + boolean succeeds = CollaborativePruner.foundConsistentDerivation; + CollaborativePruner.stats.totalExploit += 1; + if (succeeds) + CollaborativePruner.stats.successfulExploit += 1; + LogInfo.end_track(); + return succeeds; + } +} + +// ============================================================ +// Helper classes +// ============================================================ + +class MiniGrammar extends Grammar { + + public MiniGrammar(List rules) { + this.rules.addAll(rules); + } + +} diff --git a/src/edu/stanford/nlp/sempre/cprune/CPruneStats.java b/src/edu/stanford/nlp/sempre/cprune/CPruneStats.java new file mode 100644 index 0000000..6892bc3 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cprune/CPruneStats.java @@ -0,0 +1,20 @@ +package edu.stanford.nlp.sempre.cprune; + +/** + * Stores various statistic. + */ +public class CPruneStats { + public String iter; + public int totalExplore = 0; + public int successfulExplore = 0; + public int totalExploit = 0; + public int successfulExploit = 0; + + public void reset(String iter) { + this.iter = iter; + this.totalExplore = 0; + this.successfulExplore = 0; + this.totalExploit = 0; + this.successfulExploit = 0; + } +} diff --git a/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java b/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java new file mode 100644 index 0000000..cf91014 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java @@ -0,0 +1,208 @@ +package edu.stanford.nlp.sempre.cprune; + +import java.io.*; +import java.util.*; + +import fig.basic.*; +import edu.stanford.nlp.sempre.*; +import fig.basic.LogInfo; + +/** + * Static class for collaborative pruning. + */ +public class CollaborativePruner { + public static class Options { + @Option(gloss = "K = Maximum number of nearest-neighbor examples to consider (-1 to use all examples so far)") + public int maxNumNeighbors = -1; + @Option(gloss = "Load cached neighbors from this file") + public String neighborFilePath = null; + @Option(gloss = "Maximum number of matching patterns (default = use all patterns)") + public int maxPredictedPatterns = Integer.MAX_VALUE; + @Option(gloss = "Maximum number of derivations per example") + public int maxDerivations = 5000; + @Option(gloss = "Maximum number of exporation iterations") + public int maxExplorationIters = Integer.MAX_VALUE; + } + + public static Options opts = new Options(); + + public enum Mode { EXPLORE, EXPLOIT, NONE } + + public static Mode mode = Mode.NONE; + public static CPruneStats stats = new CPruneStats(); + public static CustomGrammar customGrammar = new CustomGrammar(); + + // Static class; do not instantiate + private CollaborativePruner() { throw new RuntimeException("Cannot instantiate CollaborativePruner"); } + + // Global variables + // Nearest neighbors + static Map> neighbors; + // uid => pattern + static Map consistentPattern = new HashMap<>(); + // patternString => customRuleString + static Map> customRules = new HashMap<>(); + // set of patternStrings + static Set allConsistentPatterns = new HashSet<>(); + + // Example-level variables + public static boolean foundConsistentDerivation = false; + public static Map predictedPatterns; + public static List predictedRules; + + /** + * Read the cached neighbors file. + * Line Format: ex_id [tab] neighbor_id1,neighbor_id2,... + */ + public static void loadNeighbors() { + if (opts.neighborFilePath == null) { + LogInfo.logs("neighborFilePath is null."); + return; + } + LogInfo.logs("loading Neighbors " + opts.neighborFilePath); + Map> tmpMap = new HashMap<>(); + try { + BufferedReader reader = IOUtils.openIn(opts.neighborFilePath); + String line; + while ((line = reader.readLine()) != null) { + String[] tokens = line.split("\t"); + String uid = tokens[0]; + String[] nids = tokens[1].split(","); + tmpMap.put(uid, Arrays.asList(nids)); + } + reader.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + neighbors = tmpMap; + } + + public static > Map sortByValue(Map map) { + List> list = new LinkedList>(map.entrySet()); + Collections.sort(list, new Comparator>() { + public int compare(Map.Entry o1, Map.Entry o2) { + return (o1.getValue()).compareTo(o2.getValue()); + } + }); + + Map result = new LinkedHashMap(); + for (Map.Entry entry : list) { + result.put(entry.getKey(), entry.getValue()); + } + return result; + } + + public static void initialize(Example ex, Mode mode) { + CollaborativePruner.mode = mode; + predictedRules = null; + predictedPatterns = null; + foundConsistentDerivation = false; + if (mode == Mode.EXPLOIT) { + preprocessExample(ex); + } + } + + static void preprocessExample(Example ex) { + Map patternFreqMap = new HashMap<>(); + List cachedNeighbors = neighbors.get(ex.id); + int total = 0; + + // Gather the neighbors + if (opts.maxNumNeighbors > 0) { + for (String nid : cachedNeighbors) { + // Only get examples that have been previously processed + if (!consistentPattern.containsKey(nid)) + continue; + + String neighborPattern = consistentPattern.get(nid).pattern; + if (!patternFreqMap.containsKey(neighborPattern)) + patternFreqMap.put(neighborPattern, new Pattern(neighborPattern, 0, 0)); + patternFreqMap.get(neighborPattern).frequency += 1; + total++; + if (total >= opts.maxNumNeighbors) + break; + } + } else { + for (String patternString : allConsistentPatterns) { + patternFreqMap.put(patternString, new Pattern(patternString, 0, 1)); + } + } + + // Gather the patterns + patternFreqMap = sortByValue(patternFreqMap); + int rank = 0; + Set predictedRulesStrings = new HashSet<>(); + predictedPatterns = new HashMap<>(); + LogInfo.begin_track("Predicted patterns"); + for (Map.Entry entry : patternFreqMap.entrySet()) { + Pattern newPattern = entry.getValue(); + predictedPatterns.put(newPattern.pattern, newPattern); + predictedRulesStrings.addAll(customRules.get(newPattern.pattern)); + LogInfo.logs((rank + 1) + ". " + newPattern.pattern + " (" + newPattern.frequency + ")"); + rank++; + if (rank >= opts.maxPredictedPatterns) + break; + } + + // Gather the rules + predictedRules = customGrammar.getRules(predictedRulesStrings); + LogInfo.end_track(); + } + + public static String getPatternString(Derivation deriv) { + if (deriv.cat.equals("$TOKEN") || deriv.cat.equals("$PHRASE") + || deriv.cat.equals("$LEMMA_TOKEN") || deriv.cat.equals("$LEMMA_PHRASE")) { + return deriv.cat; + } else { + return PatternInfo.convertToIndexedPattern(deriv); + } + } + + public static void addRules(String patternString, Derivation deriv, Example ex) { + if (!customRules.containsKey(patternString)) { + customRules.put(patternString, new HashSet()); + } + Set parsedCustomRules = customGrammar.addCustomRule(deriv, ex); + customRules.get(patternString).addAll(parsedCustomRules); + } + + public static void updateConsistentPattern(ValueEvaluator evaluator, Example ex, Derivation deriv) { + String uid = ex.id; + if (ex.targetValue != null) + deriv.compatibility = evaluator.getCompatibility(ex.targetValue, deriv.value); + + if (deriv.isRootCat() && deriv.compatibility == 1) { + foundConsistentDerivation = true; + LogInfo.logs("Found consistent deriv: %s", deriv); + + String patternString = getPatternString(deriv); + Pattern newConsistentPattern = new Pattern(patternString, 0, 0); + newConsistentPattern.score = deriv.getScore(); + + if (!consistentPattern.containsKey(uid)) { + addRules(patternString, deriv, ex); + consistentPattern.put(uid, newConsistentPattern); + allConsistentPatterns.add(patternString); + } else { + Pattern oldConsistentPattern = consistentPattern.get(uid); + if (newConsistentPattern.score > oldConsistentPattern.score) { + addRules(patternString, deriv, ex); + consistentPattern.put(uid, newConsistentPattern); + allConsistentPatterns.add(patternString); + } + } + } + } + + public static Pattern getConsistentPattern(Example ex) { + return consistentPattern.get(ex.id); + } + + public static Pattern getPattern(Example ex, Derivation deriv) { + if (mode == Mode.EXPLORE) + return null; + if (!deriv.isRootCat()) + return null; + return predictedPatterns.get(getPatternString(deriv)); + } +} diff --git a/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java b/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java new file mode 100644 index 0000000..d7b7697 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java @@ -0,0 +1,256 @@ +package edu.stanford.nlp.sempre.cprune; + +import java.util.*; + +import edu.stanford.nlp.sempre.*; +import fig.basic.*; + +public class CustomGrammar extends Grammar { + public static class Options { + @Option + public boolean enableTemplateDecomposition = true; + } + + public static Options opts = new Options(); + + public static final Set baseCategories = new HashSet(Arrays.asList("$Unary", "$Binary", "$Entity", "$Property")); + + ArrayList baseRules = new ArrayList<>(); + + // symbolicFormulas => symbolicFormula ID + Map symbolicFormulas = new HashMap<>(); + // indexedSymbolicFormula => customRuleString + Map> customRules = new HashMap<>(); + // customRuleString => Binarized rules + Map> customBinarizedRules = new HashMap<>(); + + public void init(Grammar initGrammar) { + baseCategories.add(Rule.tokenCat); + baseCategories.add(Rule.phraseCat); + baseCategories.add(Rule.lemmaTokenCat); + baseCategories.add(Rule.lemmaPhraseCat); + + baseRules = new ArrayList<>(); + for (Rule rule : initGrammar.getRules()) { + if (baseCategories.contains(rule.lhs)) { + baseRules.add(rule); + } + } + this.freshCatIndex = initGrammar.getFreshCatIndex(); + } + + public List getRules(Collection customRuleStrings) { + Set ruleSet = new LinkedHashSet<>(); + ruleSet.addAll(baseRules); + for (String ruleString : customRuleStrings) { + ruleSet.addAll(customBinarizedRules.get(ruleString)); + } + return new ArrayList(ruleSet); + } + + public Set addCustomRule(Derivation deriv, Example ex) { + String indexedSymbolicFormula = getIndexedSymbolicFormula(deriv); + if (customRules.containsKey(indexedSymbolicFormula)) { + return customRules.get(indexedSymbolicFormula); + } + + CPruneDerivInfo derivInfo = aggregateSymbols(deriv); + Set crossReferences = new HashSet<>(); + for (Symbol symbol : derivInfo.treeSymbols.values()) { + if (symbol.frequency > 1) { + crossReferences.add(symbol.formula); + } + } + computeCustomRules(deriv, crossReferences); + customRules.put(indexedSymbolicFormula, new HashSet(derivInfo.customRuleStrings)); + + LogInfo.begin_track("Add custom rules for formula: " + indexedSymbolicFormula); + for (String customRuleString : derivInfo.customRuleStrings) { + if (customBinarizedRules.containsKey(customRuleString)) { + LogInfo.log("Custom rule exists: " + customRuleString); + continue; + } + + rules = new ArrayList<>(); + LispTree tree = LispTree.proto.parseFromString(customRuleString); + interpretRule(tree); + customBinarizedRules.put(customRuleString, new HashSet(rules)); + + // Debug + LogInfo.begin_track("Add custom rule: " + customRuleString); + for (Rule rule : rules) { + LogInfo.log(rule.toString()); + } + LogInfo.end_track(); + } + LogInfo.end_track(); + + // Debug + System.out.println("consistent_lf\t" + ex.id + "\t" + deriv.formula.toString()); + + return customRules.get(indexedSymbolicFormula); + } + + public static String getIndexedSymbolicFormula(Derivation deriv) { + return getIndexedSymbolicFormula(deriv, deriv.formula.toString()); + } + + public static String getIndexedSymbolicFormula(Derivation deriv, String formula) { + CPruneDerivInfo derivInfo = aggregateSymbols(deriv); + int index = 1; + List symbolList = new ArrayList<>(derivInfo.treeSymbols.values()); + for (Symbol symbol : symbolList) + symbol.computeIndex(formula); + Collections.sort(symbolList); + for (Symbol symbol : symbolList) { + if (formula.equals(symbol.formula)) + formula = symbol.category + "#" + index; + formula = safeReplace(formula, symbol.formula, symbol.category + "#" + index); + index += 1; + } + return formula; + } + + // ============================================================ + // Private methods + // ============================================================ + + private static String safeReplace(String formula, String target, String replacement) { + formula = formula.replace(target + ")", replacement + ")"); + formula = formula.replace(target + " ", replacement + " "); + return formula; + } + + private static CPruneDerivInfo aggregateSymbols(Derivation deriv) { + Map tempState = deriv.getTempState(); + if (tempState.containsKey("cprune")) { + return (CPruneDerivInfo) tempState.get("cprune"); + } + CPruneDerivInfo derivInfo = new CPruneDerivInfo(); + tempState.put("cprune", derivInfo); + + Map treeSymbols = new LinkedHashMap<>(); + derivInfo.treeSymbols = treeSymbols; + if (baseCategories.contains(deriv.cat)) { + String formula = deriv.formula.toString(); + treeSymbols.put(formula, new Symbol(deriv.cat, formula, 1)); + } else { + for (Derivation child : deriv.children) { + CPruneDerivInfo childInfo = aggregateSymbols(child); + for (Symbol symbol : childInfo.treeSymbols.values()) { + if (derivInfo.treeSymbols.containsKey(symbol.formula)) { + treeSymbols.get(symbol.formula).frequency += symbol.frequency; + } else { + treeSymbols.put(symbol.formula, symbol); + } + } + } + } + return derivInfo; + } + + private CPruneDerivInfo computeCustomRules(Derivation deriv, Set crossReferences) { + CPruneDerivInfo derivInfo = (CPruneDerivInfo) deriv.getTempState().get("cprune"); + Map ruleSymbols = new LinkedHashMap<>(); + derivInfo.ruleSymbols = ruleSymbols; + derivInfo.customRuleStrings = new ArrayList<>(); + String formula = deriv.formula.toString(); + + if (baseCategories.contains(deriv.cat)) { + // Leaf node induces no custom rule + derivInfo.containsCrossReference = crossReferences.contains(formula); + // Propagate the symbol of this derivation to the parent + ruleSymbols.putAll(derivInfo.treeSymbols); + } else { + derivInfo.containsCrossReference = false; + for (Derivation child : deriv.children) { + CPruneDerivInfo childInfo = computeCustomRules(child, crossReferences); + derivInfo.containsCrossReference = derivInfo.containsCrossReference || childInfo.containsCrossReference; + } + + for (Derivation child : deriv.children) { + CPruneDerivInfo childInfo = (CPruneDerivInfo) child.getTempState().get("cprune"); + ruleSymbols.putAll(childInfo.ruleSymbols); + derivInfo.customRuleStrings.addAll(childInfo.customRuleStrings); + } + + if (opts.enableTemplateDecomposition == false || derivInfo.containsCrossReference) { + // If this node contains a cross reference + if (deriv.isRootCat()) { + // If this is the root node, then generate a custom rule + derivInfo.customRuleStrings.add(getCustomRuleString(deriv, derivInfo)); + } + } else { + if (!deriv.cat.startsWith("$Intermediate")) { + // Generate a custom rule for this node + derivInfo.customRuleStrings.add(getCustomRuleString(deriv, derivInfo)); + + // Propagate this derivation as a category to the parent + ruleSymbols.clear(); + ruleSymbols.put(formula, new Symbol(hash(deriv), deriv.formula.toString(), 1)); + } + } + } + return derivInfo; + } + + private String getCustomRuleString(Derivation deriv, CPruneDerivInfo derivInfo) { + String formula = deriv.formula.toString(); + List rhsSymbols = new ArrayList<>(derivInfo.ruleSymbols.values()); + for (Symbol symbol : rhsSymbols) + symbol.computeIndex(formula); + Collections.sort(rhsSymbols); + + String lhs = null; + if (derivInfo.containsCrossReference) + lhs = deriv.cat; + else + lhs = deriv.isRootCat() ? "$ROOT" : hash(deriv); + + LinkedList rhsList = new LinkedList<>(); + int index = 1; + for (Symbol symbol : rhsSymbols) { + if (formula.equals(symbol.formula)) { + formula = "(IdentityFn)"; + } else { + formula = safeReplace(formula, symbol.formula, "(var s" + index + ")"); + formula = "(lambda s" + index + " " + formula + ")"; + } + rhsList.addFirst(symbol.category); + index += 1; + } + String rhs = null; + if (rhsList.size() > 0) { + rhs = "(" + String.join(" ", rhsList) + ")"; + } else { + rhs = "(nothing)"; + formula = "(ConstantFn " + formula + ")"; + } + return "(rule " + lhs + " " + rhs + " " + formula + ")"; + } + + private String hash(Derivation deriv) { + if (baseCategories.contains(deriv.cat)) + return deriv.cat; + + String formula = getSymbolicFormula(deriv); + if (!symbolicFormulas.containsKey(formula)) { + symbolicFormulas.put(formula, symbolicFormulas.size() + 1); + String hashString = "$Formula" + symbolicFormulas.get(formula); + LogInfo.log("Add symbolic formula: " + hashString + " = " + formula + " (" + deriv.cat + ")"); + } + return "$Formula" + symbolicFormulas.get(formula); + } + + private static String getSymbolicFormula(Derivation deriv) { + CPruneDerivInfo derivInfo = aggregateSymbols(deriv); + String formula = deriv.formula.toString(); + for (Symbol symbol : derivInfo.treeSymbols.values()) { + if (formula.equals(symbol.formula)) + formula = symbol.category; + formula = safeReplace(formula, symbol.formula, symbol.category); + } + return formula; + } + +} diff --git a/src/edu/stanford/nlp/sempre/cprune/Pattern.java b/src/edu/stanford/nlp/sempre/cprune/Pattern.java new file mode 100644 index 0000000..485573d --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cprune/Pattern.java @@ -0,0 +1,34 @@ +package edu.stanford.nlp.sempre.cprune; + +public class Pattern implements Comparable { + public String pattern; + public Integer rank; + public Integer frequency; + public Double score; + + public Pattern(String pattern, Integer rank, Integer frequency) { + this.pattern = pattern; + this.rank = rank; + this.frequency = frequency; + } + + public Double complexity() { + return (double) (pattern.length() - pattern.replace("(@R", "***").replace("(", "").length()); + } + + @Override + public String toString() { + return "(" + pattern + ", " + rank + ", " + frequency + ")"; + } + + @Override + public int compareTo(Pattern that) { + if (this.frequency > that.frequency) { + return -1; + } else if (this.frequency < that.frequency) { + return 1; + } else { + return this.complexity().compareTo(that.complexity()); + } + } +} diff --git a/src/edu/stanford/nlp/sempre/cprune/PatternInfo.java b/src/edu/stanford/nlp/sempre/cprune/PatternInfo.java new file mode 100644 index 0000000..b9d8377 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cprune/PatternInfo.java @@ -0,0 +1,92 @@ +package edu.stanford.nlp.sempre.cprune; + +import java.util.*; +import edu.stanford.nlp.sempre.*; + +public class PatternInfo { + Set baseCategories = new HashSet(Arrays.asList("$Unary", "$Binary", "$Entity", "$Property")); + + public static String removePropertyPredicates(String formula) { + formula = RegexReplaceManager.replace(formula, "fb:cell\\.cell\\.[_0-9a-z]+", "@PPT"); + formula = formula.replace("(reverse @PPT)", "@PPT"); + + while (formula.contains("@PPT")) { + int begin = formula.indexOf("(@PPT"); + if (begin == -1) { + formula = formula.replace("@PPT", ""); + ; + break; + } + int count = 1; + for (int i = begin + 1; i < formula.length(); i++) { + if (formula.charAt(i) == '(') { + count += 1; + } else if (formula.charAt(i) == ')') { + count -= 1; + } + if (count == 0) { + int end = i; + formula = formula.substring(0, begin) + formula.substring(begin + 6, end) + formula.substring(end + 1, formula.length()); + break; + } + if (i == formula.length() - 1) { + System.out.println(formula); + } + } + } + return formula; + } + + public static String convertToPattern(Derivation deriv) { + String formula = deriv.formula.toString(); + + // These can interfere with (number 1) + formula = formula.replace("argmax (number 1) (number 1)", "argmax"); + formula = formula.replace("argmin (number 1) (number 1)", "argmin"); + + formula = removePropertyPredicates(formula); + + formula = formula.replace("fb:type.object.type fb:type.row", "@type @row"); + + formula = RegexReplaceManager.replace(formula, "!fb:[._0-9a-z]+", "(reverse $0)").replace("reverse !fb", "reverse fb"); + formula = formula.replace("fb:row.row.index", "(reverse (lambda x ((reverse @index) (var x))))"); + formula = formula.replace("fb:row.row.next", "@next"); + formula = RegexReplaceManager.replace(formula, "\\(lambda [a-z]", "\\(lambda x"); + formula = RegexReplaceManager.replace(formula, "\\(var [a-z]\\)", "\\(var x\\)"); + + formula = RegexReplaceManager.replace(formula, "fb:row\\.row\\.[_0-9a-z]+", "\\$Binary"); + formula = RegexReplaceManager.replace(formula, "fb:cell\\.cell\\.[_0-9a-z]+", "\\$Property"); + formula = RegexReplaceManager.replace(formula, "fb:cell_[_0-9a-z]+\\.[_0-9a-z]+", "\\$Entity"); + formula = RegexReplaceManager.replace(formula, "\\(number [0-9]+[.]?[0-9]+\\)", "\\$Entity"); + formula = RegexReplaceManager.replace(formula, "\\(number [0-9]+\\)", "\\$Entity"); + formula = RegexReplaceManager.replace(formula, "\\(date [-]?[0-9]+ [-]?[0-9]+ [-]?[0-9]+\\)", "\\$Entity"); + + formula = RegexReplaceManager.replace(formula, "[ ]+", " "); + formula = formula.replace("reverse", "@R"); + formula = RegexReplaceManager.replace(formula, "(<=|>=|>|<)", "@compare"); + return formula; + } + + public static String convertToIndexedPattern(Derivation deriv) { + String formula = deriv.formula.toString(); + + // These can interfere with (number 1) + formula = formula.replace("argmax (number 1) (number 1)", "argmax"); + formula = formula.replace("argmin (number 1) (number 1)", "argmin"); + + formula = removePropertyPredicates(formula); + formula = CustomGrammar.getIndexedSymbolicFormula(deriv, formula); + + formula = formula.replace("fb:type.object.type fb:type.row", "@type @row"); + + formula = RegexReplaceManager.replace(formula, "!fb:[._0-9a-z]+", "(reverse $0)").replace("reverse !fb", "reverse fb"); + formula = formula.replace("fb:row.row.index", "(reverse (lambda x ((reverse @index) (var x))))"); + formula = formula.replace("fb:row.row.next", "@next"); + formula = RegexReplaceManager.replace(formula, "\\(lambda [a-z]", "\\(lambda x"); + formula = RegexReplaceManager.replace(formula, "\\(var [a-z]\\)", "\\(var x\\)"); + formula = RegexReplaceManager.replace(formula, "[ ]+", " "); + formula = formula.replace("reverse", "@R"); + formula = RegexReplaceManager.replace(formula, "(<=|>=|>|<)", "@compare"); + return formula; + } +} diff --git a/src/edu/stanford/nlp/sempre/cprune/RegexReplaceManager.java b/src/edu/stanford/nlp/sempre/cprune/RegexReplaceManager.java new file mode 100644 index 0000000..c6493dd --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cprune/RegexReplaceManager.java @@ -0,0 +1,15 @@ +package edu.stanford.nlp.sempre.cprune; + +import java.util.*; + +public class RegexReplaceManager { + static Map dict = new HashMap<>(); + + public static String replace(String source, String regex, String replacement) { + if (!dict.containsKey(regex)) { + dict.put(regex, java.util.regex.Pattern.compile(regex)); + } + java.util.regex.Pattern regexPattern = dict.get(regex); + return regexPattern.matcher(source).replaceAll(replacement); + } +} diff --git a/src/edu/stanford/nlp/sempre/cprune/Symbol.java b/src/edu/stanford/nlp/sempre/cprune/Symbol.java new file mode 100644 index 0000000..11634be --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cprune/Symbol.java @@ -0,0 +1,26 @@ +package edu.stanford.nlp.sempre.cprune; + +public class Symbol implements Comparable { + String category; + String formula; + Integer frequency; + Integer index; + + public Symbol(String category, String formula, int frequency) { + this.category = category; + this.formula = formula; + this.frequency = frequency; + } + + public void computeIndex(String referenceString) { + index = referenceString.indexOf(formula); + if (index < 0) { + index = Integer.MAX_VALUE; + } + } + + @Override + public int compareTo(Symbol that) { + return index.compareTo(that.index); + } +} diff --git a/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java b/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java index afa098a..b3f63d1 100644 --- a/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java @@ -238,6 +238,8 @@ public final class DenotationUtils { */ public static UnaryDenotation superlativeUnary(int rank, int count, List> pairs, SuperlativeFormula.Mode mode, TypeProcessor processor) { + if (rank <= 0 || count <= 0) + LogInfo.fails("Invalid superlative (rank = %d, count = %d)", rank, count); if (pairs.isEmpty()) { if (LambdaDCSExecutor.opts.superlativesFailOnEmptyLists) throw new LambdaDCSException(Type.emptyList, "Cannot call %s on an empty list.", mode); @@ -343,7 +345,7 @@ public final class DenotationUtils { public boolean isCompatible(Value v) { return v instanceof NumberValue; } - + @Override public boolean isSortable(Collection values) { return true; @@ -387,19 +389,19 @@ public final class DenotationUtils { public boolean isCompatible(Value v) { return v instanceof DateValue; } - + @Override public boolean isSortable(Collection values) { DateValue firstDate = null; for (Value value : values) { - DateValue date = (DateValue) value; + DateValue date = (DateValue) value; if (firstDate == null) { firstDate = date; } else { if ((firstDate.year == -1) != (date.year == -1)) return false; if ((firstDate.month == -1) != (date.month == -1)) return false; if ((firstDate.day == -1) != (date.day == -1)) return false; - } + } } return true; } @@ -465,5 +467,5 @@ public final class DenotationUtils { throw new LambdaDCSException(Type.typeMismatch, "Cannot compare values"); } } - + } diff --git a/tables/grammars/extended.grammar b/tables/grammars/extended.grammar new file mode 100644 index 0000000..d45e1b5 --- /dev/null +++ b/tables/grammars/extended.grammar @@ -0,0 +1,169 @@ +# (Extended) Generic Grammar +# Use more generic compositional patterns. + +################################################################ +# Macros + +(def @R reverse) +(def @type fb:type.object.type) +(def @row fb:type.row) + +(def @next fb:row.row.next) +(def @!next !fb:row.row.next) +(def @index fb:row.row.index) +(def @!index !fb:row.row.index) + +(def @p.num fb:cell.cell.number) +(def @!p.num !fb:cell.cell.number) +(def @p.date fb:cell.cell.date) +(def @!p.date !fb:cell.cell.date) +(def @p.second fb:cell.cell.second) +(def @!p.second !fb:cell.cell.second) + +################################################################ +# Lexicon + +################################ +# Anchored Rules: Entity, Unary, Binary +(rule $Entity ($PHRASE) (FuzzyMatchFn entity) (anchored 1)) +#(rule $Binary ($PHRASE) (FuzzyMatchFn binary) (anchored 1)) +(rule $Entity ($PHRASE) (NumberFn) (anchored 1)) +(rule $Entity ($PHRASE) (DateFn) (anchored 1)) + +################################ +# Create binary from thin air +(rule $Binary (nothing) (FuzzyMatchFn any binary)) +(rule $Unary (nothing) (FuzzyMatchFn any unary)) + +################################ +# Property +(for @property (@p.num @p.date) + (rule $Property (nothing) (ConstantFn @property)) +) +(when second + (rule $Property (nothing) (ConstantFn @p.second)) +) + +################################ +# Generic RowSet +(rule $RowSet (nothing) (ConstantFn (@type @row))) + +################################ +# Anchored ValueSet +(rule $ValueSet ($Entity) (IdentityFn)) + +# [TAG] alternative: "X or Y" questions +(when alternative + (rule $ValueSet ($Entity $Entity) + (lambda e1 (lambda e2 (or (var e1) (var e2)))) + ) +) + +################################ +# Join + +(rule $RowSet ($Binary $ValueSet) (lambda b (lambda v ((var b) (var v))))) + +(rule $ValueSet ($Binary $RowSet) (lambda b (lambda r ((@R (var b)) (var r))))) + +(rule $RowSet ($Binary $Property $ValueSet) + (lambda b (lambda p (lambda v ((var b) ((var p) (var v)))))) +) + +(rule $ValueSet ($Binary $Property $RowSet) + (lambda b (lambda p (lambda r ((@R (var p)) ((@R (var b)) (var r)))))) +) + +# [TAG] movement: "next" / "previous" +(when movement + (for @movement (@next @!next) + (rule $RowSet ($RowSet) (lambda r (@movement (var r)))) + ) +) + +# [TAG] comparison: "at least" / "more than" +(when comparison + (for @comparison (< > <= >=) + (rule $RowSet ($Binary $Property $Entity) + (lambda b (lambda p (lambda e ((var b) ((var p) (@comparison (var e))))))) + ) + ) +) + +# [TAG] != : "not zero" / "same" +(when neq + (rule $RowSet ($Binary $Entity) (lambda b (lambda e ((var b) (!= (var e)))))) + (rule $RowSet ($Binary $Property $Entity) + (lambda b (lambda p (lambda e ((var b) ((var p) (!= (var e))))))) + ) +) + +################################ +# Aggregate + +(when count + (rule $SingleValue ($RowSet) (lambda r (count (var r)))) +) + +(when aggregate + (rule $SingleValue ($ValueSet) (lambda r (min (var r)))) + (rule $SingleValue ($ValueSet) (lambda r (max (var r)))) + (rule $SingleValue ($ValueSet) (lambda r (sum (var r)))) + (rule $SingleValue ($ValueSet) (lambda r (avg (var r)))) +) + +################################ +# Superlative + +(rule $FnOnRow ($Binary $Property) + (lambda b (lambda p (lambda x ((@R (var p)) ((@R (var b)) (var x)))))) +) + +(when superlative + (for @argm (argmax argmin) + (rule $RowSet ($RowSet) (lambda r (@argm 1 1 (var r) @index))) + (rule $RowSet ($RowSet $FnOnRow) (lambda r (lambda f (@argm 1 1 (var r) (@R (var f)))))) + ) +) + +################################ +# Merge + +(when merge + (rule $RowSet ($RowSet $RowSet) + (lambda r1 (lambda r2 (and (var r1) (var r2)))) + ) +) + +################################ +# Arithmatic + +(rule $FnOnValue ($Binary $Binary $Property) + (lambda b1 (lambda b2 (lambda p (lambda x ((@R (var p)) ((@R (var b2)) ((var b1) (var x)))))))) +) +(rule $FnOnValue ($Binary) + (lambda b (lambda x (count ((var b) (var x))))) +) +(rule $FnOnValue ($Binary $Property) + (lambda b (lambda p (lambda x (count ((var b) ((var p) (var x))))))) +) + +(when arithmetic + (rule $SingleValue ($FnOnValue $Entity $Entity) + (lambda f (lambda e1 (lambda e2 (- ((var f) (var e1)) ((var f) (var e2)))))) + ) +) + +################################ +# V-superlative + +(when v-superlative + (for @argm (argmax argmin) + (rule $ValueSet ($ValueSet $FnOnValue) (lambda v (lambda f (@argm 1 1 (var v) (reverse (var f)))))) + ) +) + +################################ +# ROOT +(rule $ROOT ($ValueSet) (IdentityFn)) +(rule $ROOT ($SingleValue) (IdentityFn)) From 9f97024e9d97cf5e89eb95f7f2e57a38bf47f15f Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Tue, 29 Aug 2017 03:28:48 -0700 Subject: [PATCH 11/30] Refactored the stuff in FloatingParser --- .../stanford/nlp/sempre/FloatingParser.java | 176 ++++++++++++------ .../sempre/cprune/CPruneFloatingParser.java | 10 +- 2 files changed, 125 insertions(+), 61 deletions(-) diff --git a/src/edu/stanford/nlp/sempre/FloatingParser.java b/src/edu/stanford/nlp/sempre/FloatingParser.java index 827b190..721ff88 100644 --- a/src/edu/stanford/nlp/sempre/FloatingParser.java +++ b/src/edu/stanford/nlp/sempre/FloatingParser.java @@ -89,10 +89,29 @@ public class FloatingParser extends Parser { public static Options opts = new Options(); + public boolean earlyStopOnConsistent = false; + public int earlyStopOnNumDerivs = -1; + public FloatingParser(Spec spec) { super(spec); } + /** + * Set early stopping criteria + * + * @param onConsistent + * Stop when a consistent derivation is found. (Only triggered when computeExpectedCounts = true) + * @param onNumDerivs + * Stop when the number of featurized derivations exceed this number (set to -1 to disable) + * @return + * this + */ + public FloatingParser setEarlyStopping(boolean onConsistent, int onNumDerivs) { + this.earlyStopOnConsistent = onConsistent; + this.earlyStopOnNumDerivs = onNumDerivs; + return this; + } + /** * computeCatUnaryRules, but do not topologically sort floating rules */ @@ -435,70 +454,107 @@ class FloatingParserState extends ParserState { derivations.addAll(myDerivations); } + /** + * Build derivations in a thread to allow timeout. + */ + class DerivationBuilder implements Runnable { + @Override public void run() { + // Base case ($TOKEN, $PHRASE) + for (Derivation deriv : gatherTokenAndPhraseDerivations()) { + addToChart(anchoredCell(deriv.cat, deriv.start, deriv.end), deriv); + addToChart(floatingCell(deriv.cat, 0), deriv); + } + + Set categories = new HashSet<>(); + for (Rule rule : parser.grammar.rules) + categories.add(rule.lhs); + + if (Parser.opts.verbose >= 1) + LogInfo.begin_track_printAll("Anchored"); + // Build up anchored derivations (like the BeamParser) + int numTokens = ex.numTokens(); + for (int len = 1; len <= numTokens; len++) { + for (int i = 0; i + len <= numTokens; i++) { + buildAnchored(i, i + len); + for (String cat : categories) { + String cell = anchoredCell(cat, i, i + len).toString(); + pruneCell(cell, chart.get(cell)); + } + } + } + if (Parser.opts.verbose >= 1) + LogInfo.end_track(); + + // Build up floating derivations + for (int depth = (FloatingParser.opts.initialFloatingHasZeroDepth ? 0 : 1); depth <= FloatingParser.opts.maxDepth; depth++) { + if (Parser.opts.verbose >= 1) + LogInfo.begin_track_printAll("%s = %d", FloatingParser.opts.useSizeInsteadOfDepth ? "SIZE" : "DEPTH", depth); + buildFloating(depth); + for (String cat : categories) { + String cell = floatingCell(cat, depth).toString(); + pruneCell(cell, chart.get(cell)); + } + if (Parser.opts.verbose >= 1) + LogInfo.end_track(); + // Early stopping + if (computeExpectedCounts && ((FloatingParser) parser).earlyStopOnConsistent) { + // Consistent derivation found? + String cell = floatingCell(Rule.rootCat, depth).toString(); + List rootDerivs = chart.get(cell); + if (rootDerivs != null) { + for (Derivation rootDeriv : rootDerivs) { + rootDeriv.ensureExecuted(parser.executor, ex.context); + if (parser.valueEvaluator.getCompatibility(ex.targetValue, rootDeriv.value) == 1) { + LogInfo.logs("Early stopped: consistent derivation found at depth = %d", depth); + return; + } + } + } + } + if (((FloatingParser) parser).earlyStopOnNumDerivs > 0) { + // Too many derivations generated? + if (numOfFeaturizedDerivs > ((FloatingParser) parser).earlyStopOnNumDerivs) { + LogInfo.logs("Early stopped: number of derivations exceeded at depth = %d", depth); + return; + } + } + } + } + } + + public void buildDerivations() { + DerivationBuilder derivBuilder = new DerivationBuilder(); + if (FloatingParser.opts.maxFloatingParsingTime == Integer.MAX_VALUE) { + derivBuilder.run(); + } else { + Thread parsingThread = new Thread(derivBuilder); + parsingThread.start(); + try { + parsingThread.join(FloatingParser.opts.maxFloatingParsingTime * 1000); + if (parsingThread.isAlive()) { + // This will only interrupt first or second passes, not the final candidate collection. + LogInfo.warnings("Parsing time exceeded %d seconds. Will now interrupt ...", FloatingParser.opts.maxFloatingParsingTime); + timeout = true; + parsingThread.interrupt(); + parsingThread.join(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + LogInfo.fails("FloatingParser error: %s", e); + } + } + evaluation.add("timeout", timeout); + } + + // ============================================================ + // Main entry point + // ============================================================ + @Override public void infer() { LogInfo.begin_track_printAll("FloatingParser.infer()"); ruleTime = new HashMap<>(); - // Base case ($TOKEN, $PHRASE) - for (Derivation deriv : gatherTokenAndPhraseDerivations()) { - addToChart(anchoredCell(deriv.cat, deriv.start, deriv.end), deriv); - addToChart(floatingCell(deriv.cat, 0), deriv); - } - - Set categories = new HashSet<>(); - for (Rule rule : parser.grammar.rules) - categories.add(rule.lhs); - - if (Parser.opts.verbose >= 1) - LogInfo.begin_track_printAll("Anchored"); - // Build up anchored derivations (like the BeamParser) - int numTokens = ex.numTokens(); - for (int len = 1; len <= numTokens; len++) { - for (int i = 0; i + len <= numTokens; i++) { - buildAnchored(i, i + len); - for (String cat : categories) { - String cell = anchoredCell(cat, i, i + len).toString(); - pruneCell(cell, chart.get(cell)); - } - } - } - if (Parser.opts.verbose >= 1) - LogInfo.end_track(); - - // Build up floating derivations - // Timeout if taking too long - timeout = false; - Thread parsingThread = new Thread(new Runnable() { - @Override - public void run() { - for (int depth = (FloatingParser.opts.initialFloatingHasZeroDepth ? 0 : 1); depth <= FloatingParser.opts.maxDepth; depth++) { - if (Parser.opts.verbose >= 1) - LogInfo.begin_track_printAll("%s = %d", FloatingParser.opts.useSizeInsteadOfDepth ? "SIZE" : "DEPTH", depth); - buildFloating(depth); - for (String cat : categories) { - String cell = floatingCell(cat, depth).toString(); - pruneCell(cell, chart.get(cell)); - } - if (Parser.opts.verbose >= 1) - LogInfo.end_track(); - } - } - }); - parsingThread.start(); - try { - parsingThread.join(FloatingParser.opts.maxFloatingParsingTime * 1000); - if (parsingThread.isAlive()) { - // This will only interrupt first or second passes, not the final candidate collection. - LogInfo.warnings("Parsing time exceeded %d seconds. Will now interrupt ...", FloatingParser.opts.maxFloatingParsingTime); - timeout = true; - parsingThread.interrupt(); - parsingThread.join(); - } - } catch (InterruptedException e) { - e.printStackTrace(); - LogInfo.fails("DPParser error: %s", e); - } - evaluation.add("timeout", timeout); + buildDerivations(); if (FloatingParser.opts.summarizeRuleTime) summarizeRuleTime(); diff --git a/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java b/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java index e580359..5a7d464 100644 --- a/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java +++ b/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java @@ -11,7 +11,7 @@ public class CPruneFloatingParser extends FloatingParser { public CPruneFloatingParser(Spec spec) { super(spec); - exploreParser = new FloatingParser(spec); + exploreParser = new FloatingParser(spec).setEarlyStopping(true, CollaborativePruner.opts.maxDerivations); } @Override @@ -64,6 +64,10 @@ class CPruneFloatingParserState extends ParserState { predDerivations.clear(); predDerivations.addAll(exploreParserState.predDerivations); expectedCounts = exploreParserState.expectedCounts; + if (computeExpectedCounts) { + for (Derivation deriv : predDerivations) + CollaborativePruner.updateConsistentPattern(parser.valueEvaluator, ex, deriv); + } CollaborativePruner.stats.totalExplore += 1; if (CollaborativePruner.foundConsistentDerivation) CollaborativePruner.stats.successfulExplore += 1; @@ -80,6 +84,10 @@ class CPruneFloatingParserState extends ParserState { predDerivations.clear(); predDerivations.addAll(exploitParserState.predDerivations); expectedCounts = exploitParserState.expectedCounts; + if (computeExpectedCounts) { + for (Derivation deriv : predDerivations) + CollaborativePruner.updateConsistentPattern(parser.valueEvaluator, ex, deriv); + } boolean succeeds = CollaborativePruner.foundConsistentDerivation; CollaborativePruner.stats.totalExploit += 1; if (succeeds) From 7b710ab9424ea1eac3b5028be2c6457dfde8ec91 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Tue, 29 Aug 2017 04:04:50 -0700 Subject: [PATCH 12/30] Convert to lower case before evaluation --- src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java b/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java index 7138eac..930f01b 100644 --- a/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java +++ b/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java @@ -70,12 +70,12 @@ public class TableValueEvaluator implements ValueEvaluator { if (predText == null) predText = ""; if (opts.allowNormalizedStringMatch) { String targetTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(targetText); - targetText = StringNormalizationUtils.aggressiveNormalize(targetText); + targetText = StringNormalizationUtils.aggressiveNormalize(targetText).toLowerCase(); if (!targetTextOfficial.equals(targetText)) { LogInfo.warnings("Different normalization: [%s][%s]", targetTextOfficial, targetText); } String predTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(predText); - predText = StringNormalizationUtils.aggressiveNormalize(predText); + predText = StringNormalizationUtils.aggressiveNormalize(predText).toLowerCase(); if (!predTextOfficial.equals(predText)) { LogInfo.warnings("Different normalization: [%s][%s]", predTextOfficial, predText); } From 174f57b5d4053b0170fe814710c73d12f48044b9 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Tue, 29 Aug 2017 04:19:11 -0700 Subject: [PATCH 13/30] Changed how texts are normalized --- .../nlp/sempre/tables/StringNormalizationUtils.java | 8 +++++--- .../stanford/nlp/sempre/tables/TableValueEvaluator.java | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java index b32afb7..8d83eed 100644 --- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java @@ -341,7 +341,6 @@ public final class StringNormalizationUtils { // Dashed / Parenthesized information string = simpleNormalize(string); string = string.replaceAll("\\[[^\\]]*\\]", ""); - string = string.trim().replaceAll(" - .*$", ""); string = string.trim().replaceAll("\\([^)]*\\)$", ""); return string.replaceAll("\\s+", " ").trim(); } @@ -363,7 +362,7 @@ public final class StringNormalizationUtils { // Remove citations string = string.trim().replaceAll("((? metadata = ArrayListMultimap.create(); TableColumn column = new TableColumn("Test", "test", 0); analyzeString(o, metadata, column, new HashMap<>()); - LogInfo.logs("%s %s", o, metadata); + String aggressive = aggressiveNormalize(o).toLowerCase(); + String official = officialEvaluatorNormalize(o); + LogInfo.logs("%s %s | %s %s %s", o, metadata, official, aggressive, aggressive.equals(official)); } public static void main(String[] args) { @@ -395,6 +396,7 @@ public final class StringNormalizationUtils { unitTest("twenty three"); unitTest("apple, banana, banana, BANANA"); unitTest("apple\nbanana\norange"); + unitTest("0-1\n(4-5 p)"); unitTest("21st"); unitTest("2001st"); unitTest("2,000,000 ft."); diff --git a/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java b/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java index 930f01b..b4eb137 100644 --- a/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java +++ b/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java @@ -71,12 +71,12 @@ public class TableValueEvaluator implements ValueEvaluator { if (opts.allowNormalizedStringMatch) { String targetTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(targetText); targetText = StringNormalizationUtils.aggressiveNormalize(targetText).toLowerCase(); - if (!targetTextOfficial.equals(targetText)) { + if (!targetTextOfficial.equals(targetText) && !(targetTextOfficial + ".").equals(targetText)) { LogInfo.warnings("Different normalization: [%s][%s]", targetTextOfficial, targetText); } String predTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(predText); predText = StringNormalizationUtils.aggressiveNormalize(predText).toLowerCase(); - if (!predTextOfficial.equals(predText)) { + if (!predTextOfficial.equals(predText) && !(predTextOfficial + ".").equals(predText)) { LogInfo.warnings("Different normalization: [%s][%s]", predTextOfficial, predText); } } From f57d5c16b12aa75868a607df8c45e39d872121e2 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Tue, 29 Aug 2017 04:24:15 -0700 Subject: [PATCH 14/30] Oops --- .../stanford/nlp/sempre/tables/StringNormalizationUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java index 8d83eed..19efb21 100644 --- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java @@ -364,7 +364,7 @@ public final class StringNormalizationUtils { // Remove details in parenthesis string = string.trim().replaceAll("(? Date: Tue, 29 Aug 2017 04:27:56 -0700 Subject: [PATCH 15/30] Be more lenient with parentheses --- .../stanford/nlp/sempre/tables/StringNormalizationUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java index 19efb21..1213bfa 100644 --- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java @@ -362,7 +362,7 @@ public final class StringNormalizationUtils { // Remove citations string = string.trim().replaceAll("((? Date: Tue, 29 Aug 2017 04:30:27 -0700 Subject: [PATCH 16/30] One more --- .../nlp/sempre/tables/StringNormalizationUtils.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java index 1213bfa..7607bd7 100644 --- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java @@ -341,7 +341,11 @@ public final class StringNormalizationUtils { // Dashed / Parenthesized information string = simpleNormalize(string); string = string.replaceAll("\\[[^\\]]*\\]", ""); - string = string.trim().replaceAll("\\([^)]*\\)$", ""); + String oldString; + do { + oldString = string; + string = string.trim().replaceAll("\\([^)]*\\)$", ""); + } while (!oldString.equals(string)); return string.replaceAll("\\s+", " ").trim(); } From 922946cf0b9e49767f74d756a746cf4cc835fbc2 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Tue, 29 Aug 2017 06:05:44 -0700 Subject: [PATCH 17/30] Hack superlative for now --- .../stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java b/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java index b3f63d1..9bd6c04 100644 --- a/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java @@ -269,7 +269,7 @@ public final class DenotationUtils { default: throw new LambdaDCSException(Type.invalidFormula, "Unknown superlative mode: %s", mode); } int from = Math.min(rank - 1, indices.size()), - to = Math.min(from + count, indices.size()); + to = Math.min(Math.max(from, from + count), indices.size()); for (int index : indices.subList(from, to)) answer.add(pairs.get(index).getSecond()); } From e355572acd42a8a9bce58b386999bee27fbba870 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Tue, 29 Aug 2017 13:38:45 -0700 Subject: [PATCH 18/30] Added TOPALT --- src/edu/stanford/nlp/sempre/ParserState.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/edu/stanford/nlp/sempre/ParserState.java b/src/edu/stanford/nlp/sempre/ParserState.java index 680fd40..9194bcb 100644 --- a/src/edu/stanford/nlp/sempre/ParserState.java +++ b/src/edu/stanford/nlp/sempre/ParserState.java @@ -24,7 +24,7 @@ public abstract class ParserState { } public static Options opts = new Options(); - public enum CustomExpectedCount { NONE, UNIFORM, TOP, RANDOM, } + public enum CustomExpectedCount { NONE, UNIFORM, TOP, TOPALT, RANDOM, } //// Input: specification of how to parse @@ -130,7 +130,7 @@ public abstract class ParserState { for (Derivation deriv : derivations) deriv.score += Parser.opts.derivationScoreRandom.nextDouble() * Parser.opts.derivationScoreNoise; } - + Derivation.sortByScore(derivations); // Print out information @@ -275,7 +275,7 @@ public abstract class ParserState { predScores = new double[n]; // For update schemas that choose one good and one bad candidate to update int[] goodAndBad = null; - if (opts.customExpectedCounts == CustomExpectedCount.TOP) { + if (opts.customExpectedCounts == CustomExpectedCount.TOP || opts.customExpectedCounts == CustomExpectedCount.TOPALT) { goodAndBad = getTopDerivations(derivations); if (goodAndBad == null) return; } else if (opts.customExpectedCounts == CustomExpectedCount.RANDOM) { @@ -300,6 +300,10 @@ public abstract class ParserState { trueScores[i] = (i == goodAndBad[0]) ? 0 : Double.NEGATIVE_INFINITY; predScores[i] = (i == goodAndBad[1]) ? 0 : Double.NEGATIVE_INFINITY; break; + case TOPALT: + trueScores[i] = (i == goodAndBad[0]) ? 0 : Double.NEGATIVE_INFINITY; + predScores[i] = (i == goodAndBad[0] || i == goodAndBad[1]) ? deriv.score : Double.NEGATIVE_INFINITY; + break; default: throw new RuntimeException("Unknown customExpectedCounts: " + opts.customExpectedCounts); } From c475bddba284d97c675d7c532112a715382e839f Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Wed, 30 Aug 2017 15:40:38 -0700 Subject: [PATCH 19/30] Added table executor --- run | 3 +- src/edu/stanford/nlp/sempre/Learner.java | 1 + .../tables/test/BatchTableExecutor.java | 136 ++++++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 src/edu/stanford/nlp/sempre/tables/test/BatchTableExecutor.java diff --git a/run b/run index 9a98f0f..e55c140 100755 --- a/run +++ b/run @@ -650,6 +650,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( 'alter-ex' => l('edu.stanford.nlp.sempre.tables.alter.AlteredTablesExecutor', let(:parser, 'serialized')), 'filter' => 'edu.stanford.nlp.sempre.tables.serialize.DumpFilterer', 'column' => 'edu.stanford.nlp.sempre.tables.test.TableColumnAnalyzer', + 'execute' => 'edu.stanford.nlp.sempre.tables.test.BatchTableExecutor', }), # Fig parameters selo(:cldir, 'execDir', '_OUTPATH_', '.'), @@ -674,6 +675,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( }), # Parser letDefault(:parser, 'floatsize'), + o('beamSize', 100), o('useSizeInsteadOfDepth'), sel(:parser, { 'floatsize' => l( @@ -702,7 +704,6 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( 'cprune' => l( o('Builder.parser', 'cprune.CPruneFloatingParser'), o('FloatingParser.maxDepth', 15), - o('beamSize', 100), o('maxNumNeighbors', 40), o('maxPredictedPatterns', 1000), nil), diff --git a/src/edu/stanford/nlp/sempre/Learner.java b/src/edu/stanford/nlp/sempre/Learner.java index dc45d03..5dd0f3f 100644 --- a/src/edu/stanford/nlp/sempre/Learner.java +++ b/src/edu/stanford/nlp/sempre/Learner.java @@ -312,6 +312,7 @@ public class Learner { // evaluation.add(LexiconFn.lexEval); evaluation.logStats(prefix); evaluation.putOutput(prefix); + evaluation.putOutput(prefix.replaceAll("iter=", "").replace('.', '_')); } private void printLearnerEventsIter(Example ex, int iter, String group) { diff --git a/src/edu/stanford/nlp/sempre/tables/test/BatchTableExecutor.java b/src/edu/stanford/nlp/sempre/tables/test/BatchTableExecutor.java new file mode 100644 index 0000000..1dab7f6 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/tables/test/BatchTableExecutor.java @@ -0,0 +1,136 @@ +package edu.stanford.nlp.sempre.tables.test; + +import java.io.*; +import java.util.*; + +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.tables.TableKnowledgeGraph; +import edu.stanford.nlp.sempre.tables.TableValueEvaluator; +import edu.stanford.nlp.sempre.tables.lambdadcs.LambdaDCSExecutor; +import fig.basic.*; +import fig.exec.Execution; + +/** + * Execute the specified logical forms on the specified WikiTableQuestions context. + * + * @author ppasupat + */ +public class BatchTableExecutor implements Runnable { + public static class Options { + @Option(gloss = "TSV file containing table contexts and logical forms") + public String batchInput; + @Option(gloss = "Datasets for mapping example IDs to contexts") + public List batchDatasets = Arrays.asList("lib/data/tables/data/training.examples"); + } + public static Options opts = new Options(); + + public static void main(String[] args) { + Execution.run(args, "BatchTableExecutorMain", new BatchTableExecutor(), Master.getOptionsParser()); + } + + @Override + public void run() { + if (opts.batchInput == null || opts.batchInput.isEmpty()) { + LogInfo.logs("*******************************************************************************"); + LogInfo.logs("USAGE: ./run @mode=tables @class=execute -batchInput "); + LogInfo.logs(""); + LogInfo.logs("Input file format: Each line has something like"); + LogInfo.logs(" nt-218 [tab] (count (fb:type.object.type fb:type.row))"); + LogInfo.logs("or"); + LogInfo.logs(" csv/204-csv/23.csv [tab] (count (fb:type.object.type fb:type.row))"); + LogInfo.logs(""); + LogInfo.logs("Results will also be printed to state/execs/___.exec/denotations.tsv"); + LogInfo.logs("Output format:"); + LogInfo.logs(" nt-218 [tab] (count (fb:type.object.type fb:type.row)) [tab] (list (number 10)) [tab] false"); + LogInfo.logs("where the last column indicates whether the answer is consistent with the target answer"); + LogInfo.logs("(only available when the first column is nt-___)"); + LogInfo.logs("*******************************************************************************"); + System.exit(1); + } + LambdaDCSExecutor executor = new LambdaDCSExecutor(); + ValueEvaluator evaluator = new TableValueEvaluator(); + try { + BufferedReader reader = IOUtils.openIn(opts.batchInput); + PrintWriter output = IOUtils.openOut(Execution.getFile("denotations.tsv")); + String line; + while ((line = reader.readLine()) != null) { + String[] tokens = line.split("\t"); + String answer; + try { + Formula formula = Formula.fromString(tokens[1]); + if (tokens[0].startsWith("csv")) { + TableKnowledgeGraph graph = TableKnowledgeGraph.fromFilename(tokens[0]); + ContextValue context = new ContextValue(graph); + Value denotation = executor.execute(formula, context).value; + if (denotation instanceof ListValue) + denotation = addOriginalStrings((ListValue) denotation, graph); + answer = denotation.toString(); + } else { + Example ex = exIdToExample(tokens[0]); + Value denotation = executor.execute(formula, ex.context).value; + if (denotation instanceof ListValue) + denotation = addOriginalStrings((ListValue) denotation, (TableKnowledgeGraph) ex.context.graph); + answer = denotation.toString(); + boolean correct = evaluator.getCompatibility(ex.targetValue, denotation) == 1.; + answer = denotation.toString() + "\t" + correct; + } + } catch (Exception e) { + answer = "ERROR: " + e; + } + System.out.printf("%s\t%s\t%s\n", tokens[0], tokens[1], answer); + output.printf("%s\t%s\t%s\n", tokens[0], tokens[1], answer); + } + reader.close(); + output.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Map exIdToExampleMap; + + private Example exIdToExample(String exId) { + if (exIdToExampleMap == null) { + exIdToExampleMap = new HashMap<>(); + try { + for (String filename : opts.batchDatasets) { + BufferedReader reader = IOUtils.openIn(filename); + String line; + while ((line = reader.readLine()) != null) { + LispTree tree = LispTree.proto.parseFromString(line); + if (!"id".equals(tree.child(1).child(0).value)) + throw new RuntimeException("Malformed example: " + line); + exIdToExampleMap.put(tree.child(1).child(1).value, tree); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + Object obj = exIdToExampleMap.get(exId); + if (obj == null) return null; + Example ex; + if (obj instanceof LispTree) { + ex = Example.fromLispTree((LispTree) obj, exId); + ex.preprocess(); + exIdToExampleMap.put(exId, ex); + } else { + ex = (Example) obj; + } + return ex; + } + + ListValue addOriginalStrings(ListValue answers, TableKnowledgeGraph graph) { + List values = new ArrayList<>(); + for (Value value : answers.values) { + if (value instanceof NameValue) { + NameValue name = (NameValue) value; + if (name.description == null) + value = new NameValue(name.id, graph.getOriginalString(((NameValue) value).id)); + } + values.add(value); + } + return new ListValue(values); + } + +} From 8a987add98231a1172b862a993013e28a2a36361 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Wed, 30 Aug 2017 18:21:22 -0700 Subject: [PATCH 20/30] Added more features --- run | 12 ++- src/edu/stanford/nlp/sempre/FuzzyMatchFn.java | 3 + .../sempre/tables/TableKnowledgeGraph.java | 12 +++ .../features/AnchorFeatureComputer.java | 68 +++++++++++++++ .../tables/features/ColumnCategoryInfo.java | 84 +++++++++++++++++++ .../PhraseDenotationFeatureComputer.java | 1 + .../sempre/tables/features/PhraseInfo.java | 16 ++++ .../PhrasePredicateFeatureComputer.java | 32 ++++++- .../tables/lambdadcs/DenotationUtils.java | 4 +- 9 files changed, 222 insertions(+), 10 deletions(-) create mode 100644 src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java create mode 100644 src/edu/stanford/nlp/sempre/tables/features/ColumnCategoryInfo.java diff --git a/run b/run index e55c140..18e944e 100755 --- a/run +++ b/run @@ -675,7 +675,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( }), # Parser letDefault(:parser, 'floatsize'), - o('beamSize', 100), + o('beamSize', 50), o('useSizeInsteadOfDepth'), sel(:parser, { 'floatsize' => l( @@ -728,8 +728,8 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( nil), 'editdist-fuzzy' => l( o('FuzzyMatcher.fuzzyMatcher', 'tables.match.EditDistanceFuzzyMatcher'), - o('fuzzyMatchSubstring'), o('fuzzyMatchMaxEditDistanceRatio', 0.3), - o('alsoReturnUnion'), o('alsoMatchPart'), + o('fuzzyMatchSubstring'), o('fuzzyMatchMaxEditDistanceRatio', 0.15), + o('alsoMatchPart'), nil), }), letDefault(:normalize, 1), @@ -814,10 +814,14 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( 'some' => l( # Add your own features! (only set up the feature computers) o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split), nil), - 'all' => l( # All features + 'all' => l( # All ACL 2015 features o('FeatureExtractor.featureDomains', 'custom-denotation phrase-predicate phrase-denotation headword-denotation missing-predicate'.split), o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split), nil), + 'more' => l( # All ACL 2015 features + more experimental features + o('FeatureExtractor.featureDomains', 'custom-denotation phrase-predicate phrase-denotation headword-denotation missing-predicate anchored-entity'.split), + o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer tables.features.AnchorFeatureComputer'.split), + nil), 'baseline' => l( # For the baseline classifier o('FeatureExtractor.featureDomains', 'custom-denotation phrase-denotation headword-denotation table-baseline'.split), o('FeatureExtractor.featureComputers', 'tables.baseline.TableBaselineFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split), diff --git a/src/edu/stanford/nlp/sempre/FuzzyMatchFn.java b/src/edu/stanford/nlp/sempre/FuzzyMatchFn.java index 6c98126..9f427c5 100644 --- a/src/edu/stanford/nlp/sempre/FuzzyMatchFn.java +++ b/src/edu/stanford/nlp/sempre/FuzzyMatchFn.java @@ -39,6 +39,9 @@ public class FuzzyMatchFn extends SemanticFn { } } + public FuzzyMatchFnMode getMode() { return mode; } + public boolean getMatchAny() { return matchAny; } + @Override public DerivationStream call(Example ex, Callable c) { return new LazyFuzzyMatchFnDerivs(ex, c, mode, matchAny); diff --git a/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java b/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java index 01b31ad..eca3bc3 100644 --- a/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java +++ b/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java @@ -660,6 +660,18 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl return answer; } + /** + * Return the index of the column with the specified ID. Return -1 if not found. + */ + public int getColumnIndex(String nameValueId) { + if (nameValueId.startsWith("!")) + nameValueId = nameValueId.substring(1); + for (int j = 0; j < columns.size(); j++) { + if (columns.get(j).relationNameValue.id.equals(nameValueId)) return j; + } + return -1; + } + // ============================================================ // Test // ============================================================ diff --git a/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java b/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java new file mode 100644 index 0000000..a3dd20c --- /dev/null +++ b/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java @@ -0,0 +1,68 @@ +package edu.stanford.nlp.sempre.tables.features; + +import java.util.*; + +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.tables.StringNormalizationUtils; +import edu.stanford.nlp.sempre.tables.TableKnowledgeGraph; +import edu.stanford.nlp.sempre.tables.TableCell; +import edu.stanford.nlp.sempre.tables.TableColumn; +import fig.basic.LogInfo; + +public class AnchorFeatureComputer implements FeatureComputer { + + @Override + public void extractLocal(Example ex, Derivation deriv) { + if (!(FeatureExtractor.containsDomain("anchored-entity"))) return; + if (!(deriv.rule.sem instanceof FuzzyMatchFn)) return; + FuzzyMatchFn sem = (FuzzyMatchFn) deriv.rule.sem; + if (sem.getMatchAny() || sem.getMode() != FuzzyMatchFn.FuzzyMatchFnMode.ENTITY) return; + String phrase = ((StringValue) ((ValueFormula) deriv.child(0).formula).value).value; + NameValue predicate = (NameValue) ((ValueFormula) deriv.formula).value; + TableKnowledgeGraph graph = (TableKnowledgeGraph) ex.context.graph; + extractMatchingFeatures(graph, deriv, phrase, predicate); + } + + private void extractMatchingFeatures(TableKnowledgeGraph graph, + Derivation deriv, String phrase, NameValue predicate) { + LogInfo.logs("%s -> %s", phrase, predicate); + // Type of match + String predicateString = graph.getOriginalString(predicate); + predicateString = StringNormalizationUtils.simpleNormalize(predicateString).toLowerCase(); + if (predicateString.equals(phrase)) { + deriv.addFeature("a-e", "exact"); + LogInfo.logs("%s %s exact", phrase, predicateString); + } else if (predicateString.startsWith(phrase + " ")) { + deriv.addFeature("a-e", "prefix"); + LogInfo.logs("%s %s prefix", phrase, predicateString); + } else if (predicateString.endsWith(" " + phrase)) { + deriv.addFeature("a-e", "suffix"); + LogInfo.logs("%s %s suffix", phrase, predicateString); + } else if (predicateString.contains(" " + phrase + " ")){ + deriv.addFeature("a-e", "substring"); + LogInfo.logs("%s %s substring", phrase, predicateString); + } else { + deriv.addFeature("a-e", "other"); + LogInfo.logs("%s %s other", phrase, predicateString); + } + // Does the phrase match other cells? + Set matches = new HashSet<>(); + for (TableColumn column : graph.columns) { + for (TableCell cell : column.children) { + String s = StringNormalizationUtils.simpleNormalize(cell.properties.originalString).toLowerCase(); + if (s.contains(phrase) && !cell.properties.id.equals(predicate.id)) { + matches.add(s); + } + } + } + LogInfo.logs(">> %s", matches); + if (matches.size() == 0) { + deriv.addFeature("a-e", "unique"); + } else if (matches.size() < 3) { + deriv.addFeature("a-e", "multiple;" + matches.size()); + } else { + deriv.addFeature("a-e", "multiple;>=3"); + } + } + +} diff --git a/src/edu/stanford/nlp/sempre/tables/features/ColumnCategoryInfo.java b/src/edu/stanford/nlp/sempre/tables/features/ColumnCategoryInfo.java new file mode 100644 index 0000000..d776fc2 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/tables/features/ColumnCategoryInfo.java @@ -0,0 +1,84 @@ +package edu.stanford.nlp.sempre.tables.features; + +import java.io.*; +import java.util.*; + +import edu.stanford.nlp.sempre.Example; +import edu.stanford.nlp.sempre.tables.TableKnowledgeGraph; +import fig.basic.*; + +public class ColumnCategoryInfo { + public static class Options { + @Option(gloss = "Read category information from this file") + public String tableCategoryInfo = null; + } + public static Options opts = new Options(); + + // ============================================================ + // Singleton access + // ============================================================ + + private static ColumnCategoryInfo singleton; + + public static ColumnCategoryInfo getSingleton() { + if (opts.tableCategoryInfo == null) + return null; + else if (singleton == null) + singleton = new ColumnCategoryInfo(); + return singleton; + } + + // ============================================================ + // Read data from file + // ============================================================ + + // tableId -> columnIndex -> list of (category, weight) + protected static Map>>> allCategoryInfo = null; + + private ColumnCategoryInfo() { + LogInfo.begin_track("Loading category information from %s", opts.tableCategoryInfo); + allCategoryInfo = new HashMap<>(); + try { + BufferedReader reader = IOUtils.openIn(opts.tableCategoryInfo); + String line; + while ((line = reader.readLine()) != null) { + String[] tokens = line.split("\t"); + String tableId = tokens[0]; + List>> categoryInfoForTable = allCategoryInfo.get(tableId); + if (categoryInfoForTable == null) + allCategoryInfo.put(tableId, categoryInfoForTable = new ArrayList<>()); + int columnIndex = Integer.parseInt(tokens[1]); + // Assume that the columns are ordered + assert categoryInfoForTable.size() == columnIndex; + // Read the category-weight pairs + List> categories = new ArrayList<>(); + for (int i = 2; i < tokens.length; i++) { + String[] pair = tokens[i].split(":"); + categories.add(new Pair<>(pair[0], Double.parseDouble(pair[1]))); + } + categoryInfoForTable.add(categories); + } + reader.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + LogInfo.end_track(); + } + + // ============================================================ + // Getters + // ============================================================ + + public List> get(String tableId, int columnIndex) { + return allCategoryInfo.get(tableId).get(columnIndex); + } + + public List> get(Example ex, String columnId) { + TableKnowledgeGraph graph = (TableKnowledgeGraph) ex.context.graph; + String tableId = graph.filename; + int columnIndex = graph.getColumnIndex(columnId); + if (columnIndex == -1) return null; + return allCategoryInfo.get(tableId).get(columnIndex); + } + +} diff --git a/src/edu/stanford/nlp/sempre/tables/features/PhraseDenotationFeatureComputer.java b/src/edu/stanford/nlp/sempre/tables/features/PhraseDenotationFeatureComputer.java index 5f33941..895ef39 100644 --- a/src/edu/stanford/nlp/sempre/tables/features/PhraseDenotationFeatureComputer.java +++ b/src/edu/stanford/nlp/sempre/tables/features/PhraseDenotationFeatureComputer.java @@ -113,6 +113,7 @@ public class PhraseDenotationFeatureComputer implements FeatureComputer { LogInfo.logs("%s %s %s", deriv.value, deriv.type, denotationTypes); for (String denotationType : denotationTypes) { for (PhraseInfo phraseInfo : phraseInfos) { + if (PhraseInfo.opts.forbidBorderStopWordInLexicalizedFeatures && phraseInfo.isBorderStopWord) continue; deriv.addFeature("p-d", phraseInfo.lemmaText + ";" + denotationType); } // Check original column text diff --git a/src/edu/stanford/nlp/sempre/tables/features/PhraseInfo.java b/src/edu/stanford/nlp/sempre/tables/features/PhraseInfo.java index 1ba0cd5..4c1c82c 100644 --- a/src/edu/stanford/nlp/sempre/tables/features/PhraseInfo.java +++ b/src/edu/stanford/nlp/sempre/tables/features/PhraseInfo.java @@ -2,6 +2,7 @@ package edu.stanford.nlp.sempre.tables.features; import java.util.*; import java.util.concurrent.ExecutionException; +import java.util.regex.Pattern; import com.google.common.cache.*; @@ -23,6 +24,8 @@ public class PhraseInfo { public int maxPhraseLength = 3; @Option(gloss = "Fuzzy match predicates") public boolean computeFuzzyMatchPredicates = false; + @Option(gloss = "Do not produce lexicalized features if the phrase begins or ends with a stop word") + public boolean forbidBorderStopWordInLexicalizedFeatures = false; } public static Options opts = new Options(); @@ -35,6 +38,7 @@ public class PhraseInfo { public final List nerTags; public final String canonicalPosSeq; public final List fuzzyMatchedPredicates; + public final boolean isBorderStopWord; // true if the first or last word is a stop word public PhraseInfo(Example ex, int start, int end) { this.start = start; @@ -49,6 +53,7 @@ public class PhraseInfo { lemmaText = languageInfo.lemmaPhrase(start, end).toLowerCase(); canonicalPosSeq = languageInfo.canonicalPosSeq(start, end); fuzzyMatchedPredicates = opts.computeFuzzyMatchPredicates ? getFuzzyMatchedPredicates(ex.context) : null; + isBorderStopWord = isStopWord(languageInfo.lemmaTokens.get(start)) || isStopWord(languageInfo.lemmaTokens.get(end - 1)); } private List getFuzzyMatchedPredicates(ContextValue context) { @@ -71,6 +76,17 @@ public class PhraseInfo { return matchedPredicates; } + static final Pattern ALL_PUNCT = Pattern.compile("^[^A-Za-z0-9]*$"); + static final Set STOP_WORDS = new HashSet<>(Arrays.asList( + "a", "an", "the", "be", "of", "in", "on", "do" + )); + + static boolean isStopWord(String x) { + if (ALL_PUNCT.matcher(x).matches()) return true; + if (STOP_WORDS.contains(x)) return true; + return false; + } + @Override public String toString() { return "\"" + text + "\""; diff --git a/src/edu/stanford/nlp/sempre/tables/features/PhrasePredicateFeatureComputer.java b/src/edu/stanford/nlp/sempre/tables/features/PhrasePredicateFeatureComputer.java index d7b5f71..e61c2d2 100644 --- a/src/edu/stanford/nlp/sempre/tables/features/PhrasePredicateFeatureComputer.java +++ b/src/edu/stanford/nlp/sempre/tables/features/PhrasePredicateFeatureComputer.java @@ -35,6 +35,11 @@ public class PhrasePredicateFeatureComputer implements FeatureComputer { public boolean lexicalizedPhrasePredicate = true; @Option(gloss = "Maximum ngram length for lexicalize all pair features") public int maxNforLexicalizeAllPairs = Integer.MAX_VALUE; + @Option(gloss = "phrase-category: Weight threshold") + public double phraseCategoryWeightThreshold = 0.8; + @Option(gloss = "phrase-category: Use binary features instead of continuous ones") + public boolean phraseCategoryBinary = true; + } public static Options opts = new Options(); @@ -47,7 +52,8 @@ public class PhrasePredicateFeatureComputer implements FeatureComputer { @Override public void extractLocal(Example ex, Derivation deriv) { if (!(FeatureExtractor.containsDomain("phrase-predicate") - || FeatureExtractor.containsDomain("missing-predicate"))) return; + || FeatureExtractor.containsDomain("phrase-formula") + || FeatureExtractor.containsDomain("phrase-category"))) return; // Only compute features at the root, except when the partial option is set. if (!opts.defineOnPartialDerivs && !deriv.isRoot(ex.numTokens())) return; List phraseInfos = PhraseInfo.getPhraseInfos(ex); @@ -58,7 +64,8 @@ public class PhrasePredicateFeatureComputer implements FeatureComputer { LogInfo.logs("Derivation: %s", deriv); LogInfo.logs("Predicates: %s", predicateInfos); } - if (FeatureExtractor.containsDomain("phrase-predicate")) { + if (FeatureExtractor.containsDomain("phrase-predicate") + || FeatureExtractor.containsDomain("phrase-category")) { if (opts.defineOnPartialDerivs) { deriv.getTempState().put("p-p", new ArrayList<>(predicateInfos)); // Subtract predicates from children @@ -109,7 +116,7 @@ public class PhrasePredicateFeatureComputer implements FeatureComputer { private void extractMatch(Example ex, Derivation deriv, PhraseInfo phraseInfo, String phraseString, String phraseType, PredicateInfo predicateInfo, String predicateString, String predicateType, double factor) { - if (opts.unlexicalizedPhrasePredicate) { + if (FeatureExtractor.containsDomain("phrase-predicate") && opts.unlexicalizedPhrasePredicate) { if (phraseString.equals(predicateString)) { defineFeatures(ex, deriv, phraseInfo, predicateInfo, phraseType + "=" + predicateType, phraseString, predicateString, factor); @@ -132,10 +139,27 @@ public class PhrasePredicateFeatureComputer implements FeatureComputer { } } } - if (opts.lexicalizedPhrasePredicate && phraseInfo.end - phraseInfo.start <= maxNforLexicalizeAllPairs) { + if (FeatureExtractor.containsDomain("phrase-predicate") && opts.lexicalizedPhrasePredicate + && phraseInfo.end - phraseInfo.start <= maxNforLexicalizeAllPairs + && (!PhraseInfo.opts.forbidBorderStopWordInLexicalizedFeatures || !phraseInfo.isBorderStopWord)) { deriv.addFeature("p-p", phraseType + phraseString + ";" + predicateType + predicateString, factor); } + if (FeatureExtractor.containsDomain("phrase-category") && predicateInfo.type == PredicateType.BINARY + && (!PhraseInfo.opts.forbidBorderStopWordInLexicalizedFeatures || !phraseInfo.isBorderStopWord)) { + ColumnCategoryInfo catInfo = ColumnCategoryInfo.getSingleton(); + List> categories = catInfo.get(ex, predicateInfo.predicate); + if (categories != null) { + for (Pair pair : categories) { + if (pair.getSecond() >= opts.phraseCategoryWeightThreshold) { + if (opts.phraseCategoryBinary) + deriv.addFeature("p-c", phraseType + phraseString + ";" + pair.getFirst()); + else + deriv.addFeature("p-c", phraseType + phraseString + ";" + pair.getFirst(), pair.getSecond()); + } + } + } + } } private void defineFeatures(Example ex, Derivation deriv, PhraseInfo phraseInfo, PredicateInfo predicateInfo, diff --git a/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java b/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java index 9bd6c04..f586b3b 100644 --- a/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java @@ -238,7 +238,7 @@ public final class DenotationUtils { */ public static UnaryDenotation superlativeUnary(int rank, int count, List> pairs, SuperlativeFormula.Mode mode, TypeProcessor processor) { - if (rank <= 0 || count <= 0) + if (rank <= 0 || count <= 0 || rank >= 1000000 || count >= 100000) LogInfo.fails("Invalid superlative (rank = %d, count = %d)", rank, count); if (pairs.isEmpty()) { if (LambdaDCSExecutor.opts.superlativesFailOnEmptyLists) @@ -269,7 +269,7 @@ public final class DenotationUtils { default: throw new LambdaDCSException(Type.invalidFormula, "Unknown superlative mode: %s", mode); } int from = Math.min(rank - 1, indices.size()), - to = Math.min(Math.max(from, from + count), indices.size()); + to = Math.min(from + count, indices.size()); for (int index : indices.subList(from, to)) answer.add(pairs.get(index).getSecond()); } From bc878e69e5a3b4a55d7d58787e2c607710770b49 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Wed, 30 Aug 2017 20:49:51 -0700 Subject: [PATCH 21/30] Try to fix the superlative error + refactored a bunch --- src/edu/stanford/nlp/sempre/Rule.java | 14 +-- .../sempre/cprune/CPruneFloatingParser.java | 9 +- .../sempre/cprune/CollaborativePruner.java | 79 +++++--------- .../nlp/sempre/cprune/CustomGrammar.java | 19 ++-- .../nlp/sempre/cprune/FormulaPattern.java | 102 ++++++++++++++++++ .../stanford/nlp/sempre/cprune/Pattern.java | 34 ------ .../nlp/sempre/cprune/PatternInfo.java | 92 ---------------- .../sempre/cprune/RegexReplaceManager.java | 15 --- .../tables/lambdadcs/LambdaDCSExecutor.java | 3 + 9 files changed, 156 insertions(+), 211 deletions(-) create mode 100644 src/edu/stanford/nlp/sempre/cprune/FormulaPattern.java delete mode 100644 src/edu/stanford/nlp/sempre/cprune/Pattern.java delete mode 100644 src/edu/stanford/nlp/sempre/cprune/PatternInfo.java delete mode 100644 src/edu/stanford/nlp/sempre/cprune/RegexReplaceManager.java diff --git a/src/edu/stanford/nlp/sempre/Rule.java b/src/edu/stanford/nlp/sempre/Rule.java index a871e0e..31f30b7 100644 --- a/src/edu/stanford/nlp/sempre/Rule.java +++ b/src/edu/stanford/nlp/sempre/Rule.java @@ -31,7 +31,7 @@ public class Rule { public final SemanticFn sem; // Takes derivations corresponding to RHS categories and produces a set of derivations corresponding to LHS. public List> info; // Extra info public RuleSource source = null; // for tracking where the rule comes from when they are induced - + // Cache the semanticRepn public String getSemRepn() { if (semRepn == null) semRepn = sem.getClass().getSimpleName(); @@ -51,9 +51,9 @@ public class Rule { public String toString() { if (stringRepn == null) { String semStr = sem == null? "NullSemanticFn" : sem.toString(); - int maxLength = 100; - if (semStr.length() > maxLength) - semStr = String.format("%s...(%d total)", semStr.substring(0,maxLength), semStr.length()); + //int maxLength = 100; + //if (semStr.length() > maxLength) + // semStr = String.format("%s...(%d total)", semStr.substring(0,maxLength), semStr.length()); stringRepn = lhs + " -> " + (rhs == null ? "" : Joiner.on(' ').join(rhs)) + " " + semStr; } return stringRepn; @@ -139,13 +139,13 @@ public class Rule { else return f == 1.0 ? false : !FloatingParser.opts.defaultIsFloating; } - + public boolean isInduced() { double a = getInfoTag("induced"); if (a == 1.0) return true; return false; } - + @Override public boolean equals(Object o) { if (!(o instanceof Rule)) return false; @@ -155,7 +155,7 @@ public class Rule { public int hashCode() { return this.toString().hashCode(); } - + public String toJson() { Map jsonMap = new LinkedHashMap<>(); jsonMap.put("lhs", lhs); diff --git a/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java b/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java index 5a7d464..107615f 100644 --- a/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java +++ b/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java @@ -16,7 +16,7 @@ public class CPruneFloatingParser extends FloatingParser { @Override public void onBeginDataGroup(int iter, int numIters, String group) { - if (CollaborativePruner.neighbors == null) { + if (CollaborativePruner.uidToCachedNeighbors == null) { CollaborativePruner.customGrammar.init(grammar); CollaborativePruner.loadNeighbors(); } @@ -77,8 +77,8 @@ class CPruneFloatingParserState extends ParserState { public boolean exploit() { LogInfo.begin_track("Exploit"); CollaborativePruner.initialize(ex, CollaborativePruner.Mode.EXPLOIT); - Parser exploitParser = new FloatingParser( - new Parser.Spec(new MiniGrammar(CollaborativePruner.predictedRules), parser.extractor, parser.executor, parser.valueEvaluator)); + Grammar miniGrammar = new MiniGrammar(CollaborativePruner.predictedRules); + Parser exploitParser = new FloatingParser(new Parser.Spec(miniGrammar, parser.extractor, parser.executor, parser.valueEvaluator)); ParserState exploitParserState = exploitParser.newParserState(params, ex, computeExpectedCounts); exploitParserState.infer(); predDerivations.clear(); @@ -105,6 +105,9 @@ class MiniGrammar extends Grammar { public MiniGrammar(List rules) { this.rules.addAll(rules); + LogInfo.begin_track("MiniGrammar Rules"); + for (Rule rule : rules) LogInfo.logs("%s", rule); + LogInfo.end_track(); } } diff --git a/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java b/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java index cf91014..df95aee 100644 --- a/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java +++ b/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java @@ -5,7 +5,6 @@ import java.util.*; import fig.basic.*; import edu.stanford.nlp.sempre.*; -import fig.basic.LogInfo; /** * Static class for collaborative pruning. @@ -20,7 +19,7 @@ public class CollaborativePruner { public int maxPredictedPatterns = Integer.MAX_VALUE; @Option(gloss = "Maximum number of derivations per example") public int maxDerivations = 5000; - @Option(gloss = "Maximum number of exporation iterations") + @Option(gloss = "Maximum number of times to fall back to exploration") public int maxExplorationIters = Integer.MAX_VALUE; } @@ -37,9 +36,9 @@ public class CollaborativePruner { // Global variables // Nearest neighbors - static Map> neighbors; + static Map> uidToCachedNeighbors; // uid => pattern - static Map consistentPattern = new HashMap<>(); + static Map consistentPattern = new HashMap<>(); // patternString => customRuleString static Map> customRules = new HashMap<>(); // set of patternStrings @@ -47,7 +46,7 @@ public class CollaborativePruner { // Example-level variables public static boolean foundConsistentDerivation = false; - public static Map predictedPatterns; + public static Map predictedPatterns; public static List predictedRules; /** @@ -59,8 +58,8 @@ public class CollaborativePruner { LogInfo.logs("neighborFilePath is null."); return; } - LogInfo.logs("loading Neighbors " + opts.neighborFilePath); - Map> tmpMap = new HashMap<>(); + LogInfo.begin_track("Loading cached neighbors from %s", opts.neighborFilePath); + uidToCachedNeighbors = new HashMap<>(); try { BufferedReader reader = IOUtils.openIn(opts.neighborFilePath); String line; @@ -68,28 +67,13 @@ public class CollaborativePruner { String[] tokens = line.split("\t"); String uid = tokens[0]; String[] nids = tokens[1].split(","); - tmpMap.put(uid, Arrays.asList(nids)); + uidToCachedNeighbors.put(uid, Arrays.asList(nids)); } reader.close(); } catch (IOException e) { throw new RuntimeException(e); } - neighbors = tmpMap; - } - - public static > Map sortByValue(Map map) { - List> list = new LinkedList>(map.entrySet()); - Collections.sort(list, new Comparator>() { - public int compare(Map.Entry o1, Map.Entry o2) { - return (o1.getValue()).compareTo(o2.getValue()); - } - }); - - Map result = new LinkedHashMap(); - for (Map.Entry entry : list) { - result.put(entry.getKey(), entry.getValue()); - } - return result; + LogInfo.end_track(); } public static void initialize(Example ex, Mode mode) { @@ -103,39 +87,42 @@ public class CollaborativePruner { } static void preprocessExample(Example ex) { - Map patternFreqMap = new HashMap<>(); - List cachedNeighbors = neighbors.get(ex.id); + Map patternFreqMap = new HashMap<>(); + List cachedNeighbors = uidToCachedNeighbors.get(ex.id); int total = 0; // Gather the neighbors if (opts.maxNumNeighbors > 0) { for (String nid : cachedNeighbors) { - // Only get examples that have been previously processed + // Only get examples that have been previously processed + found a consistent formula if (!consistentPattern.containsKey(nid)) continue; String neighborPattern = consistentPattern.get(nid).pattern; if (!patternFreqMap.containsKey(neighborPattern)) - patternFreqMap.put(neighborPattern, new Pattern(neighborPattern, 0, 0)); - patternFreqMap.get(neighborPattern).frequency += 1; + patternFreqMap.put(neighborPattern, new FormulaPattern(neighborPattern, 0)); + patternFreqMap.get(neighborPattern).frequency++; total++; if (total >= opts.maxNumNeighbors) break; } } else { for (String patternString : allConsistentPatterns) { - patternFreqMap.put(patternString, new Pattern(patternString, 0, 1)); + patternFreqMap.put(patternString, new FormulaPattern(patternString, 1)); } } + // Sort by frequency (more frequent = smaller; see FormulaPattern.compareTo) + List> patternFreqEntries = new ArrayList<>(patternFreqMap.entrySet()); + patternFreqEntries.sort(new ValueComparator<>(false)); + // Gather the patterns - patternFreqMap = sortByValue(patternFreqMap); + LogInfo.begin_track("Predicted patterns"); int rank = 0; Set predictedRulesStrings = new HashSet<>(); predictedPatterns = new HashMap<>(); - LogInfo.begin_track("Predicted patterns"); - for (Map.Entry entry : patternFreqMap.entrySet()) { - Pattern newPattern = entry.getValue(); + for (Map.Entry entry : patternFreqEntries) { + FormulaPattern newPattern = entry.getValue(); predictedPatterns.put(newPattern.pattern, newPattern); predictedRulesStrings.addAll(customRules.get(newPattern.pattern)); LogInfo.logs((rank + 1) + ". " + newPattern.pattern + " (" + newPattern.frequency + ")"); @@ -143,7 +130,6 @@ public class CollaborativePruner { if (rank >= opts.maxPredictedPatterns) break; } - // Gather the rules predictedRules = customGrammar.getRules(predictedRulesStrings); LogInfo.end_track(); @@ -154,7 +140,7 @@ public class CollaborativePruner { || deriv.cat.equals("$LEMMA_TOKEN") || deriv.cat.equals("$LEMMA_PHRASE")) { return deriv.cat; } else { - return PatternInfo.convertToIndexedPattern(deriv); + return FormulaPattern.convertToIndexedPattern(deriv); } } @@ -176,33 +162,20 @@ public class CollaborativePruner { LogInfo.logs("Found consistent deriv: %s", deriv); String patternString = getPatternString(deriv); - Pattern newConsistentPattern = new Pattern(patternString, 0, 0); + FormulaPattern newConsistentPattern = new FormulaPattern(patternString, 0); newConsistentPattern.score = deriv.getScore(); - if (!consistentPattern.containsKey(uid)) { + FormulaPattern oldConsistentPattern = consistentPattern.get(uid); + if (oldConsistentPattern == null || newConsistentPattern.score > oldConsistentPattern.score) { addRules(patternString, deriv, ex); consistentPattern.put(uid, newConsistentPattern); allConsistentPatterns.add(patternString); - } else { - Pattern oldConsistentPattern = consistentPattern.get(uid); - if (newConsistentPattern.score > oldConsistentPattern.score) { - addRules(patternString, deriv, ex); - consistentPattern.put(uid, newConsistentPattern); - allConsistentPatterns.add(patternString); - } } } } - public static Pattern getConsistentPattern(Example ex) { + public static FormulaPattern getConsistentPattern(Example ex) { return consistentPattern.get(ex.id); } - public static Pattern getPattern(Example ex, Derivation deriv) { - if (mode == Mode.EXPLORE) - return null; - if (!deriv.isRootCat()) - return null; - return predictedPatterns.get(getPatternString(deriv)); - } } diff --git a/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java b/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java index d7b7697..12a4421 100644 --- a/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java +++ b/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java @@ -13,10 +13,11 @@ public class CustomGrammar extends Grammar { public static Options opts = new Options(); - public static final Set baseCategories = new HashSet(Arrays.asList("$Unary", "$Binary", "$Entity", "$Property")); + public static final Set baseCategories = new HashSet(Arrays.asList( + Rule.tokenCat, Rule.phraseCat, Rule.lemmaTokenCat, Rule.lemmaPhraseCat, + "$Unary", "$Binary", "$Entity", "$Property")); ArrayList baseRules = new ArrayList<>(); - // symbolicFormulas => symbolicFormula ID Map symbolicFormulas = new HashMap<>(); // indexedSymbolicFormula => customRuleString @@ -25,11 +26,6 @@ public class CustomGrammar extends Grammar { Map> customBinarizedRules = new HashMap<>(); public void init(Grammar initGrammar) { - baseCategories.add(Rule.tokenCat); - baseCategories.add(Rule.phraseCat); - baseCategories.add(Rule.lemmaTokenCat); - baseCategories.add(Rule.lemmaPhraseCat); - baseRules = new ArrayList<>(); for (Rule rule : initGrammar.getRules()) { if (baseCategories.contains(rule.lhs)) { @@ -116,8 +112,17 @@ public class CustomGrammar extends Grammar { // ============================================================ private static String safeReplace(String formula, String target, String replacement) { + // (argmin 1 1 ...) and (argmax 1 1 ...) are troublesome + String before = formula, targetBefore = target; + formula = formula.replace("(argmin (number 1) (number 1)", "(ARGMIN"); + formula = formula.replace("(argmax (number 1) (number 1)", "(ARGMAX"); + target = target.replace("(argmin (number 1) (number 1)", "(ARGMIN"); + target = target.replace("(argmax (number 1) (number 1)", "(ARGMAX"); formula = formula.replace(target + ")", replacement + ")"); formula = formula.replace(target + " ", replacement + " "); + formula = formula.replace("(ARGMIN", "(argmin (number 1) (number 1)"); + formula = formula.replace("(ARGMAX", "(argmax (number 1) (number 1)"); + LogInfo.logs("REPLACE: [%s | %s] %s | %s", targetBefore, replacement, before, formula); return formula; } diff --git a/src/edu/stanford/nlp/sempre/cprune/FormulaPattern.java b/src/edu/stanford/nlp/sempre/cprune/FormulaPattern.java new file mode 100644 index 0000000..0ae2735 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cprune/FormulaPattern.java @@ -0,0 +1,102 @@ +package edu.stanford.nlp.sempre.cprune; + +import java.util.regex.Pattern; + +import edu.stanford.nlp.sempre.Derivation; +import fig.basic.LogInfo; + +public class FormulaPattern implements Comparable { + public String pattern; + public Integer frequency; + public Double score; + + public FormulaPattern(String pattern, Integer frequency) { + this.pattern = pattern; + this.frequency = frequency; + } + + public Double complexity() { + return (double) (pattern.length() - pattern.replace("(@R", "***").replace("(", "").length()); + } + + @Override + public String toString() { + return "(" + pattern + ", " + frequency + ")"; + } + + @Override + public int compareTo(FormulaPattern that) { + if (this.frequency > that.frequency) { + return -1; + } else if (this.frequency < that.frequency) { + return 1; + } else { + return this.complexity().compareTo(that.complexity()); + } + } + + // ============================================================ + // Utilities + // ============================================================ + + private static Pattern reverseRelation = Pattern.compile("!(fb:[._a-z0-9]+)"); + private static Pattern varName = Pattern.compile("\\((lambda|var) [a-z0-9]+"); + private static Pattern compare = Pattern.compile("(<=|>=|>|<)"); + + public static String convertToIndexedPattern(Derivation deriv) { + String formula = deriv.formula.toString(); + + // These can interfere with (number 1) + formula = formula.replace("argmax (number 1) (number 1)", "argmax"); + formula = formula.replace("argmin (number 1) (number 1)", "argmin"); + + formula = removePropertyPredicates(formula); + formula = CustomGrammar.getIndexedSymbolicFormula(deriv, formula); + + formula = formula.replace("fb:type.object.type fb:type.row", "@type @row"); + formula = reverseRelation.matcher(formula).replaceAll("(reverse $1)"); + formula = formula.replace("fb:row.row.index", "(reverse (lambda x ((reverse @index) (var x))))"); + formula = formula.replace("fb:row.row.next", "@next"); + formula = varName.matcher(formula).replaceAll("($1 x"); + formula = formula.replaceAll("\\s+", " "); + formula = formula.replace("reverse", "@R"); + formula = compare.matcher(formula).replaceAll("@compare"); + LogInfo.logs("PATTERN: %s -> %s", deriv.formula, formula); + return formula; + } + + private static Pattern cellProperty = Pattern.compile("!?fb:cell\\.cell\\.[_a-z0-9]+|\\(reverse fb:cell\\.cell\\.[_a-z0-9]+\\)"); + + /** + * Remove cell property relations (fb:cell.cell.*) + */ + public static String removePropertyPredicates(String formula) { + formula = cellProperty.matcher(formula).replaceAll("@PPT"); + while (formula.contains("@PPT")) { + int begin = formula.indexOf("(@PPT"); + if (begin == -1) { + formula = formula.replace("@PPT", ""); + break; + } + // Find the matching parenthesis + int count = 1; + for (int i = begin + 1; i < formula.length(); i++) { + if (formula.charAt(i) == '(') { + count++; + } else if (formula.charAt(i) == ')') { + count--; + if (count == 0) { + int end = i; + formula = formula.substring(0, begin) + formula.substring(begin + 6, end) + formula.substring(end + 1, formula.length()); + break; + } + } + if (i == formula.length() - 1) { + LogInfo.fails("Unbalanced parentheses: %s", formula); + } + } + } + return formula; + } + +} diff --git a/src/edu/stanford/nlp/sempre/cprune/Pattern.java b/src/edu/stanford/nlp/sempre/cprune/Pattern.java deleted file mode 100644 index 485573d..0000000 --- a/src/edu/stanford/nlp/sempre/cprune/Pattern.java +++ /dev/null @@ -1,34 +0,0 @@ -package edu.stanford.nlp.sempre.cprune; - -public class Pattern implements Comparable { - public String pattern; - public Integer rank; - public Integer frequency; - public Double score; - - public Pattern(String pattern, Integer rank, Integer frequency) { - this.pattern = pattern; - this.rank = rank; - this.frequency = frequency; - } - - public Double complexity() { - return (double) (pattern.length() - pattern.replace("(@R", "***").replace("(", "").length()); - } - - @Override - public String toString() { - return "(" + pattern + ", " + rank + ", " + frequency + ")"; - } - - @Override - public int compareTo(Pattern that) { - if (this.frequency > that.frequency) { - return -1; - } else if (this.frequency < that.frequency) { - return 1; - } else { - return this.complexity().compareTo(that.complexity()); - } - } -} diff --git a/src/edu/stanford/nlp/sempre/cprune/PatternInfo.java b/src/edu/stanford/nlp/sempre/cprune/PatternInfo.java deleted file mode 100644 index b9d8377..0000000 --- a/src/edu/stanford/nlp/sempre/cprune/PatternInfo.java +++ /dev/null @@ -1,92 +0,0 @@ -package edu.stanford.nlp.sempre.cprune; - -import java.util.*; -import edu.stanford.nlp.sempre.*; - -public class PatternInfo { - Set baseCategories = new HashSet(Arrays.asList("$Unary", "$Binary", "$Entity", "$Property")); - - public static String removePropertyPredicates(String formula) { - formula = RegexReplaceManager.replace(formula, "fb:cell\\.cell\\.[_0-9a-z]+", "@PPT"); - formula = formula.replace("(reverse @PPT)", "@PPT"); - - while (formula.contains("@PPT")) { - int begin = formula.indexOf("(@PPT"); - if (begin == -1) { - formula = formula.replace("@PPT", ""); - ; - break; - } - int count = 1; - for (int i = begin + 1; i < formula.length(); i++) { - if (formula.charAt(i) == '(') { - count += 1; - } else if (formula.charAt(i) == ')') { - count -= 1; - } - if (count == 0) { - int end = i; - formula = formula.substring(0, begin) + formula.substring(begin + 6, end) + formula.substring(end + 1, formula.length()); - break; - } - if (i == formula.length() - 1) { - System.out.println(formula); - } - } - } - return formula; - } - - public static String convertToPattern(Derivation deriv) { - String formula = deriv.formula.toString(); - - // These can interfere with (number 1) - formula = formula.replace("argmax (number 1) (number 1)", "argmax"); - formula = formula.replace("argmin (number 1) (number 1)", "argmin"); - - formula = removePropertyPredicates(formula); - - formula = formula.replace("fb:type.object.type fb:type.row", "@type @row"); - - formula = RegexReplaceManager.replace(formula, "!fb:[._0-9a-z]+", "(reverse $0)").replace("reverse !fb", "reverse fb"); - formula = formula.replace("fb:row.row.index", "(reverse (lambda x ((reverse @index) (var x))))"); - formula = formula.replace("fb:row.row.next", "@next"); - formula = RegexReplaceManager.replace(formula, "\\(lambda [a-z]", "\\(lambda x"); - formula = RegexReplaceManager.replace(formula, "\\(var [a-z]\\)", "\\(var x\\)"); - - formula = RegexReplaceManager.replace(formula, "fb:row\\.row\\.[_0-9a-z]+", "\\$Binary"); - formula = RegexReplaceManager.replace(formula, "fb:cell\\.cell\\.[_0-9a-z]+", "\\$Property"); - formula = RegexReplaceManager.replace(formula, "fb:cell_[_0-9a-z]+\\.[_0-9a-z]+", "\\$Entity"); - formula = RegexReplaceManager.replace(formula, "\\(number [0-9]+[.]?[0-9]+\\)", "\\$Entity"); - formula = RegexReplaceManager.replace(formula, "\\(number [0-9]+\\)", "\\$Entity"); - formula = RegexReplaceManager.replace(formula, "\\(date [-]?[0-9]+ [-]?[0-9]+ [-]?[0-9]+\\)", "\\$Entity"); - - formula = RegexReplaceManager.replace(formula, "[ ]+", " "); - formula = formula.replace("reverse", "@R"); - formula = RegexReplaceManager.replace(formula, "(<=|>=|>|<)", "@compare"); - return formula; - } - - public static String convertToIndexedPattern(Derivation deriv) { - String formula = deriv.formula.toString(); - - // These can interfere with (number 1) - formula = formula.replace("argmax (number 1) (number 1)", "argmax"); - formula = formula.replace("argmin (number 1) (number 1)", "argmin"); - - formula = removePropertyPredicates(formula); - formula = CustomGrammar.getIndexedSymbolicFormula(deriv, formula); - - formula = formula.replace("fb:type.object.type fb:type.row", "@type @row"); - - formula = RegexReplaceManager.replace(formula, "!fb:[._0-9a-z]+", "(reverse $0)").replace("reverse !fb", "reverse fb"); - formula = formula.replace("fb:row.row.index", "(reverse (lambda x ((reverse @index) (var x))))"); - formula = formula.replace("fb:row.row.next", "@next"); - formula = RegexReplaceManager.replace(formula, "\\(lambda [a-z]", "\\(lambda x"); - formula = RegexReplaceManager.replace(formula, "\\(var [a-z]\\)", "\\(var x\\)"); - formula = RegexReplaceManager.replace(formula, "[ ]+", " "); - formula = formula.replace("reverse", "@R"); - formula = RegexReplaceManager.replace(formula, "(<=|>=|>|<)", "@compare"); - return formula; - } -} diff --git a/src/edu/stanford/nlp/sempre/cprune/RegexReplaceManager.java b/src/edu/stanford/nlp/sempre/cprune/RegexReplaceManager.java deleted file mode 100644 index c6493dd..0000000 --- a/src/edu/stanford/nlp/sempre/cprune/RegexReplaceManager.java +++ /dev/null @@ -1,15 +0,0 @@ -package edu.stanford.nlp.sempre.cprune; - -import java.util.*; - -public class RegexReplaceManager { - static Map dict = new HashMap<>(); - - public static String replace(String source, String regex, String replacement) { - if (!dict.containsKey(regex)) { - dict.put(regex, java.util.regex.Pattern.compile(regex)); - } - java.util.regex.Pattern regexPattern = dict.get(regex); - return regexPattern.matcher(source).replaceAll(replacement); - } -} diff --git a/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutor.java b/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutor.java index 39f85d7..0d468b2 100644 --- a/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutor.java +++ b/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutor.java @@ -292,6 +292,9 @@ class LambdaDCSCoreLogic { computeUnary(superlative.rank, typeHint.unrestrictedUnary()).range()); int count = DenotationUtils.getSinglePositiveInteger( computeUnary(superlative.count, typeHint.unrestrictedUnary()).range()); + if (rank != 1 || count != 1) { + LogInfo.logs("Superlative WTF: %s | rank %d | count %d", formula, rank, count); + } Unarylike headD = computeUnary(superlative.head, typeHint); Binarylike relationD; if (superlative.relation instanceof ReverseFormula) { From 212e28fc770ad7061921dc7d14434c65253faacb Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Thu, 31 Aug 2017 00:06:31 -0700 Subject: [PATCH 22/30] Added a few more docstrings --- src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java | 4 ++++ src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java | 6 ++++++ src/edu/stanford/nlp/sempre/cprune/Symbol.java | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java b/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java index df95aee..8361aab 100644 --- a/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java +++ b/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java @@ -152,6 +152,10 @@ public class CollaborativePruner { customRules.get(patternString).addAll(parsedCustomRules); } + /** + * Get called when a (consistent) formula is found. + * Update the consistent patterns. + */ public static void updateConsistentPattern(ValueEvaluator evaluator, Example ex, Derivation deriv) { String uid = ex.id; if (ex.targetValue != null) diff --git a/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java b/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java index 12a4421..536b392 100644 --- a/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java +++ b/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java @@ -91,6 +91,9 @@ public class CustomGrammar extends Grammar { return getIndexedSymbolicFormula(deriv, deriv.formula.toString()); } + /** + * Replace symbols (e.g., fb:row.row.name) with placeholders (e.g., Binary#1). + */ public static String getIndexedSymbolicFormula(Derivation deriv, String formula) { CPruneDerivInfo derivInfo = aggregateSymbols(deriv); int index = 1; @@ -126,6 +129,9 @@ public class CustomGrammar extends Grammar { return formula; } + /** + * Cache the symbols in deriv.tempState[cprune].treeSymbols + */ private static CPruneDerivInfo aggregateSymbols(Derivation deriv) { Map tempState = deriv.getTempState(); if (tempState.containsKey("cprune")) { diff --git a/src/edu/stanford/nlp/sempre/cprune/Symbol.java b/src/edu/stanford/nlp/sempre/cprune/Symbol.java index 11634be..75d8537 100644 --- a/src/edu/stanford/nlp/sempre/cprune/Symbol.java +++ b/src/edu/stanford/nlp/sempre/cprune/Symbol.java @@ -1,5 +1,10 @@ package edu.stanford.nlp.sempre.cprune; +/** + * Represents the leaf node of the parse tree. + * + * Any sub-derivation whose category is in CustomGrammar.baseCategories becomes a Symbol. + */ public class Symbol implements Comparable { String category; String formula; From 3e7c05262182d7ee572070e08b1c642238569015 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Thu, 31 Aug 2017 03:41:19 -0700 Subject: [PATCH 23/30] Fixed bug in feature --- .../nlp/sempre/cprune/CPruneFloatingParser.java | 2 +- .../nlp/sempre/tables/TableKnowledgeGraph.java | 15 ++++++++++++--- .../tables/features/AnchorFeatureComputer.java | 3 +-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java b/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java index 107615f..5535dcc 100644 --- a/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java +++ b/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java @@ -106,7 +106,7 @@ class MiniGrammar extends Grammar { public MiniGrammar(List rules) { this.rules.addAll(rules); LogInfo.begin_track("MiniGrammar Rules"); - for (Rule rule : rules) LogInfo.logs("%s", rule); + for (Rule rule : rules) LogInfo.logs("%s %s", rule, rule.isAnchored() ? "[A]" : "[F]"); LogInfo.end_track(); } diff --git a/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java b/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java index eca3bc3..9300eb3 100644 --- a/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java +++ b/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java @@ -65,6 +65,8 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl Map relationIdToTableColumn; // "fb:cell.palo_alto_ca" --> TableCellProperties object Map cellIdToTableCellProperties; + // "fb:part.palo_alto" --> String + Map partIdToOriginalString; FuzzyMatcher fuzzyMatcher; public ExecutorCache executorCache; @@ -143,9 +145,14 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl // Collect cell properties for public access cellProperties = new HashSet<>(cellIdToTableCellProperties.values()); cellParts = new HashSet<>(); - for (TableCellProperties properties : cellProperties) - for (Value part : properties.metadata.get(TableTypeSystem.CELL_PART_VALUE)) - cellParts.add((NameValue) part); + partIdToOriginalString = new HashMap<>(); + for (TableCellProperties properties : cellProperties) { + for (Value part : properties.metadata.get(TableTypeSystem.CELL_PART_VALUE)) { + NameValue partNameValue = (NameValue) part; + cellParts.add(partNameValue); + partIdToOriginalString.put(partNameValue.id, partNameValue.description); + } + } // Precompute normalized strings for fuzzy matching fuzzyMatcher = FuzzyMatcher.getFuzzyMatcher(this); executorCache = opts.individualExecutorCache ? new ExecutorCache() : null; @@ -615,6 +622,8 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl if (nameValueId.startsWith("!")) nameValueId = nameValueId.substring(1); if (cellIdToTableCellProperties.containsKey(nameValueId)) return cellIdToTableCellProperties.get(nameValueId).originalString; + if (partIdToOriginalString.containsKey(nameValueId)) + return partIdToOriginalString.get(nameValueId); if (relationIdToTableColumn.containsKey(nameValueId)) return relationIdToTableColumn.get(nameValueId).originalString; if (nameValueId.startsWith(TableTypeSystem.CELL_SPECIFIC_TYPE_PREFIX)) { diff --git a/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java b/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java index a3dd20c..f0a44e9 100644 --- a/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java +++ b/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java @@ -25,9 +25,8 @@ public class AnchorFeatureComputer implements FeatureComputer { private void extractMatchingFeatures(TableKnowledgeGraph graph, Derivation deriv, String phrase, NameValue predicate) { - LogInfo.logs("%s -> %s", phrase, predicate); - // Type of match String predicateString = graph.getOriginalString(predicate); + LogInfo.logs("%s -> %s = %s", phrase, predicate, predicateString); predicateString = StringNormalizationUtils.simpleNormalize(predicateString).toLowerCase(); if (predicateString.equals(phrase)) { deriv.addFeature("a-e", "exact"); From 4c1bcffebd3a0c23f51d307a081a661f1f3ca4b9 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Thu, 31 Aug 2017 16:13:34 -0700 Subject: [PATCH 24/30] Also log the number of correct and incorrect derivations --- src/edu/stanford/nlp/sempre/Parser.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/edu/stanford/nlp/sempre/Parser.java b/src/edu/stanford/nlp/sempre/Parser.java index 9afa080..00fa76b 100644 --- a/src/edu/stanford/nlp/sempre/Parser.java +++ b/src/edu/stanford/nlp/sempre/Parser.java @@ -210,6 +210,7 @@ public abstract class Parser { int correctIndexAfterParse = -1; double maxCompatibility = 0.0; double[] compatibilities = null; + int numCorrect = 0, numPartialCorrect = 0, numIncorrect = 0; if (ex.targetValue != null) { compatibilities = new double[numCandidates]; @@ -221,6 +222,14 @@ public abstract class Parser { correctIndex = i; // record maximum compatibility for partial oracle maxCompatibility = Math.max(compatibilities[i], maxCompatibility); + // Count + if (compatibilities[i] == 1) { + numCorrect++; + } else if (compatibilities[i] == 0) { + numIncorrect++; + } else { + numPartialCorrect++; + } } // What if we only had parsed bottom up? for (int i = 0; i < numCandidates; i++) { @@ -334,6 +343,9 @@ public abstract class Parser { evaluation.add("numCandidates", numCandidates); // From this parse if (numCandidates > 0) evaluation.add("parsedNumCandidates", numCandidates); + evaluation.add("numCorrect", numCorrect); + evaluation.add("numPartialCorrect", numPartialCorrect); + evaluation.add("numIncorrect", numIncorrect); // Add parsing stats evaluation.add(state.evaluation); From 3ef45877f33c32f914723209569bc5dba44ebfd1 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Fri, 1 Sep 2017 17:25:41 -0700 Subject: [PATCH 25/30] Make cprune less verbose --- .../nlp/sempre/cprune/CPruneFloatingParser.java | 12 +++++++++--- .../nlp/sempre/cprune/CollaborativePruner.java | 2 ++ .../stanford/nlp/sempre/cprune/CustomGrammar.java | 5 +++-- .../stanford/nlp/sempre/cprune/FormulaPattern.java | 8 ++++++-- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java b/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java index 5535dcc..2b6ef29 100644 --- a/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java +++ b/src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java @@ -5,6 +5,9 @@ import java.util.List; import edu.stanford.nlp.sempre.*; import fig.basic.LogInfo; +/** + * A parser that first tries to exploit the macro grammar and only fall back to full search when needed. + */ public class CPruneFloatingParser extends FloatingParser { FloatingParser exploreParser; @@ -105,9 +108,12 @@ class MiniGrammar extends Grammar { public MiniGrammar(List rules) { this.rules.addAll(rules); - LogInfo.begin_track("MiniGrammar Rules"); - for (Rule rule : rules) LogInfo.logs("%s %s", rule, rule.isAnchored() ? "[A]" : "[F]"); - LogInfo.end_track(); + if (CollaborativePruner.opts.verbose >= 2) { + LogInfo.begin_track("MiniGrammar Rules"); + for (Rule rule : rules) + LogInfo.logs("%s %s", rule, rule.isAnchored() ? "[A]" : "[F]"); + LogInfo.end_track(); + } } } diff --git a/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java b/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java index 8361aab..c65a1c9 100644 --- a/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java +++ b/src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java @@ -11,6 +11,8 @@ import edu.stanford.nlp.sempre.*; */ public class CollaborativePruner { public static class Options { + @Option(gloss = "Logging verbosity") + public int verbose = 0; @Option(gloss = "K = Maximum number of nearest-neighbor examples to consider (-1 to use all examples so far)") public int maxNumNeighbors = -1; @Option(gloss = "Load cached neighbors from this file") diff --git a/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java b/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java index 536b392..1dbca83 100644 --- a/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java +++ b/src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java @@ -7,7 +7,7 @@ import fig.basic.*; public class CustomGrammar extends Grammar { public static class Options { - @Option + @Option(gloss = "Whether to decompose the templates into multiple rules") public boolean enableTemplateDecomposition = true; } @@ -125,7 +125,8 @@ public class CustomGrammar extends Grammar { formula = formula.replace(target + " ", replacement + " "); formula = formula.replace("(ARGMIN", "(argmin (number 1) (number 1)"); formula = formula.replace("(ARGMAX", "(argmax (number 1) (number 1)"); - LogInfo.logs("REPLACE: [%s | %s] %s | %s", targetBefore, replacement, before, formula); + if (CollaborativePruner.opts.verbose >= 2) + LogInfo.logs("REPLACE: [%s | %s] %s | %s", targetBefore, replacement, before, formula); return formula; } diff --git a/src/edu/stanford/nlp/sempre/cprune/FormulaPattern.java b/src/edu/stanford/nlp/sempre/cprune/FormulaPattern.java index 0ae2735..0c83d04 100644 --- a/src/edu/stanford/nlp/sempre/cprune/FormulaPattern.java +++ b/src/edu/stanford/nlp/sempre/cprune/FormulaPattern.java @@ -16,6 +16,7 @@ public class FormulaPattern implements Comparable { } public Double complexity() { + // Roughly the number of predicates return (double) (pattern.length() - pattern.replace("(@R", "***").replace("(", "").length()); } @@ -42,6 +43,7 @@ public class FormulaPattern implements Comparable { private static Pattern reverseRelation = Pattern.compile("!(fb:[._a-z0-9]+)"); private static Pattern varName = Pattern.compile("\\((lambda|var) [a-z0-9]+"); private static Pattern compare = Pattern.compile("(<=|>=|>|<)"); + private static Pattern whitespace = Pattern.compile("\\s+"); public static String convertToIndexedPattern(Derivation deriv) { String formula = deriv.formula.toString(); @@ -58,10 +60,12 @@ public class FormulaPattern implements Comparable { formula = formula.replace("fb:row.row.index", "(reverse (lambda x ((reverse @index) (var x))))"); formula = formula.replace("fb:row.row.next", "@next"); formula = varName.matcher(formula).replaceAll("($1 x"); - formula = formula.replaceAll("\\s+", " "); formula = formula.replace("reverse", "@R"); formula = compare.matcher(formula).replaceAll("@compare"); - LogInfo.logs("PATTERN: %s -> %s", deriv.formula, formula); + formula = whitespace.matcher(formula).replaceAll(" "); + + if (CollaborativePruner.opts.verbose >= 2) + LogInfo.logs("PATTERN: %s -> %s", deriv.formula, formula); return formula; } From 1dd4a4ef358763e2876170e293504cc4a6a1d0f8 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Fri, 1 Sep 2017 18:29:34 -0700 Subject: [PATCH 26/30] Fixed the floating parser to allow canonical utterance generation --- run | 7 +- .../stanford/nlp/sempre/FloatingParser.java | 64 +++++++++++-------- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/run b/run index 2362606..c940211 100755 --- a/run +++ b/run @@ -940,10 +940,8 @@ end ############################################################ # {2015-01-18} Generate utterances [Percy] addMode('genovernight', 'Generate utterances for overnight semantic parsing', lambda { |e| l( - 'fig/bin/qcreate', - letDefault(:gen, 0), - sel(:gen, l()), - 'java', '-Dmodules=core,overnight', '-Xmx10g', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.overnight.GenerationMain', + header('core,overnight'), + 'edu.stanford.nlp.sempre.overnight.GenerationMain', figOpts, o('JoinFn.typeInference', true), o('JoinFn.specializedTypeCheck', false), @@ -959,6 +957,7 @@ addMode('genovernight', 'Generate utterances for overnight semantic parsing', la o('FeatureExtractor.featureComputers','overnight.OvernightFeatureComputer'), o('OvernightFeatureComputer.featureDomains', ''), o('OvernightFeatureComputer.itemAnalysis',false), + letDefault(:gen, 0), sel(:gen, l( # For debugging the grammar o('FeatureExtractor.featureDomains', 'denotation'), diff --git a/src/edu/stanford/nlp/sempre/FloatingParser.java b/src/edu/stanford/nlp/sempre/FloatingParser.java index 4f433f2..087392c 100644 --- a/src/edu/stanford/nlp/sempre/FloatingParser.java +++ b/src/edu/stanford/nlp/sempre/FloatingParser.java @@ -379,39 +379,53 @@ class FloatingParserState extends ParserState { StopWatch stopWatch = new StopWatch().start(); String rhs1 = rule.rhs.get(0); String rhs2 = rule.rhs.get(1); - if (!Rule.isCat(rhs1) || !Rule.isCat(rhs2)) - throw new RuntimeException("Floating rules with > 1 arguments cannot have tokens on the RHS: " + rule); - if (FloatingParser.opts.useSizeInsteadOfDepth) { - derivLoop: - for (int depth1 = 0; depth1 < depth; depth1++) { // sizes must add up to depth-1 (actually size-1) - int depth2 = depth - 1 - depth1; - for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, depth1), floatingCell(rhs2, depth2))) - for (Derivation deriv1 : group.derivations1) - for (Derivation deriv2 : group.derivations2) - if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance)) - break derivLoop; - } - } else { - { - derivLoop: - for (int subDepth = 0; subDepth < depth; subDepth++) { // depth-1 <=depth-1 - for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, depth - 1), floatingCell(rhs2, subDepth))) - for (Derivation deriv1 : group.derivations1) - for (Derivation deriv2 : group.derivations2) - if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance)) - break derivLoop; - } + if (!Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // token token + if (depth == (FloatingParser.opts.initialFloatingHasZeroDepth ? 0 : 1)) { + applyFloatingRule(rule, depth, null, null, rhs1 + " " + rhs2); } - { + } else if (!Rule.isCat(rhs1) && Rule.isCat(rhs2)) { // token $Cat + List derivations = getDerivations(floatingCell(rhs2, depth - 1)); + for (Derivation deriv : derivations) + applyFloatingRule(rule, depth, deriv, null, rhs1 + " " + deriv.canonicalUtterance); + + } else if (Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // $Cat token + List derivations = getDerivations(floatingCell(rhs1, depth - 1)); + for (Derivation deriv : derivations) + applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance + " " + rhs2); + + } else { // $Cat $Cat + if (FloatingParser.opts.useSizeInsteadOfDepth) { derivLoop: - for (int subDepth = 0; subDepth < depth - 1; subDepth++) { // Date: Mon, 4 Sep 2017 15:09:23 -0700 Subject: [PATCH 27/30] Fixed the normalization stuff --- .../sempre/tables/StringNormalizationUtils.java | 11 ++++++++--- .../nlp/sempre/tables/TableValueEvaluator.java | 16 +++++++++------- .../tables/features/AnchorFeatureComputer.java | 14 +++++++------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java index 7607bd7..fbb6eb7 100644 --- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java @@ -317,7 +317,7 @@ public final class StringNormalizationUtils { .replaceAll("[‘’´`]", "'") .replaceAll("[“”«»]", "\"") .replaceAll("[•†‡]", "") - .replaceAll("[‐‑–—]", "-"); + .replaceAll("[-‐‑–—]", "-"); return string.replaceAll("\\s+", " ").trim(); } @@ -340,12 +340,17 @@ public final class StringNormalizationUtils { public static String aggressiveNormalize(String string) { // Dashed / Parenthesized information string = simpleNormalize(string); - string = string.replaceAll("\\[[^\\]]*\\]", ""); String oldString; do { oldString = string; - string = string.trim().replaceAll("\\([^)]*\\)$", ""); + // Remove citations + string = string.trim().replaceAll("((? %s = %s", phrase, predicate, predicateString); + //LogInfo.logs("%s -> %s = %s", phrase, predicate, predicateString); predicateString = StringNormalizationUtils.simpleNormalize(predicateString).toLowerCase(); if (predicateString.equals(phrase)) { deriv.addFeature("a-e", "exact"); - LogInfo.logs("%s %s exact", phrase, predicateString); + //LogInfo.logs("%s %s exact", phrase, predicateString); } else if (predicateString.startsWith(phrase + " ")) { deriv.addFeature("a-e", "prefix"); - LogInfo.logs("%s %s prefix", phrase, predicateString); + //LogInfo.logs("%s %s prefix", phrase, predicateString); } else if (predicateString.endsWith(" " + phrase)) { deriv.addFeature("a-e", "suffix"); - LogInfo.logs("%s %s suffix", phrase, predicateString); + //LogInfo.logs("%s %s suffix", phrase, predicateString); } else if (predicateString.contains(" " + phrase + " ")){ deriv.addFeature("a-e", "substring"); - LogInfo.logs("%s %s substring", phrase, predicateString); + //LogInfo.logs("%s %s substring", phrase, predicateString); } else { deriv.addFeature("a-e", "other"); - LogInfo.logs("%s %s other", phrase, predicateString); + //LogInfo.logs("%s %s other", phrase, predicateString); } // Does the phrase match other cells? Set matches = new HashSet<>(); @@ -54,7 +54,7 @@ public class AnchorFeatureComputer implements FeatureComputer { } } } - LogInfo.logs(">> %s", matches); + //LogInfo.logs(">> %s", matches); if (matches.size() == 0) { deriv.addFeature("a-e", "unique"); } else if (matches.size() < 3) { From f9807e8fec5dd6c4fe296427291098808d7ddc99 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Mon, 4 Sep 2017 15:39:42 -0700 Subject: [PATCH 28/30] Added the run command to README --- .../nlp/sempre/tables/features/PhraseInfo.java | 2 +- tables/README.md | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/edu/stanford/nlp/sempre/tables/features/PhraseInfo.java b/src/edu/stanford/nlp/sempre/tables/features/PhraseInfo.java index 4c1c82c..86f06fc 100644 --- a/src/edu/stanford/nlp/sempre/tables/features/PhraseInfo.java +++ b/src/edu/stanford/nlp/sempre/tables/features/PhraseInfo.java @@ -25,7 +25,7 @@ public class PhraseInfo { @Option(gloss = "Fuzzy match predicates") public boolean computeFuzzyMatchPredicates = false; @Option(gloss = "Do not produce lexicalized features if the phrase begins or ends with a stop word") - public boolean forbidBorderStopWordInLexicalizedFeatures = false; + public boolean forbidBorderStopWordInLexicalizedFeatures = true; } public static Options opts = new Options(); diff --git a/tables/README.md b/tables/README.md index 700944a..107e48d 100644 --- a/tables/README.md +++ b/tables/README.md @@ -41,6 +41,22 @@ Running the code Other available sets include `u-2`, ..., `u-5` (four other development splits) and `test` (actual train-test split). +Other options +------------- + +### Macro Grammar (Experimental) + +Macro grammar can be used to significantly speed up the parser. +To turn on macro grammar, run the following: + + ./run @mode=tables @data=u-1 @feat=more @parser=cprune @grammar=extended @fuzzy=editdist-fuzzy @train=1 + +Please refer to the following paper for more information: + +> Yuchen Zhang, Panupong Pasupat, Percy Liang. +> Macro Grammars and Holistic Triggering for Efficient Semantic Parsing +> Empirical Methods on Natural Language Processing (EMNLP), 2017. + Official evaluation ------------------- From 6f704027f99cdd6b397fc459d1fed8ff936b0c38 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Mon, 4 Sep 2017 16:18:23 -0700 Subject: [PATCH 29/30] Handle floating rules with non-cat RHS --- run | 7 +- .../stanford/nlp/sempre/FloatingParser.java | 65 ++++++++++++------- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/run b/run index 18e944e..e75ac09 100755 --- a/run +++ b/run @@ -967,10 +967,8 @@ end ############################################################ # {2015-01-18} Generate utterances [Percy] addMode('genovernight', 'Generate utterances for overnight semantic parsing', lambda { |e| l( - 'fig/bin/qcreate', - letDefault(:gen, 0), - sel(:gen, l()), - 'java', '-Dmodules=core,overnight', '-Xmx10g', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.overnight.GenerationMain', + header('core,overnight'), + 'edu.stanford.nlp.sempre.overnight.GenerationMain', figOpts, o('JoinFn.typeInference', true), o('JoinFn.specializedTypeCheck', false), @@ -986,6 +984,7 @@ addMode('genovernight', 'Generate utterances for overnight semantic parsing', la o('FeatureExtractor.featureComputers','overnight.OvernightFeatureComputer'), o('OvernightFeatureComputer.featureDomains', ''), o('OvernightFeatureComputer.itemAnalysis',false), + letDefault(:gen, 1), sel(:gen, l( # For debugging the grammar o('FeatureExtractor.featureDomains', 'denotation'), diff --git a/src/edu/stanford/nlp/sempre/FloatingParser.java b/src/edu/stanford/nlp/sempre/FloatingParser.java index 721ff88..6f7e5eb 100644 --- a/src/edu/stanford/nlp/sempre/FloatingParser.java +++ b/src/edu/stanford/nlp/sempre/FloatingParser.java @@ -395,39 +395,54 @@ class FloatingParserState extends ParserState { StopWatch stopWatch = new StopWatch().start(); String rhs1 = rule.rhs.get(0); String rhs2 = rule.rhs.get(1); - if (!Rule.isCat(rhs1) || !Rule.isCat(rhs2)) - throw new RuntimeException("Floating rules with > 1 arguments cannot have tokens on the RHS: " + rule); - if (FloatingParser.opts.useSizeInsteadOfDepth) { - derivLoop: - for (int depth1 = 0; depth1 < depth; depth1++) { // sizes must add up to depth-1 (actually size-1) - int depth2 = depth - 1 - depth1; - for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, depth1), floatingCell(rhs2, depth2))) - for (Derivation deriv1 : group.derivations1) - for (Derivation deriv2 : group.derivations2) - if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance)) - break derivLoop; - } - } else { - { - derivLoop: - for (int subDepth = 0; subDepth < depth; subDepth++) { // depth-1 <=depth-1 - for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, depth - 1), floatingCell(rhs2, subDepth))) - for (Derivation deriv1 : group.derivations1) - for (Derivation deriv2 : group.derivations2) - if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance)) - break derivLoop; - } + if (!Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // token token + if (depth == (FloatingParser.opts.initialFloatingHasZeroDepth ? 0 : 1)) { + applyFloatingRule(rule, depth, null, null, rhs1 + " " + rhs2); } - { + + } else if (!Rule.isCat(rhs1) && Rule.isCat(rhs2)) { // token $Cat + List derivations = getDerivations(floatingCell(rhs2, depth - 1)); + for (Derivation deriv : derivations) + applyFloatingRule(rule, depth, deriv, null, rhs1 + " " + deriv.canonicalUtterance); + + } else if (Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // $Cat token + List derivations = getDerivations(floatingCell(rhs1, depth - 1)); + for (Derivation deriv : derivations) + applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance + " " + rhs2); + + } else { // $Cat $Cat + if (FloatingParser.opts.useSizeInsteadOfDepth) { derivLoop: - for (int subDepth = 0; subDepth < depth - 1; subDepth++) { // Date: Wed, 1 Nov 2017 14:23:28 -0700 Subject: [PATCH 30/30] Updated README --- README.md | 8 +++++++- tables/README.md | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 35b4002..64dba68 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# SEMPRE 2.3.1: Semantic Parsing with Execution +# SEMPRE 2.4: Semantic Parsing with Execution ## What is semantic parsing? @@ -62,6 +62,8 @@ SEMPRE has been used in the following papers: offshoot, and does not use many of the core learning and parsing utiltiies in SEMPRE. To reproduce those results, check out SEMPRE 1.0. +Please refer to the [project page](https://nlp.stanford.edu/software/sempre/) for a more complete list. + ## Where do I go next? - If you're new to semantic parsing, you can learn more from the [background @@ -180,3 +182,7 @@ Changes from SEMPRE 2.2 to SEMPRE 2.3: Changes from SEMPRE 2.3 to SEMPRE 2.3.1: - Modified the `tables` module to resemble SEMPRE 2.1, effectively making it work again. + +Changes from SEMPRE 2.3.1 to SEMPRE 2.4: + +- Added the `cprune` package for the paper *Macro Grammars and Holistic Triggering for Efficient Semantic Parsing* (EMNLP 2017). diff --git a/tables/README.md b/tables/README.md index 107e48d..cc1c98e 100644 --- a/tables/README.md +++ b/tables/README.md @@ -57,6 +57,9 @@ Please refer to the following paper for more information: > Macro Grammars and Holistic Triggering for Efficient Semantic Parsing > Empirical Methods on Natural Language Processing (EMNLP), 2017. +Currently the module does not support model saving, and testing has to be done on the official test set. +These features will be added in the future. + Official evaluation -------------------