From 995ef44844400000dc97eb86c33955a8334c3449 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Thu, 30 Jun 2016 16:34:31 -0700 Subject: [PATCH 1/4] Only allow min, max, argmin, argmax if all dates are comparable --- .../tables/lambdadcs/DenotationUtils.java | 31 +++++++++ .../lambdadcs/InfiniteUnaryDenotation.java | 69 ++++++++----------- .../lambdadcs/LambdaDCSExecutorTest.java | 18 +++-- .../dump-parsers/write-simple-s2s-dataset.py | 10 ++- 4 files changed, 78 insertions(+), 50 deletions(-) diff --git a/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java b/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java index fb2351e..afa098a 100644 --- a/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java @@ -282,7 +282,10 @@ public final class DenotationUtils { * Processor for each data type. */ public abstract static class TypeProcessor { + // Is the value v compatible with this processor? public abstract boolean isCompatible(Value v); + // Is the collection sortable? (Is there a total order on the elements?) + public abstract boolean isSortable(Collection values); // positive if v1 > v2 | negative if v1 < v2 | 0 if v1 == v2 public int compareValues(Value v1, Value v2) { throw new LambdaDCSException(Type.typeMismatch, "Cannot compare values with " + getClass().getSimpleName()); } public Value sum(Collection values) { throw new LambdaDCSException(Type.typeMismatch, "Cannot compute sum with " + getClass().getSimpleName()); } @@ -293,6 +296,8 @@ public final class DenotationUtils { public Value div(Value v1, Value v2) { throw new LambdaDCSException(Type.typeMismatch, "Cannot compute div with " + getClass().getSimpleName()); } public Value max(Collection values) { + if (!isSortable(values)) + throw new LambdaDCSException(Type.typeMismatch, "Values cannot be sorted."); Value max = null; for (Value value : values) { if (max == null || compareValues(max, value) < 0) @@ -302,6 +307,8 @@ public final class DenotationUtils { } public Value min(Collection values) { + if (!isSortable(values)) + throw new LambdaDCSException(Type.typeMismatch, "Values cannot be sorted."); Value min = null; for (Value value : values) { if (min == null || compareValues(min, value) > 0) @@ -311,6 +318,8 @@ public final class DenotationUtils { } public List argsort(List values) { + if (!isSortable(values)) + throw new LambdaDCSException(Type.typeMismatch, "Values cannot be sorted."); List indices = new ArrayList<>(); for (int i = 0; i < values.size(); i++) indices.add(i); @@ -334,6 +343,11 @@ public final class DenotationUtils { public boolean isCompatible(Value v) { return v instanceof NumberValue; } + + @Override + public boolean isSortable(Collection values) { + return true; + } @Override public int compareValues(Value v1, Value v2) { @@ -373,6 +387,22 @@ public final class DenotationUtils { public boolean isCompatible(Value v) { return v instanceof DateValue; } + + @Override + public boolean isSortable(Collection values) { + DateValue firstDate = null; + for (Value value : values) { + DateValue date = (DateValue) value; + if (firstDate == null) { + firstDate = date; + } else { + if ((firstDate.year == -1) != (date.year == -1)) return false; + if ((firstDate.month == -1) != (date.month == -1)) return false; + if ((firstDate.day == -1) != (date.day == -1)) return false; + } + } + return true; + } @Override public int compareValues(Value v1, Value v2) { @@ -435,4 +465,5 @@ public final class DenotationUtils { throw new LambdaDCSException(Type.typeMismatch, "Cannot compare values"); } } + } diff --git a/src/edu/stanford/nlp/sempre/tables/lambdadcs/InfiniteUnaryDenotation.java b/src/edu/stanford/nlp/sempre/tables/lambdadcs/InfiniteUnaryDenotation.java index 87cc201..4bb941e 100644 --- a/src/edu/stanford/nlp/sempre/tables/lambdadcs/InfiniteUnaryDenotation.java +++ b/src/edu/stanford/nlp/sempre/tables/lambdadcs/InfiniteUnaryDenotation.java @@ -49,6 +49,11 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation { return Integer.MAX_VALUE; } + @Override + public UnaryDenotation aggregate(AggregateFormula.Mode mode) { + throw new LambdaDCSException(Type.infiniteList, "Cannot use aggregate mode %s on %s", mode, this); + } + @Override public UnaryDenotation filter(UnaryDenotation that) { return merge(that, MergeFormula.Mode.and); @@ -62,9 +67,9 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation { return (EverythingUnaryDenotation) second; } else if (second instanceof GenericDateUnaryDenotation) { if ("<".equals(binary) || ">=".equals(binary)) - return new ComparisonUnaryDenotation(binary, DenotationUtils.getSingleValue(second.aggregate(AggregateFormula.Mode.min))); + return new ComparisonUnaryDenotation(binary, ((GenericDateUnaryDenotation) second).getMin()); if (">".equals(binary) || "<=".equals(binary)) - return new ComparisonUnaryDenotation(binary, DenotationUtils.getSingleValue(second.aggregate(AggregateFormula.Mode.max))); + return new ComparisonUnaryDenotation(binary, ((GenericDateUnaryDenotation) second).getMax()); } return new ComparisonUnaryDenotation(binary, DenotationUtils.getSingleValue(second)); } @@ -108,11 +113,6 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation { } } - @Override - public UnaryDenotation aggregate(AggregateFormula.Mode mode) { - throw new LambdaDCSException(Type.infiniteList, "Cannot use aggregate mode %s on *", mode); - } - } public static final InfiniteUnaryDenotation STAR_UNARY = new EverythingUnaryDenotation(); @@ -165,11 +165,6 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation { throw new LambdaDCSException(Type.infiniteList, "Cannot use merge mode %s on %s and %s", mode, this, that); } - @Override - public UnaryDenotation aggregate(AggregateFormula.Mode mode) { - throw new LambdaDCSException(Type.infiniteList, "Cannot use aggregate mode %s on %s", mode, this); - } - @Override public boolean contains(Object o) { if (!(o instanceof Value)) return false; @@ -318,11 +313,6 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation { throw new LambdaDCSException(Type.infiniteList, "Cannot use merge mode %s on %s and %s", mode, this, that); } - @Override - public UnaryDenotation aggregate(AggregateFormula.Mode mode) { - throw new LambdaDCSException(Type.infiniteList, "Cannot use aggregate mode %s on %s", mode, this); - } - @Override public boolean contains(Object o) { if (!(o instanceof Value)) return false; @@ -381,32 +371,29 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation { } } else if (that instanceof EverythingUnaryDenotation) { return that.merge(this, mode); - } else { - // Handle some more cases? - } + } throw new LambdaDCSException(Type.infiniteList, "Cannot use merge mode %s on %s and %s", mode, this, that); } - - @Override - public UnaryDenotation aggregate(AggregateFormula.Mode mode) { - // Handle min and max - if (mode == AggregateFormula.Mode.min || mode == AggregateFormula.Mode.max) { - if (date.day != -1) { // (... ... xxx) - return new ExplicitUnaryDenotation(date); - } else if (date.month != -1) { // (... xxx -1) - if (mode == AggregateFormula.Mode.min) - return new ExplicitUnaryDenotation(new DateValue(date.year, date.month, 1)); - else - return new ExplicitUnaryDenotation(new DateValue(date.year, date.month, - YearMonth.of(date.year == -1 ? 2000 : date.year, date.month).lengthOfMonth())); - } else { // (xxx -1 -1) - if (mode == AggregateFormula.Mode.min) - return new ExplicitUnaryDenotation(new DateValue(date.year, 1, 1)); - else - return new ExplicitUnaryDenotation(new DateValue(date.year, 12, 31)); - } - } - throw new LambdaDCSException(Type.infiniteList, "Cannot use aggregate mode %s on %s", mode, this); + + public DateValue getMin() { + if (date.day != -1) + return date; + if (date.month != -1) + return new DateValue(date.year, date.month, 1); + if (date.year != -1) + return new DateValue(date.year, 1, 1); + throw new LambdaDCSException(Type.unknown, "Invalid date: (-1 -1 -1)."); + } + + public DateValue getMax() { + if (date.day != -1) + return date; + if (date.month != -1) + return new DateValue(date.year, date.month, + YearMonth.of(date.year == -1 ? 2000 : date.year, date.month).lengthOfMonth()); + if (date.year != -1) + return new DateValue(date.year, 12, 31); + throw new LambdaDCSException(Type.unknown, "Invalid date: (-1 -1 -1)."); } @Override diff --git a/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java b/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java index 41e0bc2..c2159e4 100644 --- a/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java +++ b/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java @@ -152,6 +152,8 @@ public class LambdaDCSExecutorTest { return TableKnowledgeGraph.fromFilename("tables/toy-examples/random/nikos_machlas.csv"); } else if ("csv2".equals(name)) { return TableKnowledgeGraph.fromFilename("lib/data/tables/csv/204-csv/495.tsv"); + } else if ("csv3".equals(name)) { + return TableKnowledgeGraph.fromFilename("lib/data/tables/csv/203-csv/839.tsv"); } throw new RuntimeException("Unknown graph name: " + name); } @@ -247,14 +249,18 @@ public class LambdaDCSExecutorTest { @Test(groups = "lambdaCSV2") public void lambdaOnGraphCSV2Test() { KnowledgeGraph graph = getKnowledgeGraph("csv2"); - // fb:cell.away', 'fb:row.row.venue', '!fb:row.row.result', - // '!fb:cell.cell.num2', 'avg', 'fb:row.row.index', 'fb:row.row.next', '!fb:row.row.opponent' - //runFormula(executor, - // "(!fb:row.row.opponent (fb:row.row.next (fb:row.row.index (avg (!fb:cell.cell.num2 (!fb:row.row.result (fb:row.row.venue fb:cell.away)))))))", - // graph, matches("(name fb:cell.derby_county)")); runFormula(executor, "(and (!= (and (!= fb:cell.away) fb:cell.home)) ((reverse fb:row.row.opponent) (fb:row.row.index (- (number 1) (number 1)))))", graph, matches("(name fb:cell.derby_county)")); } - + + @Test(groups = "lambdaCSV3") public void lambdaOnGraphCSV3Test() { + KnowledgeGraph graph = getKnowledgeGraph("csv3"); + runFormula(executor, + "(count (fb:type.object.type fb:type.row))", + graph, matches("(number 21)")); + runFormula(executor, + "(count (fb:row.row.opened (fb:cell.cell.date (< (date 1926 -1 -1)))))", + graph, matches("(number 6)")); + } } diff --git a/tables/dump-parsers/write-simple-s2s-dataset.py b/tables/dump-parsers/write-simple-s2s-dataset.py index bc41739..e4f832e 100755 --- a/tables/dump-parsers/write-simple-s2s-dataset.py +++ b/tables/dump-parsers/write-simple-s2s-dataset.py @@ -18,7 +18,7 @@ def get_formula(fpost): formula, size = line[:-1].split('\t') formulas.append([formula, int(size)]) if not formulas: - return None + return '' # Choose a formula best_size = min(x[1] for x in formulas) best_formulas = [x[0] for x in formulas if x[1] == best_size] @@ -34,6 +34,8 @@ def main(): help='path to postfix directory') parser.add_argument('-e', '--ex-id', action='store_true', help='add example ID as the first column') + parser.add_argument('-n', '--no-skip', action='store_true', + help='do not skip examples without formula') args = parser.parse_args() # Read annotated data @@ -48,7 +50,9 @@ def main(): count = 0 with open(args.dataset_path, 'r', 'utf8') as fin: header = fin.readline()[:-1].split('\t') - for line in fin: + for i, line in enumerate(fin): + if i % 500 == 0: + print >> sys.stderr, 'Processing Line', i, '...' record = dict(zip(header, line[:-1].split('\t'))) record = annotated[record['id']] ex_id = record['id'] @@ -56,7 +60,7 @@ def main(): with gzip.open(os.path.join(args.postfix_dir, ex_id + '.gz')) as fpost: random.seed(ex_id) formula = get_formula(fpost) - if formula: + if formula or args.no_skip: fields = [' '.join(question), formula] if args.ex_id: fields.insert(0, ex_id) From ba59a2a08301c970d10a36e02c27e8eca183c193 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Wed, 17 Aug 2016 19:39:38 -0700 Subject: [PATCH 2/4] Fixed date evaluation and NameValue string. Also added more tests. Conflicts: src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java --- .../nlp/sempre/tables/TableKnowledgeGraph.java | 8 +++++++- .../nlp/sempre/tables/TableValueEvaluator.java | 16 +++++++++++----- .../tables/lambdadcs/LambdaDCSExecutor.java | 2 ++ .../tables/lambdadcs/LambdaDCSExecutorTest.java | 11 ++++++++++- tables/dump-parsers/convert-to-postfix.py | 2 +- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java b/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java index 8609e7a..143aeca 100644 --- a/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java +++ b/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java @@ -608,13 +608,19 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl return null; } + public Value getNameValueWithOriginalString(NameValue value) { + if (value.description == null) + value = new NameValue(value.id, getOriginalString(value.id)); + return value; + } + public ListValue getListValueWithOriginalStrings(ListValue answers) { List values = new ArrayList<>(); for (Value value : answers.values) { if (value instanceof NameValue) { NameValue name = (NameValue) value; if (name.description == null) - value = new NameValue(name.id, getOriginalString(((NameValue) value).id)); + value = new NameValue(name.id, getOriginalString(name.id)); } values.add(value); } diff --git a/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java b/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java index f1d2a84..87e44a5 100644 --- a/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java +++ b/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java @@ -23,6 +23,8 @@ public class TableValueEvaluator implements ValueEvaluator { public boolean allowNormalizedStringMatch = true; @Option(gloss = "When comparing number values, only consider the value and not the unit") public boolean ignoreNumberValueUnits = true; + @Option(gloss = "Strict date evaluation (year, month, and date all have to match)") + public boolean strictDateEvaluation = false; } public static Options opts = new Options(); @@ -113,11 +115,15 @@ public class TableValueEvaluator implements ValueEvaluator { } protected boolean compareDateValues(DateValue target, DateValue pred) { - // If a field in target is not blank (-1), pred must match target on that field - if (target.year != -1 && target.year != pred.year) return false; - if (target.month != -1 && target.month != pred.month) return false; - if (target.day != -1 && target.day != pred.day) return false; - return true; + if (opts.strictDateEvaluation) { + return target.equals(pred); + } else { + // If a field in target is not blank (-1), pred must match target on that field + if (target.year != -1 && target.year != pred.year) return false; + if (target.month != -1 && target.month != pred.month) return false; + if (target.day != -1 && target.day != pred.day) return false; + return true; + } } } diff --git a/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutor.java b/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutor.java index ccaa167..bb1aace 100644 --- a/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutor.java +++ b/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutor.java @@ -205,6 +205,8 @@ class LambdaDCSCoreLogic { // Rule out binaries if (CanonicalNames.isBinary(value) && LambdaDCSExecutor.opts.executeBinary) throw new LambdaDCSException(Type.notUnary, "[Unary] Binary value %s", formula); + if (value instanceof NameValue && graph instanceof TableKnowledgeGraph) + value = ((TableKnowledgeGraph) graph).getNameValueWithOriginalString((NameValue) value); // Other cases return typeHint.applyBound(new ExplicitUnaryDenotation(value)); } diff --git a/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java b/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java index c2159e4..d603f9e 100644 --- a/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java +++ b/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java @@ -4,6 +4,7 @@ import java.util.*; import fig.basic.*; import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.corenlp.CoreNLPAnalyzer; import edu.stanford.nlp.sempre.tables.TableKnowledgeGraph; import org.testng.annotations.Test; @@ -253,8 +254,9 @@ public class LambdaDCSExecutorTest { "(and (!= (and (!= fb:cell.away) fb:cell.home)) ((reverse fb:row.row.opponent) (fb:row.row.index (- (number 1) (number 1)))))", graph, matches("(name fb:cell.derby_county)")); } - + @Test(groups = "lambdaCSV3") public void lambdaOnGraphCSV3Test() { + LanguageAnalyzer.setSingleton(new CoreNLPAnalyzer()); KnowledgeGraph graph = getKnowledgeGraph("csv3"); runFormula(executor, "(count (fb:type.object.type fb:type.row))", @@ -262,5 +264,12 @@ public class LambdaDCSExecutorTest { runFormula(executor, "(count (fb:row.row.opened (fb:cell.cell.date (< (date 1926 -1 -1)))))", graph, matches("(number 6)")); + runFormula(executor, + "(sum (- (count ((reverse fb:row.row.index) (fb:type.object.type fb:type.row))) " + + "((reverse fb:row.row.index) (fb:row.row.latitude ((reverse fb:row.row.longitude) (fb:type.object.type fb:type.row))))))", + graph, matches("(number 6)")); + runFormula(executor, + "(- (number 1926) (argmax (number 1) (number 1) ((reverse fb:cell.cell.number) (or (or (or fb:cell.1920 fb:cell.1925) fb:cell.1926) fb:cell.1946)) (reverse (lambda x (sum ((reverse fb:cell.cell.number) (fb:cell.cell.number (var x))))))))", + graph, matches("(number 6)")); } } diff --git a/tables/dump-parsers/convert-to-postfix.py b/tables/dump-parsers/convert-to-postfix.py index e64514b..3030e8e 100755 --- a/tables/dump-parsers/convert-to-postfix.py +++ b/tables/dump-parsers/convert-to-postfix.py @@ -5,7 +5,7 @@ If the file is tab-separated, only process the first column. """ -import sys, os, shutil, re, argparse, json +import sys, os, shutil, re, argparse, json, gzip from codecs import open from itertools import izip from collections import defaultdict From 254640fc84ad6ad63ef1484921a6572dabf6400d Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Wed, 17 Aug 2016 19:42:22 -0700 Subject: [PATCH 3/4] Filter the dumped logical forms based on the fixed ValueEvaluator --- .../sempre/tables/serialize/DumpFilterer.java | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 src/edu/stanford/nlp/sempre/tables/serialize/DumpFilterer.java diff --git a/src/edu/stanford/nlp/sempre/tables/serialize/DumpFilterer.java b/src/edu/stanford/nlp/sempre/tables/serialize/DumpFilterer.java new file mode 100644 index 0000000..0806c0c --- /dev/null +++ b/src/edu/stanford/nlp/sempre/tables/serialize/DumpFilterer.java @@ -0,0 +1,90 @@ +package edu.stanford.nlp.sempre.tables.serialize; + +import java.io.BufferedReader; +import java.io.File; +import java.io.PrintWriter; +import java.util.*; + +import edu.stanford.nlp.sempre.*; +import fig.basic.*; +import fig.exec.Execution; + +public class DumpFilterer implements Runnable { + public static class Options { + @Option(gloss = "verbosity") public int verbose = 0; + @Option(gloss = "input dump directory") + public String filtererInputDumpDirectory; + } + public static Options opts = new Options(); + + public static void main(String[] args) { + Execution.run(args, "DumpFiltererMain", new DumpFilterer(), Master.getOptionsParser()); + } + + Builder builder; + + @Override + public void run() { + builder = new Builder(); + builder.build(); + String outDir = Execution.getFile("filtered"); + new File(outDir).mkdirs(); + for (Pair pathPair : Dataset.opts.inPaths) { + String group = pathPair.getFirst(); + String path = pathPair.getSecond(); + // Read LispTrees + LogInfo.begin_track("Reading %s", path); + int maxExamples = Dataset.getMaxExamplesForGroup(group); + Iterator trees = LispTree.proto.parseFromFile(path); + // Go through the examples + int n = 0; + while (n < maxExamples) { + // Format: (example (id ...) (utterance ...) (targetFormula ...) (targetValue ...)) + LispTree tree = trees.next(); + if (tree == null) break; + if (tree.children.size() < 2 || !"example".equals(tree.child(0).value)) { + if ("metadata".equals(tree.child(0).value)) continue; + throw new RuntimeException("Invalid example: " + tree); + } + Example ex = Example.fromLispTree(tree, path + ":" + n); + ex.preprocess(); + LogInfo.logs("Example %s (%d): %s => %s", ex.id, n, ex.getTokens(), ex.targetValue); + n++; + processExample(ex); + } + LogInfo.end_track(); + } + } + + private void processExample(Example ex) { + File inPath = new File(opts.filtererInputDumpDirectory, ex.id + ".gz"); + File outPath = new File(Execution.getFile("filtered"), ex.id + ".gz"); + try { + BufferedReader reader = IOUtils.openInHard(inPath); + PrintWriter writer = IOUtils.openOutHard(outPath); + int inLines = 0, outLines = 0; + String line; + while ((line = reader.readLine()) != null) { + inLines++; + LispTree tree = LispTree.proto.parseFromString(line); + if (!"formula".equals(tree.child(1).child(0).value)) + throw new RuntimeException("Invalid tree: " + tree); + Formula formula = Formulas.fromLispTree(tree.child(1).child(1)); + Value value = builder.executor.execute(formula, ex.context).value; + double compatibility = builder.valueEvaluator.getCompatibility(ex.targetValue, value); + if (compatibility == 1.0) { + writer.println(tree); + outLines++; + } else if (opts.verbose >= 2) { + LogInfo.logs("Filtered out %s <= %s", value, formula); + } + } + LogInfo.logs("Filtered %d => %d", inLines, outLines); + reader.close(); + writer.close(); + } catch (Exception e) { + LogInfo.warnings("Got an error: %s", e); + } + } + +} From 7991a1801a89c8ddcf44d5f802d5de563a8f1734 Mon Sep 17 00:00:00 2001 From: Panupong Pasupat Date: Wed, 17 Aug 2016 19:42:56 -0700 Subject: [PATCH 4/4] Renamed "annotated" to "tagged" to avoid confusion --- pull-dependencies | 13 +++- run | 17 +++-- ...otatedGenerator.java => TSVGenerator.java} | 7 +- ...rator.java => TaggedDatasetGenerator.java} | 70 +++++-------------- ...nerator.java => TaggedTableGenerator.java} | 10 +-- 5 files changed, 48 insertions(+), 69 deletions(-) rename src/edu/stanford/nlp/sempre/tables/serialize/{AnnotatedGenerator.java => TSVGenerator.java} (94%) rename src/edu/stanford/nlp/sempre/tables/serialize/{AnnotatedDatasetGenerator.java => TaggedDatasetGenerator.java} (66%) rename src/edu/stanford/nlp/sempre/tables/serialize/{AnnotatedTableGenerator.java => TaggedTableGenerator.java} (92%) diff --git a/pull-dependencies b/pull-dependencies index 7febe16..e1301c8 100755 --- a/pull-dependencies +++ b/pull-dependencies @@ -150,7 +150,11 @@ addModule('corenlp', 'Stanford CoreNLP 3.6.0', lambda { 'stanford-corenlp-3.6.0-models.jar' => 'stanford-corenlp-models.jar', 'stanford-corenlp-caseless-2015-04-20-models.jar' => 'stanford-corenlp-caseless-models.jar', 'joda-time.jar' => 'joda-time.jar', - 'jollyday.jar' => 'jollyday.jar'}.each { |key, value| + 'jollyday.jar' => 'jollyday.jar', + 'ejml-0.23.jar' => 'ejml.jar', + 'slf4j-api.jar' => 'slf4j-api.jar', + 'slf4j-simple.jar' => 'slf4j-simple.jar', + }.each { |key, value| system "ln -sfv stanford-corenlp-full-2015-12-09/#{key} lib/#{value}" or exit 1 } }) @@ -170,7 +174,8 @@ addModule('corenlp-3.2.0', 'Stanford CoreNLP 3.2.0 (for backward reproducibility 'stanford-corenlp-3.2.0-models.jar' => 'stanford-corenlp-models.jar', 'stanford-corenlp-caseless-2013-06-07-models.jar' => 'stanford-corenlp-caseless-models.jar', 'joda-time.jar' => 'joda-time.jar', - 'jollyday.jar' => 'jollyday.jar'}.each { |key, value| + 'jollyday.jar' => 'jollyday.jar' + }.each { |key, value| system "ln -sfv stanford-corenlp-full-2013-06-20/#{key} lib/#{value}" or exit 1 } }) @@ -275,6 +280,10 @@ if ARGV.size == 0 $modules.each { |name,description,func| puts " #{name}: #{description}" } + puts + puts "Internal use (Stanford NLP only):" + puts " #{$0} -l ...: Get the files from the local Stanford NLP server instead" + puts " #{$0} -l -r ...: Release to the public www directory on the server" end $modules.each { |name,description,func| diff --git a/run b/run index 38d8fb8..4cfb97b 100755 --- a/run +++ b/run @@ -623,10 +623,11 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( 'dump' => 'edu.stanford.nlp.sempre.tables.serialize.SerializedDumper', 'load' => l('edu.stanford.nlp.sempre.tables.serialize.SerializedLoader', let(:parser, 'serialized')), 'stats' => 'edu.stanford.nlp.sempre.tables.test.TableStatsComputer', - 'ann-data' => 'edu.stanford.nlp.sempre.tables.serialize.AnnotatedDatasetGenerator', - 'ann-table' => 'edu.stanford.nlp.sempre.tables.serialize.AnnotatedTableGenerator', + 'tag-data' => 'edu.stanford.nlp.sempre.tables.serialize.TaggedDatasetGenerator', + 'tag-table' => 'edu.stanford.nlp.sempre.tables.serialize.TaggedTableGenerator', 'alter' => l('edu.stanford.nlp.sempre.tables.alter.BatchTableAlterer', let(:parser, 'serialized')), 'alter-ex' => l('edu.stanford.nlp.sempre.tables.alter.AlteredTablesExecutor', let(:parser, 'serialized')), + 'filter' => 'edu.stanford.nlp.sempre.tables.serialize.DumpFilterer', }), # Fig parameters selo(:cldir, 'execDir', '_OUTPATH_', 'output'), @@ -649,7 +650,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l( nil), }), # Parser - letDefault(:parser, 'floatsize'), + letDefault(:parser, 'old-floatsize'), sel(:parser, { 'old-floatsize' => l( o('Builder.parser', 'FloatingParser'), @@ -860,13 +861,11 @@ def tablesDataPaths csvDir = ['lib/data/tables/', 'WikiTableQuestions/'][e[:cldir]] datasets = { 'none' => l(), + 'train' => o('Dataset.inPaths', "train,#{baseDir}training.examples"), # Pristine test test - 'test' => l( - o('Dataset.inPaths', - "train,#{baseDir}training.examples", - "test,#{baseDir}pristine-unseen-tables.examples"), - o('Dataset.trainFrac', 1), o('Dataset.devFrac', 0), - nil), + 'test' => o('Dataset.inPaths', + "train,#{baseDir}training.examples", + "test,#{baseDir}pristine-unseen-tables.examples"), # @data=annotated can be used with @class=check only 'annotated' => o('Dataset.inPaths', "train,#{baseDir}annotated-all.examples"), 'before300' => o('Dataset.inPaths', "train,#{baseDir}training-before300.examples"), diff --git a/src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedGenerator.java b/src/edu/stanford/nlp/sempre/tables/serialize/TSVGenerator.java similarity index 94% rename from src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedGenerator.java rename to src/edu/stanford/nlp/sempre/tables/serialize/TSVGenerator.java index a492a4c..2771da5 100644 --- a/src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedGenerator.java +++ b/src/edu/stanford/nlp/sempre/tables/serialize/TSVGenerator.java @@ -5,7 +5,12 @@ import java.util.*; import edu.stanford.nlp.sempre.*; -public class AnnotatedGenerator { +/** + * Generate a TSV file for the dataset release. + * + * @author ppasupat + */ +public class TSVGenerator { protected PrintWriter out; protected void dump(String... stuff) { diff --git a/src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedDatasetGenerator.java b/src/edu/stanford/nlp/sempre/tables/serialize/TaggedDatasetGenerator.java similarity index 66% rename from src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedDatasetGenerator.java rename to src/edu/stanford/nlp/sempre/tables/serialize/TaggedDatasetGenerator.java index e895a42..b4d37c6 100644 --- a/src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedDatasetGenerator.java +++ b/src/edu/stanford/nlp/sempre/tables/serialize/TaggedDatasetGenerator.java @@ -1,6 +1,5 @@ package edu.stanford.nlp.sempre.tables.serialize; -import java.io.*; import java.util.*; import edu.stanford.nlp.sempre.*; @@ -9,7 +8,7 @@ import fig.basic.*; import fig.exec.Execution; /** - * Generate TSV files containing CoreNLP annotation of the datasets. + * Generate TSV files containing CoreNLP tags of the datasets. * * Field descriptions: * - id: unique ID of the example @@ -23,13 +22,14 @@ import fig.exec.Execution; * - nerValues: if the NER tag is numerical or temporal, the value of that * NER span will be listed here * - targetCanon: the answer, canonicalized + * - targetCanonType: type of the canonicalized answer (number, date, or string) * * @author ppasupat */ -public class AnnotatedDatasetGenerator extends AnnotatedGenerator implements Runnable { +public class TaggedDatasetGenerator extends TSVGenerator implements Runnable { public static void main(String[] args) { - Execution.run(args, "AnnotatedDatasetGeneratorMain", new AnnotatedDatasetGenerator(), + Execution.run(args, "TaggedDatasetGeneratorMain", new TaggedDatasetGenerator(), Master.getOptionsParser()); } @@ -41,7 +41,7 @@ public class AnnotatedDatasetGenerator extends AnnotatedGenerator implements Run String group = pathPair.getFirst(); String path = pathPair.getSecond(); // Open output file - String filename = Execution.getFile("annotated-" + group + ".tsv"); + String filename = Execution.getFile("tagged-" + group + ".tsv"); out = IOUtils.openOutHard(filename); dump(FIELDS); // Read LispTrees @@ -73,7 +73,8 @@ public class AnnotatedDatasetGenerator extends AnnotatedGenerator implements Run private static final String[] FIELDS = new String[] { "id", "utterance", "context", "targetValue", - "tokens", "lemmaTokens", "posTags", "nerTags", "nerValues", "targetCanon", + "tokens", "lemmaTokens", "posTags", "nerTags", "nerValues", + "targetCanon", "targetCanonType", }; @Override @@ -99,58 +100,23 @@ public class AnnotatedDatasetGenerator extends AnnotatedGenerator implements Run } } // Other information come from Example - fields[2] = serialize(((TableKnowledgeGraph) ex.context.graph).filename); + fields[2] = serialize(((TableKnowledgeGraph) ex.context.graph).filename.replace("lib/data/tables/", "")); fields[4] = serialize(ex.languageInfo.tokens); fields[5] = serialize(ex.languageInfo.lemmaTokens); fields[6] = serialize(ex.languageInfo.posTags); fields[7] = serialize(ex.languageInfo.nerTags); fields[8] = serialize(ex.languageInfo.nerValues); + // Information from target value fields[9] = serialize(ex.targetValue); + Value targetValue = ex.targetValue; + if (targetValue instanceof ListValue) + targetValue = ((ListValue) targetValue).values.get(0); + if (targetValue instanceof NumberValue) + fields[10] = "number"; + else if (targetValue instanceof DateValue) + fields[10] = "date"; + else + fields[10] = "string"; dump(fields); } - - // Helper Functions for TSV - - public static void dump(PrintWriter out, String... stuff) { - out.println(String.join("\t", stuff)); - } - - public static String serialize(String x) { - if (x == null || x.isEmpty()) return ""; - StringBuilder sb = new StringBuilder(); - for (char y : x.toCharArray()) { - if (y == '\n') sb.append("\\n"); - else if (y == '\\') sb.append("\\\\"); - else if (y == '|') sb.append("\\|"); - else sb.append(y); - } - return sb.toString().replaceAll("\\s", " ").trim(); - } - - public static String serialize(List xs) { - List serialized = new ArrayList<>(); - for (String x : xs) serialized.add(serialize(x)); - return String.join("|", serialized); - } - - public static String serialize(Value value) { - if (value instanceof ListValue) { - List xs = new ArrayList<>(); - for (Value v : ((ListValue) value).values) { - xs.add(serialize(v)); - } - return String.join("|", xs); - } else if (value instanceof DescriptionValue) { - return serialize(((DescriptionValue) value).value); - } else if (value instanceof NameValue) { - return serialize(((NameValue) value).description); - } else if (value instanceof NumberValue) { - return "" + ((NumberValue) value).value; - } else if (value instanceof DateValue) { - return ((DateValue) value).isoString(); - } else { - throw new RuntimeException("Unknown value type: " + value); - } - } - } diff --git a/src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedTableGenerator.java b/src/edu/stanford/nlp/sempre/tables/serialize/TaggedTableGenerator.java similarity index 92% rename from src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedTableGenerator.java rename to src/edu/stanford/nlp/sempre/tables/serialize/TaggedTableGenerator.java index cca2be1..36e8b65 100644 --- a/src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedTableGenerator.java +++ b/src/edu/stanford/nlp/sempre/tables/serialize/TaggedTableGenerator.java @@ -14,7 +14,7 @@ import fig.basic.LogInfo; import fig.exec.Execution; /** - * Generate TSV files containing CoreNLP annotation of the tables. + * Generate TSV files containing CoreNLP tags of the tables. * * Mandatory fields: * - row: row index (-1 is the header row) @@ -40,10 +40,10 @@ import fig.exec.Execution; * * @author ppasupat */ -public class AnnotatedTableGenerator extends AnnotatedGenerator implements Runnable { +public class TaggedTableGenerator extends TSVGenerator implements Runnable { public static void main(String[] args) { - Execution.run(args, "AnnotatedTableGeneratorMain", new AnnotatedTableGenerator(), + Execution.run(args, "TaggedTableGeneratorMain", new TaggedTableGenerator(), Master.getOptionsParser()); } @@ -65,8 +65,8 @@ public class AnnotatedTableGenerator extends AnnotatedGenerator implements Runna int batchIndex = Integer.parseInt(matcher.group(1)), dataIndex = Integer.parseInt(matcher.group(2)); TableKnowledgeGraph table = TableKnowledgeGraph.fromFilename(baseDir.relativize(file).toString()); - String outDir = Execution.getFile("annotated/" + batchIndex + "-annotated/"), - outFilename = new File(outDir, dataIndex + ".annotated").getPath(); + String outDir = Execution.getFile("tagged/" + batchIndex + "-tagged/"), + outFilename = new File(outDir, dataIndex + ".tagged").getPath(); new File(outDir).mkdirs(); out = IOUtils.openOutHard(outFilename); dumpTable(table);