natas_hw
/
sempre
mirror of https://github.com/percyliang/sempre
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/percyliang/sempre

This commit is contained in:
Sida Wang 2017-03-08 14:38:47 -08:00
parent 5616182eeb ff129650e2
commit d8bc66f9e0
14 changed files with 245 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pull-dependencies
View File

@ -157,7 +157,11 @@ addModule('corenlp', 'Stanford CoreNLP 3.6.0', lambda {
'stanford-corenlp-3.6.0-models.jar' => 'stanford-corenlp-models.jar',
'stanford-corenlp-caseless-2015-04-20-models.jar' => 'stanford-corenlp-caseless-models.jar',
'joda-time.jar' => 'joda-time.jar',
'jollyday.jar' => 'jollyday.jar'}.each { |key, value|
'jollyday.jar' => 'jollyday.jar',
'ejml-0.23.jar' => 'ejml.jar',
'slf4j-api.jar' => 'slf4j-api.jar',
'slf4j-simple.jar' => 'slf4j-simple.jar',
}.each { |key, value|
system "ln -sfv stanford-corenlp-full-2015-12-09/#{key} lib/#{value}" or exit 1
}
})
@ -177,7 +181,8 @@ addModule('corenlp-3.2.0', 'Stanford CoreNLP 3.2.0 (for backward reproducibility
'stanford-corenlp-3.2.0-models.jar' => 'stanford-corenlp-models.jar',
'stanford-corenlp-caseless-2013-06-07-models.jar' => 'stanford-corenlp-caseless-models.jar',
'joda-time.jar' => 'joda-time.jar',
'jollyday.jar' => 'jollyday.jar'}.each { |key, value|
'jollyday.jar' => 'jollyday.jar'
}.each { |key, value|
system "ln -sfv stanford-corenlp-full-2013-06-20/#{key} lib/#{value}" or exit 1
}
})
@ -299,6 +304,10 @@ if ARGV.size == 0
$modules.each { |name,description,func|
puts " #{name}: #{description}"
}
puts
puts "Internal use (Stanford NLP only):"
puts " #{$0} -l <module-1> ...: Get the files from the local Stanford NLP server instead"
puts " #{$0} -l -r <module-1> ...: Release to the public www directory on the server"
end
$modules.each { |name,description,func|

17
run
View File

@ -648,10 +648,11 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
'dump' => 'edu.stanford.nlp.sempre.tables.serialize.SerializedDumper',
'load' => l('edu.stanford.nlp.sempre.tables.serialize.SerializedLoader', let(:parser, 'serialized')),
'stats' => 'edu.stanford.nlp.sempre.tables.test.TableStatsComputer',
'ann-data' => 'edu.stanford.nlp.sempre.tables.serialize.AnnotatedDatasetGenerator',
'ann-table' => 'edu.stanford.nlp.sempre.tables.serialize.AnnotatedTableGenerator',
'tag-data' => 'edu.stanford.nlp.sempre.tables.serialize.TaggedDatasetGenerator',
'tag-table' => 'edu.stanford.nlp.sempre.tables.serialize.TaggedTableGenerator',
'alter' => l('edu.stanford.nlp.sempre.tables.alter.BatchTableAlterer', let(:parser, 'serialized')),
'alter-ex' => l('edu.stanford.nlp.sempre.tables.alter.AlteredTablesExecutor', let(:parser, 'serialized')),
'filter' => 'edu.stanford.nlp.sempre.tables.serialize.DumpFilterer',
}),
# Fig parameters
selo(:cldir, 'execDir', '_OUTPATH_', 'output'),
@ -674,7 +675,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
nil),
}),
# Parser
letDefault(:parser, 'floatsize'),
letDefault(:parser, 'old-floatsize'),
sel(:parser, {
'old-floatsize' => l(
o('Builder.parser', 'FloatingParser'),
@ -885,13 +886,11 @@ def tablesDataPaths
csvDir = ['lib/data/tables/', 'WikiTableQuestions/'][e[:cldir]]
datasets = {
'none' => l(),
'train' => o('Dataset.inPaths', "train,#{baseDir}training.examples"),
# Pristine test test
'test' => l(
o('Dataset.inPaths',
"train,#{baseDir}training.examples",
"test,#{baseDir}pristine-unseen-tables.examples"),
o('Dataset.trainFrac', 1), o('Dataset.devFrac', 0),
nil),
'test' => o('Dataset.inPaths',
"train,#{baseDir}training.examples",
"test,#{baseDir}pristine-unseen-tables.examples"),
# @data=annotated can be used with @class=check only
'annotated' => o('Dataset.inPaths', "train,#{baseDir}annotated-all.examples"),
'before300' => o('Dataset.inPaths', "train,#{baseDir}training-before300.examples"),

8
src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java
View File

@ -613,13 +613,19 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
return null;
}
public Value getNameValueWithOriginalString(NameValue value) {
if (value.description == null)
value = new NameValue(value.id, getOriginalString(value.id));
return value;
}
public ListValue getListValueWithOriginalStrings(ListValue answers) {
List<Value> values = new ArrayList<>();
for (Value value : answers.values) {
if (value instanceof NameValue) {
NameValue name = (NameValue) value;
if (name.description == null)
value = new NameValue(name.id, getOriginalString(((NameValue) value).id));
value = new NameValue(name.id, getOriginalString(name.id));
}
values.add(value);
}

16
src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java
View File

@ -23,6 +23,8 @@ public class TableValueEvaluator implements ValueEvaluator {
public boolean allowNormalizedStringMatch = true;
@Option(gloss = "When comparing number values, only consider the value and not the unit")
public boolean ignoreNumberValueUnits = true;
@Option(gloss = "Strict date evaluation (year, month, and date all have to match)")
public boolean strictDateEvaluation = false;
}
public static Options opts = new Options();
@ -113,11 +115,15 @@ public class TableValueEvaluator implements ValueEvaluator {
}
protected boolean compareDateValues(DateValue target, DateValue pred) {
// If a field in target is not blank (-1), pred must match target on that field
if (target.year != -1 && target.year != pred.year) return false;
if (target.month != -1 && target.month != pred.month) return false;
if (target.day != -1 && target.day != pred.day) return false;
return true;
if (opts.strictDateEvaluation) {
return target.equals(pred);
} else {
// If a field in target is not blank (-1), pred must match target on that field
if (target.year != -1 && target.year != pred.year) return false;
if (target.month != -1 && target.month != pred.month) return false;
if (target.day != -1 && target.day != pred.day) return false;
return true;
}
}
}

31
src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java
View File

@ -282,7 +282,10 @@ public final class DenotationUtils {
* Processor for each data type.
*/
public abstract static class TypeProcessor {
// Is the value v compatible with this processor?
public abstract boolean isCompatible(Value v);
// Is the collection sortable? (Is there a total order on the elements?)
public abstract boolean isSortable(Collection<Value> values);
// positive if v1 > v2 | negative if v1 < v2 | 0 if v1 == v2
public int compareValues(Value v1, Value v2) { throw new LambdaDCSException(Type.typeMismatch, "Cannot compare values with " + getClass().getSimpleName()); }
public Value sum(Collection<Value> values) { throw new LambdaDCSException(Type.typeMismatch, "Cannot compute sum with " + getClass().getSimpleName()); }
@ -293,6 +296,8 @@ public final class DenotationUtils {
public Value div(Value v1, Value v2) { throw new LambdaDCSException(Type.typeMismatch, "Cannot compute div with " + getClass().getSimpleName()); }
public Value max(Collection<Value> values) {
if (!isSortable(values))
throw new LambdaDCSException(Type.typeMismatch, "Values cannot be sorted.");
Value max = null;
for (Value value : values) {
if (max == null || compareValues(max, value) < 0)
@ -302,6 +307,8 @@ public final class DenotationUtils {
}
public Value min(Collection<Value> values) {
if (!isSortable(values))
throw new LambdaDCSException(Type.typeMismatch, "Values cannot be sorted.");
Value min = null;
for (Value value : values) {
if (min == null || compareValues(min, value) > 0)
@ -311,6 +318,8 @@ public final class DenotationUtils {
}
public List<Integer> argsort(List<Value> values) {
if (!isSortable(values))
throw new LambdaDCSException(Type.typeMismatch, "Values cannot be sorted.");
List<Integer> indices = new ArrayList<>();
for (int i = 0; i < values.size(); i++)
indices.add(i);
@ -334,6 +343,11 @@ public final class DenotationUtils {
public boolean isCompatible(Value v) {
return v instanceof NumberValue;
}
@Override
public boolean isSortable(Collection<Value> values) {
return true;
}
@Override
public int compareValues(Value v1, Value v2) {
@ -373,6 +387,22 @@ public final class DenotationUtils {
public boolean isCompatible(Value v) {
return v instanceof DateValue;
}
@Override
public boolean isSortable(Collection<Value> values) {
DateValue firstDate = null;
for (Value value : values) {
DateValue date = (DateValue) value;
if (firstDate == null) {
firstDate = date;
} else {
if ((firstDate.year == -1) != (date.year == -1)) return false;
if ((firstDate.month == -1) != (date.month == -1)) return false;
if ((firstDate.day == -1) != (date.day == -1)) return false;
}
}
return true;
}
@Override
public int compareValues(Value v1, Value v2) {
@ -435,4 +465,5 @@ public final class DenotationUtils {
throw new LambdaDCSException(Type.typeMismatch, "Cannot compare values");
}
}
}

69
src/edu/stanford/nlp/sempre/tables/lambdadcs/InfiniteUnaryDenotation.java
View File

@ -49,6 +49,11 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation {
return Integer.MAX_VALUE;
}
@Override
public UnaryDenotation aggregate(AggregateFormula.Mode mode) {
throw new LambdaDCSException(Type.infiniteList, "Cannot use aggregate mode %s on %s", mode, this);
}
@Override
public UnaryDenotation filter(UnaryDenotation that) {
return merge(that, MergeFormula.Mode.and);
@ -62,9 +67,9 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation {
return (EverythingUnaryDenotation) second;
} else if (second instanceof GenericDateUnaryDenotation) {
if ("<".equals(binary) || ">=".equals(binary))
return new ComparisonUnaryDenotation(binary, DenotationUtils.getSingleValue(second.aggregate(AggregateFormula.Mode.min)));
return new ComparisonUnaryDenotation(binary, ((GenericDateUnaryDenotation) second).getMin());
if (">".equals(binary) || "<=".equals(binary))
return new ComparisonUnaryDenotation(binary, DenotationUtils.getSingleValue(second.aggregate(AggregateFormula.Mode.max)));
return new ComparisonUnaryDenotation(binary, ((GenericDateUnaryDenotation) second).getMax());
}
return new ComparisonUnaryDenotation(binary, DenotationUtils.getSingleValue(second));
}
@ -108,11 +113,6 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation {
}
}
@Override
public UnaryDenotation aggregate(AggregateFormula.Mode mode) {
throw new LambdaDCSException(Type.infiniteList, "Cannot use aggregate mode %s on *", mode);
}
}
public static final InfiniteUnaryDenotation STAR_UNARY = new EverythingUnaryDenotation();
@ -165,11 +165,6 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation {
throw new LambdaDCSException(Type.infiniteList, "Cannot use merge mode %s on %s and %s", mode, this, that);
}
@Override
public UnaryDenotation aggregate(AggregateFormula.Mode mode) {
throw new LambdaDCSException(Type.infiniteList, "Cannot use aggregate mode %s on %s", mode, this);
}
@Override
public boolean contains(Object o) {
if (!(o instanceof Value)) return false;
@ -318,11 +313,6 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation {
throw new LambdaDCSException(Type.infiniteList, "Cannot use merge mode %s on %s and %s", mode, this, that);
}
@Override
public UnaryDenotation aggregate(AggregateFormula.Mode mode) {
throw new LambdaDCSException(Type.infiniteList, "Cannot use aggregate mode %s on %s", mode, this);
}
@Override
public boolean contains(Object o) {
if (!(o instanceof Value)) return false;
@ -381,32 +371,29 @@ public abstract class InfiniteUnaryDenotation extends UnaryDenotation {
}
} else if (that instanceof EverythingUnaryDenotation) {
return that.merge(this, mode);
} else {
// Handle some more cases?
}
}
throw new LambdaDCSException(Type.infiniteList, "Cannot use merge mode %s on %s and %s", mode, this, that);
}
@Override
public UnaryDenotation aggregate(AggregateFormula.Mode mode) {
// Handle min and max
if (mode == AggregateFormula.Mode.min || mode == AggregateFormula.Mode.max) {
if (date.day != -1) { // (... ... xxx)
return new ExplicitUnaryDenotation(date);
} else if (date.month != -1) { // (... xxx -1)
if (mode == AggregateFormula.Mode.min)
return new ExplicitUnaryDenotation(new DateValue(date.year, date.month, 1));
else
return new ExplicitUnaryDenotation(new DateValue(date.year, date.month,
YearMonth.of(date.year == -1 ? 2000 : date.year, date.month).lengthOfMonth()));
} else { // (xxx -1 -1)
if (mode == AggregateFormula.Mode.min)
return new ExplicitUnaryDenotation(new DateValue(date.year, 1, 1));
else
return new ExplicitUnaryDenotation(new DateValue(date.year, 12, 31));
}
}
throw new LambdaDCSException(Type.infiniteList, "Cannot use aggregate mode %s on %s", mode, this);
public DateValue getMin() {
if (date.day != -1)
return date;
if (date.month != -1)
return new DateValue(date.year, date.month, 1);
if (date.year != -1)
return new DateValue(date.year, 1, 1);
throw new LambdaDCSException(Type.unknown, "Invalid date: (-1 -1 -1).");
}
public DateValue getMax() {
if (date.day != -1)
return date;
if (date.month != -1)
return new DateValue(date.year, date.month,
YearMonth.of(date.year == -1 ? 2000 : date.year, date.month).lengthOfMonth());
if (date.year != -1)
return new DateValue(date.year, 12, 31);
throw new LambdaDCSException(Type.unknown, "Invalid date: (-1 -1 -1).");
}
@Override

2
src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutor.java
View File

@ -205,6 +205,8 @@ class LambdaDCSCoreLogic {
// Rule out binaries
if (CanonicalNames.isBinary(value) && LambdaDCSExecutor.opts.executeBinary)
throw new LambdaDCSException(Type.notUnary, "[Unary] Binary value %s", formula);
if (value instanceof NameValue && graph instanceof TableKnowledgeGraph)
value = ((TableKnowledgeGraph) graph).getNameValueWithOriginalString((NameValue) value);
// Other cases
return typeHint.applyBound(new ExplicitUnaryDenotation(value));
}

25
src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java
View File

@ -4,6 +4,7 @@ import java.util.*;
import fig.basic.*;
import edu.stanford.nlp.sempre.*;
import edu.stanford.nlp.sempre.corenlp.CoreNLPAnalyzer;
import edu.stanford.nlp.sempre.tables.TableKnowledgeGraph;
import org.testng.annotations.Test;
@ -152,6 +153,8 @@ public class LambdaDCSExecutorTest {
return TableKnowledgeGraph.fromFilename("tables/toy-examples/random/nikos_machlas.csv");
} else if ("csv2".equals(name)) {
return TableKnowledgeGraph.fromFilename("lib/data/tables/csv/204-csv/495.tsv");
} else if ("csv3".equals(name)) {
return TableKnowledgeGraph.fromFilename("lib/data/tables/csv/203-csv/839.tsv");
}
throw new RuntimeException("Unknown graph name: " + name);
}
@ -247,14 +250,26 @@ public class LambdaDCSExecutorTest {
@Test(groups = "lambdaCSV2") public void lambdaOnGraphCSV2Test() {
KnowledgeGraph graph = getKnowledgeGraph("csv2");
// fb:cell.away', 'fb:row.row.venue', '!fb:row.row.result',
// '!fb:cell.cell.num2', 'avg', 'fb:row.row.index', 'fb:row.row.next', '!fb:row.row.opponent'
//runFormula(executor,
// "(!fb:row.row.opponent (fb:row.row.next (fb:row.row.index (avg (!fb:cell.cell.num2 (!fb:row.row.result (fb:row.row.venue fb:cell.away)))))))",
// graph, matches("(name fb:cell.derby_county)"));
runFormula(executor,
"(and (!= (and (!= fb:cell.away) fb:cell.home)) ((reverse fb:row.row.opponent) (fb:row.row.index (- (number 1) (number 1)))))",
graph, matches("(name fb:cell.derby_county)"));
}
@Test(groups = "lambdaCSV3") public void lambdaOnGraphCSV3Test() {
LanguageAnalyzer.setSingleton(new CoreNLPAnalyzer());
KnowledgeGraph graph = getKnowledgeGraph("csv3");
runFormula(executor,
"(count (fb:type.object.type fb:type.row))",
graph, matches("(number 21)"));
runFormula(executor,
"(count (fb:row.row.opened (fb:cell.cell.date (< (date 1926 -1 -1)))))",
graph, matches("(number 6)"));
runFormula(executor,
"(sum (- (count ((reverse fb:row.row.index) (fb:type.object.type fb:type.row))) " +
"((reverse fb:row.row.index) (fb:row.row.latitude ((reverse fb:row.row.longitude) (fb:type.object.type fb:type.row))))))",
graph, matches("(number 6)"));
runFormula(executor,
"(- (number 1926) (argmax (number 1) (number 1) ((reverse fb:cell.cell.number) (or (or (or fb:cell.1920 fb:cell.1925) fb:cell.1926) fb:cell.1946)) (reverse (lambda x (sum ((reverse fb:cell.cell.number) (fb:cell.cell.number (var x))))))))",
graph, matches("(number 6)"));
}
}

90
src/edu/stanford/nlp/sempre/tables/serialize/DumpFilterer.java Normal file
View File

@ -0,0 +1,90 @@
package edu.stanford.nlp.sempre.tables.serialize;
import java.io.BufferedReader;
import java.io.File;
import java.io.PrintWriter;
import java.util.*;
import edu.stanford.nlp.sempre.*;
import fig.basic.*;
import fig.exec.Execution;
public class DumpFilterer implements Runnable {
public static class Options {
@Option(gloss = "verbosity") public int verbose = 0;
@Option(gloss = "input dump directory")
public String filtererInputDumpDirectory;
}
public static Options opts = new Options();
public static void main(String[] args) {
Execution.run(args, "DumpFiltererMain", new DumpFilterer(), Master.getOptionsParser());
}
Builder builder;
@Override
public void run() {
builder = new Builder();
builder.build();
String outDir = Execution.getFile("filtered");
new File(outDir).mkdirs();
for (Pair<String, String> pathPair : Dataset.opts.inPaths) {
String group = pathPair.getFirst();
String path = pathPair.getSecond();
// Read LispTrees
LogInfo.begin_track("Reading %s", path);
int maxExamples = Dataset.getMaxExamplesForGroup(group);
Iterator<LispTree> trees = LispTree.proto.parseFromFile(path);
// Go through the examples
int n = 0;
while (n < maxExamples) {
// Format: (example (id ...) (utterance ...) (targetFormula ...) (targetValue ...))
LispTree tree = trees.next();
if (tree == null) break;
if (tree.children.size() < 2 || !"example".equals(tree.child(0).value)) {
if ("metadata".equals(tree.child(0).value)) continue;
throw new RuntimeException("Invalid example: " + tree);
}
Example ex = Example.fromLispTree(tree, path + ":" + n);
ex.preprocess();
LogInfo.logs("Example %s (%d): %s => %s", ex.id, n, ex.getTokens(), ex.targetValue);
n++;
processExample(ex);
}
LogInfo.end_track();
}
}
private void processExample(Example ex) {
File inPath = new File(opts.filtererInputDumpDirectory, ex.id + ".gz");
File outPath = new File(Execution.getFile("filtered"), ex.id + ".gz");
try {
BufferedReader reader = IOUtils.openInHard(inPath);
PrintWriter writer = IOUtils.openOutHard(outPath);
int inLines = 0, outLines = 0;
String line;
while ((line = reader.readLine()) != null) {
inLines++;
LispTree tree = LispTree.proto.parseFromString(line);
if (!"formula".equals(tree.child(1).child(0).value))
throw new RuntimeException("Invalid tree: " + tree);
Formula formula = Formulas.fromLispTree(tree.child(1).child(1));
Value value = builder.executor.execute(formula, ex.context).value;
double compatibility = builder.valueEvaluator.getCompatibility(ex.targetValue, value);
if (compatibility == 1.0) {
writer.println(tree);
outLines++;
} else if (opts.verbose >= 2) {
LogInfo.logs("Filtered out %s <= %s", value, formula);
}
}
LogInfo.logs("Filtered %d => %d", inLines, outLines);
reader.close();
writer.close();
} catch (Exception e) {
LogInfo.warnings("Got an error: %s", e);
}
}
}

7
src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedGenerator.java → src/edu/stanford/nlp/sempre/tables/serialize/TSVGenerator.java
View File

@ -5,7 +5,12 @@ import java.util.*;
import edu.stanford.nlp.sempre.*;
public class AnnotatedGenerator {
/**
* Generate a TSV file for the dataset release.
*
* @author ppasupat
*/
public class TSVGenerator {
protected PrintWriter out;
protected void dump(String... stuff) {

70
src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedDatasetGenerator.java → src/edu/stanford/nlp/sempre/tables/serialize/TaggedDatasetGenerator.java
View File

@ -1,6 +1,5 @@
package edu.stanford.nlp.sempre.tables.serialize;
import java.io.*;
import java.util.*;
import edu.stanford.nlp.sempre.*;
@ -9,7 +8,7 @@ import fig.basic.*;
import fig.exec.Execution;
/**
* Generate TSV files containing CoreNLP annotation of the datasets.
* Generate TSV files containing CoreNLP tags of the datasets.
*
* Field descriptions:
* - id: unique ID of the example
@ -23,13 +22,14 @@ import fig.exec.Execution;
* - nerValues: if the NER tag is numerical or temporal, the value of that
* NER span will be listed here
* - targetCanon: the answer, canonicalized
* - targetCanonType: type of the canonicalized answer (number, date, or string)
*
* @author ppasupat
*/
public class AnnotatedDatasetGenerator extends AnnotatedGenerator implements Runnable {
public class TaggedDatasetGenerator extends TSVGenerator implements Runnable {
public static void main(String[] args) {
Execution.run(args, "AnnotatedDatasetGeneratorMain", new AnnotatedDatasetGenerator(),
Execution.run(args, "TaggedDatasetGeneratorMain", new TaggedDatasetGenerator(),
Master.getOptionsParser());
}
@ -41,7 +41,7 @@ public class AnnotatedDatasetGenerator extends AnnotatedGenerator implements Run
String group = pathPair.getFirst();
String path = pathPair.getSecond();
// Open output file
String filename = Execution.getFile("annotated-" + group + ".tsv");
String filename = Execution.getFile("tagged-" + group + ".tsv");
out = IOUtils.openOutHard(filename);
dump(FIELDS);
// Read LispTrees
@ -73,7 +73,8 @@ public class AnnotatedDatasetGenerator extends AnnotatedGenerator implements Run
private static final String[] FIELDS = new String[] {
"id", "utterance", "context", "targetValue",
"tokens", "lemmaTokens", "posTags", "nerTags", "nerValues", "targetCanon",
"tokens", "lemmaTokens", "posTags", "nerTags", "nerValues",
"targetCanon", "targetCanonType",
};
@Override
@ -99,58 +100,23 @@ public class AnnotatedDatasetGenerator extends AnnotatedGenerator implements Run
}
}
// Other information come from Example
fields[2] = serialize(((TableKnowledgeGraph) ex.context.graph).filename);
fields[2] = serialize(((TableKnowledgeGraph) ex.context.graph).filename.replace("lib/data/tables/", ""));
fields[4] = serialize(ex.languageInfo.tokens);
fields[5] = serialize(ex.languageInfo.lemmaTokens);
fields[6] = serialize(ex.languageInfo.posTags);
fields[7] = serialize(ex.languageInfo.nerTags);
fields[8] = serialize(ex.languageInfo.nerValues);
// Information from target value
fields[9] = serialize(ex.targetValue);
Value targetValue = ex.targetValue;
if (targetValue instanceof ListValue)
targetValue = ((ListValue) targetValue).values.get(0);
if (targetValue instanceof NumberValue)
fields[10] = "number";
else if (targetValue instanceof DateValue)
fields[10] = "date";
else
fields[10] = "string";
dump(fields);
}
// Helper Functions for TSV
public static void dump(PrintWriter out, String... stuff) {
out.println(String.join("\t", stuff));
}
public static String serialize(String x) {
if (x == null || x.isEmpty()) return "";
StringBuilder sb = new StringBuilder();
for (char y : x.toCharArray()) {
if (y == '\n') sb.append("\\n");
else if (y == '\\') sb.append("\\\\");
else if (y == '|') sb.append("\\|");
else sb.append(y);
}
return sb.toString().replaceAll("\\s", " ").trim();
}
public static String serialize(List<String> xs) {
List<String> serialized = new ArrayList<>();
for (String x : xs) serialized.add(serialize(x));
return String.join("|", serialized);
}
public static String serialize(Value value) {
if (value instanceof ListValue) {
List<String> xs = new ArrayList<>();
for (Value v : ((ListValue) value).values) {
xs.add(serialize(v));
}
return String.join("|", xs);
} else if (value instanceof DescriptionValue) {
return serialize(((DescriptionValue) value).value);
} else if (value instanceof NameValue) {
return serialize(((NameValue) value).description);
} else if (value instanceof NumberValue) {
return "" + ((NumberValue) value).value;
} else if (value instanceof DateValue) {
return ((DateValue) value).isoString();
} else {
throw new RuntimeException("Unknown value type: " + value);
}
}
}

10
src/edu/stanford/nlp/sempre/tables/serialize/AnnotatedTableGenerator.java → src/edu/stanford/nlp/sempre/tables/serialize/TaggedTableGenerator.java
View File

@ -14,7 +14,7 @@ import fig.basic.LogInfo;
import fig.exec.Execution;
/**
* Generate TSV files containing CoreNLP annotation of the tables.
* Generate TSV files containing CoreNLP tags of the tables.
*
* Mandatory fields:
* - row: row index (-1 is the header row)
@ -40,10 +40,10 @@ import fig.exec.Execution;
*
* @author ppasupat
*/
public class AnnotatedTableGenerator extends AnnotatedGenerator implements Runnable {
public class TaggedTableGenerator extends TSVGenerator implements Runnable {
public static void main(String[] args) {
Execution.run(args, "AnnotatedTableGeneratorMain", new AnnotatedTableGenerator(),
Execution.run(args, "TaggedTableGeneratorMain", new TaggedTableGenerator(),
Master.getOptionsParser());
}
@ -65,8 +65,8 @@ public class AnnotatedTableGenerator extends AnnotatedGenerator implements Runna
int batchIndex = Integer.parseInt(matcher.group(1)),
dataIndex = Integer.parseInt(matcher.group(2));
TableKnowledgeGraph table = TableKnowledgeGraph.fromFilename(baseDir.relativize(file).toString());
String outDir = Execution.getFile("annotated/" + batchIndex + "-annotated/"),
outFilename = new File(outDir, dataIndex + ".annotated").getPath();
String outDir = Execution.getFile("tagged/" + batchIndex + "-tagged/"),
outFilename = new File(outDir, dataIndex + ".tagged").getPath();
new File(outDir).mkdirs();
out = IOUtils.openOutHard(outFilename);
dumpTable(table);

2
tables/dump-parsers/convert-to-postfix.py
View File

@ -5,7 +5,7 @@
If the file is tab-separated, only process the first column.
"""
import sys, os, shutil, re, argparse, json
import sys, os, shutil, re, argparse, json, gzip
from codecs import open
from itertools import izip
from collections import defaultdict

10
tables/dump-parsers/write-simple-s2s-dataset.py
View File

@ -18,7 +18,7 @@ def get_formula(fpost):
formula, size = line[:-1].split('\t')
formulas.append([formula, int(size)])
if not formulas:
return None
return ''
# Choose a formula
best_size = min(x[1] for x in formulas)
best_formulas = [x[0] for x in formulas if x[1] == best_size]
@ -34,6 +34,8 @@ def main():
help='path to postfix directory')
parser.add_argument('-e', '--ex-id', action='store_true',
help='add example ID as the first column')
parser.add_argument('-n', '--no-skip', action='store_true',
help='do not skip examples without formula')
args = parser.parse_args()
# Read annotated data
@ -48,7 +50,9 @@ def main():
count = 0
with open(args.dataset_path, 'r', 'utf8') as fin:
header = fin.readline()[:-1].split('\t')
for line in fin:
for i, line in enumerate(fin):
if i % 500 == 0:
print >> sys.stderr, 'Processing Line', i, '...'
record = dict(zip(header, line[:-1].split('\t')))
record = annotated[record['id']]
ex_id = record['id']
@ -56,7 +60,7 @@ def main():
with gzip.open(os.path.join(args.postfix_dir, ex_id + '.gz')) as fpost:
random.seed(ex_id)
formula = get_formula(fpost)
if formula:
if formula or args.no_skip:
fields = [' '.join(question), formula]
if args.ex_id:
fields.insert(0, ex_id)