diff --git a/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java b/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java index 957a7fb..01b31ad 100644 --- a/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java +++ b/src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java @@ -665,18 +665,13 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl // ============================================================ public static void main(String[] args) { - //opts.baseCSVDir = "tables/toy-examples/random/"; - //String filename = "nikos_machlas.csv"; StringNormalizationUtils.opts.verbose = 5; - //LanguageAnalyzer.opts.languageAnalyzer = "corenlp.CoreNLPAnalyzer"; StringNormalizationUtils.opts.numberCanStartAnywhere = true; StringNormalizationUtils.opts.num2CanStartAnywhere = true; - opts.baseCSVDir = "lib/data/tables/"; + opts.baseCSVDir = "lib/data/WikiTableQuestions/"; String filename = "csv/200-csv/0.csv"; TableKnowledgeGraph graph = (TableKnowledgeGraph) KnowledgeGraph.fromLispTree( LispTree.proto.parseFromString("(graph tables.TableKnowledgeGraph " + filename + ")")); - //LogInfo.logs("%s", graph.toLispTree().toStringWrap()); - //LogInfo.logs("%s", graph.toTableValue().toLispTree().toStringWrap(100)); for (TableColumn column : graph.columns) { LogInfo.begin_track("%s (%s)", column.columnName, column.originalString); for (TableCell cell : column.children) { diff --git a/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java b/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java index 162de5b..3cbe95d 100644 --- a/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java +++ b/src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java @@ -150,11 +150,11 @@ public class LambdaDCSExecutorTest { "(fb:en.tampico fb:location.location.containedby fb:en.illinois)" + ")")); } else if ("csv".equals(name)) { - return TableKnowledgeGraph.fromFilename("tables/toy-examples/random/nikos_machlas.csv"); + return TableKnowledgeGraph.fromFilename("tables/toy-examples/nikos_machlas.csv"); } else if ("csv2".equals(name)) { - return TableKnowledgeGraph.fromFilename("lib/data/tables/csv/204-csv/495.tsv"); + return TableKnowledgeGraph.fromFilename("tables/toy-examples/204-495.tsv"); } else if ("csv3".equals(name)) { - return TableKnowledgeGraph.fromFilename("lib/data/tables/csv/203-csv/839.tsv"); + return TableKnowledgeGraph.fromFilename("tables/toy-examples/203-839.tsv"); } throw new RuntimeException("Unknown graph name: " + name); } @@ -230,9 +230,9 @@ public class LambdaDCSExecutorTest { @Test(groups = "lambdaCSV") public void lambdaOnGraphCSVTest() { KnowledgeGraph graph = getKnowledgeGraph("csv"); runFormula(executor, "(number 3)", graph, matches("(number 3)")); - runFormula(executor, "(!fb:row.row.score (fb:row.row.opponent fb:cell.austria))", - graph, matches("(name fb:cell.1_2)")); - runFormula(executor, "(count (fb:row.row.result fb:cell.win))", + runFormula(executor, "(!fb:row.row.score (fb:row.row.opponent fb:cell_opponent.austria))", + graph, matches("(name fb:cell_score.1_2)")); + runFormula(executor, "(count (fb:row.row.result fb:cell_result.win))", graph, matches("(number 16)")); // Depending on tie-breaking, one of these will be correct try { @@ -247,12 +247,12 @@ public class LambdaDCSExecutorTest { graph, matches("(number 2)")); } } - + @Test(groups = "lambdaCSV2") public void lambdaOnGraphCSV2Test() { KnowledgeGraph graph = getKnowledgeGraph("csv2"); runFormula(executor, - "(and (!= (and (!= fb:cell.away) fb:cell.home)) ((reverse fb:row.row.opponent) (fb:row.row.index (- (number 2) (number 1)))))", - graph, matches("(name fb:cell.derby_county)")); + "(and (!= (and (!= fb:cell_venue.away) fb:cell_venue.home)) ((reverse fb:row.row.opponent) (fb:row.row.index (- (number 2) (number 1)))))", + graph, matches("(name fb:cell_opponent.derby_county)")); } @Test(groups = "lambdaCSV3") public void lambdaOnGraphCSV3Test() { @@ -265,11 +265,9 @@ public class LambdaDCSExecutorTest { "(count (fb:row.row.opened (fb:cell.cell.date (< (date 1926 -1 -1)))))", graph, matches("(number 6)")); runFormula(executor, - "(sum (- (count ((reverse fb:row.row.index) (fb:type.object.type fb:type.row))) " + - "((reverse fb:row.row.index) (fb:row.row.latitude ((reverse fb:row.row.longitude) (fb:type.object.type fb:type.row))))))", - graph, matches("(number 3)")); - runFormula(executor, - "(- (number 1926) (argmax (number 1) (number 1) ((reverse fb:cell.cell.number) (or (or (or fb:cell.1920 fb:cell.1925) fb:cell.1926) fb:cell.1946)) (reverse (lambda x (sum ((reverse fb:cell.cell.number) (fb:cell.cell.number (var x))))))))", + "(- (number 1926) (argmax (number 1) (number 1) ((reverse fb:cell.cell.number) " + + "(or (or (or fb:cell_closed.1920 fb:cell_closed.1925) fb:cell_opened.1926) fb:cell_closed.1946)) " + + "(reverse (lambda x (sum ((reverse fb:cell.cell.number) (fb:cell.cell.number (var x))))))))", graph, matches("(number 6)")); } } diff --git a/src/edu/stanford/nlp/sempre/tables/serialize/SerializedDataset.java b/src/edu/stanford/nlp/sempre/tables/serialize/SerializedDataset.java index 11ef81e..9f75a38 100644 --- a/src/edu/stanford/nlp/sempre/tables/serialize/SerializedDataset.java +++ b/src/edu/stanford/nlp/sempre/tables/serialize/SerializedDataset.java @@ -110,7 +110,7 @@ public class SerializedDataset extends Dataset { // ============================================================ public static void main(String[] args) { - TableKnowledgeGraph.opts.baseCSVDir = "lib/data/tables/"; + TableKnowledgeGraph.opts.baseCSVDir = "lib/data/WikiTableQuestions/"; SerializedDataset dataset = new SerializedDataset(); dataset.readDir("out/sliced-dump-8-reps/representative-00-training-sliced-00000-00299/"); LazyLoadedExampleList examples = dataset.examples("representative"); diff --git a/src/edu/stanford/nlp/sempre/tables/serialize/TaggedDatasetGenerator.java b/src/edu/stanford/nlp/sempre/tables/serialize/TaggedDatasetGenerator.java index b4d37c6..360170b 100644 --- a/src/edu/stanford/nlp/sempre/tables/serialize/TaggedDatasetGenerator.java +++ b/src/edu/stanford/nlp/sempre/tables/serialize/TaggedDatasetGenerator.java @@ -100,7 +100,7 @@ public class TaggedDatasetGenerator extends TSVGenerator implements Runnable { } } // Other information come from Example - fields[2] = serialize(((TableKnowledgeGraph) ex.context.graph).filename.replace("lib/data/tables/", "")); + fields[2] = serialize(((TableKnowledgeGraph) ex.context.graph).filename.replace("lib/data/WikiTableQuestions/", "")); fields[4] = serialize(ex.languageInfo.tokens); fields[5] = serialize(ex.languageInfo.lemmaTokens); fields[6] = serialize(ex.languageInfo.posTags); diff --git a/tables/toy-examples/203-839.tsv b/tables/toy-examples/203-839.tsv new file mode 100644 index 0000000..5817cad --- /dev/null +++ b/tables/toy-examples/203-839.tsv @@ -0,0 +1,22 @@ +Station Type Latitude Longitude El. (m) Opened Closed Coordinates +Mount Wilson, California Solar 34º13'N 118º56'W 1737 1908 1920 34°13′S 118°56′W / 34.217°S 118.933°W +Hump Mountain, North Carolina Solar 36º8'N 82º0'W 1500 1917 1918 36°8′N 82°0′W / 36.133°N 82.000°W +Calama, Chile Solar 22º28'S 68º56'W 2250 1918 1920 22°28′S 68°56′W / 22.467°S 68.933°W +Mount Montezuma, Chile Solar 22º40'S 68º56'W 2711 1920 ? 22°40′S 68°56′W / 22.667°S 68.933°W +Mount Harquahala, Arizona Solar 33º48'N 113º20'W 1721 1920 1925 33°48′N 113°20′W / 33.800°N 113.333°W +Table Mountain, California Solar 34º22'N 117º41'W 2286 1925 1962 34°22′N 117°41′W / 34.367°N 117.683°W +Mount Brukkaros, Namibia Solar 25º52'S 17º48'E 1586 1926 1931 25°52′S 17°48′E / 25.867°S 17.800°E +Mount Saint Catherine, Egypt Solar 28º31'N 33º56'E 2591 1934 1937 28°31′N 33°56′E / 28.517°N 33.933°E +Burro Mountain, New Mexico Solar 32º40'N 108º33'W 2440 1938 1946 32°40′N 108°33′W / 32.667°N 108.550°W +Organ Pass, New Mexico Space Track 32º25'N 253º27'E 32°25′N 106°33′W / 32.417°N 106.550°W +Olifantsfontein, South Africa Space Track 25º58'S 28º15'E 25°58′S 28°15′E / 25.967°S 28.250°E +Woomera, Australia Space Track 31º06'S 136º46'E 31°06′S 136°46′E / 31.100°S 136.767°E +Cadiz, Spain Space Track 36º28'N 353º48'E 36°28′N 6°12′W / 36.467°N 6.200°W +Shiraz, Iran Space Track 29º38'N 52º31'E 29°38′N 52°31′E / 29.633°N 52.517°E +Curaçao, Netherlands West Indies Space Track 12º05'N 291º10'E 12°05′N 68°50′W / 12.083°N 68.833°W +Jupiter, Florida Space Track 27º01'N 279º53'E 27°01′N 80°07′W / 27.017°N 80.117°W +Haleakala, Hawaii Space Track 20º43'N 203º45'E 20°43′N 156°15′W / 20.717°N 156.250°W +Villa Dolores, Argentina Space Track 31º57'N 294º54'E 31°57′N 65°06′E / 31.950°N 65.100°E +Mitaka, Japan Space Track +Nani Tal, India Space Track +Arequipa, Peru Solar,\nSpace Track diff --git a/tables/toy-examples/204-495.tsv b/tables/toy-examples/204-495.tsv new file mode 100644 index 0000000..1f0b51a --- /dev/null +++ b/tables/toy-examples/204-495.tsv @@ -0,0 +1,41 @@ +Date Opponent Venue Result Attendance Scorers +15 August 1987 Derby County Away 0–1 17,204 — +18 August 1987 Coventry City Home 0–1 09,380 — +22 August 1987 West Ham United Home 2–2 08,073 Harford (2) +29 August 1987 Chelsea Away 0–3 16,075 — +31 August 1987 Arsenal Home 1–1 08,745 Wilson (pen) +5 September 1987 Oxford United Away 5–2 06,804 Breacker, Harford, Hill, Nwajiobi, B. Stein +12 September 1987 Everton Home 2–1 08,124 Hill, B. Stein +19 September 1987 Charlton Athletic Away 0–1 05,002 — +26 September 1987 Queens Park Rangers Away 0–2 11,175 — +3 October 1987 Manchester United Home 1–1 09,137 Harford +10 October 1987 Portsmouth Away 1–3 12,391 Harford (pen) +17 October 1987 Wimbledon Home 2–0 07,018 B. Stein, Wilson +24 October 1987 Liverpool Home 0–1 11,997 — +7 November 1987 Newcastle United Home 4–0 07,638 Nwajiobi, B. Stein, M. Stein (2) +14 November 1987 Sheffield Wednesday Away 2–0 16,960 Allinson, M. Stein +21 November 1987 Tottenham Hotspur Home 2–0 10,091 Allinson (2) +5 December 1987 Norwich City Home 1–2 07,002 B. Stein +12 December 1987 Watford Away 1–0 12,152 Foster +18 December 1987 Southampton Home 2–2 06,618 Harford, McDonough +26 December 1987 Everton Away 0–2 32,128 — +28 December 1987 Charlton Athletic Home 1–0 07,243 Wilson +1 January 1988 Chelsea Home 3–0 08,018 Harford, B. Stein, M. Stein +2 January 1988 West Ham United Away 1–1 16,716 M. Stein +16 January 1988 Derby County Home 1–0 07,175 McDonough +6 February 1988 Oxford United Home 7–4 08,063 Harford (2), McDonough, B.Stein, M.Stein (3) +13 February 1988 Arsenal Away 1–2 22,612 M.Stein +5 March 1988 Wimbledon Away 0–2 04,854 — +15 March 1988 Coventry City Away 0–4 13,711 — +29 March 1988 Portsmouth Home 4–1 06,740 B.Stein, M.Stein, Wilson, own goal +2 April 1988 Newcastle United Away 0–4 20,752 — +5 April 1988 Sheffield Wednesday Home 2–2 07,337 McDonough, B. Stein +12 April 1988 Manchester United Away 0–3 28,830 — +19 April 1988 Queens Park Rangers Home 2–1 06,735 Foster, Wilson (pen) +30 April 1988 Norwich City Away 2–2 13,171 M. Stein, Wilson (pen) +2 May 1988 Watford Home 2–1 10,409 Oldfield, Wilson (pen) +4 May 1988 Tottenham Hotspur Away 1–2 15,437 Grimes +7 May 1988 Southampton Away 1–1 12,722 Wilson +9 May 1988 Liverpool Away 1–1 30,374 Oldfield +13 May 1988 Nottingham Forest Home 1–1 09,108 Donaghy +15 May 1988 Nottingham Forest Away 1–1 13,106 Oldfield diff --git a/tables/toy-examples/random/nikos_machlas.csv b/tables/toy-examples/nikos_machlas.csv similarity index 100% rename from tables/toy-examples/random/nikos_machlas.csv rename to tables/toy-examples/nikos_machlas.csv