From 2a5365ef69ecbaedbcf53f6bf387e5260d337ac8 Mon Sep 17 00:00:00 2001 From: Sida Wang Date: Tue, 28 Mar 2017 10:13:05 -0700 Subject: [PATCH] rename, README, refactor master --- README.md | 8 +++-- {shrdlurn => interactive}/DAL.grammar | 0 {shrdlurn => interactive}/README.md | 0 .../analyze_definitions.ipynb | 0 {shrdlurn => interactive}/analyzer.py | 0 .../community-server/install-deps.py | 0 .../community-server/requirements.txt | 0 .../community-server/server.py | 0 {shrdlurn => interactive}/csscolors.lexicon | 0 .../generate_plots.ipynb | 0 {shrdlurn => interactive}/queries/README.md | 0 .../queries/freebuild1-0121.json.gz | Bin .../queries/freebuild2-0126.json.gz | Bin .../queries/freebuild2-0127.json.gz | Bin .../queries/freebuildbig-0206.def.json.gz | Bin .../queries/freebuildbig-0206.json.gz | Bin .../queries/qualifier1-0118.json.gz | Bin .../queries/qualifier2-0129.json.gz | Bin .../queries/qualifier3-0201.json.gz | Bin {shrdlurn => interactive}/queries/sidaw.json | 0 {shrdlurn => interactive}/run | 29 +++++++++--------- {shrdlurn => interactive}/testng.xml | 0 {shrdlurn => interactive}/voxelurn.grammar | 2 +- src/edu/stanford/nlp/sempre/Derivation.java | 6 ++-- src/edu/stanford/nlp/sempre/Main.java | 28 ++++++++++------- .../sempre/interactive/InteractiveServer.java | 4 +-- .../nlp/sempre/interactive/Simulator.java | 2 +- .../interactive/test/FloatingParsingTest.java | 2 +- .../interactive/test/GrammarInducerTest.java | 2 +- 29 files changed, 47 insertions(+), 36 deletions(-) rename {shrdlurn => interactive}/DAL.grammar (100%) rename {shrdlurn => interactive}/README.md (100%) rename {shrdlurn => interactive}/analyze_definitions.ipynb (100%) rename {shrdlurn => interactive}/analyzer.py (100%) rename {shrdlurn => interactive}/community-server/install-deps.py (100%) rename {shrdlurn => interactive}/community-server/requirements.txt (100%) rename {shrdlurn => interactive}/community-server/server.py (100%) rename {shrdlurn => interactive}/csscolors.lexicon (100%) rename {shrdlurn => interactive}/generate_plots.ipynb (100%) rename {shrdlurn => interactive}/queries/README.md (100%) rename {shrdlurn => interactive}/queries/freebuild1-0121.json.gz (100%) rename {shrdlurn => interactive}/queries/freebuild2-0126.json.gz (100%) rename {shrdlurn => interactive}/queries/freebuild2-0127.json.gz (100%) rename {shrdlurn => interactive}/queries/freebuildbig-0206.def.json.gz (100%) rename {shrdlurn => interactive}/queries/freebuildbig-0206.json.gz (100%) rename {shrdlurn => interactive}/queries/qualifier1-0118.json.gz (100%) rename {shrdlurn => interactive}/queries/qualifier2-0129.json.gz (100%) rename {shrdlurn => interactive}/queries/qualifier3-0201.json.gz (100%) rename {shrdlurn => interactive}/queries/sidaw.json (100%) rename {shrdlurn => interactive}/run (88%) rename {shrdlurn => interactive}/testng.xml (100%) rename {shrdlurn => interactive}/voxelurn.grammar (99%) diff --git a/README.md b/README.md index b84423f..fe01193 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ similar your system is. ./run @mode=simple You should be able to type the following into the shell and get the answer `(number 7)`: - + (execute (call + (number 3) (number 4))) To go further, check out the [tutorial](TUTORIAL.md) and then the [full @@ -133,7 +133,7 @@ For Ubuntu, follow this: make install cd .. -on OS/X you can install virtuoso using homebrew by following the instructions +on OS/X you can install virtuoso using homebrew by following the instructions [here](http://carsten.io/virtuoso-os-on-mac-os/) To have SEMPRE interact with Virtuoso, the required modules need to be compiled as follow: @@ -166,3 +166,7 @@ Changes from SEMPRE 2.0 to SEMPRE 2.1: - Added the `tables` package for the paper *Compositional semantic parsing on semi-structured tables* (ACL 2015). - Add and `overnight` package for the paper *Building a semantic parser overnight* (ACL 2015). + +Changes from SEMPRE 2.1 to SEMPRE 2.2: + +- Added the `interactive` package for the paper *Naturalizing a programming language through interaction* (ACL 2017). diff --git a/shrdlurn/DAL.grammar b/interactive/DAL.grammar similarity index 100% rename from shrdlurn/DAL.grammar rename to interactive/DAL.grammar diff --git a/shrdlurn/README.md b/interactive/README.md similarity index 100% rename from shrdlurn/README.md rename to interactive/README.md diff --git a/shrdlurn/analyze_definitions.ipynb b/interactive/analyze_definitions.ipynb similarity index 100% rename from shrdlurn/analyze_definitions.ipynb rename to interactive/analyze_definitions.ipynb diff --git a/shrdlurn/analyzer.py b/interactive/analyzer.py similarity index 100% rename from shrdlurn/analyzer.py rename to interactive/analyzer.py diff --git a/shrdlurn/community-server/install-deps.py b/interactive/community-server/install-deps.py similarity index 100% rename from shrdlurn/community-server/install-deps.py rename to interactive/community-server/install-deps.py diff --git a/shrdlurn/community-server/requirements.txt b/interactive/community-server/requirements.txt similarity index 100% rename from shrdlurn/community-server/requirements.txt rename to interactive/community-server/requirements.txt diff --git a/shrdlurn/community-server/server.py b/interactive/community-server/server.py similarity index 100% rename from shrdlurn/community-server/server.py rename to interactive/community-server/server.py diff --git a/shrdlurn/csscolors.lexicon b/interactive/csscolors.lexicon similarity index 100% rename from shrdlurn/csscolors.lexicon rename to interactive/csscolors.lexicon diff --git a/shrdlurn/generate_plots.ipynb b/interactive/generate_plots.ipynb similarity index 100% rename from shrdlurn/generate_plots.ipynb rename to interactive/generate_plots.ipynb diff --git a/shrdlurn/queries/README.md b/interactive/queries/README.md similarity index 100% rename from shrdlurn/queries/README.md rename to interactive/queries/README.md diff --git a/shrdlurn/queries/freebuild1-0121.json.gz b/interactive/queries/freebuild1-0121.json.gz similarity index 100% rename from shrdlurn/queries/freebuild1-0121.json.gz rename to interactive/queries/freebuild1-0121.json.gz diff --git a/shrdlurn/queries/freebuild2-0126.json.gz b/interactive/queries/freebuild2-0126.json.gz similarity index 100% rename from shrdlurn/queries/freebuild2-0126.json.gz rename to interactive/queries/freebuild2-0126.json.gz diff --git a/shrdlurn/queries/freebuild2-0127.json.gz b/interactive/queries/freebuild2-0127.json.gz similarity index 100% rename from shrdlurn/queries/freebuild2-0127.json.gz rename to interactive/queries/freebuild2-0127.json.gz diff --git a/shrdlurn/queries/freebuildbig-0206.def.json.gz b/interactive/queries/freebuildbig-0206.def.json.gz similarity index 100% rename from shrdlurn/queries/freebuildbig-0206.def.json.gz rename to interactive/queries/freebuildbig-0206.def.json.gz diff --git a/shrdlurn/queries/freebuildbig-0206.json.gz b/interactive/queries/freebuildbig-0206.json.gz similarity index 100% rename from shrdlurn/queries/freebuildbig-0206.json.gz rename to interactive/queries/freebuildbig-0206.json.gz diff --git a/shrdlurn/queries/qualifier1-0118.json.gz b/interactive/queries/qualifier1-0118.json.gz similarity index 100% rename from shrdlurn/queries/qualifier1-0118.json.gz rename to interactive/queries/qualifier1-0118.json.gz diff --git a/shrdlurn/queries/qualifier2-0129.json.gz b/interactive/queries/qualifier2-0129.json.gz similarity index 100% rename from shrdlurn/queries/qualifier2-0129.json.gz rename to interactive/queries/qualifier2-0129.json.gz diff --git a/shrdlurn/queries/qualifier3-0201.json.gz b/interactive/queries/qualifier3-0201.json.gz similarity index 100% rename from shrdlurn/queries/qualifier3-0201.json.gz rename to interactive/queries/qualifier3-0201.json.gz diff --git a/shrdlurn/queries/sidaw.json b/interactive/queries/sidaw.json similarity index 100% rename from shrdlurn/queries/sidaw.json rename to interactive/queries/sidaw.json diff --git a/shrdlurn/run b/interactive/run similarity index 88% rename from shrdlurn/run rename to interactive/run index 80a90f3..1caa4d2 100755 --- a/shrdlurn/run +++ b/interactive/run @@ -7,6 +7,7 @@ $: << 'fig/lib' require 'execrunner' +$path = 'interactive' $modes = [] def addMode(name, description, func) @@ -94,18 +95,18 @@ addMode('test', 'Run unit tests for interactive stuff', lambda { |e| if e[:class] l('-testclass', 'edu.stanford.nlp.sempre.interactive.test.' + e[:class]) else - 'shrdlurn/testng.xml' + "./#{$path}/testng.xml" end }, nil) }) addMode('community-dep', 'dependencies for the community server', lambda { |e| l( -l('python ./shrdlurn/community-server/install-deps.py'), +l('python ./#{$path}/community-server/install-deps.py'), nil)}) addMode('community', 'start the community server', lambda { |e| l( -l('python ./shrdlurn/community-server/server.py -- port 8403'), +l('python ./#{$path}/community-server/server.py -- port 8403'), nil)}) addMode('simulator', 'run the simulator', lambda { |e| l( @@ -130,16 +131,16 @@ addMode('simulator', 'run the simulator', lambda { |e| l( }), letDefault(:task, 'sidaw'), sel(:task, { - 'freebig' => o('logFiles', './shrdlurn/queries/freebuildbig-0206.json.gz'), - 'freebigdef' => o('logFiles', './shrdlurn/queries/freebuildbig-0206.def.json.gz'), - 'freebigcontext' => o('logFiles', './shrdlurn/queries/freebuildbig-0206.context.json.gz'), + 'freebig' => o('logFiles', "./#{$path}/queries/freebuildbig-0206.json.gz"), + 'freebigdef' => o('logFiles', "./#{$path}/queries/freebuildbig-0206.def.json.gz"), + 'freebigcontext' => o('logFiles', "./#{$path}/queries/freebuildbig-0206.context.json.gz"), - 'qual1' => o('logFiles', './shrdlurn/queries/qualifier1-0118.json.gz'), - 'qual2' => o('logFiles', './shrdlurn/queries/qualifier2-0129.json.gz'), - 'qual3' => o('logFiles', './shrdlurn/queries/qualifier3-0201.json.gz'), # both 2 and 3 - 'free1' => o('logFiles', './shrdlurn/queries/freebuild1-0121.json.gz'), - 'free2' => o('logFiles', './shrdlurn/queries/freebuild2-0127.json.gz'), - 'sidaw' => o('logFiles', './shrdlurn/queries/sidaw.json'), + 'qual1' => o('logFiles', "./#{$path}/queries/qualifier1-0118.json.gz"), + 'qual2' => o('logFiles', "./#{$path}/queries/qualifier2-0129.json.gz"), + 'qual3' => o('logFiles', "./#{$path}/queries/qualifier3-0201.json.gz"), # both 2 and 3 + 'free1' => o('logFiles', "./#{$path}/queries/freebuild1-0121.json.gz"), + 'free2' => o('logFiles', "./#{$path}/queries/freebuild2-0127.json.gz"), + 'sidaw' => o('logFiles', "./#{$path}/queries/sidaw.json"), }), nil)}) @@ -159,7 +160,7 @@ addMode('voxelurn', 'interactive semantic parsing in a VoxelWorld', lambda { |e| o('DALExecutor.printStackTrace', true), o('VoxelWorld.maxBlocks', 100000), selo(0, 'DALExecutor.worldType', 'VoxelWorld', 'CalendarWorld', 'Otherworld'), - selo(0, 'Grammar.inPaths', './shrdlurn/voxelurn.grammar', './shrdlurn/calendar.grammar'), + selo(0, 'Grammar.inPaths', "./#{$path}/voxelurn.grammar"), o('Params.initWeightsRandomly', false), o('Grammar.binarizeRules', false), @@ -213,7 +214,7 @@ addMode('voxelurn', 'interactive semantic parsing in a VoxelWorld', lambda { |e| o('Derivation.showRules', false), o('NumberFn.allowedRange', 0, 100), - o('SimpleLexicon.inPaths', 'shrdlurn/csscolors.lexicon'), + o('SimpleLexicon.inPaths', "./#{$path}/csscolors.lexicon"), # o('Grammar.tags', 'morecolors'), # o('Grammar.tags', 'morenumbers', 'morecolors'), diff --git a/shrdlurn/testng.xml b/interactive/testng.xml similarity index 100% rename from shrdlurn/testng.xml rename to interactive/testng.xml diff --git a/shrdlurn/voxelurn.grammar b/interactive/voxelurn.grammar similarity index 99% rename from shrdlurn/voxelurn.grammar rename to interactive/voxelurn.grammar index 04339ef..c5c1792 100644 --- a/shrdlurn/voxelurn.grammar +++ b/interactive/voxelurn.grammar @@ -1,6 +1,6 @@ # domain specific grammar for voxelurn -(include DAL.grammar) +(include dal.grammar) # domain specific actions: (def @put put) diff --git a/src/edu/stanford/nlp/sempre/Derivation.java b/src/edu/stanford/nlp/sempre/Derivation.java index 58cd306..c3ca601 100644 --- a/src/edu/stanford/nlp/sempre/Derivation.java +++ b/src/edu/stanford/nlp/sempre/Derivation.java @@ -47,7 +47,7 @@ public class Derivation implements SemanticFn.Callable, HasScore { /** * Information for grammar induction. - * For each descendant derivation of the body, this class tracks where in the head it matches + * For each descendant derivation of the body, this class tracks where and what in the head it matches * GrammarInfo.start, GrammarInfo.end refer to matching positions in the head, as opposed to the body * @author sidaw **/ @@ -81,8 +81,8 @@ public class Derivation implements SemanticFn.Callable, HasScore { // Information for scoring private final FeatureVector localFeatureVector; // Features - public double score = Double.NaN; // Weighted combination of features - public double prob = Double.NaN; // Probability (normalized exp of score). + double score = Double.NaN; // Weighted combination of features + double prob = Double.NaN; // Probability (normalized exp of score). // Used during parsing (by FeatureExtractor, SemanticFn) to cache arbitrary // computation across different sub-Derivations. diff --git a/src/edu/stanford/nlp/sempre/Main.java b/src/edu/stanford/nlp/sempre/Main.java index eaca2b2..4964b1e 100644 --- a/src/edu/stanford/nlp/sempre/Main.java +++ b/src/edu/stanford/nlp/sempre/Main.java @@ -2,6 +2,7 @@ package edu.stanford.nlp.sempre; import java.lang.reflect.Constructor; + import fig.basic.Option; import fig.exec.Execution; @@ -26,20 +27,25 @@ public class Main implements Runnable { learner.learn(); if (server || interactive) { - try { - Class masterClass = Class.forName(masterType); - Constructor constructor = masterClass.getConstructor(Builder.class); - Master master = (Master)constructor.newInstance(builder); - if (server) - master.runServer(); - else if (interactive) - master.runInteractivePrompt(); - } catch (Throwable t) { - t.printStackTrace(); - } + Master master = createMaster(masterType, builder); + if (server) + master.runServer(); + else if (interactive) + master.runInteractivePrompt(); } } + public Master createMaster(String masterType, Builder builder) { + try { + Class masterClass = Class.forName(masterType); + Constructor constructor = masterClass.getConstructor(Builder.class); + return (Master)constructor.newInstance(builder); + } catch (Throwable t) { + t.printStackTrace(); + } + return null; + } + public static void main(String[] args) { Execution.run(args, "Main", new Main(), Master.getOptionsParser()); } diff --git a/src/edu/stanford/nlp/sempre/interactive/InteractiveServer.java b/src/edu/stanford/nlp/sempre/interactive/InteractiveServer.java index 9618438..cd8f0c8 100644 --- a/src/edu/stanford/nlp/sempre/interactive/InteractiveServer.java +++ b/src/edu/stanford/nlp/sempre/interactive/InteractiveServer.java @@ -194,8 +194,8 @@ public class InteractiveServer { item.put("value", value.sortString()); else item.put("value", "[[]]"); - item.put("score", deriv.score); - item.put("prob", deriv.prob); + item.put("score", deriv.getScore()); + item.put("prob", deriv.getProb()); item.put("anchored", deriv.allAnchored); // used only anchored rules item.put("formula", deriv.formula.toLispTree().toString()); items.add(item); diff --git a/src/edu/stanford/nlp/sempre/interactive/Simulator.java b/src/edu/stanford/nlp/sempre/interactive/Simulator.java index adc412e..73e0263 100644 --- a/src/edu/stanford/nlp/sempre/interactive/Simulator.java +++ b/src/edu/stanford/nlp/sempre/interactive/Simulator.java @@ -92,7 +92,7 @@ public class Simulator implements Runnable { @Option public static String reqParams = "grammar=0&cite=0&learn=0"; @Option - public static List logFiles = Lists.newArrayList("./shrdlurn/commandInputs/sidaw.json.log"); + public static List logFiles = null; public void readQueries() { // T.printAllRules(); diff --git a/src/edu/stanford/nlp/sempre/interactive/test/FloatingParsingTest.java b/src/edu/stanford/nlp/sempre/interactive/test/FloatingParsingTest.java index 12e9260..c012931 100644 --- a/src/edu/stanford/nlp/sempre/interactive/test/FloatingParsingTest.java +++ b/src/edu/stanford/nlp/sempre/interactive/test/FloatingParsingTest.java @@ -54,7 +54,7 @@ public class FloatingParsingTest { DALExecutor.opts.convertNumberValues = true; DALExecutor.opts.printStackTrace = true; DALExecutor.opts.worldType = "VoxelWorld"; - Grammar.opts.inPaths = Lists.newArrayList("./shrdlurn/voxelurn.grammar"); + Grammar.opts.inPaths = Lists.newArrayList("./interactive/voxelurn.grammar"); Grammar.opts.useApplyFn = "interactive.ApplyFn"; Grammar.opts.binarizeRules = false; diff --git a/src/edu/stanford/nlp/sempre/interactive/test/GrammarInducerTest.java b/src/edu/stanford/nlp/sempre/interactive/test/GrammarInducerTest.java index 26ea937..f196ad0 100644 --- a/src/edu/stanford/nlp/sempre/interactive/test/GrammarInducerTest.java +++ b/src/edu/stanford/nlp/sempre/interactive/test/GrammarInducerTest.java @@ -54,7 +54,7 @@ public class GrammarInducerTest { Derivation.opts.showCat = true; LanguageAnalyzer.opts.languageAnalyzer = "interactive.DALAnalyzer"; - Grammar.opts.inPaths = Lists.newArrayList("./shrdlurn/voxelurn.grammar"); + Grammar.opts.inPaths = Lists.newArrayList("./interactive/voxelurn.grammar"); Grammar.opts.useApplyFn = "interactive.ApplyFn"; Grammar.opts.binarizeRules = false;