diff --git a/README.md b/README.md
index aa73ffe..6adfb22 100644
--- a/README.md
+++ b/README.md
@@ -99,7 +99,7 @@ similar your system is.
 
 3. Compile the source code (this produces `libsempre/sempre-core.jar`):
 
-        make core
+        ant core
 
 4. Run an interactive shell:
 
@@ -146,6 +146,11 @@ Changes from SEMPRE 1.0 to SEMPRE 2.0:
 - Removed fbalignment (EMNLP 2013) and paraphrase (ACL 2014) components to
   avoid confusion.  If you want to reproduce those systems, use SEMPRE 1.0.
 
+Changes from SEMPRE 2.0 to SEMPRE 2.1:
+
+- Added the `tables` package for the paper *Compositional semantic parsing on semi-structured tables* (ACL 2015).
+- Add and `overnight` package for the paper *Building a semantic parser overnight* (ACL 2015).
+
 # License
 
 SEMPRE is licensed under the [GNU General Public
diff --git a/TUTORIAL.md b/TUTORIAL.md
index b9c27ad..7229299 100644
--- a/TUTORIAL.md
+++ b/TUTORIAL.md
@@ -396,7 +396,7 @@ etc.):
 
 Compile it:
 
-    make corenlp
+    ant corenlp
 
 Now we can load the SEMPRE interactive shell with `CoreNLPAnalyzer`:
 
diff --git a/overnight/README.md b/overnight/README.md
index 00625a1..0accd50 100644
--- a/overnight/README.md
+++ b/overnight/README.md
@@ -1,25 +1,40 @@
 # Files for Building a Semantic Parser Overnight
 
-## Setting up datasets
+## Generating canonical utterances
 
-To generate canonical utterances, run
+To generate all the canonical utterances, run:
 
       ./pull-dependencies freebase overnight
-      make corenlp freebase overnight
+      ant corenlp freebase overnight
       ./run @mode=genovernight-wrapper
 
-Or, to generate each individual domain
+To generate each individual domain:
+
       ./run @mode=genovernight @gen=1 @domain=<domain>
 
-After turking and setting up the approprate example files, run
+## Training
+
+After generating the paraphrases via AMT and setting up the approprate example
+files, we train a model.
+
+Run the following to train with all the features:
 
       ./run @mode=overnight @domain=<domain>
 
-to run with all features.
+To run with a subset of the features for ablation studies:
 
-To run with partial features, use the following commands:
+Baseline:
 
-      Baseline - ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match skip-bigram root lf simpleworld
-      No Lexical features - ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match ppdb skip-bigram root lf simpleworld
-      No PPDB - ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match skip-bigram root lf alignment lexical root_lexical simpleworld
-      Full system - ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match ppdb skip-bigram root alignment lexical root_lexical lf simpleworld
+    ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match skip-bigram root lf simpleworld
+
+No Lexical features:
+
+    ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match ppdb skip-bigram root lf simpleworld
+
+No PPDB features:
+
+    ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match skip-bigram root lf alignment lexical root_lexical simpleworld
+
+Full system:
+
+    ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match ppdb skip-bigram root alignment lexical root_lexical lf simpleworld
diff --git a/run b/run
index 3ac68cf..75a1bcd 100755
--- a/run
+++ b/run
@@ -31,7 +31,8 @@ def header(modules='core', codalabDependencies=nil)
     # Queuing system
     letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')),
     # Create execution directory
-    'fig/bin/qcreate',
+    letDefault(:pooldir, 1),
+    sel(:pooldir, l(), 'fig/bin/qcreate'),
     # Run the Java command...
     'java',
     '-ea',
@@ -63,7 +64,7 @@ def balancedTrainDevSplit
   l(o('Dataset.trainFrac', 0.5), o('Dataset.devFrac', 0.5))
 end
 
-def figOpts; l(o('execDir', '_OUTPATH_'), o('overwriteExecDir'), o('addToView', 0)) end
+def figOpts; l(selo(:pooldir, 'execDir', 'exec', '_OUTPATH_'), o('overwriteExecDir'), o('addToView', 0)) end
 
 ############################################################
 # Unit tests
@@ -143,8 +144,11 @@ def freebaseOpts
       o('BinaryLexicon.binaryLexiconFilesPath', 'lib/fb_data/7/binaryInfoStringAndAlignment.txt'),
       o('BinaryLexicon.keyToSortBy', 'Intersection_size_typed'),
     nil),
-    o('UnaryLexicon.unaryLexiconFilePath', 'lib/fb_data/7/unaryInfoStringAndAlignment.txt'), # unary
-    o('EntityLexicon.entityPopularityPath', 'lib/fb_data/7/entityPopularity.txt'), # entity
+    o('UnaryLexicon.unaryLexiconFilePath','lib/fb_data/7/unaryInfoStringAndAlignment.txt'), # unary
+    o('EntityLexicon.entityPopularityPath','lib/fb_data/7/entityPopularity.txt'), # entity
+    #Jonathan - added this 3/5/2015
+    o('TypeInference.typeLookup','freebase.FreebaseTypeLookup'),
+    o('FreebaseSearch.cachePath', '/u/nlp/data/semparse/scr/cache/fbsearch/1.cache'),
   nil)
 end
 
@@ -263,7 +267,7 @@ def webquestions
     letDefault(:entitysearch, 1),
     sel(:entitysearch,  # Used for EMNLP 2013
       l(
-        cachePaths('lucene/0.cache', 'sparql/1.cache'),
+        cachePaths('lucene/0.cache', 'sparql/3.cache'),
         o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'),
         o('LexiconFn.maxEntityEntries',10),
         o('Grammar.tags', 'webquestions', 'bridge', 'join', 'inject','inexact'), # specify also strategy
@@ -285,6 +289,14 @@ addMode('freebase', 'Freebase (for EMNLP 2013, ACL 2014, TACL 2014)', lambda { |
   letDefault(:train, 0),
   letDefault(:interact, 0),
 
+  # nlpsub: for running commands on PBS
+  letDefault(:nlpsub, 0),
+  sel(:nlpsub,
+    l(),
+    l('nlpsub', '-d/scr/yonatan/sandbox/blackhole', '-nyonatan', '-c3'),
+    l('nlpsub', '-d/scr/yonatan/sandbox/blackhole', '-nyonatan', '-qjag', '-c3'),
+    l('nlpsub', '-d/scr/yonatan/sandbox/blackhole', '-nyonatan', '-qjohn', '-c3'),
+  nil),
   sel(:interact, l(), rlwrap),
   freebaseHeader,
   'edu.stanford.nlp.sempre.Main',
@@ -334,7 +346,7 @@ addMode('cacheserver', 'Start the general-purpose cache server that serves files
     letDefault(:port, 4000),
     lambda { |e| o('port', e[:port]) },
 
-    letDefault(:cachetype, 1),
+    letDefault(:cachetype, 0),
     sel(:cachetype,
       l(
         o('FileStringCache.appendMode'),
@@ -479,8 +491,22 @@ nil) })
 
 ############################################################
 # {2014-12-27} [Percy]: Overnight semantic parsing
-addMode('overnight', 'Overnight semantic parsing', lambda { |e| l(
-  'rlwrap',
+def overnightFeatureDomains
+  [
+    'match',
+    'ppdb',
+    'skip-bigram',
+    'root',
+    'alignment',
+    'lexical',
+    'root_lexical',
+    'lf',
+    'simpleworld',
+  nil].compact
+end
+
+addMode('overnight', 'Overnight semantic parsing', l(
+  rlwrap,
   header('core,freebase,overnight'),
   'edu.stanford.nlp.sempre.Main',
   figOpts,
@@ -494,7 +520,6 @@ addMode('overnight', 'Overnight semantic parsing', lambda { |e| l(
   o('Learner.maxTrainIters', 1),
   #o('printAllPredictions'),
   o('Derivation.showUtterance'),
-  letDefault(:domain, 'publications'),
   letDefault(:debug, 0),
 
   selo(1, 'maxExamples', 'train:10', 'train:MAX'),
@@ -521,46 +546,21 @@ addMode('overnight', 'Overnight semantic parsing', lambda { |e| l(
   o('Grammar.inPaths', lambda { |e| 'overnight/' + e[:domain] + '.grammar' }),
   o('SimpleWorld.domain', lambda { |e| e[:domain] }),
   o('PPDBModel.ppdbModelPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '-ppdb.txt' }),
-  #o('PPDBModel.ppdbModelPath', lambda { |e| 'overnightData/' + e[:domain] + '-ppdb.txt' }), # codalab
   o('Dataset.trainFrac', 0.8), o('Dataset.devFrac', 0.2),
   o('FloatingParser.maxDepth', 11),
   o('Parser.beamSize', 20),
-  letDefault(:alignment,1),
+  letDefault(:alignment, 1),
   sel(:alignment, 
-    o('wordAlignmentPath', 'lib/data/overnight/' + e[:domain] + '.word_alignments.heuristic'), 
-    o('wordAlignmentPath', 'lib/data/overnight/' + e[:domain] + '.word_alignments.berkeley'),
-    #o('wordAlignmentPath', 'overnightData/' + e[:domain] + '.word_alignments.heuristic'), # codalab
-    #o('wordAlignmentPath', 'overnightData/' + e[:domain] + '.word_alignments.berkeley'),
+    o('wordAlignmentPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '.word_alignments.heuristic' }), 
+    o('wordAlignmentPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '.word_alignments.berkeley' }),
   nil),
-  o('phraseAlignmentPath', 'lib/data/overnight/' + e[:domain] + '.phrase_alignments'), 
-  o('PPDBModel.ppdbModelPath', 'lib/data/overnight/' + e[:domain] + '-ppdb.txt'),
-  #o('phraseAlignmentPath', 'overnightData/' + e[:domain] + '.phrase_alignments'), # codalab
-  #o('PPDBModel.ppdbModelPath', 'overnightData/' + e[:domain] + '-ppdb.txt'),
+  o('phraseAlignmentPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '.phrase_alignments' }),
+  o('PPDBModel.ppdbModelPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '-ppdb.txt' }),
   o('DerivationPruner.pruningComputers', ['overnight.OvernightDerivationPruningComputer']),
   o('Dataset.inPaths', 
-    'train:lib/data/overnight/' + e[:domain] + '.paraphrases.train.examples',
-    'test:lib/data/overnight/' + e[:domain] + '.paraphrases.test.examples'),
-    #'train:overnightData/' + e[:domain] + '.paraphrases.train.examples', # codalab
-    #'test:overnightData/' + e[:domain] + '.paraphrases.test.examples'),
+    lambda { |e| 'train:lib/data/overnight/' + e[:domain] + '.paraphrases.train.examples' },
+    lambda { |e| 'test:lib/data/overnight/' + e[:domain] + '.paraphrases.test.examples' }),
   sel(:domain, {
-    'regex' => l(
-      letDefault(:data,0),
-      sel(:data,
-        l(o('Dataset.inPaths', 'train:lib/data/overnight/regex.paraphrases.train.examples')),
-        l(o('Dataset.inPaths', 'train:lib/data/overnight/regex.paraphrases.train.examples', 'test:lib/data/overnight/regex.examples.train.json')),
-        l(o('Dataset.inPaths', 'train:lib/data/overnight/regex.paraphrases.train.examples', 'test:lib/data/overnight/regex.examples.test.json')),
-      nil),
-      # selo(:debug, 'Dataset.inPaths', 'train:lib/data/overnight/regex.paraphrases.train.examples', 'train:regex/regex-debug.examples'),
-      o('executor', 'JavaExecutor'),
-      o('Builder.valueEvaluator', 'regex.RegexValueEvaluator'),
-      o('OvernightFeatureComputer.approxEmptyDenotation', 'true'),
-      # o('OvernightFeatureComputer.featureDomains', *regexFeatureDomains),
-      o('JavaExecutor.convertNumberValues', true),
-      o('Grammar.tags','generate','regex'),
-      o('lowerCaseTokens', false),  # case sensitivity is important for correctness of regular expressions
-      o('initialization', 'paraphrase :: match,1', 'paraphrase :: size,-0.1', 'paraphrase :: ppdb,0.3',
-        'paraphrase :: skip-bigram,0.8', 'paraphrase :: skip-ppdb,0.2','denotation :: error,-1000'),
-    nil),
     'geo880' => l(
       letDefault(:data,0),
       sel(:data,
@@ -577,8 +577,6 @@ addMode('overnight', 'Overnight semantic parsing', lambda { |e| l(
       o('FloatingParser.maxDepth', 11),
       o('Grammar.tags','generate','general', 'geo880'),
       o('SimpleLexicon.inPaths', 'lib/data/overnight/geo880.lexicon'),
-      #o('Params.l1Reg','lazy'),
-      #o('Params.l1RegCoeff',0.00001),
       o('OvernightDerivationPruningComputer.usePredicateDict'),
       o('OvernightDerivationPruningComputer.predicateDictPath','lib/data/overnight/geo880.predicate.dict'),
     nil),
@@ -612,21 +610,7 @@ addMode('overnight', 'Overnight semantic parsing', lambda { |e| l(
       o('Grammar.tags','generate','general'),
     nil),
   }),
-nil) })
-
-def overnightFeatureDomains
-  [
-    'match',
-    'ppdb',
-    'skip-bigram',
-    'root',
-    'alignment',
-    'lexical',
-    'root_lexical',
-    'lf',
-    'simpleworld',
-  nil].compact
-end
+nil))
 
 ############################################################
 # {5/27/15} [Ice]
@@ -855,6 +839,69 @@ end
 
 
 
+# {2015-01-18} Generate utterances [Percy]
+addMode('genovernight', 'Generate utterances for overnight semantic parsing', lambda { |e| l(
+  'fig/bin/qcreate',
+  letDefault(:gen, 0),
+  sel(:gen, rlwrap, l()),
+  'java', '-Dmodules=core,overnight', '-Xmx10g', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.overnight.GenerationMain',
+  figOpts,
+  o('JoinFn.typeInference', true),
+  o('JoinFn.specializedTypeCheck', false),
+
+  o('JavaExecutor.convertNumberValues', false),
+  o('JavaExecutor.printStackTrace', false),
+
+  # These domains are all based on SimpleWorld
+  required(:domain),
+  o('Grammar.inPaths', lambda { |e| 'overnight/' + e[:domain] + '.grammar' }),
+  o('SimpleWorld.domain', lambda { |e| e[:domain] }),
+  o('initialization', 'denotation :: error,-1000', 'denotation :: empty,-100', 'paraphrase :: size,+0.01', 'denotation :: value_in_formula,-100'),
+  o('FeatureExtractor.featureComputers','overnight.OvernightFeatureComputer'),
+  o('OvernightFeatureComputer.featureDomains', ''),
+  o('OvernightFeatureComputer.itemAnalysis',false),
+  sel(:gen,
+    l( # For debugging the grammar
+      o('FeatureExtractor.featureDomains', 'denotation'),
+      o('Dataset.inPaths', lambda { |e| 'train:overnight/' + e[:domain] + '-unittest.examples'}),
+      selo(:parse, 'Grammar.tags', 'generate', 'parse'),
+      o('interactive'),
+    nil),
+    l( # For generating utterances
+      o('parser', 'FloatingParser'),
+      o('maxDepth', 30), o('beamSize', 10000),
+      o('derivationScoreNoise', 1),
+      o('Dataset.inPaths', 'train:overnight/null.examples'),
+      o('Derivation.showUtterance'),
+      o('FeatureExtractor.featureDomains', 'denotation'),
+      o('printAllPredictions'),
+      o('printPredictedUtterances'),
+      o('executeAllDerivations'),
+      o('Parser.pruneErrorValues', true),
+      o('Grammar.tags', 'generate'),
+    nil),
+  nil),
+  a('Grammar.tags', 'general'),
+nil) })
+
+# Generate for all the domains
+addMode('genovernight-wrapper', 'Generate utterances for overnight semantic parsing', lambda { |e| l(
+  './run', '@mode=genovernight', '@gen=1',
+  sel(nil, {
+    'calendar' => let(:domain, 'calendar'),
+    'blocks' => let(:domain, 'blocks'),
+    'housing' => let(:domain, 'housing'),
+    'restaurants' => let(:domain, 'restaurants'),
+    'publications' => let(:domain, 'publications'),
+    'socialnetwork' => let(:domain, 'socialnetwork'),
+    'basketball' => let(:domain, 'basketball'),
+    'geo880' => let(:domain, 'geo880'),
+    'recipes' => let(:domain, 'recipes'),
+  }),
+  lambda { |e| '@domain=' + e[:domain] },
+  lambda { |e| system 'mkdir -p genovernight.out'; o('execDir', 'genovernight.out/' + e[:domain]) },
+nil) })
+
 
 ############################################################
 
diff --git a/src/edu/stanford/nlp/sempre/AbstractReinforcementParserState.java b/src/edu/stanford/nlp/sempre/AbstractReinforcementParserState.java
index a076274..cf79b83 100644
--- a/src/edu/stanford/nlp/sempre/AbstractReinforcementParserState.java
+++ b/src/edu/stanford/nlp/sempre/AbstractReinforcementParserState.java
@@ -47,9 +47,7 @@ abstract class AbstractReinforcementParserState extends ChartParserState {
       derivations.add(deriv);
       Collections.sort(derivations, Derivation.derivScoreComparator); // todo - perhaps can be removed
       return true;
-    } else {
-      return false;
-    }
+    } else return false;
   }
 
   // for [start, end) we try to create [start, end + i) or [start - i, end) and add unary rules
diff --git a/src/edu/stanford/nlp/sempre/ParserAgenda.java b/src/edu/stanford/nlp/sempre/ParserAgenda.java
index 557d012..8e31960 100644
--- a/src/edu/stanford/nlp/sempre/ParserAgenda.java
+++ b/src/edu/stanford/nlp/sempre/ParserAgenda.java
@@ -12,19 +12,19 @@ import java.util.List;
  * Data structure for agenda in reinforcement parser
  * Created by joberant on 10/20/14.
  */
-public interface ParserAgenda<E extends Comparable> extends Iterable<E> {
+public interface ParserAgenda<PrioritizedDerivationStream> extends Iterable<PrioritizedDerivationStream> {
   void sort();
-  boolean add(E item, double priority);
+  boolean add(PrioritizedDerivationStream item, double priority);
   int size();
   void clear();
-  E pop();
-  E get(int i);
-  void remove(E pds, int index);
+  PrioritizedDerivationStream pop();
+  PrioritizedDerivationStream get(int i);
+  void remove(PrioritizedDerivationStream pds, int index);
 }
 
-class ListParserAgenda<E extends Comparable> implements ParserAgenda<E> {
+class ListParserAgenda implements ParserAgenda<PrioritizedDerivationStream> {
 
-  private List<E> agenda = new ArrayList<>();
+  private List<PrioritizedDerivationStream> agenda = new ArrayList<>();
 
   @Override
   public void sort() {
@@ -32,7 +32,7 @@ class ListParserAgenda<E extends Comparable> implements ParserAgenda<E> {
   }
 
   @Override
-  public boolean add(E item, double priority) {
+  public boolean add(PrioritizedDerivationStream item, double priority) {
     return agenda.add(item);
   }
 
@@ -47,41 +47,41 @@ class ListParserAgenda<E extends Comparable> implements ParserAgenda<E> {
   }
 
   @Override
-  public E pop() {
-   // todo - replace sort with finding max
+  public PrioritizedDerivationStream pop() {
+   // todo - replace sort with finding max (check if makes it faster)
     sort();
-    E pds = agenda.get(0);
+    PrioritizedDerivationStream pds = agenda.get(0);
     remove(pds, 0);
     return pds;
   }
 
   @Override
-  public E get(int i) {
+  public PrioritizedDerivationStream get(int i) {
     return agenda.get(i);
   }
 
   @Override
-  public void remove(E pds, int index) {
-    E last = agenda.remove(agenda.size() - 1);
+  public void remove(PrioritizedDerivationStream pds, int index) {
+    PrioritizedDerivationStream last = agenda.remove(agenda.size() - 1);
     if (last != pds)
       agenda.set(index, last);
   }
 
   @Override
-  public Iterator<E> iterator() {
+  public Iterator<PrioritizedDerivationStream> iterator() {
     return agenda.iterator();
   }
 }
 
-class QueueParserAgenda<E extends Comparable> implements ParserAgenda<E> {
+class QueueParserAgenda implements ParserAgenda<PrioritizedDerivationStream> {
 
-  private PriorityQueue<E> agenda = new PriorityQueue<>();
+  private PriorityQueue<PrioritizedDerivationStream> agenda = new PriorityQueue<>();
 
   @Override
   public void sort() {  }
 
   @Override
-  public boolean add(E item, double priority) {
+  public boolean add(PrioritizedDerivationStream item, double priority) {
     return agenda.add(item, priority);
   }
 
@@ -99,22 +99,22 @@ class QueueParserAgenda<E extends Comparable> implements ParserAgenda<E> {
   }
 
   @Override
-  public E pop() {
+  public PrioritizedDerivationStream pop() {
     return agenda.next();
   }
 
   @Override
-  public E get(int i) {
+  public PrioritizedDerivationStream get(int i) {
     throw new RuntimeException("Not supported");
   }
 
   @Override
-  public void remove(E pds, int index) {
+  public void remove(PrioritizedDerivationStream pds, int index) {
     throw new RuntimeException("Not supported");
   }
 
   @Override
-  public Iterator<E> iterator() {
+  public Iterator<PrioritizedDerivationStream> iterator() {
     throw new RuntimeException("Not supported");
   }
 }
diff --git a/src/edu/stanford/nlp/sempre/ParserState.java b/src/edu/stanford/nlp/sempre/ParserState.java
index def6570..f4226c0 100644
--- a/src/edu/stanford/nlp/sempre/ParserState.java
+++ b/src/edu/stanford/nlp/sempre/ParserState.java
@@ -20,6 +20,10 @@ public abstract class ParserState {
   public static class Options {
     @Option(gloss = "Use a custom distribution for computing expected counts")
     public CustomExpectedCount customExpectedCounts = CustomExpectedCount.NONE;
+    @Option(gloss = "Whether to prune based on probability difference")
+    public boolean pruneByProbDiff = false;
+    @Option(gloss = "Difference in probability for pruning by prob diff")
+    public double probDiffPruningThresh = 100;
   }
   public static Options opts = new Options();
 
@@ -152,11 +156,22 @@ public abstract class ParserState {
       i++;
     }
 
-    // Keep only the top hypotheses
-    int beamSize = getBeamSize();
-    while (derivations.size() > beamSize) {
-      derivations.remove(derivations.size() - 1);
-      fallOffBeam = true;
+    //prune all d_i s.t  p(d_1) > CONST \cdot p(d_i)
+    if(ChartParserState.opts.pruneByProbDiff) {
+      double highestScore = derivations.get(0).score;
+      while (highestScore - derivations.get(derivations.size()-1).score > Math.log(opts.probDiffPruningThresh)) {
+        derivations.remove(derivations.size() - 1);
+        fallOffBeam = true;
+      }
+    }
+    //prune by beam size
+    else {
+      // Keep only the top hypotheses
+      int beamSize = getBeamSize();
+      while (derivations.size() > beamSize) {
+        derivations.remove(derivations.size() - 1);
+        fallOffBeam = true;
+      }
     }
   }
 
diff --git a/src/edu/stanford/nlp/sempre/ReinforcementParser.java b/src/edu/stanford/nlp/sempre/ReinforcementParser.java
index 4ca9408..107d82c 100644
--- a/src/edu/stanford/nlp/sempre/ReinforcementParser.java
+++ b/src/edu/stanford/nlp/sempre/ReinforcementParser.java
@@ -36,13 +36,11 @@ public class ReinforcementParser extends Parser {
     @Option (gloss = "Whether to update gradient only for correct moves")
     public boolean updateGradientForCorrectMovesOnly = true;
     @Option (gloss = "Low probability for which we don't unroll the stream")
-    public double lowProb = 0.001;
-    @Option (gloss = "Whether to not unroll at test time")
-    public boolean unrollWithMaxSampler = false;
+    public double lowProb = 0.01;
     @Option (gloss = "Whether to simulate the log liklihood objective")
     public boolean simulateNonRlObjective = false;
-    @Option (gloss = "Whether to never unroll")
-    public boolean noUnrolling = false;
+    @Option (gloss = "Whether to always unroll (even at test time)")
+    public boolean alwaysUnroll = false;
   }
   public static Options opts = new Options();
 
@@ -56,7 +54,6 @@ public class ReinforcementParser extends Parser {
 
   public ReinforcementParser(Spec spec) {
     super(spec);
-
     coarseParser = new CoarseParser(grammar);
 
     // generate maps from left to right and vice versa
@@ -89,7 +86,7 @@ public class ReinforcementParser extends Parser {
   @Override
   public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) {
     if (computeExpectedCounts) { // if we learn - use sampling, otherwise, use max
-     // if we simulate non RL we just take the max and not do sampling
+      // if we simulate non RL we just take the max and not do sampling
       if (opts.simulateNonRlObjective) {
         return new ReinforcementParserState.StateBuilder()
                 .parser(this)
@@ -128,7 +125,7 @@ public class ReinforcementParser extends Parser {
 // (b) whether an update needs to be done
 final class ReinforcementParserState extends AbstractReinforcementParserState {
 
-  public enum NecessaryDeriv { NECESSARY_DERIV, UNNECESSARY_DERIV, UNKNOWN };
+  public enum NecessaryDeriv { NECESSARY_DERIV, UNNECESSARY_DERIV, UNKNOWN }
   private static final double LOG_SMALL_PROB = Math.log(ReinforcementParser.opts.lowProb);
 
   private final ParserAgenda<PrioritizedDerivationStream> agenda;
@@ -140,7 +137,7 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
   List<Derivation> correctDerivations = new ArrayList<>();
   private Map<String, Double> stateSequenceExpectedCounts = new HashMap<>();
   Random randGen = new Random(1);
- // backpointers for remembering what derivations on the stream were popped before others
+  // backpointers for remembering what derivations on the stream were popped before others
   private Map<Long, Pair<ArrayList<Derivation>, Integer>> backpointerList;
   private int numItemsSampled = 0;
 
@@ -169,7 +166,7 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
     super(parser, params, ex, computeExpectedCounts);
     this.samplingStrategy = samplingStrategy;
     backpointerList = new HashMap<>();
-    agenda = samplingStrategy.equals("max") ? new QueueParserAgenda<>() : new ListParserAgenda<>();
+    agenda = samplingStrategy.equals("max") ? new QueueParserAgenda() : new ListParserAgenda();
   }
 
   private void clearState() {
@@ -184,6 +181,7 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
   }
 
 
+
   protected void addToAgenda(DerivationStream derivationStream) {
     addToAgenda(derivationStream, 0d);
   }
@@ -192,36 +190,31 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
 
     if (!derivationStream.hasNext()) return;
 
-    if (Parser.opts.unrollStream) {
-      Derivation streamHead = derivationStream.peek();
-      double streamHeadScore = streamHead.score;
-      while (derivationStream.hasNext()) {
-        Derivation deriv = derivationStream.next();
-        featurizeAndScoreDerivation(deriv);
-        if (parser.verbose(1)) {
-          if ((deriv.score - streamHeadScore) > 0.5)
-            LogInfo.logs("Unsorted stream: head=%s, head score=%s, deriv=%s, deriv score=%s",
-                    streamHead, streamHeadScore, deriv, deriv.score);
-        }
-        addToAgendaWithScore(SingleDerivationStream.constant(deriv), deriv.score, probSum);
-      }
-    } else {
+    //if it's less than one we can just add it even if we unroll everything (optimization)
+    if(!ReinforcementParser.opts.alwaysUnroll || derivationStream.estimatedSize() <= 1) {
       Derivation deriv = derivationStream.peek();
       featurizeAndScoreDerivation(deriv);
       addToAgendaWithScore(derivationStream, deriv.score, probSum);
+      if (completeDerivationsPushed % 100 == 0) // sort the agenda
+        agenda.sort();
+    }
+    else {
+      while(derivationStream.hasNext()) {
+        Derivation deriv = derivationStream.next();
+        featurizeAndScoreDerivation(deriv);
+        DerivationStream newStream = SingleDerivationStream.constant(deriv);
+        addToAgendaWithScore(newStream, deriv.score, probSum);
+        if (completeDerivationsPushed % 100 == 0) // sort the agenda
+          agenda.sort();
+      }
     }
-    if (completeDerivationsPushed % 100 == 0) // sort the agenda from time to time
-      agenda.sort();
   }
 
   // we need to override the method because parameters are prefixed with "search_"
   // this means that the score will not be the dot product and features and weights
   protected void featurizeAndScoreDerivation(Derivation deriv) {
-    if (deriv.isFeaturizedAndScored()) {
-      if (parser.verbose(5))
-        LogInfo.logs("Deriv already featurized=%s", deriv);
-      return;
-    }
+    if (deriv.isFeaturizedAndScored()) return;
+
     // Compute features
     parser.extractor.extractLocal(ex, deriv);
 
@@ -240,8 +233,8 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
   }
 
   private void addToAgendaWithScore(DerivationStream derivationStream, double derivScore, double probSum) {
-    if (derivScore == Double.NEGATIVE_INFINITY) // no need to add bad derivations
-      return;
+    if (derivScore == Double.NEGATIVE_INFINITY) return; // no need to add bad derivations - shouldn't happen
+
     Derivation deriv = derivationStream.peek(); // Score a DerivationStream based on the first item in the stream.
     double priority = derivScore - (completeDerivationsPushed++ * EPSILON);
     agenda.add(new PrioritizedDerivationStream(derivationStream, priority, probSum), priority); // when adding to agenda probsum is 0
@@ -292,31 +285,19 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
       coarseParserState = coarseParser.getCoarsePrunedChart(ex);
     LogInfo.end_track();
 
-    // draw samples to compute gradient and expected reward
-    int i = 0;
-    for ( ; i < (computeExpectedCounts ? ReinforcementParser.opts.numOfSamplesPerExample : 1); ++i) {
-      LogInfo.begin_track("ReinforcementParserState.infer, sample=%s", i);
-      if (i > 0)
-        clearState();
-      inferBySampling();
-      LogInfo.end_track();
-    }
+    // draw a sample to compute gradient and expected reward
+    LogInfo.begin_track("ReinforcementParserState.inferBySampling");
+    sampleHistoryAndInfer();
+    LogInfo.end_track();
 
     // Compute gradient
     setPredDerivations();
-    if (computeExpectedCounts) {
-      expectedCounts = ReinforcementUtils.multiplyDoubleMap(expectedCounts, 1d / i);
-      if (parser.verbose(3))
-        SempreUtils.logMap(expectedCounts, "averaged gradient");
-    }
-    objectiveValue /= i;
     if (parser.verbose(3))
-      LogInfo.logs("Expected reward (across samples) = %s", objectiveValue);
-
+      LogInfo.logs("Expected reward = %s", objectiveValue);
     visualizeChart();
   }
 
-  private void inferBySampling() {
+  private void sampleHistoryAndInfer() {
 
     // add to chart the token and phrase parts
     for (Derivation deriv : gatherTokenAndPhraseDerivations())
@@ -349,10 +330,12 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
       }
 
       // handle root derivations - get compatibility and record number of compatible derivations
-      recordRootDerivationStats(ex, numItemsSampled, sampledDerivation);
+      handleRootDerivation(ex, numItemsSampled, sampledDerivation);
+
       if (computeExpectedCounts) {
         Map<String, Double> counts = new HashMap<>();
         // add the feature vector and subtract for the time it was in the agenda unless has negative probability
+        //pretty hacky
         if (pdsAndProbability.getSecond() > -0.0001)
           sampledDerivation.incrementLocalFeatureVector(1 - pdsAndProbability.getFirst().probSum, counts);
         else
@@ -382,23 +365,23 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
 
   private void unrollHighProbStreams() {
 
-    if (ReinforcementParser.opts.noUnrolling) return;
-    if (samplingStrategy.equals("max") && !ReinforcementParser.opts.unrollWithMaxSampler) return;
+    if (samplingStrategy.equals("max")) return;
 
-    sampler.unroll(); // if multiplicative, then unroll oracle stuff
+    sampler.unroll(); // if multiplicative, then unroll oracle stuff (ignore \beta currently!)
 
     if (parser.verbose(3))
       LogInfo.begin_track("Unrolling high probability streams");
-    // get max agenda score
-    double maxScore = Double.NEGATIVE_INFINITY;
+
+    double lb=Double.NEGATIVE_INFINITY;
+    int numOfHiddenStreams = 0;
     for (PrioritizedDerivationStream pds : agenda) {
-      double agendaScore = pds.getScore();
-      if (maxScore < agendaScore)
-        maxScore = agendaScore;
+      lb = NumUtils.logAdd(lb, pds.getScore());
+      if(pds.derivStream.estimatedSize() > 1)
+        numOfHiddenStreams++;
     }
 
     if (parser.verbose(3))
-      LogInfo.logs("unrollHighProbStreams(): |agenda|=%s, max=%s", agenda.size(), maxScore);
+      LogInfo.logs("unrollHighProbStreams(): |agenda|=%s, lb=%s, |hiddenstreams|=%s", agenda.size(), lb, numOfHiddenStreams);
 
     List<Pair<DerivationStream, Double>> derivsToAdd = new ArrayList<>();
     List<Integer> indicesToRemove = new ArrayList<>();
@@ -406,7 +389,7 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
       PrioritizedDerivationStream pds = agenda.get(i);
       boolean modified = false;
       while (pds.derivStream.hasNext() && pds.derivStream.estimatedSize() > 1 &&
-              isHighProbStream(pds.derivStream, maxScore, pds.derivStream.estimatedSize())) {
+              illegalStream(pds.derivStream, lb, pds.derivStream.estimatedSize(), numOfHiddenStreams)) {
         modified = true;
         Derivation nextDeriv = pds.derivStream.next();
         updateBackpointers(pds.derivStream, nextDeriv);
@@ -414,12 +397,18 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
         DerivationStream derivStream = SingleDerivationStream.constant(nextDeriv);
         if (parser.verbose(3) && derivStream.hasNext()) {
           Derivation deriv  = derivStream.peek();
-          LogInfo.logs("unrollHighProbStreams(): add deriv=%s(%s,%s) [%s] score=%s, |stream|=%s",
+          LogInfo.logs("unrollIllegalStreams(): add deriv=%s(%s,%s) [%s] score=%s, |stream|=%s",
                   deriv.cat, deriv.start, deriv.end, deriv.formula, deriv.score, pds.derivStream.estimatedSize());
         }
         derivsToAdd.add(Pair.newPair(derivStream, pds.probSum));
-        if (pds.derivStream.hasNext())
+        //update lb
+        if (pds.derivStream.hasNext()) {
           featurizeAndScoreDerivation(pds.derivStream.peek());
+          lb = NumUtils.logAdd(lb, pds.getScore());
+        }
+        //update num of hidden streams
+        if(pds.derivStream.estimatedSize() <= 1)
+          numOfHiddenStreams--;
       }
       if (modified) {
         indicesToRemove.add(i);
@@ -441,6 +430,18 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
       LogInfo.end_track();
   }
 
+  private boolean illegalStream(DerivationStream derivStream, double logSum, int estimatedSize, int numOfHiddenStreams) {
+    Derivation deriv = derivStream.peek();
+    double firstItemLogProb = deriv.score - logSum; //log(exp(s(g_1))/L)
+    double upperBound = Math.log(estimatedSize) + Math.log(numOfHiddenStreams); //log(M(g)|G'|)
+
+    if (parser.verbose(3))
+      LogInfo.logs("IllegalStream(): score=%s, logsum=%s, |stream|=%s, |hiddenstreams|=%s, deriv=%s(%s,%s) %s, sum=%s",
+              deriv.score, logSum, estimatedSize, numOfHiddenStreams, deriv.cat, deriv.start, deriv.end, deriv.formula, firstItemLogProb+upperBound);
+
+    return (firstItemLogProb+upperBound) > LOG_SMALL_PROB;
+  }
+
   private boolean isHighProbStream(DerivationStream derivStream, double maxScore, int estimatedSize) {
     Derivation deriv = derivStream.peek();
     double gapFromMax = deriv.score - maxScore;
@@ -492,7 +493,7 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
   private void computeGradient() {
     if (predDerivations.isEmpty()) return;
 
-    double[] qDist = sampler.getDerivDistribution(predDerivations);
+    double[] qDist = sampler.getDerivDistribution(predDerivations); //uniform over correct things when \beta is high
     double[] piDist = ReinforcementUtils.expNormalize(predDerivations);
     // compute E_q(R(d))
     LogInfo.begin_track("Computing gradient");
@@ -539,7 +540,7 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
   }
 
   // info for visualizing chart
-  void visualizeChart() {
+  private void visualizeChart() {
     if (parser.chartFillOut != null && Parser.opts.visualizeChartFilling) {
       parser.chartFillOut.println(Json.writeValueAsStringHard(new ChartFillingData(ex.id, chartFillingList,
               ex.utterance, ex.numTokens())));
@@ -564,7 +565,7 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
     }
   }
 
-  private void recordRootDerivationStats(Example ex, int numItemsSampled, Derivation sampledDerivation) {
+  private void handleRootDerivation(Example ex, int numItemsSampled, Derivation sampledDerivation) {
     if (!sampledDerivation.isRoot(ex.numTokens())) return;
 
     sampledDerivation.ensureExecuted(parser.executor, ex.context);
@@ -681,11 +682,6 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
         this.firstCorrectDerivNumber = oracleDeriv.creationIndex;
 
         LogInfo.logs("OracleSampler: deriv=%s, comp=%s", oracleDeriv, oracleDeriv.compatibility);
-        if (SemanticFn.opts.trackLocalChoices) {
-          HashMap<String, Integer> choiceDiff = new LinkedHashMap<>();
-          oracleDeriv.incrementAllChoices(+1, choiceDiff);
-          FeatureVector.logChoices("Oracle deriv", choiceDiff);
-        }
         populateCorrectDerivations(oracleDeriv);
 
         if (parser.verbose(2)) {
@@ -804,6 +800,9 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
       return Pair.newPair(pds, returnProb ? prob : -1d);
     }
 
+    //todo : This unrolls all necessary derivations, and ignore \beta
+    //assumes probability mass on non-necessary things is very very small
+    //might need to be fixed if we anneal \beta to 0
     @Override
     public void unroll() {
 
@@ -869,6 +868,7 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
     }
   }
 
+  //A heuristic for choosing the oracle derivation. It'd be good to get rid of this or simplify
   public static class CorrectDerivationComparator implements Comparator<Derivation> {
     @Override
     public int compare(Derivation deriv1, Derivation deriv2) {
diff --git a/src/edu/stanford/nlp/sempre/ReinforcementUtils.java b/src/edu/stanford/nlp/sempre/ReinforcementUtils.java
index 175d4e9..8cc1285 100644
--- a/src/edu/stanford/nlp/sempre/ReinforcementUtils.java
+++ b/src/edu/stanford/nlp/sempre/ReinforcementUtils.java
@@ -12,6 +12,7 @@ import java.util.Random;
  * Utils for <code>ReinforcementParser</code>
  */
 public final class ReinforcementUtils {
+  private static double logMaxValue = Math.log(Double.MAX_VALUE);
   private ReinforcementUtils() { }
 
   // add to double map after adding prefix to all keys
@@ -141,8 +142,10 @@ public final class ReinforcementUtils {
     return res;
   }
 
-
-
-
-
+  // Return log(exp(a)-exp(b))
+  public static double logSub(double a, double b) {
+    if(a <= b) throw new RuntimeException("First argument must be strictly greater than second argument");
+    if(Double.isInfinite(b) || a-b > logMaxValue || b-a < 30) return a;
+    return a + Math.log(1d - Math.exp(b-a));
+  }
 }
diff --git a/src/edu/stanford/nlp/sempre/SingleDerivationStream.java b/src/edu/stanford/nlp/sempre/SingleDerivationStream.java
index 92bf3ee..ba57229 100644
--- a/src/edu/stanford/nlp/sempre/SingleDerivationStream.java
+++ b/src/edu/stanford/nlp/sempre/SingleDerivationStream.java
@@ -20,7 +20,8 @@ public abstract class SingleDerivationStream implements DerivationStream {
   }
 
   @Override public Derivation peek() {
-    if (nextDeriv == null) throw new RuntimeException("No more derivations!");
+    if (!hasNext()) throw new RuntimeException("No more derivations!");
+    //if (nextDeriv == null) throw new RuntimeException("No more derivations!");
     return nextDeriv;
   }
 
diff --git a/src/edu/stanford/nlp/sempre/overnight/OvernightDerivationPruningComputer.java b/src/edu/stanford/nlp/sempre/overnight/OvernightDerivationPruningComputer.java
index 14f84d4..84d0868 100644
--- a/src/edu/stanford/nlp/sempre/overnight/OvernightDerivationPruningComputer.java
+++ b/src/edu/stanford/nlp/sempre/overnight/OvernightDerivationPruningComputer.java
@@ -19,7 +19,7 @@ public class OvernightDerivationPruningComputer extends DerivationPruningCompute
   }
   public static Options opts = new Options();
 
-
+  
   public OvernightDerivationPruningComputer(DerivationPruner pruner) {
     super(pruner);
   }
diff --git a/tables/README.md b/tables/README.md
index 5393336..d356918 100644
--- a/tables/README.md
+++ b/tables/README.md
@@ -1 +1,63 @@
-This directory contains all the files relevant to semantic parsing on tables.
+Compositional Semantic Parsing on Semi-Structured Tables
+========================================================
+
+This README file describes the `tables` mode which accompanies the paper:
+
+> Panupong Pasupat, Percy Liang.  
+> Compositional Semantic Parsing on Semi-Structured Tables  
+> Association for Computational Linguistics (ACL), 2015.
+
+Further descriptions and experiments can be viewed on the CodaLab website:
+
+  https://www.codalab.org/worksheets/0xf26cd79d4d734287868923ad1067cf4c/
+
+Running the code
+----------------
+
+1. Download the dependencies and the dataset:
+
+    ./pull-dependencies corenlp
+    ./pull-dependencies tables
+    ./pull-dependencies tables-data
+
+  The dataset lives in `lib/data/tables/`
+
+2. Compile the source:
+
+    ant
+
+  This will produce JAR files in the `libsempre` directory as usual.
+
+3. The following command train and test on 100 development examples:
+
+    ./run @mode=tables @data=u-1 @feat=all @train=1 -maxex train,100 dev,100
+
+  The command should take less than an hour.
+
+  * To train on the complete development set, remove `-maxex train,100 dev,100`
+
+  * The command above uses `u-1` (80:20 split of the development data).
+  Other available sets include `u-2`, ..., `u-5` (four other development splits)
+  and `test` (actual train-test split).
+
+Other usages
+------------
+
+To launch the interactive shell, use:
+
+  ./run @mode=tables -interactive
+
+Apart from the usual shell commands, the additional command `context`
+can load the context graph for execution. For example, use:
+
+  (context (graph tables.TableKnowledgeGraph csv/204-csv/590.csv))
+
+to load lib/data/tables/csv/204-csv/590.csv
+
+The table can also be viewed in pretty-printed format by calling
+
+  ./tables/view 204 590
+
+or
+
+  ./tables/view csv/204-csv/590.csv