Update README.md

Updated as suggested in #117

DOCUMENTATION.md

@@ -146,7 +146,7 @@ The possible `Value`s, along with an example are as follows:
 
     In general:
 
-        (table (<header-string-1> ... <header-string-n>) (<row-1-value-1> ... <value-1-value-n>) ...)
+        (table (<header-string-1> ... <header-string-n>) (<row-1-value-1> ... <row-1-value-n>) ...)
 
 There are some more arcane `Value`s (see `Values.java` for a list), but they
 are not that important from the point of view of specifying a logical form.

README.md

@@ -1,4 +1,4 @@
-# SEMPRE 2.2: Semantic Parsing with Execution
+# SEMPRE 2.4: Semantic Parsing with Execution
 
 ## What is semantic parsing?
 
@@ -62,6 +62,8 @@ SEMPRE has been used in the following papers:
   offshoot, and does not use many of the core learning and parsing utiltiies in
   SEMPRE.  To reproduce those results, check out SEMPRE 1.0.
 
+Please refer to the [project page](https://nlp.stanford.edu/software/sempre/) for a more complete list.
+
 ## Where do I go next?
 
 - If you're new to semantic parsing, you can learn more from the [background
@@ -82,6 +84,8 @@ You must have the following already installed on your system.
 - Ant 1.8.2
 - Ruby 1.8.7 or 1.9
 - wget
+- make (for compiling fig and Virtuoso)
+- zip (for unzip downloaded dependencies)
 
 Other dependencies will be downloaded as you need them.  SEMPRE has been tested
 on Ubuntu Linux 12.04 and MacOS X.  Your mileage will vary depending on how
@@ -95,15 +99,18 @@ similar your system is.
 
 2. Download the minimal core dependencies (all dependencies will be placed in `lib`):
 
-        ./pull-dependencies core
+        ruby ./pull-dependencies core
 
 3. Compile the source code (this produces `libsempre/sempre-core.jar`):
 
         ant core
+        
+   If there is any issue during compilation, try deleting the directories `lib` and `fig`
+   and rerunning `./pull-dependencies core`
 
 4. Run an interactive shell:
 
-        ./run @mode=simple
+        ruby ./run @mode=simple
 
     You should be able to type the following into the shell and get the answer `(number 7)`:
 
@@ -169,4 +176,16 @@ Changes from SEMPRE 2.0 to SEMPRE 2.1:
 
 Changes from SEMPRE 2.1 to SEMPRE 2.2:
 
+- Added code for the paper *Inferring Logical Forms From Denotations* (ACL 2016).
+
+Changes from SEMPRE 2.2 to SEMPRE 2.3:
+
 - Added the `interactive` package for the paper *Naturalizing a programming language through interaction* (ACL 2017).
+
+Changes from SEMPRE 2.3 to SEMPRE 2.3.1:
+
+- Modified the `tables` module to resemble SEMPRE 2.1, effectively making it work again.
+
+Changes from SEMPRE 2.3.1 to SEMPRE 2.4:
+
+- Added the `cprune` package for the paper *Macro Grammars and Holistic Triggering for Efficient Semantic Parsing* (EMNLP 2017).

build.xml

@@ -25,7 +25,7 @@
     <antcall target="compile.released"/>
   </target>
 
-  <target name="compile.released" depends="init,core,cache,corenlp,freebase,tables,overnight"/>
+  <target name="compile.released" depends="init,core,cache,corenlp,freebase,tables,cprune,overnight"/>
 
   <!-- Compile core -->
   <target name="core" depends="init">
@@ -69,7 +69,7 @@
   </target>
 
   <!-- Compile tables -->
-  <target name="tables" depends="init,core,corenlp">
+  <target name="tables" depends="init,core,corenlp,cprune">
     <echo message="Compiling ${ant.project.name}: tables"/>
     <mkdir dir="${classes}/tables"/>
     <javac srcdir="${src}" destdir="${classes}/tables" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
@@ -78,6 +78,16 @@
     <jar destfile="${libsempre}/sempre-tables.jar" basedir="${classes}/tables"/>
   </target>
 
+  <!-- Compile cprune -->
+  <target name="cprune" depends="init,core">
+    <echo message="Compiling ${ant.project.name}: cprune"/>
+    <mkdir dir="${classes}/cprune"/>
+    <javac srcdir="${src}" destdir="${classes}/cprune" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
+      <include name="edu/stanford/nlp/sempre/cprune/"/>
+    </javac>
+    <jar destfile="${libsempre}/sempre-cprune.jar" basedir="${classes}/cprune"/>
+  </target>
+
   <!-- Compile overnight -->
   <target name="overnight" depends="init,core">
     <echo message="Compiling ${ant.project.name}: overnight"/>

freebase/scripts/extract-freebase-schema.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 require 'open-uri'
 

freebase/scripts/fbshell.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 require 'open-uri'
 require 'json'

freebase/scripts/virtuoso

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # This script provides a convenient wrapper for the Virtuoso SPARQL server.
 

interactive/community-server/requirements.txt

@@ -3,7 +3,7 @@ Flask==0.12
 Flask-Cors==3.0.2
 Flask-SocketIO==2.8.2
 greenlet==0.4.11
-httplib2==0.10.3
+httplib2==0.18.0
 itsdangerous==0.24
 Jinja2==2.9.4
 jsonschema==2.5.1

interactive/run

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # This is the main entry point for running SHRDLURN.  See
 # fig/lib/execrunner.rb for more documentation for how commands are generated.
@@ -7,6 +7,7 @@
 
 $: << 'fig/lib'
 require 'execrunner'
+$optPrefix = '-'
 $path = 'interactive'
 $output = 'interactive/output'
 $modes = []

overnight/README.md

@@ -38,3 +38,7 @@ No PPDB features:
 Full system:
 
     ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match ppdb skip-bigram root alignment lexical root_lexical lf simpleworld
+
+# Generating alignment files for a new domain
+
+[See the instructions here](https://docs.google.com/document/d/1cgqThUAyArbgjQi0P7l--kCLH10copd1EXFnk_bHkY0/edit?usp=sharing) (work in progress).

pull-dependencies

@@ -139,7 +139,7 @@ addModule('core', 'Core utilities (need to compile)', lambda {
   pull('/u/nlp/data/semparse/resources/jackson-databind-2.2.0.jar')
 
   # jLine from maven central
-  pull('http://central.maven.org/maven2/jline/jline/2.14.2/jline-2.14.2.jar')
+  pull('https://repo1.maven.org/maven2/jline/jline/2.14.6/jline-2.14.6.jar')
 })
 
 addModule('corenlp', 'Stanford CoreNLP 3.6.0', lambda {
@@ -236,11 +236,30 @@ addModule('tables', 'Semantic parsing with execution on tables', lambda {
   pull('/u/nlp/data/semparse/resources/opencsv-3.0.jar')
 })
 
-addModule('tables-data', 'Data for semantic parsing with execution on tables', lambda {
-  pull('/u/nlp/data/semparse/wikitable/data', 'data/tables/', {:symlink => true})
-  pull('/u/nlp/data/semparse/wikitable/mturk-trivia-data/csv', 'data/tables/', {:symlink => true})
+addModule('tables-data', 'WikiTableQuestions dataset v1.0.2', lambda {
+  # Compact version of the dataset
+  pull('https://github.com/ppasupat/WikiTableQuestions/releases/download/v1.0.2/WikiTableQuestions-1.0.2-compact.zip', 'data')
+  # Remove old file (for backward compatibility)
+  if File.directory?('lib/data/WikiTableQuestions')
+    system 'rm -rv lib/data/WikiTableQuestions' or exit 1
+  end
+  system "cd lib/data && unzip WikiTableQuestions-1.0.2-compact.zip" or exit 1
 })
 
+addModule('tables-data-0.5', 'WikiTableQuestions dataset v0.5 (for backward reproducibility)', lambda {
+  # Compact version of the dataset
+  pull('https://github.com/ppasupat/WikiTableQuestions/releases/download/v0.5/WikiTableQuestions-0.5-compact.zip', 'data')
+  # Remove old file (for backward compatibility)
+  if File.directory?('lib/data/WikiTableQuestions')
+    system 'rm -rv lib/data/WikiTableQuestions' or exit 1
+  end
+  system "cd lib/data && unzip WikiTableQuestions-0.5-compact.zip" or exit 1
+})
+
+addModule('tables-cprune', 'Neighbor information for applying macro grammar on tables', lambda {
+  pull('/u/nlp/data/semparse/cprune/nn_0.zip', 'data/nn_0', {:symlink => true})
+  system "cd lib/data/nn_0 && unzip nn_0.zip" or exit 1
+})
 
 addModule('overnight', 'Creating a parser for multiple domains', lambda {
   # Geo evaluation

run

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # This is the main entry point for running all SEMPRE programs.  See
 # fig/lib/execrunner.rb for more documentation for how commands are generated.
@@ -7,6 +7,7 @@
 
 $: << 'fig/lib'
 require 'execrunner'
+$optPrefix = '-'
 
 $modes = []
 def addMode(name, description, func)
@@ -551,7 +552,6 @@ addMode('overnight', 'Overnight semantic parsing', l(
   #o('initialization', 'paraphrase :: match,1', 'paraphrase :: size,-0.1', 'paraphrase :: ppdb,0.3',
   #  'paraphrase :: skip-bigram,0.8', 'paraphrase :: skip-ppdb,0.2','denotation :: error,-1000'),
   o('coarsePrune'),
-  o('OvernightDerivationPruningComputer.applyHardConstraints'),
   sel(2,
     l(), # no reg
     l(o('Params.l1Reg','lazy'),o('Params.l1RegCoeff',0)),
@@ -573,6 +573,7 @@ addMode('overnight', 'Overnight semantic parsing', l(
   o('phraseAlignmentPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '.phrase_alignments' }),
   o('PPDBModel.ppdbModelPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '-ppdb.txt' }),
   o('DerivationPruner.pruningComputers', ['overnight.OvernightDerivationPruningComputer']),
+  o('DerivationPruner.pruningStrategies', ['violateHardConstraints']),
   o('Dataset.inPaths', 
     lambda { |e| 'train:lib/data/overnight/' + e[:domain] + '.paraphrases.train.examples' },
     lambda { |e| 'test:lib/data/overnight/' + e[:domain] + '.paraphrases.test.examples' }),
@@ -593,8 +594,6 @@ addMode('overnight', 'Overnight semantic parsing', l(
       o('FloatingParser.maxDepth', 11),
       o('Grammar.tags','generate','general', 'geo880'),
       o('SimpleLexicon.inPaths', 'lib/data/overnight/geo880.lexicon'),
-      o('OvernightDerivationPruningComputer.usePredicateDict'),
-      o('OvernightDerivationPruningComputer.predicateDictPath','lib/data/overnight/geo880.predicate.dict'),
     nil),
     'calendar' => l(
       o('Grammar.tags','generate','general'),
@@ -634,25 +633,27 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
   # Add @cldir=1 to use CodaLab's directory paths
   letDefault(:cldir, 0),
   # Usual header
-  header('core,tables,corenlp'),
+  header('core,tables,corenlp,cprune'),
   # Select class
   letDefault(:class, 'main'),
   sel(:class, {
     'main' => 'edu.stanford.nlp.sempre.Main',
     'check' => 'edu.stanford.nlp.sempre.tables.test.DPDParserChecker',
-    'align' => l('edu.stanford.nlp.sempre.tables.alignment.AlignerMain', o('maxTrainIters', 11)),
     'dump' => 'edu.stanford.nlp.sempre.tables.serialize.SerializedDumper',
     'load' => l('edu.stanford.nlp.sempre.tables.serialize.SerializedLoader', let(:parser, 'serialized')),
     'stats' => 'edu.stanford.nlp.sempre.tables.test.TableStatsComputer',
     'tag-data' => 'edu.stanford.nlp.sempre.tables.serialize.TaggedDatasetGenerator',
     'tag-table' => 'edu.stanford.nlp.sempre.tables.serialize.TaggedTableGenerator',
+    'tag-fuzzy' => 'edu.stanford.nlp.sempre.tables.serialize.TaggedFuzzyGenerator',
     'alter' => l('edu.stanford.nlp.sempre.tables.alter.BatchTableAlterer', let(:parser, 'serialized')),
     'alter-ex' => l('edu.stanford.nlp.sempre.tables.alter.AlteredTablesExecutor', let(:parser, 'serialized')),
     'filter' => 'edu.stanford.nlp.sempre.tables.serialize.DumpFilterer',
+    'column' => 'edu.stanford.nlp.sempre.tables.test.TableColumnAnalyzer',
+    'execute' => 'edu.stanford.nlp.sempre.tables.test.BatchTableExecutor',
   }),
   # Fig parameters
-  selo(:cldir, 'execDir', '_OUTPATH_', 'output'),
-  o('overwriteExecDir'), o('addToView', 11), o('jarFiles', 'libsempre/*'),
+  selo(:cldir, 'execDir', '_OUTPATH_', '.'),
+  o('overwriteExecDir'), o('addToView', 15), o('jarFiles', 'libsempre/*'),
   sel(:cldir, l(), '>/dev/null'),
   # Set environment for table execution
   o('executor', 'tables.lambdadcs.LambdaDCSExecutor'),
@@ -660,6 +661,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
   o('NumberFn.unitless'), o('NumberFn.alsoTestByConversion'),
   o('TypeInference.typeLookup', 'tables.TableTypeLookup'),
   o('JoinFn.specializedTypeCheck', false), o('JoinFn.typeInference', true),
+  o('Learner.outputPredValues'),
   # Value Evaluator
   letDefault(:eval, 'value'),
   sel(:eval, {
@@ -671,58 +673,68 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
     nil),
   }),
   # Parser
-  letDefault(:parser, 'old-floatsize'),
+  letDefault(:parser, 'floatsize'),
+  o('beamSize', 50),
+  o('useSizeInsteadOfDepth'),
   sel(:parser, {
-    'old-floatsize' => l(
+    'floatsize' => l(
       o('Builder.parser', 'FloatingParser'),
-      o('useSizeInsteadOfDepth'),
       o('FloatingParser.maxDepth', 15),
     nil),
-    'old-baseline' => l(
-      o('Builder.parser', 'tables.baseline.TableBaselineParser'),
-    nil),
+    'baseline' => o('Builder.parser', 'tables.baseline.TableBaselineParser'),
     'serialized' => o('Builder.parser', 'tables.serialize.SerializedParser'),
     # ACL 2016
     'grow-dpd' => l(
       o('Builder.parser', 'tables.dpd.DPDParser'),
-      o('useSizeInsteadOfDepth'),
       o('FloatingParser.maxDepth', 8),
     nil),
     'grow-float' => l(
       o('Builder.parser', 'FloatingParser'),
-      o('useSizeInsteadOfDepth'),
       o('FloatingParser.maxDepth', 8),
       o('FloatingParser.betaReduce'), o('initialFloatingHasZeroDepth'),
     nil),
     'grow-mix' => l(
       o('Builder.parser', 'MixParser'),
       o('MixParser.parsers', 'FloatingParser', 'tables.serialize.SerializedParser:train-0xc'),
-      o('useSizeInsteadOfDepth'),
       o('FloatingParser.maxDepth', 8),
       o('FloatingParser.betaReduce'), o('initialFloatingHasZeroDepth'),
     nil),
+    # EMNLP 2017
+    'cprune' => l(
+      o('Builder.parser', 'cprune.CPruneFloatingParser'),
+      o('FloatingParser.maxDepth', 15),
+      o('maxNumNeighbors', 40),
+      o('maxPredictedPatterns', 1000),
+    nil),
   }),
-  o('Parser.verbose', 1),
+  o('Parser.verbose', 0),
   letDefault(:pruning, 1),
   sel(:pruning,
     l(),
     l(
       o('DerivationPruner.pruningStrategies', *tablesPruningStrategies),
       o('DerivationPruner.pruningComputers', 'tables.TableDerivationPruningComputer'),
-      o('recursivePruning', false),
     nil),
   nil),
   # Grammar
   tablesGrammarPaths,
-  letDefault(:fuzzy, 1),
-  sel(:fuzzy,
-    l(),
-    l(o('fuzzyMatchSubstring'), o('fuzzyMatchMaxEditDistanceRatio', 0.3), o('alsoReturnUnion'), o('alsoMatchPart')),
-  nil),
+  letDefault(:fuzzy, 'original'),
+  sel(:fuzzy, {
+    'original' => o('FuzzyMatcher.fuzzyMatcher', 'tables.match.OriginalMatcher'),
+    'editdist-exact' => l(
+      o('FuzzyMatcher.fuzzyMatcher', 'tables.match.EditDistanceFuzzyMatcher'),
+      o('fuzzyMatchMaxEditDistanceRatio', 0.0),
+    nil),
+    'editdist-fuzzy' => l(
+      o('FuzzyMatcher.fuzzyMatcher', 'tables.match.EditDistanceFuzzyMatcher'),
+      o('fuzzyMatchSubstring'), o('fuzzyMatchMaxEditDistanceRatio', 0.15),
+      o('alsoMatchPart'),
+    nil),
+  }),
   letDefault(:normalize, 1),
   sel(:normalize,
     l(),
-    l(o('genericDateValue'), o('numberCanStartAnywhere'), o('num2CanStartAnywhere'), o('NumberFn.alsoTestByIsolatedNER')),
+    l(o('genericDateValue'), o('numberCanStartAnywhere'), o('num2CanStartAnywhere')),
   nil),
   letDefault(:anchor, 1),
   sel(:anchor, {
@@ -734,19 +746,24 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
   tablesDataPaths,
   # Verbosity
   o('FeatureVector.ignoreZeroWeight'),
-  o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10), o('logFeaturesLimit', 10),
+  o('logFeaturesLimit', 10),
   o('LambdaDCSException.noErrorMessage'),
-  letDefault(:verbose, 2),
+  letDefault(:verbose, 0),
   sel(:verbose,
-    l(),
     l(
+      o('maxPrintedPredictions', 1), o('maxPrintedTrue', 1),
+    nil),
+    l(
+      o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10),
       o('putCellNameInCanonicalUtterance'), o('showUtterance'),
     nil),
     l(
+      o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10),
       o('putCellNameInCanonicalUtterance'), o('showUtterance'),
       o('summarizeRuleTime'), o('summarizeDenotations'),
     nil),
     l(
+      o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10),
       o('putCellNameInCanonicalUtterance'), o('showUtterance'),
       o('summarizeRuleTime'), o('summarizeDenotations'),
       o('showRules'),
@@ -772,6 +789,7 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
       o('combineFromFloatingParser'),
       o('maxTrainIters', 3),
       o('showValues', false), o('showFirstValue'),
+      o('customExpectedCounts', 'TOP'),
     nil),
     l(
       # for dumping derivations (@class=dump)
@@ -795,13 +813,13 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
     'some' => l(     # Add your own features! (only set up the feature computers)
       o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
     nil),
-    'all' => l(      # All features
+    'all' => l(      # All ACL 2015 features
       o('FeatureExtractor.featureDomains', 'custom-denotation phrase-predicate phrase-denotation headword-denotation missing-predicate'.split),
       o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
     nil),
-    'all-fragment' => l(
-      o('FeatureExtractor.featureDomains', 'custom-denotation phrase-denotation headword-denotation phrase-fragment'.split),
-      o('FeatureExtractor.featureComputers', 'tables.features.PhraseDenotationFeatureComputer tables.features.PhraseFragmentFeatureComputer'.split),
+    'more' => l(     # All ACL 2015 features + more experimental features
+      o('FeatureExtractor.featureDomains', 'custom-denotation phrase-predicate phrase-denotation headword-denotation missing-predicate anchored-entity'.split),
+      o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer tables.features.AnchorFeatureComputer'.split),
     nil),
     'baseline' => l( # For the baseline classifier
       o('FeatureExtractor.featureDomains', 'custom-denotation phrase-denotation headword-denotation table-baseline'.split),
@@ -824,7 +842,6 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
     'none' => l(),
     'careful' => l(
       o('maxNforLexicalizeAllPairs', 2),
-      o('lookUnderCellProperty'), o('useGenericCellType'),
       o('computeFuzzyMatchPredicates'),
     nil),
   }),
@@ -854,7 +871,7 @@ def tablesGrammarPaths
         nil),
         'combined-more' => l(
           o('Grammar.inPaths', "#{baseDir}combined.grammar"),
-          o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge u-superlative neq'.split),
+          o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge v-superlative'.split),
         nil),
         'combined-trigger' => l(    # Use trigger words for operations
           o('Grammar.inPaths', "#{baseDir}combined.grammar"),
@@ -871,6 +888,17 @@ def tablesGrammarPaths
           o('Grammar.tags', *'scoped merge-and arithmetic comparison alternative neq yearrange part closedclass scoped-2args-merge-and'.split),
           let(:anchor, 2),
         nil),
+        'grow-strict' => l(
+          o('Grammar.inPaths', "#{baseDir}grow.grammar"),
+          o('Grammar.binarizeRules', false),
+          o('Grammar.tags', *'scoped merge-and arithmetic comparison alternative neq yearrange part closedclass-generic scoped-2args-merge-and'.split),
+          let(:anchor, 2),
+        nil),
+        # EMNLP 2017
+        'extended' => l(
+          o('Grammar.inPaths', "#{baseDir}extended.grammar"),
+          o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge v-superlative'.split),
+        nil),
       }),
     nil)
   }
@@ -878,15 +906,19 @@ end
 
 def tablesDataPaths
   lambda { |e|
-    baseDir = ['lib/data/tables/data/', 'WikiTableQuestions/data/'][e[:cldir]]
-    csvDir = ['lib/data/tables/', 'WikiTableQuestions/'][e[:cldir]]
+    baseDir = ['lib/data/WikiTableQuestions/data/', 'WikiTableQuestions/data/'][e[:cldir]]
+    csvDir = ['lib/data/WikiTableQuestions/', 'WikiTableQuestions/'][e[:cldir]]
+    nnDir = ['lib/data/nn_0/', 'nn_0/'][e[:cldir]]
     datasets = {
       'none' => l(),
       'train' => o('Dataset.inPaths', "train,#{baseDir}training.examples"),
       # Pristine test test
-      'test' => o('Dataset.inPaths',
-        "train,#{baseDir}training.examples",
-        "test,#{baseDir}pristine-unseen-tables.examples"),
+      'test' => l(
+        o('Dataset.inPaths',
+          "train,#{baseDir}training.examples",
+          "test,#{baseDir}pristine-unseen-tables.examples"),
+        o('neighborFilePath', "#{nnDir}/exact_nearest_neighbors.all"),
+      nil),
       # @data=annotated can be used with @class=check only
       'annotated' => o('Dataset.inPaths', "train,#{baseDir}annotated-all.examples"),
       'before300' => o('Dataset.inPaths', "train,#{baseDir}training-before300.examples"),
@@ -898,12 +930,16 @@ def tablesDataPaths
           "train,#{baseDir}random-split-#{x}-train.examples",
           "dev,#{baseDir}random-split-#{x}-dev.examples",
         nil),
+        o('neighborFilePath', "#{nnDir}/exact_nearest_neighbors.seed-#{x}.train"),
       nil)
     end
     # That's it!
     l(
-      o('splitDevFromTrain', false),
       o('TableKnowledgeGraph.baseCSVDir', csvDir),
+      # To use the normalized values from the tagged file, which were checked by hand,
+      # add @useTaggedFile=1
+      letDefault(:useTaggedFile, 0),
+      selo(:useTaggedFile, 'TableValuePreprocessor.taggedFiles', '', "#{csvDir}/tagged/data/"),
       sel(:data, datasets),
     nil)
   }
@@ -921,8 +957,9 @@ def tablesPruningStrategies
     "mistypedMerge",
     ### Strategies that depend on the children's formulas
     "doubleNext",
-    "doubleSummarizers",
+    "multipleSuperlatives",
     "sameMerge",
+    "forwardBackward",
     "unsortedMerge",
     "typeRowMerge",
   nil].compact
@@ -932,10 +969,8 @@ end
 ############################################################
 # {2015-01-18} Generate utterances [Percy]
 addMode('genovernight', 'Generate utterances for overnight semantic parsing', lambda { |e| l(
-  'fig/bin/qcreate',
-  letDefault(:gen, 0),
-  sel(:gen, l()),
-  'java', '-Dmodules=core,overnight', '-Xmx10g', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.overnight.GenerationMain',
+  header('core,overnight'),
+  'edu.stanford.nlp.sempre.overnight.GenerationMain',
   figOpts,
   o('JoinFn.typeInference', true),
   o('JoinFn.specializedTypeCheck', false),
@@ -951,6 +986,7 @@ addMode('genovernight', 'Generate utterances for overnight semantic parsing', la
   o('FeatureExtractor.featureComputers','overnight.OvernightFeatureComputer'),
   o('OvernightFeatureComputer.featureDomains', ''),
   o('OvernightFeatureComputer.itemAnalysis',false),
+  letDefault(:gen, 1),
   sel(:gen,
     l( # For debugging the grammar
       o('FeatureExtractor.featureDomains', 'denotation'),

scripts/agenda-stats

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 ARGV.each { |e|
   e = e.sub(/\.exec$/, '')

scripts/extract-module-classes.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # Input: src
 # Output: module-classes.txt

scripts/find-hard-coded-paths.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # Heuristically find all hard-coded paths in the source code.
 # There should be no absolute paths.

scripts/fix-checkstyle.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # Hacky script for automatically fixing style errors.  This script is far from
 # perfect and you should manually inspect all changes before making changes.

scripts/verify-code-loop.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # Verifies that the codebase is sane (compiles, doesn't crash, gets reasonable
 # accuracy) every once in a while.  If something fails, an email is sent out

src/edu/stanford/nlp/sempre/BooleanValue.java

@@ -19,6 +19,9 @@ public class BooleanValue extends Value {
     return tree;
   }
 
+  @Override public String sortString() { return "" + value; }
+  @Override public String pureString() { return "" + value; }
+
   @Override public int hashCode() { return Boolean.valueOf(value).hashCode(); }
   @Override public boolean equals(Object o) {
     if (this == o) return true;

src/edu/stanford/nlp/sempre/DateValue.java

@@ -86,6 +86,7 @@ public class DateValue extends Value {
         + "-" + (month == -1 ? "xx" : String.format("%02d", month))
         + "-" + (day == -1 ? "xx" : String.format("%02d", day));
   }
+  @Override public String pureString() { return isoString(); }
 
   @Override public int hashCode() {
     int hash = 0x7ed55d16;

src/edu/stanford/nlp/sempre/DefaultDerivationPruningComputer.java

@@ -1,10 +1,9 @@
 package edu.stanford.nlp.sempre;
 
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Map;
+import java.util.*;
 
-import fig.basic.LogInfo;
+import fig.basic.LispTree;
+import fig.basic.Option;
 
 /**
  * Common pruning strategies that can be used in many semantic parsing tasks.
@@ -12,6 +11,11 @@ import fig.basic.LogInfo;
  * @author ppasupat
  */
 public class DefaultDerivationPruningComputer extends DerivationPruningComputer {
+  public static class Options {
+    @Option(gloss = "(for badSummarizerHead) allow count on sets of size 1")
+    public boolean allowCountOne = false;
+  }
+  public static Options opts = new Options();
 
   public DefaultDerivationPruningComputer(DerivationPruner pruner) {
     super(pruner);
@@ -22,6 +26,7 @@ public class DefaultDerivationPruningComputer extends DerivationPruningComputer
   public static final String nonLambdaError = "nonLambdaError";
   public static final String tooManyValues = "tooManyValues";
   public static final String doubleSummarizers = "doubleSummarizers";
+  public static final String multipleSuperlatives = "multipleSuperlatives";
   public static final String sameMerge = "sameMerge";
   public static final String mistypedMerge = "mistypedMerge";
   public static final String unsortedMerge = "unsortedMerge";
@@ -32,49 +37,116 @@ public class DefaultDerivationPruningComputer extends DerivationPruningComputer
     return Arrays.asList(
         atomic,
         emptyDenotation, nonLambdaError, tooManyValues,
-        doubleSummarizers, sameMerge, mistypedMerge, unsortedMerge, badSummarizerHead);
+        doubleSummarizers, multipleSuperlatives,
+        sameMerge, mistypedMerge, unsortedMerge, badSummarizerHead);
   }
 
-  // ============================================================
-  // Formula-based pruning
-  // ============================================================
-
   @Override
-  public String isPrunedWithoutExecution(Derivation deriv) {
+  public String isPruned(Derivation deriv) {
     // atomic: Prune atomic formula at root.
     //   e.g., Prevent "Who was taller, Lincoln or Obama" --> fb:en.lincoln generated from lexicon without any computation   
     if (containsStrategy(atomic)) {
-      if (deriv.isRoot(pruner.ex.numTokens()) && deriv.formula instanceof ValueFormula)
+      if (deriv.isRoot(ex.numTokens()) && deriv.formula instanceof ValueFormula)
         return atomic;
     }
-    return null;
-  }
-
-  // ============================================================
-  // Denotation-based Pruning
-  // ============================================================
-
-  @Override
-  public String isPrunedGeneral(Derivation deriv) {
     // emptyDenotation: Prune if the denotation is empty
     if (containsStrategy(emptyDenotation)) {
+      deriv.ensureExecuted(parser.executor, ex.context);
       if (deriv.value instanceof ListValue && ((ListValue) deriv.value).values.isEmpty())
         return emptyDenotation;
     }
     // nonLambdaError: Prune if the denotation is an error and the formula is not a partial formula
     if (containsStrategy(nonLambdaError) && !isLambdaFormula(deriv.formula)) {
-      if (deriv.value instanceof ErrorValue) {
-        if (DerivationPruner.opts.pruningVerbosity >= 5)
-          LogInfo.logs("NonLambdaError: %s => %s", deriv.formula, deriv.value);
+      deriv.ensureExecuted(parser.executor, ex.context);
+      if (deriv.value instanceof ErrorValue)
         return nonLambdaError;
-      }
     }
     // tooManyValues: Prune if the denotation has too many values (at $ROOT only)
-    if (containsStrategy(tooManyValues) && deriv.isRoot(pruner.ex.numTokens())) {
+    if (containsStrategy(tooManyValues) && deriv.isRoot(ex.numTokens())) {
       if (!(deriv.value instanceof ListValue) ||
           ((ListValue) deriv.value).values.size() > DerivationPruner.opts.maxNumValues)
         return tooManyValues;
     }
+    // doubleSummarizers: Prune when two summarizers (aggregate or superlative) are directly nested
+    // e.g., in (sum (avg ...)) and (min (argmax ...)), the outer operation is redundant
+    if (containsStrategy(doubleSummarizers)) {
+      Formula innerFormula = null;
+      if (deriv.formula instanceof SuperlativeFormula)
+        innerFormula = ((SuperlativeFormula) deriv.formula).head;
+      else if (deriv.formula instanceof AggregateFormula)
+        innerFormula = ((AggregateFormula) deriv.formula).child;
+      if (innerFormula != null &&
+          (innerFormula instanceof SuperlativeFormula || innerFormula instanceof AggregateFormula))
+        return doubleSummarizers;
+    }
+    // multipleSuperlatives: Prune when more than one superlatives are used
+    // (don't need to be adjacent)
+    if (containsStrategy(multipleSuperlatives)) {
+      List<LispTree> stack = new ArrayList<>();
+      int count = 0;
+      stack.add(deriv.formula.toLispTree());
+      while (!stack.isEmpty()) {
+        LispTree tree = stack.remove(stack.size() - 1);
+        if (tree.isLeaf()) {
+          if ("argmax".equals(tree.value) || "argmin".equals(tree.value)) {
+            count++;
+            if (count >= 2)
+              return multipleSuperlatives;
+          }
+        } else {
+          for (LispTree subtree : tree.children)
+            stack.add(subtree);
+        }
+      }
+    }
+    // sameMerge: Prune merge formulas with two identical children
+    if (containsStrategy(sameMerge) && deriv.formula instanceof MergeFormula) {
+      MergeFormula merge = (MergeFormula) deriv.formula;
+      if (merge.child1.equals(merge.child2))
+        return sameMerge;
+    }
+    // mistypedMerge: Prune merge formulas with children of different types
+    if (containsStrategy(mistypedMerge) && deriv.formula instanceof MergeFormula) {
+      MergeFormula merge = (MergeFormula) deriv.formula;
+      SemType type1 = TypeInference.inferType(merge.child1, true);
+      SemType type2 = TypeInference.inferType(merge.child2, true);
+      if (!type1.meet(type2).isValid())
+        return mistypedMerge;
+    }
+    // unsortedMerge: Prune merge formulas where the children's string forms are not lexicographically sorted.
+    //   Will remove redundant (and Y X) when (and Y X) is already present.
+    if (containsStrategy(unsortedMerge) && deriv.formula instanceof MergeFormula) {
+      MergeFormula merge = (MergeFormula) deriv.formula;
+      String child1 = merge.child1.toString(), child2 = merge.child2.toString();
+      if (child1.compareTo(child2) >= 0)
+        return unsortedMerge;
+    }
+    // badSummarizerHead: Prune if the head of a superlative or a non-count aggregate
+    // is empty or is a single object
+    if (containsStrategy(badSummarizerHead)) {
+      Formula innerFormula = null;
+      boolean isCount = false;
+      if (deriv.formula instanceof SuperlativeFormula)
+        innerFormula = ((SuperlativeFormula) deriv.formula).head;
+      else if (deriv.formula instanceof AggregateFormula) {
+        innerFormula = ((AggregateFormula) deriv.formula).child;
+        if (((AggregateFormula) deriv.formula).mode == AggregateFormula.Mode.count)
+          isCount = true;
+      }
+      if (innerFormula != null) {
+        try {
+          TypeInference.inferType(innerFormula);
+          Value innerValue = parser.executor.execute(innerFormula, ex.context).value;
+          if (innerValue instanceof ListValue) {
+            int size = ((ListValue) innerValue).values.size();
+            if (size == 0 || (size == 1 && !(opts.allowCountOne && isCount)))
+              return badSummarizerHead;
+          }
+        } catch (Exception e) {
+          // TypeInference fails; probably because of free variables. No need to do anything.
+        }
+      }
+    }
     return null;
   }
 
@@ -86,74 +158,4 @@ public class DefaultDerivationPruningComputer extends DerivationPruningComputer
     return false;
   }
 
-  // ============================================================
-  // Recursively prune formulas
-  // ============================================================
-
-  @Override
-  public String isPrunedRecursive(Derivation deriv, Formula subformula, Map<String, Object> state) {
-    // doubleSummarizers: Prune when two summarizers (aggregate or superlative) are directly nested
-    // e.g., in (sum (avg ...)) and (min (argmax ...)), the outer operation is redundant
-    if (containsStrategy(doubleSummarizers)) {
-      Formula innerFormula = null;
-      if (subformula instanceof SuperlativeFormula)
-        innerFormula = ((SuperlativeFormula) subformula).head;
-      else if (subformula instanceof AggregateFormula)
-        innerFormula = ((AggregateFormula) subformula).child;
-      if (innerFormula != null &&
-          (innerFormula instanceof SuperlativeFormula || innerFormula instanceof AggregateFormula))
-        return doubleSummarizers;
-    }
-    // sameMerge: Prune merge formulas with two identical children
-    if (containsStrategy(sameMerge) && subformula instanceof MergeFormula) {
-      MergeFormula merge = (MergeFormula) subformula;
-      if (merge.child1.equals(merge.child2))
-        return sameMerge;
-    }
-    // mistypedMerge: Prune merge formulas with children of different types
-    if (containsStrategy(mistypedMerge) && subformula instanceof MergeFormula) {
-      MergeFormula merge = (MergeFormula) subformula;
-      SemType type1 = TypeInference.inferType(merge.child1, true);
-      SemType type2 = TypeInference.inferType(merge.child2, true);
-      if (!type1.meet(type2).isValid())
-        return mistypedMerge;
-    }
-    // unsortedMerge: Prune merge formulas where the children's string forms are not lexicographically sorted.
-    //   Will remove redundant (and Y X) when (and Y X) is already present.
-    if (containsStrategy(unsortedMerge) && subformula instanceof MergeFormula) {
-      MergeFormula merge = (MergeFormula) subformula;
-      String child1 = merge.child1.toString(), child2 = merge.child2.toString();
-      if (child1.compareTo(child2) >= 0)
-        return unsortedMerge;
-    }
-    // badSummarizerHead: Prune if the head of a superlative or a non-count aggregate
-    // is empty or is a single object
-    if (containsStrategy(badSummarizerHead) && DerivationPruner.opts.ensureExecuted) {
-      Formula innerFormula = null;
-      boolean isCount = false;
-      if (subformula instanceof SuperlativeFormula)
-        innerFormula = ((SuperlativeFormula) subformula).head;
-      else if (subformula instanceof AggregateFormula) {
-        innerFormula = ((AggregateFormula) subformula).child;
-        if (((AggregateFormula) subformula).mode == AggregateFormula.Mode.count)
-          isCount = true;
-      }
-      if (innerFormula != null) {
-        try {
-          TypeInference.inferType(innerFormula);
-          Value innerValue = pruner.parser.executor.execute(innerFormula, pruner.ex.context).value;
-          if (innerValue instanceof ListValue) {
-            int size = ((ListValue) innerValue).values.size();
-            if (size == 0 || (size == 1 && !(DerivationPruner.opts.allowCountOne && isCount)))
-              return badSummarizerHead;
-          }
-        } catch (Exception e) {
-          // TypeInference fails; probably because of free variables. No need to do anything.
-        }
-      }
-    }
-    return null;
-  }
-
-
 }

src/edu/stanford/nlp/sempre/Derivation.java

@@ -48,7 +48,7 @@ public class Derivation implements SemanticFn.Callable, HasScore {
   public String canonicalUtterance;
   public boolean allAnchored = true;
   private int[] numAnchors;     // Number of times each token was anchored
-  
+
   /**
   * Information for grammar induction.
   * For each descendant derivation of the body, this class tracks where and what in the head it matches
@@ -459,17 +459,17 @@ public class Derivation implements SemanticFn.Callable, HasScore {
       return 0;
     }
   }
-  
+
   //Used to compare derivations by score, prioritizing the fully anchored.
   public static class AnchorPriorityScoreComparator implements Comparator<Derivation> {
     @Override
     public int compare(Derivation deriv1, Derivation deriv2) {
       boolean deriv1Core = deriv1.allAnchored();
       boolean deriv2Core = deriv2.allAnchored();
-    
+
       if (deriv1Core && !deriv2Core) return -1;
       if (deriv2Core && !deriv1Core) return +1;
-      
+
       if (deriv1.score > deriv2.score) return -1;
       if (deriv1.score < deriv2.score) return +1;
       // Ensure reproducible randomness

src/edu/stanford/nlp/sempre/DerivationPruner.java

@@ -2,8 +2,6 @@ package edu.stanford.nlp.sempre;
 
 import java.util.*;
 
-import com.google.common.base.Function;
-
 import fig.basic.*;
 
 /**
@@ -24,12 +22,6 @@ public class DerivationPruner {
     @Option public int pruningVerbosity = 0;
     @Option(gloss = "(for tooManyValues) maximum denotation size of the final formula")
     public int maxNumValues = 10;
-    @Option(gloss = "(for badSummarizerHead) allow count on sets of size 1")
-    public boolean allowCountOne = false;
-    @Option(gloss = "Ensure that deriv.value contains the executed denotation.")
-    public boolean ensureExecuted = true;
-    @Option(gloss = "If false, apply recursive pruning strategies only at the outermost layer of the formula")
-    public boolean recursivePruning = true;
   }
   public static Options opts = new Options();
 
@@ -92,17 +84,9 @@ public class DerivationPruner {
    */
   public boolean isPruned(Derivation deriv) {
     if (opts.pruningStrategies.isEmpty() && pruningComputers.isEmpty()) return false;
-    if (isPrunedWithoutExecution(deriv)) return true;
-    if (isPrunedGeneral(deriv)) return true;
-    if (isPrunedRecursive(deriv)) return true;
-    return false;
-  }
-
-  // Prune without executing the formula
-  boolean isPrunedWithoutExecution(Derivation deriv) {
     String matchedStrategy;
     for (DerivationPruningComputer computer : pruningComputers) {
-      if ((matchedStrategy = computer.isPrunedWithoutExecution(deriv)) != null) {
+      if ((matchedStrategy = computer.isPruned(deriv)) != null) {
         if (opts.pruningVerbosity >= 2)
           LogInfo.logs("PRUNED [%s] %s", matchedStrategy, deriv.formula);
         return true;
@@ -111,68 +95,6 @@ public class DerivationPruner {
     return false;
   }
 
-  // Prune based on the denotation (general)
-  boolean isPrunedGeneral(Derivation deriv) {
-    if (!opts.ensureExecuted) return false;
-    deriv.ensureExecuted(parser.executor, ex.context);
-    String matchedStrategy;
-    for (DerivationPruningComputer computer : pruningComputers) {
-      if ((matchedStrategy = computer.isPrunedGeneral(deriv)) != null) {
-        if (opts.pruningVerbosity >= 2)
-          LogInfo.logs("PRUNED [%s] %s", matchedStrategy, deriv.formula);
-        return true;
-      }
-    }
-    return false;
-  }
-
-  // Prune based on subformula
-  boolean isPrunedRecursive(Derivation deriv) {
-    if (!opts.recursivePruning) {
-      // If recursivePruning flag is turned off, only look at the outermost layer.
-      if (opts.ensureExecuted)
-        deriv.ensureExecuted(parser.executor, ex.context);
-      String matchedStrategy;
-      for (DerivationPruningComputer computer : pruningComputers) {
-        if ((matchedStrategy = computer.isPrunedRecursive(deriv, deriv.formula, new HashMap<>())) != null) {
-          if (opts.pruningVerbosity >= 2)
-            LogInfo.logs("PRUNED [%s] %s", matchedStrategy, deriv.formula);
-          return true;
-        }
-      }
-    } else {
-      PruningRecurser recurser = new PruningRecurser(deriv);
-      deriv.formula.forEach(recurser);
-      if (recurser.matchedStrategy != null) {
-        if (opts.pruningVerbosity >= 2)
-          LogInfo.logs("PRUNED [%s] %s", recurser.matchedStrategy, deriv.formula);
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Helper class for traversing the formula recursively.
-   */
-  class PruningRecurser implements Function<Formula, Boolean> {
-    final Derivation deriv;
-    final Map<String, Object> state = new HashMap<>();
-    String matchedStrategy = null;
-
-    public PruningRecurser(Derivation deriv) { this.deriv = deriv; }
-
-    @Override
-    public Boolean apply(Formula f) {
-      if (matchedStrategy != null) return true;     // Don't process further
-      for (DerivationPruningComputer computer : pruningComputers) {
-        if ((matchedStrategy = computer.isPrunedRecursive(deriv, f, state)) != null)
-          return true;   // Don't process further
-      }
-      return false;   // Recurse
-    }
-  }
-
   /**
    * Run isPruned with a (temporary) custom set of allowed pruning strategies.
    * If customAllowedPruningStrategies is null, all strategies are allowed.

src/edu/stanford/nlp/sempre/DerivationPruningComputer.java

@@ -1,7 +1,6 @@
 package edu.stanford.nlp.sempre;
 
 import java.util.Collection;
-import java.util.Map;
 
 /**
  * Used to prune formulas during parsing.
@@ -14,9 +13,13 @@ import java.util.Map;
 public abstract class DerivationPruningComputer {
 
   protected final DerivationPruner pruner;
+  protected final Parser parser;
+  protected final Example ex;
 
   public DerivationPruningComputer(DerivationPruner pruner) {
     this.pruner = pruner;
+    this.parser = pruner.parser;
+    this.ex = pruner.ex;
   }
 
   /**
@@ -30,38 +33,11 @@ public abstract class DerivationPruningComputer {
   }
 
   /**
-   * Prune the derivation without executing the formula.
+   * Prune the derivation.
    *
    * To add pruning strategies, override this method.
    * Return the strategy name to prune the formula, and null otherwise.
-   * This method should not execute the formula.
    */
-  public String isPrunedWithoutExecution(Derivation deriv) {
-    return null;
-  }
+  public abstract String isPruned(Derivation deriv);
 
-  /**
-   * Prune the derivation (general).
-   *
-   * The formula will already be executed (i.e., deriv.value is set).
-   *
-   * To add pruning strategies, override this method.
-   * Return the strategy name to prune the formula, and null otherwise.
-   */
-  public String isPrunedGeneral(Derivation deriv) {
-    return null;
-  }
-
-  /**
-   * Prune the derivation based on the subformula.
-   *
-   * DerivationPruner will traverse the formula and call this method on each subformula.
-   * The method can store temporary data in |state|.
-   *
-   * To add pruning strategies, override this method.
-   * Return the strategy name to prune the formula, and null otherwise.
-   */
-  public String isPrunedRecursive(Derivation deriv, Formula subformula, Map<String, Object> state) {
-    return null;
-  }
 }

src/edu/stanford/nlp/sempre/Example.java

@@ -198,7 +198,7 @@ public class Example {
 
   public void preprocess() {
     this.languageInfo = LanguageAnalyzer.getSingleton().analyze(this.utterance);
-    this.targetValue = TargetValuePreprocessor.getSingleton().preprocess(this.targetValue);
+    this.targetValue = TargetValuePreprocessor.getSingleton().preprocess(this.targetValue, this);
   }
 
   public void log() {

src/edu/stanford/nlp/sempre/ExampleUtils.java

@@ -119,6 +119,29 @@ public final class ExampleUtils {
     out.close();
   }
 
+  public static void writePredictionTSV(int iter, String group, Example ex) {
+    String basePath = "preds-iter" + iter + "-" + group + ".tsv";
+    String outPath = Execution.getFile(basePath);
+    if (outPath == null) return;
+    PrintWriter out = IOUtils.openOutAppendHard(outPath);
+
+    List<String> fields = new ArrayList<>();
+    fields.add(ex.id);
+
+    if (!ex.predDerivations.isEmpty()) {
+      Derivation deriv = ex.predDerivations.get(0);
+      if (deriv.value instanceof ListValue) {
+        List<Value> values = ((ListValue) deriv.value).values;
+        for (Value v : values) {
+          fields.add(v.pureString().replaceAll("\\s+", " ").trim());
+        }
+      }
+    }
+
+    out.println(String.join("\t", fields));
+    out.close();
+  }
+
   //read lisptree and write json
   public static void main(String[] args) {
     Dataset dataset = new Dataset();

src/edu/stanford/nlp/sempre/FloatingParser.java

@@ -84,18 +84,34 @@ public class FloatingParser extends Parser {
     @Option(gloss = "DEBUG: Print amount of time spent on each rule")
     public boolean summarizeRuleTime = false;
     @Option(gloss = "Stop the parser if it has used more than this amount of time (in seconds)")
-    public int maxFloatingParsingTime = 600;
+    public int maxFloatingParsingTime = Integer.MAX_VALUE;
   }
 
   public static Options opts = new Options();
 
-  protected List<Rule> orderedFloatingRules;
-  public List<Rule> getOrderedFloatingRules() { return orderedFloatingRules; }
+  public boolean earlyStopOnConsistent = false;
+  public int earlyStopOnNumDerivs = -1;
 
   public FloatingParser(Spec spec) {
     super(spec);
   }
 
+  /**
+   * Set early stopping criteria
+   *
+   * @param onConsistent
+   *    Stop when a consistent derivation is found. (Only triggered when computeExpectedCounts = true)
+   * @param onNumDerivs
+   *    Stop when the number of featurized derivations exceed this number (set to -1 to disable)
+   * @return
+   *    this
+   */
+  public FloatingParser setEarlyStopping(boolean onConsistent, int onNumDerivs) {
+    this.earlyStopOnConsistent = onConsistent;
+    this.earlyStopOnNumDerivs = onNumDerivs;
+    return this;
+  }
+
   /**
    * computeCatUnaryRules, but do not topologically sort floating rules
    */
@@ -379,39 +395,54 @@ class FloatingParserState extends ParserState {
       StopWatch stopWatch = new StopWatch().start();
       String rhs1 = rule.rhs.get(0);
       String rhs2 = rule.rhs.get(1);
-      if (!Rule.isCat(rhs1) || !Rule.isCat(rhs2))
-        throw new RuntimeException("Floating rules with > 1 arguments cannot have tokens on the RHS: " + rule);
 
-      if (FloatingParser.opts.useSizeInsteadOfDepth) {
-        derivLoop:
-          for (int depth1 = 0; depth1 < depth; depth1++) {  // sizes must add up to depth-1 (actually size-1)
-            int depth2 = depth - 1 - depth1;
-            for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, depth1), floatingCell(rhs2, depth2)))
-              for (Derivation deriv1 : group.derivations1)
-                for (Derivation deriv2 : group.derivations2)
-                  if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance))
-                    break derivLoop;
-          }
-      } else {
-        {
-          derivLoop:
-            for (int subDepth = 0; subDepth < depth; subDepth++) {  // depth-1 <=depth-1
-              for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, depth - 1), floatingCell(rhs2, subDepth)))
-                for (Derivation deriv1 : group.derivations1)
-                  for (Derivation deriv2 : group.derivations2)
-                    if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance))
-                      break derivLoop;
-            }
+      if (!Rule.isCat(rhs1) && !Rule.isCat(rhs2)) {  // token token
+        if (depth == (FloatingParser.opts.initialFloatingHasZeroDepth ? 0 : 1)) {
+          applyFloatingRule(rule, depth, null, null, rhs1 + " " + rhs2);
         }
-        {
+
+      } else if (!Rule.isCat(rhs1) && Rule.isCat(rhs2)) {  // token $Cat
+        List<Derivation> derivations = getDerivations(floatingCell(rhs2, depth - 1));
+        for (Derivation deriv : derivations)
+          applyFloatingRule(rule, depth, deriv, null, rhs1 + " " + deriv.canonicalUtterance);
+
+      } else if (Rule.isCat(rhs1) && !Rule.isCat(rhs2)) {  // $Cat token
+        List<Derivation> derivations = getDerivations(floatingCell(rhs1, depth - 1));
+        for (Derivation deriv : derivations)
+          applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance + " " + rhs2);
+
+      } else {  // $Cat $Cat
+        if (FloatingParser.opts.useSizeInsteadOfDepth) {
           derivLoop:
-            for (int subDepth = 0; subDepth < depth - 1; subDepth++) {  // <depth-1 depth-1
-              for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, subDepth), floatingCell(rhs2, depth - 1)))
+            for (int depth1 = 0; depth1 < depth; depth1++) {  // sizes must add up to depth-1 (actually size-1)
+              int depth2 = depth - 1 - depth1;
+              for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, depth1), floatingCell(rhs2, depth2)))
                 for (Derivation deriv1 : group.derivations1)
                   for (Derivation deriv2 : group.derivations2)
                     if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance))
                       break derivLoop;
             }
+        } else {
+          {
+            derivLoop:
+              for (int subDepth = 0; subDepth < depth; subDepth++) {  // depth-1 <=depth-1
+                for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, depth - 1), floatingCell(rhs2, subDepth)))
+                  for (Derivation deriv1 : group.derivations1)
+                    for (Derivation deriv2 : group.derivations2)
+                      if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance))
+                        break derivLoop;
+              }
+          }
+          {
+            derivLoop:
+              for (int subDepth = 0; subDepth < depth - 1; subDepth++) {  // <depth-1 depth-1
+                for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, subDepth), floatingCell(rhs2, depth - 1)))
+                  for (Derivation deriv1 : group.derivations1)
+                    for (Derivation deriv2 : group.derivations2)
+                      if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance))
+                        break derivLoop;
+              }
+          }
         }
       }
       ruleTime.put(rule, ruleTime.getOrDefault(rule, 0L) + stopWatch.stop().ms);
@@ -438,70 +469,107 @@ class FloatingParserState extends ParserState {
       derivations.addAll(myDerivations);
   }
 
+  /**
+   * Build derivations in a thread to allow timeout.
+   */
+  class DerivationBuilder implements Runnable {
+    @Override public void run() {
+      // Base case ($TOKEN, $PHRASE)
+      for (Derivation deriv : gatherTokenAndPhraseDerivations()) {
+        addToChart(anchoredCell(deriv.cat, deriv.start, deriv.end), deriv);
+        addToChart(floatingCell(deriv.cat, 0), deriv);
+      }
+
+      Set<String> categories = new HashSet<>();
+      for (Rule rule : parser.grammar.rules)
+        categories.add(rule.lhs);
+
+      if (Parser.opts.verbose >= 1)
+        LogInfo.begin_track_printAll("Anchored");
+      // Build up anchored derivations (like the BeamParser)
+      int numTokens = ex.numTokens();
+      for (int len = 1; len <= numTokens; len++) {
+        for (int i = 0; i + len <= numTokens; i++)  {
+          buildAnchored(i, i + len);
+          for (String cat : categories) {
+            String cell = anchoredCell(cat, i, i + len).toString();
+            pruneCell(cell, chart.get(cell));
+          }
+        }
+      }
+      if (Parser.opts.verbose >= 1)
+        LogInfo.end_track();
+
+      // Build up floating derivations
+      for (int depth = (FloatingParser.opts.initialFloatingHasZeroDepth ? 0 : 1); depth <= FloatingParser.opts.maxDepth; depth++) {
+        if (Parser.opts.verbose >= 1)
+          LogInfo.begin_track_printAll("%s = %d", FloatingParser.opts.useSizeInsteadOfDepth ? "SIZE" : "DEPTH", depth);
+        buildFloating(depth);
+        for (String cat : categories) {
+          String cell = floatingCell(cat, depth).toString();
+          pruneCell(cell, chart.get(cell));
+        }
+        if (Parser.opts.verbose >= 1)
+          LogInfo.end_track();
+        // Early stopping
+        if (computeExpectedCounts && ((FloatingParser) parser).earlyStopOnConsistent) {
+          // Consistent derivation found?
+          String cell = floatingCell(Rule.rootCat, depth).toString();
+          List<Derivation> rootDerivs = chart.get(cell);
+          if (rootDerivs != null) {
+            for (Derivation rootDeriv : rootDerivs) {
+              rootDeriv.ensureExecuted(parser.executor, ex.context);
+              if (parser.valueEvaluator.getCompatibility(ex.targetValue, rootDeriv.value) == 1) {
+                LogInfo.logs("Early stopped: consistent derivation found at depth = %d", depth);
+                return;
+              }
+            }
+          }
+        }
+        if (((FloatingParser) parser).earlyStopOnNumDerivs > 0) {
+          // Too many derivations generated?
+          if (numOfFeaturizedDerivs > ((FloatingParser) parser).earlyStopOnNumDerivs) {
+            LogInfo.logs("Early stopped: number of derivations exceeded at depth = %d", depth);
+            return;
+          }
+        }
+      }
+    }
+  }
+
+  public void buildDerivations() {
+    DerivationBuilder derivBuilder = new DerivationBuilder();
+    if (FloatingParser.opts.maxFloatingParsingTime == Integer.MAX_VALUE) {
+      derivBuilder.run();
+    } else {
+      Thread parsingThread = new Thread(derivBuilder);
+      parsingThread.start();
+      try {
+        parsingThread.join(FloatingParser.opts.maxFloatingParsingTime * 1000);
+        if (parsingThread.isAlive()) {
+          // This will only interrupt first or second passes, not the final candidate collection.
+          LogInfo.warnings("Parsing time exceeded %d seconds. Will now interrupt ...", FloatingParser.opts.maxFloatingParsingTime);
+          timeout = true;
+          parsingThread.interrupt();
+          parsingThread.join();
+        }
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+        LogInfo.fails("FloatingParser error: %s", e);
+      }
+    }
+    evaluation.add("timeout", timeout);
+  }
+
+  // ============================================================
+  // Main entry point
+  // ============================================================
+
   @Override public void infer() {
     LogInfo.begin_track_printAll("FloatingParser.infer()");
     ruleTime = new HashMap<>();
 
-    // Base case ($TOKEN, $PHRASE)
-    for (Derivation deriv : gatherTokenAndPhraseDerivations()) {
-      addToChart(anchoredCell(deriv.cat, deriv.start, deriv.end), deriv);
-      addToChart(floatingCell(deriv.cat, 0), deriv);
-    }
-
-    Set<String> categories = new HashSet<>();
-    for (Rule rule : parser.grammar.rules)
-      categories.add(rule.lhs);
-
-    if (Parser.opts.verbose >= 1)
-      LogInfo.begin_track_printAll("Anchored");
-    // Build up anchored derivations (like the BeamParser)
-    int numTokens = ex.numTokens();
-    for (int len = 1; len <= numTokens; len++) {
-      for (int i = 0; i + len <= numTokens; i++)  {
-        buildAnchored(i, i + len);
-        for (String cat : categories) {
-          String cell = anchoredCell(cat, i, i + len).toString();
-          pruneCell(cell, chart.get(cell));
-        }
-      }
-    }
-    if (Parser.opts.verbose >= 1)
-      LogInfo.end_track();
-
-    // Build up floating derivations
-    // Timeout if taking too long
-    timeout = false;
-    Thread parsingThread = new Thread(new Runnable() {
-      @Override
-      public void run() {
-        for (int depth = (FloatingParser.opts.initialFloatingHasZeroDepth ? 0 : 1); depth <= FloatingParser.opts.maxDepth; depth++) {
-          if (Parser.opts.verbose >= 1)
-            LogInfo.begin_track_printAll("%s = %d", FloatingParser.opts.useSizeInsteadOfDepth ? "SIZE" : "DEPTH", depth);
-          buildFloating(depth);
-          for (String cat : categories) {
-            String cell = floatingCell(cat, depth).toString();
-            pruneCell(cell, chart.get(cell));
-          }
-          if (Parser.opts.verbose >= 1)
-            LogInfo.end_track();
-        }
-      }
-    });
-    parsingThread.start();
-    try {
-      parsingThread.join(FloatingParser.opts.maxFloatingParsingTime * 1000);
-      if (parsingThread.isAlive()) {
-        // This will only interrupt first or second passes, not the final candidate collection.
-        LogInfo.warnings("Parsing time exceeded %d seconds. Will now interrupt ...", FloatingParser.opts.maxFloatingParsingTime);
-        timeout = true;
-        parsingThread.interrupt();
-        parsingThread.join();
-      }
-    } catch (InterruptedException e) {
-      e.printStackTrace();
-      LogInfo.fails("DPParser error: %s", e);
-    }
-    evaluation.add("timeout", timeout);
+    buildDerivations();
 
     if (FloatingParser.opts.summarizeRuleTime) summarizeRuleTime();
 

src/edu/stanford/nlp/sempre/Formulas.java

@@ -311,7 +311,7 @@ public abstract class Formulas {
     // recurse
     if (f instanceof AggregateFormula) {
       AggregateFormula aggFormula = (AggregateFormula) f;
-      extractSubpartsRecursive(aggFormula, res);
+      extractSubpartsRecursive(aggFormula.child, res);
     } else if (f instanceof CallFormula) {
       CallFormula callFormula = (CallFormula) f;
       extractSubpartsRecursive(callFormula.func, res);

src/edu/stanford/nlp/sempre/FuzzyMatchFn.java

@@ -39,6 +39,9 @@ public class FuzzyMatchFn extends SemanticFn {
     }
   }
 
+  public FuzzyMatchFnMode getMode() { return mode; }
+  public boolean getMatchAny() { return matchAny; }
+
   @Override
   public DerivationStream call(Example ex, Callable c) {
     return new LazyFuzzyMatchFnDerivs(ex, c, mode, matchAny);

src/edu/stanford/nlp/sempre/Grammar.java

@@ -44,7 +44,7 @@ public class Grammar {
   // All the rules in the grammar.  Each parser can read these and transform
   // them however the parser wishes.
   // This contains binarized rules
-  ArrayList<Rule> rules = new ArrayList<>();
+  protected ArrayList<Rule> rules = new ArrayList<>();
   public List<Rule> getRules() { return rules; }
 
   Map<String, LispTree> macros = new HashMap<>();  // Map from macro name to its replacement value
@@ -260,7 +260,7 @@ public class Grammar {
     return cat;
   }
 
-  private void interpretRule(LispTree tree) {
+  protected void interpretRule(LispTree tree) {
     if (tree.children.size() < 4)
       throw new RuntimeException("Invalid rule: " + tree);
 
@@ -360,7 +360,7 @@ public class Grammar {
 
   // Generate intermediate categories for binarization.
   public static final String INTERMEDIATE_PREFIX = "$Intermediate";
-  private int freshCatIndex = 0;
+  protected int freshCatIndex = 0;
   private String generateFreshCat() {
     freshCatIndex++;
     return INTERMEDIATE_PREFIX + freshCatIndex;
@@ -368,6 +368,9 @@ public class Grammar {
   public static boolean isIntermediate(String cat) {
     return cat.startsWith(INTERMEDIATE_PREFIX);
   }
+  public int getFreshCatIndex() {
+    return freshCatIndex;
+  }
 
   // Create multiple versions of this rule if there are optional RHS.
   // Restriction: must be able to split the RHS into two halves, each of

src/edu/stanford/nlp/sempre/JavaExecutor.java

@@ -98,6 +98,30 @@ public class JavaExecutor extends Executor {
     public static String plus(String a, String b, String c, String d, String e, String f, String g) {
       return a + b + c + d + e + f + g;
     }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h) {
+      return a + b + c + d + e + f + g + h;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i) {
+      return a + b + c + d + e + f + g + h + i;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j) {
+      return a + b + c + d + e + f + g + h + i + j;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j, String k) {
+      return a + b + c + d + e + f + g + h + i + j + k;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j, String k, String l) {
+      return a + b + c + d + e + f + g + h + i + j + k + l;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j, String k, String l, String m) {
+      return a + b + c + d + e + f + g + h + i + j + k + l + m;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j, String k, String l, String m, String n) {
+      return a + b + c + d + e + f + g + h + i + j + k + l + m + n;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j, String k, String l, String m, String n, String o) {
+      return a + b + c + d + e + f + g + h + i + j + k + l + m + n + o;
+    }
     private static String toString(Object x) {
       if (x instanceof String)
         return (String) x;

src/edu/stanford/nlp/sempre/Learner.java

@@ -27,6 +27,8 @@ public class Learner {
 
     @Option(gloss = "Write predDerivations to examples file (huge)")
     public boolean outputPredDerivations = false;
+    @Option(gloss = "Write predicted values to a TSV file")
+    public boolean outputPredValues = false;
 
     @Option(gloss = "Dump all features and compatibility scores")
     public boolean dumpFeaturesAndCompatibility = false;
@@ -148,7 +150,7 @@ public class Learner {
     params.update(counts);
     LogInfo.end_track();
   }
-  
+
   public void onlineLearnExampleByFormula(Example ex, List<Formula> formulas) {
     HashMap<String, Double> counts = new HashMap<>();
     for (Derivation deriv : ex.predDerivations)
@@ -222,8 +224,11 @@ public class Learner {
           addFeedback(ex);
 
         // Write out examples and predictions
-        if (opts.outputPredDerivations && Builder.opts.parser.equals("FloatingParser")) {
-          ExampleUtils.writeParaphraseSDF(iter, group, ex, opts.outputPredDerivations);
+        if (opts.outputPredDerivations) {
+          ExampleUtils.writeParaphraseSDF(iter, group, ex, true);
+        }
+        if (opts.outputPredValues) {
+          ExampleUtils.writePredictionTSV(iter, group, ex);
         }
 
         // To save memory
@@ -307,6 +312,7 @@ public class Learner {
     // evaluation.add(LexiconFn.lexEval);
     evaluation.logStats(prefix);
     evaluation.putOutput(prefix);
+    evaluation.putOutput(prefix.replaceAll("iter=", "").replace('.', '_'));
   }
 
   private void printLearnerEventsIter(Example ex, int iter, String group) {

src/edu/stanford/nlp/sempre/NameValue.java

@@ -42,6 +42,7 @@ public class NameValue extends Value {
   }
 
   @Override public String sortString() { return id; }
+  @Override public String pureString() { return description == null ? id : description; }
 
   @Override public int hashCode() { return id.hashCode(); }
   @Override public boolean equals(Object o) {

src/edu/stanford/nlp/sempre/NumberValue.java

@@ -40,6 +40,7 @@ public class NumberValue extends Value {
   }
 
   @Override public String sortString() { return "" + value; }
+  @Override public String pureString() { return "" + value; }
 
   @Override public int hashCode() { return Double.valueOf(value).hashCode(); }
   @Override public boolean equals(Object o) {

src/edu/stanford/nlp/sempre/Parser.java

@@ -56,7 +56,7 @@ public abstract class Parser {
 
     @Option(gloss = "Dump all features (for debugging)")
     public boolean dumpAllFeatures = false;
-    
+
     @Option(gloss = "Call SetEvaluation during parsing")
     public boolean callSetEvaluation = true;
   }
@@ -100,8 +100,8 @@ public abstract class Parser {
     this.valueEvaluator = spec.valueEvaluator;
 
     computeCatUnaryRules();
-    LogInfo.logs("Parser: %d catUnaryRules (sorted), %d nonCatUnaryRules (in trie)",
-        catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size());
+    LogInfo.logs("%s: %d catUnaryRules (sorted), %d nonCatUnaryRules (in trie)",
+        this.getClass().getSimpleName(), catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size());
   }
 
   // If grammar changes, then we might need to update aspects of the parser.
@@ -210,6 +210,7 @@ public abstract class Parser {
     int correctIndexAfterParse = -1;
     double maxCompatibility = 0.0;
     double[] compatibilities = null;
+    int numCorrect = 0, numPartialCorrect = 0, numIncorrect = 0;
 
     if (ex.targetValue != null) {
       compatibilities = new double[numCandidates];
@@ -221,6 +222,14 @@ public abstract class Parser {
           correctIndex = i;
         // record maximum compatibility for partial oracle
         maxCompatibility = Math.max(compatibilities[i], maxCompatibility);
+        // Count
+        if (compatibilities[i] == 1) {
+          numCorrect++;
+        } else if (compatibilities[i] == 0) {
+          numIncorrect++;
+        } else {
+          numPartialCorrect++;
+        }
       }
       // What if we only had parsed bottom up?
       for (int i = 0; i < numCandidates; i++) {
@@ -334,6 +343,9 @@ public abstract class Parser {
     evaluation.add("numCandidates", numCandidates);  // From this parse
     if (numCandidates > 0)
       evaluation.add("parsedNumCandidates", numCandidates);
+    evaluation.add("numCorrect", numCorrect);
+    evaluation.add("numPartialCorrect", numPartialCorrect);
+    evaluation.add("numIncorrect", numIncorrect);
 
     // Add parsing stats
     evaluation.add(state.evaluation);

src/edu/stanford/nlp/sempre/ParserState.java

@@ -24,7 +24,7 @@ public abstract class ParserState {
   }
   public static Options opts = new Options();
 
-  public enum CustomExpectedCount { NONE, UNIFORM, TOP, RANDOM, }
+  public enum CustomExpectedCount { NONE, UNIFORM, TOP, TOPALT, RANDOM, }
 
   //// Input: specification of how to parse
 
@@ -130,7 +130,7 @@ public abstract class ParserState {
       for (Derivation deriv : derivations)
         deriv.score += Parser.opts.derivationScoreRandom.nextDouble() * Parser.opts.derivationScoreNoise;
     }
-    
+
     Derivation.sortByScore(derivations);
 
     // Print out information
@@ -275,7 +275,7 @@ public abstract class ParserState {
     predScores = new double[n];
     // For update schemas that choose one good and one bad candidate to update
     int[] goodAndBad = null;
-    if (opts.customExpectedCounts == CustomExpectedCount.TOP) {
+    if (opts.customExpectedCounts == CustomExpectedCount.TOP || opts.customExpectedCounts == CustomExpectedCount.TOPALT) {
       goodAndBad = getTopDerivations(derivations);
       if (goodAndBad == null) return;
     } else if (opts.customExpectedCounts == CustomExpectedCount.RANDOM) {
@@ -300,6 +300,10 @@ public abstract class ParserState {
           trueScores[i] = (i == goodAndBad[0]) ? 0 : Double.NEGATIVE_INFINITY;
           predScores[i] = (i == goodAndBad[1]) ? 0 : Double.NEGATIVE_INFINITY;
           break;
+        case TOPALT:
+          trueScores[i] = (i == goodAndBad[0]) ? 0 : Double.NEGATIVE_INFINITY;
+          predScores[i] = (i == goodAndBad[0] || i == goodAndBad[1]) ? deriv.score : Double.NEGATIVE_INFINITY;
+          break;
         default:
           throw new RuntimeException("Unknown customExpectedCounts: " + opts.customExpectedCounts);
       }

src/edu/stanford/nlp/sempre/Rule.java

@@ -31,7 +31,7 @@ public class Rule {
   public final SemanticFn sem;  // Takes derivations corresponding to RHS categories and produces a set of derivations corresponding to LHS.
   public List<Pair<String, Double>> info;  // Extra info
   public RuleSource source = null; // for tracking where the rule comes from when they are induced
-  
+
   // Cache the semanticRepn
   public String getSemRepn() {
     if (semRepn == null) semRepn = sem.getClass().getSimpleName();
@@ -51,9 +51,9 @@ public class Rule {
   public String toString() {
     if (stringRepn == null) {
       String semStr = sem == null? "NullSemanticFn" : sem.toString();
-      int maxLength = 100;
-      if (semStr.length() > maxLength)
-        semStr = String.format("%s...(%d total)", semStr.substring(0,maxLength), semStr.length());
+      //int maxLength = 100;
+      //if (semStr.length() > maxLength)
+      //  semStr = String.format("%s...(%d total)", semStr.substring(0,maxLength), semStr.length());
       stringRepn = lhs + " -> " + (rhs == null ? "" : Joiner.on(' ').join(rhs)) + " " + semStr;
     }
     return stringRepn;
@@ -139,13 +139,13 @@ public class Rule {
     else
       return f == 1.0 ? false : !FloatingParser.opts.defaultIsFloating;
   }
-  
+
   public boolean isInduced() {
     double a = getInfoTag("induced");
     if (a == 1.0) return true;
     return false;
   }
-  
+
   @Override
   public boolean equals(Object o) {
     if (!(o instanceof Rule)) return false;
@@ -155,7 +155,7 @@ public class Rule {
   public int hashCode() {
     return this.toString().hashCode();
   }
-  
+
   public String toJson() {
     Map<String, Object> jsonMap = new LinkedHashMap<>();
     jsonMap.put("lhs", lhs);

src/edu/stanford/nlp/sempre/StringValue.java

@@ -20,6 +20,7 @@ public class StringValue extends Value {
   }
 
   @Override public String sortString() { return "\"" + value + "\""; }
+  @Override public String pureString() { return value; }
 
   @Override public int hashCode() { return value.hashCode(); }
   @Override public boolean equals(Object o) {

src/edu/stanford/nlp/sempre/TargetValuePreprocessor.java

@@ -27,10 +27,10 @@ public abstract class TargetValuePreprocessor {
   }
   public static void setSingleton(TargetValuePreprocessor processor) { singleton = processor; }
 
-  public abstract Value preprocess(Value value);
+  public abstract Value preprocess(Value value, Example ex);
 
 }
 
 class IdentityTargetValuePreprocessor extends TargetValuePreprocessor {
-  public Value preprocess(Value value) { return value; }
+  public Value preprocess(Value value, Example ex) { return value; }
 }

src/edu/stanford/nlp/sempre/TypeInference.java

@@ -227,11 +227,11 @@ public final class TypeInference {
 
     } else if (formula instanceof AggregateFormula) {
       AggregateFormula aggregate = (AggregateFormula) formula;
-      inferType(aggregate.child, env, SemType.anyType);
+      SemType childType = inferType(aggregate.child, env, SemType.anyType);
       if (aggregate.mode == AggregateFormula.Mode.count)
         return check(SemType.numberType.meet(type));
       else
-        return check(SemType.numberOrDateType.meet(type));
+        return check(SemType.numberOrDateType.meet(type).meet(childType));
 
     } else if (formula instanceof ArithmeticFormula) {
       ArithmeticFormula arith = (ArithmeticFormula) formula;

src/edu/stanford/nlp/sempre/UriValue.java

@@ -20,6 +20,9 @@ public class UriValue extends Value {
     return tree;
   }
 
+  @Override public String sortString() { return "" + value; }
+  @Override public String pureString() { return "" + value; }
+
   @Override public int hashCode() { return value.hashCode(); }
   @Override public boolean equals(Object o) {
     if (this == o) return true;

src/edu/stanford/nlp/sempre/Value.java

@@ -24,6 +24,9 @@ public abstract class Value {
   // (optional) String used for sorting Values. The default is to call toString()
   public String sortString() { return toString(); }
 
+  // (optional) String without the LispTree structure. The default is to call toString()
+  public String pureString() { return toString(); }
+
   @JsonCreator
   public static Value fromString(String str) {
     return Values.fromLispTree(LispTree.proto.parseFromString(str));

src/edu/stanford/nlp/sempre/cprune/CPruneDerivInfo.java

@@ -0,0 +1,13 @@
+package edu.stanford.nlp.sempre.cprune;
+
+import java.util.Map;
+import java.util.List;
+
+public class CPruneDerivInfo {
+
+  public Map<String, Symbol> treeSymbols;
+  public Map<String, Symbol> ruleSymbols;
+  public List<String> customRuleStrings;
+  public boolean containsCrossReference;
+
+}

src/edu/stanford/nlp/sempre/cprune/CPruneFloatingParser.java

@@ -0,0 +1,119 @@
+package edu.stanford.nlp.sempre.cprune;
+
+import java.util.List;
+
+import edu.stanford.nlp.sempre.*;
+import fig.basic.LogInfo;
+
+/**
+ * A parser that first tries to exploit the macro grammar and only fall back to full search when needed.
+ */
+public class CPruneFloatingParser extends FloatingParser {
+
+  FloatingParser exploreParser;
+
+  public CPruneFloatingParser(Spec spec) {
+    super(spec);
+    exploreParser = new FloatingParser(spec).setEarlyStopping(true, CollaborativePruner.opts.maxDerivations);
+  }
+
+  @Override
+  public void onBeginDataGroup(int iter, int numIters, String group) {
+    if (CollaborativePruner.uidToCachedNeighbors == null) {
+      CollaborativePruner.customGrammar.init(grammar);
+      CollaborativePruner.loadNeighbors();
+    }
+    CollaborativePruner.stats.reset(iter + "." + group);
+  }
+
+  @Override
+  public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) {
+    return new CPruneFloatingParserState(this, params, ex, computeExpectedCounts);
+  }
+
+}
+
+class CPruneFloatingParserState extends ParserState {
+
+  public CPruneFloatingParserState(Parser parser, Params params, Example ex, boolean computeExpectedCounts) {
+    super(parser, params, ex, computeExpectedCounts);
+  }
+
+  @Override
+  public void infer() {
+    LogInfo.begin_track("CPruneFloatingParser.infer()");
+    boolean exploitSucceeds = exploit();
+    if (computeExpectedCounts) {
+      LogInfo.begin_track("Summary of Collaborative Pruning");
+      LogInfo.logs("Exploit succeeds: " + exploitSucceeds);
+      LogInfo.logs("Exploit success rate: " + CollaborativePruner.stats.successfulExploit + "/" + CollaborativePruner.stats.totalExploit);
+      LogInfo.end_track();
+    }
+    // Explore only on the first training iteration
+    if (CollaborativePruner.stats.iter.equals("0.train") && computeExpectedCounts && !exploitSucceeds
+        && (CollaborativePruner.stats.totalExplore <= CollaborativePruner.opts.maxExplorationIters)) {
+      explore();
+      LogInfo.logs("Consistent pattern: " + CollaborativePruner.getConsistentPattern(ex));
+      LogInfo.logs("Explore success rate: " + CollaborativePruner.stats.successfulExplore + "/" + CollaborativePruner.stats.totalExplore);
+    }
+    LogInfo.end_track();
+  }
+
+  public void explore() {
+    LogInfo.begin_track("Explore");
+    CollaborativePruner.initialize(ex, CollaborativePruner.Mode.EXPLORE);
+    ParserState exploreParserState = ((CPruneFloatingParser) parser).exploreParser.newParserState(params, ex, computeExpectedCounts);
+    exploreParserState.infer();
+    predDerivations.clear();
+    predDerivations.addAll(exploreParserState.predDerivations);
+    expectedCounts = exploreParserState.expectedCounts;
+    if (computeExpectedCounts) {
+      for (Derivation deriv : predDerivations)
+        CollaborativePruner.updateConsistentPattern(parser.valueEvaluator, ex, deriv);
+    }
+    CollaborativePruner.stats.totalExplore += 1;
+    if (CollaborativePruner.foundConsistentDerivation)
+      CollaborativePruner.stats.successfulExplore += 1;
+    LogInfo.end_track();
+  }
+
+  public boolean exploit() {
+    LogInfo.begin_track("Exploit");
+    CollaborativePruner.initialize(ex, CollaborativePruner.Mode.EXPLOIT);
+    Grammar miniGrammar = new MiniGrammar(CollaborativePruner.predictedRules);
+    Parser exploitParser = new FloatingParser(new Parser.Spec(miniGrammar, parser.extractor, parser.executor, parser.valueEvaluator));
+    ParserState exploitParserState = exploitParser.newParserState(params, ex, computeExpectedCounts);
+    exploitParserState.infer();
+    predDerivations.clear();
+    predDerivations.addAll(exploitParserState.predDerivations);
+    expectedCounts = exploitParserState.expectedCounts;
+    if (computeExpectedCounts) {
+      for (Derivation deriv : predDerivations)
+        CollaborativePruner.updateConsistentPattern(parser.valueEvaluator, ex, deriv);
+    }
+    boolean succeeds = CollaborativePruner.foundConsistentDerivation;
+    CollaborativePruner.stats.totalExploit += 1;
+    if (succeeds)
+      CollaborativePruner.stats.successfulExploit += 1;
+    LogInfo.end_track();
+    return succeeds;
+  }
+}
+
+// ============================================================
+// Helper classes
+// ============================================================
+
+class MiniGrammar extends Grammar {
+
+  public MiniGrammar(List<Rule> rules) {
+    this.rules.addAll(rules);
+    if (CollaborativePruner.opts.verbose >= 2) {
+      LogInfo.begin_track("MiniGrammar Rules");
+      for (Rule rule : rules)
+        LogInfo.logs("%s %s", rule, rule.isAnchored() ? "[A]" : "[F]");
+      LogInfo.end_track();
+    }
+  }
+
+}

src/edu/stanford/nlp/sempre/cprune/CPruneStats.java

@@ -0,0 +1,20 @@
+package edu.stanford.nlp.sempre.cprune;
+
+/**
+ * Stores various statistic.
+ */
+public class CPruneStats {
+  public String iter;
+  public int totalExplore = 0;
+  public int successfulExplore = 0;
+  public int totalExploit = 0;
+  public int successfulExploit = 0;
+
+  public void reset(String iter) {
+    this.iter = iter;
+    this.totalExplore = 0;
+    this.successfulExplore = 0;
+    this.totalExploit = 0;
+    this.successfulExploit = 0;
+  }
+}

src/edu/stanford/nlp/sempre/cprune/CollaborativePruner.java

@@ -0,0 +1,187 @@
+package edu.stanford.nlp.sempre.cprune;
+
+import java.io.*;
+import java.util.*;
+
+import fig.basic.*;
+import edu.stanford.nlp.sempre.*;
+
+/**
+ * Static class for collaborative pruning.
+ */
+public class CollaborativePruner {
+  public static class Options {
+    @Option(gloss = "Logging verbosity")
+    public int verbose = 0;
+    @Option(gloss = "K = Maximum number of nearest-neighbor examples to consider (-1 to use all examples so far)")
+    public int maxNumNeighbors = -1;
+    @Option(gloss = "Load cached neighbors from this file")
+    public String neighborFilePath = null;
+    @Option(gloss = "Maximum number of matching patterns (default = use all patterns)")
+    public int maxPredictedPatterns = Integer.MAX_VALUE;
+    @Option(gloss = "Maximum number of derivations per example")
+    public int maxDerivations = 5000;
+    @Option(gloss = "Maximum number of times to fall back to exploration")
+    public int maxExplorationIters = Integer.MAX_VALUE;
+  }
+
+  public static Options opts = new Options();
+
+  public enum Mode { EXPLORE, EXPLOIT, NONE }
+
+  public static Mode mode = Mode.NONE;
+  public static CPruneStats stats = new CPruneStats();
+  public static CustomGrammar customGrammar = new CustomGrammar();
+
+  // Static class; do not instantiate
+  private CollaborativePruner() { throw new RuntimeException("Cannot instantiate CollaborativePruner"); }
+
+  // Global variables
+  // Nearest neighbors
+  static Map<String, List<String>> uidToCachedNeighbors;
+  // uid => pattern
+  static Map<String, FormulaPattern> consistentPattern = new HashMap<>();
+  // patternString => customRuleString
+  static Map<String, Set<String>> customRules = new HashMap<>();
+  // set of patternStrings
+  static Set<String> allConsistentPatterns = new HashSet<>();
+
+  // Example-level variables
+  public static boolean foundConsistentDerivation = false;
+  public static Map<String, FormulaPattern> predictedPatterns;
+  public static List<Rule> predictedRules;
+
+  /**
+   * Read the cached neighbors file.
+   * Line Format: ex_id [tab] neighbor_id1,neighbor_id2,...
+   */
+  public static void loadNeighbors() {
+    if (opts.neighborFilePath == null) {
+      LogInfo.logs("neighborFilePath is null.");
+      return;
+    }
+    LogInfo.begin_track("Loading cached neighbors from %s", opts.neighborFilePath);
+    uidToCachedNeighbors = new HashMap<>();
+    try {
+      BufferedReader reader = IOUtils.openIn(opts.neighborFilePath);
+      String line;
+      while ((line = reader.readLine()) != null) {
+        String[] tokens = line.split("\t");
+        String uid = tokens[0];
+        String[] nids = tokens[1].split(",");
+        uidToCachedNeighbors.put(uid, Arrays.asList(nids));
+      }
+      reader.close();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    LogInfo.end_track();
+  }
+
+  public static void initialize(Example ex, Mode mode) {
+    CollaborativePruner.mode = mode;
+    predictedRules = null;
+    predictedPatterns = null;
+    foundConsistentDerivation = false;
+    if (mode == Mode.EXPLOIT) {
+      preprocessExample(ex);
+    }
+  }
+
+  static void preprocessExample(Example ex) {
+    Map<String, FormulaPattern> patternFreqMap = new HashMap<>();
+    List<String> cachedNeighbors = uidToCachedNeighbors.get(ex.id);
+    int total = 0;
+
+    // Gather the neighbors
+    if (opts.maxNumNeighbors > 0) {
+      for (String nid : cachedNeighbors) {
+        // Only get examples that have been previously processed + found a consistent formula
+        if (!consistentPattern.containsKey(nid))
+          continue;
+
+        String neighborPattern = consistentPattern.get(nid).pattern;
+        if (!patternFreqMap.containsKey(neighborPattern))
+          patternFreqMap.put(neighborPattern, new FormulaPattern(neighborPattern, 0));
+        patternFreqMap.get(neighborPattern).frequency++;
+        total++;
+        if (total >= opts.maxNumNeighbors)
+          break;
+      }
+    } else {
+      for (String patternString : allConsistentPatterns) {
+        patternFreqMap.put(patternString, new FormulaPattern(patternString, 1));
+      }
+    }
+
+    // Sort by frequency (more frequent = smaller; see FormulaPattern.compareTo)
+    List<Map.Entry<String, FormulaPattern>> patternFreqEntries = new ArrayList<>(patternFreqMap.entrySet());
+    patternFreqEntries.sort(new ValueComparator<>(false));
+
+    // Gather the patterns
+    LogInfo.begin_track("Predicted patterns");
+    int rank = 0;
+    Set<String> predictedRulesStrings = new HashSet<>();
+    predictedPatterns = new HashMap<>();
+    for (Map.Entry<String, FormulaPattern> entry : patternFreqEntries) {
+      FormulaPattern newPattern = entry.getValue();
+      predictedPatterns.put(newPattern.pattern, newPattern);
+      predictedRulesStrings.addAll(customRules.get(newPattern.pattern));
+      LogInfo.logs((rank + 1) + ". " + newPattern.pattern + " (" + newPattern.frequency + ")");
+      rank++;
+      if (rank >= opts.maxPredictedPatterns)
+        break;
+    }
+    // Gather the rules
+    predictedRules = customGrammar.getRules(predictedRulesStrings);
+    LogInfo.end_track();
+  }
+
+  public static String getPatternString(Derivation deriv) {
+    if (deriv.cat.equals("$TOKEN") || deriv.cat.equals("$PHRASE")
+        || deriv.cat.equals("$LEMMA_TOKEN") || deriv.cat.equals("$LEMMA_PHRASE")) {
+      return deriv.cat;
+    } else {
+      return FormulaPattern.convertToIndexedPattern(deriv);
+    }
+  }
+
+  public static void addRules(String patternString, Derivation deriv, Example ex) {
+    if (!customRules.containsKey(patternString)) {
+      customRules.put(patternString, new HashSet<String>());
+    }
+    Set<String> parsedCustomRules = customGrammar.addCustomRule(deriv, ex);
+    customRules.get(patternString).addAll(parsedCustomRules);
+  }
+
+  /**
+   * Get called when a (consistent) formula is found.
+   * Update the consistent patterns.
+   */
+  public static void updateConsistentPattern(ValueEvaluator evaluator, Example ex, Derivation deriv) {
+    String uid = ex.id;
+    if (ex.targetValue != null)
+      deriv.compatibility = evaluator.getCompatibility(ex.targetValue, deriv.value);
+
+    if (deriv.isRootCat() && deriv.compatibility == 1) {
+      foundConsistentDerivation = true;
+      LogInfo.logs("Found consistent deriv: %s", deriv);
+
+      String patternString = getPatternString(deriv);
+      FormulaPattern newConsistentPattern = new FormulaPattern(patternString, 0);
+      newConsistentPattern.score = deriv.getScore();
+
+      FormulaPattern oldConsistentPattern = consistentPattern.get(uid);
+      if (oldConsistentPattern == null || newConsistentPattern.score > oldConsistentPattern.score) {
+        addRules(patternString, deriv, ex);
+        consistentPattern.put(uid, newConsistentPattern);
+        allConsistentPatterns.add(patternString);
+      }
+    }
+  }
+
+  public static FormulaPattern getConsistentPattern(Example ex) {
+    return consistentPattern.get(ex.id);
+  }
+
+}

src/edu/stanford/nlp/sempre/cprune/CustomGrammar.java

@@ -0,0 +1,268 @@
+package edu.stanford.nlp.sempre.cprune;
+
+import java.util.*;
+
+import edu.stanford.nlp.sempre.*;
+import fig.basic.*;
+
+public class CustomGrammar extends Grammar {
+  public static class Options {
+    @Option(gloss = "Whether to decompose the templates into multiple rules")
+    public boolean enableTemplateDecomposition = true;
+  }
+
+  public static Options opts = new Options();
+
+  public static final Set<String> baseCategories = new HashSet<String>(Arrays.asList(
+      Rule.tokenCat, Rule.phraseCat, Rule.lemmaTokenCat, Rule.lemmaPhraseCat,
+      "$Unary", "$Binary", "$Entity", "$Property"));
+
+  ArrayList<Rule> baseRules = new ArrayList<>();
+  // symbolicFormulas => symbolicFormula ID
+  Map<String, Integer> symbolicFormulas = new HashMap<>();
+  // indexedSymbolicFormula => customRuleString
+  Map<String, Set<String>> customRules = new HashMap<>();
+  // customRuleString => Binarized rules
+  Map<String, Set<Rule>> customBinarizedRules = new HashMap<>();
+
+  public void init(Grammar initGrammar) {
+    baseRules = new ArrayList<>();
+    for (Rule rule : initGrammar.getRules()) {
+      if (baseCategories.contains(rule.lhs)) {
+        baseRules.add(rule);
+      }
+    }
+    this.freshCatIndex = initGrammar.getFreshCatIndex();
+  }
+
+  public List<Rule> getRules(Collection<String> customRuleStrings) {
+    Set<Rule> ruleSet = new LinkedHashSet<>();
+    ruleSet.addAll(baseRules);
+    for (String ruleString : customRuleStrings) {
+      ruleSet.addAll(customBinarizedRules.get(ruleString));
+    }
+    return new ArrayList<Rule>(ruleSet);
+  }
+
+  public Set<String> addCustomRule(Derivation deriv, Example ex) {
+    String indexedSymbolicFormula = getIndexedSymbolicFormula(deriv);
+    if (customRules.containsKey(indexedSymbolicFormula)) {
+      return customRules.get(indexedSymbolicFormula);
+    }
+
+    CPruneDerivInfo derivInfo = aggregateSymbols(deriv);
+    Set<String> crossReferences = new HashSet<>();
+    for (Symbol symbol : derivInfo.treeSymbols.values()) {
+      if (symbol.frequency > 1) {
+        crossReferences.add(symbol.formula);
+      }
+    }
+    computeCustomRules(deriv, crossReferences);
+    customRules.put(indexedSymbolicFormula, new HashSet<String>(derivInfo.customRuleStrings));
+
+    LogInfo.begin_track("Add custom rules for formula: " + indexedSymbolicFormula);
+    for (String customRuleString : derivInfo.customRuleStrings) {
+      if (customBinarizedRules.containsKey(customRuleString)) {
+        LogInfo.log("Custom rule exists: " + customRuleString);
+        continue;
+      }
+
+      rules = new ArrayList<>();
+      LispTree tree = LispTree.proto.parseFromString(customRuleString);
+      interpretRule(tree);
+      customBinarizedRules.put(customRuleString, new HashSet<Rule>(rules));
+
+      // Debug
+      LogInfo.begin_track("Add custom rule: " + customRuleString);
+      for (Rule rule : rules) {
+        LogInfo.log(rule.toString());
+      }
+      LogInfo.end_track();
+    }
+    LogInfo.end_track();
+
+    // Debug
+    System.out.println("consistent_lf\t" + ex.id + "\t" + deriv.formula.toString());
+
+    return customRules.get(indexedSymbolicFormula);
+  }
+
+  public static String getIndexedSymbolicFormula(Derivation deriv) {
+    return getIndexedSymbolicFormula(deriv, deriv.formula.toString());
+  }
+
+  /**
+   * Replace symbols (e.g., fb:row.row.name) with placeholders (e.g., Binary#1).
+   */
+  public static String getIndexedSymbolicFormula(Derivation deriv, String formula) {
+    CPruneDerivInfo derivInfo = aggregateSymbols(deriv);
+    int index = 1;
+    List<Symbol> symbolList = new ArrayList<>(derivInfo.treeSymbols.values());
+    for (Symbol symbol : symbolList)
+      symbol.computeIndex(formula);
+    Collections.sort(symbolList);
+    for (Symbol symbol : symbolList) {
+      if (formula.equals(symbol.formula))
+        formula = symbol.category + "#" + index;
+      formula = safeReplace(formula, symbol.formula, symbol.category + "#" + index);
+      index += 1;
+    }
+    return formula;
+  }
+
+  // ============================================================
+  // Private methods
+  // ============================================================
+
+  private static String safeReplace(String formula, String target, String replacement) {
+    // (argmin 1 1 ...) and (argmax 1 1 ...) are troublesome
+    String before = formula, targetBefore = target;
+    formula = formula.replace("(argmin (number 1) (number 1)", "(ARGMIN");
+    formula = formula.replace("(argmax (number 1) (number 1)", "(ARGMAX");
+    target = target.replace("(argmin (number 1) (number 1)", "(ARGMIN");
+    target = target.replace("(argmax (number 1) (number 1)", "(ARGMAX");
+    formula = formula.replace(target + ")", replacement + ")");
+    formula = formula.replace(target + " ", replacement + " ");
+    formula = formula.replace("(ARGMIN", "(argmin (number 1) (number 1)");
+    formula = formula.replace("(ARGMAX", "(argmax (number 1) (number 1)");
+    if (CollaborativePruner.opts.verbose >= 2)
+      LogInfo.logs("REPLACE: [%s | %s] %s | %s", targetBefore, replacement, before, formula);
+    return formula;
+  }
+
+  /**
+   * Cache the symbols in deriv.tempState[cprune].treeSymbols
+   */
+  private static CPruneDerivInfo aggregateSymbols(Derivation deriv) {
+    Map<String, Object> tempState = deriv.getTempState();
+    if (tempState.containsKey("cprune")) {
+      return (CPruneDerivInfo) tempState.get("cprune");
+    }
+    CPruneDerivInfo derivInfo = new CPruneDerivInfo();
+    tempState.put("cprune", derivInfo);
+
+    Map<String, Symbol> treeSymbols = new LinkedHashMap<>();
+    derivInfo.treeSymbols = treeSymbols;
+    if (baseCategories.contains(deriv.cat)) {
+      String formula = deriv.formula.toString();
+      treeSymbols.put(formula, new Symbol(deriv.cat, formula, 1));
+    } else {
+      for (Derivation child : deriv.children) {
+        CPruneDerivInfo childInfo = aggregateSymbols(child);
+        for (Symbol symbol : childInfo.treeSymbols.values()) {
+          if (derivInfo.treeSymbols.containsKey(symbol.formula)) {
+            treeSymbols.get(symbol.formula).frequency += symbol.frequency;
+          } else {
+            treeSymbols.put(symbol.formula, symbol);
+          }
+        }
+      }
+    }
+    return derivInfo;
+  }
+
+  private CPruneDerivInfo computeCustomRules(Derivation deriv, Set<String> crossReferences) {
+    CPruneDerivInfo derivInfo = (CPruneDerivInfo) deriv.getTempState().get("cprune");
+    Map<String, Symbol> ruleSymbols = new LinkedHashMap<>();
+    derivInfo.ruleSymbols = ruleSymbols;
+    derivInfo.customRuleStrings = new ArrayList<>();
+    String formula = deriv.formula.toString();
+
+    if (baseCategories.contains(deriv.cat)) {
+      // Leaf node induces no custom rule
+      derivInfo.containsCrossReference = crossReferences.contains(formula);
+      // Propagate the symbol of this derivation to the parent
+      ruleSymbols.putAll(derivInfo.treeSymbols);
+    } else {
+      derivInfo.containsCrossReference = false;
+      for (Derivation child : deriv.children) {
+        CPruneDerivInfo childInfo = computeCustomRules(child, crossReferences);
+        derivInfo.containsCrossReference = derivInfo.containsCrossReference || childInfo.containsCrossReference;
+      }
+
+      for (Derivation child : deriv.children) {
+        CPruneDerivInfo childInfo = (CPruneDerivInfo) child.getTempState().get("cprune");
+        ruleSymbols.putAll(childInfo.ruleSymbols);
+        derivInfo.customRuleStrings.addAll(childInfo.customRuleStrings);
+      }
+
+      if (opts.enableTemplateDecomposition == false || derivInfo.containsCrossReference) {
+        // If this node contains a cross reference
+        if (deriv.isRootCat()) {
+          // If this is the root node, then generate a custom rule
+          derivInfo.customRuleStrings.add(getCustomRuleString(deriv, derivInfo));
+        }
+      } else {
+        if (!deriv.cat.startsWith("$Intermediate")) {
+          // Generate a custom rule for this node
+          derivInfo.customRuleStrings.add(getCustomRuleString(deriv, derivInfo));
+
+          // Propagate this derivation as a category to the parent
+          ruleSymbols.clear();
+          ruleSymbols.put(formula, new Symbol(hash(deriv), deriv.formula.toString(), 1));
+        }
+      }
+    }
+    return derivInfo;
+  }
+
+  private String getCustomRuleString(Derivation deriv, CPruneDerivInfo derivInfo) {
+    String formula = deriv.formula.toString();
+    List<Symbol> rhsSymbols = new ArrayList<>(derivInfo.ruleSymbols.values());
+    for (Symbol symbol : rhsSymbols)
+      symbol.computeIndex(formula);
+    Collections.sort(rhsSymbols);
+
+    String lhs = null;
+    if (derivInfo.containsCrossReference)
+      lhs = deriv.cat;
+    else
+      lhs = deriv.isRootCat() ? "$ROOT" : hash(deriv);
+
+    LinkedList<String> rhsList = new LinkedList<>();
+    int index = 1;
+    for (Symbol symbol : rhsSymbols) {
+      if (formula.equals(symbol.formula)) {
+        formula = "(IdentityFn)";
+      } else {
+        formula = safeReplace(formula, symbol.formula, "(var s" + index + ")");
+        formula = "(lambda s" + index + " " + formula + ")";
+      }
+      rhsList.addFirst(symbol.category);
+      index += 1;
+    }
+    String rhs = null;
+    if (rhsList.size() > 0) {
+      rhs = "(" + String.join(" ", rhsList) + ")";
+    } else {
+      rhs = "(nothing)";
+      formula = "(ConstantFn " + formula + ")";
+    }
+    return "(rule " + lhs + " " + rhs + " " + formula + ")";
+  }
+
+  private String hash(Derivation deriv) {
+    if (baseCategories.contains(deriv.cat))
+      return deriv.cat;
+
+    String formula = getSymbolicFormula(deriv);
+    if (!symbolicFormulas.containsKey(formula)) {
+      symbolicFormulas.put(formula, symbolicFormulas.size() + 1);
+      String hashString = "$Formula" + symbolicFormulas.get(formula);
+      LogInfo.log("Add symbolic formula: " + hashString + " = " + formula + "  (" + deriv.cat + ")");
+    }
+    return "$Formula" + symbolicFormulas.get(formula);
+  }
+
+  private static String getSymbolicFormula(Derivation deriv) {
+    CPruneDerivInfo derivInfo = aggregateSymbols(deriv);
+    String formula = deriv.formula.toString();
+    for (Symbol symbol : derivInfo.treeSymbols.values()) {
+      if (formula.equals(symbol.formula))
+        formula = symbol.category;
+      formula = safeReplace(formula, symbol.formula, symbol.category);
+    }
+    return formula;
+  }
+
+}

src/edu/stanford/nlp/sempre/cprune/FormulaPattern.java

@@ -0,0 +1,106 @@
+package edu.stanford.nlp.sempre.cprune;
+
+import java.util.regex.Pattern;
+
+import edu.stanford.nlp.sempre.Derivation;
+import fig.basic.LogInfo;
+
+public class FormulaPattern implements Comparable<FormulaPattern> {
+  public String pattern;
+  public Integer frequency;
+  public Double score;
+
+  public FormulaPattern(String pattern, Integer frequency) {
+    this.pattern = pattern;
+    this.frequency = frequency;
+  }
+
+  public Double complexity() {
+    // Roughly the number of predicates
+    return (double) (pattern.length() - pattern.replace("(@R", "***").replace("(", "").length());
+  }
+
+  @Override
+  public String toString() {
+    return "(" + pattern + ", " + frequency + ")";
+  }
+
+  @Override
+  public int compareTo(FormulaPattern that) {
+    if (this.frequency > that.frequency) {
+      return -1;
+    } else if (this.frequency < that.frequency) {
+      return 1;
+    } else {
+      return this.complexity().compareTo(that.complexity());
+    }
+  }
+
+  // ============================================================
+  // Utilities
+  // ============================================================
+
+  private static Pattern reverseRelation = Pattern.compile("!(fb:[._a-z0-9]+)");
+  private static Pattern varName = Pattern.compile("\\((lambda|var) [a-z0-9]+");
+  private static Pattern compare = Pattern.compile("(<=|>=|>|<)");
+  private static Pattern whitespace = Pattern.compile("\\s+");
+
+  public static String convertToIndexedPattern(Derivation deriv) {
+    String formula = deriv.formula.toString();
+
+    // These can interfere with (number 1)
+    formula = formula.replace("argmax (number 1) (number 1)", "argmax");
+    formula = formula.replace("argmin (number 1) (number 1)", "argmin");
+
+    formula = removePropertyPredicates(formula);
+    formula = CustomGrammar.getIndexedSymbolicFormula(deriv, formula);
+
+    formula = formula.replace("fb:type.object.type fb:type.row", "@type @row");
+    formula = reverseRelation.matcher(formula).replaceAll("(reverse $1)");
+    formula = formula.replace("fb:row.row.index", "(reverse (lambda x ((reverse @index) (var x))))");
+    formula = formula.replace("fb:row.row.next", "@next");
+    formula = varName.matcher(formula).replaceAll("($1 x");
+    formula = formula.replace("reverse", "@R");
+    formula = compare.matcher(formula).replaceAll("@compare");
+    formula = whitespace.matcher(formula).replaceAll(" ");
+
+    if (CollaborativePruner.opts.verbose >= 2)
+      LogInfo.logs("PATTERN: %s -> %s", deriv.formula, formula);
+    return formula;
+  }
+
+  private static Pattern cellProperty = Pattern.compile("!?fb:cell\\.cell\\.[_a-z0-9]+|\\(reverse fb:cell\\.cell\\.[_a-z0-9]+\\)");
+
+  /**
+   * Remove cell property relations (fb:cell.cell.*)
+   */
+  public static String removePropertyPredicates(String formula) {
+    formula = cellProperty.matcher(formula).replaceAll("@PPT");
+    while (formula.contains("@PPT")) {
+      int begin = formula.indexOf("(@PPT");
+      if (begin == -1) {
+        formula = formula.replace("@PPT", "");
+        break;
+      }
+      // Find the matching parenthesis
+      int count = 1;
+      for (int i = begin + 1; i < formula.length(); i++) {
+        if (formula.charAt(i) == '(') {
+          count++;
+        } else if (formula.charAt(i) == ')') {
+          count--;
+          if (count == 0) {
+            int end = i;
+            formula = formula.substring(0, begin) + formula.substring(begin + 6, end) + formula.substring(end + 1, formula.length());
+            break;
+          }
+        }
+        if (i == formula.length() - 1) {
+          LogInfo.fails("Unbalanced parentheses: %s", formula);
+        }
+      }
+    }
+    return formula;
+  }
+
+}

src/edu/stanford/nlp/sempre/cprune/Symbol.java

@@ -0,0 +1,31 @@
+package edu.stanford.nlp.sempre.cprune;
+
+/**
+ * Represents the leaf node of the parse tree.
+ *
+ * Any sub-derivation whose category is in CustomGrammar.baseCategories becomes a Symbol.
+ */
+public class Symbol implements Comparable<Symbol> {
+  String category;
+  String formula;
+  Integer frequency;
+  Integer index;
+
+  public Symbol(String category, String formula, int frequency) {
+    this.category = category;
+    this.formula = formula;
+    this.frequency = frequency;
+  }
+
+  public void computeIndex(String referenceString) {
+    index = referenceString.indexOf(formula);
+    if (index < 0) {
+      index = Integer.MAX_VALUE;
+    }
+  }
+
+  @Override
+  public int compareTo(Symbol that) {
+    return index.compareTo(that.index);
+  }
+}

src/edu/stanford/nlp/sempre/overnight/OvernightDerivationPruningComputer.java

@@ -20,7 +20,7 @@ public class OvernightDerivationPruningComputer extends DerivationPruningCompute
   }
 
   @Override
-  public String isPrunedGeneral(Derivation deriv) {
+  public String isPruned(Derivation deriv) {
     if (containsStrategy("violateHardConstraints") && violateHardConstraints(deriv)) return "violateHardConstraints";
     return null;
   }

src/edu/stanford/nlp/sempre/tables/ConvertCsvToTtl.java

@@ -1,285 +0,0 @@
-package edu.stanford.nlp.sempre.tables;
-
-import java.io.*;
-import java.util.*;
-import java.text.*;
-
-import au.com.bytecode.opencsv.CSVReader;
-import edu.stanford.nlp.sempre.*;
-import fig.basic.*;
-import fig.exec.*;
-import static fig.basic.LogInfo.*;
-
-/**
-Converts a CSV file into a TTL file to be loaded into Virtuoso.
-Also dumps a lexicon file that maps strings to Freebase constants.
-This allows us to quickly deploy semantic parsers on random CSV data that's not
-already in Freebase.
-
-Domain (e.g., paleo): everything will live under fb:<domain>.*
-The input is a set of tables.  Each table has:
-- A set of column names (each column corresponds to a property)
-- A set of row names (each row corresponds to an event).
-- A name (this determines the type of the event).
-
-Columns that contain strings are considered entities, and we define a new type
-for that based on the column name.  Important: note that different tables
-interact by virtue of having the same column name (think of joining two tables
-by string matching their column names).  Of course, this is restrictive, but
-it's good enough for now.
-
-Example (domain: foo)
-
-  table name: info
-  columns: person, age, marital_status, social_security, place_of_birth
-  ...
-
-  table name: education
-  columns: person, elementary_school, high_school, college
-  ...
-
-  Event types: fb:foo.info, fb:foo.education
-  Properties: fb:foo.info.person, fb:foo.info.age, ...
-  Entity types: fb:foo.person, ...
-  Events: fb:foo.info0, fb:foo.info1, ...
-  Entities: fb:foo.barack_obama, ...
-
-Each column is a property that could take on several values.  A column could
-have many different types (e.g., int, text, entity) depending on how the
-values are parsed, but we can't do this with one pass over the data, so we're
-punting on this for now.
-
-@author Percy Liang
-*/
-public class ConvertCsvToTtl implements Runnable {
-  @Option(required = true, gloss = "Domain (used to specify all the entities)") public String domain;
-  @Option(required = true, gloss = "Input CSV <table name>:<table path> (assume each file has a header)") public List<String> tables;
-  @Option(required = true, gloss = "Output schema ttl to this path") public String outSchemaPath;
-  @Option(required = true, gloss = "Output ttl to this path") public String outTtlPath;
-  @Option(required = true, gloss = "Output lexicon to this path") public String outLexiconPath;
-  @Option(gloss = "Only use these columns") public List<String> keepProperties;
-  @Option(gloss = "Maximum number of rows to read per file") public int maxRowsPerFile = Integer.MAX_VALUE;
-
-  public static final String ttlPrefix = "@prefix fb: <http://rdf.freebase.com/ns/>.";
-
-  // Names and types of entities
-  Map<String, String> id2name = new HashMap<String, String>();
-  Map<String, Set<String>> id2types = new HashMap<String, Set<String>>();
-
-  // Used to parse values into dates
-  List<SimpleDateFormat> dateFormats = Arrays.asList(
-    new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH),  // One used by Freebase
-    new SimpleDateFormat("EEE MMM dd HH:mm:ss z yyyy", Locale.ENGLISH),
-    new SimpleDateFormat("MM/dd/yyyy", Locale.ENGLISH));
-
-  String prependDomain(String s) { return "fb:" + domain + "." + s; }
-  String makeString(String s) {
-    if (s.length() >= 2 && s.startsWith("\"") && s.endsWith("\""))
-      s = s.substring(1, s.length() - 1);
-    s = s.replaceAll("\"", "\\\\\"");  // Quote
-    return "\"" + s + "\"@en";
-  }
-  String lexEntry(String s, String id, Set<String> types) {
-    SemType type = types == null ? SemType.anyType : SemType.newUnionSemType(types);
-    Map<String, Object> result = new HashMap<String, Object>();
-    result.put("lexeme", s);  // Note: this is not actually a lexeme, but the full rawPhrase.
-    result.put("formula", id);
-    result.put("source", "STRING_MATCH");
-    result.put("type", type);
-    return Json.writeValueAsStringHard(result);
-  }
-
-  String canonicalize(String value) {
-    if (Character.isDigit(value.charAt(value.length() - 1))) {
-      // Try to convert to integer
-      try {
-        Integer.parseInt(value);
-        return "\"" + value + "\"^^xsd:int";
-      } catch (NumberFormatException e) {
-      }
-
-      // Try to convert to double
-      try {
-        Double.parseDouble(value);
-        return "\"" + value + "\"^^xsd:double";
-      } catch (NumberFormatException e) {
-      }
-
-      // Try to convert to date
-      for (DateFormat format : dateFormats) {
-        try {
-          Date date = format.parse(value);
-          return "\"" + dateFormats.get(0).format(date) + "\"^^xsd:datetime";
-        } catch (ParseException e) {
-        }
-      }
-    }
-
-    // Try to interpret as entity if it is short enough
-    if (value.split(" ").length <= 5) {
-      String id = value;
-      id = id.replaceAll("[^\\w]", "_");  // Replace abnormal characters with _
-      id = id.replaceAll("_+", "_");  // Merge consecutive _'s
-      id = id.replaceAll("_$", "");
-      id = id.toLowerCase();
-      if (id.length() == 0) id = "null";
-      id = prependDomain(id);
-      id2name.put(id, value);
-      return id;
-    }
-
-    // Just interpret as string
-    return makeString(value);
-  }
-
-  public static void writeTriple(PrintWriter out, String arg1, String property, String arg2) {
-    out.println(arg1 + "\t" + property + "\t" + arg2 + ".");
-  }
-
-  static class Column {
-    String description;
-    String header;
-    String property;
-    int numInt, numDouble, numDate, numText, numEntity;
-
-    // Return whether we have an entity.
-    boolean add(String value) {
-      if (value.endsWith("xsd:int"))
-        numInt++;
-      else if (value.endsWith("xsd:double"))
-        numDouble++;
-      else if (value.endsWith("xsd:datetime"))
-        numDate++;
-      else if (value.endsWith("@en"))
-        numText++;
-      else {
-        numEntity++;
-        return true;
-      }
-      return false;
-    }
-
-    String getEntityType() { return header; }
-
-    String getType() {
-      int[] counts = new int[] {numInt, numDouble, numDate, numText, numEntity};
-      int max = ListUtils.max(counts);
-      // if (numInt == max) return FreebaseInfo.INT;
-      // if (numDouble == max) return FreebaseInfo.FLOAT;
-      if (numInt == max) return CanonicalNames.NUMBER;
-      if (numDouble == max) return CanonicalNames.NUMBER;
-      if (numDate == max) return CanonicalNames.DATE;
-      if (numText == max) return CanonicalNames.TEXT;
-      return getEntityType();
-    }
-
-    @Override public String toString() {
-      StringBuilder b = new StringBuilder();
-      b.append(header);
-      if (numInt > 0) b.append(", int=" + numInt);
-      if (numDouble > 0) b.append(", double=" + numDouble);
-      if (numDate > 0) b.append(", date=" + numDate);
-      if (numText > 0) b.append(", text=" + numText);
-      if (numEntity > 0) b.append(", entity=" + numEntity);
-      return b.toString();
-    }
-  }
-
-  public void run() {
-    PrintWriter schemaOut = IOUtils.openOutHard(outSchemaPath);
-    PrintWriter ttlOut = IOUtils.openOutHard(outTtlPath);
-    PrintWriter lexiconOut = IOUtils.openOutHard(outLexiconPath);
-
-    ttlOut.println(ttlPrefix);
-    int total = 0;
-    for (String pairStr : tables) {
-      int num = 0;  // The row number (standards for an event / CSV)
-      String[] pair = pairStr.split(":", 2);
-      if (pair.length != 2) throw new RuntimeException("Expected <table name>:<file name> pair, but got: " + pair);
-      String tableName = pair[0];
-      String inPath = pair[1];
-      String eventType = prependDomain(tableName);  // e.g., fb:paleo.taxon
-
-      LogInfo.begin_track("Reading %s for events of type %s", inPath, eventType);
-      Column[] columns = null;
-      try (CSVReader csv = new CSVReader(new FileReader(inPath))) {
-        for (String[] row : csv) {
-          if (num >= maxRowsPerFile) break;
-
-          // Initialize the columns
-          if (columns == null) {
-            columns = new Column[row.length];
-            for (int i = 0; i < columns.length; i++) {
-              Column c = columns[i] = new Column();
-              row[i] = row[i].trim();
-              c.description = row[i];
-              c.header = canonicalize(row[i]);
-              if (keepProperties != null && !keepProperties.contains(c.header)) {
-                c.header = null;
-                continue;
-              }
-              if (!c.header.startsWith("fb:" + domain))
-                throw new RuntimeException("Invalid (internal problem): " + c.header);
-              c.property = c.header.replace("fb:" + domain, "fb:" + domain + "." + tableName); // Property
-            }
-            continue;
-          }
-
-          // Read a row (corresponds to an event/CVT)
-          String event = prependDomain(tableName + (num++));
-          writeTriple(ttlOut, event, "fb:type.object.type", tableName);
-          for (int i = 0; i < Math.min(row.length, columns.length); i++) {  // For each column...
-            Column c = columns[i];
-            if (c.header == null || row[i].equals("")) continue;
-            row[i] = canonicalize(row[i]);
-            writeTriple(ttlOut, event, c.property, row[i]);  // Write out the assertion
-
-            if (c.add(row[i])) {  // Write out type for entities
-              MapUtils.addToSet(id2types, row[i], c.getEntityType());
-              writeTriple(ttlOut, row[i], "fb:type.object.type", c.getEntityType());
-              writeTriple(ttlOut, row[i], "fb:type.object.type", CanonicalNames.ENTITY);
-            }
-          }
-          if (num % 10000 == 0)
-            logs("Read %d rows (events)", num);
-        }
-      } catch (IOException e) {
-        throw new RuntimeException(e);
-      }
-
-      // Write out schema
-      writeTriple(schemaOut, eventType, "fb:freebase.type_hints.mediator", "\"true\"^^xsd:boolean");  // event type is a CVT
-      for (Column c : columns) {
-        if (c.header == null) continue;
-        writeTriple(schemaOut, c.property, "fb:type.object.type", "fb:type.property");
-        writeTriple(schemaOut, c.property, "fb:type.property.schema", eventType);
-        writeTriple(schemaOut, c.property, "fb:type.property.expected_type", c.getType());
-        writeTriple(schemaOut, c.property, "fb:type.object.name", makeString(c.description));
-        if (c.getType().equals(c.getEntityType())) {
-          writeTriple(schemaOut, c.getEntityType(), "fb:type.object.name", makeString(c.description));
-          writeTriple(schemaOut, c.getEntityType(), "fb:freebase.type_hints.included_types", CanonicalNames.ENTITY);
-        }
-        LogInfo.logs("%s", c);
-      }
-      LogInfo.end_track();
-      total += num;
-    }
-
-    logs("%d events, %d entities (ones with names)", total, id2name.size());
-    for (Map.Entry<String, String> e : id2name.entrySet()) {
-      writeTriple(ttlOut, e.getKey(), "fb:type.object.name", makeString(e.getValue()));
-      String k = e.getKey();
-      String s = e.getValue();
-      Set<String> types = id2types.get(k);
-      lexiconOut.println(lexEntry(s, k, types));
-    }
-
-    schemaOut.close();
-    ttlOut.close();
-    lexiconOut.close();
-  }
-
-  public static void main(String[] args) {
-    Execution.run(args, new ConvertCsvToTtl());
-  }
-}

src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java

@@ -39,17 +39,13 @@ public final class StringNormalizationUtils {
    *
    * TODO(ice): Take the homogeneity of the cells into account.
    */
-  public static void analyzeColumn(TableColumn column, Map<String, String> originalStringToPartId) {
-    // Check if any cell contains a comma-separated list
-    int numLists = 0;
-    for (TableCell cell : column.children) {
-      String[] splitted = COMMA.split(cell.properties.originalString);
-      if (splitted.length > 1) numLists++;
-    }
+  public static void analyzeColumn(TableColumn column) {
+    // Parts in the same column with the same string content gets the same id.
+    Map<String, String> originalStringToPartId = new HashMap<>();
     for (TableCell cell : column.children) {
       if (!cell.properties.metadata.isEmpty()) continue;  // Already analyzed.
       analyzeString(cell.properties.originalString, cell.properties.metadata,
-          originalStringToPartId, numLists > 0);
+          column, originalStringToPartId);
     }
   }
 
@@ -57,12 +53,13 @@ public final class StringNormalizationUtils {
   // Cell normalization
   // ============================================================
 
+  public static final Pattern STRICT_DASH = Pattern.compile("\\s*[-‐‑⁃‒–—―]\\s*");
   public static final Pattern DASH = Pattern.compile("\\s*[-‐‑⁃‒–—―/,:;]\\s*");
   public static final Pattern COMMA = Pattern.compile("\\s*(,\\s|\\n|/)\\s*");
   public static final Pattern SPACE = Pattern.compile("\\s+");
 
   public static void analyzeString(String o, Multimap<Value, Value> metadata,
-      Map<String, String> originalStringToPartId, boolean alwaysGenerateParts) {
+      TableColumn column, Map<String, String> originalStringToPartId) {
     metadata.clear();
     Value value;
     LanguageAnalyzer analyzer = LanguageAnalyzer.getSingleton();
@@ -94,11 +91,18 @@ public final class StringNormalizationUtils {
     }
     // ===== List: "apple, banana, carrot" --> fb:part.apple, etc. =====
     String[] splitted = COMMA.split(o);
-    if (splitted.length > 1 || alwaysGenerateParts) {
-      for (String x : splitted) {
-        String id = TableTypeSystem.getOrCreateName(x, originalStringToPartId,
-            (String canonicalName) -> TableTypeSystem.getPartName(canonicalName));
-        metadata.put(TableTypeSystem.CELL_PART_VALUE, new NameValue(id, x));
+    if (splitted.length > 1) {
+      for (String partName : splitted) {
+        String normalizedPartName = StringNormalizationUtils.characterNormalize(partName).toLowerCase();
+        String id = originalStringToPartId.get(normalizedPartName);
+        if (id == null) {
+          String canonicalName = TableTypeSystem.canonicalizeName(normalizedPartName);
+          id = TableTypeSystem.getUnusedName(
+              TableTypeSystem.getPartName(canonicalName, column.columnName),
+              originalStringToPartId.values());
+          originalStringToPartId.put(normalizedPartName, id);
+        }
+        metadata.put(TableTypeSystem.CELL_PART_VALUE, new NameValue(id, partName));
       }
     }
   }
@@ -197,22 +201,31 @@ public final class StringNormalizationUtils {
     return null;
   }
 
-  public static final DateTimeFormatter dateFormat = DateTimeFormat.forPattern("MMM d, yyyy");
+  public static final DateTimeFormatter americanDateFormat = DateTimeFormat.forPattern("MMM d, yyyy");
+  public static final Pattern suTimeDateFormat = Pattern.compile("([0-9X]{4})(?:-([0-9X]{2}))?(?:-([0-9X]{2}))?");
 
   /**
    * Convert string to DateValue.
    */
   public static DateValue parseDate(String s) {
+    Matcher matcher = suTimeDateFormat.matcher(s.toUpperCase());
+    if (matcher.matches()) {
+      String yS = matcher.group(1), mS = matcher.group(2), dS = matcher.group(3);
+      int y = -1, m = -1, d = -1;
+      if (!(yS == null || yS.isEmpty() || yS.contains("X"))) y = Integer.parseInt(yS);
+      if (!(mS == null || mS.isEmpty() || mS.contains("X"))) m = Integer.parseInt(mS);
+      if (!(dS == null || dS.isEmpty() || dS.contains("X"))) d = Integer.parseInt(dS);
+      if (y == -1 && m == -1 && d == -1) return null;
+      return new DateValue(y, m, d);
+    }
     try {
-      DateTime date = dateFormat.parseDateTime(s);
+      DateTime date = americanDateFormat.parseDateTime(s);
       return new DateValue(date.getYear(), date.getMonthOfYear(), date.getDayOfMonth());
     } catch (IllegalArgumentException e) {
       return null;
     }
   }
 
-  public static final Pattern suTimeDateFormat = Pattern.compile("([0-9X]{4})(?:-([0-9X]{2}))?(?:-([0-9X]{2}))?");
-
   public static DateValue parseDateWithLanguageAnalyzer(LanguageInfo languageInfo) {
     if (languageInfo.numTokens() == 0) return null;
     String nerSpan = languageInfo.getNormalizedNerSpan("DATE", 0, languageInfo.numTokens());
@@ -304,8 +317,7 @@ public final class StringNormalizationUtils {
         .replaceAll("[‘’´`]", "'")
         .replaceAll("[“”«»]", "\"")
         .replaceAll("[•†‡]", "")
-        .replaceAll("[‐‑–—]", "-")
-        .replaceAll("[\\u2E00-\\uFFFF]", "");     // (Sorry Chinese people)
+        .replaceAll("[-‐‑–—]", "-");
     return string.replaceAll("\\s+", " ").trim();
   }
 
@@ -317,13 +329,8 @@ public final class StringNormalizationUtils {
     // Citation
     string = string.replaceAll("\\[(nb ?)?\\d+\\]", "");
     string = string.replaceAll("\\*+$", "");
-    // Year in parentheses
-    string = string.replaceAll("\\(\\d* ?-? ?\\d*\\)", "");
     // Outside Quote
     string = string.replaceAll("^\"(.*)\"$", "$1");
-    // Numbering
-    if (!string.matches("^[0-9.]+$"))
-      string = string.replaceAll("^\\d+\\.", "");
     return string.replaceAll("\\s+", " ").trim();
   }
 
@@ -333,21 +340,60 @@ public final class StringNormalizationUtils {
   public static String aggressiveNormalize(String string) {
     // Dashed / Parenthesized information
     string = simpleNormalize(string);
-    string = string.replaceAll("\\[[^\\]]*\\]", "");
-    string = string.replaceAll("[\\u007F-\\uFFFF]", "");
-    string = string.trim().replaceAll(" - .*$", "");
-    string = string.trim().replaceAll("\\([^)]*\\)$", "");
+    String oldString;
+    do {
+      oldString = string;
+      // Remove citations
+      string = string.trim().replaceAll("((?<!^)\\[[^\\]]*\\]|\\[\\d+\\]|[•♦†‡*#+])*$", "");
+      // Remove details in parenthesis
+      string = string.trim().replaceAll("(?<!^)(\\s*\\([^)]*\\))*$", "");
+      // Remove outermost quotation mark
+      string = string.trim().replaceAll("^\"([^\"]*)\"$", "$1");
+    } while (!oldString.equals(string));
+    // Collapse whitespaces
     return string.replaceAll("\\s+", " ").trim();
   }
 
+  /**
+   * Normalization scheme in the official Python evaluator.
+   */
+  public static String officialEvaluatorNormalize(String string) {
+    // Remove diacritics
+    string = Normalizer.normalize(string, Normalizer.Form.NFD).replaceAll("[\u0300-\u036F]", "");
+    // Normalize quotes and dashes
+    string = string
+        .replaceAll("[‘’´`]", "'")
+        .replaceAll("[“”]", "\"")
+        .replaceAll("[‐‑‒–—−]", "-");
+    String oldString;
+    do {
+      oldString = string;
+      // Remove citations
+      string = string.trim().replaceAll("((?<!^)\\[[^\\]]*\\]|\\[\\d+\\]|[•♦†‡*#+])*$", "");
+      // Remove details in parenthesis
+      string = string.trim().replaceAll("(?<!^)(\\s*\\([^)]*\\))*$", "");
+      // Remove outermost quotation mark
+      string = string.trim().replaceAll("^\"([^\"]*)\"$", "$1");
+    } while (!oldString.equals(string));
+    // Remove final '.'
+    if (string.endsWith("."))
+      string = string.substring(0, string.length() - 1);
+    // Collapse whitespaces and convert to lower case
+    string = string.replaceAll("\\s+", " ").toLowerCase().trim();
+    return string;
+  }
+
   // ============================================================
   // Test
   // ============================================================
 
   private static void unitTest(String o) {
     Multimap<Value, Value> metadata = ArrayListMultimap.create();
-    analyzeString(o, metadata, new HashMap<>(), false);
-    LogInfo.logs("%s %s", o, metadata);
+    TableColumn column = new TableColumn("Test", "test", 0);
+    analyzeString(o, metadata, column, new HashMap<>());
+    String aggressive = aggressiveNormalize(o).toLowerCase();
+    String official = officialEvaluatorNormalize(o);
+    LogInfo.logs("%s %s | %s %s %s", o, metadata, official, aggressive, aggressive.equals(official));
   }
 
   public static void main(String[] args) {
@@ -359,6 +405,8 @@ public final class StringNormalizationUtils {
     unitTest("twenty three");
     unitTest("apple, banana, banana, BANANA");
     unitTest("apple\nbanana\norange");
+    unitTest("0-1\n(4-5 p)");
+    unitTest("\"HELLO\"");
     unitTest("21st");
     unitTest("2001st");
     unitTest("2,000,000 ft.");

src/edu/stanford/nlp/sempre/tables/TableCellProperties.java

@@ -1,7 +1,5 @@
 package edu.stanford.nlp.sempre.tables;
 
-import java.util.*;
-
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Multimap;
 
@@ -19,14 +17,12 @@ public class TableCellProperties {
   public final String originalString;
   public final NameValue nameValue;
   public final Multimap<Value, Value> metadata;
-  public final Set<TableColumn> columns;
 
   public TableCellProperties(String id, String originalString) {
     this.id = id;
     this.originalString = originalString;
     this.nameValue = new NameValue(id, originalString);
     this.metadata = ArrayListMultimap.create();
-    this.columns = new HashSet<>();
   }
 
   /** Create a copy without the columns field. */
@@ -35,7 +31,6 @@ public class TableCellProperties {
     this.originalString = old.originalString;
     this.nameValue = old.nameValue;
     this.metadata = ArrayListMultimap.create(old.metadata);
-    this.columns = new HashSet<>();
   }
 
   @Override

src/edu/stanford/nlp/sempre/tables/TableColumn.java

@@ -16,7 +16,12 @@ public class TableColumn {
   public final String originalString;
   public final String columnName;
   public final int index;
+  // Relation Name
   public final NameValue relationNameValue, relationConsecutiveNameValue;
+  // Children Cell's Type (EntitySemType)
+  public final String cellTypeString;
+  public final NameValue cellTypeValue;
+  public final SemType cellSemType;
 
   public TableColumn(String originalString, String columnName, int index) {
     this.children = new ArrayList<>();
@@ -25,6 +30,9 @@ public class TableColumn {
     this.index = index;
     this.relationNameValue = new NameValue(TableTypeSystem.getRowPropertyName(columnName), originalString);
     this.relationConsecutiveNameValue = new NameValue(TableTypeSystem.getRowConsecutivePropertyName(columnName), originalString);
+    this.cellTypeString = TableTypeSystem.getCellType(columnName);
+    this.cellTypeValue = new NameValue(this.cellTypeString, originalString);
+    this.cellSemType = SemType.newAtomicSemType(this.cellTypeString);
   }
 
   /** Create a copy without the children field. */
@@ -35,6 +43,9 @@ public class TableColumn {
     this.index = old.index;
     this.relationNameValue = old.relationNameValue;
     this.relationConsecutiveNameValue = old.relationConsecutiveNameValue;
+    this.cellTypeString = old.cellTypeString;
+    this.cellTypeValue = old.cellTypeValue;
+    this.cellSemType = old.cellSemType;
   }
 
   public static Set<String> getReservedFieldNames() {

src/edu/stanford/nlp/sempre/tables/TableDerivationPruningComputer.java

@@ -5,7 +5,6 @@ import java.util.*;
 import edu.stanford.nlp.sempre.*;
 import edu.stanford.nlp.sempre.AggregateFormula.Mode;
 import edu.stanford.nlp.sempre.tables.lambdadcs.LambdaDCSException;
-import fig.basic.LogInfo;
 
 public class TableDerivationPruningComputer extends DerivationPruningComputer {
 
@@ -42,17 +41,16 @@ public class TableDerivationPruningComputer extends DerivationPruningComputer {
   private final Formula IDENTITY = Formula.fromString("(reverse (lambda x (var x)))");
 
   @Override
-  public String isPrunedGeneral(Derivation deriv) {
+  public String isPruned(Derivation deriv) {
     // lambdaDCSError: Prune unrecoverable LambdaDCSException
     if (containsStrategy(lambdaDCSError)) {
-      if (deriv.value instanceof ErrorValue && LambdaDCSException.isUnrecoverable(((ErrorValue) deriv.value).type)) {
-        if (DerivationPruner.opts.pruningVerbosity >= 5)
-          LogInfo.logs("lambdaDCSError: %s => %s", deriv.formula, deriv.value);
+      deriv.ensureExecuted(parser.executor, ex.context);
+      if (deriv.value instanceof ErrorValue && LambdaDCSException.isUnrecoverable(((ErrorValue) deriv.value).type))
         return lambdaDCSError;
-      }
     }
     // emptyDenotation: Prune if the denotation is empty (for ScopedValue)
     if (containsStrategy(emptyDenotation)) {
+      deriv.ensureExecuted(parser.executor, ex.context);
       if (deriv.value instanceof PairListValue && ((PairListValue) deriv.value).pairs.isEmpty()) {
         return emptyDenotation;
       }
@@ -65,6 +63,7 @@ public class TableDerivationPruningComputer extends DerivationPruningComputer {
     }
     // badSummarizerHead: Prune if the head of a ScopedValue is empty or is a single object
     if (containsStrategy(badSummarizerHead)) {
+      deriv.ensureExecuted(parser.executor, ex.context);
       if (deriv.value instanceof ScopedValue) {
         Value head = ((ScopedValue) deriv.value).head;
         if ((head instanceof ListValue) && ((ListValue) head).values.size() == 1)
@@ -78,9 +77,9 @@ public class TableDerivationPruningComputer extends DerivationPruningComputer {
         if (merge.mode == MergeFormula.Mode.and) {
           Value head = null;
           if (merge.child1 instanceof MarkFormula) {
-            head = pruner.parser.executor.execute(merge.child2, pruner.ex.context).value;
+            head = parser.executor.execute(merge.child2, ex.context).value;
           } else if (merge.child2 instanceof MarkFormula) {
-            head = pruner.parser.executor.execute(merge.child1, pruner.ex.context).value;
+            head = parser.executor.execute(merge.child1, ex.context).value;
           }
           if (head != null && head.equals(deriv.value)) {
             return sameMark;
@@ -88,18 +87,14 @@ public class TableDerivationPruningComputer extends DerivationPruningComputer {
         }
       }
     }
-    return null;
-  }
-
-  @Override
-  public String isPrunedRecursive(Derivation deriv, Formula subformula, Map<String, Object> state) {
     // Prune JoinFormulas
-    if (subformula instanceof JoinFormula) {
-      JoinFormula join1 = (JoinFormula) subformula;
-      if (join1.child instanceof JoinFormula) {
-        JoinFormula join2 = (JoinFormula) join1.child;
-        String rid1 = Formulas.getBinaryId(join1.relation),
-            rid2 = Formulas.getBinaryId(join2.relation);
+    if (containsStrategy(forwardBackward) || containsStrategy(doubleNext)
+        || containsStrategy(doubleCompares) || containsStrategy(emptyJoin)) {
+      Formula current = deriv.formula;
+      String rid1 = null, rid2 = null;
+      while (current instanceof JoinFormula) {
+        rid2 = rid1;
+        rid1 = Formulas.getBinaryId(((JoinFormula) current).relation);
         if (rid1 != null && rid2 != null) {
           // forwardBackward: Prune (!relation (relation (...)))
           if (containsStrategy(forwardBackward) && (rid1.equals("!" + rid2) || rid2.equals("!" + rid1)))
@@ -116,7 +111,7 @@ public class TableDerivationPruningComputer extends DerivationPruningComputer {
           if (containsStrategy(emptyJoin)) {
             Formula test = new JoinFormula(rid1, new JoinFormula(rid2, STAR));
             try {
-              Value value = pruner.parser.executor.execute(test, pruner.ex.context).value;
+              Value value = parser.executor.execute(test, ex.context).value;
               if (value instanceof ListValue && ((ListValue) value).values.isEmpty())
                 return emptyJoin;
             } catch (RuntimeException e) {
@@ -124,16 +119,17 @@ public class TableDerivationPruningComputer extends DerivationPruningComputer {
             }
           }
         }
+        current = ((JoinFormula) current).child;
       }
     }
     // Prune merge formulas
-    else if (subformula instanceof MergeFormula) {
-      MergeFormula merge = (MergeFormula) subformula;
+    if (containsStrategy(subsetMerge) && deriv.formula instanceof MergeFormula) {
+      MergeFormula merge = (MergeFormula) deriv.formula;
       Formula child1 = merge.child1, child2 = merge.child2;
       // subsetMerge: Prune merge formulas where one child is a subset of the other
       if (containsStrategy(subsetMerge)) {
-        Value d1 = pruner.parser.executor.execute(child1, pruner.ex.context).value;
-        Value d2 = pruner.parser.executor.execute(child2, pruner.ex.context).value;
+        Value d1 = parser.executor.execute(child1, ex.context).value;
+        Value d2 = parser.executor.execute(child2, ex.context).value;
         if (d1 instanceof ListValue && d2 instanceof ListValue) {
           Set<Value> v1 = new HashSet<>(((ListValue) d1).values);
           Set<Value> v2 = new HashSet<>(((ListValue) d2).values);
@@ -149,8 +145,8 @@ public class TableDerivationPruningComputer extends DerivationPruningComputer {
         return typeRowMerge;
     }
     // Prune aggregate formulas
-    else if (subformula instanceof AggregateFormula) {
-      AggregateFormula aggregate = (AggregateFormula) subformula;
+    else if (deriv.formula instanceof AggregateFormula) {
+      AggregateFormula aggregate = (AggregateFormula) deriv.formula;
       Formula child = aggregate.child;
       // aggregateInfinite: Prune aggregates when the child is an infinite set
       if (containsStrategy(aggregateInfinite) && child instanceof JoinFormula) {
@@ -171,22 +167,23 @@ public class TableDerivationPruningComputer extends DerivationPruningComputer {
       }
     }
     // Prune superlative formulas
-    else if (subformula instanceof SuperlativeFormula) {
-      SuperlativeFormula superlative = (SuperlativeFormula) subformula;
+    else if (deriv.formula instanceof SuperlativeFormula) {
+      SuperlativeFormula superlative = (SuperlativeFormula) deriv.formula;
       Formula relation = superlative.relation;
       // superlativeIdentity: Prune superlatives when the relation is exactly (lambda x (var x))
       if (containsStrategy(superlativeIdentity) && IDENTITY.equals(relation)) {
         return superlativeIdentity;
       }
     }
-    // For ScopedFormula
-    if (!DerivationPruner.opts.recursivePruning && subformula instanceof ScopedFormula) {
-      Formula relation = ((ScopedFormula) subformula).relation;
+    // For ScopedFormula: recurse into the relation part
+    if (deriv.formula instanceof ScopedFormula) {
+      Formula relation = ((ScopedFormula) deriv.formula).relation;
       if (relation instanceof LambdaFormula) {
         relation = ((LambdaFormula) relation).body;
+        Derivation relationDeriv = new Derivation.Builder().formula(relation).createDerivation();
         String matchedStrategy;
         for (DerivationPruningComputer computer : pruner.getPruningComputers()) {
-          if ((matchedStrategy = computer.isPrunedRecursive(deriv, relation, new HashMap<>())) != null) {
+          if ((matchedStrategy = computer.isPruned(relationDeriv)) != null) {
             return matchedStrategy;
           }
         }

src/edu/stanford/nlp/sempre/tables/TableKnowledgeGraph.java

@@ -20,10 +20,16 @@ import fig.basic.*;
  *
  * - Each row becomes an entity
  * - Each cell becomes an entity
- * - Each column becomes a relation between a row and a cell
- *     e.g., (fb:row.r5 fb:row.row.nationality fb:cell.canada)
- * - Each cell has relations pointing to different normalization nodes
- *     e.g., (fb:cell.3_km fb:cell.cell.number (number 3))
+ * - Each column becomes a property between a row and a cell
+ *     e.g., (row5 nationality canada)
+ * - Rows have several special properties (next, index)
+ *
+ * === Special Row Properties ===
+ * - name = fb:row.row.next  | type = (-> fb:type.row fb:type.row)
+ * - name = fb:row.row.index | type = (-> fb:type.int fb:type.row)
+ *
+ * === Special Cell Properties ===
+ * - name = fb:cell.cell.number | type = (-> fb:type.number fb:type.cell)
  *
  * @author ppasupat
  */
@@ -38,6 +44,8 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
     public boolean forbidNextOnManyRows = true;
     @Option(gloss = "Set up executor cache for each graph (must manually clear, or else will get memory overflow)")
     public boolean individualExecutorCache = false;
+    @Option(gloss = "Have the row index starts at 1 instead of 0")
+    public boolean rowIndexStartsAt1 = true;
   }
   public static Options opts = new Options();
 
@@ -57,6 +65,8 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
   Map<String, TableColumn> relationIdToTableColumn;
   // "fb:cell.palo_alto_ca" --> TableCellProperties object
   Map<String, TableCellProperties> cellIdToTableCellProperties;
+  // "fb:part.palo_alto" --> String
+  Map<String, String> partIdToOriginalString;
 
   FuzzyMatcher fuzzyMatcher;
   public ExecutorCache executorCache;
@@ -78,10 +88,8 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
     this.filename = filename;
     // Used column names (no two columns have the same id)
     Set<String> usedColumnNames = new HashSet<>();
-    // Cells with the same string content gets the same id.
-    Map<String, String> originalStringToCellId = new HashMap<>();
-    // Parts with the same string content gets the same id.
-    Map<String, String> originalStringToPartId = new HashMap<>();
+    // Cells in the same column with the same string content gets the same id.
+    Map<Pair<TableColumn, String>, String> columnAndOriginalStringToCellId = new HashMap<>();
     // Go though the data
     for (String[] record : data) {
       if (columns == null) {
@@ -107,32 +115,44 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
           LogInfo.warnings("Table has %d columns but row has %d cells: %s | %s", columns.size(),
               record.length, columns, Fmt.D(record));
         }
-        TableRow currentRow = new TableRow(rows.size());
+        int rowIndex = opts.rowIndexStartsAt1 ? rows.size() + 1 : rows.size();
+        TableRow currentRow = new TableRow(rowIndex);
         rowIdToTableRow.put(currentRow.nameValue.id, currentRow);
         rows.add(currentRow);
         for (int i = 0; i < columns.size(); i++) {
           TableColumn column = columns.get(i);
-          String cellString = (i < record.length) ? record[i] : "";
+          String cellName = (i < record.length) ? record[i] : "";
           // Create a NameValue
-          String id = TableTypeSystem.getOrCreateName(cellString, originalStringToCellId,
-              (String canonicalName) -> TableTypeSystem.getCellName(canonicalName));
+          String normalizedCellName = StringNormalizationUtils.characterNormalize(cellName).toLowerCase();
+          Pair<TableColumn, String> columnAndOriginalString = new Pair<>(column, normalizedCellName);
+          String id = columnAndOriginalStringToCellId.get(columnAndOriginalString);
+          if (id == null) {
+            String canonicalName = TableTypeSystem.canonicalizeName(normalizedCellName);
+            id = TableTypeSystem.getUnusedName(
+                TableTypeSystem.getCellName(canonicalName, column.columnName),
+                cellIdToTableCellProperties.keySet());
+            columnAndOriginalStringToCellId.put(columnAndOriginalString, id);
+            cellIdToTableCellProperties.put(id, new TableCellProperties(id, cellName));
+          }
           TableCellProperties properties = cellIdToTableCellProperties.get(id);
-          if (properties == null)
-            cellIdToTableCellProperties.put(id, properties = new TableCellProperties(id, cellString));
           TableCell.createAndAddTo(properties, column, currentRow);
-          properties.columns.add(column);
         }
       }
     }
     // Generate cell properties by analyzing cell content in each column
     for (TableColumn column : columns)
-      StringNormalizationUtils.analyzeColumn(column, originalStringToPartId);
+      StringNormalizationUtils.analyzeColumn(column);
     // Collect cell properties for public access
     cellProperties = new HashSet<>(cellIdToTableCellProperties.values());
     cellParts = new HashSet<>();
-    for (TableCellProperties properties : cellProperties)
-      for (Value part : properties.metadata.get(TableTypeSystem.CELL_PART_VALUE))
-        cellParts.add((NameValue) part);
+    partIdToOriginalString = new HashMap<>();
+    for (TableCellProperties properties : cellProperties) {
+      for (Value part : properties.metadata.get(TableTypeSystem.CELL_PART_VALUE)) {
+        NameValue partNameValue = (NameValue) part;
+        cellParts.add(partNameValue);
+        partIdToOriginalString.put(partNameValue.id, partNameValue.description);
+      }
+    }
     // Precompute normalized strings for fuzzy matching
     fuzzyMatcher = FuzzyMatcher.getFuzzyMatcher(this);
     executorCache = opts.individualExecutorCache ? new ExecutorCache() : null;
@@ -314,8 +334,6 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
 
   public static final NameValue TYPE = new NameValue(CanonicalNames.TYPE);
   public static final NameValue ROW_TYPE = new NameValue(TableTypeSystem.ROW_TYPE);
-  public static final NameValue CELL_TYPE = new NameValue(TableTypeSystem.CELL_TYPE);
-  public static final NameValue PART_TYPE = new NameValue(TableTypeSystem.PART_TYPE);
 
   /** Return all y such that x in firsts and (x,r,y) in graph */
   @Override
@@ -345,6 +363,7 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
    *   X-Y = cell-row, primitive-cell
    */
   /** Return all (x,y) such that y in seconds and (x,r,y) in graph */
+  // TODO(ice): Check correctness
   @Override
   public List<Pair<Value, Value>> filterSecond(Value r, Collection<Value> seconds) {
     List<Pair<Value, Value>> answer = new ArrayList<>();
@@ -371,7 +390,7 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
             if (!(value instanceof NameValue)) continue;
             TableRow row = rowIdToTableRow.get(((NameValue) value).id);
             if (row == null) continue;
-            int i = row.index;
+            int i = opts.rowIndexStartsAt1 ? row.index - 1 : row.index;
             if (i + 1 >= rows.size()) continue;
             answer.add(new Pair<>(rows.get(i + 1).nameValue, row.nameValue));
           }
@@ -394,7 +413,7 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
         }
       } else if (TableTypeSystem.isCellProperty(r)) {
         ////////////////////////////////////////////////////////////
-        // (!fb:cell.cell.number fb:cell.5_dollars) --> (number 5)
+        // (!fb:cell.cell.number fb:cell_id.5) --> 5
         if (seconds.size() == Integer.MAX_VALUE) {
           for (TableColumn column : columns) {
             for (TableCell cell : column.children) {
@@ -463,12 +482,6 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
           if (second.equals(ROW_TYPE)) {
             for (TableRow row : rows)
               answer.add(new Pair<>(row.nameValue, second));
-          } else if (second.equals(CELL_TYPE)) {
-            for (TableCellProperties properties : cellProperties)
-              answer.add(new Pair<>(properties.nameValue, second));
-          } else if (second.equals(PART_TYPE)) {
-            for (NameValue part : cellParts)
-              answer.add(new Pair<>(part, second));
           }
         }
       } else if (r.equals(TableTypeSystem.ROW_NEXT_VALUE)) {
@@ -487,7 +500,7 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
             if (!(value instanceof NameValue)) continue;
             TableRow row = rowIdToTableRow.get(((NameValue) value).id);
             if (row == null) continue;
-            int i = row.index;
+            int i = opts.rowIndexStartsAt1 ? row.index - 1 : row.index;
             if (i - 1 < 0) continue;
             answer.add(new Pair<>(rows.get(i - 1).nameValue, row.nameValue));
           }
@@ -506,6 +519,7 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
             double x = ((NumberValue) value).value;
             if (Math.abs(x - Math.round(x)) > 1e-6) continue;    // Ignore non-integers
             int i = (int) x;
+            if (opts.rowIndexStartsAt1) i--;
             if (i < 0 || i >= rows.size()) continue;
             TableRow row = rows.get(i);
             answer.add(new Pair<>(row.nameValue, row.indexValue));
@@ -513,7 +527,7 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
         }
       } else if (TableTypeSystem.isCellProperty(r)) {
         ////////////////////////////////////////////////////////////
-        // (fb:cell.cell.number (number 5)) --> {fb:cell.5 fb:cell.5_dollars, ...}
+        // (fb:cell.cell.number (number 5)) --> {fb:cell_id.5 fb:cell_population.5, ...}
         // Possibly with repeated id (if there are multiple cells with that id)
         for (TableColumn column : columns) {
           for (TableCell cell : column.children) {
@@ -608,8 +622,15 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
     if (nameValueId.startsWith("!")) nameValueId = nameValueId.substring(1);
     if (cellIdToTableCellProperties.containsKey(nameValueId))
       return cellIdToTableCellProperties.get(nameValueId).originalString;
+    if (partIdToOriginalString.containsKey(nameValueId))
+      return partIdToOriginalString.get(nameValueId);
     if (relationIdToTableColumn.containsKey(nameValueId))
       return relationIdToTableColumn.get(nameValueId).originalString;
+    if (nameValueId.startsWith(TableTypeSystem.CELL_SPECIFIC_TYPE_PREFIX)) {
+      String property = nameValueId.replace(TableTypeSystem.CELL_SPECIFIC_TYPE_PREFIX, TableTypeSystem.ROW_PROPERTY_NAME_PREFIX);
+      if (relationIdToTableColumn.containsKey(property))
+        return relationIdToTableColumn.get(property).originalString;
+    }
     return null;
   }
 
@@ -636,29 +657,28 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
    * Return a list of rows that contain a cell with the specified NameValue ID.
    */
   public List<Integer> getRowsOfCellId(String nameValueId) {
+    String property = TableTypeSystem.getPropertyOfEntity(nameValueId);
+    if (property == null) return null;
+    TableColumn column = relationIdToTableColumn.get(property);
+    if (column == null) return null;
     List<Integer> answer = new ArrayList<>();
-    TableCellProperties properties = cellIdToTableCellProperties.get(nameValueId);
-    if (properties == null) return answer;
-    for (TableColumn column : properties.columns) {
-      for (int i = 0; i < column.children.size(); i++) {
-        if (column.children.get(i).properties.id.equals(nameValueId))
-          answer.add(i);
-      }
+    for (int i = 0; i < column.children.size(); i++) {
+      if (column.children.get(i).properties.id.equals(nameValueId))
+        answer.add(i);
     }
     return answer;
   }
 
   /**
-   * Return a list of columns that contain a cell with the specified NameValue ID.
+   * Return the index of the column with the specified ID. Return -1 if not found.
    */
-  public List<String> getColumnsOfCellId(String nameValueId) {
-    List<String> answer = new ArrayList<>();
-    TableCellProperties properties = cellIdToTableCellProperties.get(nameValueId);
-    if (properties == null) return answer;
-    for (TableColumn column : properties.columns) {
-      answer.add(column.columnName);
+  public int getColumnIndex(String nameValueId) {
+    if (nameValueId.startsWith("!"))
+      nameValueId = nameValueId.substring(1);
+    for (int j = 0; j < columns.size(); j++) {
+      if (columns.get(j).relationNameValue.id.equals(nameValueId)) return j;
     }
-    return answer;
+    return -1;
   }
 
   // ============================================================
@@ -666,18 +686,13 @@ public class TableKnowledgeGraph extends KnowledgeGraph implements FuzzyMatchabl
   // ============================================================
 
   public static void main(String[] args) {
-    //opts.baseCSVDir = "tables/toy-examples/random/";
-    //String filename = "nikos_machlas.csv";
     StringNormalizationUtils.opts.verbose = 5;
-    //LanguageAnalyzer.opts.languageAnalyzer = "corenlp.CoreNLPAnalyzer";
     StringNormalizationUtils.opts.numberCanStartAnywhere = true;
     StringNormalizationUtils.opts.num2CanStartAnywhere = true;
-    opts.baseCSVDir = "lib/data/tables/";
+    opts.baseCSVDir = "lib/data/WikiTableQuestions/";
     String filename = "csv/200-csv/0.csv";
     TableKnowledgeGraph graph = (TableKnowledgeGraph) KnowledgeGraph.fromLispTree(
         LispTree.proto.parseFromString("(graph tables.TableKnowledgeGraph " + filename + ")"));
-    //LogInfo.logs("%s", graph.toLispTree().toStringWrap());
-    //LogInfo.logs("%s", graph.toTableValue().toLispTree().toStringWrap(100));
     for (TableColumn column : graph.columns) {
       LogInfo.begin_track("%s (%s)", column.columnName, column.originalString);
       for (TableCell cell : column.children) {

src/edu/stanford/nlp/sempre/tables/TableTypeSystem.java

@@ -1,19 +1,17 @@
 package edu.stanford.nlp.sempre.tables;
 
 import java.util.*;
-import java.util.function.Function;
 
 import edu.stanford.nlp.sempre.*;
 
 /**
  * Typing System for table. Affects naming convention and how the types of formulas are inferred.
  *
- *    ROW: name = fb:row.r[index]        | type = fb:type.row
- *   CELL: name = fb:cell.[string]       | type = fb:type.cell
- * COLUMN: name = fb:row.row.[fieldName] | type = (-> fb:type.cell fb:type.row)
- *   PART: name = fb:part.[string]       | type = fb:type.part
+ *      ROW: name = fb:row.r[index]              | type = fb:type.row
+ *     CELL: name = fb:cell_[fieldName].[string] | type = (union fb:type.cell fb:column.[fieldName])
+ * PROPERTY: name = fb:row.row.[fieldName]       | type = (-> (union fb:type.cell fb:column.[fieldName]) fb:type.row)
  *
- * Identical strings in different cells are mapped to the same name.
+ * Note that the same string in different columns are mapped to different names.
  *
  * @author ppasupat
  */
@@ -27,10 +25,12 @@ public abstract class TableTypeSystem {
   // Type names
   public static final String ROW_TYPE = "fb:type.row";
   public static final SemType ROW_SEMTYPE = SemType.newAtomicSemType(ROW_TYPE);
-  public static final String CELL_TYPE = "fb:type.cell";
-  public static final SemType CELL_SEMTYPE = SemType.newAtomicSemType(CELL_TYPE);
-  public static final String PART_TYPE = "fb:type.part";
-  public static final SemType PART_SEMTYPE = SemType.newAtomicSemType(PART_TYPE);
+  public static final String CELL_GENERIC_TYPE = "fb:type.cell";
+  public static final SemType CELL_GENERIC_SEMTYPE = SemType.newAtomicSemType(CELL_GENERIC_TYPE);
+  public static final String CELL_SPECIFIC_TYPE_PREFIX = "fb:column";
+  public static final String PART_GENERIC_TYPE = "fb:type.part";
+  public static final SemType PART_GENERIC_SEMTYPE = SemType.newAtomicSemType(PART_GENERIC_TYPE);
+  public static final String PART_SPECIFIC_TYPE_PREFIX = "fb:part";
 
   // Row relations
   public static final String ROW_PROPERTY_NAME_PREFIX = "fb:row.row";
@@ -49,17 +49,13 @@ public abstract class TableTypeSystem {
   public static final NameValue CELL_NUMBER_VALUE = new NameValue("fb:cell.cell.number");
   public static final NameValue CELL_DATE_VALUE = new NameValue("fb:cell.cell.date");
   public static final NameValue CELL_NUM2_VALUE = new NameValue("fb:cell.cell.num2");
-  public static final NameValue CELL_STR1_VALUE = new NameValue("fb:cell.cell.str1");
-  public static final NameValue CELL_STR2_VALUE = new NameValue("fb:cell.cell.str2");
   public static final NameValue CELL_PART_VALUE = new NameValue("fb:cell.cell.part");
   public static final Map<Value, SemType> CELL_PROPERTIES = new HashMap<>();
   static {
-    CELL_PROPERTIES.put(CELL_NUMBER_VALUE, SemType.newFuncSemType(CanonicalNames.NUMBER, CELL_TYPE));
-    CELL_PROPERTIES.put(CELL_DATE_VALUE, SemType.newFuncSemType(CanonicalNames.DATE, CELL_TYPE));
-    CELL_PROPERTIES.put(CELL_NUM2_VALUE, SemType.newFuncSemType(CanonicalNames.NUMBER, CELL_TYPE));
-    CELL_PROPERTIES.put(CELL_STR1_VALUE, SemType.newFuncSemType(PART_TYPE, CELL_TYPE));
-    CELL_PROPERTIES.put(CELL_STR2_VALUE, SemType.newFuncSemType(PART_TYPE, CELL_TYPE));
-    CELL_PROPERTIES.put(CELL_PART_VALUE, SemType.newFuncSemType(PART_TYPE, CELL_TYPE));
+    CELL_PROPERTIES.put(CELL_NUMBER_VALUE, SemType.newFuncSemType(CanonicalNames.NUMBER, CELL_GENERIC_TYPE));
+    CELL_PROPERTIES.put(CELL_DATE_VALUE, SemType.newFuncSemType(CanonicalNames.DATE, CELL_GENERIC_TYPE));
+    CELL_PROPERTIES.put(CELL_NUM2_VALUE, SemType.newFuncSemType(CanonicalNames.NUMBER, CELL_GENERIC_TYPE));
+    CELL_PROPERTIES.put(CELL_PART_VALUE, SemType.newFuncSemType(PART_GENERIC_TYPE, CELL_GENERIC_TYPE));
   }
 
   // ============================================================
@@ -95,29 +91,18 @@ public abstract class TableTypeSystem {
   }
 
   /**
-   * Look up the (normalized) "original string" in originalStringToId.
-   * If found, return the found id (for creating a NameValue).
-   * If not, create a new id, add it to originalStringToId, and return the id.
-   *
-   * CanonicalNameToId should map the canonical name (e.g., "palo_alto")
-   * to the actual id (e.g., "fb:cell.palo_alto")
+   * When id = [prefix]_[1].[2], get [1]. For example:
+   * - fb:row_[tableId].r[index] --> [tableId]
+   * - fb:cell_[fieldName].[string] --> [fieldName]
    */
-  public static String getOrCreateName(String originalString, Map<String, String> originalStringToId,
-      Function<String, String> canonicalNameToId) {
-    String normalized = StringNormalizationUtils.characterNormalize(originalString).toLowerCase();
-    String id = originalStringToId.get(normalized);
-    if (id == null) {
-      String canonicalName = TableTypeSystem.canonicalizeName(normalized);
-      id = TableTypeSystem.getUnusedName(
-          canonicalNameToId.apply(canonicalName), originalStringToId.values());
-      originalStringToId.put(normalized, id);
-    }
-    return id;
+  public static String getIdAfterUnderscore(String id, String prefix) {
+    return id.substring(prefix.length() + 1).split("\\.", 2)[0];
   }
 
   /**
-   * When id = {prefix}{string without .}.{name}, get {name}
-   * (Note that {prefix} can contain ".")
+   * When id = [prefix]_[1].[2], get [2]. For example:
+   * - fb:row_[tableId].r[index] --> r[index]
+   * - fb:cell.[string] or fb:cell_[fieldName].[string] --> [string]
    */
   public static String getIdAfterPeriod(String id, String prefix) {
     return id.substring(prefix.length()).split("\\.", 2)[1];
@@ -143,12 +128,20 @@ public abstract class TableTypeSystem {
     return ROW_NAME_PREFIX + ".r" + index;
   }
 
-  public static String getCellName(String id) {
-    return CELL_NAME_PREFIX + "." + id;
+  public static String getCellName(String id, String fieldName) {
+    return CELL_NAME_PREFIX + "_" + fieldName + "." + id;
+  }
+  
+  public static String getPartName(String id, String fieldName) {
+    return PART_NAME_PREFIX + "_" + fieldName + "." + id;
   }
 
-  public static String getPartName(String id) {
-    return PART_NAME_PREFIX + "." + id;
+  public static String getCellType(String fieldName) {
+    return CELL_SPECIFIC_TYPE_PREFIX + "." + fieldName;
+  }
+
+  public static String getPartType(String fieldName) {
+    return PART_SPECIFIC_TYPE_PREFIX + "." + fieldName;
   }
 
   public static String getRowPropertyName(String fieldName) {
@@ -159,25 +152,36 @@ public abstract class TableTypeSystem {
     return ROW_CONSECUTIVE_PROPERTY_NAME_PREFIX + "." + fieldName;
   }
 
-  public static SemType getEntityTypeFromId(String id) {
-    if (id.startsWith(ROW_NAME_PREFIX)) return ROW_SEMTYPE;
-    if (id.startsWith(CELL_NAME_PREFIX)) return CELL_SEMTYPE;
-    if (id.startsWith(PART_NAME_PREFIX)) return PART_SEMTYPE;
+  public static SemType getEntityTypeFromId(String entity) {
+    if (entity.startsWith(CELL_NAME_PREFIX)) {
+      String fieldName = getIdAfterUnderscore(entity, CELL_NAME_PREFIX);
+      return SemType.newUnionSemType(CELL_GENERIC_TYPE, getCellType(fieldName));
+    } else if (entity.startsWith(PART_NAME_PREFIX)) {
+      String fieldName = getIdAfterUnderscore(entity, PART_NAME_PREFIX);
+      return SemType.newUnionSemType(PART_GENERIC_TYPE, getPartType(fieldName));
+    }
     return null;
   }
 
-  public static SemType getPropertyTypeFromId(String id) {
-    NameValue value = new NameValue(id);
-    // Predefined properties
-    SemType type = ROW_RELATIONS.get(value);
-    if (type != null) return type;
-    type = CELL_PROPERTIES.get(value);
-    if (type != null) return type;
-    // Column-based properties
-    if (id.startsWith(ROW_PROPERTY_NAME_PREFIX))
-      return SemType.newFuncSemType(CELL_TYPE, ROW_TYPE);
-    if (id.startsWith(ROW_CONSECUTIVE_PROPERTY_NAME_PREFIX))
-      return SemType.newFuncSemType(CanonicalNames.NUMBER, ROW_TYPE);
+  public static SemType getPropertyTypeFromId(String property) {
+    if (property.startsWith(ROW_PROPERTY_NAME_PREFIX)) {
+      SemType rowPropertyType = ROW_RELATIONS.get(new NameValue(property));
+      if (rowPropertyType != null) return rowPropertyType;
+      String fieldName = getIdAfterPeriod(property, ROW_PROPERTY_NAME_PREFIX);
+      return new FuncSemType(SemType.newUnionSemType(getCellType(fieldName), CELL_GENERIC_TYPE), ROW_SEMTYPE);
+    }
+    if (property.startsWith(CELL_PROPERTY_NAME_PREFIX)) {
+      SemType cellPropertyType = CELL_PROPERTIES.get(new NameValue(property));
+      return cellPropertyType;
+    }
+    return null;
+  }
+
+  public static String getPropertyOfEntity(String entity) {
+    if (entity.startsWith(CELL_NAME_PREFIX)) {
+      String fieldName = getIdAfterUnderscore(entity, CELL_NAME_PREFIX);
+      return getRowPropertyName(fieldName);
+    }
     return null;
   }
 

src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java

@@ -25,6 +25,8 @@ public class TableValueEvaluator implements ValueEvaluator {
     public boolean ignoreNumberValueUnits = true;
     @Option(gloss = "Strict date evaluation (year, month, and date all have to match)")
     public boolean strictDateEvaluation = false;
+    @Option(gloss = "Check if the normalized text matches the official evaluator")
+    public boolean checkStringNormalization = false;
   }
   public static Options opts = new Options();
 
@@ -69,8 +71,16 @@ public class TableValueEvaluator implements ValueEvaluator {
         String predText = (pred instanceof NameValue) ? ((NameValue) pred).description : ((DescriptionValue) pred).value;
         if (predText == null) predText = "";
         if (opts.allowNormalizedStringMatch) {
-          targetText = StringNormalizationUtils.aggressiveNormalize(targetText);
-          predText = StringNormalizationUtils.aggressiveNormalize(predText);
+          targetText = StringNormalizationUtils.aggressiveNormalize(targetText).toLowerCase();
+          predText = StringNormalizationUtils.aggressiveNormalize(predText).toLowerCase();
+          if (opts.checkStringNormalization) {
+            String targetTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(targetText);
+            String predTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(predText);
+            if (!targetTextOfficial.equals(targetText) && !(targetTextOfficial + ".").equals(targetText))
+              LogInfo.warnings("Different normalization: [%s][%s]", targetTextOfficial, targetText);
+            if (!predTextOfficial.equals(predText) && !(predTextOfficial + ".").equals(predText))
+              LogInfo.warnings("Different normalization: [%s][%s]", predTextOfficial, predText);
+          }
         }
         return targetText.equals(predText);
       } else if (pred instanceof NumberValue) {

src/edu/stanford/nlp/sempre/tables/TableValuePreprocessor.java

@@ -1,5 +1,9 @@
 package edu.stanford.nlp.sempre.tables;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
 import java.util.*;
 
 import edu.stanford.nlp.sempre.*;
@@ -8,15 +12,20 @@ import fig.basic.*;
 public class TableValuePreprocessor extends TargetValuePreprocessor {
   public static class Options {
     @Option(gloss = "Verbosity") public int verbose = 0;
+    @Option(gloss = "Read preprocessed values from these .tagged files")
+    public List<String> taggedFiles = new ArrayList<>();
   }
   public static Options opts = new Options();
 
   @Override
-  public Value preprocess(Value value) {
+  public Value preprocess(Value value, Example ex) {
+    if (!opts.taggedFiles.isEmpty() && ex != null) {
+      return getFromTaggedFile(ex.id);
+    }
     if (value instanceof ListValue) {
       List<Value> values = new ArrayList<>();
       for (Value entry : ((ListValue) value).values) {
-        values.add(preprocess(entry));
+        values.add(preprocessSingle(entry));
       }
       return new ListValue(values);
     } else {
@@ -67,4 +76,76 @@ public class TableValuePreprocessor extends TargetValuePreprocessor {
     return new DescriptionValue(origString);
   }
 
+  // ============================================================
+  // Get preprocessed value from tagged file
+  // ============================================================
+
+  Map<String, Value> idToValue = null;
+
+  public Value getFromTaggedFile(String id) {
+    if (idToValue == null) readTaggedFiles();
+    return idToValue.get(id);
+  }
+
+  protected void readTaggedFiles() {
+    LogInfo.begin_track("Reading .tagged files");
+    idToValue = new HashMap<>();
+    for (String path : opts.taggedFiles) {
+      File file = new File(path);
+      if (file.isDirectory()) {
+        for (File subpath : file.listFiles())
+          readTaggedFile(subpath.toString());
+      } else {
+        readTaggedFile(path);
+      }
+    }
+    LogInfo.logs("Read %d entries", idToValue.size());
+    LogInfo.end_track();
+  }
+
+  protected void readTaggedFile(String path) {
+    LogInfo.begin_track("Reading %s", path);
+    try (BufferedReader reader = new BufferedReader(new FileReader(path))) {
+      // Read header
+      String[] header = reader.readLine().split("\t", -1);
+      int exIdIndex = 0, targetCanonIndex = 0;
+      while (!"id".equals(header[exIdIndex]))
+        exIdIndex++;
+      while (!"targetCanon".equals(header[targetCanonIndex]))
+        targetCanonIndex++;
+      // Read each line
+      String line;
+      while ((line = reader.readLine()) != null) {
+        String[] fields = line.split("\t", -1);     // Include trailing spaces
+        String[] rawValues = fields[targetCanonIndex].split("\\|");
+        List<Value> values = new ArrayList<>();
+        for (String rawValue : rawValues) {
+          values.add(simpleCanonicalize(rawValue));
+        }
+        idToValue.put(fields[exIdIndex], new ListValue(values));
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    LogInfo.end_track();
+  }
+
+  /**
+   * Like canonicalize, but assume that the string is already well-formed:
+   * - A number should look like a float
+   * - A date should be in the ISO format
+   * - Otherwise, the value is treated as a string.
+   */
+  protected Value simpleCanonicalize(String origString) {
+    Value answer;
+    // Try converting to a number.
+    answer = StringNormalizationUtils.parseNumberStrict(origString);
+    if (answer != null) return answer;
+    // Try converting to a date.
+    answer = StringNormalizationUtils.parseDate(origString);
+    if (answer != null) return answer;
+    // Just treat as a description string
+    return new DescriptionValue(origString);
+  }
+
 }

src/edu/stanford/nlp/sempre/tables/alignment/AlignerMain.java

@@ -1,157 +0,0 @@
-package edu.stanford.nlp.sempre.tables.alignment;
-
-import java.io.PrintWriter;
-import java.util.*;
-
-import edu.stanford.nlp.sempre.*;
-import edu.stanford.nlp.sempre.tables.TableFormulaEvaluator;
-import edu.stanford.nlp.sempre.tables.alignment.BitextData.BitextDataGroup;
-import fig.basic.*;
-import fig.exec.*;
-
-public class AlignerMain implements Runnable {
-  public static class Options {
-    @Option public int verbose = 0;
-    @Option public AlignerName aligner = AlignerName.IBM1_Z_TO_X;
-    @Option(gloss = "probability that the target aligns to null source")
-    public NullWordHandling nullWordHandling = NullWordHandling.UNIFORM;
-    @Option(gloss = "if nullWordHandling == FIXED, use this as the null word probability")
-    public double fixedNullWordProb = 0.1;
-  }
-  public static Options opts = new Options();
-
-  public static enum AlignerName {
-    IBM1_X_TO_Z,   // IBM model 1 from x to z
-    IBM1_Z_TO_X,   // IBM model 1 from z to x
-  }
-  public static enum NullWordHandling {
-    FIXED,      // Fixed as a constant
-    UNIFORM,    // = 1 / (source length + 1)
-    TRAINED,    // Treat as a word
-  }
-
-  public static final double epsilon = 1e-6;
-
-  public static void main(String[] args) {
-    Execution.run(args, "AlignerMainMain", new AlignerMain(), Master.getOptionsParser());
-  }
-
-  protected TableFormulaEvaluator evaluator;
-  protected AlignmentComputer alignmentComputer;
-  protected BitextData bitextData;
-
-  @Override
-  public void run() {
-    Builder builder = new Builder();
-    builder.build();
-    evaluator = (TableFormulaEvaluator) builder.valueEvaluator;
-    Dataset dataset = new Dataset();
-    dataset.read();
-    // Train the aligner. Only the "train" group is used.
-    bitextData = new BitextData(dataset.examples("train"));
-    switch (opts.aligner) {
-      case IBM1_X_TO_Z: alignmentComputer = new IBM1XToZAlignmentComputer(); break;
-      case IBM1_Z_TO_X: alignmentComputer = new IBM1ZToXAlignmentComputer(); break;
-      default: throw new RuntimeException("Unknown aligner: " + opts.aligner);
-    }
-    // Align and dump the results
-    alignmentComputer.align(bitextData);
-    dumpModel();
-    //computeAllScores();
-    evaluate(dataset.examples("dev"), "dev");
-    evaluate(dataset.examples("test"), "test");
-  }
-
-  /**
-   * Dump the model parameters into a file.
-   */
-  protected void dumpModel() {
-    String filename = Execution.getFile("alignment");
-    LogInfo.begin_track("Writing to %s", filename);
-    try (PrintWriter out = new PrintWriter(filename)) {
-      alignmentComputer.dump(out);
-    } catch (Exception e) {
-      e.printStackTrace();
-      LogInfo.fail(e);
-    }
-    LogInfo.end_track();
-  }
-
-  /**
-   * Compute the candidate scores for all examples in the dataset.
-   */
-  protected void computeAllScores() {
-    PrintWriter out = IOUtils.openOutHard(Execution.getFile("aligned-formulas.gz"));
-    for (BitextDataGroup group : bitextData.bitextDataGroups) {
-      List<Pair<Formula, Double>> scores = alignmentComputer.score(group);
-      Collections.sort(scores, new Pair.ReverseSecondComparator<Formula, Double>());
-      // Log to stdout
-      LogInfo.begin_track("%s", group.id);
-      LogInfo.logs("Tokens: %s", group.tokens);
-      for (Pair<Formula, Double> pair : scores)
-        LogInfo.logs("%10.3f : %s", pair.getSecond(), pair.getFirst());
-      LogInfo.end_track();
-      // Dump to gzip file
-      out.printf("########## Example %s ##########\n", group.id);
-      out.println("(example");
-      out.println("  " + LispTree.proto.newList("id", group.id));
-      out.println("  " + LispTree.proto.newList("utterance", group.ex.utterance));
-      out.println("  " + LispTree.proto.newList("targetValue", group.ex.targetValue.toLispTree()));
-      out.println("  " + group.ex.context.toLispTree());
-      out.println("  (derivations");
-      for (Pair<Formula, Double> pair : scores) {
-        if (pair.getSecond().isInfinite()) continue;
-        LispTree tree = LispTree.proto.newList();
-        tree.addChild(LispTree.proto.newLeaf("derivation"));
-        tree.addChild(LispTree.proto.newList("formula", pair.getFirst().toLispTree()));
-        tree.addChild(LispTree.proto.newList("score", String.format("%.3f", pair.getSecond())));
-        out.println("    " + tree);
-      }
-      out.println("  )");
-      out.println(")");
-    }
-    out.close();
-  }
-
-  protected void evaluate(List<Example> examples, String group) {
-    if (examples == null || examples.isEmpty()) return;
-    LogInfo.begin_track("AlignerMain.evaluate(%s)", group);
-    Evaluation totalEvaluation = new Evaluation();
-    for (Example ex : examples) {
-      LogInfo.begin_track("%s", ex.id);
-      LogInfo.logs("Utterance: %s", ex.utterance);
-      List<Pair<Formula, Double>> scores = alignmentComputer.score(new BitextDataGroup(ex));
-      Collections.sort(scores, new Pair.ReverseSecondComparator<Formula, Double>());
-      boolean correct = false, oracle = false;
-      if (scores.isEmpty()) {
-        LogInfo.logs("The beam is empty.");
-      } else {
-        Formula bestFormula = scores.get(0).getFirst();
-        evaluator.log(ex, bestFormula);
-        double compatibility = evaluator.getCompatibilityAnnotationStrict(ex, bestFormula);
-        if (compatibility > 0)
-          correct = true;
-        for (Pair<Formula, Double> pair : scores) {
-          if (evaluator.getCompatibilityAnnotationStrict(ex, pair.getFirst()) > 0) {
-            oracle = true;
-            break;
-          }
-        }
-      }
-      Evaluation evaluation = new Evaluation();
-      evaluation.add("correct", correct);
-      evaluation.add("oracle", oracle);
-      evaluation.add("numCandidates", scores.size());
-      if (!scores.isEmpty())
-        evaluation.add("parsedNumCandidates", scores.size());
-      LogInfo.logs("Current: %s", evaluation.summary());
-      totalEvaluation.add(evaluation);
-      LogInfo.logs("Cumulative(%s): %s", group, totalEvaluation.summary());
-      LogInfo.end_track();
-    }
-    totalEvaluation.logStats(group);
-    totalEvaluation.putOutput(group);
-    LogInfo.end_track();
-  }
-
-}

src/edu/stanford/nlp/sempre/tables/alignment/AlignmentComputer.java

@@ -1,23 +0,0 @@
-package edu.stanford.nlp.sempre.tables.alignment;
-
-import java.io.PrintWriter;
-import java.util.List;
-
-import edu.stanford.nlp.sempre.Formula;
-import edu.stanford.nlp.sempre.tables.alignment.BitextData.BitextDataGroup;
-import edu.stanford.nlp.sempre.tables.alignment.BitextData.BitextDatum;
-import fig.basic.Pair;
-
-public interface AlignmentComputer {
-  // Given a BitextData, train an alignment model.
-  public void align(BitextData bitextData);
-
-  // Print the parameters
-  public void dump(PrintWriter out);
-
-  // Compute the scores of all formulas z in an example (x, y)
-  public List<Pair<Formula, Double>> score(BitextDataGroup group);
-
-  // Compute the score of a single formula z in an example (x, y)
-  double score(BitextDatum datum);
-}

src/edu/stanford/nlp/sempre/tables/alignment/BitextData.java

@@ -1,76 +0,0 @@
-package edu.stanford.nlp.sempre.tables.alignment;
-
-import java.util.*;
-
-import edu.stanford.nlp.sempre.*;
-import fig.basic.*;
-
-/**
- * Preprocess examples for training an alignment model.
- *
- * @author ppasupat
- */
-public class BitextData {
-
-  public final List<BitextDataGroup> bitextDataGroups;
-  public final Set<String> allTokens, allPredicates;
-
-  public BitextData(List<Example> examples) {
-    LogInfo.begin_track("Creating BitextData");
-    bitextDataGroups = new ArrayList<>();
-    for (Example ex : examples) {
-      if (!ex.predDerivations.isEmpty())
-        bitextDataGroups.add(new BitextDataGroup(ex));
-    }
-    // Collect all tokens and all predicates
-    allTokens = new HashSet<>();
-    allPredicates = new HashSet<>();
-    for (BitextDataGroup group : bitextDataGroups) {
-      allTokens.addAll(group.tokens);
-      for (BitextDatum datum : group.bitextDatums) {
-        allPredicates.addAll(datum.predicates);
-      }
-    }
-    LogInfo.end_track();
-  }
-
-  public static class BitextDataGroup {
-    public final Example ex;
-    public final String id;
-    public final List<String> tokens;
-    public final List<BitextDatum> bitextDatums;
-
-    public BitextDataGroup(Example ex) {
-      this.ex = ex;
-      id = ex.id;
-      tokens = new ArrayList<>(ex.languageInfo.tokens);
-      bitextDatums = new ArrayList<>();
-      for (Derivation d : ex.predDerivations) {
-        bitextDatums.add(new BitextDatum(this, d));
-      }
-    }
-  }
-
-  public static class BitextDatum {
-    public final BitextDataGroup group;
-    public final Formula formula;
-    public final List<String> predicates;
-
-    public BitextDatum(BitextDataGroup group, Derivation d) {
-      this.group = group;
-      formula = d.formula;
-      predicates = new ArrayList<>();
-      traversePredicates(formula.toLispTree());
-    }
-
-    private void traversePredicates(LispTree t) {
-      if (t.isLeaf()) {
-        predicates.add(t.value);
-      } else {
-        for (LispTree child : t.children)
-          traversePredicates(child);
-      }
-    }
-  }
-
-}

src/edu/stanford/nlp/sempre/tables/alignment/DoubleMap.java

@@ -1,79 +0,0 @@
-package edu.stanford.nlp.sempre.tables.alignment;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import fig.basic.Pair;
-
-class DoubleMap {
-  private Map<Pair<String, String>, Double> m;
-
-  public DoubleMap() {
-    m = new HashMap<>();
-  }
-
-  public double get(Pair<String, String> k) {
-    Double v = m.get(k);
-    return v == null ? 0.0 : v;
-  }
-  public double get(String k1, String k2) { return get(new Pair<>(k1, k2)); }
-
-  public void put(Pair<String, String> k, double v) {
-    m.put(k, v);
-  }
-  public void put(String k1, String k2, double v) { put(new Pair<>(k1, k2), v); }
-
-  public void incr(Pair<String, String> k, double v) {
-    Double oldV = m.get(k);
-    m.put(k, (oldV == null ? 0.0 : oldV) + v);
-  }
-  public void incr(String k1, String k2, double v) { incr(new Pair<>(k1, k2), v); }
-
-  public void reverseKeys() {
-    Map<Pair<String, String>, Double> newM = new HashMap<>();
-    for (Map.Entry<Pair<String, String>, Double> entry : m.entrySet()) {
-      newM.put(entry.getKey().reverse(), entry.getValue());
-    }
-    m = newM;
-  }
-
-  public DoubleMap getReverseKeys() {
-    DoubleMap newM = new DoubleMap();
-    for (Map.Entry<Pair<String, String>, Double> entry : m.entrySet()) {
-      newM.put(entry.getKey().reverse(), entry.getValue());
-    }
-    return newM;
-  }
-
-  public Set<Map.Entry<Pair<String, String>, Double>> entrySet() {
-    return m.entrySet();
-  }
-
-  public static DoubleMap product(DoubleMap dm1, DoubleMap dm2) {
-    DoubleMap dmp = new DoubleMap();
-    for (Map.Entry<Pair<String, String>, Double> entry : dm1.entrySet()) {
-      double v1 = entry.getValue(), v2 = dm2.get(entry.getKey());
-      if (v1 * v2 > 0) dmp.put(entry.getKey(), v1 * v2);
-    }
-    return dmp;
-  }
-
-  // A DoubleMap that returns the same value always.
-  static class ConstantDoubleMap extends DoubleMap {
-    private final double value;
-
-    public ConstantDoubleMap(double value) {
-      this.value = value;
-    }
-
-    public double get(Pair<String, String> k) { return value; }
-    public double get(String k1, String k2) { return value; }
-    public void put(Pair<String, String> k, double v) { throw new RuntimeException("cannot put"); }
-    public void put(String k1, String k2, double v) { throw new RuntimeException("cannot put"); }
-    public void incr(Pair<String, String> k, double v) { throw new RuntimeException("cannot incr"); }
-    public void incr(String k1, String k2, double v) { throw new RuntimeException("cannot incr"); }
-    public void reverseKeys() { }
-    public DoubleMap getReverseKeys() { return this; }
-    public Set<Map.Entry<Pair<String, String>, Double>> entrySet() { throw new RuntimeException("cannot entrySet"); }
-  }
-}

src/edu/stanford/nlp/sempre/tables/alignment/IBM1AlignmentComputer.java

@@ -1,230 +0,0 @@
-package edu.stanford.nlp.sempre.tables.alignment;
-
-import java.io.PrintWriter;
-import java.util.*;
-
-import edu.stanford.nlp.sempre.Formula;
-import edu.stanford.nlp.sempre.Learner;
-import edu.stanford.nlp.sempre.tables.alignment.AlignerMain.NullWordHandling;
-import edu.stanford.nlp.sempre.tables.alignment.BitextData.BitextDataGroup;
-import edu.stanford.nlp.sempre.tables.alignment.BitextData.BitextDatum;
-import fig.basic.*;
-import fig.exec.Execution;
-
-/**
- * IBM Model 1.
- *
- * Each target word independently aligns to a source word (or NULL).
- *
- * @author ppasupat
- */
-public abstract class IBM1AlignmentComputer implements AlignmentComputer {
-  public static class Options {
-    @Option(gloss = "When computing p(t|s), use the sum over all alignments instead of argmax")
-    public boolean marginalizeAlignments = false;
-  }
-  public static Options opts = new Options();
-
-  // genProbs[w_s, w_t]
-  // = P_theta(w_t|w_s) = probability of aligning target word w_t to source word w_s
-  protected DoubleMap genProbs;
-
-  protected abstract Collection<String> getAllSources(BitextData bitextData);
-  protected abstract Collection<String> getAllTargets(BitextData bitextData);
-  protected abstract List<String> getSource(BitextDatum datum);
-  protected abstract List<String> getTarget(BitextDatum datum);
-
-  protected double getNullProb(String target, int sourcesSize) {
-    switch (AlignerMain.opts.nullWordHandling) {
-      case FIXED: return AlignerMain.opts.fixedNullWordProb;
-      case UNIFORM: return 1.0 / sourcesSize;
-      case TRAINED: return genProbs.get(null, target);
-      default:
-        throw new RuntimeException("Unknown nullWordHandling " + AlignerMain.opts.nullWordHandling);
-    }
-  }
-
-  @Override
-  public void align(BitextData bitextData) {
-    LogInfo.begin_track("IBM1AlignmentComputer.align()");
-    NullWordHandling nullWordHandling = AlignerMain.opts.nullWordHandling;
-
-    // Initialize
-    LogInfo.begin_track("Initialize ...");
-    Set<String> allSources = new HashSet<>(getAllSources(bitextData));
-    Set<String> allTargets = new HashSet<>(getAllTargets(bitextData));
-    allSources.add(null);
-    genProbs = new DoubleMap.ConstantDoubleMap(1.0 / allTargets.size());
-    LogInfo.end_track();
-
-    // Run EM
-    for (int iter = 0; iter < Learner.opts.maxTrainIters; iter++) {
-      LogInfo.begin_track("EM Iteration %d", iter);
-      Execution.putOutput("iter", iter);
-      // parameters --> alignments
-      // alignmentSoftCounts[w_s, w_t]
-      // = soft count of the number of times w_t aligns to w_s
-      DoubleMap alignmentSoftCounts = new DoubleMap();
-      // sourceToMarginalSoftCounts[w_s]
-      // sum of alignmentSoftCounts[w_s, w_t] over all w_t
-      Map<String, Double> sourceToMarginalSoftCounts = new HashMap<>();
-      for (int groupIndex = 0; groupIndex < bitextData.bitextDataGroups.size(); groupIndex++) {
-        BitextDataGroup group = bitextData.bitextDataGroups.get(groupIndex);
-        LogInfo.logs("Example %s (%d): %s", group.id, groupIndex, group.tokens);
-        Execution.putOutput("example", groupIndex);
-        double weight = 1.0 / group.bitextDatums.size();
-        for (BitextDatum datum : group.bitextDatums) {
-          List<String> sources = new ArrayList<>(getSource(datum));
-          // Add a null word in front
-          sources.add(0, null);
-          List<String> targets = getTarget(datum);
-          double[] probs = new double[sources.size()];
-          for (String target : targets) {
-            double normalizer = 1e-10;
-            for (int i = 1; i < sources.size(); i++) {
-              probs[i] = genProbs.get(sources.get(i), target);
-              normalizer += probs[i];
-            }
-            // Handle NULL source
-            probs[0] = getNullProb(target, sources.size());
-            normalizer += probs[0];
-            for (int i = 0; i < sources.size(); i++) {
-              double softCount = probs[i] * weight / normalizer;
-              alignmentSoftCounts.incr(sources.get(i), target, softCount);
-              MapUtils.incr(sourceToMarginalSoftCounts, sources.get(i), softCount);
-            }
-          }
-        }
-      }
-      // alignments --> parameters
-      genProbs = new DoubleMap();
-      for (Map.Entry<Pair<String, String>, Double> entry : alignmentSoftCounts.entrySet()) {
-        String source = entry.getKey().getFirst(), target = entry.getKey().getSecond();
-        double alignmentSoftCount = entry.getValue();
-        if (nullWordHandling != NullWordHandling.TRAINED && source == null) continue;
-        if (alignmentSoftCount <= 0) continue;
-        double prob = alignmentSoftCount / sourceToMarginalSoftCounts.get(source);
-        if (prob > AlignerMain.epsilon)
-          genProbs.put(source, target, prob);
-      }
-      LogInfo.end_track();
-    }
-
-    LogInfo.end_track();
-  }
-
-  @Override
-  public void dump(PrintWriter out) {
-    for (Map.Entry<Pair<String, String>, Double> entry : genProbs.entrySet()) {
-      double value = entry.getValue();
-      if (value < AlignerMain.epsilon)
-        continue;
-      out.printf("%s\t%s\t%.6f\n", entry.getKey().getFirst(), entry.getKey().getSecond(), value);
-    }
-  }
-
-  @Override
-  public List<Pair<Formula, Double>> score(BitextDataGroup group) {
-    List<Pair<Formula, Double>> scores = new ArrayList<>();
-    for (BitextDatum datum : group.bitextDatums) {
-      scores.add(new Pair<>(datum.formula, score(datum)));
-    }
-    return scores;
-  }
-
-  @Override
-  public double score(BitextDatum datum) {
-    List<String> sources = new ArrayList<>(getSource(datum));
-    // Add a null word in front
-    sources.add(0, null);
-    List<String> targets = getTarget(datum);
-    if (AlignerMain.opts.verbose >= 2)
-      LogInfo.begin_track("%s | %s", sources, targets);
-    double score = 0;
-    for (String target : targets) {
-      double alignmentScore = getNullProb(target, sources.size());
-      if (opts.marginalizeAlignments) {
-        // Sum over all alignments
-        for (int i = 1; i < sources.size(); i++) {
-          String source = sources.get(i);
-          alignmentScore += genProbs.get(source, target);
-        }
-        if (AlignerMain.opts.verbose >= 2)
-          LogInfo.logs("[%10.3f] %20s", alignmentScore, target);
-      } else {
-        // Use only the best alignment
-        String bestSource = null;
-        for (int i = 1; i < sources.size(); i++) {
-          String source = sources.get(i);
-          double challenger = genProbs.get(source, target);
-          if (challenger > alignmentScore) {
-            alignmentScore = challenger;
-            bestSource = source;
-          }
-        }
-        if (AlignerMain.opts.verbose >= 2)
-          LogInfo.logs("[%10.3f] %20s : %s", alignmentScore, target, bestSource);
-      }
-      score += Math.log(alignmentScore);
-    }
-    if (AlignerMain.opts.verbose >= 2)
-      LogInfo.end_track();
-    // Normalize by length
-    score -= targets.size() * Math.log(sources.size());
-    return score;
-  }
-
-}
-
-/**
- * IBM Model 1 that models p(formula|utterance) [x ==> z]
- */
-class IBM1XToZAlignmentComputer extends IBM1AlignmentComputer {
-
-  @Override
-  protected Collection<String> getAllSources(BitextData bitextData) {
-    return bitextData.allTokens;
-  }
-
-  @Override
-  protected Collection<String> getAllTargets(BitextData bitextData) {
-    return bitextData.allPredicates;
-  }
-
-  @Override
-  protected List<String> getSource(BitextDatum datum) {
-    return datum.group.tokens;
-  }
-
-  @Override
-  protected List<String> getTarget(BitextDatum datum) {
-    return datum.predicates;
-  }
-
-}
-
-/**
- * IBM Model 1 that models p(utterance|formula). [z ==> x]
- */
-class IBM1ZToXAlignmentComputer extends IBM1AlignmentComputer {
-
-  @Override
-  protected Collection<String> getAllSources(BitextData bitextData) {
-    return bitextData.allPredicates;
-  }
-
-  @Override
-  protected Collection<String> getAllTargets(BitextData bitextData) {
-    return bitextData.allTokens;
-  }
-
-  @Override
-  protected List<String> getSource(BitextDatum datum) {
-    return datum.predicates;
-  }
-
-  @Override
-  protected List<String> getTarget(BitextDatum datum) {
-    return datum.group.tokens;
-  }
-}

src/edu/stanford/nlp/sempre/tables/alter/AggregatedTurkData.java

@@ -93,7 +93,7 @@ public class AggregatedTurkData {
     List<Value> values = new ArrayList<>();
     for (String x : response.split("\\|"))
       values.add(new DescriptionValue(StringNormalizationUtils.unescapeTSV(x)));
-    return TargetValuePreprocessor.getSingleton().preprocess(new ListValue(values));
+    return TargetValuePreprocessor.getSingleton().preprocess(new ListValue(values), null);
   }
 
 }

src/edu/stanford/nlp/sempre/tables/baseline/TableBaselineFeatureComputer.java

@@ -52,15 +52,11 @@ public class TableBaselineFeatureComputer implements FeatureComputer {
     }
     // Define features
     for (String predicted : predictedEntities) {
-      List<String> pCols = graph.getColumnsOfCellId(predicted);
-      // TODO: use all columns in pCols
-      String pProp = pCols.get(0);
+      String pProp = TableTypeSystem.getPropertyOfEntity(predicted);
       List<Integer> pRows = graph.getRowsOfCellId(predicted);
-      if (opts.verbosity >= 2) LogInfo.logs("[p] %s %s %s", predicted, pCols, pRows);
+      if (opts.verbosity >= 2) LogInfo.logs("[p] %s %s %s", predicted, pProp, pRows);
       for (String mentioned : mentionedEntities) {
-        List<String> mCols = graph.getColumnsOfCellId(mentioned);
-        // TODO: use all columns in mCols
-        String mProp = mCols.get(0);
+        String mProp = TableTypeSystem.getPropertyOfEntity(mentioned);
         List<Integer> mRows = graph.getRowsOfCellId(mentioned);
         if (opts.verbosity >= 2) LogInfo.logs("[m] %s %s %s", mentioned, mProp, mRows);
         // Same column as ENTITY + offset

src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java

@@ -0,0 +1,67 @@
+package edu.stanford.nlp.sempre.tables.features;
+
+import java.util.*;
+
+import edu.stanford.nlp.sempre.*;
+import edu.stanford.nlp.sempre.tables.StringNormalizationUtils;
+import edu.stanford.nlp.sempre.tables.TableKnowledgeGraph;
+import edu.stanford.nlp.sempre.tables.TableCell;
+import edu.stanford.nlp.sempre.tables.TableColumn;
+import fig.basic.LogInfo;
+
+public class AnchorFeatureComputer implements FeatureComputer {
+
+  @Override
+  public void extractLocal(Example ex, Derivation deriv) {
+    if (!(FeatureExtractor.containsDomain("anchored-entity"))) return;
+    if (!(deriv.rule.sem instanceof FuzzyMatchFn)) return;
+    FuzzyMatchFn sem = (FuzzyMatchFn) deriv.rule.sem;
+    if (sem.getMatchAny() || sem.getMode() != FuzzyMatchFn.FuzzyMatchFnMode.ENTITY) return;
+    String phrase = ((StringValue) ((ValueFormula<?>) deriv.child(0).formula).value).value;
+    NameValue predicate = (NameValue) ((ValueFormula<?>) deriv.formula).value;
+    TableKnowledgeGraph graph = (TableKnowledgeGraph) ex.context.graph;
+    extractMatchingFeatures(graph, deriv, phrase, predicate);
+  }
+
+  private void extractMatchingFeatures(TableKnowledgeGraph graph,
+      Derivation deriv, String phrase, NameValue predicate) {
+    String predicateString = graph.getOriginalString(predicate);
+    //LogInfo.logs("%s -> %s = %s", phrase, predicate, predicateString);
+    predicateString = StringNormalizationUtils.simpleNormalize(predicateString).toLowerCase();
+    if (predicateString.equals(phrase)) {
+      deriv.addFeature("a-e", "exact");
+      //LogInfo.logs("%s %s exact", phrase, predicateString);
+    } else if (predicateString.startsWith(phrase + " ")) {
+      deriv.addFeature("a-e", "prefix");
+      //LogInfo.logs("%s %s prefix", phrase, predicateString);
+    } else if (predicateString.endsWith(" " + phrase)) {
+      deriv.addFeature("a-e", "suffix");
+      //LogInfo.logs("%s %s suffix", phrase, predicateString);
+    } else if (predicateString.contains(" " + phrase + " ")){
+      deriv.addFeature("a-e", "substring");
+      //LogInfo.logs("%s %s substring", phrase, predicateString);
+    } else {
+      deriv.addFeature("a-e", "other");
+      //LogInfo.logs("%s %s other", phrase, predicateString);
+    }
+    // Does the phrase match other cells?
+    Set<String> matches = new HashSet<>();
+    for (TableColumn column : graph.columns) {
+      for (TableCell cell : column.children) {
+        String s = StringNormalizationUtils.simpleNormalize(cell.properties.originalString).toLowerCase();
+        if (s.contains(phrase) && !cell.properties.id.equals(predicate.id)) {
+          matches.add(s);
+        }
+      }
+    }
+    //LogInfo.logs(">> %s", matches);
+    if (matches.size() == 0) {
+      deriv.addFeature("a-e", "unique");
+    } else if (matches.size() < 3) {
+      deriv.addFeature("a-e", "multiple;" + matches.size());
+    } else {
+      deriv.addFeature("a-e", "multiple;>=3");
+    }
+  }
+
+}

src/edu/stanford/nlp/sempre/tables/features/ColumnCategoryInfo.java

@@ -0,0 +1,84 @@
+package edu.stanford.nlp.sempre.tables.features;
+
+import java.io.*;
+import java.util.*;
+
+import edu.stanford.nlp.sempre.Example;
+import edu.stanford.nlp.sempre.tables.TableKnowledgeGraph;
+import fig.basic.*;
+
+public class ColumnCategoryInfo {
+  public static class Options {
+    @Option(gloss = "Read category information from this file")
+    public String tableCategoryInfo = null;
+  }
+  public static Options opts = new Options();
+
+  // ============================================================
+  // Singleton access
+  // ============================================================
+
+  private static ColumnCategoryInfo singleton;
+
+  public static ColumnCategoryInfo getSingleton() {
+    if (opts.tableCategoryInfo == null)
+      return null;
+    else if (singleton == null)
+      singleton = new ColumnCategoryInfo();
+    return singleton;
+  }
+
+  // ============================================================
+  // Read data from file
+  // ============================================================
+
+  // tableId -> columnIndex -> list of (category, weight)
+  protected static Map<String, List<List<Pair<String, Double>>>> allCategoryInfo = null;
+
+  private ColumnCategoryInfo() {
+    LogInfo.begin_track("Loading category information from %s", opts.tableCategoryInfo);
+    allCategoryInfo = new HashMap<>();
+    try {
+      BufferedReader reader = IOUtils.openIn(opts.tableCategoryInfo);
+      String line;
+      while ((line = reader.readLine()) != null) {
+        String[] tokens = line.split("\t");
+        String tableId = tokens[0];
+        List<List<Pair<String, Double>>> categoryInfoForTable = allCategoryInfo.get(tableId);
+        if (categoryInfoForTable == null)
+          allCategoryInfo.put(tableId, categoryInfoForTable = new ArrayList<>());
+        int columnIndex = Integer.parseInt(tokens[1]);
+        // Assume that the columns are ordered
+        assert categoryInfoForTable.size() == columnIndex;
+        // Read the category-weight pairs
+        List<Pair<String, Double>> categories = new ArrayList<>();
+        for (int i = 2; i < tokens.length; i++) {
+          String[] pair = tokens[i].split(":");
+          categories.add(new Pair<>(pair[0], Double.parseDouble(pair[1])));
+        }
+        categoryInfoForTable.add(categories);
+      }
+      reader.close();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    LogInfo.end_track();
+  }
+
+  // ============================================================
+  // Getters
+  // ============================================================
+
+  public List<Pair<String, Double>> get(String tableId, int columnIndex) {
+    return allCategoryInfo.get(tableId).get(columnIndex);
+  }
+
+  public List<Pair<String, Double>> get(Example ex, String columnId) {
+    TableKnowledgeGraph graph = (TableKnowledgeGraph) ex.context.graph;
+    String tableId = graph.filename;
+    int columnIndex = graph.getColumnIndex(columnId);
+    if (columnIndex == -1) return null;
+    return allCategoryInfo.get(tableId).get(columnIndex);
+  }
+
+}

src/edu/stanford/nlp/sempre/tables/features/FormulaFeatureComputer.java

@@ -1,56 +0,0 @@
-package edu.stanford.nlp.sempre.tables.features;
-
-import java.util.Random;
-
-import edu.stanford.nlp.sempre.*;
-import fig.basic.LispTree;
-
-/**
- * Features on formulas.
- *
- * Originally written as baselines for correct formula evaluation.
- *
- * @author ppasupat
- */
-public class FormulaFeatureComputer implements FeatureComputer {
-
-  @Override
-  public void extractLocal(Example ex, Derivation deriv) {
-    if (!deriv.isRoot(ex.numTokens())) return;
-    if (FeatureExtractor.containsDomain("formula-random")) {
-      // Uniform random value
-      deriv.addFeature("formula-random", "random", new Random(deriv.formula.hashCode()).nextDouble());
-    }
-    if (FeatureExtractor.containsDomain("formula-length")) {
-      // Formula length
-      int length = findLength(Formulas.betaReduction(deriv.formula).toLispTree());
-      deriv.addFeature("formula-length", "length", length);
-    }
-  }
-
-  /**
-   * Find the length of a formula. Contains discounts for some types of formulas.
-   * (e.g., the (number 1) (number 1) in superlatives are ignored)
-   */
-  int findLength(LispTree tree) {
-    if (tree.isLeaf()) {
-      if ("reverse".equals(tree.value) || "var".equals(tree.value) || "lambda".equals(tree.value))
-        return 0;
-      return 1;
-    }
-    if ("argmax".equals(tree.child(0).value) || "argmin".equals(tree.child(0).value)) {
-      int length = 1;
-      for (int i = 3; i < tree.children.size(); i++)
-        length += findLength(tree.child(i));
-      return length;
-    } else {
-      int length = 0;
-      for (LispTree child : tree.children)
-        length += findLength(child);
-      return length;
-    }
-  }
-
-
-
-}

src/edu/stanford/nlp/sempre/tables/features/HeadwordInfo.java

@@ -1,5 +1,9 @@
 package edu.stanford.nlp.sempre.tables.features;
 
+import java.util.concurrent.ExecutionException;
+
+import com.google.common.cache.*;
+
 import edu.stanford.nlp.sempre.*;
 
 /**
@@ -26,35 +30,65 @@ public class HeadwordInfo {
   }
 
   public String toString() {
-    return "(" + questionWord + "," + headword + ")";
+    return "Q=" + questionWord + ",H=" + headword;
   }
 
   public String questionWordTuple() {
-    return "(" + questionWord + ",*)";
+    return "Q=" + questionWord;
   }
 
   public String headwordTuple() {
-    return "(*," + headword + ")";
+    return "H=" + headword;
   }
 
-  // Heuristics: find the first N* after the first W*
-  // Example: tell me [who] is the first [person] ... --> person
-  public static HeadwordInfo analyze(LanguageInfo langInfo) {
-    String questionWord = null;
-    for (int i = 0; i < langInfo.numTokens(); i++) {
-      String posTag = langInfo.posTags.get(i);
-      if (posTag.startsWith("W")) {
-        questionWord = langInfo.lemmaTokens.get(i);
-        if (questionWord.equals("how")) {
-          // Possibly "how many", "how much", ...
-          if (i + 1 < langInfo.numTokens() && langInfo.posTags.get(i + 1).startsWith("J"))
-            questionWord += " " + langInfo.lemmaTokens.get(i + 1);
+  // Caching
+  private static final LoadingCache<Example, HeadwordInfo> cache = CacheBuilder
+      .newBuilder().maximumSize(20)
+      .build(new CacheLoader<Example, HeadwordInfo>() {
+        @Override
+        public HeadwordInfo load(Example ex) throws Exception {
+          LanguageInfo langInfo = ex.languageInfo;
+          String questionWord = "", headWord = "";
+          for (int i = 0; i < langInfo.numTokens(); i++) {
+            String token = langInfo.lemmaTokens.get(i), posTag = langInfo.posTags.get(i);
+            if (posTag.startsWith("W")) {
+              if ("who".equals(token) || "where".equals(token) || "when".equals(token)) {
+                // These are treated as head words
+                headWord = token;
+                //LogInfo.logs("HEADWORD: %s => %s | %s", ex.utterance, questionWord, headWord);
+                return new HeadwordInfo(questionWord.trim(), headWord.trim());
+              }
+              questionWord += " " + token;
+              if (token.equals("how")) {
+                // Possibly "how many", "how much", ...
+                if (i + 1 < langInfo.numTokens() && langInfo.posTags.get(i + 1).startsWith("J"))
+                  questionWord += " " + langInfo.lemmaTokens.get(i + 1);
+              }
+            } else if (posTag.startsWith("N") && !questionWord.isEmpty()) {
+              if ("number".equals(token)) {
+                questionWord += " " + token;
+              } else {
+                headWord += " " + token;
+                while (i + 1 < langInfo.numTokens() && langInfo.posTags.get(i + 1).startsWith("N")) {
+                  i++;
+                  headWord += " " + langInfo.lemmaTokens.get(i);
+                }
+                //LogInfo.logs("HEADWORD: %s => %s | %s", ex.utterance, questionWord, headWord);
+                return new HeadwordInfo(questionWord.trim(), headWord.trim());
+              }
+            }
+          }
+          //LogInfo.logs("HEADWORD: %s => NULL", ex.utterance);
+          return new HeadwordInfo("", "");
         }
-      } else if (posTag.startsWith("N") && questionWord != null) {
-        return new HeadwordInfo(questionWord, langInfo.lemmaTokens.get(i));
-      }
+      });
+
+  public static HeadwordInfo getHeadwordInfo(Example ex) {
+    try {
+      return cache.get(ex);
+    } catch (ExecutionException e) {
+      throw new RuntimeException(e.getCause());
     }
-    return null;
   }
 
 }

src/edu/stanford/nlp/sempre/tables/features/PhraseDenotationFeatureComputer.java

@@ -16,10 +16,6 @@ public class PhraseDenotationFeatureComputer implements FeatureComputer {
   public static class Options {
     @Option(gloss = "Verbosity")
     public int verbose = 0;
-    @Option(gloss = "Look for the type under the first cell property")
-    public boolean lookUnderCellProperty = false;
-    @Option(gloss = "Define features for generic cell types too")
-    public boolean useGenericCellType = false;
   }
   public static Options opts = new Options();
 
@@ -41,14 +37,12 @@ public class PhraseDenotationFeatureComputer implements FeatureComputer {
     // Type based on SemType
     populateSemType("", deriv.type, denotationTypes);
     // Look for the type under the first cell property
-    if (opts.lookUnderCellProperty) {
-      Formula formula = deriv.formula;
-      if (formula instanceof JoinFormula) {
-        JoinFormula join = (JoinFormula) formula;
-        String property = getCellProperty(join.relation);
-        if (property != null) {
-          populateSemType(property + "/", TypeInference.inferType(join.child), denotationTypes);
-        }
+    Formula formula = deriv.formula;
+    if (formula instanceof JoinFormula) {
+      JoinFormula join = (JoinFormula) formula;
+      String property = getCellProperty(join.relation);
+      if (property != null) {
+        populateSemType(property + "/", TypeInference.inferType(join.child), denotationTypes);
       }
     }
     if (denotationTypes.isEmpty()) denotationTypes.add("OTHER");
@@ -62,11 +56,9 @@ public class PhraseDenotationFeatureComputer implements FeatureComputer {
     } else {
       for (LispTree subtree : tree.children) {
         if (!subtree.isLeaf()) continue;
-        // TODO: not sure if this is correct
-        if (subtree.value.startsWith(TableTypeSystem.CELL_TYPE)) {
+        if (subtree.value.startsWith(TableTypeSystem.CELL_SPECIFIC_TYPE_PREFIX)) {
           denotationTypes.add(prefix + subtree.value);
-          if (opts.useGenericCellType)
-            denotationTypes.add(prefix + TableTypeSystem.CELL_TYPE);
+          denotationTypes.add(prefix + TableTypeSystem.CELL_GENERIC_TYPE);
         }
       }
     }
@@ -121,6 +113,7 @@ public class PhraseDenotationFeatureComputer implements FeatureComputer {
       LogInfo.logs("%s %s %s", deriv.value, deriv.type, denotationTypes);
     for (String denotationType : denotationTypes) {
       for (PhraseInfo phraseInfo : phraseInfos) {
+        if (PhraseInfo.opts.forbidBorderStopWordInLexicalizedFeatures && phraseInfo.isBorderStopWord) continue;
         deriv.addFeature("p-d", phraseInfo.lemmaText + ";" + denotationType);
       }
       // Check original column text
@@ -146,8 +139,8 @@ public class PhraseDenotationFeatureComputer implements FeatureComputer {
 
   private void extractHeadwordDenotationFeatures(Example ex, Derivation deriv, Collection<String> denotationTypes) {
     if (!FeatureExtractor.containsDomain("headword-denotation")) return;
-    HeadwordInfo headwordInfo = HeadwordInfo.analyze(ex.languageInfo);
-    if (headwordInfo == null) return;
+    HeadwordInfo headwordInfo = HeadwordInfo.getHeadwordInfo(ex);
+    if (headwordInfo.questionWord.isEmpty() && headwordInfo.headword.isEmpty()) return;
     if (opts.verbose >= 2)
       LogInfo.logs("%s [%s] | %s %s %s", ex.utterance, headwordInfo, deriv.value, deriv.type, denotationTypes);
     for (String denotationType : denotationTypes) {

src/edu/stanford/nlp/sempre/tables/features/PhraseFragmentFeatureComputer.java

@@ -1,274 +0,0 @@
-package edu.stanford.nlp.sempre.tables.features;
-
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.google.common.base.Function;
-
-import edu.stanford.nlp.sempre.*;
-import edu.stanford.nlp.sempre.tables.ScopedFormula;
-import edu.stanford.nlp.sempre.tables.serialize.LazyLoadedExampleList;
-import fig.basic.LogInfo;
-import fig.basic.Option;
-
-/**
- * Extract features of the form (n-gram, logical form fragment)
- *
- * @author ppasupat
- */
-public class PhraseFragmentFeatureComputer implements FeatureComputer {
-  public static class Options {
-    @Option(gloss = "verbosity")
-    public int verbose = 0;
-    @Option(gloss = "number of recursion levels")
-    public int recursionLevels = 1;
-    @Option(gloss = "Define unlexicalized phrase-fragment features")
-    public boolean unlexicalizedPhraseFragment = true;
-    @Option(gloss = "Define lexicalized phrase-fragment features")
-    public boolean lexicalizedPhraseFragment = true;
-    @Option(gloss = "Forbid type of predicate (%row.row) that comes alone")
-    public boolean forbidSingletonPredicateType = true;
-  }
-  public static Options opts = new Options();
-
-  @Override
-  public void extractLocal(Example ex, Derivation deriv) {
-    if (!FeatureExtractor.containsDomain("phrase-fragment")) return;
-    List<PhraseInfo> phraseInfos = PhraseInfo.getPhraseInfos(ex);
-    extractPhraseFragment(ex, deriv, phraseInfos);
-  }
-
-  private void extractPhraseFragment(Example ex, Derivation deriv, List<PhraseInfo> phraseInfos) {
-    Formula formula = deriv.formula;
-    if (opts.verbose >= 1) {
-      List<Formula> children = new ArrayList<>();
-      for (Derivation child : deriv.children)
-        children.add(child.formula);
-      LogInfo.logs("%s => %s", formula, children);
-    }
-    // For formulas from SerializedParser, we need to define all features in one pass
-    if (LazyLoadedExampleList.SERIALIZED_ROOT.equals(deriv.cat)) {
-      extractPhraseFragment(ex, deriv, phraseInfos, formula, opts.recursionLevels, "<R>@ROOT");
-      formula.forEach(new Function<Formula, Boolean>() {
-        @Override
-        public Boolean apply(Formula subformula) {
-          if (!(subformula instanceof ReverseFormula || subformula instanceof LambdaFormula))
-            extractPhraseFragment(ex, deriv, phraseInfos, subformula, opts.recursionLevels, "<R>");
-          return false;
-        }
-      });
-      return;
-    }
-    // Special for ROOT
-    if (Rule.rootCat.equals(deriv.cat)) {
-      extractPhraseFragment(ex, deriv, phraseInfos, formula, opts.recursionLevels, "<R>@ROOT");
-    }
-    // Check type raising (formula == the single child); do not define features here
-    if (deriv.children.size() == 1 && formula.equals(deriv.children.get(0).formula)) {
-      if (opts.verbose >= 1)
-        LogInfo.logs("TYPE RAISE [%s]: %s", deriv.rule, formula);
-      return;
-    }
-    // Define features based on the formula type
-    extractPhraseFragment(ex, deriv, phraseInfos, formula, opts.recursionLevels, "<R>");
-  }
-
-  private void extractPhraseFragment(Example ex, Derivation deriv, List<PhraseInfo> phraseInfos, Formula formula,
-      int level, String fragment, Value... placeholders) {
-    if (level < 0)
-      throw new RuntimeException("Level = " + level + " < 0");
-    // Look at the formula type
-    int N = placeholders.length;
-    if (formula instanceof ValueFormula) {
-      Value value = ((ValueFormula<?>) formula).value;
-      if (value instanceof NumberValue) {
-        if (!opts.forbidSingletonPredicateType || !"<R>".equals(fragment))
-          extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "(number *)"), placeholders);
-        extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "#" + N), C(placeholders, value));
-
-      } else if (value instanceof DateValue) {
-        if (!opts.forbidSingletonPredicateType || !"<R>".equals(fragment))
-          extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "(date *)"), placeholders);
-        extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "#" + N), C(placeholders, value));
-
-      } else if (value instanceof NameValue) {
-        if (!opts.forbidSingletonPredicateType || !"<R>".equals(fragment))
-          extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, getType((NameValue) value)), placeholders);
-        extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "#" + N), C(placeholders, value));
-
-      }
-
-    } else if (formula instanceof JoinFormula) {
-      JoinFormula join = (JoinFormula) formula;
-      Formula relation = join.relation;
-      if (relation instanceof ReverseFormula && ((ReverseFormula) relation).child instanceof ValueFormula) {
-        Value relationValue = ((ValueFormula<?>) ((ReverseFormula) relation).child).value;
-        extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "(!#" + N + " *)"), C(placeholders, relationValue));
-        if (level > 0)
-          extractPhraseFragment(ex, deriv, phraseInfos, join.child, level - 1, R(fragment, "(!#" + N + " <R>)"), C(placeholders, relationValue));
-
-      } else if (relation instanceof ValueFormula) {
-        Value relationValue = ((ValueFormula<?>) relation).value;
-        extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "(#" + N + " *)"), C(placeholders, relationValue));
-        if (level > 0)
-          extractPhraseFragment(ex, deriv, phraseInfos, join.child, level - 1, R(fragment, "(#" + N + " <R>)"), C(placeholders, relationValue));
-
-      } else {
-        throw new RuntimeException("[Phrase-Fragment] Unrecognized JoinFormula: " + formula);
-      }
-
-    } else if (formula instanceof MergeFormula) {
-      MergeFormula merge = (MergeFormula) formula;
-      extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "(" + merge.mode + " * *)"), placeholders);
-      if (level > 0) {
-        extractPhraseFragment(ex, deriv, phraseInfos, merge.child1, level - 1, R(fragment, "(" + merge.mode + " <R> *)"), placeholders);
-        extractPhraseFragment(ex, deriv, phraseInfos, merge.child2, level - 1, R(fragment, "(" + merge.mode + " <R> *)"), placeholders);
-      }
-
-    } else if (formula instanceof AggregateFormula) {
-      AggregateFormula aggregate = (AggregateFormula) formula;
-      extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "(" + aggregate.mode + " *)"), placeholders);
-      if (level > 0)
-        extractPhraseFragment(ex, deriv, phraseInfos, aggregate.child, level - 1, R(fragment, "(" + aggregate.mode + " <R>)"), placeholders);
-
-    } else if (formula instanceof SuperlativeFormula) {
-      SuperlativeFormula superlative = (SuperlativeFormula) formula;
-      extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "(" + superlative.mode + " * *)"), placeholders);
-      if (level > 0) {
-        extractPhraseFragment(ex, deriv, phraseInfos, superlative.head, level - 1, R(fragment, "(" + superlative.mode + " <R> *)"), placeholders);
-        if (superlative.relation instanceof ReverseFormula && ((ReverseFormula) superlative.relation).child instanceof LambdaFormula) {
-          // (argmax 1 1 * (reverse (lambda (var x) ...)))
-          Formula lambdaBody = ((LambdaFormula) ((ReverseFormula) superlative.relation).child).body;
-          extractPhraseFragment(ex, deriv, phraseInfos, lambdaBody, level - 1, R(fragment, "(" + superlative.mode + " * R<R>)"), placeholders);
-        } else if (superlative.relation instanceof LambdaFormula) {
-          // (argmax 1 1 * (lambda (var x) ...))
-          Formula lambdaBody = ((LambdaFormula) superlative.relation).body;
-          extractPhraseFragment(ex, deriv, phraseInfos, lambdaBody, level - 1, R(fragment, "(" + superlative.mode + " * L<R>)"), placeholders);
-        } else {
-          // (argmax 1 1 * ...), such as (argmax 1 1 * @index)
-          extractPhraseFragment(ex, deriv, phraseInfos, superlative.relation, level - 1, R(fragment, "(" + superlative.mode + " * <R>)"), placeholders);
-        }
-      }
-
-    } else if (formula instanceof ArithmeticFormula) {
-      ArithmeticFormula arithmetic = (ArithmeticFormula) formula;
-      extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "(" + arithmetic.mode + " * *)"), placeholders);
-      if (level > 0) {
-        extractPhraseFragment(ex, deriv, phraseInfos, arithmetic.child1, level - 1, R(fragment, "(" + arithmetic.mode + " <R> *)"), placeholders);
-        extractPhraseFragment(ex, deriv, phraseInfos, arithmetic.child2, level - 1, R(fragment, "(" + arithmetic.mode + " <R> *)"), placeholders);
-      }
-
-    } else if (formula instanceof VariableFormula) {
-      extractPhraseFragment(ex, deriv, phraseInfos, R(fragment, "%x"), placeholders);
-
-    } else if (formula instanceof ScopedFormula) {
-      ScopedFormula scoped = (ScopedFormula) formula;
-      if (level > 0) {
-        extractPhraseFragment(ex, deriv, phraseInfos, scoped.head, level - 1, R(fragment, "(scoped <R> *)"), placeholders);
-        if (scoped.relation instanceof LambdaFormula) {
-          Formula lambdaBody = ((LambdaFormula) scoped.relation).body;
-          extractPhraseFragment(ex, deriv, phraseInfos, lambdaBody, level - 1, R(fragment, "(scoped * L<R>)"), placeholders);
-        } else {
-          extractPhraseFragment(ex, deriv, phraseInfos, scoped.relation, level - 1, R(fragment, "(scoped * <R>)"), placeholders);
-        }
-      }
-
-    } else if (formula instanceof MarkFormula) {
-      // TODO: Handle mark formula
-    } else if (formula instanceof ReverseFormula) {
-      throw new RuntimeException("[Phrase-Fragment] ReverseFormula not handled: " + formula);
-    } else if (formula instanceof LambdaFormula) {
-      throw new RuntimeException("[Phrase-Fragment] LambdaFormula not handled: " + formula);
-    } else {
-      throw new RuntimeException("[Phrase-Fragment] Cannot handle formula " + formula);
-    }
-  }
-
-  private String R(String fragment, String recursedFragment) {
-    return fragment.replace("<R>", recursedFragment);
-  }
-
-  private Value[] C(Value[] oldValues, Value newValue) {
-    Value[] combined = new Value[oldValues.length + 1];
-    for (int i = 0; i < oldValues.length; i++) combined[i] = oldValues[i];
-    combined[oldValues.length] = newValue;
-    return combined;
-  }
-
-  private static final Pattern UNARY_PATTERN = Pattern.compile("^fb:([^.]*)\\.[^.]*$");
-  private static final Pattern BINARY_PATTERN = Pattern.compile("^fb:([^.]*\\.[^.]*)\\.[^.]*$");
-
-  private String getType(NameValue name) {
-    String id = name.id;
-    if (CanonicalNames.COMPARATORS.contains(id)) return "%COMP";
-    if (CanonicalNames.COLON.equals(id)) return "%COLON";
-    if (CanonicalNames.isUnary(id)) {
-      Matcher matcher = UNARY_PATTERN.matcher(id);
-      if (matcher.matches()) return "%" + matcher.group(1);
-    } else if (CanonicalNames.isBinary(id)) {
-      Matcher matcher = BINARY_PATTERN.matcher(id);
-      if (matcher.matches()) return "%" + matcher.group(1);
-    }
-    throw new RuntimeException("[getType] Unhandled NameValue: " + name);
-  }
-
-  private void extractPhraseFragment(Example ex, Derivation deriv, List<PhraseInfo> phraseInfos, String fragment, Value... placeholders) {
-    if (opts.verbose >= 1)
-      LogInfo.logs(">>> %s %s", fragment, Arrays.asList(placeholders));
-    String[] placeholderStrings = new String[placeholders.length],
-        originalStrings = new String[placeholders.length];
-    for (int i = 0; i < placeholders.length; i++) {
-      Value placeholder = placeholders[i];
-      if (placeholder instanceof NameValue) {
-        placeholderStrings[i] = ((NameValue) placeholder).id.replace("fb:", "");
-        originalStrings[i] = PredicateInfo.getOriginalString(((NameValue) placeholder).id, ex);
-      } else if (placeholder instanceof NumberValue) {
-        originalStrings[i] = placeholderStrings[i] = "" + ((NumberValue) placeholder).value;
-      } else if (placeholder instanceof DateValue) {
-        originalStrings[i] = placeholderStrings[i] = ((DateValue) placeholder).isoString();
-      } else {
-        throw new RuntimeException("[placeholder] Unhandled Value: " + placeholder);
-      }
-    }
-    String defaultFragment = fragment;
-    for (int i = 0; i < placeholderStrings.length; i++) {
-      defaultFragment = defaultFragment.replace("#" + i, placeholderStrings[i]);
-    }
-    String[] placeholderStringsUnlex = new String[placeholders.length];
-    for (PhraseInfo phraseInfo : phraseInfos) {
-      if (opts.lexicalizedPhraseFragment)
-        deriv.addFeature("p-f", phraseInfo.lemmaText + ";" + defaultFragment);
-      if (opts.unlexicalizedPhraseFragment) {
-        boolean matched = false;
-        for (int i = 0; i < placeholders.length; i++) {
-          Value placeholder = placeholders[i];
-          if (placeholder instanceof NameValue) {
-            if (originalStrings[i] != null && originalStrings[i].equals(phraseInfo.lemmaText)) {
-              placeholderStringsUnlex[i] = "$";
-              matched = true;
-            } else {
-              placeholderStringsUnlex[i] = placeholderStrings[i];
-            }
-          } else if (placeholder instanceof NumberValue || placeholder instanceof DateValue) {
-            if (originalStrings[i] != null && originalStrings[i].equals(phraseInfo.normalizedNerSpan)) {
-              placeholderStringsUnlex[i] = "$";
-              matched = true;
-            } else {
-              placeholderStringsUnlex[i] = placeholderStrings[i];
-            }
-          } else {
-            throw new RuntimeException("[placeholder] Unhandled Value: " + placeholder);
-          }
-        }
-        if (matched) {
-          String unlexFragment = fragment;
-          for (int i = 0; i < placeholderStrings.length; i++) {
-            unlexFragment = unlexFragment.replace("#" + i, placeholderStringsUnlex[i]);
-          }
-          deriv.addFeature("p-fu", unlexFragment);
-        }
-      }
-    }
-  }
-}

src/edu/stanford/nlp/sempre/tables/features/PhraseInfo.java

@@ -2,6 +2,7 @@ package edu.stanford.nlp.sempre.tables.features;
 
 import java.util.*;
 import java.util.concurrent.ExecutionException;
+import java.util.regex.Pattern;
 
 import com.google.common.cache.*;
 
@@ -23,6 +24,8 @@ public class PhraseInfo {
     public int maxPhraseLength = 3;
     @Option(gloss = "Fuzzy match predicates")
     public boolean computeFuzzyMatchPredicates = false;
+    @Option(gloss = "Do not produce lexicalized features if the phrase begins or ends with a stop word")
+    public boolean forbidBorderStopWordInLexicalizedFeatures = true;
   }
   public static Options opts = new Options();
 
@@ -33,9 +36,9 @@ public class PhraseInfo {
   public final List<String> lemmaTokens;
   public final List<String> posTags;
   public final List<String> nerTags;
-  public final String normalizedNerSpan;
   public final String canonicalPosSeq;
   public final List<String> fuzzyMatchedPredicates;
+  public final boolean isBorderStopWord;     // true if the first or last word is a stop word
 
   public PhraseInfo(Example ex, int start, int end) {
     this.start = start;
@@ -46,14 +49,11 @@ public class PhraseInfo {
     lemmaTokens = languageInfo.tokens.subList(start, end);
     posTags = languageInfo.posTags.subList(start, end);
     nerTags = languageInfo.nerTags.subList(start, end);
-    if (nerTags.get(0) == null)
-      normalizedNerSpan = null;
-    else
-      normalizedNerSpan = languageInfo.getNormalizedNerSpan(nerTags.get(0), start, end);
     text = languageInfo.phrase(start, end).toLowerCase();
     lemmaText = languageInfo.lemmaPhrase(start, end).toLowerCase();
     canonicalPosSeq = languageInfo.canonicalPosSeq(start, end);
     fuzzyMatchedPredicates = opts.computeFuzzyMatchPredicates ? getFuzzyMatchedPredicates(ex.context) : null;
+    isBorderStopWord = isStopWord(languageInfo.lemmaTokens.get(start)) || isStopWord(languageInfo.lemmaTokens.get(end - 1));
   }
 
   private List<String> getFuzzyMatchedPredicates(ContextValue context) {
@@ -76,6 +76,17 @@ public class PhraseInfo {
     return matchedPredicates;
   }
 
+  static final Pattern ALL_PUNCT = Pattern.compile("^[^A-Za-z0-9]*$");
+  static final Set<String> STOP_WORDS = new HashSet<>(Arrays.asList(
+      "a", "an", "the", "be", "of", "in", "on", "do"
+      ));
+
+  static boolean isStopWord(String x) {
+    if (ALL_PUNCT.matcher(x).matches()) return true;
+    if (STOP_WORDS.contains(x)) return true;
+    return false;
+  }
+
   @Override
   public String toString() {
     return "\"" + text + "\"";

src/edu/stanford/nlp/sempre/tables/features/PhrasePredicateFeatureComputer.java

@@ -35,6 +35,11 @@ public class PhrasePredicateFeatureComputer implements FeatureComputer {
     public boolean lexicalizedPhrasePredicate = true;
     @Option(gloss = "Maximum ngram length for lexicalize all pair features")
     public int maxNforLexicalizeAllPairs = Integer.MAX_VALUE;
+    @Option(gloss = "phrase-category: Weight threshold")
+    public double phraseCategoryWeightThreshold = 0.8;
+    @Option(gloss = "phrase-category: Use binary features instead of continuous ones")
+    public boolean phraseCategoryBinary = true;
+
   }
   public static Options opts = new Options();
 
@@ -47,7 +52,8 @@ public class PhrasePredicateFeatureComputer implements FeatureComputer {
   @Override
   public void extractLocal(Example ex, Derivation deriv) {
     if (!(FeatureExtractor.containsDomain("phrase-predicate")
-        || FeatureExtractor.containsDomain("missing-predicate"))) return;
+        || FeatureExtractor.containsDomain("phrase-formula")
+        || FeatureExtractor.containsDomain("phrase-category"))) return;
     // Only compute features at the root, except when the partial option is set.
     if (!opts.defineOnPartialDerivs && !deriv.isRoot(ex.numTokens())) return;
     List<PhraseInfo> phraseInfos = PhraseInfo.getPhraseInfos(ex);
@@ -58,7 +64,8 @@ public class PhrasePredicateFeatureComputer implements FeatureComputer {
       LogInfo.logs("Derivation: %s", deriv);
       LogInfo.logs("Predicates: %s", predicateInfos);
     }
-    if (FeatureExtractor.containsDomain("phrase-predicate")) {
+    if (FeatureExtractor.containsDomain("phrase-predicate")
+        || FeatureExtractor.containsDomain("phrase-category")) {
       if (opts.defineOnPartialDerivs) {
         deriv.getTempState().put("p-p", new ArrayList<>(predicateInfos));
         // Subtract predicates from children
@@ -109,7 +116,7 @@ public class PhrasePredicateFeatureComputer implements FeatureComputer {
   private void extractMatch(Example ex, Derivation deriv,
       PhraseInfo phraseInfo, String phraseString, String phraseType,
       PredicateInfo predicateInfo, String predicateString, String predicateType, double factor) {
-    if (opts.unlexicalizedPhrasePredicate) {
+    if (FeatureExtractor.containsDomain("phrase-predicate") && opts.unlexicalizedPhrasePredicate) {
       if (phraseString.equals(predicateString)) {
         defineFeatures(ex, deriv, phraseInfo, predicateInfo, phraseType + "=" + predicateType,
             phraseString, predicateString, factor);
@@ -132,10 +139,27 @@ public class PhrasePredicateFeatureComputer implements FeatureComputer {
         }
       }
     }
-    if (opts.lexicalizedPhrasePredicate && phraseInfo.end - phraseInfo.start <= maxNforLexicalizeAllPairs) {
+    if (FeatureExtractor.containsDomain("phrase-predicate") && opts.lexicalizedPhrasePredicate
+        && phraseInfo.end - phraseInfo.start <= maxNforLexicalizeAllPairs
+        && (!PhraseInfo.opts.forbidBorderStopWordInLexicalizedFeatures || !phraseInfo.isBorderStopWord)) {
       deriv.addFeature("p-p",
           phraseType + phraseString + ";" + predicateType + predicateString, factor);
     }
+    if (FeatureExtractor.containsDomain("phrase-category") && predicateInfo.type == PredicateType.BINARY
+        && (!PhraseInfo.opts.forbidBorderStopWordInLexicalizedFeatures || !phraseInfo.isBorderStopWord)) {
+      ColumnCategoryInfo catInfo = ColumnCategoryInfo.getSingleton();
+      List<Pair<String, Double>> categories = catInfo.get(ex, predicateInfo.predicate);
+      if (categories != null) {
+        for (Pair<String, Double> pair : categories) {
+          if (pair.getSecond() >= opts.phraseCategoryWeightThreshold) {
+            if (opts.phraseCategoryBinary)
+              deriv.addFeature("p-c", phraseType + phraseString + ";" + pair.getFirst());
+            else
+              deriv.addFeature("p-c", phraseType + phraseString + ";" + pair.getFirst(), pair.getSecond());
+          }
+        }
+      }
+    }
   }
 
   private void defineFeatures(Example ex, Derivation deriv, PhraseInfo phraseInfo, PredicateInfo predicateInfo,

src/edu/stanford/nlp/sempre/tables/features/WordEmbeddingFeatureComputer.java

@@ -1,103 +0,0 @@
-package edu.stanford.nlp.sempre.tables.features;
-
-import java.io.*;
-import java.util.*;
-
-import edu.stanford.nlp.sempre.*;
-import fig.basic.*;
-
-/**
- * Compute vector space paraphrase features as in the paraphrase paper (Berant and Liang, 2014).
- *
- * The features are computed on all (phrase n-gram, predicate n-gram) pairs.
- *
- * @author ppasupat
- */
-public class WordEmbeddingFeatureComputer implements FeatureComputer {
-  public static class Options {
-    @Option(gloss = "Verbosity")
-    public int verbosity = 0;
-    @Option(gloss = "Path to file containing word vectors, one per line")
-    public String wordVectorFile;
-    @Option(gloss = "Vector dimension")
-    public int vecCapacity = 50;
-  }
-  public static Options opts = new Options();
-
-  private Map<String, double[]> wordVectors;
-
-  public WordEmbeddingFeatureComputer() {
-    wordVectors = new HashMap<>();
-    LogInfo.begin_track("Loading word embedding files ...");
-    try (BufferedReader reader = IOUtils.openInHard(opts.wordVectorFile)) {
-      String header = null, line;
-      while ((line = reader.readLine()) != null) {
-        String[] tokens = line.split("\\s+");
-        // Some word embedding files have a header which includes the number of
-        // words and the number of dimensions.  Ignore this.
-        if (header == null && tokens.length == 2) {
-          header = line;
-          continue;
-        }
-        if (tokens.length - 1 != opts.vecCapacity)
-          throw new RuntimeException("Expected " + opts.vecCapacity + " tokens, but got " + (tokens.length - 1) + ": " + line);
-        double[] vector = new double[opts.vecCapacity];
-        for (int i = 1; i < tokens.length; ++i)
-          vector[i - 1] = Double.parseDouble(tokens[i]);
-        wordVectors.put(tokens[0], vector);
-      }
-    } catch (IOException e) {
-      e.printStackTrace();
-      LogInfo.fail(e);
-    }
-    LogInfo.logs("%d words loaded (%d dimensions)", wordVectors.size(), opts.vecCapacity);
-    LogInfo.end_track();
-  }
-
-  @Override
-  public void extractLocal(Example ex, Derivation deriv) {
-    if (!(FeatureExtractor.containsDomain("word-embedding"))) return;
-    if (!deriv.isRoot(ex.numTokens())) return;
-    // Utterance
-    double[] utteranceVector = new double[opts.vecCapacity];
-    int numUtteranceTokens = 0;
-    for (String token : ex.languageInfo.tokens) {
-      double[] tokenVector = wordVectors.get(token);
-      if (tokenVector == null) {
-        if (opts.verbosity >= 2) LogInfo.logs("Utterance word not found: %s", token);
-        continue;
-      }
-      ListUtils.addMut(utteranceVector, tokenVector);
-      numUtteranceTokens++;
-    }
-    if (numUtteranceTokens == 0) return;
-    ListUtils.multMut(utteranceVector, 1.0 / numUtteranceTokens);
-    // Predicates
-    double[] predicateVector = new double[opts.vecCapacity];
-    int numPredicateTokens = 0;
-    for (PredicateInfo predicateInfo : PredicateInfo.getPredicateInfos(ex, deriv)) {
-      if (predicateInfo.originalString == null) continue;
-      String s = predicateInfo.originalString.toLowerCase().replaceAll("[^a-z0-9]+", " ").trim();
-      if (opts.verbosity >= 2) LogInfo.logs("|%s| ==> |%s|", predicateInfo.originalString, s);
-      if (s.isEmpty()) continue;
-      for (String token : s.split(" ")) {
-        double[] tokenVector = wordVectors.get(token);
-        if (tokenVector == null) {
-          if (opts.verbosity >= 2) LogInfo.logs("Predicate word not found: %s", token);
-          continue;
-        }
-        ListUtils.addMut(predicateVector, tokenVector);
-        numPredicateTokens++;
-      }
-    }
-    if (numPredicateTokens == 0) return;
-    ListUtils.multMut(predicateVector, 1.0 / numPredicateTokens);
-    // Outer product
-    for (int i = 0; i < opts.vecCapacity; i++) {
-      for (int j = 0; j < opts.vecCapacity; j++) {
-        deriv.addFeature("w-e", "" + i + "," + j, utteranceVector[i] * predicateVector[j]);
-      }
-    }
-  }
-
-}

src/edu/stanford/nlp/sempre/tables/lambdadcs/DenotationUtils.java

@@ -238,6 +238,8 @@ public final class DenotationUtils {
    */
   public static UnaryDenotation superlativeUnary(int rank, int count, List<Pair<Value, Value>> pairs,
       SuperlativeFormula.Mode mode, TypeProcessor processor) {
+    if (rank <= 0 || count <= 0 || rank >= 1000000 || count >= 100000)
+      LogInfo.fails("Invalid superlative (rank = %d, count = %d)", rank, count);
     if (pairs.isEmpty()) {
       if (LambdaDCSExecutor.opts.superlativesFailOnEmptyLists)
         throw new LambdaDCSException(Type.emptyList, "Cannot call %s on an empty list.", mode);
@@ -343,7 +345,7 @@ public final class DenotationUtils {
     public boolean isCompatible(Value v) {
       return v instanceof NumberValue;
     }
-    
+
     @Override
     public boolean isSortable(Collection<Value> values) {
       return true;
@@ -387,19 +389,19 @@ public final class DenotationUtils {
     public boolean isCompatible(Value v) {
       return v instanceof DateValue;
     }
-    
+
     @Override
     public boolean isSortable(Collection<Value> values) {
       DateValue firstDate = null;
       for (Value value : values) {
-        DateValue date = (DateValue) value; 
+        DateValue date = (DateValue) value;
         if (firstDate == null) {
           firstDate = date;
         } else {
           if ((firstDate.year == -1) != (date.year == -1)) return false;
           if ((firstDate.month == -1) != (date.month == -1)) return false;
           if ((firstDate.day == -1) != (date.day == -1)) return false;
-        }  
+        }
       }
       return true;
     }
@@ -465,5 +467,5 @@ public final class DenotationUtils {
       throw new LambdaDCSException(Type.typeMismatch, "Cannot compare values");
     }
   }
- 
+
 }

src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutor.java

@@ -26,6 +26,8 @@ public class LambdaDCSExecutor extends Executor {
     public boolean executeBinary = false;
     @Option(gloss = "Generic DateValue: (date -1 5 -1) in formula also matches (date -1 5 12)")
     public boolean genericDateValue = false;
+    @Option(gloss = "If the result is empty, return an ErrorValue instead of an empty ListValue")
+    public boolean failOnEmptyLists = false;
     @Option(gloss = "Return all ties on (argmax 1 1 ...) and (argmin 1 1 ...)")
     public boolean superlativesReturnAllTopTies = true;
     @Option(gloss = "Aggregates (sum, avg, max, min) throw an error on empty lists")
@@ -148,6 +150,11 @@ class LambdaDCSCoreLogic {
           cache.put(graph, formula, denotation);
         }
         answer = denotation.toValue();
+        if (answer instanceof ListValue) {
+          answer = ((ListValue) answer).getUnique();
+          if (LambdaDCSExecutor.opts.failOnEmptyLists && ((ListValue) answer).values.isEmpty())
+            answer = ErrorValue.empty;
+        }
       } catch (LambdaDCSException e) {
         if (LambdaDCSExecutor.opts.executeBinary && e.type == Type.notUnary) {
           try {
@@ -285,6 +292,9 @@ class LambdaDCSCoreLogic {
           computeUnary(superlative.rank, typeHint.unrestrictedUnary()).range());
       int count = DenotationUtils.getSinglePositiveInteger(
           computeUnary(superlative.count, typeHint.unrestrictedUnary()).range());
+      if (rank != 1 || count != 1) {
+        LogInfo.logs("Superlative WTF: %s | rank %d | count %d", formula, rank, count);
+      }
       Unarylike headD = computeUnary(superlative.head, typeHint);
       Binarylike relationD;
       if (superlative.relation instanceof ReverseFormula) {

src/edu/stanford/nlp/sempre/tables/lambdadcs/LambdaDCSExecutorTest.java

@@ -150,11 +150,11 @@ public class LambdaDCSExecutorTest {
               "(fb:en.tampico fb:location.location.containedby fb:en.illinois)" +
           ")"));
     } else if ("csv".equals(name)) {
-      return TableKnowledgeGraph.fromFilename("tables/toy-examples/random/nikos_machlas.csv");
+      return TableKnowledgeGraph.fromFilename("tables/toy-examples/nikos_machlas.csv");
     } else if ("csv2".equals(name)) {
-      return TableKnowledgeGraph.fromFilename("lib/data/tables/csv/204-csv/495.tsv");
+      return TableKnowledgeGraph.fromFilename("tables/toy-examples/204-495.tsv");
     } else if ("csv3".equals(name)) {
-      return TableKnowledgeGraph.fromFilename("lib/data/tables/csv/203-csv/839.tsv");
+      return TableKnowledgeGraph.fromFilename("tables/toy-examples/203-839.tsv");
     }
     throw new RuntimeException("Unknown graph name: " + name);
   }
@@ -230,9 +230,9 @@ public class LambdaDCSExecutorTest {
   @Test(groups = "lambdaCSV") public void lambdaOnGraphCSVTest() {
     KnowledgeGraph graph = getKnowledgeGraph("csv");
     runFormula(executor, "(number 3)", graph, matches("(number 3)"));
-    runFormula(executor, "(!fb:row.row.score (fb:row.row.opponent fb:cell.austria))",
-        graph, matches("(name fb:cell.1_2)"));
-    runFormula(executor, "(count (fb:row.row.result fb:cell.win))",
+    runFormula(executor, "(!fb:row.row.score (fb:row.row.opponent fb:cell_opponent.austria))",
+        graph, matches("(name fb:cell_score.1_2)"));
+    runFormula(executor, "(count (fb:row.row.result fb:cell_result.win))",
         graph, matches("(number 16)"));
     // Depending on tie-breaking, one of these will be correct
     try {
@@ -247,12 +247,12 @@ public class LambdaDCSExecutorTest {
           graph, matches("(number 2)"));
     }
   }
-  
+
   @Test(groups = "lambdaCSV2") public void lambdaOnGraphCSV2Test() {
     KnowledgeGraph graph = getKnowledgeGraph("csv2");
     runFormula(executor,
-        "(and (!= (and (!= fb:cell.away) fb:cell.home)) ((reverse fb:row.row.opponent) (fb:row.row.index (- (number 1) (number 1)))))",
-        graph, matches("(name fb:cell.derby_county)"));
+        "(and (!= (and (!= fb:cell_venue.away) fb:cell_venue.home)) ((reverse fb:row.row.opponent) (fb:row.row.index (- (number 2) (number 1)))))",
+        graph, matches("(name fb:cell_opponent.derby_county)"));
   }
 
   @Test(groups = "lambdaCSV3") public void lambdaOnGraphCSV3Test() {
@@ -265,11 +265,9 @@ public class LambdaDCSExecutorTest {
         "(count (fb:row.row.opened (fb:cell.cell.date (< (date 1926 -1 -1)))))",
         graph, matches("(number 6)"));
     runFormula(executor,
-        "(sum (- (count ((reverse fb:row.row.index) (fb:type.object.type fb:type.row))) " +
-            "((reverse fb:row.row.index) (fb:row.row.latitude ((reverse fb:row.row.longitude) (fb:type.object.type fb:type.row))))))",
-        graph, matches("(number 6)"));
-    runFormula(executor,
-        "(- (number 1926) (argmax (number 1) (number 1) ((reverse fb:cell.cell.number) (or (or (or fb:cell.1920 fb:cell.1925) fb:cell.1926) fb:cell.1946)) (reverse (lambda x (sum ((reverse fb:cell.cell.number) (fb:cell.cell.number (var x))))))))",
+        "(- (number 1926) (argmax (number 1) (number 1) ((reverse fb:cell.cell.number) "
+        + "(or (or (or fb:cell_closed.1920 fb:cell_closed.1925) fb:cell_opened.1926) fb:cell_closed.1946)) "
+        + "(reverse (lambda x (sum ((reverse fb:cell.cell.number) (fb:cell.cell.number (var x))))))))",
         graph, matches("(number 6)"));
   }
 }

src/edu/stanford/nlp/sempre/tables/match/ClosedClassFn.java

@@ -87,15 +87,16 @@ public class ClosedClassFn extends SemanticFn {
           .formula(formula).type(type).createDerivation();
     }
 
-    protected final String NULL = TableTypeSystem.getCellName("null");
-
     protected Collection<Formula> createGenericFormulas() {
       List<Formula> formulas = new ArrayList<>();
       // Find out if the table has a null cell
-      for (TableCellProperties properties : graph.cellProperties) {
-        if (NULL.equals(properties.id)) {
-          formulas.add(new ValueFormula<>(properties.nameValue));
-          break;
+      for (TableColumn column : graph.columns) {
+        for (TableCell cell : column.children) {
+          if (cell.properties.id.endsWith(".null")) {
+            formulas.add(new ValueFormula<>(cell.properties.nameValue));
+            break;
+            
+          }
         }
       }
       if (ClosedClassFn.opts.verbose >= 2) {
@@ -114,7 +115,7 @@ public class ClosedClassFn extends SemanticFn {
         boolean hasRepeats = false;
         Set<Value> values = new HashSet<>();
         for (TableCell cell : column.children) {
-          if (NULL.equals(cell.properties.id)) continue;
+          if (cell.properties.id.endsWith(".null")) continue;
           if (values.contains(cell.properties.nameValue))
             hasRepeats = true;
           else

src/edu/stanford/nlp/sempre/tables/match/OriginalMatcher.java

@@ -0,0 +1,131 @@
+package edu.stanford.nlp.sempre.tables.match;
+
+import java.util.*;
+
+import edu.stanford.nlp.sempre.*;
+import edu.stanford.nlp.sempre.FuzzyMatchFn.FuzzyMatchFnMode;
+import edu.stanford.nlp.sempre.tables.StringNormalizationUtils;
+import edu.stanford.nlp.sempre.tables.TableCell;
+import edu.stanford.nlp.sempre.tables.TableColumn;
+import edu.stanford.nlp.sempre.tables.TableKnowledgeGraph;
+import edu.stanford.nlp.sempre.tables.TableTypeSystem;
+import fig.basic.MapUtils;
+import fig.basic.Option;
+import fig.basic.Pair;
+
+/**
+ * Original matcher used in ACL 2015. Only does exact matches.
+ *
+ * @author ppasupat
+ */
+public class OriginalMatcher extends FuzzyMatcher {
+  public static class Options {
+    @Option(gloss = "Do not fuzzy match if the query matches more than this number of formulas (prevent overgeneration)")
+    public int maxMatchedCandidates = Integer.MAX_VALUE;
+  }
+  public static Options opts = new Options();
+
+  public OriginalMatcher(TableKnowledgeGraph graph) {
+    super(graph);
+    precomputeForMatching();
+  }
+
+  private static Collection<String> getAllCollapsedForms(String original) {
+    Set<String> collapsedForms = new HashSet<>();
+    collapsedForms.add(StringNormalizationUtils.collapseNormalize(original));
+    String normalized = StringNormalizationUtils.aggressiveNormalize(original);
+    collapsedForms.add(StringNormalizationUtils.collapseNormalize(normalized));
+    collapsedForms.remove("");
+    return collapsedForms;
+  }
+
+  private static String getCanonicalCollapsedForm(String original) {
+    return StringNormalizationUtils.collapseNormalize(original);
+  }
+
+  // Map normalized strings to Values
+  // ENTITIY --> ValueFormula fb:cell.___ or other primitive format
+  //   UNARY --> JoinFormula (type fb:column.___)
+  //  BINARY --> ValueFormula fb:row.row.___
+  Set<Formula> allEntityFormulas, allUnaryFormulas, allBinaryFormulas;
+  Map<String, Set<Formula>> phraseToEntityFormulas, phraseToUnaryFormulas, phraseToBinaryFormulas;
+
+  protected void precomputeForMatching() {
+    allEntityFormulas = new HashSet<>();
+    allUnaryFormulas = new HashSet<>();
+    allBinaryFormulas = new HashSet<>();
+    phraseToEntityFormulas = new HashMap<>();
+    phraseToUnaryFormulas = new HashMap<>();
+    phraseToBinaryFormulas = new HashMap<>();
+    for (TableColumn column : graph.columns) {
+      // unary and binary
+      Formula unary = new JoinFormula(
+          new ValueFormula<>(CanonicalNames.reverseProperty(column.relationNameValue)),
+          new JoinFormula(new ValueFormula<>(new NameValue(CanonicalNames.TYPE)),
+                          new ValueFormula<>(new NameValue(TableTypeSystem.ROW_TYPE)))
+      );
+      Formula binary = new ValueFormula<>(column.relationNameValue);
+      allUnaryFormulas.add(unary);
+      allBinaryFormulas.add(binary);
+      for (String s : getAllCollapsedForms(column.originalString)) {
+        MapUtils.addToSet(phraseToUnaryFormulas, s, unary);
+        MapUtils.addToSet(phraseToBinaryFormulas, s, binary);
+      }
+      // entity
+      for (TableCell cell : column.children) {
+        Formula entity = new ValueFormula<>(cell.properties.nameValue);
+        allEntityFormulas.add(entity);
+        for (String s : getAllCollapsedForms(cell.properties.originalString))
+          MapUtils.addToSet(phraseToEntityFormulas, s, entity);
+      }
+    }
+  }
+  
+  // ============================================================
+  // Internal methods
+  // ============================================================
+  
+  Map<Pair<String, FuzzyMatchFnMode>, FuzzyMatchCache> cacheMap = new HashMap<>();
+
+  @Override
+  protected FuzzyMatchCache cacheSentence(List<String> sentence, FuzzyMatchFnMode mode) {
+    String joined = String.join(" ", sentence);
+    FuzzyMatchCache cache = cacheMap.get(new Pair<>(joined, mode));
+    if (cache != null) return cache;
+    // Compute a new FuzzyMatchCache
+    cache = new FuzzyMatchCache();
+    for (int i = 0; i < sentence.size(); i++) {
+      for (int j = i + 1; j < sentence.size(); j++) {
+        String term = String.join(" ", sentence.subList(i, j));
+        cache.addAll(i, j, getFuzzyMatchedFormulasInternal(term, mode));
+      }
+    }
+    cacheMap.put(new Pair<>(joined, mode), cache);
+    return cache;
+  }
+
+  @Override
+  protected Collection<Formula> getFuzzyMatchedFormulasInternal(String term,
+      FuzzyMatchFnMode mode) {
+    String normalized = getCanonicalCollapsedForm(term);
+    Set<Formula> answer;
+    switch (mode) {
+      case ENTITY: answer = phraseToEntityFormulas.get(normalized); break;
+      case UNARY:  answer = phraseToUnaryFormulas.get(normalized);  break;
+      case BINARY: answer = phraseToBinaryFormulas.get(normalized); break;
+      default: throw new RuntimeException("Unknown FuzzyMatchMode " + mode);
+    }
+    return (answer == null || answer.size() > opts.maxMatchedCandidates) ? Collections.emptySet() : answer;
+  }
+
+  @Override
+  protected Collection<Formula> getAllFormulasInternal(FuzzyMatchFnMode mode) {
+    switch (mode) {
+      case ENTITY: return allEntityFormulas;
+      case UNARY:  return allUnaryFormulas;
+      case BINARY: return allBinaryFormulas;
+      default: throw new RuntimeException("Unknown FuzzyMatchMode " + mode);
+    }
+  }
+
+}

src/edu/stanford/nlp/sempre/tables/serialize/SerializedDataset.java

@@ -110,7 +110,7 @@ public class SerializedDataset extends Dataset {
   // ============================================================
 
   public static void main(String[] args) {
-    TableKnowledgeGraph.opts.baseCSVDir = "lib/data/tables/";
+    TableKnowledgeGraph.opts.baseCSVDir = "lib/data/WikiTableQuestions/";
     SerializedDataset dataset = new SerializedDataset();
     dataset.readDir("out/sliced-dump-8-reps/representative-00-training-sliced-00000-00299/");
     LazyLoadedExampleList examples = dataset.examples("representative");

src/edu/stanford/nlp/sempre/tables/serialize/TaggedDatasetGenerator.java

@@ -100,7 +100,7 @@ public class TaggedDatasetGenerator extends TSVGenerator implements Runnable {
       }
     }
     // Other information come from Example
-    fields[2] = serialize(((TableKnowledgeGraph) ex.context.graph).filename.replace("lib/data/tables/", ""));
+    fields[2] = serialize(((TableKnowledgeGraph) ex.context.graph).filename.replace("lib/data/WikiTableQuestions/", ""));
     fields[4] = serialize(ex.languageInfo.tokens);
     fields[5] = serialize(ex.languageInfo.lemmaTokens);
     fields[6] = serialize(ex.languageInfo.posTags);

src/edu/stanford/nlp/sempre/tables/serialize/TaggedFuzzyGenerator.java

@@ -0,0 +1,109 @@
+package edu.stanford.nlp.sempre.tables.serialize;
+
+import java.util.*;
+
+import edu.stanford.nlp.sempre.*;
+import edu.stanford.nlp.sempre.FuzzyMatchFn.FuzzyMatchFnMode;
+import edu.stanford.nlp.sempre.SemanticFn.CallInfo;
+import edu.stanford.nlp.sempre.tables.TableKnowledgeGraph;
+import fig.basic.*;
+import fig.exec.Execution;
+
+/**
+ * Generate TSV files containing information about fuzzy matched objects.
+ *
+ * @author ppasupat
+ */
+public class TaggedFuzzyGenerator extends TSVGenerator implements Runnable {
+
+  public static void main(String[] args) {
+    Execution.run(args, "TaggedFuzzyGeneratorMain", new TaggedFuzzyGenerator(),
+        Master.getOptionsParser());
+  }
+
+  private Grammar grammar = new Grammar();
+  
+  @Override
+  public void run() {
+    // Read grammar
+    grammar.read(Grammar.opts.inPaths);
+    // Read dataset
+    LogInfo.begin_track("Dataset.read");
+    for (Pair<String, String> pathPair : Dataset.opts.inPaths) {
+      String group = pathPair.getFirst();
+      String path = pathPair.getSecond();
+      // Open output file
+      String filename = Execution.getFile("fuzzy-" + group + ".tsv");
+      out = IOUtils.openOutHard(filename);
+      dump(FIELDS);
+      // Read LispTrees
+      LogInfo.begin_track("Reading %s", path);
+      int maxExamples = Dataset.getMaxExamplesForGroup(group);
+      Iterator<LispTree> trees = LispTree.proto.parseFromFile(path);
+      // Go through the examples
+      int n = 0;
+      while (n < maxExamples) {
+        // Format: (example (id ...) (utterance ...) (targetFormula ...) (targetValue ...))
+        LispTree tree = trees.next();
+        if (tree == null) break;
+        if (tree.children.size() < 2 || !"example".equals(tree.child(0).value)) {
+          if ("metadata".equals(tree.child(0).value)) continue;
+          throw new RuntimeException("Invalid example: " + tree);
+        }
+        Example ex = Example.fromLispTree(tree, path + ":" + n);
+        ex.preprocess();
+        LogInfo.begin_track("Example %s (%d): %s => %s", ex.id, n, ex.getTokens(), ex.targetValue);
+        n++;
+        dumpExample(ex, tree);
+        LogInfo.end_track();
+      }
+      out.close();
+      LogInfo.logs("Finished dumping to %s", filename);
+      LogInfo.end_track();
+    }
+    LogInfo.end_track();
+  }
+
+  private static final String[] FIELDS = new String[] {
+    "id", "type", "start", "end", "phrase", "fragment"
+  };
+
+  @Override
+  protected void dump(String... stuff) {
+    assert stuff.length == FIELDS.length;
+    super.dump(stuff);
+  }
+  
+  private void dumpExample(Example ex, LispTree tree) {
+    int n = ex.numTokens();
+    for (int i = 0; i < n; i++) {
+      StringBuilder sb = new StringBuilder(ex.token(i));
+      for (int j = i; j < n; j++) {
+        String term = sb.toString();
+        Derivation deriv = 
+            new Derivation.Builder()
+                    .cat(Rule.phraseCat).start(i).end(j)
+                    .rule(Rule.nullRule)
+                    .children(Derivation.emptyList)
+                    .withStringFormulaFrom(term)
+                    .canonicalUtterance(term)
+                    .createDerivation();
+        List<Derivation> children = new ArrayList<>();
+        children.add(deriv);
+        // Get the derived derivations
+        for (Rule rule : grammar.getRules()) {
+          CallInfo c = new CallInfo(rule.lhs, i, j + 1, rule, children);
+          Iterator<Derivation> itr = rule.sem.call(ex, c);
+          while (itr.hasNext()) {
+            deriv = itr.next();
+            LogInfo.logs("Found %s %s -> %s", rule.lhs, term, deriv.formula);
+            dump(ex.id, rule.lhs.substring(1), "" + i, "" + (j + 1), term, deriv.formula.toString());
+          }
+        }
+        if (j + 1 < n)
+          sb.append(" ").append(ex.token(j + 1));
+      }
+    }
+  }
+  
+}

src/edu/stanford/nlp/sempre/tables/test/BatchTableExecutor.java

@@ -0,0 +1,136 @@
+package edu.stanford.nlp.sempre.tables.test;
+
+import java.io.*;
+import java.util.*;
+
+import edu.stanford.nlp.sempre.*;
+import edu.stanford.nlp.sempre.tables.TableKnowledgeGraph;
+import edu.stanford.nlp.sempre.tables.TableValueEvaluator;
+import edu.stanford.nlp.sempre.tables.lambdadcs.LambdaDCSExecutor;
+import fig.basic.*;
+import fig.exec.Execution;
+
+/**
+ * Execute the specified logical forms on the specified WikiTableQuestions context.
+ *
+ * @author ppasupat
+ */
+public class BatchTableExecutor implements Runnable {
+  public static class Options {
+    @Option(gloss = "TSV file containing table contexts and logical forms")
+    public String batchInput;
+    @Option(gloss = "Datasets for mapping example IDs to contexts")
+    public List<String> batchDatasets = Arrays.asList("lib/data/tables/data/training.examples");
+  }
+  public static Options opts = new Options();
+
+  public static void main(String[] args) {
+    Execution.run(args, "BatchTableExecutorMain", new BatchTableExecutor(), Master.getOptionsParser());
+  }
+
+  @Override
+  public void run() {
+    if (opts.batchInput == null || opts.batchInput.isEmpty()) {
+      LogInfo.logs("*******************************************************************************");
+      LogInfo.logs("USAGE: ./run @mode=tables @class=execute -batchInput <filename>");
+      LogInfo.logs("");
+      LogInfo.logs("Input file format: Each line has something like");
+      LogInfo.logs("  nt-218    [tab]   (count (fb:type.object.type fb:type.row))");
+      LogInfo.logs("or");
+      LogInfo.logs("  csv/204-csv/23.csv    [tab]   (count (fb:type.object.type fb:type.row))");
+      LogInfo.logs("");
+      LogInfo.logs("Results will also be printed to state/execs/___.exec/denotations.tsv");
+      LogInfo.logs("Output format:");
+      LogInfo.logs("  nt-218    [tab]   (count (fb:type.object.type fb:type.row))   [tab]   (list (number 10))   [tab]   false");
+      LogInfo.logs("where the last column indicates whether the answer is consistent with the target answer");
+      LogInfo.logs("(only available when the first column is nt-___)");
+      LogInfo.logs("*******************************************************************************");
+      System.exit(1);
+    }
+    LambdaDCSExecutor executor = new LambdaDCSExecutor();
+    ValueEvaluator evaluator = new TableValueEvaluator();
+    try {
+      BufferedReader reader = IOUtils.openIn(opts.batchInput);
+      PrintWriter output = IOUtils.openOut(Execution.getFile("denotations.tsv"));
+      String line;
+      while ((line = reader.readLine()) != null) {
+        String[] tokens = line.split("\t");
+        String answer;
+        try {
+          Formula formula = Formula.fromString(tokens[1]);
+          if (tokens[0].startsWith("csv")) {
+            TableKnowledgeGraph graph = TableKnowledgeGraph.fromFilename(tokens[0]);
+            ContextValue context = new ContextValue(graph);
+            Value denotation = executor.execute(formula, context).value;
+            if (denotation instanceof ListValue)
+              denotation = addOriginalStrings((ListValue) denotation, graph);
+            answer = denotation.toString();
+          } else {
+            Example ex = exIdToExample(tokens[0]);
+            Value denotation = executor.execute(formula, ex.context).value;
+            if (denotation instanceof ListValue)
+              denotation = addOriginalStrings((ListValue) denotation, (TableKnowledgeGraph) ex.context.graph);
+            answer = denotation.toString();
+            boolean correct = evaluator.getCompatibility(ex.targetValue, denotation) == 1.;
+            answer = denotation.toString() + "\t" + correct;
+          }
+        } catch (Exception e) {
+          answer = "ERROR: " + e;
+        }
+        System.out.printf("%s\t%s\t%s\n", tokens[0], tokens[1], answer);
+        output.printf("%s\t%s\t%s\n", tokens[0], tokens[1], answer);
+      }
+      reader.close();
+      output.close();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private Map<String, Object> exIdToExampleMap;
+
+  private Example exIdToExample(String exId) {
+    if (exIdToExampleMap == null) {
+      exIdToExampleMap = new HashMap<>();
+      try {
+        for (String filename : opts.batchDatasets) {
+          BufferedReader reader = IOUtils.openIn(filename);
+          String line;
+          while ((line = reader.readLine()) != null) {
+            LispTree tree = LispTree.proto.parseFromString(line);
+            if (!"id".equals(tree.child(1).child(0).value))
+              throw new RuntimeException("Malformed example: " + line);
+            exIdToExampleMap.put(tree.child(1).child(1).value, tree);
+          }
+        }
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    Object obj = exIdToExampleMap.get(exId);
+    if (obj == null) return null;
+    Example ex;
+    if (obj instanceof LispTree) {
+      ex = Example.fromLispTree((LispTree) obj, exId);
+      ex.preprocess();
+      exIdToExampleMap.put(exId, ex);
+    } else {
+      ex = (Example) obj;
+    }
+    return ex;
+  }
+
+  ListValue addOriginalStrings(ListValue answers, TableKnowledgeGraph graph) {
+    List<Value> values = new ArrayList<>();
+    for (Value value : answers.values) {
+      if (value instanceof NameValue) {
+        NameValue name = (NameValue) value;
+        if (name.description == null)
+          value = new NameValue(name.id, graph.getOriginalString(((NameValue) value).id));
+      }
+      values.add(value);
+    }
+    return new ListValue(values);
+  }
+
+}

src/edu/stanford/nlp/sempre/tables/test/CustomExample.java

@@ -108,10 +108,6 @@ public class CustomExample extends Example {
     formulaMacros.put("@!p.date", "!" + TableTypeSystem.CELL_DATE_VALUE.id);
     formulaMacros.put("@p.num2", TableTypeSystem.CELL_NUM2_VALUE.id);
     formulaMacros.put("@!p.num2", "!" + TableTypeSystem.CELL_NUM2_VALUE.id);
-    formulaMacros.put("@p.str1", TableTypeSystem.CELL_STR1_VALUE.id);
-    formulaMacros.put("@!p.str1", "!" + TableTypeSystem.CELL_STR1_VALUE.id);
-    formulaMacros.put("@p.str2", TableTypeSystem.CELL_STR2_VALUE.id);
-    formulaMacros.put("@!p.str2", "!" + TableTypeSystem.CELL_STR2_VALUE.id);
     formulaMacros.put("@p.part", TableTypeSystem.CELL_PART_VALUE.id);
     formulaMacros.put("@!p.part", "!" + TableTypeSystem.CELL_PART_VALUE.id);
   }
@@ -144,9 +140,9 @@ public class CustomExample extends Example {
       if ((match = regexReversedProperty.matcher(value)).matches())
         return LispTree.proto.newLeaf("!" + TableTypeSystem.getRowPropertyName(match.group(1)));
       if ((match = regexEntity.matcher(value)).matches())
-        return LispTree.proto.newLeaf(TableTypeSystem.getCellName(match.group(1)));
+        return LispTree.proto.newLeaf(TableTypeSystem.CELL_NAME_PREFIX + "." + match.group(1));
       if ((match = regexPart.matcher(value)).matches())
-        return LispTree.proto.newLeaf(TableTypeSystem.getPartName(match.group(1)));
+        return LispTree.proto.newLeaf(TableTypeSystem.PART_NAME_PREFIX + "." + match.group(1));
       if (value.contains(".") && !(value.startsWith("fb:") || value.startsWith("!fb:")))
         throw new RuntimeException("Unhandled '.': " + value);
       return orig;

src/edu/stanford/nlp/sempre/tables/test/Sandbox.java

@@ -1,8 +0,0 @@
-package edu.stanford.nlp.sempre.tables.test;
-
-public class Sandbox {
-
-  public static void main(String[] args) {
-  }
-
-}

src/edu/stanford/nlp/sempre/tables/test/TableColumnAnalyzer.java

@@ -0,0 +1,237 @@
+package edu.stanford.nlp.sempre.tables.test;
+
+import java.io.*;
+import java.util.*;
+import java.util.regex.*;
+
+import edu.stanford.nlp.sempre.*;
+import edu.stanford.nlp.sempre.tables.*;
+import fig.basic.*;
+import fig.exec.Execution;
+
+/**
+ * Analyze table columns and print out any hard-to-process column.
+ *
+ * @author ppasupat
+ */
+public class TableColumnAnalyzer implements Runnable {
+  public static class Options {
+    @Option(gloss = "Maximum number of tables to process (for debugging)")
+    public int maxNumTables = Integer.MAX_VALUE;
+    @Option(gloss = "Load Wikipedia article titles from this file")
+    public String wikiTitles = null;
+  }
+  public static Options opts = new Options();
+
+  public static void main(String[] args) {
+    Execution.run(args, "TableColumnAnalyzerMain", new TableColumnAnalyzer(), Master.getOptionsParser());
+  }
+
+  PrintWriter out;
+  PrintWriter outCompact;
+
+  @Override
+  public void run() {
+    out = IOUtils.openOutHard(Execution.getFile("column-stats.tsv"));
+    outCompact = IOUtils.openOutHard(Execution.getFile("column-compact.tsv"));
+    Map<String, List<String>> tableIdToExIds = getTableIds();
+    int tablesProcessed = 0;
+    for (Map.Entry<String, List<String>> entry : tableIdToExIds.entrySet()) {
+      Execution.putOutput("example", tablesProcessed);
+      String tableId = entry.getKey(),
+          tableIdAbbrev = tableId.replaceAll("csv/(\\d+)-csv/(\\d+)\\.csv", "$1-$2");
+      LogInfo.begin_track("Processing %s ...", tableId);
+      TableKnowledgeGraph graph = TableKnowledgeGraph.fromFilename(tableId);
+      out.printf("%s\tIDS\t%s\n", tableIdAbbrev, String.join(" ", entry.getValue()));
+      out.printf("%s\tCOLUMNS\t%d\n", tableIdAbbrev, graph.numColumns());
+      for (int i = 0; i < graph.numColumns(); i++) {
+        analyzeColumn(graph, graph.columns.get(i), tableIdAbbrev + "\t" + i);
+      }
+      LogInfo.end_track();
+      if (tablesProcessed++ >= opts.maxNumTables) break;
+    }
+    out.close();
+    outCompact.close();
+  }
+
+  protected Map<String, List<String>> getTableIds() {
+    Map<String, List<String>> tableIdToExIds = new LinkedHashMap<>();
+    LogInfo.begin_track_printAll("Collect table IDs");
+    for (Pair<String, String> pathPair : Dataset.opts.inPaths) {
+      String group = pathPair.getFirst();
+      String path = pathPair.getSecond();
+      Execution.putOutput("group", group);
+      LogInfo.begin_track("Reading %s", path);
+      Iterator<LispTree> trees = LispTree.proto.parseFromFile(path);
+      while (trees.hasNext()) {
+        LispTree tree = trees.next();
+        if ("metadata".equals(tree.child(0).value)) continue;
+        String exId = null, tableId = null;
+        for (int i = 1; i < tree.children.size(); i++) {
+          LispTree arg = tree.child(i);
+          String label = arg.child(0).value;
+          if ("id".equals(label)) {
+            exId = arg.child(1).value;
+          } else if ("context".equals(label)) {
+            tableId = arg.child(1).child(2).value;
+          }
+        }
+        if (exId != null && tableId != null) {
+          List<String> exIdsForTable = tableIdToExIds.get(tableId);
+          if (exIdsForTable == null)
+            tableIdToExIds.put(tableId, exIdsForTable = new ArrayList<>());
+          exIdsForTable.add(exId);
+        }
+      }
+      LogInfo.end_track();
+    }
+    LogInfo.end_track();
+    LogInfo.logs("Got %d IDs", tableIdToExIds.size());
+    return tableIdToExIds;
+  }
+
+  protected void analyzeColumn(TableKnowledgeGraph graph, TableColumn column, String printPrefix) {
+    List<String> escapedCells = new ArrayList<>();
+    // Print the header
+    String h = column.originalString, escapedH = StringNormalizationUtils.escapeTSV(h);
+    out.printf("%s\t0\t%s\n", printPrefix, escapedH);
+    escapedCells.add(escapedH);
+    // Print the cells
+    Map<String, Integer> typeCounts = new HashMap<>();
+    for (int j = 0; j < column.children.size(); j++) {
+      TableCell cell = column.children.get(j);
+      String c = cell.properties.originalString, escapedC = StringNormalizationUtils.escapeTSV(c);
+      escapedCells.add(escapedC);
+      // Infer the type
+      List<String> types = analyzeCell(c);
+      for (String type : types)
+        MapUtils.incr(typeCounts, type);
+      out.printf("%s\t%d\t%s\t%s\n", printPrefix, j + 1, String.join("|", types), escapedC);
+    }
+    // Analyze the common types
+    List<String> commonTypes = new ArrayList<>();
+    for (Map.Entry<String, Integer> entry : typeCounts.entrySet()) {
+      if (entry.getValue() == column.children.size()) {
+        commonTypes.add(entry.getKey());
+      } else if (entry.getValue() == column.children.size() - 1) {
+        commonTypes.add("ALMOST-" + entry.getKey());
+      }
+    }
+    outCompact.printf("%s\t%s\t%s\n", String.join("|", commonTypes), printPrefix, String.join("\t", escapedCells));
+  }
+
+  // ============================================================
+  // Cell analysis
+  // ============================================================
+
+  public static final Pattern ORDINAL = Pattern.compile("^(\\d+)(st|nd|rd|th)$");
+
+  protected List<String> analyzeCell(String c) {
+    List<String> types = new ArrayList<>();
+    LanguageInfo languageInfo = LanguageAnalyzer.getSingleton().analyze(c);
+    {
+      // Integer
+      NumberValue n = StringNormalizationUtils.parseNumberStrict(c);
+      if (n != null) {
+        // Number
+        types.add("num");
+        // Integer
+        double value = n.value;
+        if (Math.abs(value - Math.round(value)) < 1e-9) {
+          types.add("int");
+          if (c.matches("^[12]\\d\\d\\d$")) {
+            // Year?
+            types.add("year");
+          }
+        }
+      }
+    }
+    {
+      // Ordinal
+      Matcher m = ORDINAL.matcher(c);
+      if (m.matches()) {
+        types.add("ordinal");
+      }
+    }
+    {
+      // Integer-Integer
+      String[] splitted = StringNormalizationUtils.STRICT_DASH.split(c);
+      if (splitted.length == 2 && splitted[0].matches("^[0-9]+$") && splitted[1].matches("^[0-9]+$")) {
+        types.add("2ints");
+      }
+    }
+    {
+      // Date
+      DateValue date = StringNormalizationUtils.parseDateWithLanguageAnalyzer(languageInfo);
+      if (date != null) {
+        types.add("date");
+        // Also more detailed date type
+        types.add("date-"
+            + (date.year != -1  ? "Y" : "")
+            + (date.month != -1 ? "M" : "")
+            + (date.day != -1   ? "D" : ""));
+      }
+    }
+    {
+      // Quoted text
+      if (c.matches("^[“”\"].*[“”\"]$")) {
+        types.add("quoted");
+      }
+    }
+    if (opts.wikiTitles != null) {
+      // Wikipedia titles
+      WikipediaTitleLibrary library = WikipediaTitleLibrary.getSingleton();
+      if (library.contains(c)) {
+        types.add("wiki");
+      }
+    }
+    {
+      // POS and NER
+      types.add("POS=" + String.join("-", languageInfo.posTags));
+      types.add("NER=" + String.join("-", languageInfo.nerTags));
+    }
+    return types;
+  }
+
+  // ============================================================
+  // Helper class: Wikipedia titles
+  // ============================================================
+
+  public static class WikipediaTitleLibrary {
+
+    private static WikipediaTitleLibrary _singleton = null;
+
+    public static WikipediaTitleLibrary getSingleton() {
+      if (_singleton == null)
+        _singleton = new WikipediaTitleLibrary();
+      return _singleton;
+    }
+
+    Set<String> titles = new HashSet<>();
+
+    private WikipediaTitleLibrary() {
+      assert opts.wikiTitles != null;
+      LogInfo.begin_track("Reading Wikipedia article titles from %s ...", opts.wikiTitles);
+      try {
+        BufferedReader reader = IOUtils.openIn(opts.wikiTitles);
+        String line;
+        while ((line = reader.readLine()) != null) {
+          titles.add(line);
+          if (titles.size() <= 10) {
+            LogInfo.logs("Example title: %s", line);
+          }
+        }
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+      LogInfo.logs("Read %d titles", titles.size());
+      LogInfo.end_track();
+    }
+
+    public boolean contains(String c) {
+      return titles.contains(c.toLowerCase().trim());
+    }
+  }
+
+
+}

src/edu/stanford/nlp/sempre/tables/test/TableStatsComputer.java

@@ -88,7 +88,7 @@ public class TableStatsComputer implements Runnable {
         outputFields.add("number");
         // (and (@type @cell) (@p.num ___))
         Formula formula = new MergeFormula(Mode.and,
-            new JoinFormula(Formula.fromString(CanonicalNames.TYPE), Formula.fromString(TableTypeSystem.CELL_TYPE)),
+            new JoinFormula(Formula.fromString(CanonicalNames.TYPE), Formula.fromString(TableTypeSystem.CELL_GENERIC_TYPE)),
             new JoinFormula(Formula.fromString(TableTypeSystem.CELL_NUMBER_VALUE.id), new ValueFormula<Value>(value)));
         Value result = builder.executor.execute(formula, ex.context).value;
         inTable = result instanceof ListValue && !((ListValue) result).values.isEmpty();
@@ -97,7 +97,7 @@ public class TableStatsComputer implements Runnable {
         outputFields.add("date");
         // (and (@type @cell) (@p.num ___))
         Formula formula = new MergeFormula(Mode.and,
-            new JoinFormula(Formula.fromString(CanonicalNames.TYPE), Formula.fromString(TableTypeSystem.CELL_TYPE)),
+            new JoinFormula(Formula.fromString(CanonicalNames.TYPE), Formula.fromString(TableTypeSystem.CELL_GENERIC_TYPE)),
             new JoinFormula(Formula.fromString(TableTypeSystem.CELL_DATE_VALUE.id), new ValueFormula<Value>(value)));
         Value result = builder.executor.execute(formula, ex.context).value;
         inTable = result instanceof ListValue && !((ListValue) result).values.isEmpty();

src/edu/stanford/nlp/sempre/test/ParserTest.java

@@ -107,21 +107,13 @@ public class ParserTest {
     // TODO(chaganty): test more thoroughly
   }
 
-  @Test public void checkFloatingNumDerivations() {
-    // Make it behave like the BeamParser
-    FloatingParser.opts.defaultIsFloating = false;
-    ParseTest p;
-    p = ABCTest();
-    p.test(new FloatingParser(p.getParserSpec()));
-    p = ArithmeticTest();
-    p.test(new FloatingParser(p.getParserSpec()));
-
-    // If floating, should get more hypotheses
+  @Test(groups = "floating") public void checkFloatingNumDerivations() {
     FloatingParser.opts.defaultIsFloating = true;
+    FloatingParser.opts.useSizeInsteadOfDepth = true;
     Parser parser = new FloatingParser(ABCTest().getParserSpec());
     FloatingParser.opts.maxDepth = 2;
     checkNumDerivations(parser, "ignore", null, 3);
-    FloatingParser.opts.maxDepth = 3;
+    FloatingParser.opts.maxDepth = 4;
     checkNumDerivations(parser, "ignore", null, 3 + 3 * 3);
   }
 
@@ -143,12 +135,35 @@ public class ParserTest {
   @Test void checkRankingSimple() {
     checkRankingArithmetic(new BeamParser(ArithmeticTest().getParserSpec()));
   }
-  @Test void checkRankingReinforcement() {
+  @Test(groups = "reinforcement") void checkRankingReinforcement() {
     checkRankingArithmetic(new ReinforcementParser(ArithmeticTest().getParserSpec()));
   }
-  @Test void checkRankingFloating() {
-    FloatingParser.opts.defaultIsFloating = false;
-    checkRankingArithmetic(new FloatingParser(ArithmeticTest().getParserSpec()));
+
+  @Test(groups = "floating") public void checkRankingFloating() {
+    FloatingParser.opts.defaultIsFloating = true;
+    FloatingParser.opts.maxDepth = 4;
+    FloatingParser.opts.useAnchorsOnce = true;
+    Parser parser = new FloatingParser(new ParseTest(TestUtils.makeArithmeticFloatingGrammar()) {
+      @Override public void test(Parser parser) {}
+    }.getParserSpec());
+    Params params = new Params();
+    Map<String, Double> features = new HashMap<>();
+    features.put("rule :: $Operator -> nothing (ConstantFn (lambda y (lambda x (call + (var x) (var y)))))", 1.0);
+    features.put("rule :: $Operator -> nothing (ConstantFn (lambda y (lambda x (call * (var x) (var y)))))", -1.0);
+    params.update(features);
+    /*
+     * Expected LFs:
+     *    2          3
+     *    2 + 3      3 + 2
+     *    2 * 3      3 * 2
+     */
+    checkNumDerivations(parser, params, "2 and 3", "(number 5)", 6);
+
+    params = new Params();
+    features.put("rule :: $Operator -> nothing (ConstantFn (lambda y (lambda x (call + (var x) (var y)))))", -1.0);
+    features.put("rule :: $Operator -> nothing (ConstantFn (lambda y (lambda x (call * (var x) (var y)))))", 1.0);
+    params.update(features);
+    checkNumDerivations(parser, params, "2 and 3", "(number 6)", 6);
   }
 
   // TODO(chaganty): verify the parser gradients

src/edu/stanford/nlp/sempre/test/TestUtils.java

@@ -33,6 +33,16 @@ public final class TestUtils {
     return g;
   }
 
+  public static Grammar makeArithmeticFloatingGrammar() {
+    Grammar g = new Grammar();
+    g.addStatement("(rule $Expr ($TOKEN) (NumberFn) (anchored 1))");
+    g.addStatement("(rule $Expr ($Expr $Partial) (JoinFn backward))");
+    g.addStatement("(rule $Partial ($Operator $Expr) (JoinFn forward))");
+    g.addStatement("(rule $Operator (nothing) (ConstantFn (lambda y (lambda x (call + (var x) (var y))))))");
+    g.addStatement("(rule $Operator (nothing) (ConstantFn (lambda y (lambda x (call * (var x) (var y))))))");
+    g.addStatement("(rule $ROOT ($Expr) (IdentityFn))");
+    return g;
+  }
 
   public static Grammar makeNumberConcatGrammar() {
     Grammar g = new Grammar();

tables/README.md

@@ -21,26 +21,67 @@ Running the code
         ./pull-dependencies tables
         ./pull-dependencies tables-data
 
-  The dataset lives in `lib/data/tables/`
+  The dataset lives in `lib/data/WikiTableQuestions/`
 
 2. Compile the source:
 
         ant tables
 
-  This will produce JAR files in the `libsempre` directory as usual.
+  This will produce JAR files in the `libsempre` directory.
 
 3. The following command train and test on 100 development examples:
 
-        ./run @mode=tables @data=u-1 @feat=all @train=1 -maxex train,100 dev,100
+        ./run @mode=tables @data=u-1 @feat=all @train=1 -maxExamples train:100 dev:100
 
-  The command should take less than an hour.
+  The command should take less than 30 minutes.
 
-  * To train on the complete development set, remove `-maxex train,100 dev,100`
+  * To train on the complete development set, remove `-maxExamples train:100 dev:100`
 
   * The command above uses `u-1` (80:20 split of the development data).
   Other available sets include `u-2`, ..., `u-5` (four other development splits)
   and `test` (actual train-test split).
 
+Other options
+-------------
+
+### Macro Grammar (Experimental)
+
+Macro grammar can be used to significantly speed up the parser.
+To turn on macro grammar, run the following:
+
+    ./run @mode=tables @data=u-1 @feat=more @parser=cprune @grammar=extended @fuzzy=editdist-fuzzy @train=1
+
+Please refer to the following paper for more information:
+
+> Yuchen Zhang, Panupong Pasupat, Percy Liang.  
+> Macro Grammars and Holistic Triggering for Efficient Semantic Parsing  
+> Empirical Methods on Natural Language Processing (EMNLP), 2017.
+
+Currently the module does not support model saving, and testing has to be done on the official test set.
+These features will be added in the future.
+
+Official evaluation
+-------------------
+
+The official evaluation script in the WikiTableQuestions dataset is slightly
+more lenient than the SEMPRE one (`tables.TableValueEvaluator`).
+In particular, the SEMPRE evaluator enforces that the type of the predicted
+denotation must match the correct answer type, while the official one allows
+type conversion.
+
+To get the official number of a trained model, run
+
+    ./pull-dependencies tables-cprune
+    ./run @mode=tables @data=u-1 @feat=all @train=0 -Derivation.showValues -Builder.inParamsPath path/to/params
+
+(Change the `@data` other options to match the ones used during training.)
+
+This should produce an execution directory (in `state/execs/` by default)
+with a log file in it. Then run
+
+    ./tables/log-parsers/get-predictions.py path/to/log > predictions
+    ./lib/data/WikiTableQuestions/evaluator.py -t ./lib/data/WikiTableQuestions/tagged-data predictions
+
 Other usages
 ------------
 
@@ -62,3 +103,44 @@ The table can also be viewed in pretty-printed format by calling
 or
 
     ./tables/view csv/204-csv/590.csv
+
+Generating `.tagged` Files
+--------------------------
+
+The [WikiTableQuestions](https://github.com/ppasupat/WikiTableQuestions)
+dataset contains CoreNLP tagged files.
+Here is how they were generated.
+
+The classes responsible for generating `.tagged` files are `TaggedDatasetGenerator`
+and `TaggedTableGenerator` in `edu.stanford.nlp.sempre.tables.serialize`.
+
+Suppose the LispTree `.examples` files is at `my-data.examples`, and the
+CSV table files all have paths of the form
+`table-base-directory/csv/___-csv/___.csv` where `___` are numbers.
+(The path pattern is currently hard-coded.)
+
+Please see the WikiTableQuestions dataset package for how the LispTree
+`.examples` file should be formatted (these files are stored in `data/`),
+and how the CSV table files are stored (these are stored in `csv/`; only
+.csv files are needed; .tsv, .html, and .table are not needed).
+
+For dataset `.tagged` files:
+
+```
+./run @mode=tables @class=tag-data \
+    -dataset.inpaths anygroupname:my-data.examples \
+    -baseCSVDir table-base-directory @useTaggedFile=0
+```
+
+This will invoke `TaggedDatasetGenerator`.
+The results should be in `states/exec/___.exec/___.tagged`.
+
+For table `.tagged` files:
+
+```
+./run @mode=tables @class=tag-table \
+    -baseCSVDir table-base-directory @useTaggedFile=0
+```
+
+This will invoke `TaggedTableGenerator`.
+The results should be in `states/exec/___.exec/tagged/...`

tables/dump-parsers/convert-to-postfix.py

@@ -85,25 +85,23 @@ def convert(tree, args):
                     recurse(subtree[1])
                     answer.append(opr)
                     if not args.implicit_join:
-                        answer.append('.')
+                        answer.append('join')
         else:   # Join with a complex construct
             assert len(subtree) == 2, str(subtree)
-            if args.implicit_join:
-                # Only allows ((reverse ...) ...)
-                assert subtree[0][0] == 'reverse', str(subtree)
-                assert len(subtree[0]) == 2, str(subtree)
-                recurse(subtree[1])
-                answer.append('!' + subtree[0][1])
-            else:
-                recurse(subtree[1])
-                recurse(subtree[0])
-                answer.append('.')
+            # Only allows ((reverse ...) ...)
+            assert subtree[0][0] == 'reverse', str(subtree)
+            assert len(subtree[0]) == 2, str(subtree)
+            recurse(subtree[1])
+            answer.append('!' + subtree[0][1])
+            if not args.implicit_join:
+                answer.append('join')
     recurse(tree)
     return answer
 
 def process(line, args):
-    line = line[:-1].split('\t')
-    print '\t'.join([' '.join(convert(lisptree.parse(line[0]), args))] + line[1:])
+    line = line.rstrip('\n').split('\t')
+    line[args.field] = ' '.join(convert(lisptree.parse(line[args.field]), args))
+    print '\t'.join(line)
 
 def main():
     parser = argparse.ArgumentParser()
@@ -113,6 +111,8 @@ def main():
             help='Do not output "lambda reverse" for superlatives')
     parser.add_argument('-t', '--collapse-type-row', action='store_true',
             help='Collapse "(type row)" into a single token')
+    parser.add_argument('-f', '--field', type=int, default=0,
+            help='Field index (tab-separated; 0-based) containing the logical form')
     parser.add_argument('infile')
     args = parser.parse_args()
 

tables/dump-parsers/convert-to-prefix.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Convert LispTree formulas to Prefix encoding.
+
+Assume implicit join.
+Skip formulas that cannot be converted.
+
+If the file is tab-separated, only process the first column.
+"""
+
+import sys, os, shutil, re, argparse, json, gzip
+from codecs import open
+from itertools import izip
+from collections import defaultdict
+
+import fileinput
+import lisptree
+
+NUMBER_PREFIX = 'N'
+DATE_PREFIX = 'D'
+TYPE_ROW = 'type-row'
+AGGREGATE = ['count', 'min', 'max', 'sum', 'avg']
+MERGE_ARITH = ['and', 'or', '-']
+SUPERLATIVE = ['argmin', 'argmax']
+
+def convert(tree, args):
+    answer = []
+    def recurse(subtree):
+        if isinstance(subtree, basestring):
+            answer.append(subtree)
+            return
+        if isinstance(subtree[0], basestring):
+            opr = subtree[0]
+            if opr in AGGREGATE:
+                assert len(subtree) == 2, str(subtree)
+                answer.append(opr)
+                recurse(subtree[1])
+            elif opr in MERGE_ARITH:
+                assert len(subtree) == 3, str(subtree)
+                if opr == '-':
+                    opr = 'diff'
+                answer.append(opr)
+                recurse(subtree[1])
+                recurse(subtree[2])
+            elif opr in SUPERLATIVE:
+                assert len(subtree) in (3, 5), str(subtree)
+                if len(subtree) == 3:
+                    u, b = subtree[1], subtree[2]
+                else:
+                    u, b = subtree[3], subtree[4]
+                assert b[0] == 'reverse'
+                assert b[1][0] == 'lambda'
+                answer.append(opr)
+                recurse(u)
+                recurse(b[1][2])
+            elif opr == 'lambda':
+                assert False, 'Cannot convert lambda'
+                #assert len(subtree) == 3, str(subtree)
+                #answer.append(opr)
+                #recurse(subtree[2])
+            elif opr == 'reverse':
+                assert False, 'Cannot convert reverse'
+                #assert len(subtree) == 2, str(subtree)
+                #answer.append(opr)
+                #recurse(subtree[1])
+            elif opr == 'var':
+                assert len(subtree) == 2, str(subtree)
+                answer.append(subtree[1])
+            elif opr == 'number':
+                assert len(subtree) == 2, str(subtree)
+                answer.append(NUMBER_PREFIX + subtree[1])
+            elif opr == 'date':
+                assert len(subtree) == 4, str(subtree)
+                answer.append(DATE_PREFIX + '-'.join(
+                    'XX' if x == '-1' else x for x in subtree[1:4]))
+            else:    # Join with a name
+                assert len(subtree) == 2, str(subtree)
+                if ('fb:type.object.type' == opr and
+                        'fb:type.row' == subtree[1]):
+                    answer.append(TYPE_ROW)
+                else:
+                    answer.append(opr)
+                    recurse(subtree[1])
+        else:   # Join with a complex construct
+            assert len(subtree) == 2, str(subtree)
+            # Only allows ((reverse ...) ...)
+            assert subtree[0][0] == 'reverse', str(subtree)
+            assert len(subtree[0]) == 2, str(subtree)
+            answer.append('!' + subtree[0][1])
+            recurse(subtree[1])
+    recurse(tree)
+    return answer
+
+def prenormalize(lf, args):
+    lf = lf[:]
+    # Mark all normalization relations
+    for i in xrange(len(lf)):
+        if lf[i].startswith('fb:cell.cell.'):
+            assert lf[i-1].startswith('fb:row.row.')
+            lf[i-1] = 'fb:row.{}.{}'.format(
+                    lf[i][len('fb:cell.cell.'):],
+                    lf[i-1][len('fb:row.row.'):])
+            lf[i] = ''
+        elif lf[i].startswith('!fb:cell.cell.'):
+            assert lf[i+1].startswith('!fb:row.row.')
+            lf[i+1] = '!fb:row.{}.{}'.format(
+                    lf[i][len('!fb:cell.cell.'):],
+                    lf[i+1][len('!fb:row.row.'):])
+            lf[i] = ''
+    return [x for x in lf if x]
+
+def process(line, args):
+    line = line.rstrip('\n').split('\t')
+    lf = lisptree.parse(line[args.field])
+    lf = convert(lf, args)
+    lf = prenormalize(lf, args)
+    line[args.field] = ' '.join(lf)
+    print '\t'.join(line)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--field', type=int, default=0,
+            help='Field index (tab-separated; 0-based) containing the logical form')
+    parser.add_argument('infile')
+    args = parser.parse_args()
+
+    if args.infile != '-':
+        opener = gzip.open if args.infile.endswith('.gz') else open 
+        with opener(args.infile, 'r', 'utf8') as fin:
+            for line in fin:
+                process(line, args)
+    else:
+        for line in sys.stdin:
+            try:
+                process(line, args)
+            except Exception as e:
+                print >> sys.stderr, 'ERROR:', line.rstrip()
+
+if __name__ == '__main__':
+    main()
+

tables/grammars/extended.grammar

@@ -0,0 +1,169 @@
+# (Extended) Generic Grammar
+# Use more generic compositional patterns.
+
+################################################################
+# Macros
+
+(def @R reverse)
+(def @type fb:type.object.type)
+(def @row fb:type.row)
+
+(def @next fb:row.row.next)
+(def @!next !fb:row.row.next)
+(def @index fb:row.row.index)
+(def @!index !fb:row.row.index)
+
+(def @p.num fb:cell.cell.number)
+(def @!p.num !fb:cell.cell.number)
+(def @p.date fb:cell.cell.date)
+(def @!p.date !fb:cell.cell.date)
+(def @p.second fb:cell.cell.second) 
+(def @!p.second !fb:cell.cell.second)
+
+################################################################
+# Lexicon
+
+################################
+# Anchored Rules: Entity, Unary, Binary
+(rule $Entity ($PHRASE) (FuzzyMatchFn entity) (anchored 1))
+#(rule $Binary ($PHRASE) (FuzzyMatchFn binary) (anchored 1))
+(rule $Entity ($PHRASE) (NumberFn) (anchored 1))
+(rule $Entity ($PHRASE) (DateFn) (anchored 1))
+
+################################
+# Create binary from thin air
+(rule $Binary (nothing) (FuzzyMatchFn any binary))
+(rule $Unary (nothing) (FuzzyMatchFn any unary))
+
+################################
+# Property
+(for @property (@p.num @p.date)
+  (rule $Property (nothing) (ConstantFn @property))
+)
+(when second
+  (rule $Property (nothing) (ConstantFn @p.second))
+)
+
+################################
+# Generic RowSet
+(rule $RowSet (nothing) (ConstantFn (@type @row)))
+
+################################
+# Anchored ValueSet
+(rule $ValueSet ($Entity) (IdentityFn))
+
+# [TAG] alternative: "X or Y" questions
+(when alternative
+  (rule $ValueSet ($Entity $Entity)
+    (lambda e1 (lambda e2 (or (var e1) (var e2))))
+  )
+)
+
+################################
+# Join
+
+(rule $RowSet ($Binary $ValueSet) (lambda b (lambda v ((var b) (var v)))))
+
+(rule $ValueSet ($Binary $RowSet) (lambda b (lambda r ((@R (var b)) (var r)))))
+
+(rule $RowSet ($Binary $Property $ValueSet)
+  (lambda b (lambda p (lambda v ((var b) ((var p) (var v))))))
+)
+
+(rule $ValueSet ($Binary $Property $RowSet)
+  (lambda b (lambda p (lambda r ((@R (var p)) ((@R (var b)) (var r))))))
+)
+
+# [TAG] movement: "next" / "previous"
+(when movement
+  (for @movement (@next @!next)
+    (rule $RowSet ($RowSet) (lambda r (@movement (var r))))
+  )
+)
+
+# [TAG] comparison: "at least" / "more than"
+(when comparison
+  (for @comparison (< > <= >=)
+    (rule $RowSet ($Binary $Property $Entity)
+      (lambda b (lambda p (lambda e ((var b) ((var p) (@comparison (var e)))))))
+    )
+  )
+)
+
+# [TAG] != : "not zero" / "same"
+(when neq
+  (rule $RowSet ($Binary $Entity) (lambda b (lambda e ((var b) (!= (var e))))))
+  (rule $RowSet ($Binary $Property $Entity)
+    (lambda b (lambda p (lambda e ((var b) ((var p) (!= (var e)))))))
+  )
+)
+
+################################
+# Aggregate
+
+(when count
+  (rule $SingleValue ($RowSet) (lambda r (count (var r))))
+)
+
+(when aggregate
+  (rule $SingleValue ($ValueSet) (lambda r (min (var r))))
+  (rule $SingleValue ($ValueSet) (lambda r (max (var r))))
+  (rule $SingleValue ($ValueSet) (lambda r (sum (var r))))
+  (rule $SingleValue ($ValueSet) (lambda r (avg (var r))))
+)
+
+################################
+# Superlative
+
+(rule $FnOnRow ($Binary $Property)
+  (lambda b (lambda p (lambda x ((@R (var p)) ((@R (var b)) (var x))))))
+)
+
+(when superlative
+  (for @argm (argmax argmin)
+    (rule $RowSet ($RowSet) (lambda r (@argm 1 1 (var r) @index)))
+    (rule $RowSet ($RowSet $FnOnRow) (lambda r (lambda f (@argm 1 1 (var r) (@R (var f))))))
+  )
+)
+
+################################
+# Merge
+
+(when merge
+  (rule $RowSet ($RowSet $RowSet)
+    (lambda r1 (lambda r2 (and (var r1) (var r2))))
+  )
+)
+
+################################
+# Arithmatic
+
+(rule $FnOnValue ($Binary $Binary $Property)
+  (lambda b1 (lambda b2 (lambda p (lambda x ((@R (var p)) ((@R (var b2)) ((var b1) (var x))))))))
+)
+(rule $FnOnValue ($Binary)
+  (lambda b (lambda x (count ((var b) (var x)))))
+)
+(rule $FnOnValue ($Binary $Property)
+  (lambda b (lambda p (lambda x (count ((var b) ((var p) (var x)))))))
+)
+
+(when arithmetic
+  (rule $SingleValue ($FnOnValue $Entity $Entity)
+    (lambda f (lambda e1 (lambda e2 (- ((var f) (var e1)) ((var f) (var e2))))))
+  )
+)
+
+################################
+# V-superlative
+
+(when v-superlative
+  (for @argm (argmax argmin)
+    (rule $ValueSet ($ValueSet $FnOnValue) (lambda v (lambda f (@argm 1 1 (var v) (reverse (var f))))))
+  )
+)
+
+################################
+# ROOT
+(rule $ROOT ($ValueSet) (IdentityFn))
+(rule $ROOT ($SingleValue) (IdentityFn))

tables/grammars/fuzzy-dump.grammar

@@ -0,0 +1,25 @@
+# For dumping fuzzy matched objects
+
+################################################################
+# Macros
+
+(def @R reverse)
+(def @type fb:type.object.type)
+(def @row fb:type.row)
+(def @numdate (union fb:type.number fb:type.datetime))
+
+(def @next fb:row.row.next)
+(def @index fb:row.row.index)
+(def @p.num fb:cell.cell.number)
+(def @p.date fb:cell.cell.date)
+(def @p.num2 fb:cell.cell.num2)
+(def @p.part fb:cell.cell.part)
+
+################################################################
+# Anchored Rules
+
+(rule $ENTITY ($PHRASE) (FuzzyMatchFn entity) (anchored 1))
+(rule $BINARY ($PHRASE) (FuzzyMatchFn binary) (anchored 1))
+(rule $NUMBER ($PHRASE) (NumberFn) (anchored 1))
+(rule $DATE ($PHRASE) (DateFn) (anchored 1))
+(rule $DATERANGE ($PHRASE) (DateRangeFn) (anchored 1))

tables/grammars/grow.grammar

@@ -52,9 +52,11 @@
 # Set of all rows
 (rule $Set (nothing) (ConstantFn (@type @row)))
 
-(when closedclass
+(when (or closedclass closedclass-generic)
   # Generate things like "null"
   (rule $Set (nothing) (tables.match.ClosedClassFn generic))
+)
+(when (or closedclass closedclass-column)
   # If a column contains only a few unique things, generate them
   (rule $Set (nothing) (tables.match.ClosedClassFn column))
 )

tables/log-parsers/get-predictions.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Get predictions from the log file of SEMPRE."""
+
+import sys, os, shutil, re, argparse
+
+PATTERN = re.compile(r'Pred@0000: '
+        r'\(derivation \(formula (.*)\)\) '
+        r'\(value (.*)\) '
+        r'\(type (.*)\)\) \[score=(.*), prob=(.*), comp=(.*)\]')
+
+def lisptree_to_python_object(charbuffer):
+    """Convert the lisptree to Python object.
+
+    Args:
+        charbuffer: REVERSED list of characters of the lisptree string.
+        Characters will be consumed from the list.
+    """
+    c = charbuffer.pop()
+    if c == '(':
+        answer = []
+        while charbuffer[-1] != ')':
+            if charbuffer[-1] == ' ':
+                charbuffer.pop()
+            else:
+                answer.append(lisptree_to_python_object(charbuffer))
+        assert charbuffer.pop() == ')'
+        return answer
+    elif c == '"':
+        answer = []
+        while charbuffer[-1] != '"':
+            c = charbuffer.pop()
+            if c == '\\':
+                answer.append(charbuffer.pop())
+            else:
+                answer.append(c)
+        assert charbuffer.pop() == '"'
+        return ''.join(answer)
+    else:
+        answer = [c if c != '\\' else charbuffer.pop()]
+        while charbuffer[-1] not in (' ', ')'):
+            c = charbuffer.pop()
+            if c == '\\':
+                answer.append(charbuffer.pop())
+            else:
+                assert c != '('
+                answer.append(c)
+        return ''.join(answer)
+
+def lisptree_to_values(tree):
+    assert tree.startswith('(list ') and tree.endswith(')')
+    tree = lisptree_to_python_object(list(tree.decode('utf8'))[::-1])
+    assert tree[0] == 'list'
+    answer = []
+    for subtree in tree[1:]:
+        if subtree[0] == 'number':
+            answer.append(float(subtree[1]))
+        elif subtree[0] == 'date':
+            answer.append('{}-{}-{}'.format(
+                int(subtree[1]) if subtree[1] != '-1' else 'xx',
+                int(subtree[2]) if subtree[2] != '-1' else 'xx',
+                int(subtree[3]) if subtree[3] != '-1' else 'xx'))
+        else:
+            assert subtree[0] == 'name'
+            answer.append(re.sub('\s+', ' ', subtree[2]).strip())
+    return '\t'.join(unicode(x) for x in answer)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('infile', help='log file')
+    parser.add_argument('iteration', help='iteration to extract')
+    args = parser.parse_args()
+
+    prefix = 'iter=%s:' % args.iteration
+    ex_id = None
+    with open(args.infile) as fin:
+        for line in fin:
+            line = line.strip()
+            if line.startswith(prefix):
+                if ex_id is not None:
+                    # No prediction for the previous example
+                    print ex_id
+                ex_id = line.split()[3]
+            elif ex_id is not None and line.startswith('Pred@0000:'):
+                match = PATTERN.match(line)
+                formula, denotation, deno_type, score, prob, comp = match.groups()
+                denotation = lisptree_to_values(denotation)
+                print u'{}\t{}'.format(ex_id, denotation)
+                ex_id = None
+    if ex_id is not None:
+        print '\t'.join([ex_id, 'None'])
+
+if __name__ == '__main__':
+    main()
+

tables/table-alter/checker-summarize.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys, os, shutil, re, argparse, json
+from codecs import open
+from itertools import izip
+from collections import defaultdict
+
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-o', '--outfile')
+    parser.add_argument('checker_filenames', nargs='+')    
+    args = parser.parse_args()
+
+    data = []
+
+    for filename in args.checker_filenames:
+        with open(filename) as fin:
+            for line in fin:
+                line = line.strip()
+                if not line:
+                    continue
+                ex_id, annotation, parser = line.split('\t')
+                data.append((ex_id, annotation, parser))
+
+    stats = {
+        'n':          len(data),
+        'good':       sum(x[1] == 'good' for x in data),
+        'incorrect':  sum(x[1] == 'incorrect' for x in data),
+        'no':         sum(x[1] == 'no' for x in data),
+        'good_yes':   sum(x[1] == 'good' and x[2] == 'yes' for x in data),
+        'good_reach': sum(x[1] == 'good' and x[2] == 'reach' for x in data),
+        'good_no':    sum(x[1] == 'good' and x[2] == 'no' for x in data),
+        }
+    # compute percentage
+    for key in list(stats):
+        stats[key + '_percent'] = stats[key] * 100. / stats['n']
+    if args.outfile:
+        with open(args.outfile, 'w') as fout:
+            print >> fout, json.dumps(stats)
+    else:
+        print json.dumps(stats)
+
+if __name__ == '__main__':
+    main()
+

tables/table-alter/fictitious-dump-lf.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Dump the best logical form for each example."""
+
+import sys, os, shutil, re, argparse, json, gzip
+from codecs import open
+from itertools import izip
+from collections import defaultdict
+
+################################################
+# Data storage
+
+CHECK_VALUE_TO_FLAG = {
+    '(name Ao)': 'Ao',  # Turkers agree on an answer; LF agrees
+    '(name Ax)': 'Ax',  # Turkers agree on an answer; LF disagrees
+    '(name Bo)': 'Bo',  # Turkers agree that there's no answer; LF agrees
+    '(name Bx)': 'Bx',  # Turkers agree that there's no answer; LF disagrees
+    '(name X)': 'X',   # Turkers cannot reach a consensus 
+    'null': '?',
+    }
+
+def read_check_denotations_file(filename):
+    """Return n, k, d, denotations, grid"""
+    with gzip.open(filename) as fin:
+        n, k, d = [int(x) for x in fin.readline().split()]
+        denotations = []
+        for i in xrange(d):
+            denotations.append(CHECK_VALUE_TO_FLAG[fin.readline().strip()])
+        denotations.append(None)
+        grid = []
+        for i in xrange(n):
+            row = [denotations[int(x)] for x in fin.readline()[:-1].split()]
+            grid.append(row)
+        denotations.pop()
+        return n, k, d, denotations, grid
+
+def clean(x):
+    """Remove original strings from NameValues."""
+    x = x.decode('utf8').replace(ur'\"', '')
+    x = re.sub(ur'\(name fb:(?:cell|part)\.([^ )]*)\)', r'(name \1)', x)
+    x = re.sub(ur'\(name fb:(?:cell|part)\.([^ )]*) [^")]*\)', r'(name \1)', x)
+    x = re.sub(ur'\(name fb:(?:cell|part)\.([^ )]*) "[^"]*"\)', r'(name \1)', x)
+    return x
+
+
+################################################
+# Experiments
+
+# Scheme is a function that takes in a check vector (on turked tables)
+#   and returns either True (Turked result matches the EC)
+#   or False (Turked result filters out the EC).
+SCHEMES = {
+    # Here are the different schemes
+    # [e] LF must agree with everything the Turkers agree on
+    #     (including agreeing upon not having an answer)
+    'e': lambda vector: all(x != 'Ax' and x != 'Bx' for x in vector),
+    # [A] When the Turkers agree on an answer, LF must agree with that
+    #     Also the Turkers must agree on the original table
+    'A': lambda vector: vector and vector[0][0] == 'A' and all(x != 'Ax' for x in vector),
+    # [a] When the Turkers agree on an answer, LF must agree with that
+    'a': lambda vector: all(x != 'Ax' for x in vector),
+    # [l] Allow LF to disagree with Turkers at most once
+    'l': lambda vector: sum([x == 'Ax' for x in vector], 0) <= 1,
+    }
+
+def process(ex_id, filename, scheme):
+    print >> sys.stderr, 'Processing', ex_id
+
+    # Read the "check" file: agreement with the Turked data
+    n, k, d, denotations, check_grid = read_check_denotations_file(filename)
+    if not n:
+        return 0, []
+
+    # Which tables did we Turk on?
+    turked_tables = [i for (i,x) in enumerate(check_grid[0]) if x is not None]
+    # turk_flags contains only A, B, or X
+    turk_flags = ''.join(check_grid[0][i][0] for i in turked_tables)
+    print >> sys.stderr, turked_tables, turk_flags
+
+    lf_matched = []
+    for i, vector in enumerate(check_grid):
+        if scheme([vector[x] for x in turked_tables]):
+            lf_matched.append(i)
+    return n, lf_matched
+
+def get_best_lf(ex_id, filename, indices, return_all=False):
+    lfs = []
+    with gzip.open(filename) as fin:
+        for line in fin:
+            line = line.strip()
+            if not line.startswith('(derivation '):
+                continue
+            # Just find matching parentheses
+            # Wow this is an awful hack
+            line = re.sub(r'^\(derivation \(formula ', '', line)
+            num_open = 0
+            stuff = []
+            for char in line:
+                stuff.append(char)
+                if char == '(':
+                    num_open += 1
+                elif char == ')':
+                    num_open -= 1
+                    if not num_open:
+                        break
+            formula = ''.join(stuff)
+            size = int(re.search(r'\$ROOT:(\d+)', line).group(1))
+            lfs.append((size, formula))
+    lfs = [lfs[i] for i in indices]
+    lfs.sort()
+    return lfs if return_all else lfs[0]
+
+################################################
+# Main entry
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-s', '--scheme', choices=list(SCHEMES), default='a')
+    # ./run @mode=tables @class=dump @parser=grow-dpd -cheat -numExamplesPerFile 1
+    parser.add_argument('dump_dir')
+    # ./run @mode=tables @class=alter-ex -dumpdir [above] -turkedDataPath ...
+    parser.add_argument('turk_ex_dir')
+    parser.add_argument('-a', '--dump-all',
+            help='Dump all matching LFs. Specify the base directory here')
+    args = parser.parse_args()
+
+    if args.dump_all:
+        assert os.path.isdir(args.dump_all)
+    
+    # Note relevant files
+    check_denotations_files = {}
+    path = os.path.join(args.turk_ex_dir, 'check-denotations')
+    for filename in os.listdir(path):
+        match = re.match(r'nt-(\d+)\.gz', filename)
+        assert match, 'Invalid filename: ' + os.path.join(path, filename)
+        ex_id = int(match.group(1))
+        assert ex_id not in check_denotations_files, 'Repeated example id: ' + ex_id
+        check_denotations_files[ex_id] = os.path.join(path, filename)
+    print >> sys.stderr, 'Found', len(check_denotations_files), 'check-denotations files'
+    
+    dump_files = {}
+    path = args.dump_dir
+    for filename in os.listdir(path):
+        if re.match(r'dumped-.*.gz', filename):
+            with gzip.open(os.path.join(path, filename)) as fin:
+                ex_id = None
+                for line in fin:
+                    line = line.strip()
+                    if line.strip().startswith('(id '):
+                        match = re.match(r'\(id nt-(\d+)\)', line)
+                        assert match, 'Invalid ID: ' + line
+                        ex_id = int(match.group(1))
+                        break
+                assert ex_id is not None, 'ID not found in ' + filename
+                dump_files[ex_id] = os.path.join(path, filename)
+    print >> sys.stderr, 'Found', len(check_denotations_files), 'dump files'
+
+    for ex_id, filename in sorted(check_denotations_files.items()):
+        n, matched = process(ex_id, filename, SCHEMES[args.scheme])
+        if matched:
+            print >> sys.stderr, ex_id, n, len(matched)
+            if args.dump_all:
+                lfs = get_best_lf(ex_id, dump_files[ex_id], matched, True)
+                path = os.path.join(args.dump_all, 'nt-' + str(ex_id) + '.gz')
+                with gzip.open(path, 'w') as fout:
+                    for size, lf in lfs:
+                        print >> fout, lf
+            else:
+                size, best_lf = get_best_lf(ex_id, dump_files[ex_id], matched)
+                print '\t'.join(unicode(x) for x in ['nt-' + str(ex_id), best_lf])
+
+if __name__ == '__main__':
+    main()
+

tables/toy-examples/203-839.tsv

@@ -0,0 +1,22 @@
+Station	Type	Latitude	Longitude	El. (m)	Opened	Closed	Coordinates
+Mount Wilson, California	Solar	34º13'N	118º56'W	1737	1908	1920	34°13′S 118°56′W / 34.217°S 118.933°W
+Hump Mountain, North Carolina	Solar	36º8'N	82º0'W	1500	1917	1918	36°8′N 82°0′W / 36.133°N 82.000°W
+Calama, Chile	Solar	22º28'S	68º56'W	2250	1918	1920	22°28′S 68°56′W / 22.467°S 68.933°W
+Mount Montezuma, Chile	Solar	22º40'S	68º56'W	2711	1920	?	22°40′S 68°56′W / 22.667°S 68.933°W
+Mount Harquahala, Arizona	Solar	33º48'N	113º20'W	1721	1920	1925	33°48′N 113°20′W / 33.800°N 113.333°W
+Table Mountain, California	Solar	34º22'N	117º41'W	2286	1925	1962	34°22′N 117°41′W / 34.367°N 117.683°W
+Mount Brukkaros, Namibia	Solar	25º52'S	17º48'E	1586	1926	1931	25°52′S 17°48′E / 25.867°S 17.800°E
+Mount Saint Catherine, Egypt	Solar	28º31'N	33º56'E	2591	1934	1937	28°31′N 33°56′E / 28.517°N 33.933°E
+Burro Mountain, New Mexico	Solar	32º40'N	108º33'W	2440	1938	1946	32°40′N 108°33′W / 32.667°N 108.550°W
+Organ Pass, New Mexico	Space Track	32º25'N	253º27'E				32°25′N 106°33′W / 32.417°N 106.550°W
+Olifantsfontein, South Africa	Space Track	25º58'S	28º15'E				25°58′S 28°15′E / 25.967°S 28.250°E
+Woomera, Australia	Space Track	31º06'S	136º46'E				31°06′S 136°46′E / 31.100°S 136.767°E
+Cadiz, Spain	Space Track	36º28'N	353º48'E				36°28′N 6°12′W / 36.467°N 6.200°W
+Shiraz, Iran	Space Track	29º38'N	52º31'E				29°38′N 52°31′E / 29.633°N 52.517°E
+Curaçao, Netherlands West Indies	Space Track	12º05'N	291º10'E				12°05′N 68°50′W / 12.083°N 68.833°W
+Jupiter, Florida	Space Track	27º01'N	279º53'E				27°01′N 80°07′W / 27.017°N 80.117°W
+Haleakala, Hawaii	Space Track	20º43'N	203º45'E				20°43′N 156°15′W / 20.717°N 156.250°W
+Villa Dolores, Argentina	Space Track	31º57'N	294º54'E				31°57′N 65°06′E / 31.950°N 65.100°E
+Mitaka, Japan	Space Track						
+Nani Tal, India	Space Track						
+Arequipa, Peru	Solar,\nSpace Track						

tables/toy-examples/204-495.tsv

@@ -0,0 +1,41 @@
+Date	Opponent	Venue	Result	Attendance	Scorers
+15 August 1987	Derby County	Away	0–1	17,204	—
+18 August 1987	Coventry City	Home	0–1	09,380	—
+22 August 1987	West Ham United	Home	2–2	08,073	Harford (2)
+29 August 1987	Chelsea	Away	0–3	16,075	—
+31 August 1987	Arsenal	Home	1–1	08,745	Wilson (pen)
+5 September 1987	Oxford United	Away	5–2	06,804	Breacker, Harford, Hill, Nwajiobi, B. Stein
+12 September 1987	Everton	Home	2–1	08,124	Hill, B. Stein
+19 September 1987	Charlton Athletic	Away	0–1	05,002	—
+26 September 1987	Queens Park Rangers	Away	0–2	11,175	—
+3 October 1987	Manchester United	Home	1–1	09,137	Harford
+10 October 1987	Portsmouth	Away	1–3	12,391	Harford (pen)
+17 October 1987	Wimbledon	Home	2–0	07,018	B. Stein, Wilson
+24 October 1987	Liverpool	Home	0–1	11,997	—
+7 November 1987	Newcastle United	Home	4–0	07,638	Nwajiobi, B. Stein, M. Stein (2)
+14 November 1987	Sheffield Wednesday	Away	2–0	16,960	Allinson, M. Stein
+21 November 1987	Tottenham Hotspur	Home	2–0	10,091	Allinson (2)
+5 December 1987	Norwich City	Home	1–2	07,002	B. Stein
+12 December 1987	Watford	Away	1–0	12,152	Foster
+18 December 1987	Southampton	Home	2–2	06,618	Harford, McDonough
+26 December 1987	Everton	Away	0–2	32,128	—
+28 December 1987	Charlton Athletic	Home	1–0	07,243	Wilson
+1 January 1988	Chelsea	Home	3–0	08,018	Harford, B. Stein, M. Stein
+2 January 1988	West Ham United	Away	1–1	16,716	M. Stein
+16 January 1988	Derby County	Home	1–0	07,175	McDonough
+6 February 1988	Oxford United	Home	7–4	08,063	Harford (2), McDonough, B.Stein, M.Stein (3)
+13 February 1988	Arsenal	Away	1–2	22,612	M.Stein
+5 March 1988	Wimbledon	Away	0–2	04,854	—
+15 March 1988	Coventry City	Away	0–4	13,711	—
+29 March 1988	Portsmouth	Home	4–1	06,740	B.Stein, M.Stein, Wilson, own goal
+2 April 1988	Newcastle United	Away	0–4	20,752	—
+5 April 1988	Sheffield Wednesday	Home	2–2	07,337	McDonough, B. Stein
+12 April 1988	Manchester United	Away	0–3	28,830	—
+19 April 1988	Queens Park Rangers	Home	2–1	06,735	Foster, Wilson (pen)
+30 April 1988	Norwich City	Away	2–2	13,171	M. Stein, Wilson (pen)
+2 May 1988	Watford	Home	2–1	10,409	Oldfield, Wilson (pen)
+4 May 1988	Tottenham Hotspur	Away	1–2	15,437	Grimes
+7 May 1988	Southampton	Away	1–1	12,722	Wilson
+9 May 1988	Liverpool	Away	1–1	30,374	Oldfield
+13 May 1988	Nottingham Forest	Home	1–1	09,108	Donaghy
+15 May 1988	Nottingham Forest	Away	1–1	13,106	Oldfield