Update README.md

Updated as suggested in #117

.gitignore

@@ -1,9 +1,13 @@
 lib
 fig
+sfig
+refdb
 virtuoso-opensource
-module-classes.txt
+
 classes
 libsempre
+sempre.jar
+module-classes.txt
 
 state
 out
@@ -21,3 +25,22 @@ semparse.iml
 *.cache
 *.DS_Store
 java.hprof.txt
+
+# Don't put papers here
+/papers
+
+# Symlinks
+/c
+/e
+/t
+/x
+scr
+rdf
+
+# interactive outputs
+int-output*
+int-backup
+interactive/.ipynb_checkpoints
+# Community server logs
+community-server/data
+community-server/data-backup

DOCUMENTATION.md

@@ -146,7 +146,7 @@ The possible `Value`s, along with an example are as follows:
 
     In general:
 
-        (table (<header-string-1> ... <header-string-n>) (<row-1-value-1> ... <value-1-value-n>) ...)
+        (table (<header-string-1> ... <header-string-n>) (<row-1-value-1> ... <row-1-value-n>) ...)
 
 There are some more arcane `Value`s (see `Values.java` for a list), but they
 are not that important from the point of view of specifying a logical form.
@@ -945,7 +945,7 @@ which are used in our first sematic parsing applications, but they probably
 should be avoided unless you're specifically doing Freebase QA.  Even in that
 case, the main thing you should think about is:
 
-    (rule $Entity ($PHRASE) (LexiconFn fbsearch)
+    (rule $Entity ($PHRASE) (LexiconFn fbsearch))
 
 which uses the Freebase Search API to look up entities.  Be aware here that the
 API will generously return many candidate entities for any string you give it,

LICENSE.txt

@@ -1,340 +1,12 @@
-		    GNU GENERAL PUBLIC LICENSE
-		       Version 2, June 1991
+Copyright (c) 2013, Stanford University.
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.
-                       59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License.  You may obtain a copy of the
+License at
 
-			    Preamble
+    http://www.apache.org/licenses/LICENSE-2.0
 
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Library General Public License instead.)  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
-
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
-
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
-
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
-
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-		    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
-
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
-
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
-
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
-
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
-
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
-
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
-
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
-
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
-
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
-
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
-
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-			    NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-		     END OF TERMS AND CONDITIONS
-
-	    How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) 19yy  <name of author>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) 19yy name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  <signature of Ty Coon>, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Library General
-Public License instead of this License.
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations under the License.

README.md

@@ -1,4 +1,4 @@
-# SEMPRE 2.1: Semantic Parsing with Execution
+# SEMPRE 2.4: Semantic Parsing with Execution
 
 ## What is semantic parsing?
 
@@ -24,7 +24,7 @@ A virtual travel agent task:
 - Denotation: `(list ...)`
 
 By parsing utterances into logical forms, we obtain a rich representation that
-enables mucher deeper, context-aware understanding beyond the words.  With the
+enables much deeper, context-aware understanding beyond the words.  With the
 rise of natural language interfaces, semantic parsers are becoming increasingly
 more powerful and useful.
 
@@ -62,6 +62,8 @@ SEMPRE has been used in the following papers:
   offshoot, and does not use many of the core learning and parsing utiltiies in
   SEMPRE.  To reproduce those results, check out SEMPRE 1.0.
 
+Please refer to the [project page](https://nlp.stanford.edu/software/sempre/) for a more complete list.
+
 ## Where do I go next?
 
 - If you're new to semantic parsing, you can learn more from the [background
@@ -82,6 +84,8 @@ You must have the following already installed on your system.
 - Ant 1.8.2
 - Ruby 1.8.7 or 1.9
 - wget
+- make (for compiling fig and Virtuoso)
+- zip (for unzip downloaded dependencies)
 
 Other dependencies will be downloaded as you need them.  SEMPRE has been tested
 on Ubuntu Linux 12.04 and MacOS X.  Your mileage will vary depending on how
@@ -95,18 +99,21 @@ similar your system is.
 
 2. Download the minimal core dependencies (all dependencies will be placed in `lib`):
 
-        ./pull-dependencies core
+        ruby ./pull-dependencies core
 
 3. Compile the source code (this produces `libsempre/sempre-core.jar`):
 
         ant core
+        
+   If there is any issue during compilation, try deleting the directories `lib` and `fig`
+   and rerunning `./pull-dependencies core`
 
 4. Run an interactive shell:
 
-        ./run @mode=simple
+        ruby ./run @mode=simple
 
     You should be able to type the following into the shell and get the answer `(number 7)`:
-   
+
         (execute (call + (number 3) (number 4)))
 
 To go further, check out the [tutorial](TUTORIAL.md) and then the [full
@@ -118,24 +125,40 @@ If you will be using natural language to query databases (e.g., Freebase), then
 you will also need to setup your own Virtuoso database (unless someone already
 has done this for you):
 
-    # For Ubuntu, make sure these dependencies are installed
+For Ubuntu, follow this:
+
     sudo apt-get install -y automake gawk gperf libtool bison flex libssl-dev
 
     # Clone the repository
-    git clone https://github.com/openlink/virtuoso-opensource
+    ./pull-dependencies virtuoso
+
+    # Make and install
     cd virtuoso-opensource
-    git checkout tags/v7.0.0
-
-    # Configure
     ./autogen.sh
-    mv INSTALL INSTALL.txt  # Avoid conflict on case-insensitive file systems
     ./configure --prefix=$PWD/install
-
-    # Make (this takes a while)
     make
     make install
     cd ..
 
+on OS/X you can install virtuoso using homebrew by following the instructions
+[here](http://carsten.io/virtuoso-os-on-mac-os/)
+
+To have SEMPRE interact with Virtuoso, the required modules need to be compiled as follow:
+
+    ./pull-dependencies core corenlp freebase
+    ant freebase
+
+# Contribute
+
+To contribute code or resource to SEMPRE:
+
+- Create a fork of the repository. If you already have a fork,
+  it is a good idea to sync with the upstream repository first.
+- Push your changes to a new branch in your fork.
+- Start a pull request: go to your branch on the GitHub website,
+  then click "New pull request". Please specify the `develop` branch
+  of the upstream repository.
+
 # ChangeLog
 
 Changes from SEMPRE 1.0 to SEMPRE 2.0:
@@ -151,10 +174,18 @@ Changes from SEMPRE 2.0 to SEMPRE 2.1:
 - Added the `tables` package for the paper *Compositional semantic parsing on semi-structured tables* (ACL 2015).
 - Add and `overnight` package for the paper *Building a semantic parser overnight* (ACL 2015).
 
-# License
+Changes from SEMPRE 2.1 to SEMPRE 2.2:
 
-SEMPRE is licensed under the [GNU General Public
-License](http://www.gnu.org/licenses/gpl-2.0.html) (v2 or later).
-Note that this is the /full/ GPL, which allows many free uses, but not
-its use in distributed [proprietary
-software](http://www.gnu.org/licenses/gpl-faq.html#GPLInProprietarySystem).
+- Added code for the paper *Inferring Logical Forms From Denotations* (ACL 2016).
+
+Changes from SEMPRE 2.2 to SEMPRE 2.3:
+
+- Added the `interactive` package for the paper *Naturalizing a programming language through interaction* (ACL 2017).
+
+Changes from SEMPRE 2.3 to SEMPRE 2.3.1:
+
+- Modified the `tables` module to resemble SEMPRE 2.1, effectively making it work again.
+
+Changes from SEMPRE 2.3.1 to SEMPRE 2.4:
+
+- Added the `cprune` package for the paper *Macro Grammars and Holistic Triggering for Efficient Semantic Parsing* (EMNLP 2017).

TUTORIAL.md

@@ -17,7 +17,8 @@ Recall that in semantic parsing, *natural language utterances* are mapped into
 *logical forms* (think programs), which are executed to produce some
 *denotation* (think return value).
 
-We have assumed you have already downloaded SEMPRE and can open up a shell:
+We have assumed you have already [installed](README.md#installation)
+SEMPRE and can open up a shell:
 
     ./run @mode=simple
 
@@ -32,7 +33,7 @@ shell command and executes it.  To see which command is run, do:
 
 This should print out:
 
-    rlwrap java -cp libsempre/*:lib/* -ea edu.stanford.nlp.sempre.Main -interactive
+    java -cp libsempre/*:lib/* -ea edu.stanford.nlp.sempre.Main -Main.interactive
 
 You can pass in additional options:
 
@@ -552,7 +553,7 @@ where $s$ and $o$ are entities and $p$ is a property.  For example:
 is one triple.  If we think of the entities as nodes in a directed graph, the
 each triple is a directed edge between two nodes labeled with the property.
 
-See `data/tutorial.ttl` for an example of a tiny subset of the Freebase graph
+See `freebase/data/tutorial.ttl` for an example of a tiny subset of the Freebase graph
 pertaining to geography about California.
 
 First, pull the dependencies needed for Freebase:
@@ -586,7 +587,7 @@ To stop the server:
 ### Setting up a copy of Freebase
 
 The best case is someone already installed Freebase for you and handed you a
-host:port.  Otherwise, to run your own copy of the Freebase graph (a
+host:port.  Otherwise, to run your own copy of the entire Freebase graph (a
 2013 snapshot), read on.
 
 Download it (this is really big and takes a LONG time):
@@ -611,12 +612,9 @@ We assume you have started the Virtuoso database:
 
 Then start up a prompt:
 
-    ./run @mode=simple-freebase @sparqlserver=localhost:3001
-
-The simplest logical formula in lambda DCS is a single entity:
-
-    fb:en.california
+    ./run @mode=simple-freebase-nocache @sparqlserver=localhost:3001
 
+The simplest logical formula in lambda DCS is a single entity such as `fb:en.california`.
 To execute this query, simply type the following into the interactive prompt:
 
     (execute fb:en.california)
@@ -753,7 +751,7 @@ is the basis for the overloading.
 
 Now start the interactive prompt:
 
-    ./run @mode=simple-freebase @sparqlserver=localhost:3001 -Grammar.inPaths freebase/data/tutorial-freebase.grammar -SimpleLexicon.inPaths freebase/data/tutorial-freebase.lexicon
+    ./run @mode=simple-freebase-nocache @sparqlserver=localhost:3001 -Grammar.inPaths freebase/data/tutorial-freebase.grammar -SimpleLexicon.inPaths freebase/data/tutorial-freebase.lexicon
 
 We should be able to parse the following utterances:
 

build.xml

@@ -25,7 +25,7 @@
     <antcall target="compile.released"/>
   </target>
 
-  <target name="compile.released" depends="init,core,cache,corenlp,freebase,tables,overnight"/>
+  <target name="compile.released" depends="init,core,cache,corenlp,freebase,tables,cprune,overnight"/>
 
   <!-- Compile core -->
   <target name="core" depends="init">
@@ -69,7 +69,7 @@
   </target>
 
   <!-- Compile tables -->
-  <target name="tables" depends="init,core,corenlp">
+  <target name="tables" depends="init,core,corenlp,cprune">
     <echo message="Compiling ${ant.project.name}: tables"/>
     <mkdir dir="${classes}/tables"/>
     <javac srcdir="${src}" destdir="${classes}/tables" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
@@ -78,6 +78,16 @@
     <jar destfile="${libsempre}/sempre-tables.jar" basedir="${classes}/tables"/>
   </target>
 
+  <!-- Compile cprune -->
+  <target name="cprune" depends="init,core">
+    <echo message="Compiling ${ant.project.name}: cprune"/>
+    <mkdir dir="${classes}/cprune"/>
+    <javac srcdir="${src}" destdir="${classes}/cprune" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
+      <include name="edu/stanford/nlp/sempre/cprune/"/>
+    </javac>
+    <jar destfile="${libsempre}/sempre-cprune.jar" basedir="${classes}/cprune"/>
+  </target>
+
   <!-- Compile overnight -->
   <target name="overnight" depends="init,core">
     <echo message="Compiling ${ant.project.name}: overnight"/>
@@ -88,6 +98,25 @@
     <jar destfile="${libsempre}/sempre-overnight.jar" basedir="${classes}/overnight"/>
   </target>
 
+  <!-- Compile interactive -->
+  <target name="interactive" depends="init,core">
+    <echo message="Compiling ${ant.project.name}: interactive"/>
+    <mkdir dir="${classes}/interactive"/>
+    <javac srcdir="${src}" destdir="${classes}/interactive" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
+      <include name="edu/stanford/nlp/sempre/interactive/"/>
+    </javac>
+    <jar destfile="${libsempre}/sempre-interactive.jar" basedir="${classes}/interactive"/>
+  </target>
+
+  <!-- Compile geo880 -->
+  <target name="geo880" depends="init,core,corenlp,tables">
+    <echo message="Compiling ${ant.project.name}: geo880"/>
+    <mkdir dir="${classes}/geo880"/>
+    <javac srcdir="${src}" destdir="${classes}/geo880" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
+      <include name="edu/stanford/nlp/sempre/geo880/"/>
+    </javac>
+    <jar destfile="${libsempre}/sempre-geo880.jar" basedir="${classes}/geo880"/>
+  </target>
 
   <!-- Clean up -->
   <target name="clean">

freebase/data/tutorial-freebase.lexicon

@@ -0,0 +1,6 @@
+{"lexeme": "california", "formula": "fb:en.california"}
+{"lexeme": "the golden state", "formula": "fb:en.california"}
+{"lexeme": "cities", "formula": "(fb:type.object.type fb:location.citytown)"}
+{"lexeme": "towns", "formula": "(fb:type.object.type fb:location.citytown)"}
+{"lexeme": "in", "formula": "fb:location.location.containedby"}
+{"lexeme": "located in", "formula": "fb:location.location.containedby"}

freebase/data/tutorial.ttl

@@ -1,21 +1,21 @@
-@prefix fb: <http://rdf.freebase.com/ns/>.
+@prefix fb: <http://rdf.freebase.com/ns/> .
 
-fb:en.los_angeles	fb:location.location.containedby	fb:en.california.
-fb:en.san_francisco	fb:location.location.containedby	fb:en.california.
-fb:en.mount_whitney	fb:location.location.containedby	fb:en.california.
+fb:en.los_angeles	fb:location.location.containedby	fb:en.california	.
+fb:en.san_francisco	fb:location.location.containedby	fb:en.california	.
+fb:en.mount_whitney	fb:location.location.containedby	fb:en.california	.
 
-fb:en.seattle	fb:type.object.type	fb:location.citytown.
-fb:en.san_francisco	fb:type.object.type	fb:location.citytown.
-fb:en.los_angeles	fb:type.object.type	fb:location.citytown.
-fb:en.mount_whitney	fb:type.object.type	fb:geography.mountain.
-fb:en.california	fb:type.object.type	fb:location.us_state.
+fb:en.seattle	fb:type.object.type	fb:location.citytown	.
+fb:en.san_francisco	fb:type.object.type	fb:location.citytown	.
+fb:en.los_angeles	fb:type.object.type	fb:location.citytown	.
+fb:en.mount_whitney	fb:type.object.type	fb:geography.mountain	.
+fb:en.california	fb:type.object.type	fb:location.us_state	.
 
-fb:en.seattle	fb:type.object.name	"Seattle"@en.
-fb:en.san_francisco	fb:type.object.name	"San Francisco"@en.
-fb:en.los_angeles	fb:type.object.name	"Los Angeles"@en.
-fb:en.mount_whitney	fb:type.object.name	"Mount Whitney"@en.
-fb:en.california	fb:type.object.name	"California"@en.
+fb:en.seattle	fb:type.object.name	"Seattle"@en	.
+fb:en.san_francisco	fb:type.object.name	"San Francisco"@en	.
+fb:en.los_angeles	fb:type.object.name	"Los Angeles"@en	.
+fb:en.mount_whitney	fb:type.object.name	"Mount Whitney"@en	.
+fb:en.california	fb:type.object.name	"California"@en	.
 
-fb:en.seattle	fb:location.location.area	"369.2"^^xsd:double.
-fb:en.san_francisco	fb:location.location.area	"600.6"@en.
-fb:en.los_angeles	fb:location.location.area	"1301.97"^^xsd:double.
+fb:en.seattle	fb:location.location.area	"369.2"^^xsd:double	.
+fb:en.san_francisco	fb:location.location.area	"600.6"@en	.
+fb:en.los_angeles	fb:location.location.area	"1301.97"^^xsd:double	.

freebase/scripts/extract-freebase-schema.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 require 'open-uri'
 

freebase/scripts/fbshell.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 require 'open-uri'
 require 'json'

freebase/scripts/virtuoso

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # This script provides a convenient wrapper for the Virtuoso SPARQL server.
 
@@ -151,7 +151,7 @@ def add
     # Virtuoso can't handle prefixes in the second argument, so need to expand.
     tmp = "virtuoso-#{port}-#{i}.ttl"
     log "==== Step A: Converting #{ttl} to #{tmp}..."
-    run "time #{chunkCmd} -indices #{i} | sed -r -e 's/\\tfb:([^\\t]*).$/\\t\\<http:\\/\\/rdf.freebase.com\\/ns\\/\\1\\>./' > #{tmp}"
+    run "time #{chunkCmd} -indices #{i} | sed -r -e 's/\\tfb:([^\\t ]*)[\\t ]*\\.$/\\t\\<http:\\/\\/rdf.freebase.com\\/ns\\/\\1\\>./' > #{tmp}"
 
     # Index the ttl file
     log "==== Step B: Indexing #{tmp}..."

interactive/README.md

@@ -0,0 +1,115 @@
+# README
+
+This `interactive` package is the code for our paper
+*Naturalizing a programming language through interaction* (ACL 2017).
+A live demo is at [www.voxelurn.com](http://www.voxelurn.com).
+
+voxelurn is a language interface to a voxel world.
+This server handles commands used to learn from definitions, and other interactive queries.
+In this setting, the system begin with the dependency-based action language (`dal.grammar`), and gradually expand the language through interacting with it users.
+
+## Overview of the components
+
+### sempre.interactive
+
+The `edu.stanford.nlp.sempre.interactive` package live in this repo contains code for
+* running interactive commands (such as query, accept, reject, definition)
+* executor for the dependency-based action (DAL) language
+* voxelurn specific code in `edu.stanford.nlp.sempre.interactive.voxelurn` for actually   generating the voxel and manipulating them
+Utilties and resources such as the grammar and run script are in this directory, and the code in in the regular `sempre/src` directory.
+
+### voxelurn client
+
+It queries the server, and renders the voxels to a browser. A live version is at [www.voxelurn.com](http://www.voxelurn.com), which queries our server. You can also find a client for localhost at [http://local.voxelurn.com](http://local.voxelurn.com), which is the same client, but with queries going to `http://localhost:8410` instead.
+Code for the client at `https://github.com/sidaw/shrdlurn`. See its [README.md](https://github.com/sidaw/shrdlurn/blob/master/README.md) if you want to work with and build the client yourself.
+
+
+### voxelurn community server
+Located at `interactive/community-server`, the community server
+handles other functionalities such as logging client actions, leaderboard, submiting structures, authentication etc. and generally functions not related to parsing. This server is needed for running interactive experiments, but is not required just for trying out voxelurn.
+
+
+## Running the SEMPRE server for Voxelurn
+
+0. Setup SEMPRE dependencies and compile
+
+         ./pull-dependencies core
+         ant interactive
+
+1. Start the server
+
+        ./interactive/run @mode=voxelurn -server -interactive
+
+  things in the core language such as `add red left`, `repeat 3 [select left]` should work.
+
+2. Feed the server existing definitions, which should take less than 2 minutes.
+
+        ./interactive/run @mode=simulator @server=local @sandbox=none @task=freebuilddef -maxQueries 2496
+
+  try `add dancer`  now.
+
+### Interacting with the server
+
+After you run the above, there are 3 ways to interact and try your own commands.
+
+* The visual way is to use the client: [http://local.voxelurn.com](http://local.voxelurn.com).
+  Code for the client is at `https://github.com/sidaw/shrdlurn` (see its [README.md](https://github.com/sidaw/shrdlurn/blob/master/README.md)).
+  Try `[add dancer; front 5] 3 times`.
+
+* Hit `Ctrl-D` on the terminal running the server, and type `add red top`, or `add green monster`
+
+* On a browser, type `http://localhost:8410/sempre?q=(:q add green monster)`
+
+
+## Experiments in ACL2017
+
+1. Start the server
+
+       ./interactive/run @mode=voxelurn -server -interactive
+
+2. Feed the server all the query logs
+
+       ./interactive/run @mode=simulator @server=local @sandbox=none @task=freebuild -maxQueries 103874
+
+  This currently takes just under 30 minutes. Decrease maxQuery for a quicker experiment. This generate `plotInfo.json` in `./state/execs/${lastExec}.exec/` where `lastExec` is `cat ./state/lastExec`.
+
+3. Taking `../state/execs/${lastExec}.exec/plotInfo.json` as input, we can analyze the data and produce some plots using the following ipython notebook
+
+       ipython notebook interactive/analyze_data.ipynb
+
+  which prints out basic statistics and generates the plots used in our paper. The plots are saved at `../state/execs/${lastExec}.exec/`
+
+
+## Misc.
+
+There are some unit tests
+
+    ./interactive/run @mode=test
+
+To specify a specific test class and verbosity
+
+    ./interactive/run @mode=test @class=DALExecutorTest -verbose 5
+
+Clean up or backup data
+
+    ./interactive/run @mode=backup # save previous data logs
+    ./interactive/run @mode=trash # deletes previous data logs
+
+Data, in .gz can be found in queries.
+
+* `./interactive/queries/freebuild.def.json.gz`
+has 2495 definitions combining just over 10k utterances.
+* `./interactive/queries/freebuild.json.gz` has 103873 queries made during the main experiment.
+
+## Voxelurn community server (optional and in development)
+
+This server helps with client side logging, leaderboard, authentication etc. basically anything that is not directly related to parsing.
+This component is only required if you want to run the interactive experiment yourself. It is fairly coupled with the [voxelurn  client](http://github.com/sidaw/shrdlurn), which sends the request to this server.
+
+    cd interactive/community-server
+    python install-deps.py
+    python server.py --port 8403
+
+    # required keys for authentication
+    export SEMPRE_JWT_SECRET=ANY_RANDOM_SEQEUNCE
+    export SLACK_OAUTH_SECRET=OAUTH_KEY_FROM_SLACK

interactive/analyze_data.ipynb

@@ -0,0 +1,424 @@
+{
+ "cells": [
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "A line looks like this\n",
+    "  \"time\": \"2017-01-21T05:31:57.474\",\n",
+    "  \"id\": \"AMT_A1HKYY6XI2OHO1\",\n",
+    "  \"log\": \"(:q \\\"repeat 10 [ Ebony Wall; select front]\\\")\",\n",
+    "  \"stats.type\": \"q\",\n",
+    "  \"stats.size\": 1,\n",
+    "  \"stats.status\": \"Induced\",\n",
+    "  \"queryCount\": 2707"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import csv\n",
+    "import numpy as np\n",
+    "import pandas as pd  \n",
+    "import matplotlib\n",
+    "import matplotlib.pyplot as plt\n",
+    "import json\n",
+    "import os\n",
+    "import subprocess\n",
+    "from collections import OrderedDict\n",
+    "%matplotlib inline  \n",
+    "\n",
+    "with open('../state/lastExec', 'rb') as lastExec:\n",
+    "    lastExecInd = lastExec.readline().strip()\n",
+    "print lastExecInd\n",
+    "     \n",
+    "rows = []; \n",
+    "execInd = lastExecInd;\n",
+    "execPath = '../state/execs/%s.exec/' % execInd\n",
+    "#print 'analyzing: ' + execPath\n",
+    "def printOptions():\n",
+    "    with open(os.path.join(execPath,'options.map')) as optionsfile:\n",
+    "        opts = filter(lambda l: 'file' in l or 'logFiles' in l  or 'reqParams' in l, optionsfile.readlines());\n",
+    "        for opt in opts: print opt.strip()\n",
+    "    # egrep 'file|Simulator'\n",
+    "printOptions()\n",
+    "\n",
+    "with open('../state/execs/%s.exec/plotInfo.json' % execInd, 'rb') as jsonfile:\n",
+    "    json_lines = jsonfile.readlines()\n",
+    "\n",
+    "rawrows = [json.loads(l) for l in json_lines]\n",
+    "rows = [r for r in rawrows if r.has_key('stats.type')]\n",
+    "\n",
+    "print '%d queries in plotInfo' % len(json_lines)\n",
+    "\n",
+    "def print_defstats():\n",
+    "    filtered_rows = [r for r in rows if r['stats.type'] == 'def' and r['stats.num_rules'] >= 1]\n",
+    "    total_failed = np.sum([r['stats.num_failed'] for r in filtered_rows])\n",
+    "    total_body = np.sum([r['stats.num_body'] for r in filtered_rows])\n",
+    "\n",
+    "    stats =  {\\\n",
+    "           'total_def_queries': len(filtered_rows), \\\n",
+    "           'total_rules': np.sum([r['stats.num_rules'] for r in filtered_rows]), \\\n",
+    "           'total_failed': total_failed, \\\n",
+    "           'total_body': total_body\n",
+    "           }\n",
+    "    print ''\n",
+    "    print stats\n",
+    "    print 'failpercent: %.4f' % (float(total_failed)/float(total_body))\n",
+    "print_defstats()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "\n",
+    "\n",
+    "def print_stats():\n",
+    "    filtered_rows = [r for r in rows if r['stats.type'] == 'accept']\n",
+    "    induced_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status']=='Induced']\n",
+    "    core_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status']=='Core']\n",
+    "    none_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status']=='Nothing']\n",
+    "\n",
+    "    stats =  {\\\n",
+    "           'accepted': len(filtered_rows), \\\n",
+    "           'induced': len(induced_rows), \\\n",
+    "           'inducedp':len(induced_rows)/float(len(filtered_rows)),\\\n",
+    "           'core': len(core_rows), \\\n",
+    "           'corep':len(core_rows)/float(len(filtered_rows)),\\\n",
+    "           'none': len(none_rows), \\\n",
+    "           'nonep':len(none_rows)/float(len(filtered_rows))\\\n",
+    "           }\n",
+    "    print '\\nCount of accepted query / percentage'\n",
+    "    print ' total:{accepted}\\n induced:{induced}({inducedp:.4f})\\n core:{core}({corep:.4f})\\n none:{none}({nonep:.4f}),'.format(**stats)\n",
+    "    #print 'check %f' % (stats['inducedp']+stats['corep']+stats['nonep'])\n",
+    "    \n",
+    "    statscorrect = {\n",
+    "        'accepted0': np.mean([1 if r['stats.rank']==0 and r['stats.status']!='Nothing' else 0 for r in filtered_rows]),\\\n",
+    "        'accepted1': np.mean([1 if r['stats.rank']>=0 and r['stats.status']!='Nothing'  else 0 for r in filtered_rows]),\\\n",
+    "        'induced0': np.mean([1 if r['stats.rank']==0 and r['stats.status']=='Induced' else 0 for r in filtered_rows]),\\\n",
+    "        'induced1': np.mean([1 if r['stats.rank']>=0 and r['stats.status']=='Induced' else 0 for r in filtered_rows]),\\\n",
+    "        'core0': np.mean([1 if r['stats.rank']==0 and r['stats.status']=='Core' else 0 for r in filtered_rows]),\\\n",
+    "        'core1': np.mean([1 if r['stats.rank']>=0 and r['stats.status']=='Core' else 0 for r in filtered_rows])\\\n",
+    "    }\n",
+    "    print '\\nranked 1 accepted / found accepted'\n",
+    "    print ' total:{accepted0:.4f}/{accepted1:.4f}\\n induced:{induced0:.4f}/{induced1:.4f}\\n core:{core0:.4f}/{core1:.4f}'.format(**statscorrect)\n",
+    "\n",
+    "    token_types = set();\n",
+    "    rows_types = [r for r in rows if r['stats.type'] == 'q'];\n",
+    "    for r in rows_types:\n",
+    "        token_types |= set(r['q'].split(' '))\n",
+    "        \n",
+    "    print '\\nThere are %d token types of %d query types' % (len(token_types), len(rows_types))\n",
+    "print_stats()\n",
+    "\n",
+    "def percent_core():\n",
+    "    allq = [r for r in rows if r['stats.type'] == 'accept'];\n",
+    "    is_status = [100 if r['stats.status'] == 'Induced' else 0 for r in allq]\n",
+    "    print 'percent_induced %f' % np.mean(is_status)\n",
+    "    print 'percent_induced_last10k %f' % np.mean(is_status[len(is_status)-10000:])\n",
+    "percent_core()\n",
+    "\n",
+    "# print filtered_rows[0]\n",
+    "\n",
+    "\n",
+    "\n",
+    "def percent_error():\n",
+    "    allerror = [r for r in rawrows if r.has_key('stats.error')];\n",
+    "    alluerror = [r for r in rawrows if r.has_key('stats.uncaught_error')];\n",
+    "    print 'errors %d (%d uncaught)' % (len(allerror), len(alluerror))\n",
+    "percent_error()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "\n",
+    "def plot_reset():\n",
+    "    global p\n",
+    "    p = {'color': 'r', 'linewidth': 2, 'alpha':0.5}\n",
+    "                       #, 'marker':'*', 'markersize':0.3}\n",
+    "plot_reset()\n",
+    "def savefig(filename = 'fig.pdf'):\n",
+    "    plt.savefig(os.path.join(execPath,filename) ,  bbox_inches=\"tight\")\n",
+    "\n",
+    "def plot_cumavg(x, y, xlabel='query#', ylabel='recall', title=None):\n",
+    "    y_cum = np.cumsum(y).tolist()\n",
+    "    #print accepts_np[:,1]\n",
+    "    y_cumavg = [cum / float(count+1) for count,cum in enumerate(y_cum)]\n",
+    "    #N = 500;\n",
+    "    #y_cumavg = np.convolve(np.array(y), np.ones((N,))/N, mode='same').tolist()\n",
+    "    #print accept_rate \n",
+    "    #plt.scatter(means_baseline[0:], means[0:], s=colors, alpha=0.8, c='r')\n",
+    "    plt.plot(x, y_cumavg, **p)\n",
+    "    \n",
+    "    plt.xlabel(xlabel, fontsize=12)\n",
+    "    plt.ylabel(ylabel, fontsize=12)\n",
+    "    # plt.xlim(0, 0.65)\n",
+    "    plt.ylim(0, max(y_cumavg)*1.02)\n",
+    "    plt.xlim(0, len(rows)*1.02)\n",
+    "    xp = np.linspace(0, 0.65, 300)\n",
+    "    \n",
+    "    #plt.gca().set_aspect('equal', adjustable='box')\n",
+    "    plottitle = title if title is not None else '%s_vs_%s.pdf' % (xlabel, ylabel)\n",
+    "    # plt.savefig(os.path.join(execPath,plottitle) ,  bbox_inches=\"tight\")\n",
+    "    \n",
+    "def print_avg(x, name = 'unnamed'):\n",
+    "    print 'avg(%s): %f' % (name, reduce(lambda a,b: a+b, x) / float(len(x)));\n",
+    "def average_stat(stat = 'stats.size', type = 'accept'):\n",
+    "    filtered_rows = [r for r in rows if r['stats.type'] == type]\n",
+    "    query_counts = [r['queryCount'] for r in filtered_rows]\n",
+    "    stats = [r[stat] for r in filtered_rows]\n",
+    "    plot_cumavg(query_counts, stats, xlabel='query#', ylabel=stat.replace('stats.','').replace('size','# parses'));\n",
+    "    print_avg(stats, stat)\n",
+    "plt.figure()\n",
+    "p['color'] = 'b'\n",
+    "average_stat(stat = 'stats.size')\n",
+    "#savefig('ambiguity.pdf')\n",
+    "\n",
+    "\n",
+    "def precent_status(filtered_rows, status = 'Core'):\n",
+    "    query_counts = [r['queryCount'] for r in filtered_rows]\n",
+    "    is_status = [100 if r['stats.status'] == status else 0 for r in filtered_rows]\n",
+    "    print_avg(is_status, 'percent of status ' + status)\n",
+    "    plot_cumavg(query_counts, is_status, xlabel='query #', ylabel='percent');\n",
+    "\n",
+    "\n",
+    "def plotCoreInducedNone(filtered_rows):\n",
+    "    p['color'] = 'g'; p['label'] = 'induced';\n",
+    "    precent_status(filtered_rows,status = 'Induced');\n",
+    "    p['color'] = 'b'; p['label'] = 'core';\n",
+    "    precent_status(filtered_rows, status = 'Core');\n",
+    "    plt.legend(frameon=False)\n",
+    "    \n",
+    "plt.figure()\n",
+    "plotCoreInducedNone([r for r in rows if r['stats.type'] == 'accept'])\n",
+    "# savefig('parse_status_accepted.pdf')\n",
+    "\n",
+    "def plotCoreInducedNone(filtered_rows):\n",
+    "    p['color'] = 'r'; p['label'] = 'none';\n",
+    "    precent_status(filtered_rows, status = 'Nothing');\n",
+    "    p['color'] = 'g'; p['label'] = 'induced';\n",
+    "    precent_status(filtered_rows,status = 'Induced');\n",
+    "    p['color'] = 'b'; p['label'] = 'core';\n",
+    "    precent_status(filtered_rows, status = 'Core');\n",
+    "    plt.legend(frameon=False)\n",
+    "plt.figure()\n",
+    "plotCoreInducedNone([r for r in rows if r['stats.type'] == 'q' and r.has_key('stats.status')])\n",
+    "plt.ylim(0, 70)\n",
+    "savefig('parse_status_q.pdf')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "def top_users(counts, line):\n",
+    "    id = 'sessionId'\n",
+    "    if line[id] in counts:\n",
+    "        counts[line[id]] = counts[line[id]] + 1\n",
+    "    else:\n",
+    "        counts[line[id]] = 1\n",
+    "    return counts\n",
+    "\n",
+    "accept_all = [r for r in rows if r['stats.type']=='accept']\n",
+    "accept_nothing = [r for r in rows if r['stats.type']=='accept' and r['stats.status']=='Nothing']\n",
+    "\n",
+    "for r in accept_nothing[:5]:\n",
+    "    print '{q}'.format(**r)\n",
+    "    \n",
+    "sorted(reduce(top_users, rows, {}).items(), key=lambda x: -x[1]);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "def precent_status_user(filtered_rows):\n",
+    "    query_counts = [r['queryCount'] for r in filtered_rows]\n",
+    "    is_status = [100 if r['stats.status'] == 'Induced' else 0 for r in filtered_rows]\n",
+    "    plot_cumavg(query_counts, is_status, xlabel='query #', ylabel='precent induced');\n",
+    "\n",
+    "rows_to_count = [r for r in rows if r['stats.type'] == 'accept']\n",
+    "ranked_users = sorted(reduce(top_users, rows_to_count[5000:], {}).items(), key=lambda x: -x[1])\n",
+    "\n",
+    "plt.figure()\n",
+    "topnum = 5;\n",
+    "plot_reset()\n",
+    "p['alpha'] = 1;\n",
+    "p['linewidth'] = 5;\n",
+    "p['label'] = 'all'; \n",
+    "p['color'] = 'k';\n",
+    "precent_status_user([r for r in rows if r['stats.type'] == 'accept']);    \n",
+    "\n",
+    "colors = ['c', 'r', 'm', 'y', 'b', 'g']\n",
+    "plot_reset()\n",
+    "p['alpha'] = 1;\n",
+    "p['linewidth'] = 5;\n",
+    "p['alpha'] = 0.5;\n",
+    "p['marker'] = 'o';\n",
+    "p['markersize'] = 1;\n",
+    "\n",
+    "for g in enumerate(ranked_users[0:5]):\n",
+    "    # (0, (u'AMT_A1HKYY6XI2OHO1', 2830))\n",
+    "    p['label'] = '#%d' % (g[0]+1);\n",
+    "    #plotsetting['alpha'] = 1-float(g[0])/topnum;\n",
+    "    p['color'] = colors[g[0]];\n",
+    "    # print plotsetting['color']\n",
+    "    precent_status_user([r for r in rows if r['stats.type'] == 'accept' and r['sessionId'] == g[1][0]])\n",
+    "    print g\n",
+    "\n",
+    "plt.ylim(-0.1, 100)\n",
+    "plt.legend(frameon=False, loc='lower right')\n",
+    "savefig('parse_status_topuser.pdf');"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "def expressivity(status = 'Core'):\n",
+    "    filtered_rows = [r for r in rows if r['stats.type'] == 'accept' and r['stats.status'] == status]\n",
+    "    query_counts = [r['queryCount'] for r in filtered_rows]\n",
+    "    len_formula = [r['stats.len_formula'] for r in filtered_rows]\n",
+    "    len_utterance = [r['stats.len_utterance'] for r in filtered_rows]\n",
+    "    form_per_q = [float(ls[0])/ls[1] for ls in zip(len_formula, len_utterance)]\n",
+    "    plot_cumavg(query_counts, form_per_q, xlabel='query#', ylabel='\"expressiveness\"');\n",
+    "\n",
+    "    #plot_cumavg(query_counts, len_utterance, xlabel='query#', ylabel='length');\n",
+    "    print_avg(form_per_q, 'formula length')\n",
+    "plt.figure()\n",
+    "plot_reset()\n",
+    "p['color'] = 'b'; p['label'] = 'core'; expressivity('Core')\n",
+    "p['color'] = 'g'; p['label'] = 'induced'; expressivity('Induced')\n",
+    "plt.legend(frameon=False, loc='upper left')\n",
+    "# savefig('expressiveness.pdf')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "def expressivity_by_users(filtered_rows):\n",
+    "    query_counts = [r['queryCount'] for r in filtered_rows]\n",
+    "    len_formula = [r['stats.len_formula'] for r in filtered_rows]\n",
+    "    len_utterance = [r['stats.len_utterance'] for r in filtered_rows]\n",
+    "    form_per_q = [float(ls[0])/ls[1] for ls in zip(len_formula, len_utterance)]\n",
+    "    plot_cumavg(query_counts, form_per_q, xlabel='query#', ylabel='len(z) / len(x)');\n",
+    "\n",
+    "plot_reset()\n",
+    "plt.figure()\n",
+    "p['alpha'] = 1;\n",
+    "p['linewidth'] = 5;\n",
+    "p['label'] = 'all'; \n",
+    "p['color'] = 'k';\n",
+    "expressivity_by_users([r for r in rows if r['stats.type'] == 'accept' ])\n",
+    "    \n",
+    "colors = ['c', 'r', 'm', 'y', 'b']\n",
+    "plot_reset()\n",
+    "p['alpha'] = 1;\n",
+    "p['linewidth'] = 5;\n",
+    "p['alpha'] = 0.5;\n",
+    "p['marker'] = 'o';\n",
+    "p['markersize'] = 1;\n",
+    "rows_to_count = [r for r in rows if r['stats.type'] == 'accept']\n",
+    "ranked_users = sorted(reduce(top_users, rows_to_count, {}).items(), key=lambda x: -x[1])\n",
+    "for g in enumerate(ranked_users[0:5]):\n",
+    "    print g\n",
+    "    p['label'] = '#%d' % (g[0]+1);\n",
+    "    p['color'] = colors[g[0]];\n",
+    "    expressivity_by_users([r for r in rows if r['stats.type'] == 'accept' \\\n",
+    "                     and r['sessionId'] == g[1][0]])\n",
+    "\n",
+    "plt.ylim([0,30])\n",
+    "plt.legend(frameon=False, loc='lower right')\n",
+    "savefig('expressiveness_by_user.pdf')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "def query_reformulation_by_user():\n",
+    "    q_all = [r for r in rows if r['stats.type']=='q']\n",
+    "    q_nothing = [r for r in rows if r['stats.type']=='q' and r.has_key('stats.status') and r['stats.status']=='Nothing']\n",
+    "    print 'q_nothing: %d / %d totallines: %d' % (len(q_nothing), len(q_all), len(json_lines))\n",
+    "    ranked_users = sorted(reduce(top_users, q_all, {}).items(), key=lambda x: -x[1])\n",
+    "\n",
+    "    num_user = 3;\n",
+    "    num_query = 100;\n",
+    "    for g in (ranked_users[0:num_user]):\n",
+    "        print g\n",
+    "        print '********************'\n",
+    "        rows_userg = [r for r in rows if r['stats.type']=='q' and r['sessionId']==g[0] and r.has_key('stats.status') ]\n",
+    "        print_count = 0\n",
+    "        prev_nothing = False\n",
+    "        for r in rows_userg[-num_query:]:\n",
+    "            if prev_nothing or r['stats.status']=='Nothing':\n",
+    "                print_count = print_count + 1\n",
+    "                if print_count>100: break\n",
+    "                print r['stats.status'] + ':\\t'  + r['q'].replace('(:q \"','').replace('\")','')\n",
+    "            prev_nothing = True if r['stats.status']=='Nothing' else False\n",
+    "\n",
+    "query_reformulation_by_user()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

interactive/community-server/install-deps.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+"""
+Installs the needed dependencies for the community-server.
+
+Ideally, you would use virtualenv.
+"""
+
+import pip
+
+print("Installing community-server dependencies...")
+
+pip.main(['install', '-r', 'requirements.txt'])

interactive/community-server/requirements.txt

@@ -0,0 +1,15 @@
+eventlet==0.20.1
+Flask==0.12
+Flask-Cors==3.0.2
+Flask-SocketIO==2.8.2
+greenlet==0.4.11
+httplib2==0.18.0
+itsdangerous==0.24
+Jinja2==2.9.4
+jsonschema==2.5.1
+pep8==1.7.0
+pexpect==4.2.0
+PyJWT==1.4.2
+python-socketio==1.6.2
+requests==2.10.0
+Werkzeug==0.11.15

interactive/community-server/server.py

@@ -0,0 +1,573 @@
+#!/usr/bin/env python
+
+"""
+SHRDLURN - Community & Logging Server
+
+# Instructions
+
+You can run the server by running ```./server.py --port <PORT_NUMBER>```
+"""
+
+import json
+import sys
+import time
+import os
+import random
+import eventlet
+import glob
+from optparse import OptionParser
+from flask import Flask, request, session
+from flask_cors import CORS
+from flask_socketio import SocketIO, emit, join_room, leave_room
+import jwt
+import requests
+
+# Setup flask
+app = Flask(__name__)
+app.config['SECRET_KEY'] = os.urandom(24)
+
+# We need to enable CORS support to handle CORS flights from the frontend
+CORS(app)
+
+# The community server runs through websockets to enable real-time updates
+socketio = SocketIO(app)
+
+# Hardcoded folders for the data (mirrored in data_rotate.py)
+DATA_FOLDER = "community-server/data/"
+LOG_FOLDER = os.path.join(DATA_FOLDER, "log/")
+STRUCTS_FOLDER = os.path.join(DATA_FOLDER, "structs/")
+
+CITATION_FOLDER = "../int-output/citation"
+
+# Scoring function parameters
+GRAVITY = 1.1  # higher the gravity, the faster old structs lose score
+TIME_INTERVAL = 7200.0  # break off by every 30 minutes
+
+# Default port for the server
+DEFAULT_PORT = 8406
+
+JWT_SECRET = os.environ['SEMPRE_JWT_SECRET']
+SLACK_SECRET = os.environ['SLACK_OAUTH_SECRET']
+
+
+@app.route("/")
+def index():
+    return "Hello World! ~ SHRDLURN Community Server"
+
+
+def is_safe_path(basedir, path, follow_symlinks=True):
+    """https://security.openstack.org/guidelines/dg_using-file-paths.html"""
+    # resolves symbolic links
+    if follow_symlinks:
+        return os.path.realpath(path).startswith(os.path.realpath(basedir))
+
+    return os.path.abspath(path).startswith(os.path.abspath(basedir))
+
+
+def score_struct(timestamp, upvotesN):
+    """We use the HN formula to score structures for ranking.
+
+    Formula is: (P + 1) / ((T + 2)^GRAVITY)
+    where: - P: the number of unique upvotes for the structure
+           - T: the amount of TIME_INTERVALs that have elapsed since the
+             structure was submitted
+           - GRAVITY: a constant to determine the weight of T v. P
+    """
+    time_ago = (current_unix_time() / TIME_INTERVAL) - \
+        (int(timestamp) / TIME_INTERVAL)
+    return (upvotesN + 1) / ((time_ago + 2) ** GRAVITY)
+
+
+def current_unix_time():
+    """Returns the number of seconds since the epoch."""
+    return int(time.time())
+
+
+def emit_structs():
+    """Walk through the STRUCTS_FOLDER directory and read each struct and emit
+    it to the user one by one."""
+
+    structs = []
+    for uid in [name for name in os.listdir(STRUCTS_FOLDER) if os.path.isdir(os.path.join(STRUCTS_FOLDER, name))]:
+        uid_folder = os.path.join(STRUCTS_FOLDER, uid)
+
+        count = 0
+        for name in os.listdir(uid_folder):
+            if count > 100:
+                break
+
+            path = os.path.join(uid_folder, name)
+            if not os.path.isfile(path):
+                continue
+
+            fname = name[:-5]
+
+            try:
+                with open(path, 'r') as f:
+                    lines = f.readlines()
+
+                    upvotes = json.loads(lines[0].strip())
+                    timestamp = json.loads(lines[1].strip())
+                    struct = json.loads(lines[2].strip())
+                    image = lines[3].strip()
+
+                    score = score_struct(timestamp, len(upvotes))
+
+                    message = {"uid": uid, "id": fname, "score": score, "upvotes": [
+                        up for up in upvotes], "struct": struct, "image": image}
+
+                    structs.append(message)
+                    count += 1
+            except:
+                pass
+
+    emit("structs", structs)
+
+
+def emit_user_structs_count(uid):
+    """"Emits a count of the total number of user structs in the folder."""
+    path = os.path.join(STRUCTS_FOLDER, uid)
+    if not is_safe_path(STRUCTS_FOLDER, path):
+        return
+        structs = [name[:-5] for name in os.listdir(path) if os.path.isfile(
+            os.path.join(path, name)) and os.path.join(path, name).endswith(".json")]
+    if os.path.isdir(path):
+        structs = [name[:-5] for name in os.listdir(path) if os.path.isfile(
+            os.path.join(path, name)) and os.path.join(path, name).endswith(".json")]
+        emit("user_structs", {"structs": structs})
+
+
+def emit_utterances():
+    """Emit a list of the last 11 utterances for the 5 most recent turkers."""
+    latest_5 = []
+    for dirname, subdirs, files in os.walk(LOG_FOLDER):
+        for fname in files:
+            path = os.path.join(dirname, fname)
+
+            mtime = os.stat(path).st_mtime
+            file_info = (mtime, fname[:-5], path)
+
+            if len(latest_5) < 3:
+                latest_5.append(file_info)
+            else:
+                earliest_time = latest_5[0][0]
+                earliest_idx = 0
+                for idx, l in enumerate(latest_5):
+                    if l[0] < earliest_time:
+                        earliest_time = l[0]
+                        earliest_idx = idx
+
+                if mtime > earliest_time:
+                    latest_5[earliest_idx] = file_info
+
+    for (time, uid, path) in sorted(latest_5, key=lambda s: int(s[0]), reverse=True):
+        uid = uid
+        utts = []
+        count = 0
+        for line in reverse_readline(path):
+            data = json.loads(line)
+            if (data["type"] == "accept" or data["type"] == "define"):
+                utts.append(line)
+                count += 1
+
+            if count > 10:
+                break
+
+        message = {"uid": uid, "utterances": utts}
+        emit("utterances", message)
+
+
+def h_index(citations):
+    """https://github.com/kamyu104/LeetCode/blob/master/Python/h-index.py"""
+    n = len(citations)
+    count = [0] * (n + 1)
+    for x in citations:
+        # Put all x >= n in the same bucket.
+        if x >= n:
+            count[n] += 1
+        else:
+            count[x] += 1
+
+    h = 0
+    for i in reversed(xrange(0, n + 1)):
+        h += count[i]
+        if h >= i:
+            return i
+    return h
+
+
+def compute_citations(dir):
+    citations = []
+    for fname in os.listdir(dir):
+        if not fname.endswith(".json"):
+            continue
+
+        path = os.path.join(dir, fname)
+
+        with open(path, 'r') as f:
+            data = json.load(f)
+            citations.append(data)
+
+    citation_numbers = [citation["cite"] + citation["self"]
+                        for citation in citations]
+    citation_score = h_index(citation_numbers)
+
+    return (citations, citation_score)
+
+
+def emit_top_builders():
+    top_5_builders = []
+    for uid in os.listdir(CITATION_FOLDER):
+        subdir = os.path.join(CITATION_FOLDER, uid)
+        if not os.path.isdir(subdir):
+            continue
+
+        top_5_builders = sorted(
+            top_5_builders, key=lambda b: b[1], reverse=True)
+
+        (citations, citation_score) = compute_citations(subdir)
+
+        top_5_builders = sorted(
+            top_5_builders, key=lambda b: b[1], reverse=True)
+        if len(top_5_builders) < 10 or citation_score > top_5_builders[9][1]:
+            # If there are more than 5 citations with cites, only return those
+            citations = sorted(
+                citations, key=lambda c: c["cite"] + c["self"], reverse=True)[:10]
+            # if len(citations_with_cites) >= 6:
+            #     citations = citations_with_cites
+
+            # Sort them by score and return the top 7.
+            citations = sorted(
+                citations, key=lambda c: c["cite"] + c["self"], reverse=True)[:10]
+
+            struct = (uid, citation_score, citations)
+            if len(top_5_builders) < 10:
+                top_5_builders.append(struct)
+            else:
+                top_5_builders[9] = struct
+
+    emit("top_builders", {"top_builders": top_5_builders},
+         broadcast=True, room="community")
+
+
+def log(message):
+    """Logs the given message by writing it in the uid's JSON log file."""
+    uid = message["uid"] if 'uid' in message else "NULL_session"
+
+    user = current_user(message['token'])
+    if user:
+        uid = user['id']
+
+    path = os.path.join(LOG_FOLDER, uid + ".json")
+
+    if not is_safe_path(LOG_FOLDER, path):
+        print("NOT SAFE!", path)
+        return
+
+    # Add a timestamp to the log
+    message["timestamp"] = current_unix_time()
+
+    # Append the log to the end of the file
+    with open(path, 'a') as f:
+        json.dump(message, f)
+        f.write('\n')
+
+
+@socketio.on('getscore')
+def get_score(data):
+    user = current_user(data['token'])
+    if not user:
+        return
+
+    uid = user['id']
+    subdir = os.path.join(CITATION_FOLDER, uid)
+    if (os.path.isdir(subdir) and is_safe_path(CITATION_FOLDER, subdir)):
+        (citations, score) = compute_citations(subdir)
+        emit("score", {"score": score})
+
+
+@socketio.on('delete_struct')
+def delete_struct(data):
+    user = current_user(data['token'])
+
+    if not user:
+        return
+
+    uid = user['id']
+
+    struct_id = data["id"]
+    struct_path = struct_id + ".json"
+    subdir = os.path.join(STRUCTS_FOLDER, uid)
+    if not (is_safe_path(STRUCTS_FOLDER, subdir)):
+        return
+    path = os.path.join(subdir, struct_path)
+    if (is_safe_path(subdir, path) and os.path.isfile(path)):
+        delete_dir = os.path.join(STRUCTS_FOLDER, uid, "deleted")
+        make_dir_if_necessary(delete_dir)
+        os.rename(path, os.path.join(delete_dir, struct_path))
+
+
+@socketio.on('join')
+def on_join(data):
+    """When a user joins the "community" room, emit to them the list of
+    the top 5 most recent users' most recent 11 utterances and all of the
+    submitted structs."""
+
+    room = data['room']
+    join_room(room)
+
+    if (room == "community"):
+        # And then we emit the most recent 5 users' utterances per file
+        emit_utterances()
+
+        # and also emit the top builders when first joining
+        emit_top_builders()
+
+        # We iterate through all the shared structs and emit them one by one
+        emit_structs()
+
+
+@socketio.on('leave')
+def on_leave(data):
+    """A user can leave a room"""
+    # username = data['sessionId']
+    room = data['room']
+    leave_room(room)
+
+
+@socketio.on('share')
+def handle_share(data):
+    """Users can share structs. We save this struct in STRUCTS_FOLDER/UID/SCORE_ID.json
+
+    where UID is the uid of the user who submitted the struct, SCORE is the
+    current score of the struct and ID is the unique index (auto-incremented) of
+    this particular struct.."""
+
+    user = current_user(data['token'])
+    if not user:
+        return
+
+    uid = user['id']
+
+    user_structs_folder = os.path.join(STRUCTS_FOLDER, uid)
+    if not is_safe_path(STRUCTS_FOLDER, user_structs_folder):
+        return
+    make_dir_if_necessary(user_structs_folder)
+    new_struct_path = os.path.join(user_structs_folder, data["id"] + ".json")
+    if not is_safe_path(user_structs_folder, new_struct_path):
+        return
+
+    submission_time = current_unix_time()
+    score = score_struct(submission_time, 0)
+
+    with open(new_struct_path, 'w') as f:
+        f.write("[]\n")  # it starts with no upvoters
+        f.write(str(submission_time) + "\n")  # timestamp of submission
+        f.write(json.dumps(data["struct"]) + "\n")  # the actual struct
+        f.write(data["image"])  # the png of the struct
+
+    # Broadcast addition to the "community" room
+    message = {"uid": uid, "id": data["id"], "score": score, "upvotes": [
+    ], "struct": data["struct"], "image": data["image"]}
+    emit("struct", message, broadcast=True, room="community")
+
+
+@socketio.on('upvote')
+def upvote(data):
+    """Users can upvote other users' structures."""
+
+    user = current_user(data['token'])
+
+    # if no authenticated user, do nothing
+    if not user:
+        return
+
+    subdir = os.path.join(STRUCTS_FOLDER, data["struct_uid"])
+    if not is_safe_path(STRUCTS_FOLDER, subdir):
+        return
+    struct_path = os.path.join(subdir, str(data["id"]) + ".json")
+
+    # if the struct does not exist, do nothing
+    if not (is_safe_path(subdir, struct_path) or os.path.isfile(struct_path)):
+        print("not", struct_path)
+        return
+
+    # Read the first line of the file to get the number of upvotes
+    upvotes = []
+    score = 0
+    with open(struct_path, 'r+') as f:
+        upvotes = json.loads(f.readline().strip())
+
+        # If the user has not already upvoted this, add them
+        if user['id'] not in upvotes:
+            upvotes.append(user['id'])
+
+            timestamp = f.readline()
+            struct = f.readline()
+            image = f.readline()
+
+            # reset file to top
+            f.seek(0)
+
+            # write file back with updated upvotes
+            f.write(json.dumps(upvotes) + "\n")
+            f.write(timestamp)  # rewrite the timestamp
+            f.write(struct)  # rewrite the actual struct
+            f.write(image)  # rewrite the image
+
+            message = {"uid": data["struct_uid"],
+                       "id": data["id"], "up": user['id'], "score": score}
+
+            f.truncate()  # truncate to ensure flush appropriate
+
+            # calculate score
+            score = score_struct(timestamp, len(upvotes))
+
+            # and then broadcast the new upvote to the room:
+            message = {"uid": data["struct_uid"],
+                       "id": data["id"], "up": user['id'], "score": score}
+            emit("upvote", message, broadcast=True, room="community")
+
+
+@socketio.on('log')
+def handle_log(data):
+    """Receive a log message in the form of {"type": LOG_TYPE, "msg": LOG_OBJECT}
+
+    If the log type is an accept of utterance, then broadcast that to all
+    community-connected clients."""
+
+    if "type" not in data:
+        # If the log object is improper, don't do anything.
+        return
+
+    log(data)
+
+    # If the message is an accept or define type, broadcast it to all
+    # community-connected clients so they can update their display.
+    if data["type"] == "accept":
+        emit("new_accept", {"uid": data['uid'], "query": data["msg"]["query"], "timestamp": current_unix_time()},
+             broadcast=True, room="community")
+    elif data["type"] == "define":
+        emit("new_define", {"uid": data['uid'], "defined": data["msg"]["defineAs"], "timestamp": current_unix_time()},
+             broadcast=True, room="community")
+
+
+@socketio.on('session')
+def get_session(data):
+    """On every new connection, the client should transmit the sessionId to tell
+    the server that a new session has started. This sessionId is then used for
+    all future authentication by storing it as uid in the session global
+    context variable."""
+    session.uid = data['uid']
+    log({"type": "connect", "token": ""})
+
+
+@socketio.on('getstructcount')
+def getstructcount(data):
+    emit_user_structs_count(data['uid'])
+
+
+@socketio.on('connect')
+def connect():
+    """Return an ok if connection worked"""
+    emit('ok', {'data': 'Connected'})
+
+
+@socketio.on('disconnect')
+def disconnect():
+    """Log the fact that a user disconnected."""
+    if 'uid' in session:
+        log({"uid": session.uid, "type": "disconnect"})
+
+
+@socketio.on('sign_in')
+def sign_in(data):
+    r = requests.get("https://slack.com/api/oauth.access", params={
+                     'code': data['code'], 'client_id': '130265636855.151294060356', 'client_secret': SLACK_SECRET})
+    data = r.json()
+    session['access_token'] = data['access_token']
+    user = data['user']
+    session['user'] = user
+
+    encoded = jwt.encode(
+        {'name': user['name'], 'email': user['email'], 'id': user['id']}, JWT_SECRET, algorithm='HS256').decode('utf-8')
+
+    emit('sign_in', {
+         "name": user['name'], "email": user['email'], 'id': user['id'], 'token': encoded})
+
+
+@socketio.on('get_user')
+def get_user(data):
+    user = current_user(data['token'])
+    if user:
+        emit('sign_in', {
+            "name": user['name'], "email": user['email'], 'id': user['id'], 'token': data['token']
+        })
+    else:
+        emit('sign_in_failed')
+
+
+def current_user(token):
+    try:
+        return jwt.decode(token, JWT_SECRET, algorithms=['HS256'])
+    except:
+        return False
+
+
+# http://stackoverflow.com/questions/2301789/read-a-file-in-reverse-order-using-python
+def reverse_readline(filename, buf_size=8192):
+    """a generator that returns the lines of a file in reverse order"""
+    with open(filename) as fh:
+        segment = None
+        offset = 0
+        fh.seek(0, os.SEEK_END)
+        file_size = remaining_size = fh.tell()
+        while remaining_size > 0:
+            offset = min(file_size, offset + buf_size)
+            fh.seek(file_size - offset)
+            buffer = fh.read(min(remaining_size, buf_size))
+            remaining_size -= buf_size
+            lines = buffer.split('\n')
+            # the first line of the buffer is probably not a complete line so
+            # we'll save it and append it to the last line of the next buffer
+            # we read
+            if segment is not None:
+                # if the previous chunk starts right from the beginning of line
+                # do not concact the segment to the last line of new chunk
+                # instead, yield the segment first
+                if buffer[-1] is not '\n':
+                    lines[-1] += segment
+                else:
+                    yield segment
+            segment = lines[0]
+            for index in range(len(lines) - 1, 0, -1):
+                if len(lines[index]):
+                    yield lines[index]
+        # Don't yield None if the file was empty
+        if segment is not None:
+            yield segment
+
+
+def make_dir_if_necessary(dir_name):
+    """Creates the directory if not already created"""
+    if not os.path.exists(dir_name):
+        os.makedirs(dir_name)
+
+
+if __name__ == "__main__":
+    # Create any missing directories
+    make_dir_if_necessary(DATA_FOLDER)
+    make_dir_if_necessary(LOG_FOLDER)
+    make_dir_if_necessary(STRUCTS_FOLDER)
+
+    # Parse arguments
+    parser = OptionParser()
+    parser.add_option("-p", "--port", dest="port",
+                      help="port number to run the server", default=DEFAULT_PORT)
+    (options, args) = parser.parse_args()
+
+    # Run the server
+    # NB: socketio.run uses eventlet to run a production webserver
+    # so, make sure that "eventlet" is installed, or else it will default to
+    # the werkzeug development server which is unsafe and slow.
+    socketio.run(app, host='0.0.0.0', port=int(options.port))

interactive/csscolors.lexicon

@@ -0,0 +1,148 @@
+{"lexeme": "AliceBlue", "formula": "aliceblue", "type": "color"}
+{"lexeme": "AntiqueWhite", "formula": "antiquewhite", "type": "color"}
+{"lexeme": "Aqua", "formula": "aqua", "type": "color"}
+{"lexeme": "Aquamarine", "formula": "aquamarine", "type": "color"}
+{"lexeme": "Azure", "formula": "azure", "type": "color"}
+{"lexeme": "Beige", "formula": "beige", "type": "color"}
+{"lexeme": "Bisque", "formula": "bisque", "type": "color"}
+{"lexeme": "Black", "formula": "black", "type": "color"}
+{"lexeme": "BlanchedAlmond", "formula": "blanchedalmond", "type": "color"}
+{"lexeme": "Blue", "formula": "blue", "type": "color"}
+{"lexeme": "BlueViolet", "formula": "blueviolet", "type": "color"}
+{"lexeme": "Brown", "formula": "brown", "type": "color"}
+{"lexeme": "BurlyWood", "formula": "burlywood", "type": "color"}
+{"lexeme": "CadetBlue", "formula": "cadetblue", "type": "color"}
+{"lexeme": "Chartreuse", "formula": "chartreuse", "type": "color"}
+{"lexeme": "Chocolate", "formula": "chocolate", "type": "color"}
+{"lexeme": "Coral", "formula": "coral", "type": "color"}
+{"lexeme": "CornflowerBlue", "formula": "cornflowerblue", "type": "color"}
+{"lexeme": "Cornsilk", "formula": "cornsilk", "type": "color"}
+{"lexeme": "Crimson", "formula": "crimson", "type": "color"}
+{"lexeme": "Cyan", "formula": "cyan", "type": "color"}
+{"lexeme": "DarkBlue", "formula": "darkblue", "type": "color"}
+{"lexeme": "DarkCyan", "formula": "darkcyan", "type": "color"}
+{"lexeme": "DarkGoldenRod", "formula": "darkgoldenrod", "type": "color"}
+{"lexeme": "DarkGray", "formula": "darkgray", "type": "color"}
+{"lexeme": "DarkGrey", "formula": "darkgrey", "type": "color"}
+{"lexeme": "DarkGreen", "formula": "darkgreen", "type": "color"}
+{"lexeme": "DarkKhaki", "formula": "darkkhaki", "type": "color"}
+{"lexeme": "DarkMagenta", "formula": "darkmagenta", "type": "color"}
+{"lexeme": "DarkOliveGreen", "formula": "darkolivegreen", "type": "color"}
+{"lexeme": "DarkOrange", "formula": "darkorange", "type": "color"}
+{"lexeme": "DarkOrchid", "formula": "darkorchid", "type": "color"}
+{"lexeme": "DarkRed", "formula": "darkred", "type": "color"}
+{"lexeme": "DarkSalmon", "formula": "darksalmon", "type": "color"}
+{"lexeme": "DarkSeaGreen", "formula": "darkseagreen", "type": "color"}
+{"lexeme": "DarkSlateBlue", "formula": "darkslateblue", "type": "color"}
+{"lexeme": "DarkSlateGray", "formula": "darkslategray", "type": "color"}
+{"lexeme": "DarkSlateGrey", "formula": "darkslategrey", "type": "color"}
+{"lexeme": "DarkTurquoise", "formula": "darkturquoise", "type": "color"}
+{"lexeme": "DarkViolet", "formula": "darkviolet", "type": "color"}
+{"lexeme": "DeepPink", "formula": "deeppink", "type": "color"}
+{"lexeme": "DeepSkyBlue", "formula": "deepskyblue", "type": "color"}
+{"lexeme": "DimGray", "formula": "dimgray", "type": "color"}
+{"lexeme": "DimGrey", "formula": "dimgrey", "type": "color"}
+{"lexeme": "DodgerBlue", "formula": "dodgerblue", "type": "color"}
+{"lexeme": "FireBrick", "formula": "firebrick", "type": "color"}
+{"lexeme": "FloralWhite", "formula": "floralwhite", "type": "color"}
+{"lexeme": "ForestGreen", "formula": "forestgreen", "type": "color"}
+{"lexeme": "Fuchsia", "formula": "fuchsia", "type": "color"}
+{"lexeme": "Gainsboro", "formula": "gainsboro", "type": "color"}
+{"lexeme": "GhostWhite", "formula": "ghostwhite", "type": "color"}
+{"lexeme": "Gold", "formula": "gold", "type": "color"}
+{"lexeme": "GoldenRod", "formula": "goldenrod", "type": "color"}
+{"lexeme": "Gray", "formula": "gray", "type": "color"}
+{"lexeme": "Grey", "formula": "grey", "type": "color"}
+{"lexeme": "Green", "formula": "green", "type": "color"}
+{"lexeme": "GreenYellow", "formula": "greenyellow", "type": "color"}
+{"lexeme": "HoneyDew", "formula": "honeydew", "type": "color"}
+{"lexeme": "HotPink", "formula": "hotpink", "type": "color"}
+{"lexeme": "IndianRed", "formula": "indianred", "type": "color"}
+{"lexeme": "Indigo", "formula": "indigo", "type": "color"}
+{"lexeme": "Ivory", "formula": "ivory", "type": "color"}
+{"lexeme": "Khaki", "formula": "khaki", "type": "color"}
+{"lexeme": "Lavender", "formula": "lavender", "type": "color"}
+{"lexeme": "LavenderBlush", "formula": "lavenderblush", "type": "color"}
+{"lexeme": "LawnGreen", "formula": "lawngreen", "type": "color"}
+{"lexeme": "LemonChiffon", "formula": "lemonchiffon", "type": "color"}
+{"lexeme": "LightBlue", "formula": "lightblue", "type": "color"}
+{"lexeme": "LightCoral", "formula": "lightcoral", "type": "color"}
+{"lexeme": "LightCyan", "formula": "lightcyan", "type": "color"}
+{"lexeme": "LightGoldenRodYellow", "formula": "lightgoldenrodyellow", "type": "color"}
+{"lexeme": "LightGray", "formula": "lightgray", "type": "color"}
+{"lexeme": "LightGrey", "formula": "lightgrey", "type": "color"}
+{"lexeme": "LightGreen", "formula": "lightgreen", "type": "color"}
+{"lexeme": "LightPink", "formula": "lightpink", "type": "color"}
+{"lexeme": "LightSalmon", "formula": "lightsalmon", "type": "color"}
+{"lexeme": "LightSeaGreen", "formula": "lightseagreen", "type": "color"}
+{"lexeme": "LightSkyBlue", "formula": "lightskyblue", "type": "color"}
+{"lexeme": "LightSlateGray", "formula": "lightslategray", "type": "color"}
+{"lexeme": "LightSlateGrey", "formula": "lightslategrey", "type": "color"}
+{"lexeme": "LightSteelBlue", "formula": "lightsteelblue", "type": "color"}
+{"lexeme": "LightYellow", "formula": "lightyellow", "type": "color"}
+{"lexeme": "Lime", "formula": "lime", "type": "color"}
+{"lexeme": "LimeGreen", "formula": "limegreen", "type": "color"}
+{"lexeme": "Linen", "formula": "linen", "type": "color"}
+{"lexeme": "Magenta", "formula": "magenta", "type": "color"}
+{"lexeme": "Maroon", "formula": "maroon", "type": "color"}
+{"lexeme": "MediumAquaMarine", "formula": "mediumaquamarine", "type": "color"}
+{"lexeme": "MediumBlue", "formula": "mediumblue", "type": "color"}
+{"lexeme": "MediumOrchid", "formula": "mediumorchid", "type": "color"}
+{"lexeme": "MediumPurple", "formula": "mediumpurple", "type": "color"}
+{"lexeme": "MediumSeaGreen", "formula": "mediumseagreen", "type": "color"}
+{"lexeme": "MediumSlateBlue", "formula": "mediumslateblue", "type": "color"}
+{"lexeme": "MediumSpringGreen", "formula": "mediumspringgreen", "type": "color"}
+{"lexeme": "MediumTurquoise", "formula": "mediumturquoise", "type": "color"}
+{"lexeme": "MediumVioletRed", "formula": "mediumvioletred", "type": "color"}
+{"lexeme": "MidnightBlue", "formula": "midnightblue", "type": "color"}
+{"lexeme": "MintCream", "formula": "mintcream", "type": "color"}
+{"lexeme": "MistyRose", "formula": "mistyrose", "type": "color"}
+{"lexeme": "Moccasin", "formula": "moccasin", "type": "color"}
+{"lexeme": "NavajoWhite", "formula": "navajowhite", "type": "color"}
+{"lexeme": "Navy", "formula": "navy", "type": "color"}
+{"lexeme": "OldLace", "formula": "oldlace", "type": "color"}
+{"lexeme": "Olive", "formula": "olive", "type": "color"}
+{"lexeme": "OliveDrab", "formula": "olivedrab", "type": "color"}
+{"lexeme": "Orange", "formula": "orange", "type": "color"}
+{"lexeme": "OrangeRed", "formula": "orangered", "type": "color"}
+{"lexeme": "Orchid", "formula": "orchid", "type": "color"}
+{"lexeme": "PaleGoldenRod", "formula": "palegoldenrod", "type": "color"}
+{"lexeme": "PaleGreen", "formula": "palegreen", "type": "color"}
+{"lexeme": "PaleTurquoise", "formula": "paleturquoise", "type": "color"}
+{"lexeme": "PaleVioletRed", "formula": "palevioletred", "type": "color"}
+{"lexeme": "PapayaWhip", "formula": "papayawhip", "type": "color"}
+{"lexeme": "PeachPuff", "formula": "peachpuff", "type": "color"}
+{"lexeme": "Peru", "formula": "peru", "type": "color"}
+{"lexeme": "Pink", "formula": "pink", "type": "color"}
+{"lexeme": "Plum", "formula": "plum", "type": "color"}
+{"lexeme": "PowderBlue", "formula": "powderblue", "type": "color"}
+{"lexeme": "Purple", "formula": "purple", "type": "color"}
+{"lexeme": "RebeccaPurple", "formula": "rebeccapurple", "type": "color"}
+{"lexeme": "Red", "formula": "red", "type": "color"}
+{"lexeme": "RosyBrown", "formula": "rosybrown", "type": "color"}
+{"lexeme": "RoyalBlue", "formula": "royalblue", "type": "color"}
+{"lexeme": "SaddleBrown", "formula": "saddlebrown", "type": "color"}
+{"lexeme": "Salmon", "formula": "salmon", "type": "color"}
+{"lexeme": "SandyBrown", "formula": "sandybrown", "type": "color"}
+{"lexeme": "SeaGreen", "formula": "seagreen", "type": "color"}
+{"lexeme": "SeaShell", "formula": "seashell", "type": "color"}
+{"lexeme": "Sienna", "formula": "sienna", "type": "color"}
+{"lexeme": "Silver", "formula": "silver", "type": "color"}
+{"lexeme": "SkyBlue", "formula": "skyblue", "type": "color"}
+{"lexeme": "SlateBlue", "formula": "slateblue", "type": "color"}
+{"lexeme": "SlateGray", "formula": "slategray", "type": "color"}
+{"lexeme": "SlateGrey", "formula": "slategrey", "type": "color"}
+{"lexeme": "Snow", "formula": "snow", "type": "color"}
+{"lexeme": "SpringGreen", "formula": "springgreen", "type": "color"}
+{"lexeme": "SteelBlue", "formula": "steelblue", "type": "color"}
+{"lexeme": "Tan", "formula": "tan", "type": "color"}
+{"lexeme": "Teal", "formula": "teal", "type": "color"}
+{"lexeme": "Thistle", "formula": "thistle", "type": "color"}
+{"lexeme": "Tomato", "formula": "tomato", "type": "color"}
+{"lexeme": "Turquoise", "formula": "turquoise", "type": "color"}
+{"lexeme": "Violet", "formula": "violet", "type": "color"}
+{"lexeme": "Wheat", "formula": "wheat", "type": "color"}
+{"lexeme": "White", "formula": "white", "type": "color"}
+{"lexeme": "WhiteSmoke", "formula": "whitesmoke", "type": "color"}
+{"lexeme": "Yellow", "formula": "yellow", "type": "color"}
+{"lexeme": "YellowGreen", "formula": "yellowgreen", "type": "color"}

interactive/dal.grammar

@@ -0,0 +1,104 @@
+# Grammar for Dependency-based Action Language (DAL)
+# which is based on lambda DCS, and designed to interpolate
+# well with NL
+
+(def @select select)
+(def @update update)
+(def @remove remove)
+(def @R reverse)
+
+(rule $ROOT ($Actions) (IdentityFn) (anchored 1))
+(rule $ROOT ($Actions) (interactive.BlockFn isolate) (anchored 1))
+(rule $ROOT ($Actions) (interactive.BlockFn block) (anchored 1))
+
+(rule $Actions ($Actions ; $Action) (lambda a1 (lambda a2 (:s (var a1) (var a2)))) (anchored 1))
+(rule $Actions ($Action) (IdentityFn) (anchored 1))
+(rule $Action ({ $Actions }) (lambda a (:blk (var a))) (anchored 1))
+(rule $Action (return { $Actions }) (lambda a (:blkr (var a))) (anchored 1))
+(rule $Action ([ $Actions ]) (IdentityFn) (anchored 1))
+
+(rule $Action (repeat $Numbers $Action) (lambda n (lambda a (:loop (var n) (var a)))) (anchored 1))
+(rule $Action (if $ValueSets $Action) (lambda b (lambda a (:if (var b) (var a)))) (anchored 1))
+(rule $Action (if $Sets $Action) (lambda b (lambda a (:if (var b) (var a)))) (anchored 1))
+
+# basic actions, local scope "for has color red [add red left; add red top]"
+# (rule $Action (for $Sets $Action ) (lambda s (lambda a (:for (var s) (var a)))) (anchored 1))
+(rule $Action (foreach $Sets $Action ) (lambda s (lambda a (:foreach (var s) (var a)))) (anchored 1))
+(rule $Action (foreach $Action ) (lambda a (:foreach this (var a))) (anchored 1))
+(rule $Action (isolate $Action ) (lambda a (:isolate (var a))) (anchored 1))
+(rule $Action (while $Sets $Action ) (lambda s (lambda a (:while (var s) (var a)))) (anchored 1))
+
+# global actions
+(rule $Action (select $Sets) (lambda s (: @select (var s))) (anchored 1))
+(rule $Action (select) (ConstantFn (: @select)) (anchored 1))
+(rule $Action (update $Rel $ValueSets) (lambda r (lambda v (: @update (var r) (var v)))) (anchored 1))
+(rule $Action (remove) (ConstantFn (: @remove)) (anchored 1))
+(rule $Action (remove $Sets) (lambda s (: @remove (var s))) (anchored 1))
+
+############### unary sets
+(rule $Set (selected) this (anchored 1))
+(rule $Set (previous) prev (anchored 1))
+(rule $Set (this) this (anchored 1))
+(rule $Set (all) * (anchored 1))
+(rule $Set (nothing) nothing (anchored 1))
+(rule $Set (none) nothing (anchored 1))
+
+############### basic set operations
+# just use or and filter, which does and
+(for @op (or and)
+(rule $Sets ($Sets @op $Set) (lambda s1 (lambda s2
+					(@op (var s1) (var s2)))) (anchored 1))
+(rule $ValueSets ($ValueSets @op $ValueSet) (lambda s1 (lambda s2
+					(@op (var s1) (var s2)))) (anchored 1))
+)
+
+# this would require universe of each type
+# (rule $ValueSet (not $ValueSet) (lambda s (not (var s))) (anchored 1))
+
+(rule $Set (not $Sets) (lambda s (not (var s))) (anchored 1))
+
+(rule $ValueSets ($ValueSet) (IdentityFn) (anchored 1))
+(rule $ValueSet ([ $ValueSets ]) (IdentityFn) (anchored 1))
+
+(rule $Sets ($Set) (IdentityFn) (anchored 1))
+(rule $Set ([ $Sets ]) (IdentityFn) (anchored 1))
+
+############### Joins
+(rule $ValueSet ($Rel of $Set) (lambda r (lambda s
+			   ((@R (var r)) (var s)))) (anchored 1))
+
+(rule $Set (has $Rel $ValueSet) (lambda r (lambda s
+			   ((var r) (var s)))) (anchored 1))
+
+############### Arithmetic
+(for @op (+ -)
+(rule $Numbers ($Numbers @op $Number) (lambda n1  (lambda n2
+(@op (var n1) (var n2)))) (anchored 1))
+)
+
+(rule $Number ([ $Numbers ]) (IdentityFn) (anchored 1))
+(rule $Numbers ( $Number ) (IdentityFn) (anchored 1))
+
+#############################
+# Floating rules
+#############################
+
+# (rule $Action ($Numbers $Actions) (lambda n (lambda a (:loop (var n) (var a)))) (floating 1)) # do add red top 3 times
+# (rule $Action ($Sets $Actions) (lambda s (lambda a (:foreach (var s) (var a)))) (floating 1))
+# (rule $Action ($Actions $Actions) (lambda a1 (lambda a2 (:s (var a1) (var a2)))) (floating 1)) # add red top then remove them
+
+# let floating rules continue to propagate, and generate a lot of candidates
+# (rule $Actions ($Action) (IdentityFn) (floating 1))
+
+# (rule $ValueSets ($Rel $Sets) (lambda r (lambda s
+#			   ((reverse (var r)) (var s)))) (floating 1))
+
+# (rule $Action ($Sets) (lambda s (: @select (var s))) (floating 1))
+
+# (rule $Set ($Rel $ValueSets) (lambda r (lambda s
+# 			   ((var r) (var s)))) (floating 1))
+# (rule $Action ($Rel $ValueSets) (lambda r (lambda v (: @update (var r) (var v)))) (floating 1))
+# (rule $Action (nothing) (ConstantFn (: @remove)) (floating 1))
+
+# (rule $FROOT ($Action) (IdentityFn) (floating 1))
+# (rule $FROOT ($Actions) (IdentityFn) (floating 1))

interactive/queries/README.md

@@ -0,0 +1,23 @@
+# README
+
+## Processed
+
+* freebuild.def.json.gz contains just the 2495 definition queries in freebuild.json.gz
+
+        gzcat freebuild.json.gz | grep '(:def'
+
+* freebuild.json.gz is the main data file. Obtained by:
+
+      awk '{print "{\"qid\":" NR "," substr($0,2)}' freebuildbig-0206 > freebuild.id.json
+      jq -c '{"qid":.qid, "q":.q, "sessionId": .sessionId[:10], "time":.time}' freebuild.id.json > freebuild.json
+
+
+## Raw queries
+
+* freebuildbig-0206.def is the raw query log, without context
+
+* qualifier3-0201: 30 turkers, 1 rejection
+
+* qualifiers: turkers had to build a fixed target
+
+* freebuild[12]: qualified turkers can build whatever they want

interactive/queries/sidaw.json

@@ -0,0 +1,185 @@
+{"time":"2017-01-22T00:14:52.709","sessionId":"sidaw","q":"(:q \"add red top 3 times\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.739","sessionId":"sidaw","q":"(:q \"add red top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.786","sessionId":"sidaw","q":"(:accept \"add red top\" \"(: add red top)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.797","sessionId":"sidaw","q":"(:q \"add red top 3 times\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.805","sessionId":"sidaw","q":"(:q \"repeat 3 [add red top]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.812","sessionId":"sidaw","q":"(:accept \"repeat 3 [add red top]\" \"(:loop (number 3) (: add red top))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.820","sessionId":"sidaw","q":"(:def_ret \"add red top 3 times\" \"[[\\\"repeat 3 [add red top]\\\",\\\"(:loop (number 3) (: add red top))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.852","sessionId":"sidaw","q":"(:q \"add cardinal\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.858","sessionId":"sidaw","q":"(:q \"add red\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.863","sessionId":"sidaw","q":"(:accept \"add red\" \"(: add red here)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.868","sessionId":"sidaw","q":"(:def_ret \"add cardinal\" \"[[\\\"add red\\\",\\\"(: add red here)\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.884","sessionId":"sidaw","q":"(:q \"yellow stick size 5\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.889","sessionId":"sidaw","q":"(:q \"repeat 5 [ add yellow; select top]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.898","sessionId":"sidaw","q":"(:accept \"repeat 5 [ add yellow; select top]\" \"(:loop (number 5) (:s (: add yellow here) (: select (call adj top))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.905","sessionId":"sidaw","q":"(:def_ret \"yellow stick size 5\" \"[[\\\"repeat 5 [ add yellow; select top]\\\",\\\"(:loop (number 5) (:s (: add yellow here) (: select (call adj top))))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.922","sessionId":"sidaw","q":"(:q \"yellow beam left 5\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.929","sessionId":"sidaw","q":"(:q \"{repeat  5 [ add yellow; select left]}\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.939","sessionId":"sidaw","q":"(:accept \"{repeat  5 [ add yellow; select left]}\" \"(:blk (:loop (number 5) (:s (: add yellow here) (: select (call adj left)))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.947","sessionId":"sidaw","q":"(:def_ret \"yellow beam left 5\" \"[[\\\"{repeat  5 [ add yellow; select left]}\\\",\\\"(:blk (:loop (number 5) (:s (: add yellow here) (: select (call adj left)))))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.962","sessionId":"sidaw","q":"(:q \"move down\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.968","sessionId":"sidaw","q":"(:q \"move bot\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.978","sessionId":"sidaw","q":"(:q \"select bot\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.984","sessionId":"sidaw","q":"(:accept \"select bot\" \"(: select (call adj bot))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.990","sessionId":"sidaw","q":"(:def_ret \"move down\" \"[[\\\"select bot\\\",\\\"(: select (call adj bot))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:52.999","sessionId":"sidaw","q":"(:q \"move down\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.006","sessionId":"sidaw","q":"(:q \"move bot\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.012","sessionId":"sidaw","q":"(:q \"move right\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.015","sessionId":"sidaw","q":"(:accept \"move right\" \"(: move right)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.021","sessionId":"sidaw","q":"(:q \"move up\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.028","sessionId":"sidaw","q":"(:q \"move top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.032","sessionId":"sidaw","q":"(:accept \"move top\" \"(: move top)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.036","sessionId":"sidaw","q":"(:def_ret \"move up\" \"[[\\\"move top\\\",\\\"(: move top)\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.046","sessionId":"sidaw","q":"(:q \"move up\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.051","sessionId":"sidaw","q":"(:accept \"move up\" \"(: move top)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.057","sessionId":"sidaw","q":"(:accept \"move up\" \"(not *)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.063","sessionId":"sidaw","q":"(:q \"select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.066","sessionId":"sidaw","q":"(:accept \"select top\" \"(: select (call adj top))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.070","sessionId":"sidaw","q":"(:def_ret \"move up\" \"[[\\\"select top\\\",\\\"(: select (call adj top))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.081","sessionId":"sidaw","q":"(:q \"move up\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.090","sessionId":"sidaw","q":"(:q \"select yellow\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.096","sessionId":"sidaw","q":"(:q \"select has color yellow\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.102","sessionId":"sidaw","q":"(:accept \"select has color yellow\" \"(: select (color yellow))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.106","sessionId":"sidaw","q":"(:def_ret \"select yellow\" \"[[\\\"select has color yellow\\\",\\\"(: select (color yellow))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.115","sessionId":"sidaw","q":"(:q \"update color red\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.119","sessionId":"sidaw","q":"(:accept \"update color red\" \"(: update color red)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.124","sessionId":"sidaw","q":"(:q \"select origin\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.128","sessionId":"sidaw","q":"(:accept \"select origin\" \"(: select (call origin))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.135","sessionId":"sidaw","q":"(:q \"select red\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.139","sessionId":"sidaw","q":"(:accept \"select red\" \"(: select (color red))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.148","sessionId":"sidaw","q":"(:q \"foreach red [move right]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.155","sessionId":"sidaw","q":"(:q \"foreach has color red [move right]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.162","sessionId":"sidaw","q":"(:accept \"foreach has color red [move right]\" \"(:foreach (color red) (: move right))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.169","sessionId":"sidaw","q":"(:def_ret \"foreach red [move right]\" \"[[\\\"foreach has color red [move right]\\\",\\\"(:foreach (color red) (: move right))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.182","sessionId":"sidaw","q":"(:q \"remove red\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.187","sessionId":"sidaw","q":"(:q \"remove has color red\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.191","sessionId":"sidaw","q":"(:accept \"remove has color red\" \"(: remove (color red))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.195","sessionId":"sidaw","q":"(:def_ret \"remove red\" \"[[\\\"remove has color red\\\",\\\"(: remove (color red))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.204","sessionId":"sidaw","q":"(:q \"select none\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.207","sessionId":"sidaw","q":"(:accept \"select none\" \"(: select nothing)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.213","sessionId":"sidaw","q":"(:q \"select origin\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.216","sessionId":"sidaw","q":"(:accept \"select origin\" \"(: select (call origin))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.220","sessionId":"sidaw","q":"(:q \"yellow sq size 3\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.225","sessionId":"sidaw","q":"(:q \"yellow beam back 3\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.230","sessionId":"sidaw","q":"(:q \"repeat 3 [yellow beam back 3; select left]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.238","sessionId":"sidaw","q":"(:accept \"repeat 3 [yellow beam back 3; select left]\" \"(:loop (number 3) (:s (:blk (:loop (number 3) (:s (: add yellow here) (: select (call adj back))))) (: select (call adj left))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.250","sessionId":"sidaw","q":"(:q \"select front\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.255","sessionId":"sidaw","q":"(:accept \"select front\" \"(: select (call adj front))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.259","sessionId":"sidaw","q":"(:def \"yellow sq size 3\" \"[[\\\"repeat 3 [yellow beam back 3; select left]\\\",\\\"(:loop (number 3) (:s (:blk (:loop (number 3) (:s (: add yellow here) (: select (call adj back))))) (: select (call adj left))))\\\"],[\\\"select front\\\",\\\"(: select (call adj front))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.276","sessionId":"sidaw","q":"(:q \"yellow sq size 3\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.281","sessionId":"sidaw","q":"(:accept \"yellow sq size 3\" \"(:blk (:loop (number 3) (:s (:blk (:loop (number 3) (:s (: add yellow here) (: select (call adj back))))) (: select (call adj left)))) (: select (call adj front)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.291","sessionId":"sidaw","q":"(:q \"select origin\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.295","sessionId":"sidaw","q":"(:accept \"select origin\" \"(: select (call origin))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.303","sessionId":"sidaw","q":"(:q \"red sq size 4\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.311","sessionId":"sidaw","q":"(:accept \"red sq size 4\" \"(:blk (:loop (number 4) (:s (:blk (:loop (number 4) (:s (: add red here) (: select (call adj back))))) (: select (call adj left)))) (: select (call adj front)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.327","sessionId":"sidaw","q":"(:q \"red cube size 4\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.331","sessionId":"sidaw","q":"(:q \"red sq size 4\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.337","sessionId":"sidaw","q":"(:accept \"red sq size 4\" \"(:blk (:loop (number 4) (:s (:blk (:loop (number 4) (:s (: add red here) (: select (call adj back))))) (: select (call adj left)))) (: select (call adj front)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.345","sessionId":"sidaw","q":"(:q \"repeat 4-1 [select top; red sq size 4]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.358","sessionId":"sidaw","q":"(:accept \"repeat 4-1 [select top; red sq size 4]\" \"(:loop (- (number 4) (number 1)) (:s (: select (call adj top)) (:blk (:loop (number 4) (:s (:blk (:loop (number 4) (:s (: add red here) (: select (call adj back))))) (: select (call adj left)))) (: select (call adj front)))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.377","sessionId":"sidaw","q":"(:def \"red cube size 4\" \"[[\\\"red sq size 4\\\",\\\"(:blk (:loop (number 4) (:s (:blk (:loop (number 4) (:s (: add red here) (: select (call adj back))))) (: select (call adj left)))) (: select (call adj front)))\\\"],[\\\"repeat 4-1 [select top; red sq size 4]\\\",\\\"(:loop (- (number 4) (number 1)) (:s (: select (call adj top)) (:blk (:loop (number 4) (:s (:blk (:loop (number 4) (:s (: add red here) (: select (call adj back))))) (: select (call adj left)))) (: select (call adj front)))))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.412","sessionId":"sidaw","q":"(:q \"brown cube size 4\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.436","sessionId":"sidaw","q":"(:q \"brown cube size 5\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.462","sessionId":"sidaw","q":"(:q \"move back\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.471","sessionId":"sidaw","q":"(:accept \"move back\" \"(: move back)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.490","sessionId":"sidaw","q":"(:q \"select very back of very top of red\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.498","sessionId":"sidaw","q":"(:q \"select very back of very top of has color red\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.508","sessionId":"sidaw","q":"(:accept \"select very back of very top of has color red\" \"(: select (call veryx back (call veryx top (color red))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.522","sessionId":"sidaw","q":"(:q \"add blue top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.528","sessionId":"sidaw","q":"(:accept \"add blue top\" \"(: add blue top)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.542","sessionId":"sidaw","q":"(:q \"select blue\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.549","sessionId":"sidaw","q":"(:accept \"select blue\" \"(: select (color blue))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.561","sessionId":"sidaw","q":"(:q \"select this and not very left and not very right\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.573","sessionId":"sidaw","q":"(:accept \"select this and not very left and not very right\" \"(: select (and (and this (not (call veryx left))) (not (call veryx right))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.593","sessionId":"sidaw","q":"(:q \"add yellow top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.599","sessionId":"sidaw","q":"(:accept \"add yellow top\" \"(: add yellow top)\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.605","sessionId":"sidaw","q":"(:q \"brown beam top 5\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.609","sessionId":"sidaw","q":"(:q \"brown beam top 10\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.613","sessionId":"sidaw","q":"(:q \"brown beam top 12\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.616","sessionId":"sidaw","q":"(:q \"brown beam top 9\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.621","sessionId":"sidaw","q":"(:q \"brown beam top 10\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.624","sessionId":"sidaw","q":"(:accept \"brown beam top 10\" \"(:blk (:loop (number 10) (:s (: add brown here) (: select (call adj top)))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.630","sessionId":"sidaw","q":"(:q \"green branch 3\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.637","sessionId":"sidaw","q":"(:q \"repeat 3 [add green left; add green right; add green front; add green back]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.644","sessionId":"sidaw","q":"(:accept \"repeat 3 [add green left; add green right; add green front; add green back]\" \"(:loop (number 3) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.651","sessionId":"sidaw","q":"(:def_ret \"green branch 3\" \"[[\\\"repeat 3 [add green left; add green right; add green front; add green back]\\\",\\\"(:loop (number 3) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back)))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.664","sessionId":"sidaw","q":"(:q \"select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.666","sessionId":"sidaw","q":"(:accept \"select top\" \"(: select (call adj top))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.672","sessionId":"sidaw","q":"(:q \"green branch 3\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.678","sessionId":"sidaw","q":"(:q \"green branch 4\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.682","sessionId":"sidaw","q":"(:accept \"green branch 4\" \"(:loop (number 4) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.695","sessionId":"sidaw","q":"(:q \"select top of top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.699","sessionId":"sidaw","q":"(:accept \"select top of top\" \"(: select (call adj top (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.704","sessionId":"sidaw","q":"(:q \"yellow branch 5\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.712","sessionId":"sidaw","q":"(:q \"green branch 5\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.717","sessionId":"sidaw","q":"(:accept \"green branch 5\" \"(:loop (number 5) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.728","sessionId":"sidaw","q":"(:q \"select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.734","sessionId":"sidaw","q":"(:accept \"select top\" \"(: select (call adj top))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.745","sessionId":"sidaw","q":"(:q \"green branch 3\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.751","sessionId":"sidaw","q":"(:accept \"green branch 3\" \"(:loop (number 3) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.763","sessionId":"sidaw","q":"(:q \"select top of top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.769","sessionId":"sidaw","q":"(:accept \"select top of top\" \"(: select (call adj top (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.780","sessionId":"sidaw","q":"(:q \"green branch 2\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.785","sessionId":"sidaw","q":"(:accept \"green branch 2\" \"(:loop (number 2) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.801","sessionId":"sidaw","q":"(:q \"select top of top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.808","sessionId":"sidaw","q":"(:accept \"select top of top\" \"(: select (call adj top (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.818","sessionId":"sidaw","q":"(:q \"green branch\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.826","sessionId":"sidaw","q":"(:q \"green branch 1\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.831","sessionId":"sidaw","q":"(:accept \"green branch 1\" \"(:loop (number 1) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.839","sessionId":"sidaw","q":"(:q \"brown beam top 10\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.844","sessionId":"sidaw","q":"(:accept \"brown beam top 10\" \"(:blk (:loop (number 10) (:s (: add brown here) (: select (call adj top)))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.850","sessionId":"sidaw","q":"(:q \"green branch 1; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.854","sessionId":"sidaw","q":"(:accept \"green branch 1; select top\" \"(:s (:loop (number 1) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.862","sessionId":"sidaw","q":"(:q \"green branch 2; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.866","sessionId":"sidaw","q":"(:accept \"green branch 2; select top\" \"(:s (:loop (number 2) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.872","sessionId":"sidaw","q":"(:q \"green branch 5; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.876","sessionId":"sidaw","q":"(:accept \"green branch 5; select top\" \"(:s (:loop (number 5) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.885","sessionId":"sidaw","q":"(:q \"green branch 4; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.891","sessionId":"sidaw","q":"(:accept \"green branch 4; select top\" \"(:s (:loop (number 4) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.903","sessionId":"sidaw","q":"(:q \"green branch 3; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.909","sessionId":"sidaw","q":"(:accept \"green branch 3; select top\" \"(:s (:loop (number 3) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.922","sessionId":"sidaw","q":"(:q \"green branch 2; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.927","sessionId":"sidaw","q":"(:accept \"green branch 2; select top\" \"(:s (:loop (number 2) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.941","sessionId":"sidaw","q":"(:q \"green branch 1; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.946","sessionId":"sidaw","q":"(:accept \"green branch 1; select top\" \"(:s (:loop (number 1) (:s (:s (:s (: add green left) (: add green right)) (: add green front)) (: add green back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.960","sessionId":"sidaw","q":"(:q \"red branch 1; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.966","sessionId":"sidaw","q":"(:accept \"red branch 1; select top\" \"(:s (:loop (number 1) (:s (:s (:s (: add red left) (: add red right)) (: add red front)) (: add red back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.980","sessionId":"sidaw","q":"(:q \"red branch 2; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:53.988","sessionId":"sidaw","q":"(:accept \"red branch 2; select top\" \"(:s (:loop (number 2) (:s (:s (:s (: add red left) (: add red right)) (: add red front)) (: add red back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:54.004","sessionId":"sidaw","q":"(:q \"red branch 3; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:54.012","sessionId":"sidaw","q":"(:accept \"red branch 3; select top\" \"(:s (:loop (number 3) (:s (:s (:s (: add red left) (: add red right)) (: add red front)) (: add red back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:54.030","sessionId":"sidaw","q":"(:q \"red branch 4; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:54.041","sessionId":"sidaw","q":"(:accept \"red branch 4; select top\" \"(:s (:loop (number 4) (:s (:s (:s (: add red left) (: add red right)) (: add red front)) (: add red back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:54.060","sessionId":"sidaw","q":"(:q \"red branch 5; select top\")","remote":"localhost"}
+{"time":"2017-01-22T00:14:54.073","sessionId":"sidaw","q":"(:accept \"red branch 5; select top\" \"(:s (:loop (number 5) (:s (:s (:s (: add red left) (: add red right)) (: add red front)) (: add red back))) (: select (call adj top)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:16:42.444","sessionId":"w3nFZG1uYN","q":"(:q \"while [has color red] [select up; add yellow]\")","remote":"localhost"}
+{"time":"2017-01-22T00:16:45.576","sessionId":"w3nFZG1uYN","q":"(:accept \"while [has color red] [select up; add yellow]\" \"(:while (color red) (:s (: select (call adj top)) (: add yellow here)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:17:16.687","sessionId":"w3nFZG1uYN","q":"(:q \"while [this and has color red] [select up; add yellow]\")","remote":"localhost"}
+{"time":"2017-01-22T00:17:25.866","sessionId":"w3nFZG1uYN","q":"(:q \"red cube size 5\")","remote":"localhost"}
+{"time":"2017-01-22T00:17:26.172","sessionId":"w3nFZG1uYN","q":"(:accept \"red cube size 5\" \"(:blk (:blk (:loop (number 5) (:s (:blk (:loop (number 5) (:s (: add red here) (: select (call adj back))))) (: select (call adj left)))) (: select (call adj front))) (:loop (- (number 5) (number 1)) (:s (: select (call adj top)) (:blk (:loop (number 5) (:s (:blk (:loop (number 5) (:s (: add red here) (: select (call adj back))))) (: select (call adj left)))) (: select (call adj front))))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:17:27.580","sessionId":"w3nFZG1uYN","q":"(:q \"while [this and has color red] [select up; add yellow]\")","remote":"localhost"}
+{"time":"2017-01-22T00:17:42.205","sessionId":"w3nFZG1uYN","q":"(:q \"while [this and has color red] [add yellow; select up]\")","remote":"localhost"}
+{"time":"2017-01-22T00:17:43.220","sessionId":"w3nFZG1uYN","q":"(:accept \"while [this and has color red] [add yellow; select up]\" \"(:while (and this (color red)) (:s (: add yellow here) (: select (call adj top))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:19:24.781","sessionId":"w3nFZG1uYN","q":"(:q \"replace the red with yellow\")","remote":"localhost"}
+{"time":"2017-01-22T00:19:33.117","sessionId":"w3nFZG1uYN","q":"(:def_ret \"replace the red with yellow\" \"[[\\\"while [this and has color red] [add yellow; select up]\\\",\\\"(:while (and this (color red)) (:s (: add yellow here) (: select (call adj top))))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:19:37.579","sessionId":"w3nFZG1uYN","q":"(:q \"select origin\")","remote":"localhost"}
+{"time":"2017-01-22T00:19:38.099","sessionId":"w3nFZG1uYN","q":"(:accept \"select origin\" \"(: select (call origin))\")","remote":"localhost"}
+{"time":"2017-01-22T00:19:41.179","sessionId":"w3nFZG1uYN","q":"(:q \"select right of right\")","remote":"localhost"}
+{"time":"2017-01-22T00:19:45.019","sessionId":"w3nFZG1uYN","q":"(:q \"select left of left\")","remote":"localhost"}
+{"time":"2017-01-22T00:19:45.439","sessionId":"w3nFZG1uYN","q":"(:accept \"select left of left\" \"(: select (call adj left (call adj left)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:19:52.202","sessionId":"w3nFZG1uYN","q":"(:q \"replace the red with orange\")","remote":"localhost"}
+{"time":"2017-01-22T00:19:52.897","sessionId":"w3nFZG1uYN","q":"(:accept \"replace the red with orange\" \"(:while (and this (color red)) (:s (: add orange here) (: select (call adj top))))\")","remote":"localhost"}
+{"time":"2017-01-22T00:20:50.449","sessionId":"w3nFZG1uYN","q":"(:q \"add red\")","remote":"localhost"}
+{"time":"2017-01-22T00:20:50.777","sessionId":"w3nFZG1uYN","q":"(:accept \"add red\" \"(: add red here)\")","remote":"localhost"}
+{"time":"2017-01-22T00:20:55.194","sessionId":"w3nFZG1uYN","q":"(:q \"select left 6 times\")","remote":"localhost"}
+{"time":"2017-01-22T00:20:55.638","sessionId":"w3nFZG1uYN","q":"(:accept \"select left 6 times\" \"(:loop (number 6) (: select (call adj left)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:21:07.717","sessionId":"w3nFZG1uYN","q":"(:q \"move right to the red block\")","remote":"localhost"}
+{"time":"2017-01-22T00:21:38.542","sessionId":"w3nFZG1uYN","q":"(:q \"while [has color red and not this] [move right]\")","remote":"localhost"}
+{"time":"2017-01-22T00:21:45.078","sessionId":"w3nFZG1uYN","q":"(:q \"while [has color red and not this] [select right]\")","remote":"localhost"}
+{"time":"2017-01-22T00:21:46.845","sessionId":"w3nFZG1uYN","q":"(:accept \"while [has color red and not this] [select right]\" \"(:while (and (color red) (not this)) (: select (call adj right)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:21:53.183","sessionId":"w3nFZG1uYN","q":"(:def_ret \"move right to the red block\" \"[[\\\"while [has color red and not this] [select right]\\\",\\\"(:while (and (color red) (not this)) (: select (call adj right)))\\\"]]\")","remote":"localhost"}
+{"time":"2017-01-22T00:21:58.891","sessionId":"w3nFZG1uYN","q":"(:q \"move left 3 times\")","remote":"localhost"}
+{"time":"2017-01-22T00:22:03.707","sessionId":"w3nFZG1uYN","q":"(:q \"select left 3 times\")","remote":"localhost"}
+{"time":"2017-01-22T00:22:04.077","sessionId":"w3nFZG1uYN","q":"(:accept \"select left 3 times\" \"(:loop (number 3) (: select (call adj left)))\")","remote":"localhost"}
+{"time":"2017-01-22T00:22:07.389","sessionId":"w3nFZG1uYN","q":"(:q \"move right to the red block\")","remote":"localhost"}
+{"time":"2017-01-22T00:22:07.797","sessionId":"w3nFZG1uYN","q":"(:accept \"move right to the red block\" \"(:while (and (color red) (not this)) (: select (call adj right)))\")","remote":"localhost"}

interactive/run

@@ -0,0 +1,233 @@
+#!/usr/bin/env ruby
+
+# This is the main entry point for running SHRDLURN.  See
+# fig/lib/execrunner.rb for more documentation for how commands are generated.
+# There are a bunch of modes that this script can be invoked with, which
+# loosely correspond to the modules.
+
+$: << 'fig/lib'
+require 'execrunner'
+$optPrefix = '-'
+$path = 'interactive'
+$output = 'interactive/output'
+$modes = []
+def addMode(name, description, func)
+  $modes << [name, description, func]
+end
+
+def codalab(dependencies=nil)
+  # Set @cl=1 to run job on CodaLab
+  dependencies ||= l(':fig', ':lib', ':module-classes.txt', ':libsempre')
+  l(
+    letDefault(:cl, 0),
+    sel(:cl,
+      l(),
+      l('cl', 'run', dependencies, '---', 'LC_ALL=C.UTF-8'),
+    nil),
+  nil)
+end
+
+def header(modules='core', codalabDependencies=nil)
+  l(
+    codalab(codalabDependencies),
+    # Queuing system
+    letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')),
+    # Create execution directory
+    letDefault(:pooldir, 1),
+    sel(:pooldir, l(), 'fig/bin/qcreate'),
+    # Run the Java command...
+    'java',
+    '-ea',
+    '-Dmodules='+modules,
+    # Memory size
+    letDefault(:memsize, 'default'),
+    sel(:memsize, {
+      'tiny' => l('-Xms2G', '-Xmx4G'),
+      'low' => l('-Xms5G', '-Xmx7G'),
+      'default' => l('-Xms8G', '-Xmx10G'),
+      'medium' => l('-Xms12G', '-Xmx14G'),
+      'high' => l('-Xms20G', '-Xmx24G'),
+      'higher' => l('-Xms40G', '-Xmx50G'),
+      'impressive' => l('-Xms75G', '-Xmx90G'),
+    }),
+    # Classpath
+    '-cp', 'libsempre/*:lib/*',
+    # Profiling
+    letDefault(:prof, 0), sel(:prof, l(), '-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt'),
+    # Debugging
+    letDefault(:debug, 0), sel(:debug, l(), l('-Xdebug', '-Xrunjdwp:server=y,transport=dt_socket,suspend=y,address=8898')),
+  nil)
+end
+
+def figOpts; l(selo(:pooldir, 'execDir', 'exec', '_OUTPATH_'), o('overwriteExecDir'), o('addToView', 0)) end
+
+############################################################
+# Unit tests
+
+addMode('backup', 'small commands like run community server, backup, or simulator', lambda { |e| l(
+  lambda { |e| system 'echo "backing up with mv"'},
+  lambda { |e| system 'mkdir -p ./int-backup/'},
+  letDefault(:msg, 'backing up, no message'),
+  lambda { |e| l('echo ', :msg, '> ./int-backup/message')},
+  lambda { |e| l('echo ', '`date +%Y-%m-%d.%H:%M:%S`', '>> ./int-backup/message')},
+  lambda { |e| system 'mv int-output int-backup/`date +%Y-%m-%d.%H:%M:%S`'},
+  lambda { |e| system 'mkdir -p ./int-output'},
+nil)})
+
+addMode('backup-data', 'put community-server into trash with time stamp', lambda { |e| l(
+  lambda { |e| system 'echo "backing up data with mv"'},
+  lambda { |e| system 'mkdir -p ./community-server/data-backup'},
+  lambda { |e| system 'mv ./community-server/data ./community-server/data-backup/`date +%Y-%m-%d.%H:%M:%S`'},
+nil)})
+
+addMode('trash', 'put int-output into trash with time stamp', lambda { |e| l(
+  lambda { |e| system 'echo "trashing int-output with time stamp"'},
+  lambda { |e| system 'mv int-output int-output-trash-`date +%Y-%m-%d.%H:%M:%S`'},
+  lambda { |e| system 'rm -rf int-output-trash-*'},
+  lambda { |e| system 'mkdir -p ./int-output'},
+nil)})
+
+addMode('test', 'Run unit tests for interactive stuff', lambda { |e|
+  l(
+    'java', '-ea', '-Xmx12g', '-cp', 'libsempre/*:lib/*',
+    letDefault(:debug, 0), sel(:debug, l(), l('-Xdebug', '-Xrunjdwp:server=y,transport=dt_socket,suspend=y,address=8898')),
+    'org.testng.TestNG',
+    lambda { |e|
+      if e[:class]
+        l('-testclass', 'edu.stanford.nlp.sempre.interactive.test.' + e[:class])
+      else
+        "./#{$path}/testng.xml"
+      end
+    },
+  nil)
+})
+
+addMode('simulator', 'run the simulator', lambda { |e| l(
+  # rlwrap,
+  header('core,interactive'),
+  'edu.stanford.nlp.sempre.interactive.Simulator',
+  figOpts,
+  letDefault(:server, 'local'),
+  sel(:server, {
+    'local' => o('serverURL', 'http://localhost:8410'),
+    'remote' => o('serverURL', 'http://jonsson.stanford.edu:8410')
+  }),
+  # set to 0 to enable logging
+  o('numThreads', 1),
+  letDefault(:sandbox, 'full'),
+  sel(:sandbox, {
+    'all' => o('reqParams', 'grammar=0\&cite=0\&learn=0\&logging=0'),
+    'nolog' => o('reqParams', 'grammar=0\&cite=0\&learn=1\&logging=0'),
+    'nolearn' => o('reqParams', 'grammar=1\&cite=1\&learn=0\&logging=0'),
+    'none' => o('reqParams', 'grammar=1\&cite=1\&learn=1\&logging=0'),
+    'nocite' => o('reqParams', 'grammar=1\&cite=0\&learn=1\&logging=0'),
+  }),
+  letDefault(:task, 'sidaw'),
+  sel(:task, {
+    'freebuild' => o('logFiles', "./#{$path}/queries/freebuild.json.gz"),
+    'freebuilddef' => o('logFiles', "./#{$path}/queries/freebuild.def.json.gz"),
+
+    'qual1' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier1-0118.json.gz"),
+    'qual2' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier2-0129.json.gz"),
+    'qual3' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier3-0201.json.gz"), #  both 2 and 3
+    'free1' => o('logFiles', "./#{$path}/queries/rawqueries/freebuild1-0121.json.gz"),
+    'free2' => o('logFiles', "./#{$path}/queries/rawqueries/freebuild2-0127.json.gz"),
+    }),
+nil)})
+
+############################################################
+# {2016-07-02} [sidaw]: interactive semantic parsing
+addMode('voxelurn', 'interactive semantic parsing in a VoxelWorld', lambda { |e| l(
+  #rlwrap,
+  header('core,interactive'),
+  'edu.stanford.nlp.sempre.Main',
+  #figOpts,
+  o('server'),
+  o('masterType', 'edu.stanford.nlp.sempre.interactive.InteractiveMaster'),
+  o('Executor', 'interactive.DALExecutor'),
+  o('LanguageAnalyzer', 'interactive.DALAnalyzer'),
+  o('DALExecutor.convertNumberValues', true),
+  o('DALExecutor.printStackTrace', true),
+  o('VoxelWorld.maxBlocks', 100000),
+  selo(0, 'DALExecutor.worldType', 'VoxelWorld', 'CalendarWorld', 'Otherworld'),
+  selo(0, 'Grammar.inPaths', "./#{$path}/voxelurn.grammar"),
+
+  o('Params.initWeightsRandomly', false),
+  o('Grammar.binarizeRules', false),
+  o('Grammar.useApplyFn', 'interactive.ApplyFn'),
+
+  o('LanguageAnalyzer.lowerCaseTokens', true),
+
+  o('Parser.pruneErrorValues', true),
+  o('Parser', 'interactive.InteractiveBeamParser'),
+  o('Parser.callSetEvaluation', false),
+  o('Parser.coarsePrune', true),
+
+  o('Parser.beamSize', 50),
+  o('InteractiveBeamParser.maxNewTreesPerSpan', 5001),
+  o('ParserState.customExpectedCounts', 'None'),
+
+  selo(0, 'InteractiveBeamParser.floatStrategy', 'Never', 'NoParse', 'Always'),
+  o('InteractiveBeamParser.trackedCats', 'Number', 'Numbers', 'Color',  'Direction', 'Set', 'Sets', 'Action', 'Actions'),
+
+  o('Derivation.derivComparator', 'AnchorPriorityScoreComparator'),
+  o('Params.l1Reg', 'nonlazy'),
+  o('Params.l1RegCoeff', 0.0001),
+
+  o('Params.initStepSize', 0.1),
+  o('Params.adaptiveStepSize', true),
+  #o('Params.stepSizeReduction', 0.25),
+
+  o('FeatureExtractor.featureComputers', 'interactive.DALFeatureComputer'),
+  o('FeatureExtractor.featureDomains', ':rule', ':span', ':stats', ':scope', ':social', ':window'),
+  # o('FeatureExtractor.featureDomains', ':rule'),
+
+  o('InteractiveMaster.intOutputPath', './int-output/'),
+  o('InteractiveMaster.onlyInteractive', true),
+  o('InteractiveUtils.citationPath', './int-output/citation/'),
+
+  o('InteractiveMaster.useAligner', false),
+  o('InteractiveMaster.maxSequence', 20),
+  o('InteractiveMaster.maxChars', 200),
+
+  o('DefinitionAligner.strategies', 'ExactExclusion'),
+
+  o('InteractiveServer.numThreads', 16),
+  o('InteractiveServer.maxCandidates', 50),
+  o('InteractiveServer.queryLogPath', './int-output/query.log'),
+  o('InteractiveServer.responseLogPath', './int-output/response.log'),
+  o('InteractiveServer.port', 8410),
+
+  o('GrammarInducer.useBestPacking', true),
+  o('GrammarInducer.useSimplePacking', true),
+  o('GrammarInducer.maxNonterminals', 3),
+
+  o('Derivation.showTypes', false),
+  o('Derivation.showValues', false),
+  o('Derivation.showRules', false),
+  o('Derivation.anchoredBonus', 1.0),
+
+  o('NumberFn.allowedRange', 0, 100),
+  o('SimpleLexicon.inPaths', "./#{$path}/csscolors.lexicon"),
+
+  lambda { |e| system 'mkdir -p ./int-output/'; nil},
+  lambda { |e| system 'mkdir -p ./int-output/log/'; nil},
+  lambda { |e| system 'mkdir -p ./int-output/citation/'; nil},
+nil) })
+############################################################
+
+if ARGV.size == 0
+  puts "#{$0} @mode=<mode> [options]"
+  puts
+  puts 'This is the main entry point for all interactive related modes.'
+  puts "Modes:"
+  $modes.each { |name,description,func|
+    puts "  #{name}: #{description}"
+  }
+end
+
+modesMap = {}
+$modes.each { |name,description,func|
+  modesMap[name] = func
+}
+run!(sel(:mode, modesMap))

interactive/testng.xml

@@ -0,0 +1,16 @@
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
+
+<suite name="SempreInteractiveTestSuite" verbose="10">
+  <test name="InteractiveTests">
+    <classes>
+      <class name="edu.stanford.nlp.sempre.interactive.test.GrammarInducerTest">
+      </class>
+			<class name="edu.stanford.nlp.sempre.interactive.test.DALExecutorTest">
+      </class>
+			<!--
+			<class name="edu.stanford.nlp.sempre.interactive.test.FloatingParsingTest">
+			</class>
+			-->
+    </classes>
+  </test>
+</suite>

interactive/voxelurn.grammar

@@ -0,0 +1,74 @@
+# domain specific grammar for voxelurn
+
+(include dal.grammar)
+
+# domain specific actions:
+(def @put put)
+(def @base base)
+(def @add add)
+(def @move move)
+
+# syntactic sugar
+(def @veryx veryx)
+
+# typed lambdas
+# Action a, ItemSet i, ValueSet v, Rel r, Direction d, Color c, Number n
+# Rule.defaultIsFloating should be false
+
+# utilities
+(rule $Action (base $Number $Number) (lambda x (lambda y (: @base (var x) (var y)))) (anchored 1))
+(rule $Action (noop) (ConstantFn (: noop)) (anchored 1))
+
+(rule $Set (origin) (ConstantFn (call origin)) (anchored 1))
+
+# primitive actions
+(rule $Action (add $Color) (lambda c (: @add (var c) here)) (anchored 1))
+(rule $Action (add $Color $Direction) (lambda c (lambda d (: @add (var c) (var d)))) (anchored 1))
+(rule $Action (move $Direction) (lambda d (: @move (var d))) (anchored 1))
+
+# primitive actions with arguments
+
+# cube to cube relations
+(rule $Set ($Direction of $Set) (lambda d (lambda s
+			   (call adj (var d) (var s)))) (anchored 1))
+(rule $Set ($Extreme of $Set) (lambda e (lambda s
+			   (call @veryx (var e) (var s)))) (anchored 1))
+(rule $Set ($Direction) (lambda d
+			   (call adj (var d))) (anchored 1))
+(rule $Set ($Extreme) (lambda e
+			   (call @veryx (var e))) (anchored 1))
+
+(for @dir (top bot left right front back)
+  (rule $Direction (@dir) @dir (anchored 1))
+)
+(rule $Extreme (very $Direction) (IdentityFn) (anchored 1))
+
+(for @rel (col row height color name age)
+  (rule $Rel (@rel) (ConstantFn @rel) (anchored 1))
+)
+
+(rule $ValueSet ($Color) (IdentityFn) (anchored 1))
+(rule $ValueSet ($Number) (IdentityFn) (anchored 1))
+
+# set the range via NumberFn options
+
+(rule $Number ($TOKEN) (NumberFn) (anchored 1))
+(rule $Color ($TOKEN) (SimpleLexiconFn (type color)) (anchored 1))
+
+####### domain specific floating rules
+# red means has color red
+# (rule $Sets ($Color) (lambda c (color (var c))))
+
+# various actions can be performed
+# (rule $Action ($Color) (lambda c (: @add (var c) here)))
+# (rule $Action ($Color $Direction) (lambda c (lambda d (: @add (var c) (var d)))))
+# (rule $Action ($Direction) (lambda d (: @move (var d))))
+
+# (rule $Set ($Direction $Sets) (lambda d (lambda s
+#			   (call adj (var d) (var s)))))
+# (rule $Set ($Extreme $Sets) (lambda e (lambda s
+#			   (call @veryx (var e) (var s)))))
+# (rule $Set ($Direction) (lambda d
+#			   (call adj (var d))))
+# (rule $Set ($Extreme) (lambda e
+#			   (call @veryx (var e))))

overnight/README.md

@@ -14,7 +14,7 @@ To generate each individual domain:
 
 ## Training
 
-After generating the paraphrases via AMT and setting up the approprate example
+After generating the paraphrases via AMT and setting up the appropriate example
 files, we train a model.
 
 Run the following to train with all the features:
@@ -38,3 +38,7 @@ No PPDB features:
 Full system:
 
     ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match ppdb skip-bigram root alignment lexical root_lexical lf simpleworld
+
+# Generating alignment files for a new domain
+
+[See the instructions here](https://docs.google.com/document/d/1cgqThUAyArbgjQi0P7l--kCLH10copd1EXFnk_bHkY0/edit?usp=sharing) (work in progress).

pull-dependencies

@@ -1,23 +1,38 @@
 #!/usr/bin/env ruby
 
-# SEMPRE depends on several library/data files into *lib*.  Run this script to
+# SEMPRE depends on several library/data files into |lib|.  Run this script to
 # copy those dependencies to your local directory.  This allows you to run
 # SEMPRE from anywhere.  This file consists of a set of modules (which loosely
 # correspond to the code modules).
+#
+# The master copy of these dependencies are stored on the Stanford NLP machines.
+#
+# Usage:
+#   ./pull-dependencies <module-1> ... <module-n>
+#
+# For developers with ssh access to NLP machines, there are two more local commands:
+# - Copy or link |sourcePath| into lib/|dir|.
+#   ./pull-dependencies -l <module-1> ... <module-n>
+# - Deploy the dependencies to the NLP machines's public www.
+#   ./pull-dependencies -l -r <module-1> ... <module-n>
 
-# The master copy of these dependencies are stored on the Stanford NLP
-# machines.
-
+# Specify the version of the dependencies
+# (To developer: Update this before releasing a new version!)
 $version = '2.0'
-$isPublic = true
 
+$isLocal = ARGV.index('-l')
+$isRelease = ARGV.index('-r')
+if $isRelease and not $isLocal
+  puts "ERROR: To release, must use both -l and -r"
+  exit 1
+end
+ARGV.delete_if { |x| x == '-l' or x == '-r' }
 
 def isZip(name)
   # Directories are zipped
   name.end_with?('.exec') or name !~ /\./
 end
 
-# - Download the dependencies from the Stanford SEMPRE servers.
 def pull(sourcePath, dir=nil, opts={})
   puts sourcePath
   destDir = 'lib' + (dir ? '/' + dir : '')
@@ -26,9 +41,13 @@ def pull(sourcePath, dir=nil, opts={})
   name = File.basename(sourcePath)
   ext = isZip(name) ? '.zip' : ''
 
-  if $isPublic
+  if not $isLocal and not $isRelease
     # Download url => localPath
-    url = 'http://nlp.stanford.edu/software/sempre/dependencies-' + $version + sourcePath + ext
+    if sourcePath.start_with?('http://') || sourcePath.start_with?('https://')
+      url = sourcePath
+    else
+      url = 'http://nlp.stanford.edu/software/sempre/dependencies-' + $version + sourcePath + ext
+    end
     localPath = destDir + '/' + name + ext
     system "mkdir -p #{File.dirname(localPath)}" or exit 1
     system "wget -c '#{url}' -O #{localPath}" or exit 1
@@ -38,6 +57,38 @@ def pull(sourcePath, dir=nil, opts={})
       system "rm #{localPath}" or exit 1
     end
   else
+    rsyncOpts = '-rlptDzi'  # Preserve everything except groups and permissions
+    if $isRelease
+      # Copy sourcePath to cluster
+      baseDeployPath = '/u/apache/htdocs/static/software/sempre/dependencies-' + $version
+      deployPath = baseDeployPath + sourcePath + ext
+      system "mkdir -p #{File.dirname(deployPath)}" or exit 1
+      if File.exists?(sourcePath)
+        if isZip(name)
+          system "cd #{File.dirname(sourcePath)} && zip -r #{deployPath} #{File.basename(sourcePath)}" or exit 1
+        else
+          if opts[:symlink]
+            system "ln -sf #{File.expand_path(sourcePath)} #{deployPath}" or exit 1
+          else
+            system "rsync #{rsyncOpts} #{sourcePath} #{deployPath}" or exit 1
+          end
+        end
+      else
+        system "rsync #{rsyncOpts} jamie.stanford.edu:#{sourcePath} #{deployPath}" or exit 1
+      end
+      system "chmod -R og=u #{baseDeployPath}" #or exit 1
+    else
+      # Download sourcePath from cluster to destDir
+      if File.exists?(sourcePath)
+        if opts[:symlink]
+          system "ln -sf #{File.expand_path(sourcePath)} #{destDir}" or exit 1
+        else
+          system "rsync #{rsyncOpts} #{sourcePath} #{destDir}" or exit 1
+        end
+      else
+        system "rsync #{rsyncOpts} jamie.stanford.edu:#{sourcePath} #{destDir}" or exit 1
+      end
+    end
   end
 end
 
@@ -86,17 +137,54 @@ addModule('core', 'Core utilities (need to compile)', lambda {
   pull('/u/nlp/data/semparse/resources/jackson-core-2.2.0.jar')
   pull('/u/nlp/data/semparse/resources/jackson-annotations-2.2.0.jar')
   pull('/u/nlp/data/semparse/resources/jackson-databind-2.2.0.jar')
+
+  # jLine from maven central
+  pull('https://repo1.maven.org/maven2/jline/jline/2.14.6/jline-2.14.6.jar')
 })
 
-addModule('corenlp', 'Stanford CoreNLP (code and modules)', lambda {
+addModule('corenlp', 'Stanford CoreNLP 3.6.0', lambda {
+  pull('/u/nlp/data/semparse/resources/stanford-corenlp-full-2015-12-09.zip', '', {:symlink => true})
+  if not File.exists?('lib/stanford-corenlp-full-2015-12-09')
+    system "cd lib && unzip stanford-corenlp-full-2015-12-09.zip" or exit 1
+  end
+  pull('/u/nlp/data/semparse/resources/stanford-corenlp-caseless-2015-04-20-models.jar',
+       'stanford-corenlp-full-2015-12-09', {:symlink => true})
+  # Remove old file (for backward compatibility)
+  if Dir.glob('lib/stanford-corenlp*.jar').any?
+    system 'rm -v lib/stanford-corenlp*.jar' or exit 1
+  end
+  {'stanford-corenlp-3.6.0.jar' => 'stanford-corenlp.jar',
+   'stanford-corenlp-3.6.0-models.jar' => 'stanford-corenlp-models.jar',
+   'stanford-corenlp-caseless-2015-04-20-models.jar' => 'stanford-corenlp-caseless-models.jar',
+   'joda-time.jar' => 'joda-time.jar',
+   'jollyday.jar' => 'jollyday.jar',
+   'ejml-0.23.jar' => 'ejml.jar',
+   'slf4j-api.jar' => 'slf4j-api.jar',
+   'slf4j-simple.jar' => 'slf4j-simple.jar',
+  }.each { |key, value|
+    system "ln -sfv stanford-corenlp-full-2015-12-09/#{key} lib/#{value}" or exit 1
+  }
+})
+
+addModule('corenlp-3.2.0', 'Stanford CoreNLP 3.2.0 (for backward reproducibility)', lambda {
   pull('/u/nlp/data/semparse/resources/stanford-corenlp-full-2013-06-20.zip', '', {:symlink => true})
   if not File.exists?('lib/stanford-corenlp-full-2013-06-20')
     system "cd lib && unzip stanford-corenlp-full-2013-06-20.zip" or exit 1
   end
-  ['stanford-corenlp-3.2.0.jar', 'stanford-corenlp-3.2.0-models.jar', 'joda-time.jar', 'jollyday.jar'].each { |file|
-    system "ln -sfv stanford-corenlp-full-2013-06-20/#{file} lib" or exit 1
+  pull('/u/nlp/data/semparse/resources/stanford-corenlp-caseless-2013-06-07-models.jar',
+       'stanford-corenlp-full-2013-06-20', {:symlink => true})
+  # Remove old file (for backward compatibility)
+  if Dir.glob('lib/stanford-corenlp*.jar').any?
+    system 'rm -v lib/stanford-corenlp*.jar' or exit 1
+  end
+  {'stanford-corenlp-3.2.0.jar' => 'stanford-corenlp.jar',
+   'stanford-corenlp-3.2.0-models.jar' => 'stanford-corenlp-models.jar',
+   'stanford-corenlp-caseless-2013-06-07-models.jar' => 'stanford-corenlp-caseless-models.jar',
+   'joda-time.jar' => 'joda-time.jar',
+   'jollyday.jar' => 'jollyday.jar'
+  }.each { |key, value|
+    system "ln -sfv stanford-corenlp-full-2013-06-20/#{key} lib/#{value}" or exit 1
   }
-  pull('/u/nlp/data/semparse/resources/stanford-corenlp-caseless-2013-06-07-models.jar', '', {:symlink => true})
 })
 
 addModule('freebase', 'Freebase: need to construct Freebase schemas', lambda {
@@ -148,11 +236,30 @@ addModule('tables', 'Semantic parsing with execution on tables', lambda {
   pull('/u/nlp/data/semparse/resources/opencsv-3.0.jar')
 })
 
-addModule('tables-data', 'Data for semantic parsing with execution on tables', lambda {
-  pull('/u/nlp/data/semparse/wikitable/data', 'data/tables/', {:symlink => true})
-  pull('/u/nlp/data/semparse/wikitable/mturk-trivia-data/csv', 'data/tables/', {:symlink => true})
+addModule('tables-data', 'WikiTableQuestions dataset v1.0.2', lambda {
+  # Compact version of the dataset
+  pull('https://github.com/ppasupat/WikiTableQuestions/releases/download/v1.0.2/WikiTableQuestions-1.0.2-compact.zip', 'data')
+  # Remove old file (for backward compatibility)
+  if File.directory?('lib/data/WikiTableQuestions')
+    system 'rm -rv lib/data/WikiTableQuestions' or exit 1
+  end
+  system "cd lib/data && unzip WikiTableQuestions-1.0.2-compact.zip" or exit 1
 })
 
+addModule('tables-data-0.5', 'WikiTableQuestions dataset v0.5 (for backward reproducibility)', lambda {
+  # Compact version of the dataset
+  pull('https://github.com/ppasupat/WikiTableQuestions/releases/download/v0.5/WikiTableQuestions-0.5-compact.zip', 'data')
+  # Remove old file (for backward compatibility)
+  if File.directory?('lib/data/WikiTableQuestions')
+    system 'rm -rv lib/data/WikiTableQuestions' or exit 1
+  end
+  system "cd lib/data && unzip WikiTableQuestions-0.5-compact.zip" or exit 1
+})
+
+addModule('tables-cprune', 'Neighbor information for applying macro grammar on tables', lambda {
+  pull('/u/nlp/data/semparse/cprune/nn_0.zip', 'data/nn_0', {:symlink => true})
+  system "cd lib/data/nn_0 && unzip nn_0.zip" or exit 1
+})
 
 addModule('overnight', 'Creating a parser for multiple domains', lambda {
   # Geo evaluation
@@ -189,7 +296,19 @@ addModule('overnight', 'Creating a parser for multiple domains', lambda {
   pull('/u/nlp/data/semparse/overnight/calendar/eval/calendar.test.turk.examples', 'data/overnight/', {:symlink => true})
 })
 
+addModule('esslli_2016', 'Data for ESSLLI 2016 semantic parsing class', lambda {
+  pull('/u/nlp/data/semparse/esslli_2016', 'data/esslli_2016/', {:symlink => true})
+})
 
+addModule('geo880', 'Data, lexicon, grammars and KB for geo880', lambda {
+  pull('/u/nlp/data/semparse/geo880/geo880-test.examples', 'data/geo880', {:symlink => true})
+  pull('/u/nlp/data/semparse/geo880/geo880-test.preprocessed.examples', 'data/geo880', {:symlink => true})
+  pull('/u/nlp/data/semparse/geo880/geo880-train.preprocessed.examples', 'data/geo880', {:symlink => true})
+  pull('/u/nlp/data/semparse/geo880/geo880.grammar', 'data/geo880', {:symlink => true})
+  pull('/u/nlp/data/semparse/geo880/geo880.lexicon', 'data/geo880', {:symlink => true})
+  pull('/u/nlp/data/semparse/geo880/geo880.kg', 'data/geo880', {:symlink => true})
+  pull('/u/nlp/data/semparse/geo880/geo880.type_hierarchy', 'data/geo880', {:symlink => true})
+})
 ############################################################
 
 if ARGV.size == 0
@@ -199,6 +318,10 @@ if ARGV.size == 0
   $modules.each { |name,description,func|
     puts "  #{name}: #{description}"
   }
+  puts
+  puts "Internal use (Stanford NLP only):"
+  puts "  #{$0} -l <module-1> ...: Get the files from the local Stanford NLP server instead"
+  puts "  #{$0} -l -r <module-1> ...: Release to the public www directory on the server"
 end
 
 $modules.each { |name,description,func|

run

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # This is the main entry point for running all SEMPRE programs.  See
 # fig/lib/execrunner.rb for more documentation for how commands are generated.
@@ -7,6 +7,7 @@
 
 $: << 'fig/lib'
 require 'execrunner'
+$optPrefix = '-'
 
 $modes = []
 def addMode(name, description, func)
@@ -20,7 +21,7 @@ def codalab(dependencies=nil)
     letDefault(:cl, 0),
     sel(:cl,
       l(),
-      l('cl', 'run', dependencies, '---'),
+      l('cl', 'run', dependencies, '---', 'LC_ALL=C.UTF-8'),
     nil),
   nil)
 end
@@ -55,8 +56,6 @@ def header(modules='core', codalabDependencies=nil)
   nil)
 end
 
-def rlwrap; system('which rlwrap') ? 'rlwrap' : nil end
-
 def unbalancedTrainDevSplit
   l(o('Dataset.trainFrac', 0.8), o('Dataset.devFrac', 0.2))
 end
@@ -264,7 +263,7 @@ def webquestions
     o('Parser.beamSize', 200), # {07/03/13}: WebQuestions is too slow to run with default 500, so set to 200 for now...
 
     # Caching
-    letDefault(:entitysearch, 1),
+    letDefault(:entitysearch, 0),
     sel(:entitysearch,  # Used for EMNLP 2013
       l(
         cachePaths('lucene/0.cache', 'sparql/3.cache'),
@@ -297,7 +296,7 @@ addMode('freebase', 'Freebase (for EMNLP 2013, ACL 2014, TACL 2014)', lambda { |
     l('nlpsub', '-d/scr/yonatan/sandbox/blackhole', '-nyonatan', '-qjag', '-c3'),
     l('nlpsub', '-d/scr/yonatan/sandbox/blackhole', '-nyonatan', '-qjohn', '-c3'),
   nil),
-  sel(:interact, l(), rlwrap),
+  sel(:interact, l()),
   freebaseHeader,
   'edu.stanford.nlp.sempre.Main',
   freebaseOpts,
@@ -308,11 +307,6 @@ addMode('freebase', 'Freebase (for EMNLP 2013, ACL 2014, TACL 2014)', lambda { |
     'free917' => free917,
   }),
 
-  # Training
-  sel(:train, l(), l(
-    letDefault(:agenda, 0),
-    sel(:agenda, l(), agendaExperiments, agendaFree917Experiments),
-  nil)),
 
   sel(:interact, l(), l(
     # After training, run interact, which loads up a set of parameters and
@@ -451,19 +445,27 @@ nil) })
 
 # Just start a simple interactive shell to try out SEMPRE commands
 addMode('simple', 'Simple shell', lambda { |e| l(
-  codalab, rlwrap, 'java', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
+  codalab, 'java', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
   o('Main.interactive'),
 nil) })
 
 addMode('simple-sparql', 'Simple shell for querying SPARQL', lambda { |e| l(
-  codalab, rlwrap, 'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
+  codalab, 'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
   o('executor', 'freebase.SparqlExecutor'),
   sparqlOpts,
   o('Main.interactive'),
 nil) })
 
+addMode('simple-lambdadcs', 'Simple shell for querying with the LambdaDCSExecutor', lambda { |e| l(
+  codalab, 'java', '-Dmodules=core,tables,corenlp', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
+  o('executor', 'tables.lambdadcs.LambdaDCSExecutor'),
+  o('FeatureExtractor.featureDomains', 'denotation lexAlign joinPos skipPos'.split),
+  o('LanguageAnalyzer.languageAnalyzer', 'corenlp.CoreNLPAnalyzer'),
+  o('Main.interactive'),
+nil) })
+
 addMode('simple-freebase', 'Simple shell for using Freebase', lambda { |e| l(
-  rlwrap, 'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
+  'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
   o('executor', 'freebase.SparqlExecutor'),
   letDefault(:sparqlserver, 'freebase.cloudapp.net:3093'),
   letDefault(:cacheserver, 'freebase.cloudapp.net:4000'),
@@ -488,6 +490,22 @@ addMode('simple-freebase', 'Simple shell for using Freebase', lambda { |e| l(
   o('Main.interactive'),
 nil) })
 
+addMode('simple-freebase-nocache', 'Simple shell for using Freebase (without a cache server)', lambda { |e| l(
+  'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
+  o('executor', 'freebase.SparqlExecutor'),
+  letDefault(:sparqlserver, 'freebase.cloudapp.net:3093'),
+  sparqlOpts,
+  o('FeatureExtractor.featureDomains', 'rule'),
+  o('Parser.coarsePrune'),
+  o('JoinFn.typeInference'),
+  o('UnaryLexicon.unaryLexiconFilePath', '/dev/null'),
+  o('BinaryLexicon.binaryLexiconFilesPath', '/dev/null'),
+  #o('JoinFn.showTypeCheckFailures'),  # Use this to debug
+  o('Grammar.inPaths', 'freebase/data/demo1.grammar'),  # Override with your own custom grammar
+  #o('SparqlExecutor.includeSupportingInfo'),  # Show full information
+  o('Main.interactive'),
+nil) })
+
 
 ############################################################
 # {2014-12-27} [Percy]: Overnight semantic parsing
@@ -506,7 +524,6 @@ def overnightFeatureDomains
 end
 
 addMode('overnight', 'Overnight semantic parsing', l(
-  rlwrap,
   header('core,freebase,overnight'),
   'edu.stanford.nlp.sempre.Main',
   figOpts,
@@ -535,7 +552,6 @@ addMode('overnight', 'Overnight semantic parsing', l(
   #o('initialization', 'paraphrase :: match,1', 'paraphrase :: size,-0.1', 'paraphrase :: ppdb,0.3',
   #  'paraphrase :: skip-bigram,0.8', 'paraphrase :: skip-ppdb,0.2','denotation :: error,-1000'),
   o('coarsePrune'),
-  o('OvernightDerivationPruningComputer.applyHardConstraints'),
   sel(2,
     l(), # no reg
     l(o('Params.l1Reg','lazy'),o('Params.l1RegCoeff',0)),
@@ -557,6 +573,7 @@ addMode('overnight', 'Overnight semantic parsing', l(
   o('phraseAlignmentPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '.phrase_alignments' }),
   o('PPDBModel.ppdbModelPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '-ppdb.txt' }),
   o('DerivationPruner.pruningComputers', ['overnight.OvernightDerivationPruningComputer']),
+  o('DerivationPruner.pruningStrategies', ['violateHardConstraints']),
   o('Dataset.inPaths', 
     lambda { |e| 'train:lib/data/overnight/' + e[:domain] + '.paraphrases.train.examples' },
     lambda { |e| 'test:lib/data/overnight/' + e[:domain] + '.paraphrases.test.examples' }),
@@ -577,8 +594,6 @@ addMode('overnight', 'Overnight semantic parsing', l(
       o('FloatingParser.maxDepth', 11),
       o('Grammar.tags','generate','general', 'geo880'),
       o('SimpleLexicon.inPaths', 'lib/data/overnight/geo880.lexicon'),
-      o('OvernightDerivationPruningComputer.usePredicateDict'),
-      o('OvernightDerivationPruningComputer.predicateDictPath','lib/data/overnight/geo880.predicate.dict'),
     nil),
     'calendar' => l(
       o('Grammar.tags','generate','general'),
@@ -618,42 +633,81 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
   # Add @cldir=1 to use CodaLab's directory paths
   letDefault(:cldir, 0),
   # Usual header
-  rlwrap, header('core,tables,corenlp'),
+  header('core,tables,corenlp,cprune'),
   # Select class
   letDefault(:class, 'main'),
   sel(:class, {
     'main' => 'edu.stanford.nlp.sempre.Main',
-    'check' => 'edu.stanford.nlp.sempre.tables.test.DPParserChecker',
-    'align' => 'edu.stanford.nlp.sempre.tables.alignment.IBMAligner',
+    'check' => 'edu.stanford.nlp.sempre.tables.test.DPDParserChecker',
     'dump' => 'edu.stanford.nlp.sempre.tables.serialize.SerializedDumper',
-    'load' => 'edu.stanford.nlp.sempre.tables.serialize.SerializedLoader',
+    'load' => l('edu.stanford.nlp.sempre.tables.serialize.SerializedLoader', let(:parser, 'serialized')),
     'stats' => 'edu.stanford.nlp.sempre.tables.test.TableStatsComputer',
+    'tag-data' => 'edu.stanford.nlp.sempre.tables.serialize.TaggedDatasetGenerator',
+    'tag-table' => 'edu.stanford.nlp.sempre.tables.serialize.TaggedTableGenerator',
+    'tag-fuzzy' => 'edu.stanford.nlp.sempre.tables.serialize.TaggedFuzzyGenerator',
+    'alter' => l('edu.stanford.nlp.sempre.tables.alter.BatchTableAlterer', let(:parser, 'serialized')),
+    'alter-ex' => l('edu.stanford.nlp.sempre.tables.alter.AlteredTablesExecutor', let(:parser, 'serialized')),
+    'filter' => 'edu.stanford.nlp.sempre.tables.serialize.DumpFilterer',
+    'column' => 'edu.stanford.nlp.sempre.tables.test.TableColumnAnalyzer',
+    'execute' => 'edu.stanford.nlp.sempre.tables.test.BatchTableExecutor',
   }),
   # Fig parameters
   selo(:cldir, 'execDir', '_OUTPATH_', '.'),
-  o('overwriteExecDir'), o('addToView', 0), o('jarFiles', 'libsempre/*'),
+  o('overwriteExecDir'), o('addToView', 15), o('jarFiles', 'libsempre/*'),
   sel(:cldir, l(), '>/dev/null'),
   # Set environment for table execution
   o('executor', 'tables.lambdadcs.LambdaDCSExecutor'),
-  o('Builder.valueEvaluator', 'tables.TableValueEvaluator'),
-  o('TargetValuePreprocessor.targetValuePreprocessor', 'tables.TableValuePreprocessor'),
+  o('targetValuePreprocessor', 'tables.TableValuePreprocessor'),
   o('NumberFn.unitless'), o('NumberFn.alsoTestByConversion'),
   o('TypeInference.typeLookup', 'tables.TableTypeLookup'),
   o('JoinFn.specializedTypeCheck', false), o('JoinFn.typeInference', true),
+  o('Learner.outputPredValues'),
+  # Value Evaluator
+  letDefault(:eval, 'value'),
+  sel(:eval, {
+    'value' => o('Builder.valueEvaluator', 'tables.TableValueEvaluator'),
+    'denotation' => o('Builder.valueEvaluator', 'tables.TableValueEvaluator'),  # alias of 'value'
+    'formula' => l(
+      o('Builder.valueEvaluator', 'tables.TableFormulaEvaluator'),
+      o('fallBackToValueEvaluator', false),
+    nil),
+  }),
   # Parser
   letDefault(:parser, 'floatsize'),
+  o('beamSize', 50),
+  o('useSizeInsteadOfDepth'),
   sel(:parser, {
     'floatsize' => l(
       o('Builder.parser', 'FloatingParser'),
-      o('useSizeInsteadOfDepth'),
       o('FloatingParser.maxDepth', 15),
     nil),
-    'baseline' => l(
-      o('Builder.parser', 'tables.baseline.TableBaselineParser'),
+    'baseline' => o('Builder.parser', 'tables.baseline.TableBaselineParser'),
+    'serialized' => o('Builder.parser', 'tables.serialize.SerializedParser'),
+    # ACL 2016
+    'grow-dpd' => l(
+      o('Builder.parser', 'tables.dpd.DPDParser'),
+      o('FloatingParser.maxDepth', 8),
+    nil),
+    'grow-float' => l(
+      o('Builder.parser', 'FloatingParser'),
+      o('FloatingParser.maxDepth', 8),
+      o('FloatingParser.betaReduce'), o('initialFloatingHasZeroDepth'),
+    nil),
+    'grow-mix' => l(
+      o('Builder.parser', 'MixParser'),
+      o('MixParser.parsers', 'FloatingParser', 'tables.serialize.SerializedParser:train-0xc'),
+      o('FloatingParser.maxDepth', 8),
+      o('FloatingParser.betaReduce'), o('initialFloatingHasZeroDepth'),
+    nil),
+    # EMNLP 2017
+    'cprune' => l(
+      o('Builder.parser', 'cprune.CPruneFloatingParser'),
+      o('FloatingParser.maxDepth', 15),
+      o('maxNumNeighbors', 40),
+      o('maxPredictedPatterns', 1000),
     nil),
-    'dummy' => o('Builder.parser', 'tables.serialize.DummyParser'),
   }),
-  o('FloatingParser.useAnchorsOnce', true),
+  o('Parser.verbose', 0),
   letDefault(:pruning, 1),
   sel(:pruning,
     l(),
@@ -664,17 +718,54 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
   nil),
   # Grammar
   tablesGrammarPaths,
+  letDefault(:fuzzy, 'original'),
+  sel(:fuzzy, {
+    'original' => o('FuzzyMatcher.fuzzyMatcher', 'tables.match.OriginalMatcher'),
+    'editdist-exact' => l(
+      o('FuzzyMatcher.fuzzyMatcher', 'tables.match.EditDistanceFuzzyMatcher'),
+      o('fuzzyMatchMaxEditDistanceRatio', 0.0),
+    nil),
+    'editdist-fuzzy' => l(
+      o('FuzzyMatcher.fuzzyMatcher', 'tables.match.EditDistanceFuzzyMatcher'),
+      o('fuzzyMatchSubstring'), o('fuzzyMatchMaxEditDistanceRatio', 0.15),
+      o('alsoMatchPart'),
+    nil),
+  }),
+  letDefault(:normalize, 1),
+  sel(:normalize,
+    l(),
+    l(o('genericDateValue'), o('numberCanStartAnywhere'), o('num2CanStartAnywhere')),
+  nil),
+  letDefault(:anchor, 1),
+  sel(:anchor, {
+    1 => o('FloatingParser.useAnchorsOnce', true),
+    2 => l(o('FloatingParser.useAnchorsOnce', false), o('FloatingParser.useMaxAnchors', 2)),
+  }),
   # Dataset
   letDefault(:data, 'none'),
-  letDefault(:unseen, 0),
   tablesDataPaths,
   # Verbosity
   o('FeatureVector.ignoreZeroWeight'),
-  o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10), o('logFeaturesLimit', 10),
+  o('logFeaturesLimit', 10),
+  o('LambdaDCSException.noErrorMessage'),
   letDefault(:verbose, 0),
   sel(:verbose,
-    l(),
     l(
+      o('maxPrintedPredictions', 1), o('maxPrintedTrue', 1),
+    nil),
+    l(
+      o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10),
+      o('putCellNameInCanonicalUtterance'), o('showUtterance'),
+    nil),
+    l(
+      o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10),
+      o('putCellNameInCanonicalUtterance'), o('showUtterance'),
+      o('summarizeRuleTime'), o('summarizeDenotations'),
+    nil),
+    l(
+      o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10),
+      o('putCellNameInCanonicalUtterance'), o('showUtterance'),
+      o('summarizeRuleTime'), o('summarizeDenotations'),
       o('showRules'),
       o('Parser.verbose', 2),
       o('JoinFn.verbose', 3),
@@ -691,25 +782,26 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
   # Training
   letDefault(:train, 0),
   sel(:train,
-    l(),
+    l(
+      let(:l1, 0),
+    nil),
     l(
       o('combineFromFloatingParser'),
-      sel(:unseen, unbalancedTrainDevSplit, l()),
       o('maxTrainIters', 3),
       o('showValues', false), o('showFirstValue'),
+      o('customExpectedCounts', 'TOP'),
     nil),
     l(
       # for dumping derivations (@class=dump)
       # force unbalancedTrainDevSplit + combine from floating parser
-      o('combineFromFloatingParser'), o('DPParser.cheat'),
-      sel(:unseen, unbalancedTrainDevSplit, l()),
+      o('combineFromFloatingParser'), o('DPDParser.cheat'),
     nil),
   nil),
   # Regularization
   letDefault(:l1, 1),
   sel(:l1,
     l(),
-    l(o('Params.l1Reg','lazy'), o('Params.l1RegCoeff', '3e-5')),
+    l(o('Params.l1Reg','lazy'), o('Params.l1RegCoeff', '3e-5')),    # Default
     l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0, 0.00001, 0.0001, 0.001, 0.01)),
     l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0.00001, 0.00003, 0.0001, 0.0003)),
     l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0.00001, 0.00003, 0.0005)),
@@ -721,10 +813,14 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
     'some' => l(     # Add your own features! (only set up the feature computers)
       o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
     nil),
-    'all' => l(      # All features
+    'all' => l(      # All ACL 2015 features
       o('FeatureExtractor.featureDomains', 'custom-denotation phrase-predicate phrase-denotation headword-denotation missing-predicate'.split),
       o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
     nil),
+    'more' => l(     # All ACL 2015 features + more experimental features
+      o('FeatureExtractor.featureDomains', 'custom-denotation phrase-predicate phrase-denotation headword-denotation missing-predicate anchored-entity'.split),
+      o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer tables.features.AnchorFeatureComputer'.split),
+    nil),
     'baseline' => l( # For the baseline classifier
       o('FeatureExtractor.featureDomains', 'custom-denotation phrase-denotation headword-denotation table-baseline'.split),
       o('FeatureExtractor.featureComputers', 'tables.baseline.TableBaselineFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
@@ -744,14 +840,8 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
   letDefault(:featOp, 'careful'),
   sel(:featOp, {
     'none' => l(),
-    'base' => l(
-      o('usePredicateLemma'), o('usePhraseLemmaOnly'),
-    nil),
     'careful' => l(
-      o('usePredicateLemma'), o('usePhraseLemmaOnly'),
       o('maxNforLexicalizeAllPairs', 2),
-      o('traverseWithFormulaTypes'), o('reverseNameValueConversion', 'allBang'),
-      o('lookUnderCellProperty'), o('useGenericCellType'),
       o('computeFuzzyMatchPredicates'),
     nil),
   }),
@@ -763,6 +853,7 @@ def tablesGrammarPaths
     l(
       letDefault(:grammar, 'combined-all'),
       sel(:grammar, {
+        'custom' => l(),
         'restrict' => o('Grammar.inPaths', "#{baseDir}restrict.grammar"),
         'simple' => o('Grammar.inPaths', "#{baseDir}simple.grammar"),
         'combined' => o('Grammar.inPaths', "#{baseDir}combined.grammar"),
@@ -778,10 +869,36 @@ def tablesGrammarPaths
           o('Grammar.inPaths', "#{baseDir}combined.grammar"),
           o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge'.split),
         nil),
+        'combined-more' => l(
+          o('Grammar.inPaths', "#{baseDir}combined.grammar"),
+          o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge v-superlative'.split),
+        nil),
         'combined-trigger' => l(    # Use trigger words for operations
           o('Grammar.inPaths', "#{baseDir}combined.grammar"),
           o('Grammar.tags', *'t-alternative t-movement t-comparison t-count t-aggregate t-superlative t-arithmetic merge'.split),
         nil),
+        # ACL 2016
+        'grow-custom' => l(
+          o('Grammar.inPaths', "#{baseDir}grow.grammar"),
+          o('Grammar.binarizeRules', false),
+        nil),
+        'grow-default' => l(
+          o('Grammar.inPaths', "#{baseDir}grow.grammar"),
+          o('Grammar.binarizeRules', false),
+          o('Grammar.tags', *'scoped merge-and arithmetic comparison alternative neq yearrange part closedclass scoped-2args-merge-and'.split),
+          let(:anchor, 2),
+        nil),
+        'grow-strict' => l(
+          o('Grammar.inPaths', "#{baseDir}grow.grammar"),
+          o('Grammar.binarizeRules', false),
+          o('Grammar.tags', *'scoped merge-and arithmetic comparison alternative neq yearrange part closedclass-generic scoped-2args-merge-and'.split),
+          let(:anchor, 2),
+        nil),
+        # EMNLP 2017
+        'extended' => l(
+          o('Grammar.inPaths', "#{baseDir}extended.grammar"),
+          o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge v-superlative'.split),
+        nil),
       }),
     nil)
   }
@@ -789,32 +906,40 @@ end
 
 def tablesDataPaths
   lambda { |e|
-    baseDir = ['lib/data/tables/data/', 'WikiTableQuestions/data/'][e[:cldir]]
-    csvDir = ['lib/data/tables/', 'WikiTableQuestions/'][e[:cldir]]
+    baseDir = ['lib/data/WikiTableQuestions/data/', 'WikiTableQuestions/data/'][e[:cldir]]
+    csvDir = ['lib/data/WikiTableQuestions/', 'WikiTableQuestions/'][e[:cldir]]
+    nnDir = ['lib/data/nn_0/', 'nn_0/'][e[:cldir]]
     datasets = {
       'none' => l(),
+      'train' => o('Dataset.inPaths', "train,#{baseDir}training.examples"),
       # Pristine test test
       'test' => l(
         o('Dataset.inPaths',
           "train,#{baseDir}training.examples",
           "test,#{baseDir}pristine-unseen-tables.examples"),
-        o('Dataset.trainFrac', 1), o('Dataset.devFrac', 0),
-        let(:unseen, 1),
+        o('neighborFilePath', "#{nnDir}/exact_nearest_neighbors.all"),
       nil),
+      # @data=annotated can be used with @class=check only
+      'annotated' => o('Dataset.inPaths', "train,#{baseDir}annotated-all.examples"),
+      'before300' => o('Dataset.inPaths', "train,#{baseDir}training-before300.examples"),
     }
     # Development sets: 80:20 random split of training data
     ['1', '2', '3', '4', '5'].each do |x|
       datasets['u-' + x] = l(
         o('Dataset.inPaths',
-          "train,#{baseDir}random-split-seed-#{x}-train.examples",
-          "dev,#{baseDir}random-split-seed-#{x}-test.examples",
+          "train,#{baseDir}random-split-#{x}-train.examples",
+          "dev,#{baseDir}random-split-#{x}-dev.examples",
         nil),
-        let(:unseen, 1),
+        o('neighborFilePath', "#{nnDir}/exact_nearest_neighbors.seed-#{x}.train"),
       nil)
     end
     # That's it!
     l(
       o('TableKnowledgeGraph.baseCSVDir', csvDir),
+      # To use the normalized values from the tagged file, which were checked by hand,
+      # add @useTaggedFile=1
+      letDefault(:useTaggedFile, 0),
+      selo(:useTaggedFile, 'TableValuePreprocessor.taggedFiles', '', "#{csvDir}/tagged/data/"),
       sel(:data, datasets),
     nil)
   }
@@ -822,29 +947,30 @@ end
 
 def tablesPruningStrategies
   [
-    # Formula
-    "singleton",
+    ### Critical strategies
+    "emptyDenotation",
+    "nonLambdaError",
+    ### Strategies that do not depend on the children's actual formulas
+    "atomic",
+    "tooManyValues",
+    "badSummarizerHead",
+    "mistypedMerge",
+    ### Strategies that depend on the children's formulas
+    "doubleNext",
     "multipleSuperlatives",
     "sameMerge",
     "forwardBackward",
-    "doubleNext",
-    # Denotation
-    "emptyDenotation",
-    "nonLambdaError",
-    "tooManyValues",
-    "badSuperlativeHead",
-    "mistypedMerge",
+    "unsortedMerge",
+    "typeRowMerge",
   nil].compact
 end
 
 
-
+############################################################
 # {2015-01-18} Generate utterances [Percy]
 addMode('genovernight', 'Generate utterances for overnight semantic parsing', lambda { |e| l(
-  'fig/bin/qcreate',
-  letDefault(:gen, 0),
-  sel(:gen, rlwrap, l()),
-  'java', '-Dmodules=core,overnight', '-Xmx10g', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.overnight.GenerationMain',
+  header('core,overnight'),
+  'edu.stanford.nlp.sempre.overnight.GenerationMain',
   figOpts,
   o('JoinFn.typeInference', true),
   o('JoinFn.specializedTypeCheck', false),
@@ -860,6 +986,7 @@ addMode('genovernight', 'Generate utterances for overnight semantic parsing', la
   o('FeatureExtractor.featureComputers','overnight.OvernightFeatureComputer'),
   o('OvernightFeatureComputer.featureDomains', ''),
   o('OvernightFeatureComputer.itemAnalysis',false),
+  letDefault(:gen, 1),
   sel(:gen,
     l( # For debugging the grammar
       o('FeatureExtractor.featureDomains', 'denotation'),
@@ -902,6 +1029,72 @@ addMode('genovernight-wrapper', 'Generate utterances for overnight semantic pars
   lambda { |e| system 'mkdir -p genovernight.out'; o('execDir', 'genovernight.out/' + e[:domain]) },
 nil) })
 
+addMode('geo880', 'Semantic parsing on the geo880 dataset', lambda { |e| l(
+  # Usual header
+  header('core,tables,corenlp,geo880'),
+  'edu.stanford.nlp.sempre.Main',
+  # Fig parameters
+  figOpts,
+  o('executor', 'tables.lambdadcs.LambdaDCSExecutor'),
+  o('JoinFn.specializedTypeCheck', false), o('JoinFn.typeInference', false),
+  # Parser
+  o('Builder.parser', 'BeamParser'),
+  o('Parser.coarsePrune'),
+ 
+  # Evaluation
+  o('Builder.valueEvaluator', 'geo880.Geo880ValueEvaluator'),
+
+  # Grammar
+  o('Grammar.inPaths','lib/data/geo880/geo880.grammar'),
+
+  # Type hierarchy
+  o('Geo880TypeLookup.typeHierarchyPath', 'lib/data/geo880/geo880.type_hierarchy'),
+  o('TypeInference.typeLookup','geo880.Geo880TypeLookup'),
+
+  # Yrkvpba
+  o('SimpleLexicon.inPaths', 'lib/data/geo880/geo880.lexicon'),
+  
+  # Learner
+  o('Learner.maxTrainIters', 3),
+
+  # Dataset
+  letDefault(:data, 0),
+  sel(:data,
+    l(o('Dataset.inPaths', 'train,lib/data/geo880/geo880-train.preprocessed.examples'), unbalancedTrainDevSplit), # (0) train 0.8, dev 0.2
+    l(o('Dataset.inPaths', 'train,lib/data/geo880/geo880-train.examples', 'test,lib/data/geo880/geo880-test.preprocessed/examples')), # (1) Don't run on test yet!
+  nil),
+  # Load the graph
+  o('Dataset.globalGraphPath', 'lib/data/geo880/geo880.kg'),
+  # Verbosity
+  letDefault(:verbose, 0),
+  sel(:verbose,
+    l(),
+    l(
+      o('showRules'),
+      o('Parser.verbose', 2),
+      o('JoinFn.verbose', 3),
+      o('JoinFn.showTypeCheckFailures'),
+    nil),
+  nil),
+  # Language Analyzer
+  l(o('LanguageAnalyzer', 'corenlp.CoreNLPAnalyzer'), o('annotators', *'tokenize ssplit pos lemma ner'.split)),
+  # Regularization
+  letDefault(:l1, 0),
+  sel(:l1,
+    l(),
+    l(o('Params.l1Reg','lazy'), o('Params.l1RegCoeff', '3e-5')),
+    l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0, 0.00001, 0.0001, 0.001, 0.01)),
+  nil),
+  # Features
+  letDefault(:feat, 'freebase'),
+  sel(:feat, {
+    'none' => l(),   # No features (random)
+    'freebase' => l(
+      o('FeatureExtractor.featureDomains', 'rule opCount constant whType span lemmaAndBinaries denotation lexAlign joinPos skipPos'.split),
+#      o('FeatureExtractor.featureDomains', 'rule opCount constant whType lemmaAndBinaries denotation lexAlign joinPos skipPos'.split),
+    nil),
+  }),
+nil) })
 
 ############################################################
 

scripts/agenda-stats

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 ARGV.each { |e|
   e = e.sub(/\.exec$/, '')

scripts/extract-module-classes.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # Input: src
 # Output: module-classes.txt

scripts/find-hard-coded-paths.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # Heuristically find all hard-coded paths in the source code.
 # There should be no absolute paths.

scripts/fix-checkstyle.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # Hacky script for automatically fixing style errors.  This script is far from
 # perfect and you should manually inspect all changes before making changes.

scripts/verify-code-loop.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 # Verifies that the codebase is sane (compiles, doesn't crash, gets reasonable
 # accuracy) every once in a while.  If something fails, an email is sent out

src/edu/stanford/nlp/sempre/ActionFormula.java

@@ -0,0 +1,130 @@
+package edu.stanford.nlp.sempre;
+
+import java.util.List;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+
+import fig.basic.LispTree;
+
+/**
+ * An ActionFormula represent a compositional action used in the interactive
+ * package : is used as a prefix to denote an ActionFormula primitive (:
+ * actioname args) sequential (:s ActionFormula ActionFormula ...) repeat (:loop
+ * Number ActionFormula) conditional (:if Set ActionFormula) block scoping (:blk
+ * ActionFormula)
+ * 
+ * @author sidaw
+ */
+public class ActionFormula extends Formula {
+  public enum Mode {
+    primitive(":"), // (: remove *)
+    sequential(":s"), // (:s (: add red top) (: remove this))
+    repeat(":loop"), // (:loop (count (has color green)) (: add red top))
+    conditional(":if"), // (:if (count (has color green)) (: add red top))
+    whileloop(":while"), // (:while (count (has color green)) (: add red top))
+    forset(":for"), // (:for (and this (color red)) (:s (: add red top) (: add
+                    // yellow top) (: remove)))
+    foreach(":foreach"), // (:foreach * (add ((reverse color) this) top))
+
+    // primitives for declaring variables
+    // let(":let"), // (:let X *),
+    // set(":set"), // (:set X *)
+
+    block(":blk"), // start a block of code (like {}) with a new scope
+    blockr(":blkr"), // also return a result after finishing the block
+    isolate(":isolate"), other(":?");
+
+    private final String value;
+
+    Mode(String value) {
+      this.value = value;
+    }
+
+    @Override
+    public String toString() {
+      return this.value;
+    }
+  };
+
+  public final Mode mode;
+  public final List<Formula> args;
+
+  public ActionFormula(Mode mode, List<Formula> args) {
+    this.mode = mode;
+    this.args = args;
+  }
+
+  public static Mode parseMode(String mode) {
+    if (mode == null)
+      return null;
+    for (Mode m : Mode.values()) {
+      // LogInfo.logs("mode string %s \t== %s \t!= %s", m.toString(), mode,
+      // m.name());
+      if (m.toString().equals(mode))
+        return m;
+    }
+    if (mode.startsWith(":"))
+      throw new RuntimeException("Unsupported ActionFormula mode");
+    return null;
+  }
+
+  @Override
+  public LispTree toLispTree() {
+    LispTree tree = LispTree.proto.newList();
+    tree.addChild(this.mode.toString());
+    for (Formula arg : args)
+      tree.addChild(arg.toLispTree());
+    return tree;
+  }
+
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) {
+      for (Formula arg : args)
+        arg.forEach(func);
+    }
+  }
+
+  @Override
+  public Formula map(Function<Formula, Formula> transform) {
+    Formula result = transform.apply(this);
+    if (result != null)
+      return result;
+    List<Formula> newArgs = Lists.newArrayList();
+    for (Formula arg : args)
+      newArgs.add(arg.map(transform));
+    return new ActionFormula(this.mode, newArgs);
+  }
+
+  @Override
+  public List<Formula> mapToList(Function<Formula, List<Formula>> transform, boolean alwaysRecurse) {
+    List<Formula> res = transform.apply(this);
+    if (res.isEmpty() || alwaysRecurse) {
+      for (Formula arg : args)
+        res.addAll(arg.mapToList(transform, alwaysRecurse));
+    }
+    return res;
+  }
+
+  @SuppressWarnings({ "equalshashcode" })
+  @Override
+  public boolean equals(Object thatObj) {
+    if (!(thatObj instanceof ActionFormula))
+      return false;
+    ActionFormula that = (ActionFormula) thatObj;
+    if (!this.mode.equals(that.mode))
+      return false;
+    if (!this.args.equals(that.args))
+      return false;
+    return true;
+  }
+
+  @Override
+  public int computeHashCode() {
+    int hash = 0x7ed55d16;
+    hash = hash * 0xd3a2646c + mode.hashCode();
+    hash = hash * 0xd3a2646c + args.hashCode();
+    return hash;
+  }
+}

src/edu/stanford/nlp/sempre/AggregateFormula.java

@@ -36,6 +36,12 @@ public class AggregateFormula extends Formula {
     return null;
   }
 
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) child.forEach(func);
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> func) {
     Formula result = func.apply(this);
     return result == null ? new AggregateFormula(mode, child.map(func)) : result;

src/edu/stanford/nlp/sempre/ArithmeticFormula.java

@@ -32,6 +32,12 @@ public class ArithmeticFormula extends Formula {
     return tree;
   }
 
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) { child1.forEach(func); child2.forEach(func); }
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> func) {
     Formula result = func.apply(this);
     return result == null ? new ArithmeticFormula(mode, child1.map(func), child2.map(func)) : result;

src/edu/stanford/nlp/sempre/AtomicSemType.java

@@ -1,7 +1,6 @@
 package edu.stanford.nlp.sempre;
 
 import fig.basic.LispTree;
-import java.util.*;
 
 // Represents an atomic type (strings, entities, numbers, dates, etc.).
 public class AtomicSemType extends SemType {

src/edu/stanford/nlp/sempre/BooleanValue.java

@@ -19,6 +19,9 @@ public class BooleanValue extends Value {
     return tree;
   }
 
+  @Override public String sortString() { return "" + value; }
+  @Override public String pureString() { return "" + value; }
+
   @Override public int hashCode() { return Boolean.valueOf(value).hashCode(); }
   @Override public boolean equals(Object o) {
     if (this == o) return true;

src/edu/stanford/nlp/sempre/CallFormula.java

@@ -35,6 +35,16 @@ public class CallFormula extends Formula {
     return tree;
   }
 
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) {
+      this.func.forEach(func);
+      for (Formula arg: args)
+        arg.forEach(func);
+    }
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> transform) {
     Formula result = transform.apply(this);
     if (result != null) return result;

src/edu/stanford/nlp/sempre/CanonicalNames.java

@@ -5,7 +5,8 @@ import java.util.*;
 /**
  * List of canonical names that we borrowed from Freebase.
  *
- * These names and helper methods are independent from the Freebase schema.
+ * These names and helper methods are independent from the Freebase schema
+ * (even though the names begin with "fb:").
  *
  * @author ppasupat
  */
@@ -31,33 +32,86 @@ public final class CanonicalNames {
   public static final String TYPE = "fb:type.object.type";
   public static final String NAME = "fb:type.object.name";
 
+  // Special Unary: star (*)
+  public static final String STAR = "*";
+
+  // Special Binaries: comparison
+  public static final Map<String, String> COMPARATOR_REVERSE = new HashMap<>();
+  static {
+    COMPARATOR_REVERSE.put("!=", "!=");     // a != b implies b != a
+    COMPARATOR_REVERSE.put("<", ">=");
+    COMPARATOR_REVERSE.put(">", "<=");
+    COMPARATOR_REVERSE.put("<=", ">");
+    COMPARATOR_REVERSE.put(">=", "<");
+  }
+  public static final Set<String> COMPARATORS = COMPARATOR_REVERSE.keySet();
+
+  // Special Binary: colon (:)
+  public static final String COLON = ":";
+
+  // SemType for special unaries and binaries
+  public static final Map<String, SemType> SPECIAL_SEMTYPES = new HashMap<>();
+  static {
+    SPECIAL_SEMTYPES.put("*", SemType.anyType);
+    SPECIAL_SEMTYPES.put("!=", SemType.anyAnyFunc);
+    SPECIAL_SEMTYPES.put("<", SemType.compareFunc);
+    SPECIAL_SEMTYPES.put(">", SemType.compareFunc);
+    SPECIAL_SEMTYPES.put("<=", SemType.compareFunc);
+    SPECIAL_SEMTYPES.put(">=", SemType.compareFunc);
+    SPECIAL_SEMTYPES.put(":", SemType.anyAnyFunc);
+  }
+
   // Unary: fb:domain.type [contains exactly one period]
-  // Binary: fb:domain.type.property, <, >, etc.
+  // Special Unary: star (*)
   public static boolean isUnary(String s) {
+    if (STAR.equals(s)) return true;
     int i = s.indexOf('.');
     if (i == -1) return false;
     i = s.indexOf('.', i + 1);
     if (i == -1) return true;
     return false;
   }
+  public static boolean isUnary(Value value) {
+    return value instanceof NameValue && isUnary((((NameValue) value).id));
+  }
 
+  // Binary: fb:domain.type.property [contains two periods]
+  // Also catch reversed binary shorthand [!fb:people.person.parent]
+  // Special Binaries: comparison (<, >, etc.) and colon (:)
   public static boolean isBinary(String s) {
+    if (COMPARATORS.contains(s) || COLON.equals(s)) return true;
     int i = s.indexOf('.');
     if (i == -1) return false;
     i = s.indexOf('.', i + 1);
     if (i == -1) return false;
     return true;
   }
+  public static boolean isBinary(Value value) {
+    return value instanceof NameValue && isBinary((((NameValue) value).id));
+  }
 
   // Return whether |property| is the name of a reverse property.
   // Convention: ! is the prefix for reverses.
   public static boolean isReverseProperty(String property) {
     return property.startsWith("!") && !property.equals("!=");
   }
+  public static boolean isReverseProperty(Value value) {
+    return value instanceof NameValue && isReverseProperty(((NameValue) value).id);
+  }
+
+  // Return the reverse property as a String
   public static String reverseProperty(String property) {
-    if (isReverseProperty(property)) return property.substring(1);
+    if (COMPARATORS.contains(property))
+      return COMPARATOR_REVERSE.get(property);
+    if (isReverseProperty(property))
+      return property.substring(1);
     else return "!" + property;
   }
+  public static NameValue reverseProperty(Value value) {
+    if (!(value instanceof NameValue))
+      throw new RuntimeException("Cannot call reverseProperty on " + value);
+    return new NameValue(reverseProperty(((NameValue) value).id));
+  }
 
 
 }

src/edu/stanford/nlp/sempre/CatSizeBound.java

@@ -0,0 +1,78 @@
+package edu.stanford.nlp.sempre;
+
+import java.util.*;
+
+import fig.basic.*;
+
+/**
+ * Given the maximum formula size in a floating grammar, compute the maximum size
+ * that each floating grammar category can have.
+ *
+ * For example, if the grammar looks like this:
+ *   $ROOT -> $A | $A $B
+ *   $A -> $C $A
+ *   $B -> $C
+ *   $C -> $D $B | nothing
+ *   ...
+ * and the maximum formula size (for $ROOT) is 10, then the maximum formula sizes for
+ * $A, $B, $C and $D are 9, 9, 8, and 7, respectively.
+ *
+ * The bound is = maxFormulaSize - (shortest distance from $ROOT to cat)
+ *
+ * @author ppasupat
+ */
+public class CatSizeBound {
+  public static class Options {
+    @Option(gloss = "verbosity") public int verbose = 0;
+  }
+  public static Options opts = new Options();
+
+  private final int maxFormulaSize;
+  private final Map<String, Integer> bound = new HashMap<>();
+
+  public CatSizeBound(int maxFormulaSize, Grammar grammar) {
+    this(maxFormulaSize, grammar.getRules());
+  }
+
+  public CatSizeBound(int maxFormulaSize, List<Rule> rules) {
+    this.maxFormulaSize = maxFormulaSize;
+    if (!FloatingParser.opts.useSizeInsteadOfDepth) {
+      LogInfo.warnings("Currently CatSizeBound is usable only when useSizeInsteadOfDepth = true.");
+      return;
+    }
+    // Construct graph
+    Map<String, Set<String>> graph = new HashMap<>();
+    for (Rule rule : rules) {
+      if (!Rule.isCat(rule.lhs))
+        throw new RuntimeException("Non-cat found in LHS of rule " + rule);
+      for (String rhsCat : rule.rhs) {
+        if (Rule.isCat(rhsCat))
+          MapUtils.addToSet(graph, rule.lhs, rhsCat);
+      }
+    }
+    // Breadth first search
+    bound.put(Rule.rootCat, maxFormulaSize);
+    Queue<String> queue = new ArrayDeque<>();
+    queue.add(Rule.rootCat);
+    while (!queue.isEmpty()) {
+      String cat = queue.remove();
+      if (!graph.containsKey(cat)) continue;
+      for (String rhsCat : graph.get(cat)) {
+        if (bound.containsKey(rhsCat)) continue;
+        bound.put(rhsCat, bound.get(cat) - 1);
+        queue.add(rhsCat);
+      }
+    }
+    if (opts.verbose >= 1) {
+      LogInfo.begin_track("CatSizeBound: distances");
+      for (Map.Entry<String, Integer> entry : bound.entrySet())
+        LogInfo.logs("%25s : %2d", entry.getKey(), entry.getValue());
+      LogInfo.end_track();
+    }
+  }
+
+  public int getBound(String cat) {
+    return bound.getOrDefault(cat, maxFormulaSize);
+  }
+
+}

src/edu/stanford/nlp/sempre/ChartParserState.java

@@ -22,7 +22,7 @@ public abstract class ChartParserState extends ParserState {
   protected final Map<String, List<Derivation>>[][] chart;
 
   // For visualizing how chart is filled
-  List<CatSpan> chartFillingList = new ArrayList<>();
+  protected List<CatSpan> chartFillingList = new ArrayList<>();
 
   protected String[][] phrases; // the phrases in the example
 
@@ -74,7 +74,7 @@ public abstract class ChartParserState extends ParserState {
     }
   }
 
-  void addToChart(Derivation deriv) {
+  protected void addToChart(Derivation deriv) {
     if (parser.verbose(3)) LogInfo.logs("addToChart %s: %s", deriv.cat, deriv);
 
     if (Parser.opts.pruneErrorValues && deriv.value instanceof ErrorValue) return;
@@ -90,6 +90,10 @@ public abstract class ChartParserState extends ParserState {
     }
   }
 
+  public Map<String, List<Derivation>>[][] getChart() {
+    return chart;
+  }
+
   // TODO(joberant): move to visualization utility class
   public static class CatSpan {
     @JsonProperty

src/edu/stanford/nlp/sempre/ChildDerivationsGroup.java

@@ -0,0 +1,36 @@
+package edu.stanford.nlp.sempre;
+
+import java.util.List;
+
+/**
+ * A group containing one or two lists of potential child derivations.
+ *
+ * The motivation is to group potential child derivations based on type compatibility.
+ * For example, when building (and __ __), considering all pairs of derivations
+ * is time-wasting since a lot of pairs don't type-check. We instead group
+ * derivations by type, and only apply the rule to the pairs that type-check.
+ * 
+ * This idea also extends to one-argument rules. For example, for (sum ___),
+ * we should only look at child derivations with number type.
+ *
+ * During parsing, for each DerivationGroup:
+ * - For a one-argument rule (derivations2 == null):
+ *   Apply the rule on all derivations in derivations1
+ * - For a two-argument rule (derivations2 != null):
+ *   Apply the rule to all pairs (d1, d2) where d1 is in derivations1 and d2 is in derivations2
+ *
+ * @author ppasupat
+ */
+public class ChildDerivationsGroup {
+  public final List<Derivation> derivations1, derivations2;
+
+  public ChildDerivationsGroup(List<Derivation> derivations1) {
+    this.derivations1 = derivations1;
+    this.derivations2 = null;
+  }
+
+  public ChildDerivationsGroup(List<Derivation> derivations1, List<Derivation> derivations2) {
+    this.derivations1 = derivations1;
+    this.derivations2 = derivations2;
+  }
+}

src/edu/stanford/nlp/sempre/ContextValue.java

@@ -68,7 +68,7 @@ public class ContextValue extends Value {
   }
 
   public ContextValue(KnowledgeGraph graph) {
-    this(null, null, null, graph);
+    this(null, null, new ArrayList(), graph);
   }
 
   // Example:
@@ -107,8 +107,11 @@ public class ContextValue extends Value {
       tree.addChild(LispTree.proto.newList("user", user));
     if (date != null)
       tree.addChild(date.toLispTree());
+		// When logging examples, logging the entire graph takes too much screen space.
+		// I don't think that we ever deserialize a graph from a serialized context,
+		// so this should be fine.
     if (graph != null)
-      tree.addChild(graph.toLispTree());
+      tree.addChild(graph.toShortLispTree());
     for (Exchange e : exchanges)
       tree.addChild(LispTree.proto.newList("exchange", e.toLispTree()));
     return tree;

src/edu/stanford/nlp/sempre/Dataset.java

@@ -38,6 +38,9 @@ public class Dataset {
 
     @Option(gloss = "Only keep examples which have at most this number of tokens")
     public int maxTokens = Integer.MAX_VALUE;
+
+    @Option(gloss = "Path to a knowledge graph that will be uploaded as global context")
+    public String globalGraphPath;
   }
 
   public static Options opts = new Options();
@@ -96,10 +99,22 @@ public class Dataset {
         return;
       }
     }
-
     readLispTreeFromPathPairs(pathPairs);
+    updateGlobalContext();
   }
 
+  private void updateGlobalContext() {
+    if (opts.globalGraphPath != null) {
+      KnowledgeGraph graph = NaiveKnowledgeGraph.fromFile(opts.globalGraphPath);
+      for (String group : allExamples.keySet()) {
+        for (Example ex : allExamples.get(group)) {
+          ex.setContext(new ContextValue(graph));
+        }
+      }
+    }
+  }
+
+
   private void readJsonFromPathPairs(List<Pair<String, String>> pathPairs) {
     List<GroupInfo> groups = Lists.newArrayListWithCapacity(pathPairs.size());
     for (Pair<String, String> pathPair : pathPairs) {
@@ -213,8 +228,10 @@ public class Dataset {
     while (examples.size() < maxExamples && trees.hasNext()) {
       // Format: (example (id ...) (utterance ...) (targetFormula ...) (targetValue ...))
       LispTree tree = trees.next();
-      if (tree.children.size() < 2 && !"example".equals(tree.child(0).value))
+      if (tree.children.size() < 2 || !"example".equals(tree.child(0).value)) {
+        if ("metadata".equals(tree.child(0).value)) continue;
         throw new RuntimeException("Invalid example: " + tree);
+      }
 
       Example ex = Example.fromLispTree(tree, path + ":" + n);  // Specify a default id if it doesn't exist
       n++;
@@ -241,7 +258,7 @@ public class Dataset {
     LogInfo.end_track();
   }
 
-  private static int getMaxExamplesForGroup(String group) {
+  public static int getMaxExamplesForGroup(String group) {
     int maxExamples = Integer.MAX_VALUE;
     for (Pair<String, Integer> maxPair : opts.maxExamples)
       if (maxPair.getFirst().equals(group))

src/edu/stanford/nlp/sempre/DateRangeFn.java

@@ -0,0 +1,64 @@
+package edu.stanford.nlp.sempre;
+
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Parse strings representing date ranges such as "20th century".
+ * The result has the form (and (< ...) (>= ...)).
+ * 
+ * Currently only supports patterns like "1990's" and "1800s"
+ *
+ * @author ppasupat
+ */
+public class DateRangeFn extends SemanticFn {
+
+  @Override
+  public DerivationStream call(Example ex, Callable c) {
+    return new LazyDateRangeFnDerivs(ex, c);
+  }
+
+  public static class LazyDateRangeFnDerivs extends MultipleDerivationStream {
+    Example ex;
+    Callable c;
+
+    int index = 0;
+    List<Formula> formulas;
+
+    public LazyDateRangeFnDerivs(Example ex, Callable c) {
+      this.ex = ex;
+      this.c = c;
+    }
+
+    @Override
+    public Derivation createDerivation() {
+      if (formulas == null)
+        populateFormulas();
+
+      if (index >= formulas.size()) return null;
+      Formula formula = formulas.get(index++);
+
+      return new Derivation.Builder().withCallable(c)
+          .formula(formula).type(SemType.numberType).createDerivation();
+    }
+
+    private static final Pattern YEAR_RANGE = Pattern.compile("^(\\d+0+)\\s*'?s$");
+
+    // TODO: Handle more cases
+    private void populateFormulas() {
+      formulas = new ArrayList<>();
+      String query = c.childStringValue(0);
+      Matcher matcher = YEAR_RANGE.matcher(query);
+      if (!matcher.matches()) return;
+      int year = Integer.parseInt(matcher.group(1)), range = 10;
+      while (year % range == 0) {
+        // Put "<" before ">=" to keep the children of MergeFormula sorted
+        formulas.add(new MergeFormula(MergeFormula.Mode.and,
+            new JoinFormula(new ValueFormula<Value>(new NameValue("<")), new ValueFormula<>(new NumberValue(year + range))),
+            new JoinFormula(new ValueFormula<Value>(new NameValue(">=")), new ValueFormula<>(new NumberValue(year)))));
+        range *= 10;
+      }
+    }
+  }
+}

src/edu/stanford/nlp/sempre/DateValue.java

@@ -80,6 +80,14 @@ public class DateValue extends Value {
     return tree;
   }
 
+  @Override public String sortString() { return "" + year + "/" + month + "/" + day; }
+  public String isoString() {
+    return "" + (year == -1 ? "xxxx" : String.format("%04d", year))
+        + "-" + (month == -1 ? "xx" : String.format("%02d", month))
+        + "-" + (day == -1 ? "xx" : String.format("%02d", day));
+  }
+  @Override public String pureString() { return isoString(); }
+
   @Override public int hashCode() {
     int hash = 0x7ed55d16;
     hash = hash * 0xd3a2646c + year;

src/edu/stanford/nlp/sempre/DefaultDerivationPruningComputer.java

@@ -0,0 +1,161 @@
+package edu.stanford.nlp.sempre;
+
+import java.util.*;
+
+import fig.basic.LispTree;
+import fig.basic.Option;
+
+/**
+ * Common pruning strategies that can be used in many semantic parsing tasks.
+ *
+ * @author ppasupat
+ */
+public class DefaultDerivationPruningComputer extends DerivationPruningComputer {
+  public static class Options {
+    @Option(gloss = "(for badSummarizerHead) allow count on sets of size 1")
+    public boolean allowCountOne = false;
+  }
+  public static Options opts = new Options();
+
+  public DefaultDerivationPruningComputer(DerivationPruner pruner) {
+    super(pruner);
+  }
+
+  public static final String atomic = "atomic";
+  public static final String emptyDenotation = "emptyDenotation";
+  public static final String nonLambdaError = "nonLambdaError";
+  public static final String tooManyValues = "tooManyValues";
+  public static final String doubleSummarizers = "doubleSummarizers";
+  public static final String multipleSuperlatives = "multipleSuperlatives";
+  public static final String sameMerge = "sameMerge";
+  public static final String mistypedMerge = "mistypedMerge";
+  public static final String unsortedMerge = "unsortedMerge";
+  public static final String badSummarizerHead = "badSummarizerHead";
+
+  @Override
+  public Collection<String> getAllStrategyNames() {
+    return Arrays.asList(
+        atomic,
+        emptyDenotation, nonLambdaError, tooManyValues,
+        doubleSummarizers, multipleSuperlatives,
+        sameMerge, mistypedMerge, unsortedMerge, badSummarizerHead);
+  }
+
+  @Override
+  public String isPruned(Derivation deriv) {
+    // atomic: Prune atomic formula at root.
+    //   e.g., Prevent "Who was taller, Lincoln or Obama" --> fb:en.lincoln generated from lexicon without any computation   
+    if (containsStrategy(atomic)) {
+      if (deriv.isRoot(ex.numTokens()) && deriv.formula instanceof ValueFormula)
+        return atomic;
+    }
+    // emptyDenotation: Prune if the denotation is empty
+    if (containsStrategy(emptyDenotation)) {
+      deriv.ensureExecuted(parser.executor, ex.context);
+      if (deriv.value instanceof ListValue && ((ListValue) deriv.value).values.isEmpty())
+        return emptyDenotation;
+    }
+    // nonLambdaError: Prune if the denotation is an error and the formula is not a partial formula
+    if (containsStrategy(nonLambdaError) && !isLambdaFormula(deriv.formula)) {
+      deriv.ensureExecuted(parser.executor, ex.context);
+      if (deriv.value instanceof ErrorValue)
+        return nonLambdaError;
+    }
+    // tooManyValues: Prune if the denotation has too many values (at $ROOT only)
+    if (containsStrategy(tooManyValues) && deriv.isRoot(ex.numTokens())) {
+      if (!(deriv.value instanceof ListValue) ||
+          ((ListValue) deriv.value).values.size() > DerivationPruner.opts.maxNumValues)
+        return tooManyValues;
+    }
+    // doubleSummarizers: Prune when two summarizers (aggregate or superlative) are directly nested
+    // e.g., in (sum (avg ...)) and (min (argmax ...)), the outer operation is redundant
+    if (containsStrategy(doubleSummarizers)) {
+      Formula innerFormula = null;
+      if (deriv.formula instanceof SuperlativeFormula)
+        innerFormula = ((SuperlativeFormula) deriv.formula).head;
+      else if (deriv.formula instanceof AggregateFormula)
+        innerFormula = ((AggregateFormula) deriv.formula).child;
+      if (innerFormula != null &&
+          (innerFormula instanceof SuperlativeFormula || innerFormula instanceof AggregateFormula))
+        return doubleSummarizers;
+    }
+    // multipleSuperlatives: Prune when more than one superlatives are used
+    // (don't need to be adjacent)
+    if (containsStrategy(multipleSuperlatives)) {
+      List<LispTree> stack = new ArrayList<>();
+      int count = 0;
+      stack.add(deriv.formula.toLispTree());
+      while (!stack.isEmpty()) {
+        LispTree tree = stack.remove(stack.size() - 1);
+        if (tree.isLeaf()) {
+          if ("argmax".equals(tree.value) || "argmin".equals(tree.value)) {
+            count++;
+            if (count >= 2)
+              return multipleSuperlatives;
+          }
+        } else {
+          for (LispTree subtree : tree.children)
+            stack.add(subtree);
+        }
+      }
+    }
+    // sameMerge: Prune merge formulas with two identical children
+    if (containsStrategy(sameMerge) && deriv.formula instanceof MergeFormula) {
+      MergeFormula merge = (MergeFormula) deriv.formula;
+      if (merge.child1.equals(merge.child2))
+        return sameMerge;
+    }
+    // mistypedMerge: Prune merge formulas with children of different types
+    if (containsStrategy(mistypedMerge) && deriv.formula instanceof MergeFormula) {
+      MergeFormula merge = (MergeFormula) deriv.formula;
+      SemType type1 = TypeInference.inferType(merge.child1, true);
+      SemType type2 = TypeInference.inferType(merge.child2, true);
+      if (!type1.meet(type2).isValid())
+        return mistypedMerge;
+    }
+    // unsortedMerge: Prune merge formulas where the children's string forms are not lexicographically sorted.
+    //   Will remove redundant (and Y X) when (and Y X) is already present.
+    if (containsStrategy(unsortedMerge) && deriv.formula instanceof MergeFormula) {
+      MergeFormula merge = (MergeFormula) deriv.formula;
+      String child1 = merge.child1.toString(), child2 = merge.child2.toString();
+      if (child1.compareTo(child2) >= 0)
+        return unsortedMerge;
+    }
+    // badSummarizerHead: Prune if the head of a superlative or a non-count aggregate
+    // is empty or is a single object
+    if (containsStrategy(badSummarizerHead)) {
+      Formula innerFormula = null;
+      boolean isCount = false;
+      if (deriv.formula instanceof SuperlativeFormula)
+        innerFormula = ((SuperlativeFormula) deriv.formula).head;
+      else if (deriv.formula instanceof AggregateFormula) {
+        innerFormula = ((AggregateFormula) deriv.formula).child;
+        if (((AggregateFormula) deriv.formula).mode == AggregateFormula.Mode.count)
+          isCount = true;
+      }
+      if (innerFormula != null) {
+        try {
+          TypeInference.inferType(innerFormula);
+          Value innerValue = parser.executor.execute(innerFormula, ex.context).value;
+          if (innerValue instanceof ListValue) {
+            int size = ((ListValue) innerValue).values.size();
+            if (size == 0 || (size == 1 && !(opts.allowCountOne && isCount)))
+              return badSummarizerHead;
+          }
+        } catch (Exception e) {
+          // TypeInference fails; probably because of free variables. No need to do anything.
+        }
+      }
+    }
+    return null;
+  }
+
+  // Helper function: return true if the result is clearly a binary
+  private boolean isLambdaFormula(Formula formula) {
+    if (formula instanceof LambdaFormula) return true;
+    if (formula instanceof ValueFormula &&
+        CanonicalNames.isBinary(((ValueFormula<?>) formula).value)) return true;
+    return false;
+  }
+
+}

src/edu/stanford/nlp/sempre/Derivation.java

@@ -1,9 +1,9 @@
 package edu.stanford.nlp.sempre;
 
-import fig.basic.*;
-
 import java.util.*;
 
+import fig.basic.*;
+
 /**
  * A Derivation corresponds to the production of a (partial) logical form
  * |formula| from a span of the utterance [start, end). Contains the formula and
@@ -24,8 +24,14 @@ public class Derivation implements SemanticFn.Callable, HasScore {
     public boolean showRules = false;
     @Option(gloss = "When printing derivations, to show canonical utterance")
     public boolean showUtterance = false;
+    @Option(gloss = "When printing derivations, show the category")
+    public boolean showCat = false;
     @Option(gloss = "When executing, show formulae (for debugging)")
     public boolean showExecutions = false;
+    @Option(gloss = "Pick the comparator used to sort derivations")
+    public String derivComparator = "ScoredDerivationComparator";
+    @Option(gloss = "bonus score for being all anchored")
+    public double anchoredBonus = 0.0;
   }
 
   public static Options opts = new Options();
@@ -40,7 +46,24 @@ public class Derivation implements SemanticFn.Callable, HasScore {
   // Floating cell information
   // TODO(yushi): make fields final
   public String canonicalUtterance;
-  private boolean[] anchoredTokens;   // Tokens which anchored rules are defined on
+  public boolean allAnchored = true;
+  private int[] numAnchors;     // Number of times each token was anchored
+
+  /**
+  * Information for grammar induction.
+  * For each descendant derivation of the body, this class tracks where and what in the head it matches
+  * GrammarInfo.start, GrammarInfo.end refer to matching positions in the head, as opposed to the body
+  * @author sidaw
+  **/
+  public class GrammarInfo {
+    public boolean anchored = false;
+    public boolean matched = false;
+    public int start = -1, end = -1;
+    public Formula formula;
+    public List<Derivation> matches = new ArrayList<>();
+  }
+  public GrammarInfo grammarInfo = new GrammarInfo();
+
 
   // If this derivation is composed of other derivations
   public final Rule rule;  // Which rule was used to produce this derivation?  Set to nullRule if not.
@@ -63,6 +86,7 @@ public class Derivation implements SemanticFn.Callable, HasScore {
   // Information for scoring
   private final FeatureVector localFeatureVector;  // Features
   double score = Double.NaN;  // Weighted combination of features
+  double prob = Double.NaN;  // Probability (normalized exp of score).
 
   // Used during parsing (by FeatureExtractor, SemanticFn) to cache arbitrary
   // computation across different sub-Derivations.
@@ -78,8 +102,6 @@ public class Derivation implements SemanticFn.Callable, HasScore {
 
   // Number in [0, 1] denoting how correct the value is.
   public double compatibility = Double.NaN;
-  // Probability (normalized exp of score).
-  public double prob = Double.NaN;
 
   // Miscellaneous statistics
   int maxBeamPosition = -1;  // Lowest position that this tree or any of its children is on the beam (after sorting)
@@ -94,7 +116,9 @@ public class Derivation implements SemanticFn.Callable, HasScore {
   // we can break ties consistently for reproducible results.
   long creationIndex;
   public static long numCreated = 0;  // Incremented for each derivation we create.
-  public static final Comparator<Derivation> derivScoreComparator = new ScoredDerivationComparator();
+  @SuppressWarnings("unchecked")
+  public static final Comparator<Derivation> derivScoreComparator =
+      (Comparator<Derivation>)Utils.newInstanceHard(SempreUtils.resolveClassName("Derivation$" + opts.derivComparator));
 
   public static final List<Derivation> emptyList = Collections.emptyList();
 
@@ -152,6 +176,24 @@ public class Derivation implements SemanticFn.Callable, HasScore {
       return this;
     }
 
+    public Builder withAllFrom(Derivation deriv) {
+      this.cat = deriv.cat;
+      this.start = deriv.start;
+      this.end = deriv.end;
+      this.rule = deriv.rule;
+      this.children = deriv.children == null ? null : new ArrayList<>(deriv.children);
+      this.formula = deriv.formula;
+      this.type = deriv.type;
+      this.localFeatureVector = deriv.localFeatureVector;
+      this.score = deriv.score;
+      this.value = deriv.value;
+      this.executorStats = deriv.executorStats;
+      this.compatibility = deriv.compatibility;
+      this.prob = deriv.prob;
+      this.canonicalUtterance = deriv.canonicalUtterance;
+      return this;
+    }
+
     public Derivation createDerivation() {
       return new Derivation(
           cat, start, end, rule, children, formula, type,
@@ -224,7 +266,12 @@ public class Derivation implements SemanticFn.Callable, HasScore {
   public void addFeatures(FeatureVector fv) { this.localFeatureVector.add(fv); }
 
   public double localScore(Params params) {
-    return localFeatureVector.dotProduct(params);
+    return localFeatureVector.dotProduct(params) + (this.allAnchored()? opts.anchoredBonus : 0.0);
+  }
+
+  // SHOULD NOT BE USED except during test time if the memory is desperately needed.
+  public void clearFeatures() {
+    localFeatureVector.clear();
   }
 
   /**
@@ -290,6 +337,9 @@ public class Derivation implements SemanticFn.Callable, HasScore {
     if (opts.showUtterance && canonicalUtterance != null) {
       tree.addChild(LispTree.proto.newList("canonicalUtterance", canonicalUtterance));
     }
+    if (opts.showCat && cat != null) {
+      tree.addChild(LispTree.proto.newList("cat", cat));
+    }
     return tree;
   }
 
@@ -410,6 +460,25 @@ public class Derivation implements SemanticFn.Callable, HasScore {
     }
   }
 
+  //Used to compare derivations by score, prioritizing the fully anchored.
+  public static class AnchorPriorityScoreComparator implements Comparator<Derivation> {
+    @Override
+    public int compare(Derivation deriv1, Derivation deriv2) {
+      boolean deriv1Core = deriv1.allAnchored();
+      boolean deriv2Core = deriv2.allAnchored();
+
+      if (deriv1Core && !deriv2Core) return -1;
+      if (deriv2Core && !deriv1Core) return +1;
+
+      if (deriv1.score > deriv2.score) return -1;
+      if (deriv1.score < deriv2.score) return +1;
+      // Ensure reproducible randomness
+      if (deriv1.creationIndex < deriv2.creationIndex) return -1;
+      if (deriv1.creationIndex > deriv2.creationIndex) return +1;
+      return 0;
+    }
+  }
+
   // for debugging
   public void printDerivationRecursively() {
     LogInfo.logs("Deriv: %s(%s,%s) %s", cat, start, end, formula);
@@ -448,27 +517,63 @@ public class Derivation implements SemanticFn.Callable, HasScore {
         child.clearTempState();
   }
 
-  // Compute anchoredTokens and return the result
-  // anchoredTokens[>= anchoredTokens.length] are False by default
-  public boolean[] getAnchoredTokens() {
-    if (anchoredTokens == null) {
+  /**
+   * Return an int array numAnchors where numAnchors[i] is
+   * the number of times we anchored on token i.
+   *
+   * numAnchors[>= numAnchors.length] are 0 by default.
+   */
+  public int[] getNumAnchors() {
+    if (numAnchors == null) {
       if (rule.isAnchored()) {
-        anchoredTokens = new boolean[end];
-        for (int i = start; i < end; i++) anchoredTokens[i] = true;
+        numAnchors = new int[end];
+        for (int i = start; i < end; i++) numAnchors[i] = 1;
       } else {
-        anchoredTokens = new boolean[0];
+        numAnchors = new int[0];
         for (Derivation child : children) {
-          boolean[] childAnchoredTokens = child.getAnchoredTokens();
-          if (anchoredTokens.length < childAnchoredTokens.length) {
-            boolean[] newAnchoredTokens = new boolean[childAnchoredTokens.length];
-            for (int i = 0; i < anchoredTokens.length; i++) newAnchoredTokens[i] = anchoredTokens[i];
-            anchoredTokens = newAnchoredTokens;
+          int[] childNumAnchors = child.getNumAnchors();
+          if (numAnchors.length < childNumAnchors.length) {
+            int[] newNumAnchors = new int[childNumAnchors.length];
+            for (int i = 0; i < numAnchors.length; i++)
+              newNumAnchors[i] = numAnchors[i];
+            numAnchors = newNumAnchors;
           }
-          for (int i = 0; i < childAnchoredTokens.length; i++)
-            anchoredTokens[i] = anchoredTokens[i] || childAnchoredTokens[i];
+          for (int i = 0; i < childNumAnchors.length; i++)
+            numAnchors[i] += childNumAnchors[i];
         }
       }
     }
-    return anchoredTokens.clone();
+    return numAnchors;
+  }
+
+  /**
+   * Return a boolean array anchoredTokens where anchoredTokens[i]
+   * indicates whether we have anchored on token i.
+   *
+   * anchoredTokens[>= anchoredTokens.length] are False by default
+   */
+  public boolean[] getAnchoredTokens() {
+    int[] numAnchors = getNumAnchors();
+    boolean[] anchoredTokens = new boolean[numAnchors.length];
+    for (int i = 0; i < numAnchors.length; i++)
+      anchoredTokens[i] = (numAnchors[i] > 0);
+    return anchoredTokens;
+  }
+
+  public Derivation betaReduction() {
+    Formula reduced = Formulas.betaReduction(formula);
+    return new Builder().withAllFrom(this).formula(reduced).createDerivation();
+  }
+
+  public boolean allAnchored() {
+    if (rule.isInduced() || !this.allAnchored) {
+      this.allAnchored = false;
+      return false;
+    } else {
+      for (Derivation child : children) {
+        if (child.allAnchored() == false) return false;
+      }
+      return true;
+    }
   }
 }

src/edu/stanford/nlp/sempre/DerivationPruner.java

@@ -15,21 +15,27 @@ import fig.basic.*;
 
 public class DerivationPruner {
   public static class Options {
-    @Option public List<String> pruningStrategies = new ArrayList<>();
-    @Option public List<String> pruningComputers = new ArrayList<>();
+    @Option(gloss = "Pruning strategies to use")
+    public List<String> pruningStrategies = new ArrayList<>();
+    @Option(gloss = "DerivationPruningComputer subclasses to look for pruning strategies")
+    public List<String> pruningComputers = new ArrayList<>();
     @Option public int pruningVerbosity = 0;
-    @Option public int maxNumValues = 10;
+    @Option(gloss = "(for tooManyValues) maximum denotation size of the final formula")
+    public int maxNumValues = 10;
   }
   public static Options opts = new Options();
 
   public final Parser parser;
   public final Example ex;
   private List<DerivationPruningComputer> pruningComputers = new ArrayList<>();
-  private List<String> customAllowedDomains;
+  // If not null, limit the pruning strategies to this list in addition to opts.pruningStrategies.
+  private List<String> customAllowedPruningStrategies;
+  private final Set<String> allStrategyNames;
 
   public DerivationPruner(ParserState parserState) {
     this.parser = parserState.parser;
     this.ex = parserState.ex;
+    this.pruningComputers.add(new DefaultDerivationPruningComputer(this));
     for (String pruningComputer : opts.pruningComputers) {
       try {
         Class<?> pruningComputerClass = Class.forName(SempreUtils.resolveClassName(pruningComputer));
@@ -42,173 +48,64 @@ public class DerivationPruner {
         throw new RuntimeException("Error while instantiating pruning computer: " + pruningComputer);
       }
     }
+    // Compile the list of all strategies
+    allStrategyNames = new HashSet<>();
+    for (DerivationPruningComputer computer : pruningComputers)
+      allStrategyNames.addAll(computer.getAllStrategyNames());
+    for (String strategy : opts.pruningStrategies) {
+      if (!allStrategyNames.contains(strategy))
+        LogInfo.fails("Pruning strategy '%s' not found!", strategy);
+    }
   }
 
-  public void setCustomAllowedDomains(List<String> customAllowedDomains) {
-    this.customAllowedDomains = customAllowedDomains;
+  /**
+   * Set additional restrictions on the pruning strategies.
+   *
+   * If customAllowedPruningStrategies is not null, the pruning strategy must be in both
+   * opts.pruningStrategies and customAllowedPruningStrategies in order to be used.
+   *
+   * Useful when some pruning strategies can break the parsing mechanism.
+   */
+  public void setCustomAllowedPruningStrategies(List<String> customAllowedPruningStrategies) {
+    this.customAllowedPruningStrategies = customAllowedPruningStrategies;
   }
 
   protected boolean containsStrategy(String name) {
-     return opts.pruningStrategies.contains(name) &&
-         (customAllowedDomains == null || customAllowedDomains.contains(name));
+    return opts.pruningStrategies.contains(name) &&
+        (customAllowedPruningStrategies == null || customAllowedPruningStrategies.contains(name));
+  }
+  
+  public List<DerivationPruningComputer> getPruningComputers() {
+    return new ArrayList<>(pruningComputers);
   }
 
+  /**
+   * Return true if the derivation should be pruned. Otherwise, return false.
+   */
   public boolean isPruned(Derivation deriv) {
     if (opts.pruningStrategies.isEmpty() && pruningComputers.isEmpty()) return false;
-    if (pruneFormula(deriv)) return true;
-    if (pruneDenotation(deriv)) return true;
-    for (DerivationPruningComputer pruningComputer : pruningComputers) {
-      if (pruningComputer.isPruned(deriv)) return true;
-    }
-    return false;
-  }
-
-  // ============================================================
-  // Formula-based Pruning
-  // ============================================================
-
-  private boolean pruneFormula(Derivation deriv) {
-    return pruneSingleton(deriv) || pruneSuperlatives(deriv) || pruneMerges(deriv);
-  }
-
-  /**
-   * Prune singleton formula at the root.
-   */
-  private boolean pruneSingleton(Derivation deriv) {
-    if (!containsStrategy("singleton")) return false;
-    return deriv.isRoot(ex.numTokens()) && deriv.formula instanceof ValueFormula;
-  }
-
-  /**
-   * Prune strings of multiple superlatives.
-   */
-  private boolean pruneSuperlatives(Derivation deriv) {
-    if (containsStrategy("doubleSuperlatives")) {
-      // Prune if there is an arg{max|min} whose head has arg{max|min}
-      if (deriv.formula instanceof SuperlativeFormula) {
-        SuperlativeFormula superlative = (SuperlativeFormula) deriv.formula;
-        if (superlative.head instanceof SuperlativeFormula) {
-          if (opts.pruningVerbosity >= 2)
-            LogInfo.logs("PRUNED [doubleSuperlatives] %s", deriv.formula);
-          return true;
-        }
-      }
-    }
-    if (containsStrategy("multipleSuperlatives")) {
-      // Prune if there are more than arg{max|min} appearing in the formula (don't need to be adjacent)
-      List<LispTree> stack = new ArrayList<>();
-      int count = 0;
-      stack.add(deriv.formula.toLispTree());
-      while (!stack.isEmpty()) {
-        LispTree tree = stack.remove(stack.size() - 1);
-        if (tree.isLeaf()) {
-          if ("argmax".equals(tree.value) || "argmin".equals(tree.value)) {
-            count++;
-            if (count >= 2) {
-              if (opts.pruningVerbosity >= 2)
-                LogInfo.logs("PRUNED [multipleSuperlatives] %s", deriv.formula);
-              return true;
-            }
-          }
-        } else {
-          for (LispTree subtree : tree.children)
-            stack.add(subtree);
-        }
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Prune merges.
-   */
-  private boolean pruneMerges(Derivation deriv) {
-    if (!(deriv.formula instanceof MergeFormula)) return false;
-    MergeFormula merge = (MergeFormula) deriv.formula;
-    if (containsStrategy("sameMerge")) {
-      if (merge.child1.equals(merge.child2)) {
+    String matchedStrategy;
+    for (DerivationPruningComputer computer : pruningComputers) {
+      if ((matchedStrategy = computer.isPruned(deriv)) != null) {
         if (opts.pruningVerbosity >= 2)
-          LogInfo.logs("PRUNED [sameMerge] %s", deriv.formula);
+          LogInfo.logs("PRUNED [%s] %s", matchedStrategy, deriv.formula);
         return true;
       }
     }
     return false;
   }
 
-  // ============================================================
-  // Denotation-based Pruning
-  // ============================================================
-
   /**
-   * Pruning based on denotations.
+   * Run isPruned with a (temporary) custom set of allowed pruning strategies.
+   * If customAllowedPruningStrategies is null, all strategies are allowed.
+   * If customAllowedPruningStrategies is empty, no pruning happens.
    */
-  private boolean pruneDenotation(Derivation deriv) {
-    return pruneFinalDenotation(deriv) || prunePartialDenotation(deriv);
+  public boolean isPruned(Derivation deriv, List<String> customAllowedPruningStategies) {
+    List<String> old = this.customAllowedPruningStrategies;
+    this.customAllowedPruningStrategies = customAllowedPruningStategies;
+    boolean answer = isPruned(deriv);
+    this.customAllowedPruningStrategies = old;
+    return answer;
   }
 
-  private boolean pruneFinalDenotation(Derivation deriv) {
-    Formula formula = deriv.formula;
-    // Prune if the denotation is an empty list
-    if (containsStrategy("emptyDenotation")) {
-      deriv.ensureExecuted(parser.executor, ex.context);
-      if (deriv.value instanceof ListValue) {
-        if (((ListValue) deriv.value).values.isEmpty()) {
-          if (opts.pruningVerbosity >= 3)
-            LogInfo.logs("PRUNED [emptyDenotation] %s", formula);
-          return true;
-        }
-      }
-    }
-    // Prune if the denotation is an error and the formula is not a partial formula
-    if (containsStrategy("nonLambdaError") && !(deriv.formula instanceof LambdaFormula)) {
-      deriv.ensureExecuted(parser.executor, ex.context);
-      if (deriv.value instanceof ErrorValue) {
-        if (opts.pruningVerbosity >= 3)
-          LogInfo.logs("PRUNED [nonLambdaError] %s", formula);
-        return true;
-      }
-    }
-    // Prune if the denotation has too many values
-    if (containsStrategy("tooManyValues") && deriv.isRoot(ex.numTokens())) {
-      deriv.ensureExecuted(parser.executor, ex.context);
-      if (deriv.value instanceof ListValue) {
-        if (((ListValue) deriv.value).values.size() > opts.maxNumValues) {
-          if (opts.pruningVerbosity >= 3)
-            LogInfo.logs("PRUNED [tooManyValues] %s", formula);
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
-  private boolean prunePartialDenotation(Derivation deriv) {
-    Formula formula = deriv.formula;
-    if (containsStrategy("badSuperlativeHead")) {
-      Formula head = null;
-      if (formula instanceof AggregateFormula)
-        head = ((AggregateFormula) formula).child;
-      else if (formula instanceof SuperlativeFormula)
-        head = ((SuperlativeFormula) formula).head;
-      if (head != null) {
-        Value headValue = parser.executor.execute(head, ex.context).value;
-        if (headValue instanceof ListValue && ((ListValue) headValue).values.size() < 2) {
-          if (opts.pruningVerbosity >= 3)
-            LogInfo.logs("PRUNED [badSuperlativeHead] %s", formula);
-          return true;
-        }
-      }
-    }
-    if (containsStrategy("mistypedMerge") && formula instanceof MergeFormula) {
-      MergeFormula merge = (MergeFormula) formula;
-      SemType type1 = TypeInference.inferType(merge.child1);
-      SemType type2 = TypeInference.inferType(merge.child2);
-      if (!type1.meet(type2).isValid()) {
-        if (opts.pruningVerbosity >= 2)
-          LogInfo.logs("PRUNED [mistypedMerge] %s", deriv.formula);
-        return true;
-      }
-    }
-    return false;
-  }
 }

src/edu/stanford/nlp/sempre/DerivationPruningComputer.java

@@ -1,5 +1,7 @@
 package edu.stanford.nlp.sempre;
 
+import java.util.Collection;
+
 /**
  * Used to prune formulas during parsing.
  *
@@ -11,15 +13,31 @@ package edu.stanford.nlp.sempre;
 public abstract class DerivationPruningComputer {
 
   protected final DerivationPruner pruner;
+  protected final Parser parser;
+  protected final Example ex;
 
   public DerivationPruningComputer(DerivationPruner pruner) {
     this.pruner = pruner;
+    this.parser = pruner.parser;
+    this.ex = pruner.ex;
   }
 
+  /**
+   * Return a collection of all strategy names used in this class.
+   */
+  abstract public Collection<String> getAllStrategyNames();
+
+  // Shorthand
   protected boolean containsStrategy(String name) {
     return pruner.containsStrategy(name);
   }
 
-  public abstract boolean isPruned(Derivation deriv);
+  /**
+   * Prune the derivation.
+   *
+   * To add pruning strategies, override this method.
+   * Return the strategy name to prune the formula, and null otherwise.
+   */
+  public abstract String isPruned(Derivation deriv);
 
 }

src/edu/stanford/nlp/sempre/Example.java

@@ -36,7 +36,8 @@ public class Example {
 
   // What we should try to predict.
   @JsonProperty public Formula targetFormula;  // Logical form (e.g., database query)
-  @JsonProperty public Value targetValue;  // Denotation (e.g., answer)
+  public List<Formula> alternativeFormulas;    // Alternative logical form (less canonical)
+  @JsonProperty public Value targetValue;      // Denotation (e.g., answer)
 
   //// Information after preprocessing (e.g., tokenization, POS tagging, NER, syntactic parsing, etc.).
   public LanguageInfo languageInfo = null;
@@ -46,7 +47,7 @@ public class Example {
   // Predicted derivations (sorted by score).
   public List<Derivation> predDerivations;
 
-  // Temporary state while parsing an Example (see Derivation.java for analogous struture).
+  // Temporary state while parsing an Example (see Derivation.java for analogous structure).
   private Map<String, Object> tempState;
 
   // Statistics relating to processing the example.
@@ -58,7 +59,6 @@ public class Example {
     private ContextValue context;
     private Formula targetFormula;
     private Value targetValue;
-    private List<Derivation> predDerivations;
     private LanguageInfo languageInfo;
 
     public Builder setId(String id) { this.id = id; return this; }
@@ -103,6 +103,12 @@ public class Example {
 
   public void setContext(ContextValue context) { this.context = context; }
   public void setTargetFormula(Formula targetFormula) { this.targetFormula = targetFormula; }
+  public void setAlternativeFormulas(List<Formula> alternativeFormulas) { this.alternativeFormulas = alternativeFormulas; }
+  public void addAlternativeFormula(Formula alternativeFormula) {
+    if (this.alternativeFormulas == null)
+      this.alternativeFormulas = new ArrayList<>();
+    this.alternativeFormulas.add(alternativeFormula);
+  }
   public void setTargetValue(Value targetValue) { this.targetValue = targetValue; }
 
   public String spanString(int start, int end) {
@@ -153,18 +159,35 @@ public class Example {
     for (int i = 1; i < tree.children.size(); i++) {
       LispTree arg = tree.child(i);
       String label = arg.child(0).value;
-      if ("posTags".equals(label) || "nerTags".equals(label) || "url".equals(label)) {
-        // Do nothing
-      } else if ("tokens".equals(label)) {
-        int n = arg.child(1).children.size();
-        for (int j = 0; j < n; j++)
-          ex.languageInfo.tokens.add(arg.child(1).child(j).value);
+      if ("tokens".equals(label)) {
+        for (LispTree child : arg.child(1).children)
+          ex.languageInfo.tokens.add(child.value);
+      } else if ("lemmaTokens".equals(label)) {
+        for (LispTree child : arg.child(1).children)
+          ex.languageInfo.lemmaTokens.add(child.value);
+      } else if ("posTags".equals(label)) {
+        for (LispTree child : arg.child(1).children)
+          ex.languageInfo.posTags.add(child.value);
+      } else if ("nerTags".equals(label)) {
+        for (LispTree child : arg.child(1).children)
+          ex.languageInfo.nerTags.add(child.value);
+      } else if ("nerValues".equals(label)) {
+        for (LispTree child : arg.child(1).children)
+          ex.languageInfo.nerValues.add("null".equals(child.value) ? null : child.value);
+      } else if ("alternativeFormula".equals(label)) {
+        ex.addAlternativeFormula(Formulas.fromLispTree(arg.child(1)));
       } else if ("evaluation".equals(label)) {
         ex.evaluation = Evaluation.fromLispTree(arg.child(1));
       } else if ("predDerivations".equals(label)) {
+        // Featurized
         ex.predDerivations = new ArrayList<>();
         for (int j = 1; j < arg.children.size(); j++)
           ex.predDerivations.add(derivationFromLispTree(arg.child(j)));
+      } else if ("rawDerivations".equals(label) || "derivations".equals(label)) {
+        // Unfeaturized
+        ex.predDerivations = new ArrayList<>();
+        for (int j = 1; j < arg.children.size(); j++)
+          ex.predDerivations.add(rawDerivationFromLispTree(arg.child(j)));
       } else if (!Sets.newHashSet("id", "utterance", "targetFormula", "targetValue", "targetValues", "context", "original").contains(label)) {
         throw new RuntimeException("Invalid example argument: " + arg);
       }
@@ -175,7 +198,7 @@ public class Example {
 
   public void preprocess() {
     this.languageInfo = LanguageAnalyzer.getSingleton().analyze(this.utterance);
-    this.targetValue = TargetValuePreprocessor.getSingleton().preprocess(this.targetValue);
+    this.targetValue = TargetValuePreprocessor.getSingleton().preprocess(this.targetValue, this);
   }
 
   public void log() {
@@ -195,6 +218,21 @@ public class Example {
     LogInfo.end_track();
   }
 
+  public void logWithoutContext() {
+    LogInfo.begin_track("Example: %s", utterance);
+    LogInfo.logs("Tokens: %s", getTokens());
+    LogInfo.logs("Lemmatized tokens: %s", getLemmaTokens());
+    LogInfo.logs("POS tags: %s", languageInfo.posTags);
+    LogInfo.logs("NER tags: %s", languageInfo.nerTags);
+    LogInfo.logs("NER values: %s", languageInfo.nerValues);
+    if (targetFormula != null)
+      LogInfo.logs("targetFormula: %s", targetFormula);
+    if (targetValue != null)
+      LogInfo.logs("targetValue: %s", targetValue);
+    LogInfo.logs("Dependency children: %s", languageInfo.dependencyChildren);
+    LogInfo.end_track();
+  }
+
   public List<Derivation> getCorrectDerivations() {
     List<Derivation> res = new ArrayList<>();
     for (Derivation deriv : predDerivations) {
@@ -242,7 +280,14 @@ public class Example {
     return tree;
   }
 
-  private static Derivation derivationFromLispTree(LispTree item) {
+  /**
+   * Parse a featurized derivation.
+   *
+   * Format:
+   *   ({compatibility} {prob} {score} {value|null} {formula} {features})
+   *   where {features} = (({key} {value}) ({key} {value}) ...)
+   */
+  public static Derivation derivationFromLispTree(LispTree item) {
     Derivation.Builder b = new Derivation.Builder()
         .cat(Rule.rootCat)
         .start(-1)
@@ -271,7 +316,7 @@ public class Example {
     return b.createDerivation();
   }
 
-  private static LispTree derivationToLispTree(Derivation deriv) {
+  public static LispTree derivationToLispTree(Derivation deriv) {
     LispTree item = LispTree.proto.newList();
 
     item.addChild(deriv.compatibility + "");
@@ -290,6 +335,44 @@ public class Example {
     return item;
   }
 
+  /**
+   * Parse a LispTree with the format created by deriv.toLispTree().
+   * Due to the complexity, rules and children are not parsed.
+   *
+   * Format:
+   *   (derivation [(formula {formula})] [(value {value})] [(type {type})]
+   *               [(canonicalUtterance {canonicalUtterance})])
+   * @param item
+   * @return
+   */
+  public static Derivation rawDerivationFromLispTree(LispTree item) {
+    Derivation.Builder b = new Derivation.Builder()
+    .cat(Rule.rootCat)
+    .start(-1).end(-1)
+    .rule(Rule.nullRule)
+    .children(new ArrayList<Derivation>());
+    for (int i = 1; i < item.children.size(); i++) {
+      LispTree arg = item.child(i);
+      String label = arg.child(0).value;
+      if ("formula".equals(label)) {
+        b.formula(Formulas.fromLispTree(arg.child(1)));
+      } else if ("value".equals(label)) {
+        b.value(Values.fromLispTree(arg.child(1)));
+      } else if ("type".equals(label)) {
+        b.type(SemType.fromLispTree(arg.child(1)));
+      } else if ("canonicalUtterance".equals(label)) {
+        b.canonicalUtterance(arg.child(1).value);
+      } else {
+        throw new RuntimeException("Invalid example argument: " + arg);
+      }
+    }
+    return b.createDerivation();
+  }
+
+  public static LispTree rawDerivationToLispTree(Derivation deriv) {
+    return deriv.toLispTree();
+  }
+
   public Map<String, Object> getTempState() {
     // Create the tempState if it doesn't exist.
     if (tempState == null)
@@ -299,4 +382,13 @@ public class Example {
   public void clearTempState() {
     tempState = null;
   }
+
+  /**
+   * Clean up things to save memory
+   */
+  public void clean() {
+    predDerivations.clear();
+    if (context.graph != null)
+      context.graph.clean();
+  }
 }

src/edu/stanford/nlp/sempre/ExampleUtils.java

@@ -23,7 +23,7 @@ public final class ExampleUtils {
   }
 
   public static void writeJson(List<Example> examples, String outPath) throws IOException {
-    PrintWriter out = edu.stanford.nlp.io.IOUtils.getPrintWriter(outPath);
+    PrintWriter out = IOUtils.openOutHard(outPath);
     out.println("[");
     for (int i = 0; i < examples.size(); ++i) {
       Example ex = examples.get(i);
@@ -119,6 +119,29 @@ public final class ExampleUtils {
     out.close();
   }
 
+  public static void writePredictionTSV(int iter, String group, Example ex) {
+    String basePath = "preds-iter" + iter + "-" + group + ".tsv";
+    String outPath = Execution.getFile(basePath);
+    if (outPath == null) return;
+    PrintWriter out = IOUtils.openOutAppendHard(outPath);
+
+    List<String> fields = new ArrayList<>();
+    fields.add(ex.id);
+
+    if (!ex.predDerivations.isEmpty()) {
+      Derivation deriv = ex.predDerivations.get(0);
+      if (deriv.value instanceof ListValue) {
+        List<Value> values = ((ListValue) deriv.value).values;
+        for (Value v : values) {
+          fields.add(v.pureString().replaceAll("\\s+", " ").trim());
+        }
+      }
+    }
+
+    out.println(String.join("\t", fields));
+    out.close();
+  }
+
   //read lisptree and write json
   public static void main(String[] args) {
     Dataset dataset = new Dataset();

src/edu/stanford/nlp/sempre/FeatureExtractor.java

@@ -211,10 +211,12 @@ public class FeatureExtractor {
     List<String> nonEntityLemmas = new LinkedList<>();
     extractNonEntityLemmas(ex, deriv, nonEntityLemmas);
     List<String> binaries = extractBinaries(deriv.formula);
-    String binariesStr = Joiner.on('_').join(binaries);
-    for (String nonEntityLemma : nonEntityLemmas) {
-      deriv.addFeature("lemmaAndBinaries", "nonEntitylemmas=" + nonEntityLemma +
-              ",binaries=" + binariesStr);
+    if (!binaries.isEmpty()) {
+      String binariesStr = Joiner.on('_').join(binaries);
+      for (String nonEntityLemma : nonEntityLemmas) {
+        deriv.addFeature("lemmaAndBinaries", "nonEntitylemmas=" + nonEntityLemma +
+          ",binaries=" + binariesStr);
+      }
     }
   }
 

src/edu/stanford/nlp/sempre/FeatureVector.java

@@ -2,6 +2,7 @@ package edu.stanford.nlp.sempre;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
+
 import fig.basic.*;
 
 import java.util.*;

src/edu/stanford/nlp/sempre/FloatingParser.java

@@ -16,53 +16,142 @@ import static fig.basic.LogInfo.logs;
  * where many of the words are unaccounted for.
  *
  * Assume the Grammar is binarized and only has rules of the following form:
+ *
  *   $Cat => token
  *   $Cat => $Cat
  *   $Cat => token token
  *   $Cat => token $Cat
  *   $Cat => $Cat token
  *   $Cat => $Cat $Cat
- * Each rule is either anchored or floating or both.
+ *
+ * Each rule can be either anchored or floating (or technically, both).
+ * For floating rules, tokens on the RHS are ignored.
  *
  * Chart cells are either:
- * - anchored: (cat, start, end) [these are effectively at depth 0]
- * - floating: (cat, depth) [depends on anchored cells as base cases]
+ * - anchored: (cat, start, end)    [these are effectively at depth 0]
+ * - floating: (cat, depth or size) [depends on anchored cells as base cases]
  *
- * Rules:
+ * With rules:
  *   cat => cat1 cat2 [binary]
  *   cat => cat1 [unary]
- * Combinations:
- *   (cat1, start, end) => (cat, start, end)
- *   (cat1, depth) => (cat, depth)
  *
+ * Anchored Combinations:
+ *   (cat1, start, end) => (cat, start, end)
  *   (cat1, start, mid), (cat2, mid, end) => (cat, start, end)
- *   (cat1, start, end), (cat2, depth) => (cat, depth + 1)
- *   (cat1, depth), (cat2, start, end) => (cat, depth + 1)
- *   (cat1, depth1), (cat2, depth2) => (cat, max(depth1, depth2)+1)
+ *   (cat, start, end) => (cat, 0)   [anchored => floating]
+ *
+ * Floating Combinations:
+ *   [nothing] => (cat, 1)           [from $Cat => token]
+ *   (cat1, depth) => (cat, depth + 1)
+ *   (cat1, depth1), (cat2, depth2) => (cat, max(depth1, depth2) + 1)
+ *
+ * If --useSizeInsteadOfDepth is turned on, the floating combinations become:
+ *   [nothing] => (cat, 1)           [from $Cat => token]
+ *   (cat1, size) => (cat, size + 1)
+ *   (cat1, size1), (cat2, size2) => (cat, size1 + size2 + 1)
  *
  * @author Percy Liang
  */
 public class FloatingParser extends Parser {
   public static class Options {
-    @Option public int maxDepth = 10;
-    @Option public boolean defaultIsFloating = true;
-    @Option (gloss = "Flag specifying whether anchored spans/tokens can only be used once in a derivation")
-    public boolean useAnchorsOnce = false;
-    @Option (gloss = "Flag specifying whether floating rules are allowed to be applied consecutively")
-    public boolean consecutiveRules = true;
-    @Option (gloss = "Whether to always execute the derivation")
-    public boolean executeAllDerivations = false;
-    @Option (gloss = "Whether to output a file with all utterances predicted")
-    public boolean printPredictedUtterances = false;
+    // Floating rules
+    @Option(gloss = "Whether rules without the (anchored 1) or (floating 1) tag should be anchored or floating")
+    public boolean defaultIsFloating = true;
+    @Option(gloss = "Limit on formula depth (or formula size when --useSizeInsteadOfDepth is true)")
+    public int maxDepth = 10;
     @Option(gloss = "Put a limit on formula size instead of formula depth")
     public boolean useSizeInsteadOfDepth = false;
+    @Option(gloss = "Whether floating rules are allowed to be applied consecutively")
+    public boolean consecutiveRules = true;
+    @Option(gloss = "Whether floating rule (rule $A (a)) should have depth 0 or 1")
+    public boolean initialFloatingHasZeroDepth = false;
+    @Option(gloss = "Filter child derivations using the type information from SemanticFn")
+    public boolean filterChildDerivations = true;
+    // Anchored rules
+    @Option(gloss = "Whether anchored spans/tokens can only be used once in a derivation")
+    public boolean useAnchorsOnce = false;
+    @Option(gloss = "Each span can be anchored this number of times (unused if useAnchorsOnce is active)")
+    public int useMaxAnchors = -1;
+    // Other options
+    @Option(gloss = "Whether to always execute the derivation")
+    public boolean executeAllDerivations = false;
+    @Option(gloss = "Whether to output a file with all utterances predicted")
+    public boolean printPredictedUtterances = false;
     @Option(gloss = "Custom beam size at training time (default = Parser.beamSize)")
     public int trainBeamSize = -1;
+    @Option(gloss = "Whether to beta reduce the formula")
+    public boolean betaReduce = false;
+    @Option(gloss = "DEBUG: Print amount of time spent on each rule")
+    public boolean summarizeRuleTime = false;
+    @Option(gloss = "Stop the parser if it has used more than this amount of time (in seconds)")
+    public int maxFloatingParsingTime = Integer.MAX_VALUE;
   }
 
   public static Options opts = new Options();
 
-  public FloatingParser(Spec spec) { super(spec); }
+  public boolean earlyStopOnConsistent = false;
+  public int earlyStopOnNumDerivs = -1;
+
+  public FloatingParser(Spec spec) {
+    super(spec);
+  }
+
+  /**
+   * Set early stopping criteria
+   *
+   * @param onConsistent
+   *    Stop when a consistent derivation is found. (Only triggered when computeExpectedCounts = true)
+   * @param onNumDerivs
+   *    Stop when the number of featurized derivations exceed this number (set to -1 to disable)
+   * @return
+   *    this
+   */
+  public FloatingParser setEarlyStopping(boolean onConsistent, int onNumDerivs) {
+    this.earlyStopOnConsistent = onConsistent;
+    this.earlyStopOnNumDerivs = onNumDerivs;
+    return this;
+  }
+
+  /**
+   * computeCatUnaryRules, but do not topologically sort floating rules
+   */
+  @Override
+  protected void computeCatUnaryRules() {
+    // Handle anchored catUnaryRules
+    catUnaryRules = new ArrayList<>();
+    Map<String, List<Rule>> graph = new HashMap<>();  // Node from LHS to list of rules
+    for (Rule rule : grammar.rules)
+      if (rule.isCatUnary() && rule.isAnchored())
+        MapUtils.addToList(graph, rule.lhs, rule);
+
+    // Topologically sort catUnaryRules so that B->C occurs before A->B
+    Map<String, Boolean> done = new HashMap<>();
+    for (String node : graph.keySet())
+      traverse(catUnaryRules, node, graph, done);
+
+    // Add floating catUnaryRules
+    for (Rule rule : grammar.rules)
+      if (rule.isCatUnary() && rule.isFloating())
+        catUnaryRules.add(rule);
+  }
+
+  // Helper function for transitive closure of floating rules.
+  protected void traverseFloatingRules(List<Rule> orderedFloatingRules,
+      String node, Map<String, List<Rule>> graph, Map<String, Boolean> done) {
+    Boolean d = done.get(node);
+    if (Boolean.TRUE.equals(d)) return;
+    if (Boolean.FALSE.equals(d))
+      throw new RuntimeException("Found cycle of floating rules involving " + node);
+    done.put(node, false);
+    for (Rule rule : MapUtils.getList(graph, node)) {
+      for (String rhsCat : rule.rhs) {
+        if (Grammar.isIntermediate(rhsCat))
+          traverseFloatingRules(orderedFloatingRules, rhsCat, graph, done);
+      }
+      orderedFloatingRules.add(rule);
+    }
+    done.put(node, true);
+  }
 
   public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) {
     return new FloatingParserState(this, params, ex, computeExpectedCounts);
@@ -79,18 +168,21 @@ public class FloatingParser extends Parser {
  * @author Percy Liang
  */
 class FloatingParserState extends ParserState {
-  // cell => list of derivations, formula set
-  // Examples of state:
-  //   (category, depth)
-  //   (category, depth, set of tokens)
 
+  // cell => list of derivations
+  // Anchored cells: cat[start,end]
+  // Floating cells: cat:depth
   private final Map<Object, List<Derivation>> chart = new HashMap<>();
 
   private final DerivationPruner pruner;
+  private final CatSizeBound catSizeBound;
+  private Map<Rule, Long> ruleTime;
+  private boolean timeout = false;
 
   public FloatingParserState(FloatingParser parser, Params params, Example ex, boolean computeExpectedCounts) {
     super(parser, params, ex, computeExpectedCounts);
     pruner = new DerivationPruner(this);
+    catSizeBound = new CatSizeBound(FloatingParser.opts.maxDepth, parser.grammar);
   }
 
   @Override
@@ -100,14 +192,12 @@ class FloatingParserState extends ParserState {
     return Parser.opts.beamSize;
   }
 
-
-
   // Construct state names.
   private Object floatingCell(String cat, int depth) {
-    return cat + ":" + depth;
+    return (cat + ":" + depth).intern();
   }
   private Object anchoredCell(String cat, int start, int end) {
-    return cat + "[" + start + "," + end + "]";
+    return (cat + "[" + start + "," + end + "]").intern();
   }
   private Object cell(String cat, int start, int end, int depth) {
     return (start != -1) ? anchoredCell(cat, start, end) : floatingCell(cat, depth);
@@ -122,7 +212,18 @@ class FloatingParserState extends ParserState {
     MapUtils.addToList(chart, cell, deriv);
   }
 
-  private void applyRule(Rule rule, int start, int end, int depth, Derivation child1, Derivation child2, String canonicalUtterance) {
+  private boolean isRootRule(Rule rule) {
+    return Rule.rootCat.equals(rule.lhs);
+  }
+
+  private boolean applyRule(Rule rule, int start, int end, int depth,
+      Derivation child1, Derivation child2, String canonicalUtterance) {
+    if (timeout && !isRootRule(rule)) return false;
+    applyRuleActual(rule, start, end, depth, child1, child2, canonicalUtterance);
+    return true;
+  }
+
+  private void applyRuleActual(Rule rule, int start, int end, int depth, Derivation child1, Derivation child2, String canonicalUtterance) {
     if (Parser.opts.verbose >= 5) logs("applyRule %s [%s:%s] depth=%s, %s %s", rule, start, end, depth, child1, child2);
     List<Derivation> children;
     if (child1 == null)  // 0-ary
@@ -131,9 +232,14 @@ class FloatingParserState extends ParserState {
       children = Collections.singletonList(child1);
     else {
       // Optional: ensure that each anchor is only used once per derivation.
-      if (FloatingParser.opts.useAnchorsOnce &&
-              FloatingRuleUtils.derivationAnchorsOverlap(child1, child2))
-        return;
+      if (FloatingParser.opts.useAnchorsOnce) {
+        if (FloatingRuleUtils.derivationAnchorsOverlap(child1, child2))
+          return;
+      } else if (FloatingParser.opts.useMaxAnchors >= 0) {
+        if (FloatingRuleUtils.maxNumAnchorOverlaps(child1, child2)
+            > FloatingParser.opts.useMaxAnchors)
+          return;
+      }
       children = ListUtils.newList(child1, child2);
     }
 
@@ -145,9 +251,10 @@ class FloatingParserState extends ParserState {
     }
 
     DerivationStream results = rule.sem.call(ex,
-            new SemanticFn.CallInfo(rule.lhs, start, end, rule, children));
+        new SemanticFn.CallInfo(rule.lhs, start, end, rule, children));
     while (results.hasNext()) {
       Derivation newDeriv = results.next();
+      if (FloatingParser.opts.betaReduce) newDeriv = newDeriv.betaReduction();
       newDeriv.canonicalUtterance = canonicalUtterance;
 
       // make sure we execute
@@ -162,14 +269,17 @@ class FloatingParserState extends ParserState {
     }
   }
 
-  private void applyAnchoredRule(Rule rule, int start, int end, Derivation child1, Derivation child2, String canonicalUtterance) {
-    applyRule(rule, start, end, -1, child1, child2, canonicalUtterance);
+  private boolean applyAnchoredRule(Rule rule, int start, int end, Derivation child1, Derivation child2, String canonicalUtterance) {
+    return applyRule(rule, start, end, -1, child1, child2, canonicalUtterance);
   }
 
-  private void applyFloatingRule(Rule rule, int depth, Derivation child1, Derivation child2, String canonicalUtterance) {
-    applyRule(rule, -1, -1, depth, child1, child2, canonicalUtterance);
+  private boolean applyFloatingRule(Rule rule, int depth, Derivation child1, Derivation child2, String canonicalUtterance) {
+    return applyRule(rule, -1, -1, depth, child1, child2, canonicalUtterance);
   }
 
+  /**
+   * Return a collection of Derivation.
+   */
   private List<Derivation> getDerivations(Object cell) {
     List<Derivation> derivations = chart.get(cell);
     // logs("getDerivations %s => %s", cell, derivations);
@@ -177,6 +287,26 @@ class FloatingParserState extends ParserState {
     return derivations;
   }
 
+  /**
+   * Return a collection of DerivationGroup.
+   *
+   * The rule should be applied on all derivations (or all pairs of derivations) in each DerivationGroup.
+   */
+  private Collection<ChildDerivationsGroup> getFilteredDerivations(Rule rule, Object cell1, Object cell2) {
+    List<Derivation> derivations1 = getDerivations(cell1),
+        derivations2 = (cell2 == null) ? null : getDerivations(cell2);
+    if (!FloatingParser.opts.filterChildDerivations)
+      return Collections.singleton(new ChildDerivationsGroup(derivations1, derivations2));
+    // Try to filter down the number of partial logical forms
+    if (rule.getSem().supportFilteringOnTypeData())
+      return rule.getSem().getFilteredDerivations(derivations1, derivations2);
+    return Collections.singleton(new ChildDerivationsGroup(derivations1, derivations2));
+  }
+
+  private Collection<ChildDerivationsGroup> getFilteredDerivations(Rule rule, Object cell) {
+    return getFilteredDerivations(rule, cell, null);
+  }
+
   // Build derivations over span |start|, |end|.
   private void buildAnchored(int start, int end) {
     // Apply unary tokens on spans (rule $A (a))
@@ -184,8 +314,10 @@ class FloatingParserState extends ParserState {
       if (!rule.isAnchored()) continue;
       if (rule.rhs.size() != 1 || rule.isCatUnary()) continue;
       boolean match = (end - start == 1) && ex.token(start).equals(rule.rhs.get(0));
-      if (match)
-        applyAnchoredRule(rule, start, end, null, null, rule.rhs.get(0));
+      if (!match) continue;
+      StopWatch stopWatch = new StopWatch().start();
+      applyAnchoredRule(rule, start, end, null, null, rule.rhs.get(0));
+      ruleTime.put(rule, ruleTime.getOrDefault(rule, 0L) + stopWatch.stop().ms);
     }
 
     // Apply binaries on spans (rule $A ($B $C)), ...
@@ -194,6 +326,7 @@ class FloatingParserState extends ParserState {
         if (!rule.isAnchored()) continue;
         if (rule.rhs.size() != 2) continue;
 
+        StopWatch stopWatch = new StopWatch().start();
         String rhs1 = rule.rhs.get(0);
         String rhs2 = rule.rhs.get(1);
         boolean match1 = (mid - start == 1) && ex.token(start).equals(rhs1);
@@ -221,6 +354,7 @@ class FloatingParserState extends ParserState {
             for (Derivation deriv2 : derivations2)
               applyAnchoredRule(rule, start, end, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
         }
+        ruleTime.put(rule, ruleTime.getOrDefault(rule, 0L) + stopWatch.stop().ms);
       }
     }
 
@@ -228,79 +362,104 @@ class FloatingParserState extends ParserState {
     // Important: do this in topologically sorted order and after all the binaries are done.
     for (Rule rule : parser.catUnaryRules) {
       if (!rule.isAnchored()) continue;
+      StopWatch stopWatch = new StopWatch().start();
       List<Derivation> derivations = getDerivations(anchoredCell(rule.rhs.get(0), start, end));
-      for (Derivation deriv : derivations) {
+      for (Derivation deriv : derivations)
         applyAnchoredRule(rule, start, end, deriv, null, deriv.canonicalUtterance);
-      }
+      ruleTime.put(rule, ruleTime.getOrDefault(rule, 0L) + stopWatch.stop().ms);
     }
   }
 
   // Build floating derivations of exactly depth |depth|.
   private void buildFloating(int depth) {
-    // Apply unary tokens on spans (rule $A (a))
-    if (depth == 1) {
+    // Build a floating predicate from thin air
+    // (rule $A (a)); note that "a" is ignored
+    if (depth == (FloatingParser.opts.initialFloatingHasZeroDepth ? 0 : 1)) {
       for (Rule rule : parser.grammar.rules) {
+        if (timeout && !isRootRule(rule)) continue;
         if (!rule.isFloating()) continue;
         if (rule.rhs.size() != 1 || rule.isCatUnary()) continue;
+        StopWatch stopWatch = new StopWatch().start();
         applyFloatingRule(rule, depth, null, null, rule.rhs.get(0));
+        ruleTime.put(rule, ruleTime.getOrDefault(rule, 0L) + stopWatch.stop().ms);
       }
     }
 
     // Apply binaries on spans (rule $A ($B $C)), ...
     for (Rule rule : parser.grammar.rules) {
+      if (timeout && !isRootRule(rule)) continue;
       if (!rule.isFloating()) continue;
       if (rule.rhs.size() != 2) continue;
+      if (catSizeBound.getBound(rule.lhs) < depth) continue;
 
+      StopWatch stopWatch = new StopWatch().start();
       String rhs1 = rule.rhs.get(0);
       String rhs2 = rule.rhs.get(1);
 
       if (!Rule.isCat(rhs1) && !Rule.isCat(rhs2)) {  // token token
-        if (depth == 1)
+        if (depth == (FloatingParser.opts.initialFloatingHasZeroDepth ? 0 : 1)) {
           applyFloatingRule(rule, depth, null, null, rhs1 + " " + rhs2);
+        }
+
       } else if (!Rule.isCat(rhs1) && Rule.isCat(rhs2)) {  // token $Cat
         List<Derivation> derivations = getDerivations(floatingCell(rhs2, depth - 1));
         for (Derivation deriv : derivations)
           applyFloatingRule(rule, depth, deriv, null, rhs1 + " " + deriv.canonicalUtterance);
+
       } else if (Rule.isCat(rhs1) && !Rule.isCat(rhs2)) {  // $Cat token
         List<Derivation> derivations = getDerivations(floatingCell(rhs1, depth - 1));
         for (Derivation deriv : derivations)
           applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance + " " + rhs2);
+
       } else {  // $Cat $Cat
         if (FloatingParser.opts.useSizeInsteadOfDepth) {
-          for (int depth1 = 0; depth1 < depth; depth1++) {
-            int depth2 = depth - 1 - depth1;
-            List<Derivation> derivations1 = getDerivations(floatingCell(rhs1, depth1));
-            List<Derivation> derivations2 = getDerivations(floatingCell(rhs2, depth2));
-            for (Derivation deriv1 : derivations1)
-              for (Derivation deriv2 : derivations2)
-                applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
-          }
+          derivLoop:
+            for (int depth1 = 0; depth1 < depth; depth1++) {  // sizes must add up to depth-1 (actually size-1)
+              int depth2 = depth - 1 - depth1;
+              for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, depth1), floatingCell(rhs2, depth2)))
+                for (Derivation deriv1 : group.derivations1)
+                  for (Derivation deriv2 : group.derivations2)
+                    if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance))
+                      break derivLoop;
+            }
         } else {
-          for (int subDepth = 0; subDepth < depth; subDepth++) {  // depth-1 <=depth-1
-            List<Derivation> derivations1 = getDerivations(floatingCell(rhs1, depth - 1));
-            List<Derivation> derivations2 = getDerivations(floatingCell(rhs2, subDepth));
-            for (Derivation deriv1 : derivations1)
-              for (Derivation deriv2 : derivations2)
-                applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
+          {
+            derivLoop:
+              for (int subDepth = 0; subDepth < depth; subDepth++) {  // depth-1 <=depth-1
+                for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, depth - 1), floatingCell(rhs2, subDepth)))
+                  for (Derivation deriv1 : group.derivations1)
+                    for (Derivation deriv2 : group.derivations2)
+                      if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance))
+                        break derivLoop;
+              }
           }
-          for (int subDepth = 0; subDepth < depth - 1; subDepth++) {  // <depth-1 depth-1
-            List<Derivation> derivations1 = getDerivations(floatingCell(rhs1, subDepth));
-            List<Derivation> derivations2 = getDerivations(floatingCell(rhs2, depth - 1));
-            for (Derivation deriv1 : derivations1)
-              for (Derivation deriv2 : derivations2)
-                applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
+          {
+            derivLoop:
+              for (int subDepth = 0; subDepth < depth - 1; subDepth++) {  // <depth-1 depth-1
+                for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rhs1, subDepth), floatingCell(rhs2, depth - 1)))
+                  for (Derivation deriv1 : group.derivations1)
+                    for (Derivation deriv2 : group.derivations2)
+                      if (!applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance))
+                        break derivLoop;
+              }
           }
         }
       }
+      ruleTime.put(rule, ruleTime.getOrDefault(rule, 0L) + stopWatch.stop().ms);
     }
 
     // Apply unary categories on spans (rule $A ($B))
-    // Important: do this in topologically sorted order and after all the binaries are done.
     for (Rule rule : parser.catUnaryRules) {
+      if (timeout && !isRootRule(rule)) continue;
       if (!rule.isFloating()) continue;
-      List<Derivation> derivations = getDerivations(floatingCell(rule.rhs.get(0), depth - 1));
-      for (Derivation deriv : derivations)
-        applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance);
+      if (catSizeBound.getBound(rule.lhs) < depth) continue;
+      StopWatch stopWatch = new StopWatch().start();
+      derivLoop:
+        for (ChildDerivationsGroup group : getFilteredDerivations(rule, floatingCell(rule.rhs.get(0), depth - 1)))
+          for (Derivation deriv : group.derivations1)
+            if (!applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance))
+              break derivLoop;
+      ruleTime.put(rule, ruleTime.getOrDefault(rule, 0L) + stopWatch.stop().ms);
     }
   }
 
@@ -310,43 +469,113 @@ class FloatingParserState extends ParserState {
       derivations.addAll(myDerivations);
   }
 
-  @Override public void infer() {
-    LogInfo.begin_track("FloatingParser.infer()");
+  /**
+   * Build derivations in a thread to allow timeout.
+   */
+  class DerivationBuilder implements Runnable {
+    @Override public void run() {
+      // Base case ($TOKEN, $PHRASE)
+      for (Derivation deriv : gatherTokenAndPhraseDerivations()) {
+        addToChart(anchoredCell(deriv.cat, deriv.start, deriv.end), deriv);
+        addToChart(floatingCell(deriv.cat, 0), deriv);
+      }
 
-    // Base case ($TOKEN, $PHRASE)
-    for (Derivation deriv : gatherTokenAndPhraseDerivations()) {
-      addToChart(anchoredCell(deriv.cat, deriv.start, deriv.end), deriv);
-      addToChart(floatingCell(deriv.cat, 0), deriv);
-    }
+      Set<String> categories = new HashSet<>();
+      for (Rule rule : parser.grammar.rules)
+        categories.add(rule.lhs);
 
-    Set<String> categories = new HashSet<>();
-    for (Rule rule : parser.grammar.rules)
-      categories.add(rule.lhs);
+      if (Parser.opts.verbose >= 1)
+        LogInfo.begin_track_printAll("Anchored");
+      // Build up anchored derivations (like the BeamParser)
+      int numTokens = ex.numTokens();
+      for (int len = 1; len <= numTokens; len++) {
+        for (int i = 0; i + len <= numTokens; i++)  {
+          buildAnchored(i, i + len);
+          for (String cat : categories) {
+            String cell = anchoredCell(cat, i, i + len).toString();
+            pruneCell(cell, chart.get(cell));
+          }
+        }
+      }
+      if (Parser.opts.verbose >= 1)
+        LogInfo.end_track();
 
-    // Build up anchored derivations (like the BeamParser)
-    int numTokens = ex.numTokens();
-    for (int len = 1; len <= numTokens; len++) {
-      for (int i = 0; i + len <= numTokens; i++)  {
-        buildAnchored(i, i + len);
+      // Build up floating derivations
+      for (int depth = (FloatingParser.opts.initialFloatingHasZeroDepth ? 0 : 1); depth <= FloatingParser.opts.maxDepth; depth++) {
+        if (Parser.opts.verbose >= 1)
+          LogInfo.begin_track_printAll("%s = %d", FloatingParser.opts.useSizeInsteadOfDepth ? "SIZE" : "DEPTH", depth);
+        buildFloating(depth);
         for (String cat : categories) {
-          String cell = anchoredCell(cat, i, i + len).toString();
+          String cell = floatingCell(cat, depth).toString();
           pruneCell(cell, chart.get(cell));
         }
+        if (Parser.opts.verbose >= 1)
+          LogInfo.end_track();
+        // Early stopping
+        if (computeExpectedCounts && ((FloatingParser) parser).earlyStopOnConsistent) {
+          // Consistent derivation found?
+          String cell = floatingCell(Rule.rootCat, depth).toString();
+          List<Derivation> rootDerivs = chart.get(cell);
+          if (rootDerivs != null) {
+            for (Derivation rootDeriv : rootDerivs) {
+              rootDeriv.ensureExecuted(parser.executor, ex.context);
+              if (parser.valueEvaluator.getCompatibility(ex.targetValue, rootDeriv.value) == 1) {
+                LogInfo.logs("Early stopped: consistent derivation found at depth = %d", depth);
+                return;
+              }
+            }
+          }
+        }
+        if (((FloatingParser) parser).earlyStopOnNumDerivs > 0) {
+          // Too many derivations generated?
+          if (numOfFeaturizedDerivs > ((FloatingParser) parser).earlyStopOnNumDerivs) {
+            LogInfo.logs("Early stopped: number of derivations exceeded at depth = %d", depth);
+            return;
+          }
+        }
       }
     }
+  }
 
-    // Build up floating derivations
-    for (int depth = 1; depth <= FloatingParser.opts.maxDepth; depth++) {
-      buildFloating(depth);
-      for (String cat : categories) {
-        String cell = floatingCell(cat, depth).toString();
-        pruneCell(cell, chart.get(cell));
+  public void buildDerivations() {
+    DerivationBuilder derivBuilder = new DerivationBuilder();
+    if (FloatingParser.opts.maxFloatingParsingTime == Integer.MAX_VALUE) {
+      derivBuilder.run();
+    } else {
+      Thread parsingThread = new Thread(derivBuilder);
+      parsingThread.start();
+      try {
+        parsingThread.join(FloatingParser.opts.maxFloatingParsingTime * 1000);
+        if (parsingThread.isAlive()) {
+          // This will only interrupt first or second passes, not the final candidate collection.
+          LogInfo.warnings("Parsing time exceeded %d seconds. Will now interrupt ...", FloatingParser.opts.maxFloatingParsingTime);
+          timeout = true;
+          parsingThread.interrupt();
+          parsingThread.join();
+        }
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+        LogInfo.fails("FloatingParser error: %s", e);
       }
     }
+    evaluation.add("timeout", timeout);
+  }
+
+  // ============================================================
+  // Main entry point
+  // ============================================================
+
+  @Override public void infer() {
+    LogInfo.begin_track_printAll("FloatingParser.infer()");
+    ruleTime = new HashMap<>();
+
+    buildDerivations();
+
+    if (FloatingParser.opts.summarizeRuleTime) summarizeRuleTime();
 
     // Collect final predicted derivations
     addToDerivations(anchoredCell(Rule.rootCat, 0, numTokens), predDerivations);
-    for (int depth = 1; depth <= FloatingParser.opts.maxDepth; depth++)
+    for (int depth = 0; depth <= FloatingParser.opts.maxDepth; depth++)
       addToDerivations(floatingCell(Rule.rootCat, depth), predDerivations);
 
     // Compute gradient with respect to the predicted derivations
@@ -357,8 +586,8 @@ class FloatingParserState extends ParserState {
     }
 
     // Example summary
-    if (Parser.opts.verbose >= 1) {
-      LogInfo.begin_track("Summary of Example %s", ex.getUtterance());
+    if (Parser.opts.verbose >= 2) {
+      LogInfo.begin_track_printAll("Summary of Example %s", ex.getUtterance());
       for (Derivation deriv : predDerivations)
         LogInfo.logs("Generated: canonicalUtterance=%s, value=%s", deriv.canonicalUtterance, deriv.value);
       LogInfo.end_track();
@@ -387,6 +616,7 @@ class FloatingParserState extends ParserState {
     evaluation.add("numCells", chart.size());
   }
 
+  @SuppressWarnings("unused")
   private void visualizeAnchoredChart(Set<String> categories) {
     for (String cat : categories) {
       for (int len = 1; len <= numTokens; ++len) {
@@ -399,4 +629,14 @@ class FloatingParserState extends ParserState {
       }
     }
   }
+
+  private void summarizeRuleTime() {
+    List<Map.Entry<Rule, Long>> entries = new ArrayList<>(ruleTime.entrySet());
+    entries.sort(new ValueComparator<>(true));
+    LogInfo.begin_track_printAll("Rule time");
+    for (Map.Entry<Rule, Long> entry : entries) {
+      LogInfo.logs("%9d : %s", entry.getValue(), entry.getKey());
+    }
+    LogInfo.end_track();
+  }
 }

src/edu/stanford/nlp/sempre/FloatingRuleUtils.java

@@ -31,20 +31,20 @@ public final class FloatingRuleUtils {
    * that spans) [2, 4] then we have an overlap.
    */
   public static boolean derivationAnchorsOverlap(Derivation a, Derivation b) {
-    /*
-    List<Derivation> aRoots = getDerivationAnchors(a);
-    List<Derivation> bRoots = getDerivationAnchors(b);
-    for (Derivation aRoot : aRoots) {
-      for (Derivation bRoot : bRoots) {
-        if (aRoot.start < bRoot.end && bRoot.start < aRoot.end)
-          return true;
-      }
-    }
-    return false;
-    */
     boolean[] aAnchors = a.getAnchoredTokens(), bAnchors = b.getAnchoredTokens();
     for (int i = 0; i < aAnchors.length && i < bAnchors.length; i++)
       if (aAnchors[i] && bAnchors[i]) return true;
     return false;
   }
+
+  public static int maxNumAnchorOverlaps(Derivation a, Derivation b) {
+    int[] aAnchors = a.getNumAnchors(), bAnchors = b.getNumAnchors();
+    int n = Math.max(aAnchors.length, bAnchors.length);
+    int maxAnchors = 0;
+    for (int i = 0; i < n; i++)
+      maxAnchors = Math.max(maxAnchors,
+          (i < aAnchors.length ? aAnchors[i] : 0) +
+          (i < bAnchors.length ? bAnchors[i] : 0));
+    return maxAnchors;
+  }
 }

src/edu/stanford/nlp/sempre/Formula.java

@@ -23,6 +23,10 @@ public abstract class Formula {
   // Serialize as LispTree.
   public abstract LispTree toLispTree();
 
+  // Recursively perform some operation on each formula.
+  // Apply to formulas.  If |func| returns false, then recurse on children.
+  public abstract void forEach(Function<Formula, Boolean> func);
+
   // Recursively perform some operation on each formula.
   // Apply to formulas.  If |func| returns null, then recurse on children.
   public abstract Formula map(Function<Formula, Formula> func);

src/edu/stanford/nlp/sempre/Formulas.java

@@ -74,6 +74,16 @@ public abstract class Formulas {
       if (mode != null)
         return new ArithmeticFormula(mode, fromLispTree(tree.child(1)), fromLispTree(tree.child(2)));
     }
+    
+    { // ActionFormula
+      ActionFormula.Mode mode = ActionFormula.parseMode(func);
+      if (mode != null) {
+        List<Formula> args = Lists.newArrayList();
+        for (int i = 1; i < tree.children.size(); i++)
+          args.add(fromLispTree(tree.child(i)));
+        return new ActionFormula(mode, args);
+      }
+    }
 
     // Default is join: (fb:type.object.type fb:people.person)
     if (tree.children.size() != 2)
@@ -85,7 +95,7 @@ public abstract class Formulas {
   private static Formula parseIntToFormula(LispTree tree) {
     try {
       int i = Integer.parseInt(tree.value);
-      double d = (double) i;
+      double d = i;
       NumberValue value = new NumberValue(d);
       return new ValueFormula(value);
     } catch (NumberFormatException e) {
@@ -260,6 +270,27 @@ public abstract class Formulas {
     return (int) getDouble(formula);
   }
 
+  /**
+   * If the formula represents a binary (e.g., fb:a.b.c or <=),
+   *   return the ID of the binary as a string.
+   * If the formula represents a reversed binary (e.g., !fb:a.b.c or (reverse fb:a.b.c)),
+   *   return "!" + ID of the binary.
+   * Otherwise, return null.
+   */
+  public static String getBinaryId(Formula formula) {
+    if (formula instanceof ReverseFormula) {
+      String childId = getBinaryId(((ReverseFormula) formula).child);
+      if (childId == null) return null;
+      return CanonicalNames.reverseProperty(childId);
+    } else if (formula instanceof ValueFormula) {
+      Value v = ((ValueFormula<?>) formula).value;
+      if (v instanceof NameValue) {
+        return ((NameValue) v).id;
+      }
+    }
+    return null;
+  }
+
   public static ValueFormula<NameValue> newNameFormula(String id) {
     return new ValueFormula<NameValue>(new NameValue(id));
   }
@@ -280,7 +311,7 @@ public abstract class Formulas {
     // recurse
     if (f instanceof AggregateFormula) {
       AggregateFormula aggFormula = (AggregateFormula) f;
-      extractSubpartsRecursive(aggFormula, res);
+      extractSubpartsRecursive(aggFormula.child, res);
     } else if (f instanceof CallFormula) {
       CallFormula callFormula = (CallFormula) f;
       extractSubpartsRecursive(callFormula.func, res);
@@ -360,8 +391,19 @@ public abstract class Formulas {
   // !fb:people.person.place_of_birth <=> fb:people.person.place_of_birth
   private static ValueFormula<NameValue> reverseNameFormula(ValueFormula<NameValue> formula) {
     String id = formula.value.id;
-    return new ValueFormula<>(
-            new NameValue(CanonicalNames.isReverseProperty(id) ?  id.substring(1) : "!" + id));
+    return new ValueFormula<>(new NameValue(CanonicalNames.reverseProperty(id)));
   }
 
+  // Try to simplify reverse subformulas within the specified formula
+  public static Formula simplifyReverses(Formula formula) {
+    return formula.map(new Function<Formula, Formula>() {
+      public Formula apply(Formula formula) {
+        if (formula instanceof ReverseFormula)
+          return reverseFormula(((ReverseFormula) formula).child);
+        return null;
+      }
+    });
+  }
+
+
 }

src/edu/stanford/nlp/sempre/FuzzyMatchFn.java

@@ -5,7 +5,7 @@ import java.util.*;
 import fig.basic.*;
 
 /**
- * Similar to LexiconFn, but list all approximate matches from a TableKnowledgeGraph or PuzzleKnowledgeGraph.
+ * Similar to LexiconFn, but list all approximate matches from a FuzzyMatchable instance.
  *
  * @author ppasupat
  */
@@ -39,6 +39,9 @@ public class FuzzyMatchFn extends SemanticFn {
     }
   }
 
+  public FuzzyMatchFnMode getMode() { return mode; }
+  public boolean getMatchAny() { return matchAny; }
+
   @Override
   public DerivationStream call(Example ex, Callable c) {
     return new LazyFuzzyMatchFnDerivs(ex, c, mode, matchAny);
@@ -50,9 +53,10 @@ public class FuzzyMatchFn extends SemanticFn {
 
   public static class LazyFuzzyMatchFnDerivs extends MultipleDerivationStream {
     final Example ex;
-    final KnowledgeGraph graph;
+    final FuzzyMatchable matchable;
     final Callable c;
     final String query;
+    final List<String> sentence;
     final FuzzyMatchFnMode mode;
     final boolean matchAny;
 
@@ -61,9 +65,19 @@ public class FuzzyMatchFn extends SemanticFn {
 
     public LazyFuzzyMatchFnDerivs(Example ex, Callable c, FuzzyMatchFnMode mode, boolean matchAny) {
       this.ex = ex;
-      this.graph = (ex.context == null) ? null : ex.context.graph;
+      if (ex.context != null && ex.context.graph != null && ex.context.graph instanceof FuzzyMatchable)
+        this.matchable = (FuzzyMatchable) ex.context.graph;
+      else
+        this.matchable = null;
       this.c = c;
       this.query = (matchAny || c.getChildren().isEmpty()) ? null : c.childStringValue(0);
+      if (c.getRule().rhs.size() == 1 && Rule.phraseCat.equals(c.getRule().rhs.get(0))) {
+        sentence = ex.getTokens();
+      } else if (c.getRule().rhs.size() == 1 && Rule.lemmaPhraseCat.equals(c.getRule().rhs.get(0))) {
+        sentence = ex.getLemmaTokens();
+      } else {
+        sentence = null;
+      }
       this.mode = mode;
       this.matchAny = matchAny;
       if (opts.verbose >= 2)
@@ -73,15 +87,17 @@ public class FuzzyMatchFn extends SemanticFn {
 
     @Override
     public Derivation createDerivation() {
-      if (graph == null) return null;
+      if (matchable == null) return null;
       if (query == null && !matchAny) return null;
 
       // Compute the formulas if not computed yet
       if (formulas == null) {
         if (matchAny)
-          formulas = new ArrayList<>(graph.getAllFormulas(mode));
+          formulas = new ArrayList<>(matchable.getAllFormulas(mode));
+        else if (sentence != null)
+          formulas = new ArrayList<>(matchable.getFuzzyMatchedFormulas(sentence, c.getStart(), c.getEnd(), mode));
         else
-          formulas = new ArrayList<>(graph.getFuzzyMatchedFormulas(query, mode));
+          formulas = new ArrayList<>(matchable.getFuzzyMatchedFormulas(query, mode));
       }
 
       // Use the next formula to create a derivation

src/edu/stanford/nlp/sempre/FuzzyMatchable.java

@@ -0,0 +1,39 @@
+package edu.stanford.nlp.sempre;
+
+import java.util.*;
+
+import edu.stanford.nlp.sempre.FuzzyMatchFn.FuzzyMatchFnMode;
+
+/**
+ * Interface for knowledge sources that, given a phrase, can retrieve all its
+ * predicates that (fuzzily) match the phrase.
+ *
+ * @author ppasupat
+ */
+public interface FuzzyMatchable {
+
+  /**
+   * Return all entities / unaries / binaries that approximately match the
+   * string formed by joining sentence[startIndex], ..., sentence[endIndex-1]
+   * with spaces.
+   *
+   * This allows the algorithm to consider the context of the term being matched.
+   *
+   * One possible implementation, which ignores the context, is calling
+   * getFuzzyMatchedFormulas(term, mode) where
+   * term = String.join(" ", sentence.subList(startIndex, endIndex))
+   */
+  public abstract Collection<Formula> getFuzzyMatchedFormulas(
+      List<String> sentence, int startIndex, int endIndex, FuzzyMatchFnMode mode);
+
+  /**
+   * Return all entities / unaries / binaries that approximately match the term
+   */
+  public abstract Collection<Formula> getFuzzyMatchedFormulas(String term, FuzzyMatchFnMode mode);
+
+  /**
+   * Return all possible entities / unaries / binaries
+   */
+  public abstract Collection<Formula> getAllFormulas(FuzzyMatchFnMode mode);
+
+}

src/edu/stanford/nlp/sempre/Grammar.java

@@ -3,6 +3,7 @@ package edu.stanford.nlp.sempre;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
+
 import fig.basic.*;
 import fig.exec.Execution;
 
@@ -13,13 +14,14 @@ import java.util.*;
 /**
  * The grammar is a set of rules of the form:
  *   (rule lhsCat (rhs ... rhs) semanticFn (key value) ... (key value))
-
+ *
  * Some programming-language-esque features are supported to make life easier.
  * All grammar-file variables should start with '@' (both loop variables and global definitions).
+ *
  * Includes (reads in commands from the given file):
  *   (include base.grammar)
  * Control flow:
- *   (when (and tag1 (not tag2)) ...)
+ *   (when (and tag1 (not tag2) (or tag3 tag4)) ...)
  * Macro definitions:
  *   (def @type fb:type.object.type)
  * Iteration:
@@ -33,6 +35,8 @@ public class Grammar {
     @Option(gloss = "Variables which are used to interpret the grammar file")
     public List<String> tags = new ArrayList<>();
     @Option public boolean binarizeRules = true;
+    @Option(gloss = "Specifiy which ApplyFn to use: defaults to JoinFn when null")
+    public String useApplyFn = null;
   }
 
   public static Options opts = new Options();
@@ -40,10 +44,11 @@ public class Grammar {
   // All the rules in the grammar.  Each parser can read these and transform
   // them however the parser wishes.
   // This contains binarized rules
-  ArrayList<Rule> rules = new ArrayList<>();
+  protected ArrayList<Rule> rules = new ArrayList<>();
   public List<Rule> getRules() { return rules; }
 
   Map<String, LispTree> macros = new HashMap<>();  // Map from macro name to its replacement value
+  Set<String> validTags = new TreeSet<>();         // All tags in the grammar file
 
   // Verbatim copy of all the lines read, so we can preserve the grammar file.
   List<String> statements = new ArrayList<>();
@@ -80,6 +85,15 @@ public class Grammar {
         }
       }
     }
+
+    // Check if all tags are defined in a grammar file
+    LogInfo.logs("Valid tags: %s", validTags);
+    LogInfo.logs("Used tags: %s", new TreeSet<>(opts.tags));
+    for (String tag : opts.tags) {
+      if (!validTags.contains(tag)) {
+        LogInfo.warnings("Tag %s not defined in grammar", tag);
+      }
+    }
   }
 
   /**
@@ -100,12 +114,24 @@ public class Grammar {
     addStatement(stmt, null);
   }
 
+  private static boolean isValidVar(String var) {
+    return var.startsWith("@");
+  }
+
+  private static void checkIsValidVar(String var) {
+    if (!isValidVar("@"))
+      throw new RuntimeException("Invalid variable: '" + var + "' doesn't start with '@'");
+  }
+
   // Replace all leaves of LispTree with value in macros if exists
   private static LispTree applyMacros(Map<String, LispTree> macros, LispTree tree) {
     if (tree.isLeaf()) {
       LispTree replacement = macros.get(tree.value);
       if (replacement != null) return replacement;
-      return tree;
+      if (isValidVar(tree.value))
+        throw new RuntimeException("Undefined macro: " + tree.value);
+      else
+        return tree;
     }
     LispTree newTree = LispTree.proto.newList();
     for (LispTree child : tree.children)
@@ -126,8 +152,11 @@ public class Grammar {
       statements.add(line);
 
     Iterator<LispTree> trees = LispTree.proto.parseFromFile(path);
-    while (trees.hasNext())
-      interpret(path, trees.next(), tags);
+    while (trees.hasNext()) {
+      LispTree tree = trees.next();
+      interpret(path, tree, tags);
+      collectValidTags(tree);
+    }
   }
 
   public void write() {
@@ -180,18 +209,28 @@ public class Grammar {
   private boolean interpretBoolean(LispTree tree, Set<String> tags) {
     if (tree.isLeaf())
       return tags.contains(tree.value);
-    if ("not".equals(tree.child(0).value))
+    if ("not".equals(tree.child(0).value)) {
+      if (tree.children.size() != 2)
+        throw new RuntimeException("Too many arguments for not: " + tree);
       return !interpretBoolean(tree.child(1), tags);
-    if ("and".equals(tree.child(0).value))
-      return interpretBoolean(tree.child(1), tags) && interpretBoolean(tree.child(2), tags);
-    if ("or".equals(tree.child(0).value))
-      return interpretBoolean(tree.child(1), tags) || interpretBoolean(tree.child(2), tags);
+    }
+    if ("and".equals(tree.child(0).value)) {
+      for (int i = 1; i < tree.children.size(); i++)
+        if (!interpretBoolean(tree.child(i), tags)) return false;
+      return true;
+    }
+    if ("or".equals(tree.child(0).value)) {
+      for (int i = 1; i < tree.children.size(); i++)
+        if (interpretBoolean(tree.child(i), tags)) return true;
+      return false;
+    }
     throw new RuntimeException("Expected a single tag, but got: " + tree);
   }
 
+
   public void interpretMacroDef(LispTree tree) {
     if (tree.children.size() != 3 || !tree.child(1).isLeaf()) {
-      throw new RuntimeException("Invalid usage: (def |name| |value|)");
+      throw new RuntimeException("Invalid usage (def |name| |value|): " + tree);
     }
     String var = tree.child(1).value;
     checkIsValidVar(var);
@@ -200,9 +239,8 @@ public class Grammar {
 
   public void interpretFor(String path, LispTree tree, Set<String> tags) {
     if (tree.children.size() <= 3 || !tree.child(1).isLeaf()) {
-      throw new RuntimeException("Invalid usage: (for |var| (|value| ... |value|) |statement| ...)");
+      throw new RuntimeException("Invalid usage (for |var| (|value| ... |value|) |statement| ...) " + tree);
     }
-
     String var = tree.child(1).value;
     checkIsValidVar(var);
     List<LispTree> values = applyMacros(tree.child(2)).children;
@@ -216,7 +254,13 @@ public class Grammar {
     else macros.put(var, old);
   }
 
-  private void interpretRule(LispTree tree) {
+  private static String checkCatName(String cat) {
+    if (isIntermediate(cat))
+      LogInfo.warnings("Category '%s' starts with '$Intermediate'; please avoid this unless you know what you are doing.");
+    return cat;
+  }
+
+  protected void interpretRule(LispTree tree) {
     if (tree.children.size() < 4)
       throw new RuntimeException("Invalid rule: " + tree);
 
@@ -226,7 +270,7 @@ public class Grammar {
     // Parse LHS
     if (!tree.child(1).isLeaf())
       throw new RuntimeException("Invalid LHS: " + tree.child(1));
-    String lhs = tree.child(1).value;
+    String lhs = checkCatName(tree.child(1).value);
 
     // Parse RHS
     List<String> rhs = Lists.newArrayList();
@@ -238,12 +282,12 @@ public class Grammar {
       LispTree child = rhsTree.child(i);
       boolean isOptional = false;
       if (child.isLeaf()) {  // $PHRASE
-        rhs.add(child.value);
+        rhs.add(checkCatName(child.value));
       } else {
         // e.g., ($PHRASE optional)
         // First item is the token/category; the rest of the items
         // specify parameters (currently, only parameter is optional).
-        rhs.add(child.child(0).value);
+        rhs.add(checkCatName(child.child(0).value));
         for (int j = 1; j < child.children.size(); j++)
           if (child.child(j).value.equals("optional"))
             isOptional = true;
@@ -271,24 +315,61 @@ public class Grammar {
     addRule(rule, isOptionals);
   }
 
-  public synchronized boolean addRule(Rule rule) {
+  public synchronized void addRule(Rule rule) {
     List<Boolean> isOptionals = new ArrayList<>();
-    for (String rhs : rule.rhs) isOptionals.add(false);
-    return addRule(rule, isOptionals);
+    for (int i = 0; i < rule.rhs.size(); i++) isOptionals.add(false);
+    addRule(rule, isOptionals);
   }
 
   // Add a rule to the grammar.
-  public synchronized boolean addRule(Rule rule, List<Boolean> isOptionals) {
+  public synchronized void addRule(Rule rule, List<Boolean> isOptionals) {
     rules.addAll(binarizeRule(rule, isOptionals));
-    return true;
+  }
+
+  private void collectValidTags(LispTree tree) {
+    if (tree.isLeaf())
+      throw new RuntimeException("Expected list, got " + tree);
+    try {
+      String command = tree.child(0).value;
+      if ("when".equals(command)) {
+        collectValidTagsBoolean(tree.child(1));
+        for (int i = 2; i < tree.children.size(); i++)
+          collectValidTags(tree.child(i));
+      } else if ("for".equals(command)) {
+        for (int i = 3; i < tree.children.size(); i++)
+          collectValidTags(tree.child(i));
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw new RuntimeException("Error on " + tree + ": " + e);
+    }
+  }
+
+  private void collectValidTagsBoolean(LispTree tree) {
+    if (tree.isLeaf()) {
+      validTags.add(tree.value);
+      return;
+    }
+    String command = tree.child(0).value;
+    if ("not".equals(command))
+      collectValidTagsBoolean(tree.child(1));
+    else if ("and".equals(command) || "or".equals(command))
+      for (int i = 1; i < tree.children.size(); i++)
+        collectValidTagsBoolean(tree.child(i));
   }
 
   // Generate intermediate categories for binarization.
-  private int freshCatIndex = 0;
+  public static final String INTERMEDIATE_PREFIX = "$Intermediate";
+  protected int freshCatIndex = 0;
   private String generateFreshCat() {
     freshCatIndex++;
-    return "$Intermediate" + freshCatIndex;
-    // return "$I" + freshCatIndex + ":" + content;
+    return INTERMEDIATE_PREFIX + freshCatIndex;
+  }
+  public static boolean isIntermediate(String cat) {
+    return cat.startsWith(INTERMEDIATE_PREFIX);
+  }
+  public int getFreshCatIndex() {
+    return freshCatIndex;
   }
 
   // Create multiple versions of this rule if there are optional RHS.
@@ -299,6 +380,14 @@ public class Grammar {
   private List<Rule> binarizeRule(Rule rule, List<Boolean> isOptionals) {
     List<Rule> newRules = new ArrayList<>();
 
+    // Don't binarize: do same as before
+    if (!opts.binarizeRules) {
+      if (isOptionals.contains(true))
+        throw new RuntimeException("Can't have optionals if don't binarize: " + rule + " " + isOptionals);
+      newRules.add(rule);
+      return newRules;
+    }
+
     // Special case: JoinFn with an arg0Fn but with multiple non-optional categories.
     // In this case, we need to use arg0Fn on just the first such category, and
     // then use function application on the rest.
@@ -336,19 +425,11 @@ public class Grammar {
       }
     }
 
-    // Don't binarize: do same as before
-    if (!opts.binarizeRules) {
-      if (isOptionals.contains(true))
-        throw new RuntimeException("Can't have optionals if don't binarize: " + rule + " " + isOptionals);
-      newRules.add(rule);
-      return newRules;
-    }
-
     if (!isOptionals.contains(false))
       throw new RuntimeException("Can't have all RHS items be optional: " + rule + " " + isOptionals);
 
-    // Unaries: don't need to binarize
-    if (rule.rhs.size() == 1) {
+    // Zero-aries (for floating rules) and Unaries: don't need to binarize
+    if (rule.rhs.size() <= 1) {
       newRules.add(rule);
       return newRules;
     }
@@ -448,13 +529,20 @@ public class Grammar {
 
     String name = tree.child(0).value;
 
-    // Syntactic sugar: (lambda x (var x)) => (JoinFn betaReduce forward (arg0 (lambda x (var x))))
+    // Syntactic sugar: (lambda x (f (var x))) => (useApplyFn (lambda x (f (var x))))
+    // defaults to (lambda x (var x)) => (JoinFn betaReduce forward (arg0 (lambda x (var x))))
     if (name.equals("lambda")) {
       LispTree newTree = LispTree.proto.newList();
-      newTree.addChild("JoinFn");
-      newTree.addChild("betaReduce");
-      newTree.addChild("forward");
-      newTree.addChild(LispTree.proto.newList("arg0", tree));
+
+      if (Grammar.opts.useApplyFn == null) {
+        newTree.addChild("JoinFn");
+        newTree.addChild("betaReduce");
+        newTree.addChild("forward");
+        newTree.addChild(LispTree.proto.newList("arg0", tree));
+      } else {
+        newTree.addChild(Grammar.opts.useApplyFn);
+        newTree.addChild(tree);
+      }
       tree = newTree;
       name = tree.child(0).value;
     }
@@ -472,8 +560,4 @@ public class Grammar {
     return fn;
   }
 
-  private void checkIsValidVar(String var) {
-    if (!var.startsWith("@"))
-      LogInfo.errors("Invalid variable: '%s' doesn't start with '@'", var);
-  }
 }

src/edu/stanford/nlp/sempre/JavaExecutor.java

@@ -1,7 +1,9 @@
 package edu.stanford.nlp.sempre;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+
 import fig.basic.MapUtils;
 import fig.basic.Option;
 
@@ -23,6 +25,11 @@ public class JavaExecutor extends Executor {
   public static class Options {
     @Option(gloss = "Whether to convert NumberValue to int/double") public boolean convertNumberValues = true;
     @Option(gloss = "Print stack trace on exception") public boolean printStackTrace = false;
+    // the actual function will be called with the current ContextValue as its last argument if marked by contextPrefix
+    @Option(gloss = "Formula in the grammar whose name startsWith contextPrefix is context sensitive") 
+    public String contextPrefix = "context:";
+    @Option(gloss = "Reduce verbosity by automatically appending, for example, edu.stanford.nlp.sempre to java calls")
+    public String classPathPrefix = ""; // e.g. "edu.stanford.nlp.sempre";
   }
   public static Options opts = new Options();
 
@@ -91,6 +98,30 @@ public class JavaExecutor extends Executor {
     public static String plus(String a, String b, String c, String d, String e, String f, String g) {
       return a + b + c + d + e + f + g;
     }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h) {
+      return a + b + c + d + e + f + g + h;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i) {
+      return a + b + c + d + e + f + g + h + i;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j) {
+      return a + b + c + d + e + f + g + h + i + j;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j, String k) {
+      return a + b + c + d + e + f + g + h + i + j + k;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j, String k, String l) {
+      return a + b + c + d + e + f + g + h + i + j + k + l;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j, String k, String l, String m) {
+      return a + b + c + d + e + f + g + h + i + j + k + l + m;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j, String k, String l, String m, String n) {
+      return a + b + c + d + e + f + g + h + i + j + k + l + m + n;
+    }
+    public static String plus(String a, String b, String c, String d, String e, String f, String g, String h, String i, String j, String k, String l, String m, String n, String o) {
+      return a + b + c + d + e + f + g + h + i + j + k + l + m + n + o;
+    }
     private static String toString(Object x) {
       if (x instanceof String)
         return (String) x;
@@ -134,13 +165,13 @@ public class JavaExecutor extends Executor {
     private static Object apply(LambdaFormula func, Object x) {
       // Apply the function func to x.  In order to do that, need to convert x into a value.
       Formula formula = Formulas.lambdaApply(func, new ValueFormula<Value>(toValue(x)));
-      return defaultExecutor.processFormula(formula);
+      return defaultExecutor.processFormula(formula, null);
     }
     private static Object apply(LambdaFormula func, Object x, Object y) {
       // Apply the function func to x and y.  In order to do that, need to convert x into a value.
       Formula formula = Formulas.lambdaApply(func, new ValueFormula<Value>(toValue(x)));
       formula = Formulas.lambdaApply((LambdaFormula) formula, new ValueFormula<Value>(toValue(y)));
-      return defaultExecutor.processFormula(formula);
+      return defaultExecutor.processFormula(formula, null);
     }
 
     public static List<Integer> range(int start, int end) {
@@ -155,9 +186,8 @@ public class JavaExecutor extends Executor {
     // We can do beta reduction here since macro substitution preserves the
     // denotation (unlike for lambda DCS).
     formula = Formulas.betaReduction(formula);
-
     try {
-      return new Response(toValue(processFormula(formula)));
+      return new Response(toValue(processFormula(formula, context)));
     } catch (Exception e) {
       // Comment this out if we expect lots of innocuous type checking failures
       if (opts.printStackTrace) e.printStackTrace();
@@ -165,25 +195,33 @@ public class JavaExecutor extends Executor {
     }
   }
 
-  private Object processFormula(Formula formula) {
+  private Object processFormula(Formula formula, ContextValue context) {
     if (formula instanceof ValueFormula)  // Unpack value and convert to object (e.g., for ints)
       return toObject(((ValueFormula) formula).value);
 
     if (formula instanceof CallFormula) {  // Invoke the function.
-      //LogInfo.logs("formula=%s", formula);
       // Recurse
       CallFormula call = (CallFormula) formula;
-      Object func = processFormula(call.func);
+      Object func = processFormula(call.func, context);
       List<Object> args = Lists.newArrayList();
       for (Formula arg : call.args) {
-        args.add(processFormula(arg));
+        args.add(processFormula(arg, context));
       }
 
       if (!(func instanceof NameValue))
         throw new RuntimeException("Invalid func: " + call.func + " => " + func);
 
       String id = ((NameValue) func).id;
+      if (id.indexOf(opts.contextPrefix) != -1) {
+        args.add(context);
+        id = id.replace(opts.contextPrefix, "");
+      }
       id = MapUtils.get(shortcuts, id, id);
+      
+      // classPathPrefix, like edu.stanford.nlp.sempre.interactive
+      if (!Strings.isNullOrEmpty(opts.classPathPrefix) && !id.startsWith(".") && !id.startsWith(opts.classPathPrefix)) {
+        id = opts.classPathPrefix + "." + id;
+      }
 
       if (id.startsWith(".")) // Instance method
         return invoke(id.substring(1), args.get(0), args.subList(1, args.size()).toArray(new Object[0]));

src/edu/stanford/nlp/sempre/JoinFn.java

@@ -23,10 +23,10 @@ public class JoinFn extends SemanticFn {
   public static class Options {
     @Option(gloss = "Verbose") public int verbose = 0;
     @Option public boolean showTypeCheckFailures = false;
-    @Option public boolean typeInference = false;
+    @Option public boolean typeInference = true;
     // TODO(joberant): this flag is for backward compatibility.  If we don't
     // need it for the new results, get rid of it.
-    @Option public boolean specializedTypeCheck = true;
+    @Option public boolean specializedTypeCheck = false;
   }
 
   public static Options opts = new Options();

src/edu/stanford/nlp/sempre/JoinFormula.java

@@ -36,6 +36,12 @@ public class JoinFormula extends Formula {
     return tree;
   }
 
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) { relation.forEach(func); child.forEach(func); }
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> func) {
     Formula result = func.apply(this);
     return result == null ? new JoinFormula(relation.map(func), child.map(func)) : result;

src/edu/stanford/nlp/sempre/KnowledgeGraph.java

@@ -47,31 +47,6 @@ public abstract class KnowledgeGraph {
   // Helper methods
   // ============================================================
 
-  /**
-   * Return the reversed relation if |r| is of the form |!relation|.
-   * Otherwise, return null.
-   */
-  public static Value isReversedRelation(Value r) {
-    if (r instanceof NameValue) {
-      String id = ((NameValue) r).id;
-      if (id.startsWith("!")) return new NameValue(id.substring(1));
-    }
-    return null;
-  }
-
-  /** Convert between |r| and |!r| (r must be a NameValue) */
-  public static Value getReversedPredicate(Value r) {
-    if (r instanceof NameValue) {
-      String id = ((NameValue) r).id;
-      if (id.startsWith("!"))
-        return new NameValue(id.substring(1));
-      else
-        return new NameValue("!" + id);
-    } else {
-      throw new BadFormulaException("Cannot reverse " + r + " which is not a NameValue");
-    }
-  }
-
   /** Reverse the pairs */
   public static List<Pair<Value, Value>> getReversedPairs(Collection<Pair<Value, Value>> pairs) {
     List<Pair<Value, Value>> reversed = new ArrayList<>();
@@ -85,6 +60,7 @@ public abstract class KnowledgeGraph {
   // ============================================================
 
   public abstract LispTree toLispTree();
+  public abstract LispTree toShortLispTree();
   @Override public String toString() { return toLispTree().toString(); }
 
   /** Return all y such that x in firsts and (x,r,y) in graph */
@@ -99,14 +75,7 @@ public abstract class KnowledgeGraph {
   /** Return all (x,y) such that y in seconds and (x,r,y) in graph */
   public abstract List<Pair<Value, Value>> filterSecond(Value r, Collection<Value> seconds);
 
-  // ============================================================
-  // Fuzzy Matching
-  // ============================================================
-
-  /** Return all entities / unaries / binaries that approximately match the given term */
-  public abstract Collection<Formula> getFuzzyMatchedFormulas(String term, FuzzyMatchFn.FuzzyMatchFnMode mode);
-
-  /** Return all entities / unaries / binaries */
-  public abstract Collection<Formula> getAllFormulas(FuzzyMatchFn.FuzzyMatchFnMode mode);
+  /** Clean up cached data. Default to doing nothing. */
+  public void clean() { }
 
 }

src/edu/stanford/nlp/sempre/LambdaFormula.java

@@ -26,6 +26,12 @@ public class LambdaFormula extends Formula {
     return tree;
   }
 
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) body.forEach(func);
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> func) {
     Formula result = func.apply(this);
     return result == null ? new LambdaFormula(var, body.map(func)) : result;

src/edu/stanford/nlp/sempre/Learner.java

@@ -27,6 +27,8 @@ public class Learner {
 
     @Option(gloss = "Write predDerivations to examples file (huge)")
     public boolean outputPredDerivations = false;
+    @Option(gloss = "Write predicted values to a TSV file")
+    public boolean outputPredValues = false;
 
     @Option(gloss = "Dump all features and compatibility scores")
     public boolean dumpFeaturesAndCompatibility = false;
@@ -41,10 +43,16 @@ public class Learner {
     @Option(gloss = "Initialize with these parameters")
     public List<Pair<String, Double>> initialization;
 
-    @Option(gloss = "whether to update weights")
+    @Option(gloss = "Whether to update weights")
     public boolean updateWeights = true;
-    @Option(gloss = "whether to check gradient")
+    @Option(gloss = "Whether to check gradient")
     public boolean checkGradient = false;
+
+    @Option(gloss = "Whether to skip the 'train' group in the last iteration and non-'train' groups in other iterations")
+    public boolean skipUnnecessaryGroups = false;
+
+    @Option(gloss = "Number of threads to parallelize")
+    public int numParallelThreads = 1;
   }
   public static Options opts = new Options();
 
@@ -87,7 +95,7 @@ public class Learner {
    */
   public void learn(int numIters, Map<String, List<Evaluation>> evaluations) {
     LogInfo.begin_track("Learner.learn()");
-   // if when we start we have parameters already - need to sort the semantic functions.
+    // if when we start we have parameters already - need to sort the semantic functions.
     if (!params.isEmpty())
       sortOnFeedback();
     // For each iteration, go through the groups and parse (updating if train).
@@ -108,28 +116,31 @@ public class Learner {
       for (String group : dataset.groups()) {
         boolean lastIter = (iter == numIters);
         boolean updateWeights = opts.updateWeights && group.equals("train") && !lastIter;  // Don't train on last iteration
-        Evaluation eval = processExamples(
-                iter,
-                group,
-                dataset.examples(group),
-                updateWeights);
+        if (opts.skipUnnecessaryGroups) {
+          if ((group.equals("train") && lastIter) || (!group.equals("train") && !lastIter))
+            continue;
+        }
+        // Allow the parser to change behavior based on current group and iteration
+        parser.onBeginDataGroup(iter, numIters, group);
+        Evaluation eval = processExamples(iter, group, dataset.examples(group), updateWeights);
         MapUtils.addToList(evaluations, group, eval);
         meanEvaluations.get(group).add(eval);
         StopWatchSet.logStats();
+        writeParams(iter);
       }
-
-      // Write out parameters
-      String path = Execution.getFile("params." + iter);
-      if (path != null) {
-        params.write(path);
-        Utils.systemHard("ln -sf params." + iter + " " + Execution.getFile("params"));
-      }
-
       LogInfo.end_track();
     }
     LogInfo.end_track();
   }
 
+  private void writeParams(int iter) {
+    String path = Execution.getFile("params." + iter);
+    if (path != null) {
+      params.write(path);
+      Utils.systemHard("ln -sf params." + iter + " " + Execution.getFile("params"));
+    }
+  }
+
   public void onlineLearnExample(Example ex) {
     LogInfo.begin_track("onlineLearnExample: %s derivations", ex.predDerivations.size());
     HashMap<String, Double> counts = new HashMap<>();
@@ -140,9 +151,16 @@ public class Learner {
     LogInfo.end_track();
   }
 
+  public void onlineLearnExampleByFormula(Example ex, List<Formula> formulas) {
+    HashMap<String, Double> counts = new HashMap<>();
+    for (Derivation deriv : ex.predDerivations)
+      deriv.compatibility = formulas.contains(deriv.formula)? 1 : 0;
+    ParserState.computeExpectedCounts(ex.predDerivations, counts);
+    params.update(counts);
+  }
+
   private Evaluation processExamples(int iter, String group,
-                                     List<Example> examples,
-                                     boolean computeExpectedCounts) {
+      List<Example> examples, boolean computeExpectedCounts) {
     Evaluation evaluation = new Evaluation();
 
     if (examples.size() == 0)
@@ -152,65 +170,83 @@ public class Learner {
 
     Execution.putOutput("group", group);
     LogInfo.begin_track_printAll(
-            "Processing %s: %s examples", prefix, examples.size());
+        "Processing %s: %s examples", prefix, examples.size());
     LogInfo.begin_track("Examples");
 
-    Map<String, Double> counts = new HashMap<>();
-    int batchSize = 0;
-    for (int e = 0; e < examples.size(); e++) {
+    if (opts.numParallelThreads > 1) {
+      // Parallelize!
+      Parallelizer<Example> paral = new Parallelizer<>(opts.numParallelThreads);
+      LearnerParallelProcessor processor = new LearnerParallelProcessor(
+          parser, params, prefix, computeExpectedCounts, evaluation);
+      LogInfo.begin_threads();
+      paral.process(examples, processor);
+      LogInfo.end_threads();
 
-      Example ex = examples.get(e);
+    } else {
+      // Original code (single-threaded)
 
-      LogInfo.begin_track_printAll(
-              "%s: example %s/%s: %s", prefix, e, examples.size(), ex.id);
-      ex.log();
-      Execution.putOutput("example", e);
+      Map<String, Double> counts = new HashMap<>();
+      int batchSize = 0;
+      for (int e = 0; e < examples.size(); e++) {
 
-      ParserState state = parseExample(params, ex, computeExpectedCounts);
-      if (computeExpectedCounts) {
-        if (opts.checkGradient) {
-          LogInfo.begin_track("Checking gradient");
-          checkGradient(ex, state);
-          LogInfo.end_track();
+        Example ex = examples.get(e);
+
+        LogInfo.begin_track_printAll(
+            "%s: example %s/%s: %s", prefix, e, examples.size(), ex.id);
+        ex.log();
+        Execution.putOutput("example", e);
+
+        ParserState state = parseExample(params, ex, computeExpectedCounts);
+        if (computeExpectedCounts) {
+          if (opts.checkGradient) {
+            LogInfo.begin_track("Checking gradient");
+            checkGradient(ex, state);
+            LogInfo.end_track();
+          }
+
+          SempreUtils.addToDoubleMap(counts, state.expectedCounts);
+
+          batchSize++;
+          if (batchSize >= opts.batchSize) {
+            // Gathered enough examples, update parameters
+            updateWeights(counts);
+            batchSize = 0;
+          }
         }
 
-        SempreUtils.addToDoubleMap(counts, state.expectedCounts);
+        LogInfo.logs("Current: %s", ex.evaluation.summary());
+        evaluation.add(ex.evaluation);
+        LogInfo.logs("Cumulative(%s): %s", prefix, evaluation.summary());
 
-        batchSize++;
-        if (batchSize >= opts.batchSize) {
-          // Gathered enough examples, update parameters
-          updateWeights(counts);
-          batchSize = 0;
+        printLearnerEventsIter(ex, iter, group);
+        LogInfo.end_track();
+        if (opts.addFeedback && computeExpectedCounts)
+          addFeedback(ex);
+
+        // Write out examples and predictions
+        if (opts.outputPredDerivations) {
+          ExampleUtils.writeParaphraseSDF(iter, group, ex, true);
+        }
+        if (opts.outputPredValues) {
+          ExampleUtils.writePredictionTSV(iter, group, ex);
         }
-      }
-     // }
 
-      LogInfo.logs("Current: %s", ex.evaluation.summary());
-      evaluation.add(ex.evaluation);
-      LogInfo.logs("Cumulative(%s): %s", prefix, evaluation.summary());
-
-      printLearnerEventsIter(ex, iter, group);
-      LogInfo.end_track();
-      if (opts.addFeedback && computeExpectedCounts)
-        addFeedback(ex);
-
-      // Write out examples and predictions
-      if (opts.outputPredDerivations && Builder.opts.parser.equals("FloatingParser")) {
-        ExampleUtils.writeParaphraseSDF(iter, group, ex, opts.outputPredDerivations);
+        // To save memory
+        ex.predDerivations.clear();
       }
 
-      // To save memory
-      ex.predDerivations.clear();
+      if (computeExpectedCounts && batchSize > 0)
+        updateWeights(counts);
+
     }
 
-    if (computeExpectedCounts && batchSize > 0)
-      updateWeights(counts);
     params.finalizeWeights();
     if (opts.sortOnFeedback && computeExpectedCounts)
       sortOnFeedback();
 
     LogInfo.end_track();
     logEvaluationStats(evaluation, prefix);
+    evaluation.putOutput(prefix.replace('.', '-'));
     printLearnerEventsSummary(evaluation, iter, group);
     ExampleUtils.writeEvaluationSDF(iter, group, evaluation, examples.size());
     LogInfo.end_track();
@@ -273,9 +309,10 @@ public class Learner {
   // Print summary over all examples
   private void logEvaluationStats(Evaluation evaluation, String prefix) {
     LogInfo.logs("Stats for %s: %s", prefix, evaluation.summary());
-   // evaluation.add(LexiconFn.lexEval);
+    // evaluation.add(LexiconFn.lexEval);
     evaluation.logStats(prefix);
     evaluation.putOutput(prefix);
+    evaluation.putOutput(prefix.replaceAll("iter=", "").replace('.', '_'));
   }
 
   private void printLearnerEventsIter(Example ex, int iter, String group) {

src/edu/stanford/nlp/sempre/LearnerParallelProcessor.java

@@ -0,0 +1,77 @@
+package edu.stanford.nlp.sempre;
+
+import java.util.*;
+
+import fig.basic.Evaluation;
+import fig.basic.LogInfo;
+import fig.basic.Parallelizer;
+import fig.basic.StopWatchSet;
+import fig.exec.Execution;
+
+/**
+ * Parallel version of the Learner.
+ *
+ * Most of the codes are copied from the paraphrase package.
+ *
+ * @author ppasupat
+ */
+public class LearnerParallelProcessor implements Parallelizer.Processor<Example> {
+
+  private final Parser parser;
+  private final String prefix;
+  private final boolean computeExpectedCounts;
+  private Params params;         // this is common to threads and should be synchronized
+  private Evaluation evaluation; // this is common to threads and should be synchronized
+
+  public LearnerParallelProcessor(Parser parser, Params params, String prefix, boolean computeExpectedCounts, Evaluation evaluation) {
+    this.prefix = prefix;
+    this.parser = parser;
+    this.computeExpectedCounts = computeExpectedCounts;
+    this.params = params;
+    this.evaluation = evaluation;
+  }
+
+  @Override
+  public void process(Example ex, int i, int n) {
+    LogInfo.begin_track_printAll(
+        "%s: example %s/%s: %s", prefix, i, n, ex.id);
+    ex.log();
+    Execution.putOutput("example", i);
+
+    StopWatchSet.begin("Parser.parse");
+    ParserState state = parser.parse(params, ex, computeExpectedCounts);
+    StopWatchSet.end();
+
+    if (computeExpectedCounts) {
+      Map<String, Double> counts = new HashMap<>();
+      SempreUtils.addToDoubleMap(counts, state.expectedCounts);
+
+      // Gathered enough examples, update parameters
+      StopWatchSet.begin("Learner.updateWeights");
+      LogInfo.begin_track("Updating learner weights");
+      if (Learner.opts.verbose >= 2)
+        SempreUtils.logMap(counts, "gradient");
+      double sum = 0;
+      for (double v : counts.values()) sum += v * v;
+      LogInfo.logs("L2 norm: %s", Math.sqrt(sum));
+      synchronized (params) {
+        params.update(counts);
+      }
+      counts.clear();
+      LogInfo.end_track();
+      StopWatchSet.end();
+    }
+
+    LogInfo.logs("Current: %s", ex.evaluation.summary());
+    synchronized (evaluation) {
+      evaluation.add(ex.evaluation);
+      LogInfo.logs("Cumulative(%s): %s", prefix, evaluation.summary());
+    }
+
+    LogInfo.end_track();
+
+    // To save memory
+    ex.clean();
+  }
+
+}

src/edu/stanford/nlp/sempre/ListValue.java

@@ -1,11 +1,10 @@
 package edu.stanford.nlp.sempre;
 
+import java.util.*;
+
 import fig.basic.LispTree;
 import fig.basic.LogInfo;
 
-import java.util.ArrayList;
-import java.util.List;
-
 public class ListValue extends Value {
   public final List<Value> values;
 
@@ -35,9 +34,26 @@ public class ListValue extends Value {
     if (this == o) return true;
     if (o == null || getClass() != o.getClass()) return false;
     ListValue that = (ListValue) o;
-    if (!values.equals(that.values)) return false;
-    return true;
+    return values.equals(that.values);
   }
 
   @Override public int hashCode() { return values.hashCode(); }
+
+  // Sorted on string representation
+  public ListValue getSorted() {
+    List<Value> sorted = new ArrayList<>(values);
+    Collections.sort(sorted,
+        (Value v1, Value v2) -> (
+            v1 == null ? "null" : v1.sortString()).compareTo(v2 == null ? "null" : v2.sortString()));
+    return new ListValue(sorted);
+  }
+
+  // Unique
+  public ListValue getUnique() {
+    List<Value> sorted = new ArrayList<>(new HashSet<>(values));
+    Collections.sort(sorted,
+        (Value v1, Value v2) -> (
+            v1 == null ? "null" : v1.sortString()).compareTo(v2 == null ? "null" : v2.sortString()));
+    return new ListValue(sorted);
+  }
 }

src/edu/stanford/nlp/sempre/Main.java

@@ -1,5 +1,8 @@
 package edu.stanford.nlp.sempre;
 
+import java.lang.reflect.Constructor;
+
+
 import fig.basic.Option;
 import fig.exec.Execution;
 
@@ -11,6 +14,7 @@ import fig.exec.Execution;
 public class Main implements Runnable {
   @Option public boolean interactive = false;
   @Option public boolean server = false;
+  @Option public String masterType = "edu.stanford.nlp.sempre.Master";
 
   public void run() {
     Builder builder = new Builder();
@@ -22,16 +26,24 @@ public class Main implements Runnable {
     Learner learner = new Learner(builder.parser, builder.params, dataset);
     learner.learn();
 
-    if (server) {
-      Master master = new Master(builder);
-      Server server = new Server(master);
-      server.run();
+    if (server || interactive) {
+      Master master = createMaster(masterType, builder);
+      if (server)
+        master.runServer();
+      if (interactive)
+        master.runInteractivePrompt();
     }
+  }
 
-    if (interactive) {
-      Master master = new Master(builder);
-      master.runInteractivePrompt();
+  public Master createMaster(String masterType, Builder builder) {
+    try {
+      Class<?> masterClass = Class.forName(masterType);
+      Constructor<?> constructor = masterClass.getConstructor(Builder.class);
+      return (Master)constructor.newInstance(builder);
+    } catch (Throwable t) {
+      t.printStackTrace();
     }
+    return null;
   }
 
   public static void main(String[] args) {

src/edu/stanford/nlp/sempre/MarkFormula.java

@@ -33,6 +33,12 @@ public class MarkFormula extends Formula {
     return tree;
   }
 
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) body.forEach(func);
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> func) {
     Formula result = func.apply(this);
     return result == null ? new MarkFormula(var, body.map(func)) : result;

src/edu/stanford/nlp/sempre/Master.java

@@ -4,6 +4,7 @@ import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import fig.basic.*;
+import jline.console.ConsoleReader;
 
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -40,22 +41,24 @@ public class Master {
     @Option(gloss = "Write out new grammar rules")
     public String newGrammarPath;
   }
-
   public static Options opts = new Options();
-
+  
   public class Response {
     // Example that was parsed, if any.
-    Example ex;
+    public Example ex;
 
     // Which derivation we're selecting to show
     int candidateIndex = -1;
 
     // Detailed information
-    List<String> lines = new ArrayList<>();
+    public Map<String, Object> stats = new LinkedHashMap<>();
+    public List<String> lines = new ArrayList<>();
 
     public String getFormulaAnswer() {
       if (ex.getPredDerivations().size() == 0)
         return "(no answer)";
+      else if (candidateIndex == -1)
+        return "(not selected)";
       else {
         Derivation deriv = getDerivation();
         return deriv.getFormula() + " => " + deriv.getValue();
@@ -64,6 +67,8 @@ public class Master {
     public String getAnswer() {
       if (ex.getPredDerivations().size() == 0)
         return "(no answer)";
+      else if (candidateIndex == -1)
+        return "(not selected)";
       else {
         Derivation deriv = getDerivation();
         deriv.ensureExecuted(builder.executor, ex.context);
@@ -79,9 +84,9 @@ public class Master {
     }
   }
 
-  private Builder builder;
-  private Learner learner;
-  private HashMap<String, Session> sessions = new LinkedHashMap<>();
+  protected Builder builder;
+  protected Learner learner;
+  protected HashMap<String, Session> sessions = new LinkedHashMap<>();
 
   public Master(Builder builder) {
     this.builder = builder;
@@ -106,7 +111,7 @@ public class Master {
     return session;
   }
 
-  void printHelp() {
+  protected void printHelp() {
     LogInfo.log("Enter an utterance to parse or one of the following commands:");
     LogInfo.log("  (help): show this help message");
     LogInfo.log("  (status): prints out status of the system");
@@ -126,31 +131,32 @@ public class Master {
     LogInfo.log("Press Ctrl-D to exit.");
   }
 
+  public void runServer() {
+    Server server = new Server(this);
+    server.run();;
+  }
+  
   public void runInteractivePrompt() {
     Session session = getSession("stdin");
 
     if (opts.printHelp)
       printHelp();
-
-    while (true) {
-      LogInfo.stdout.print("> ");
-      LogInfo.stdout.flush();
+    try {
+      ConsoleReader reader = new ConsoleReader();
+      reader.setPrompt("> ");
       String line;
-      try {
-        line = LogInfo.stdin.readLine();
-      } catch (IOException e) {
-        throw new RuntimeException(e);
-      }
-      if (line == null) break;
-
-      int indent = LogInfo.getIndLevel();
-      try {
-        processQuery(session, line);
-      } catch (Throwable t) {
-        while (LogInfo.getIndLevel() > indent)
-          LogInfo.end_track();
-        t.printStackTrace();
+      while ((line = reader.readLine()) != null) {
+        int indent = LogInfo.getIndLevel();
+        try {
+          processQuery(session, line);
+        } catch (Throwable t) {
+          while (LogInfo.getIndLevel() > indent)
+            LogInfo.end_track();
+          t.printStackTrace();
+        }
       }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
     }
   }
 
@@ -229,7 +235,7 @@ public class Master {
     builder.parser.parse(builder.params, ex, false);
 
     response.ex = ex;
-    ex.log();
+    ex.logWithoutContext();
     if (ex.predDerivations.size() > 0) {
       response.candidateIndex = 0;
       printDerivation(response.getDerivation());
@@ -377,7 +383,14 @@ public class Master {
       } else {
         session.context = new ContextValue(tree);
       }
-    } else {
+    } else if (command.equals("loadgraph")) {
+      if (tree.children.size() != 2 || !tree.child(1).isLeaf())
+        throw new RuntimeException("Invalid argument: argument should be a file path");
+      KnowledgeGraph graph = NaiveKnowledgeGraph.fromFile(tree.child(1).value);
+      session.context = new ContextValue(session.context.user, session.context.date,
+        session.context.exchanges, graph);
+    }
+    else {
       LogInfo.log("Invalid command: " + tree);
     }
   }

src/edu/stanford/nlp/sempre/MergeFormula.java

@@ -30,6 +30,12 @@ public class MergeFormula extends Formula {
     return tree;
   }
 
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) { child1.forEach(func); child2.forEach(func); }
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> func) {
     Formula result = func.apply(this);
     return result == null ? new MergeFormula(mode, child1.map(func), child2.map(func)) : result;

src/edu/stanford/nlp/sempre/MixParser.java

@@ -0,0 +1,155 @@
+package edu.stanford.nlp.sempre;
+
+import java.util.*;
+import java.util.regex.Pattern;
+
+import fig.basic.LogInfo;
+import fig.basic.Option;
+import fig.basic.Pair;
+
+/**
+ * A parser that mixes the derivation lists from other parsers.
+ *
+ * @author ppasupat
+ */
+public class MixParser extends Parser {
+  public static class Options {
+    @Option(gloss = "verbosity")
+    public int verbose = 1;
+
+    /**
+     * Syntax: [className]:[options]
+     * - className also includes the package name (BeamParser, FloatingParser, tables.dpd.DPDParser, ...)
+     * - options is a comma-separated list of [group] or [group]-[iter1]-[iter2]-...
+     *   [iter1], [iter2], ... are iteration indices (0-based), "xc" (computing expected counts),
+     *   "nxc" (not computing expected counts), index + "xc", or index + "nxc"
+     *
+     * Example: Using
+     *      -MixParser.parsers FloatingParser tables.serialize.SerializedParser:train-0-2-3,dev
+     *   will cause derivations from SerializedParser to be mixed in during all dev iterations
+     *   and training iterations 0, 2, and 3.
+     */
+    @Option(gloss = "list of parsers to use along with options")
+    public List<String> parsers = new ArrayList<>();
+  }
+  public static Options opts = new Options();
+
+  final List<Pair<Parser, MixParserOption>> parsers;
+  int iter, numIters;
+  String group;
+
+  public MixParser(Spec spec) {
+    super(spec);
+    parsers = new ArrayList<>();
+    for (String parserAndOptions : opts.parsers) {
+      if (opts.verbose >= 1)
+        LogInfo.logs("Adding parser %s", parserAndOptions);
+      String[] tokens = parserAndOptions.split(":");
+      if (tokens.length > 2)
+        throw new RuntimeException("Invalid parser options: " + parserAndOptions);
+      String parserName = tokens[0];
+      Parser parser;
+      try {
+        Class<?> parserClass = Class.forName(SempreUtils.resolveClassName(parserName));
+        parser = ((Parser) parserClass.getConstructor(spec.getClass()).newInstance(spec));
+      } catch (ClassNotFoundException e1) {
+        throw new RuntimeException("Illegal parser: " + parserName);
+      } catch (Exception e) {
+        e.printStackTrace();
+        throw new RuntimeException("Error while instantiating parser: " + parserName + "\n" + e);
+      }
+      if (tokens.length > 1)
+        parsers.add(new Pair<>(parser, new MixParserOption(this, tokens[1])));
+      else
+        parsers.add(new Pair<>(parser, new MixParserOption(this)));
+    }
+  }
+
+  // Don't do it.
+  @Override protected void computeCatUnaryRules() {
+    catUnaryRules = Collections.emptyList();
+  };
+
+  @Override
+  public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) {
+    return new MixParserState(this, params, ex, computeExpectedCounts);
+  }
+
+  @Override
+  public void onBeginDataGroup(int iter, int numIters, String group) {
+    this.iter = iter;
+    this.numIters = numIters;
+    this.group = group;
+  }
+}
+
+class MixParserOption {
+  private final MixParser mixParser;
+  private boolean allowedAll = false;
+  private List<Pair<String, String>> allowedGroupsAndIter = new ArrayList<>();
+
+  public MixParserOption(MixParser mixParser) {
+    this.mixParser = mixParser;
+    // Allow in all groups
+    allowedAll = true;
+  }
+
+  public MixParserOption(MixParser mixParser, String optionString) {
+    this.mixParser = mixParser;
+    String[] tokens = optionString.split(",");
+    for (String option : tokens) {
+      String[] subtokens = option.split("-");
+      if (subtokens.length == 1)
+        allowedGroupsAndIter.add(new Pair<>(subtokens[0], "all"));
+      else
+        for (int i = 1; i < subtokens.length; i++) {
+          if (!Pattern.matches("^([0-9]*(n?xc)?)$", subtokens[i]))
+            throw new RuntimeException("Invalid iteration options: " + subtokens[i]);
+          allowedGroupsAndIter.add(new Pair<>(subtokens[0], subtokens[i]));
+        }
+    }
+  }
+
+  public boolean isAllowed(boolean computeExpectedCounts) {
+    if (allowedAll) return true;
+    String xcFlag = computeExpectedCounts ? "xc" : "nxc";
+    return allowedGroupsAndIter.contains(new Pair<>(mixParser.group, "all"))
+        || allowedGroupsAndIter.contains(new Pair<>(mixParser.group, "" + mixParser.iter))
+        || allowedGroupsAndIter.contains(new Pair<>(mixParser.group, xcFlag))
+        || allowedGroupsAndIter.contains(new Pair<>(mixParser.group, "" + mixParser.iter + xcFlag));
+  }
+}
+
+class MixParserState extends ParserState {
+
+  public MixParserState(Parser parser, Params params, Example ex, boolean computeExpectedCounts) {
+    super(parser, params, ex, computeExpectedCounts);
+  }
+
+  @Override
+  public void infer() {
+    for (Pair<Parser, MixParserOption> pair : ((MixParser) parser).parsers) {
+      if (!pair.getSecond().isAllowed(computeExpectedCounts)) {
+        if (MixParser.opts.verbose >= 1)
+          LogInfo.logs("Skipping %s", pair.getFirst().getClass().getSimpleName());
+        continue;
+      }
+      if (MixParser.opts.verbose >= 1)
+        LogInfo.begin_track("Using %s", pair.getFirst().getClass().getSimpleName());
+      ParserState parserState = pair.getFirst().newParserState(params, ex, false);
+      parserState.infer();
+      predDerivations.addAll(parserState.predDerivations);
+      if (MixParser.opts.verbose >= 1) {
+        LogInfo.logs("Number of derivations: %d", parserState.predDerivations.size());
+        LogInfo.end_track();
+      }
+
+    }
+    ensureExecuted();
+    if (computeExpectedCounts) {
+      expectedCounts = new HashMap<>();
+      ParserState.computeExpectedCounts(predDerivations, expectedCounts);
+    }
+  }
+
+}

src/edu/stanford/nlp/sempre/NaiveKnowledgeGraph.java

@@ -84,25 +84,14 @@ public class NaiveKnowledgeGraph extends KnowledgeGraph {
     }
   }
 
-  @Override
-  public Collection<Formula> getFuzzyMatchedFormulas(String term, FuzzyMatchFn.FuzzyMatchFnMode mode) {
-    throw new RuntimeException("Not implemented yet");
-  }
-
-  @Override
-  public Collection<Formula> getAllFormulas(FuzzyMatchFn.FuzzyMatchFnMode mode) {
-    throw new RuntimeException("Not implemented yet");
-  }
-
   // ============================================================
   // Queries
   // ============================================================
 
   @Override
   public List<Value> joinFirst(Value r, Collection<Value> firsts) {
-    Value reversed = isReversedRelation(r);
-    if (reversed != null)
-      return joinSecond(reversed, firsts);
+    if (CanonicalNames.isReverseProperty(r))
+      return joinSecond(CanonicalNames.reverseProperty(r), firsts);
     List<Value> seconds = new ArrayList<>();
     List<KnowledgeGraphTriple> relationTriples = relationToTriples.get(r);
     if (relationTriples != null) {
@@ -116,9 +105,8 @@ public class NaiveKnowledgeGraph extends KnowledgeGraph {
 
   @Override
   public List<Value> joinSecond(Value r, Collection<Value> seconds) {
-    Value reversed = isReversedRelation(r);
-    if (reversed != null)
-      return joinFirst(reversed, seconds);
+    if (CanonicalNames.isReverseProperty(r))
+      return joinFirst(CanonicalNames.reverseProperty(r), seconds);
     List<Value> firsts = new ArrayList<>();
     List<KnowledgeGraphTriple> relationTriples = relationToTriples.get(r);
     if (relationTriples != null) {
@@ -132,9 +120,8 @@ public class NaiveKnowledgeGraph extends KnowledgeGraph {
 
   @Override
   public List<Pair<Value, Value>> filterFirst(Value r, Collection<Value> firsts) {
-    Value reversed = isReversedRelation(r);
-    if (reversed != null)
-      return getReversedPairs(filterSecond(reversed, firsts));
+    if (CanonicalNames.isReverseProperty(r))
+      return getReversedPairs(filterSecond(CanonicalNames.reverseProperty(r), firsts));
     List<Pair<Value, Value>> pairs = new ArrayList<>();
     List<KnowledgeGraphTriple> relationTriples = relationToTriples.get(r);
     if (relationTriples != null) {
@@ -148,9 +135,8 @@ public class NaiveKnowledgeGraph extends KnowledgeGraph {
 
   @Override
   public List<Pair<Value, Value>> filterSecond(Value r, Collection<Value> seconds) {
-    Value reversed = isReversedRelation(r);
-    if (reversed != null)
-      return getReversedPairs(filterFirst(reversed, seconds));
+    if (CanonicalNames.isReverseProperty(r))
+      return getReversedPairs(filterFirst(CanonicalNames.reverseProperty(r), seconds));
     List<Pair<Value, Value>> pairs = new ArrayList<>();
     List<KnowledgeGraphTriple> relationTriples = relationToTriples.get(r);
     if (relationTriples != null) {
@@ -184,6 +170,10 @@ public class NaiveKnowledgeGraph extends KnowledgeGraph {
     return new NaiveKnowledgeGraph(triples);
   }
 
+  public static KnowledgeGraph fromFile(String path) {
+    return fromLispTree(LispTree.proto.parseFromFile(path).next());
+  }
+
   @Override
   public LispTree toLispTree() {
     LispTree tree = LispTree.proto.newList();
@@ -194,4 +184,16 @@ public class NaiveKnowledgeGraph extends KnowledgeGraph {
     }
     return tree;
   }
+
+  @Override
+  public LispTree toShortLispTree() {
+    if (triples.size() > 1000) {
+      LispTree tree = LispTree.proto.newList();
+      tree.addChild("graph");
+      tree.addChild("NaiveKnowledgeGraph");
+      tree.addChild(("TooManyTriples"));
+      return tree;
+    }
+    return toLispTree();
+  }
 }

src/edu/stanford/nlp/sempre/NameValue.java

@@ -41,6 +41,9 @@ public class NameValue extends Value {
     return tree;
   }
 
+  @Override public String sortString() { return id; }
+  @Override public String pureString() { return description == null ? id : description; }
+
   @Override public int hashCode() { return id.hashCode(); }
   @Override public boolean equals(Object o) {
     if (this == o) return true;

src/edu/stanford/nlp/sempre/NotFormula.java

@@ -22,6 +22,12 @@ public class NotFormula extends Formula {
     return tree;
   }
 
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) child.forEach(func);
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> func) {
     Formula result = func.apply(this);
     return result == null ? new NotFormula(child.map(func)) : result;

src/edu/stanford/nlp/sempre/NumberFn.java

@@ -2,6 +2,7 @@ package edu.stanford.nlp.sempre;
 
 import java.util.ArrayList;
 import java.util.List;
+
 import fig.basic.*;
 
 /**
@@ -14,6 +15,10 @@ public class NumberFn extends SemanticFn {
     @Option(gloss = "Omit units") public boolean unitless = false;
     @Option(gloss = "Also test numbers by try converting to float (instead of using NER tags)")
     public boolean alsoTestByConversion = false;
+    @Option(gloss = "Also test numbers by applying NER on just the phrase")
+    public boolean alsoTestByIsolatedNER = false;
+    @Option(gloss = "range of allowed numbers. e.g. null: no limits, Lists.newArrayList(0,100): 0-100 inclusive")
+    public List<Double> allowedRange = null;
   }
   public static Options opts = new Options();
 
@@ -36,12 +41,54 @@ public class NumberFn extends SemanticFn {
   public DerivationStream call(final Example ex, final Callable c) {
     return new SingleDerivationStream() {
       public Derivation createDerivation() {
-        // Numbers: If it is an integer, set its type to integer.  Otherwise, use float.
-        if (request("NUMBER")) {
-          String value = ex.languageInfo.getNormalizedNerSpan("NUMBER", c.getStart(), c.getEnd());
+        // Test using NER span
+        Derivation deriv = check(ex.languageInfo, c.getStart(), c.getEnd());
+        if (deriv != null) return deriv;
+
+        // Test by converting string to number directly (don't look at NER)
+        if (opts.alsoTestByConversion && request("NUMBER") & c.getEnd() - c.getStart() == 1) {
+          String value = ex.languageInfo.tokens.get(c.getStart());
           if (value != null) {
             try {
               NumberValue numberValue = new NumberValue(Double.parseDouble(value));
+              SemType type = numberValue.value == (int) numberValue.value ? SemType.intType : SemType.floatType;
+              return new Derivation.Builder()
+              .withCallable(c)
+              .formula(new ValueFormula<>(numberValue))
+              .type(type)
+              .createDerivation();
+            } catch (NumberFormatException e) {
+              // Don't issue warnings; most spans are not numbers
+            }
+          }
+        }
+
+        // Test by applying NER on just the phrase
+        if (opts.alsoTestByIsolatedNER) {
+          String phrase = ex.phraseString(c.getStart(), c.getEnd());
+          LanguageInfo languageInfo = LanguageAnalyzer.getSingleton().analyze(phrase);
+          deriv = check(languageInfo, 0, languageInfo.numTokens());
+          if (deriv != null)
+            return deriv;
+        }
+
+        return null;
+      }
+
+      public Derivation check(LanguageInfo languageInfo, int start, int end) {
+        // Numbers: If it is an integer, set its type to integer.  Otherwise, use float.
+        if (request("NUMBER")) {
+          String value = languageInfo.getNormalizedNerSpan("NUMBER", start, end);
+          if (value != null) {
+            try {
+              NumberValue numberValue = new NumberValue(Double.parseDouble(value));
+              if (opts.allowedRange != null) {
+                if (numberValue.value < opts.allowedRange.get(0) || numberValue.value > opts.allowedRange.get(1)) {
+                  LogInfo.warnings("NumberFn: %f is outside of the allowed range %s", numberValue.value, opts.allowedRange);
+                  return null;
+                }
+              }
+
               SemType type = numberValue.value == (int) numberValue.value ? SemType.intType : SemType.floatType;
               return new Derivation.Builder()
                       .withCallable(c)
@@ -56,7 +103,7 @@ public class NumberFn extends SemanticFn {
 
         // Ordinals
         if (request("ORDINAL")) {
-          String value = ex.languageInfo.getNormalizedNerSpan("ORDINAL", c.getStart(), c.getEnd());
+          String value = languageInfo.getNormalizedNerSpan("ORDINAL", start, end);
           if (value != null) {
             try {
               NumberValue numberValue = (opts.unitless ?
@@ -76,7 +123,7 @@ public class NumberFn extends SemanticFn {
 
         // Percents
         if (request("PERCENT")) {
-          String value = ex.languageInfo.getNormalizedNerSpan("PERCENT", c.getStart(), c.getEnd());
+          String value = languageInfo.getNormalizedNerSpan("PERCENT", start, end);
           if (value != null) {
             try {
               NumberValue numberValue = (opts.unitless ?
@@ -96,7 +143,7 @@ public class NumberFn extends SemanticFn {
 
         // Money
         if (request("MONEY")) {
-          String value = ex.languageInfo.getNormalizedNerSpan("MONEY", c.getStart(), c.getEnd());
+          String value = languageInfo.getNormalizedNerSpan("MONEY", start, end);
           if (value != null) {
             try {
               NumberValue numberValue = (opts.unitless ?
@@ -114,24 +161,6 @@ public class NumberFn extends SemanticFn {
           }
         }
 
-        // Test by converting string to number directly (don't look at NER)
-        if (opts.alsoTestByConversion && request("NUMBER") & c.getEnd() - c.getStart() == 1) {
-          String value = ex.languageInfo.tokens.get(c.getStart());
-          if (value != null) {
-            try {
-              NumberValue numberValue = new NumberValue(Double.parseDouble(value));
-              SemType type = numberValue.value == (int) numberValue.value ? SemType.intType : SemType.floatType;
-              return new Derivation.Builder()
-                      .withCallable(c)
-                      .formula(new ValueFormula<>(numberValue))
-                      .type(type)
-                      .createDerivation();
-            } catch (NumberFormatException e) {
-              // Don't issue warnings; most spans are not numbers
-            }
-          }
-        }
-
         return null;
       }
     };

src/edu/stanford/nlp/sempre/NumberValue.java

@@ -39,6 +39,9 @@ public class NumberValue extends Value {
     return tree;
   }
 
+  @Override public String sortString() { return "" + value; }
+  @Override public String pureString() { return "" + value; }
+
   @Override public int hashCode() { return Double.valueOf(value).hashCode(); }
   @Override public boolean equals(Object o) {
     if (this == o) return true;

src/edu/stanford/nlp/sempre/PairListValue.java

@@ -0,0 +1,68 @@
+package edu.stanford.nlp.sempre;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import fig.basic.LispTree;
+import fig.basic.LogInfo;
+import fig.basic.Pair;
+
+/**
+ * Represent a binary using a list of pairs.
+ *
+ * @author ppasupat
+ */
+public class PairListValue extends Value {
+  public final List<Pair<Value, Value>> pairs;
+
+  public PairListValue(LispTree tree) {
+    pairs = new ArrayList<>();
+    for (int i = 1; i < tree.children.size(); i++)
+      pairs.add(new Pair<>(
+          Values.fromLispTree(tree.child(i).child(0)),
+          Values.fromLispTree(tree.child(i).child(1))));
+  }
+
+  public PairListValue(List<Pair<Value, Value>> pairs) { this.pairs = pairs; }
+
+  protected static final LispTree NULL_LEAF = LispTree.proto.newLeaf(null);
+
+  public LispTree toLispTree() {
+    LispTree tree = LispTree.proto.newList();
+    tree.addChild("pairs");
+    for (Pair<Value, Value> pair : pairs) {
+      Value first = pair.getFirst(), second = pair.getSecond();
+      tree.addChild(LispTree.proto.newList(
+          first == null ? NULL_LEAF : first.toLispTree(), second == null ? NULL_LEAF : second.toLispTree()));
+    }
+    return tree;
+  }
+
+  public void log() {
+    for (Pair<Value, Value> pair : pairs)
+      LogInfo.logs("%s | %s", pair.getFirst(), pair.getSecond());
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    PairListValue that = (PairListValue) o;
+    return pairs.equals(that.pairs);
+  }
+
+  @Override public int hashCode() { return pairs.hashCode(); }
+
+  // Sorted on string representation
+  public PairListValue getSorted() {
+    List<Pair<Value, Value>> sorted = new ArrayList<>(pairs);
+    Collections.sort(sorted,
+        (Pair<Value, Value> p1, Pair<Value, Value> p2) -> getQuickStringOfPair(p1).compareTo(getQuickStringOfPair(p2)));
+    return new PairListValue(sorted);
+  }
+  private static String getQuickStringOfPair(Pair<Value, Value> pair) {
+    Value v1 = pair.getFirst(), v2 = pair.getSecond();
+    return (v1 == null ? "null" : v1.sortString()) + " " + (v2 == null ? "null" : v2.sortString());
+  }
+}

src/edu/stanford/nlp/sempre/Parser.java

@@ -51,11 +51,14 @@ public abstract class Parser {
     @Option(gloss = "Source of random noise")
     public Random derivationScoreRandom = new Random(1);
 
-    @Option (gloss = "Prune away error denotations")
+    @Option(gloss = "Prune away error denotations")
     public boolean pruneErrorValues = false;
 
     @Option(gloss = "Dump all features (for debugging)")
     public boolean dumpAllFeatures = false;
+
+    @Option(gloss = "Call SetEvaluation during parsing")
+    public boolean callSetEvaluation = true;
   }
 
   public static final Options opts = new Options();
@@ -88,7 +91,7 @@ public abstract class Parser {
   public List<Rule> getCatUnaryRules() { return catUnaryRules; }
 
   // TODO(joberant): move this to a separate class in charge of visualizing charts
-  PrintWriter chartFillOut = null;  // For printing a machine-readable json file
+  public PrintWriter chartFillOut = null;  // For printing a machine-readable json file
 
   public Parser(Spec spec) {
     this.grammar = spec.grammar;
@@ -97,7 +100,8 @@ public abstract class Parser {
     this.valueEvaluator = spec.valueEvaluator;
 
     computeCatUnaryRules();
-    LogInfo.logs("Parser: %d catUnaryRules (sorted), %d nonCatUnaryRules (in trie)", catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size());
+    LogInfo.logs("%s: %d catUnaryRules (sorted), %d nonCatUnaryRules (in trie)",
+        this.getClass().getSimpleName(), catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size());
   }
 
   // If grammar changes, then we might need to update aspects of the parser.
@@ -106,7 +110,7 @@ public abstract class Parser {
       catUnaryRules.add(rule);
   }
 
-  private void computeCatUnaryRules() {
+  protected void computeCatUnaryRules() {
     // Handle catUnaryRules
     catUnaryRules = new ArrayList<>();
     Map<String, List<Rule>> graph = new HashMap<>();  // Node from LHS to list of rules
@@ -121,10 +125,8 @@ public abstract class Parser {
   }
 
   // Helper function for transitive closure of unary rules.
-  private void traverse(List<Rule> catUnaryRules,
-                          String node,
-                          Map<String, List<Rule>> graph,
-                          Map<String, Boolean> done) {
+  protected void traverse(List<Rule> catUnaryRules, String node,
+      Map<String, List<Rule>> graph, Map<String, Boolean> done) {
     Boolean d = done.get(node);
     if (Boolean.TRUE.equals(d)) return;
     if (Boolean.FALSE.equals(d))
@@ -137,6 +139,15 @@ public abstract class Parser {
     done.put(node, true);
   }
 
+  /**
+   * Override this method to change the parser's behavior based on current
+   * group name and iteration number. This method will be called at the
+   * beginning of each data group.
+   */
+  public void onBeginDataGroup(int iter, int numIters, String group) {
+    // DEFAULT: Do nothing.
+  }
+
   // Main thing for parsers to implement.
   public abstract ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts);
   public Params getSearchParams(Params params) { return params; }
@@ -154,7 +165,7 @@ public abstract class Parser {
     // Parse
     StopWatch watch = new StopWatch();
     watch.start();
-    LogInfo.begin_track("Parser.parse: parse");
+    LogInfo.begin_track_printAll("Parser.parse: parse");
     ParserState state = newParserState(params, ex, computeExpectedCounts);
     state.infer();
     LogInfo.end_track();
@@ -166,14 +177,14 @@ public abstract class Parser {
     Derivation.sortByScore(ex.predDerivations);
 
     // Evaluate
-    ex.evaluation = new Evaluation();
-    addToEvaluation(state, ex.evaluation);
-
+    if (opts.callSetEvaluation) {
+      ex.evaluation = new Evaluation();
+      addToEvaluation(state, ex.evaluation);
+    }
     // Clean up temporary state used during parsing
     ex.clearTempState();
     for (Derivation deriv : ex.predDerivations)
       deriv.clearTempState();
-
     return state;
   }
 
@@ -199,22 +210,30 @@ public abstract class Parser {
     int correctIndexAfterParse = -1;
     double maxCompatibility = 0.0;
     double[] compatibilities = null;
+    int numCorrect = 0, numPartialCorrect = 0, numIncorrect = 0;
+
     if (ex.targetValue != null) {
       compatibilities = new double[numCandidates];
       for (int i = 0; i < numCandidates; i++) {
         Derivation deriv = predDerivations.get(i);
         compatibilities[i] = deriv.compatibility;
-
         // Must be fully compatible to count as correct.
         if (compatibilities[i] == 1 && correctIndex == -1)
           correctIndex = i;
         // record maximum compatibility for partial oracle
         maxCompatibility = Math.max(compatibilities[i], maxCompatibility);
+        // Count
+        if (compatibilities[i] == 1) {
+          numCorrect++;
+        } else if (compatibilities[i] == 0) {
+          numIncorrect++;
+        } else {
+          numPartialCorrect++;
+        }
       }
       // What if we only had parsed bottom up?
       for (int i = 0; i < numCandidates; i++) {
-        Derivation deriv = predDerivations.get(i);
-        if (deriv.compatibility == 1) {
+        if (compatibilities[i] == 1) {
           correctIndexAfterParse = i;
           break;
         }
@@ -233,13 +252,12 @@ public abstract class Parser {
     double topMass = 0;
     if (ex.targetValue != null) {
       while (numTop < numCandidates &&
-              Math.abs(predDerivations.get(numTop).score - predDerivations.get(0).score) < 1e-10) {
+          Math.abs(predDerivations.get(numTop).score - predDerivations.get(0).score) < 1e-10) {
         topMass += probs[numTop];
         numTop++;
       }
     }
-    double correct = 0;
-    double partCorrect = 0;
+    double correct = 0, partCorrect = 0;
     if (ex.targetValue != null) {
       for (int i = 0; i < numTop; i++) {
         if (compatibilities[i] == 1) correct += probs[i] / topMass;
@@ -273,8 +291,8 @@ public abstract class Parser {
         boolean print = printAllPredictions || (numPrintedSoFar < opts.maxPrintedTrue);
         if (print) {
           LogInfo.logs(
-                  "True@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
-                  Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
+              "True@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
+              Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
           numPrintedSoFar++;
           if (opts.dumpAllFeatures) FeatureVector.logFeatureWeights("Features", deriv.getAllFeatureVector(), state.params);
         }
@@ -288,8 +306,8 @@ public abstract class Parser {
         boolean print = printAllPredictions || (numPrintedSoFar < opts.maxPrintedTrue);
         if (print) {
           LogInfo.logs(
-                  "Part@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
-                  Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
+              "Part@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
+              Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
           numPrintedSoFar++;
           if (opts.dumpAllFeatures) FeatureVector.logFeatureWeights("Features", deriv.getAllFeatureVector(), state.params);
         }
@@ -303,8 +321,8 @@ public abstract class Parser {
       boolean print = printAllPredictions || ((probs[i] >= probs[0] / 2 || i < 10) && i < opts.maxPrintedPredictions);
       if (print) {
         LogInfo.logs(
-                "Pred@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
-                Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
+            "Pred@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
+            Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
         // LogInfo.logs("Derivation tree: %s", deriv.toRecursiveString());
         if (opts.dumpAllFeatures) FeatureVector.logFeatureWeights("Features", deriv.getAllFeatureVector(), state.params);
       }
@@ -325,6 +343,9 @@ public abstract class Parser {
     evaluation.add("numCandidates", numCandidates);  // From this parse
     if (numCandidates > 0)
       evaluation.add("parsedNumCandidates", numCandidates);
+    evaluation.add("numCorrect", numCorrect);
+    evaluation.add("numPartialCorrect", numPartialCorrect);
+    evaluation.add("numIncorrect", numIncorrect);
 
     // Add parsing stats
     evaluation.add(state.evaluation);

src/edu/stanford/nlp/sempre/ParserState.java

@@ -1,14 +1,7 @@
 package edu.stanford.nlp.sempre;
 
-import fig.basic.Fmt;
-import fig.basic.LogInfo;
-import fig.basic.NumUtils;
-import fig.basic.Evaluation;
-import fig.basic.Option;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+import fig.basic.*;
 
 /**
  * Actually does the parsing.  Main method is infer(), whose job is to fill in
@@ -20,14 +13,18 @@ public abstract class ParserState {
   public static class Options {
     @Option(gloss = "Use a custom distribution for computing expected counts")
     public CustomExpectedCount customExpectedCounts = CustomExpectedCount.NONE;
+    @Option(gloss = "For customExpectedCounts = TOP, only update if good < bad + margin")
+    public double contrastiveMargin = 1e6;    // default = always update
     @Option(gloss = "Whether to prune based on probability difference")
     public boolean pruneByProbDiff = false;
     @Option(gloss = "Difference in probability for pruning by prob diff")
     public double probDiffPruningThresh = 100;
+    @Option(gloss = "Throw features away after scoring to save memory")
+    public boolean throwFeaturesAway = false;
   }
   public static Options opts = new Options();
 
-  public enum CustomExpectedCount { NONE, UNIFORM, TOP, RANDOM, }
+  public enum CustomExpectedCount { NONE, UNIFORM, TOP, TOPALT, RANDOM, }
 
   //// Input: specification of how to parse
 
@@ -79,9 +76,12 @@ public abstract class ParserState {
     // Compute score
     deriv.computeScoreLocal(params);
 
+    if (opts.throwFeaturesAway)
+      deriv.clearFeatures();
+
     if (parser.verbose(5)) {
       LogInfo.logs("featurizeAndScoreDerivation(score=%s) %s %s: %s [rule: %s]",
-              Fmt.D(deriv.score), deriv.cat, ex.spanString(deriv.start, deriv.end), deriv, deriv.rule);
+          Fmt.D(deriv.score), deriv.cat, ex.spanString(deriv.start, deriv.end), deriv, deriv.rule);
     }
     numOfFeaturizedDerivs++;
   }
@@ -99,7 +99,8 @@ public abstract class ParserState {
       maxCellSize = derivations.size();
       maxCellDescription = cellDescription;
       if (maxCellSize > 5000)
-        LogInfo.logs("ParserState.pruneCell %s: %s entries", maxCellDescription, maxCellSize);
+        LogInfo.logs("ParserState.pruneCell %s: maxCellSize = %s entries (not pruned yet)",
+            maxCellDescription, maxCellSize);
     }
 
     // The extra code blocks in here that set |deriv.maxXBeamPosition|
@@ -133,14 +134,12 @@ public abstract class ParserState {
     Derivation.sortByScore(derivations);
 
     // Print out information
-    if (parser.opts.verbose >= 3) {
+    if (Parser.opts.verbose >= 3) {
       LogInfo.begin_track("ParserState.pruneCell(%s): %d derivations", cellDescription, derivations.size());
       for (Derivation deriv : derivations) {
-        LogInfo.logs("%s(%s,%s): %s %s, [score=%s]", deriv.cat, deriv.start, deriv.end, deriv.formula,
-                deriv.canonicalUtterance, deriv.score);
+        LogInfo.logs("%s(%s,%s): %s %s, [score=%s] allAnchored: %s", deriv.cat, deriv.start, deriv.end, deriv.formula,
+            deriv.canonicalUtterance, deriv.score, deriv.allAnchored());
       }
-
-
       LogInfo.end_track();
     }
 
@@ -168,6 +167,9 @@ public abstract class ParserState {
     else {
       // Keep only the top hypotheses
       int beamSize = getBeamSize();
+      if (derivations.size() > beamSize && Parser.opts.verbose >= 1) {
+        LogInfo.logs("ParserState.pruneCell %s: Pruning %d -> %d derivations", cellDescription, derivations.size(), beamSize);
+      }
       while (derivations.size() > beamSize) {
         derivations.remove(derivations.size() - 1);
         fallOffBeam = true;
@@ -228,7 +230,7 @@ public abstract class ParserState {
   }
 
   // Ensure that all the logical forms are executed and compatibilities are computed.
-  protected void ensureExecuted() {
+  public void ensureExecuted() {
     LogInfo.begin_track("Parser.ensureExecuted");
     // Execute predicted derivations to get value.
     for (Derivation deriv : predDerivations) {
@@ -273,7 +275,7 @@ public abstract class ParserState {
     predScores = new double[n];
     // For update schemas that choose one good and one bad candidate to update
     int[] goodAndBad = null;
-    if (opts.customExpectedCounts == CustomExpectedCount.TOP) {
+    if (opts.customExpectedCounts == CustomExpectedCount.TOP || opts.customExpectedCounts == CustomExpectedCount.TOPALT) {
       goodAndBad = getTopDerivations(derivations);
       if (goodAndBad == null) return;
     } else if (opts.customExpectedCounts == CustomExpectedCount.RANDOM) {
@@ -298,6 +300,10 @@ public abstract class ParserState {
           trueScores[i] = (i == goodAndBad[0]) ? 0 : Double.NEGATIVE_INFINITY;
           predScores[i] = (i == goodAndBad[1]) ? 0 : Double.NEGATIVE_INFINITY;
           break;
+        case TOPALT:
+          trueScores[i] = (i == goodAndBad[0]) ? 0 : Double.NEGATIVE_INFINITY;
+          predScores[i] = (i == goodAndBad[0] || i == goodAndBad[1]) ? deriv.score : Double.NEGATIVE_INFINITY;
+          break;
         default:
           throw new RuntimeException("Unknown customExpectedCounts: " + opts.customExpectedCounts);
       }
@@ -331,11 +337,14 @@ public abstract class ParserState {
         }
       }
     }
-    return (chosenGood == -1 || chosenBad == -1) ? null : new int[] {chosenGood, chosenBad};
+    if (chosenGood == -1 || chosenBad == -1 || chosenGoodScore >= chosenBadScore + opts.contrastiveMargin)
+      return null;
+    return new int[] {chosenGood, chosenBad};
   }
 
   private static int[] getRandomDerivations(List<Derivation> derivations) {
     int chosenGood = -1, chosenBad = -1, numGoodSoFar = 0, numBadSoFar = 0;
+    // Get a uniform random sample from the stream
     for (int i = 0; i < derivations.size(); i++) {
       Derivation deriv = derivations.get(i);
       if (deriv.compatibility == 1) {

src/edu/stanford/nlp/sempre/PrimitiveFormula.java

@@ -11,10 +11,19 @@ import java.util.List;
  * @author Percy Liang
  */
 public abstract class PrimitiveFormula extends Formula {
+
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    func.apply(this);
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> func) {
     Formula result = func.apply(this);
     return result == null ? this : result;
   }
+
+  @Override
   public List<Formula> mapToList(Function<Formula, List<Formula>> func, boolean alwaysRecurse) {
     return func.apply(this);
   }

src/edu/stanford/nlp/sempre/ReverseFormula.java

@@ -26,6 +26,12 @@ public class ReverseFormula extends Formula {
     return tree;
   }
 
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) child.forEach(func);
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> func) {
     Formula result = func.apply(this);
     return result == null ? new ReverseFormula(child.map(func)) : result;

src/edu/stanford/nlp/sempre/Rule.java

@@ -5,7 +5,9 @@ import com.google.common.collect.Lists;
 import fig.basic.LispTree;
 import fig.basic.Pair;
 
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * A rule specifies how to take a right hand of terminals and non-terminals.
@@ -23,11 +25,12 @@ public class Rule {
   public static final String phraseCat = "$PHRASE";  // Sequence of tokens
   public static final String lemmaTokenCat = "$LEMMA_TOKEN";  // Lemmatized version
   public static final String lemmaPhraseCat = "$LEMMA_PHRASE";  // Lemmatized version
-
+  public static final List<String> specialCats = Lists.newArrayList(rootCat, tokenCat, phraseCat, lemmaTokenCat, lemmaPhraseCat);
   public final String lhs;  // Left-hand side: category.
   public final List<String> rhs;  // Right-hand side: sequence of categories (have $ prefix) and tokens.
   public final SemanticFn sem;  // Takes derivations corresponding to RHS categories and produces a set of derivations corresponding to LHS.
   public List<Pair<String, Double>> info;  // Extra info
+  public RuleSource source = null; // for tracking where the rule comes from when they are induced
 
   // Cache the semanticRepn
   public String getSemRepn() {
@@ -46,8 +49,13 @@ public class Rule {
 
   @Override
   public String toString() {
-    if (stringRepn == null)
-      stringRepn = lhs + " -> " + (rhs == null ? "" : Joiner.on(' ').join(rhs)) + " " + sem;
+    if (stringRepn == null) {
+      String semStr = sem == null? "NullSemanticFn" : sem.toString();
+      //int maxLength = 100;
+      //if (semStr.length() > maxLength)
+      //  semStr = String.format("%s...(%d total)", semStr.substring(0,maxLength), semStr.length());
+      stringRepn = lhs + " -> " + (rhs == null ? "" : Joiner.on(' ').join(rhs)) + " " + semStr;
+    }
     return stringRepn;
   }
   private String stringRepn;  // Cache toString()
@@ -95,11 +103,13 @@ public class Rule {
       for (Pair<String, Double> p : info)
         tree.addChild(LispTree.proto.newList(p.getFirst(), "" + p.getSecond()));
     }
+    if (source != null)
+      tree.addChild(source.toJson());
     return tree;
   }
 
   /* Extract tag info */
-  private double getInfoTag(String infoTag) {
+  public double getInfoTag(String infoTag) {
     if (info != null) {
       for (Pair<String, Double> p : info) {
         if (p.getFirst().equals(infoTag)) return p.getSecond();
@@ -129,4 +139,35 @@ public class Rule {
     else
       return f == 1.0 ? false : !FloatingParser.opts.defaultIsFloating;
   }
+
+  public boolean isInduced() {
+    double a = getInfoTag("induced");
+    if (a == 1.0) return true;
+    return false;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!(o instanceof Rule)) return false;
+    return ((Rule)o).toString().equals(this.toString());
+  }
+  @Override
+  public int hashCode() {
+    return this.toString().hashCode();
+  }
+
+  public String toJson() {
+    Map<String, Object> jsonMap = new LinkedHashMap<>();
+    jsonMap.put("lhs", lhs);
+    jsonMap.put("rhs", rhs);
+    if (source != null) {
+      jsonMap.put("source", source);
+    }
+    if (info != null) {
+      for (Pair<String, Double> p : info)
+        jsonMap.put(p.getFirst(), p.getSecond());
+    }
+    jsonMap.put("sem", sem.toString());
+    return Json.writeValueAsStringHard(jsonMap);
+  }
 }

src/edu/stanford/nlp/sempre/RuleSource.java

@@ -0,0 +1,48 @@
+package edu.stanford.nlp.sempre;
+
+import java.time.LocalDateTime;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Tracking where the rule comes from in the grammar induction process.
+ *
+ * @author sidaw
+ */
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class RuleSource {
+  @JsonProperty
+  public String uid;
+  @JsonProperty
+  public LocalDateTime time;
+  @JsonProperty
+  public String head;
+  @JsonProperty
+  public List<String> body;
+
+  @JsonProperty
+  public int cite = 0;
+  @JsonProperty
+  public int self = 0;
+  @JsonProperty
+  public boolean align = false;
+  @JsonProperty
+  public String alignInfo = "";
+
+  public RuleSource(String uid, String head, List<String> body) {
+    this.uid = uid;
+    this.head = head;
+    this.body = body;
+    this.time = LocalDateTime.now();
+  }
+
+  public String toJson() {
+    return Json.writeValueAsStringHard(this);
+  }
+
+}

src/edu/stanford/nlp/sempre/SemType.java

@@ -108,5 +108,7 @@ public abstract class SemType {
   public static final SemType entityType = new AtomicSemType(CanonicalNames.ENTITY);
   public static final SemType anyType = new AtomicSemType(CanonicalNames.ANY);
 
+  public static final FuncSemType topTopFunc = new FuncSemType(topType, topType);
   public static final FuncSemType anyAnyFunc = new FuncSemType(anyType, anyType);
+  public static final FuncSemType compareFunc = new FuncSemType(numberOrDateType, numberOrDateType);
 }

src/edu/stanford/nlp/sempre/SemanticFn.java

@@ -2,8 +2,10 @@ package edu.stanford.nlp.sempre;
 
 import fig.basic.LispTree;
 import fig.basic.Option;
+import fig.basic.Utils;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 
 /**
@@ -79,10 +81,22 @@ public abstract class SemanticFn {
   public LispTree toLispTree() { return tree; }
   @Override public String toString() { return tree.toString(); }
 
- // default does nothing
+  // default does nothing
   public void addFeedback(Example ex) { return; }
 
- // default does nothing
+  // default does nothing
   public void sortOnFeedback(Params params) { return; }
 
+  /*
+   * Filter on type data to save time.
+   * Return a collection of DerivationGroup. The rule will be applied on each DerivationGroup.
+   *
+   * See an example in tables.grow.ApplyFn
+   */
+  public boolean supportFilteringOnTypeData() { return false; }
+  public Collection<ChildDerivationsGroup> getFilteredDerivations(
+      List<Derivation> derivations1, List<Derivation> derivations2) {
+    throw new UnsupportedOperationException();
+  }
+
 }

src/edu/stanford/nlp/sempre/Session.java

@@ -2,6 +2,11 @@ package edu.stanford.nlp.sempre;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Strings;
+
+import fig.basic.Option;
 
 /**
  * A Session contains the information specific to a user.
@@ -13,11 +18,23 @@ import java.util.List;
  */
 public class Session {
   public final String id;  // Session id
-  String remoteHost;  // Where we connected from
-  String format;  // html or json
-  ContextValue context;  // Current context used to create new examples
+  public static class Options {
+    // path for default parameters, if using a different set for each session
+    @Option public String inParamsPath;
+  }
+  public String remoteHost;  // Where we connected from
+  public String format;  // html or json
+  public ContextValue context;  // Current context used to create new examples
   Example lastEx;  // Last example that we processed
-
+  
+  // if every user have their own model
+  Params params;
+  Learner learner;
+  public Map<String,String> reqParams;
+  
+  public static Options opts = new Options();
+  
+  // per session parameters
   public Session(String id) {
     this.id = id;
     context = new ContextValue(id, DateValue.now(), new ArrayList<ContextValue.Exchange>());
@@ -58,9 +75,29 @@ public class Session {
       newExchanges.add(context.exchanges.get(i));
     return context.withNewExchange(newExchanges);
   }
-
+  
+  public void useIndependentLearner(Builder builder) {
+    this.params = new Params();
+    if (!Strings.isNullOrEmpty(opts.inParamsPath))
+      this.params.read(opts.inParamsPath);
+    this.learner = new Learner(builder.parser, this.params, new Dataset());
+  }
+ 
   @Override
   public String toString() {
     return String.format("%s: %s; last: %s", id, context, lastEx);
   }
+  
+  // Decides if we write out any logs
+  public boolean isLogging() { return defaultTrue("logging");}
+  public boolean isWritingCitation() { return defaultTrue("cite");}
+  public boolean isWritingGrammar() { return defaultTrue("grammar");}
+  public boolean isLearning() { return defaultTrue("learn");}
+  public boolean isStatsing() { return defaultTrue("stats");}
+  
+  private boolean defaultTrue(String key) {
+    if (this.reqParams == null) return true;
+    if (!this.reqParams.containsKey(key)) return true;
+    return !this.reqParams.get(key).equals("0");
+  }
 }

src/edu/stanford/nlp/sempre/StringValue.java

@@ -19,6 +19,9 @@ public class StringValue extends Value {
     return tree;
   }
 
+  @Override public String sortString() { return "\"" + value + "\""; }
+  @Override public String pureString() { return value; }
+
   @Override public int hashCode() { return value.hashCode(); }
   @Override public boolean equals(Object o) {
     if (this == o) return true;

src/edu/stanford/nlp/sempre/SuperlativeFormula.java

@@ -44,6 +44,12 @@ public class SuperlativeFormula extends Formula {
     return tree;
   }
 
+  @Override
+  public void forEach(Function<Formula, Boolean> func) {
+    if (!func.apply(this)) { rank.forEach(func); count.forEach(func); head.forEach(func); relation.forEach(func); }
+  }
+
+  @Override
   public Formula map(Function<Formula, Formula> func) {
     Formula result = func.apply(this);
     return result == null ? new SuperlativeFormula(mode, rank.map(func), count.map(func), head.map(func), relation.map(func)) : result;

src/edu/stanford/nlp/sempre/TargetValuePreprocessor.java

@@ -27,10 +27,10 @@ public abstract class TargetValuePreprocessor {
   }
   public static void setSingleton(TargetValuePreprocessor processor) { singleton = processor; }
 
-  public abstract Value preprocess(Value value);
+  public abstract Value preprocess(Value value, Example ex);
 
 }
 
 class IdentityTargetValuePreprocessor extends TargetValuePreprocessor {
-  public Value preprocess(Value value) { return value; }
+  public Value preprocess(Value value, Example ex) { return value; }
 }