diff --git a/tables/README.md b/tables/README.md index 88bdb55..2197eb8 100644 --- a/tables/README.md +++ b/tables/README.md @@ -21,13 +21,13 @@ Running the code ./pull-dependencies tables ./pull-dependencies tables-data - The dataset lives in `lib/data/tables/` + The dataset lives in `lib/data/WikiTableQuestions/` 2. Compile the source: ant tables - This will produce JAR files in the `libsempre` directory as usual. + This will produce JAR files in the `libsempre` directory. 3. The following command train and test on 100 development examples: @@ -41,6 +41,27 @@ Running the code Other available sets include `u-2`, ..., `u-5` (four other development splits) and `test` (actual train-test split). +Official evaluation +------------------- + +The official evaluation script in the WikiTableQuestions dataset is slightly +more lenient than the SEMPRE one (`tables.TableValueEvaluator`). +In particular, the SEMPRE evaluator enforces that the type of the predicted +denotation must match the correct answer type, while the official one allows +type conversion. + +To get the official number of a trained model, run + + ./run @mode=tables @data=u-1 @feat=all @train=0 -Derivation.showValues -Builder.inParamsPath path/to/params + +(Change the `@data` other options to match the ones used during training.) + +This should produce an execution directory (in `state/execs/` by default) +with a log file in it. Then run + + ./tables/log-parsers/get-predictions.py path/to/log > predictions + ./lib/data/WikiTableQuestions/evaluator.py -t ./lib/data/WikiTableQuestions/tagged-data predictions + Other usages ------------ diff --git a/tables/log-parsers/get-predictions.py b/tables/log-parsers/get-predictions.py new file mode 100755 index 0000000..6b482fe --- /dev/null +++ b/tables/log-parsers/get-predictions.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Get predictions from the log file of SEMPRE.""" + +import sys, os, shutil, re, argparse + +PATTERN = re.compile(r'Pred@0000: ' + r'\(derivation \(formula (.*)\)\) ' + r'\(value (.*)\) ' + r'\(type (.*)\)\) \[score=(.*), prob=(.*), comp=(.*)\]') + +def lisptree_to_python_object(charbuffer): + """Convert the lisptree to Python object. + + Args: + charbuffer: REVERSED list of characters of the lisptree string. + Characters will be consumed from the list. + """ + c = charbuffer.pop() + if c == '(': + answer = [] + while charbuffer[-1] != ')': + if charbuffer[-1] == ' ': + charbuffer.pop() + else: + answer.append(lisptree_to_python_object(charbuffer)) + assert charbuffer.pop() == ')' + return answer + elif c == '"': + answer = [] + while charbuffer[-1] != '"': + c = charbuffer.pop() + if c == '\\': + answer.append(charbuffer.pop()) + else: + answer.append(c) + assert charbuffer.pop() == '"' + return ''.join(answer) + else: + answer = [c if c != '\\' else charbuffer.pop()] + while charbuffer[-1] not in (' ', ')'): + c = charbuffer.pop() + if c == '\\': + answer.append(charbuffer.pop()) + else: + assert c != '(' + answer.append(c) + return ''.join(answer) + +def lisptree_to_values(tree): + assert tree.startswith('(list ') and tree.endswith(')') + tree = lisptree_to_python_object(list(tree.decode('utf8'))[::-1]) + assert tree[0] == 'list' + answer = [] + for subtree in tree[1:]: + if subtree[0] == 'number': + answer.append(float(subtree[1])) + elif subtree[0] == 'date': + answer.append('{}-{}-{}'.format( + int(subtree[1]) if subtree[1] != '-1' else 'xx', + int(subtree[2]) if subtree[2] != '-1' else 'xx', + int(subtree[3]) if subtree[3] != '-1' else 'xx')) + else: + assert subtree[0] == 'name' + answer.append(re.sub('\s+', ' ', subtree[2]).strip()) + return '\t'.join(unicode(x) for x in answer) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('infile', help='log file') + parser.add_argument('iteration', help='iteration to extract') + args = parser.parse_args() + + prefix = 'iter=%s:' % args.iteration + ex_id = None + with open(args.infile) as fin: + for line in fin: + line = line.strip() + if line.startswith(prefix): + if ex_id is not None: + # No prediction for the previous example + print ex_id + ex_id = line.split()[3] + elif ex_id is not None and line.startswith('Pred@0000:'): + match = PATTERN.match(line) + formula, denotation, deno_type, score, prob, comp = match.groups() + denotation = lisptree_to_values(denotation) + print u'{}\t{}'.format(ex_id, denotation) + ex_id = None + if ex_id is not None: + print '\t'.join([ex_id, 'None']) + +if __name__ == '__main__': + main() +