mirror of https://github.com/percyliang/sempre
Added more files related to tables experiments
This commit is contained in:
Panupong Pasupat
parent
2071579b8c
commit
53d02315ab
143
run
143
run
|
|
@ -618,58 +618,115 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
|
|||
letDefault(:class, 'main'),
|
||||
sel(:class, {
|
||||
'main' => 'edu.stanford.nlp.sempre.Main',
|
||||
'check' => 'edu.stanford.nlp.sempre.tables.test.DPParserChecker',
|
||||
'align' => 'edu.stanford.nlp.sempre.tables.alignment.IBMAligner',
|
||||
'check' => 'edu.stanford.nlp.sempre.tables.test.DPDParserChecker',
|
||||
'align' => l('edu.stanford.nlp.sempre.tables.alignment.AlignerMain', o('maxTrainIters', 11)),
|
||||
'dump' => 'edu.stanford.nlp.sempre.tables.serialize.SerializedDumper',
|
||||
'load' => l('edu.stanford.nlp.sempre.tables.serialize.SerializedLoader', let(:parser, 'dummy')),
|
||||
'load' => l('edu.stanford.nlp.sempre.tables.serialize.SerializedLoader', let(:parser, 'serialized')),
|
||||
'stats' => 'edu.stanford.nlp.sempre.tables.test.TableStatsComputer',
|
||||
'ann-data' => 'edu.stanford.nlp.sempre.tables.serialize.AnnotatedDatasetGenerator',
|
||||
'ann-table' => 'edu.stanford.nlp.sempre.tables.serialize.AnnotatedTableGenerator',
|
||||
'alter' => l('edu.stanford.nlp.sempre.tables.alter.BatchTableAlterer', let(:parser, 'serialized')),
|
||||
'alter-ex' => l('edu.stanford.nlp.sempre.tables.alter.AlteredTablesExecutor', let(:parser, 'serialized')),
|
||||
}),
|
||||
# Fig parameters
|
||||
selo(:cldir, 'execDir', '_OUTPATH_', '.'),
|
||||
o('overwriteExecDir'), o('addToView', 0), o('jarFiles', 'libsempre/*'),
|
||||
selo(:cldir, 'execDir', '_OUTPATH_', 'exec'),
|
||||
o('overwriteExecDir'), o('addToView', 11), o('jarFiles', 'libsempre/*'),
|
||||
sel(:cldir, l(), '>/dev/null'),
|
||||
# Set environment for table execution
|
||||
o('executor', 'tables.lambdadcs.LambdaDCSExecutor'),
|
||||
o('Builder.valueEvaluator', 'tables.TableValueEvaluator'),
|
||||
o('TargetValuePreprocessor.targetValuePreprocessor', 'tables.TableValuePreprocessor'),
|
||||
o('targetValuePreprocessor', 'tables.TableValuePreprocessor'),
|
||||
o('NumberFn.unitless'), o('NumberFn.alsoTestByConversion'),
|
||||
o('TypeInference.typeLookup', 'tables.TableTypeLookup'),
|
||||
o('JoinFn.specializedTypeCheck', false), o('JoinFn.typeInference', true),
|
||||
# Value Evaluator
|
||||
letDefault(:eval, 'value'),
|
||||
sel(:eval, {
|
||||
'value' => o('Builder.valueEvaluator', 'tables.TableValueEvaluator'),
|
||||
'denotation' => o('Builder.valueEvaluator', 'tables.TableValueEvaluator'), # alias of 'value'
|
||||
'formula' => l(
|
||||
o('Builder.valueEvaluator', 'tables.TableFormulaEvaluator'),
|
||||
o('fallBackToValueEvaluator', false),
|
||||
nil),
|
||||
}),
|
||||
# Parser
|
||||
letDefault(:parser, 'floatsize'),
|
||||
sel(:parser, {
|
||||
'floatsize' => l(
|
||||
'old-floatsize' => l(
|
||||
o('Builder.parser', 'FloatingParser'),
|
||||
o('useSizeInsteadOfDepth'),
|
||||
o('FloatingParser.maxDepth', 15),
|
||||
nil),
|
||||
'baseline' => l(
|
||||
'old-baseline' => l(
|
||||
o('Builder.parser', 'tables.baseline.TableBaselineParser'),
|
||||
nil),
|
||||
'dummy' => o('Builder.parser', 'tables.serialize.DummyParser'),
|
||||
'serialized' => o('Builder.parser', 'tables.serialize.SerializedParser'),
|
||||
# ACL 2016
|
||||
'grow-dp' => l(
|
||||
o('Builder.parser', 'tables.dpd.DPDParser'),
|
||||
o('useSizeInsteadOfDepth'),
|
||||
o('FloatingParser.maxDepth', 8),
|
||||
nil),
|
||||
'grow-float' => l(
|
||||
o('Builder.parser', 'FloatingParser'),
|
||||
o('useSizeInsteadOfDepth'),
|
||||
o('FloatingParser.maxDepth', 8),
|
||||
o('FloatingParser.betaReduce'), o('initialFloatingHasZeroDepth'),
|
||||
nil),
|
||||
'grow-mix' => l(
|
||||
o('Builder.parser', 'MixParser'),
|
||||
o('MixParser.parsers', 'FloatingParser', 'tables.serialize.SerializedParser:train-0xc'),
|
||||
o('useSizeInsteadOfDepth'),
|
||||
o('FloatingParser.maxDepth', 8),
|
||||
o('FloatingParser.betaReduce'), o('initialFloatingHasZeroDepth'),
|
||||
nil),
|
||||
}),
|
||||
o('FloatingParser.useAnchorsOnce', true),
|
||||
o('Parser.verbose', 1),
|
||||
letDefault(:pruning, 1),
|
||||
sel(:pruning,
|
||||
l(),
|
||||
l(
|
||||
o('DerivationPruner.pruningStrategies', *tablesPruningStrategies),
|
||||
o('DerivationPruner.pruningComputers', 'tables.TableDerivationPruningComputer'),
|
||||
o('recursivePruning', false),
|
||||
nil),
|
||||
nil),
|
||||
# Grammar
|
||||
tablesGrammarPaths,
|
||||
letDefault(:fuzzy, 1),
|
||||
sel(:fuzzy,
|
||||
l(),
|
||||
l(o('fuzzyMatchSubstring'), o('fuzzyMatchMaxEditDistanceRatio', 0.3), o('alsoReturnUnion'), o('alsoMatchPart')),
|
||||
nil),
|
||||
letDefault(:normalize, 1),
|
||||
sel(:normalize,
|
||||
l(),
|
||||
l(o('genericDateValue'), o('numberCanStartAnywhere'), o('num2CanStartAnywhere'), o('NumberFn.alsoTestByIsolatedNER')),
|
||||
nil),
|
||||
letDefault(:anchor, 1),
|
||||
sel(:anchor, {
|
||||
1 => o('FloatingParser.useAnchorsOnce', true),
|
||||
2 => l(o('FloatingParser.useAnchorsOnce', false), o('FloatingParser.useMaxAnchors', 2)),
|
||||
}),
|
||||
# Dataset
|
||||
letDefault(:data, 'none'),
|
||||
letDefault(:unseen, 0),
|
||||
tablesDataPaths,
|
||||
tableDataPaths,
|
||||
# Verbosity
|
||||
o('FeatureVector.ignoreZeroWeight'),
|
||||
o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10), o('logFeaturesLimit', 10),
|
||||
letDefault(:verbose, 0),
|
||||
o('LambdaDCSException.noErrorMessage'),
|
||||
letDefault(:verbose, 2),
|
||||
sel(:verbose,
|
||||
l(),
|
||||
l(
|
||||
o('putCellNameInCanonicalUtterance'), o('showUtterance'),
|
||||
nil),
|
||||
l(
|
||||
o('putCellNameInCanonicalUtterance'), o('showUtterance'),
|
||||
o('summarizeRuleTime'), o('summarizeDenotations'),
|
||||
nil),
|
||||
l(
|
||||
o('putCellNameInCanonicalUtterance'), o('showUtterance'),
|
||||
o('summarizeRuleTime'), o('summarizeDenotations'),
|
||||
o('showRules'),
|
||||
o('Parser.verbose', 2),
|
||||
o('JoinFn.verbose', 3),
|
||||
|
|
@ -686,25 +743,25 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
|
|||
# Training
|
||||
letDefault(:train, 0),
|
||||
sel(:train,
|
||||
l(),
|
||||
l(
|
||||
let(:l1, 0),
|
||||
nil),
|
||||
l(
|
||||
o('combineFromFloatingParser'),
|
||||
sel(:unseen, unbalancedTrainDevSplit, l()),
|
||||
o('maxTrainIters', 3),
|
||||
o('showValues', false), o('showFirstValue'),
|
||||
nil),
|
||||
l(
|
||||
# for dumping derivations (@class=dump)
|
||||
# force unbalancedTrainDevSplit + combine from floating parser
|
||||
o('combineFromFloatingParser'), o('DPParser.cheat'),
|
||||
sel(:unseen, unbalancedTrainDevSplit, l()),
|
||||
o('combineFromFloatingParser'), o('DPDParser.cheat'),
|
||||
nil),
|
||||
nil),
|
||||
# Regularization
|
||||
letDefault(:l1, 1),
|
||||
sel(:l1,
|
||||
l(),
|
||||
l(o('Params.l1Reg','lazy'), o('Params.l1RegCoeff', '3e-5')),
|
||||
l(o('Params.l1Reg','lazy'), o('Params.l1RegCoeff', '3e-5')), # Default
|
||||
l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0, 0.00001, 0.0001, 0.001, 0.01)),
|
||||
l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0.00001, 0.00003, 0.0001, 0.0003)),
|
||||
l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0.00001, 0.00003, 0.0005)),
|
||||
|
|
@ -720,6 +777,10 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
|
|||
o('FeatureExtractor.featureDomains', 'custom-denotation phrase-predicate phrase-denotation headword-denotation missing-predicate'.split),
|
||||
o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
|
||||
nil),
|
||||
'all-fragment' => l(
|
||||
o('FeatureExtractor.featureDomains', 'custom-denotation phrase-denotation headword-denotation phrase-fragment'.split),
|
||||
o('FeatureExtractor.featureComputers', 'tables.features.PhraseDenotationFeatureComputer tables.features.PhraseFragmentFeatureComputer'.split),
|
||||
nil),
|
||||
'baseline' => l( # For the baseline classifier
|
||||
o('FeatureExtractor.featureDomains', 'custom-denotation phrase-denotation headword-denotation table-baseline'.split),
|
||||
o('FeatureExtractor.featureComputers', 'tables.baseline.TableBaselineFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
|
||||
|
|
@ -741,7 +802,6 @@ addMode('tables', 'QA on HTML tables', lambda { |e| l(
|
|||
'none' => l(),
|
||||
'careful' => l(
|
||||
o('maxNforLexicalizeAllPairs', 2),
|
||||
o('traverseWithFormulaTypes'), o('reverseNameValueConversion', 'allBang'),
|
||||
o('lookUnderCellProperty'), o('useGenericCellType'),
|
||||
o('computeFuzzyMatchPredicates'),
|
||||
nil),
|
||||
|
|
@ -754,6 +814,7 @@ def tablesGrammarPaths
|
|||
l(
|
||||
letDefault(:grammar, 'combined-all'),
|
||||
sel(:grammar, {
|
||||
'custom' => l(),
|
||||
'restrict' => o('Grammar.inPaths', "#{baseDir}restrict.grammar"),
|
||||
'simple' => o('Grammar.inPaths', "#{baseDir}simple.grammar"),
|
||||
'combined' => o('Grammar.inPaths', "#{baseDir}combined.grammar"),
|
||||
|
|
@ -769,10 +830,25 @@ def tablesGrammarPaths
|
|||
o('Grammar.inPaths', "#{baseDir}combined.grammar"),
|
||||
o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge'.split),
|
||||
nil),
|
||||
'combined-more' => l(
|
||||
o('Grammar.inPaths', "#{baseDir}combined.grammar"),
|
||||
o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge u-superlative neq'.split),
|
||||
nil),
|
||||
'combined-trigger' => l( # Use trigger words for operations
|
||||
o('Grammar.inPaths', "#{baseDir}combined.grammar"),
|
||||
o('Grammar.tags', *'t-alternative t-movement t-comparison t-count t-aggregate t-superlative t-arithmetic merge'.split),
|
||||
nil),
|
||||
# ACL 2016
|
||||
'grow-custom' => l(
|
||||
o('Grammar.inPaths', "#{baseDir}grow.grammar"),
|
||||
o('Grammar.binarizeRules', false),
|
||||
nil),
|
||||
'grow-default' => l(
|
||||
o('Grammar.inPaths', "#{baseDir}grow.grammar"),
|
||||
o('Grammar.binarizeRules', false),
|
||||
o('Grammar.tags', *'scoped merge-and arithmetic comparison alternative neq yearrange part closedclass scoped-2args-merge-and'.split),
|
||||
let(:anchor, 2),
|
||||
nil),
|
||||
}),
|
||||
nil)
|
||||
}
|
||||
|
|
@ -790,21 +866,20 @@ def tablesDataPaths
|
|||
"train,#{baseDir}training.examples",
|
||||
"test,#{baseDir}pristine-unseen-tables.examples"),
|
||||
o('Dataset.trainFrac', 1), o('Dataset.devFrac', 0),
|
||||
let(:unseen, 1),
|
||||
nil),
|
||||
}
|
||||
# Development sets: 80:20 random split of training data
|
||||
['1', '2', '3', '4', '5'].each do |x|
|
||||
datasets['u-' + x] = l(
|
||||
o('Dataset.inPaths',
|
||||
"train,#{baseDir}random-split-seed-#{x}-train.examples",
|
||||
"dev,#{baseDir}random-split-seed-#{x}-test.examples",
|
||||
"train,#{baseDir}random-split-#{x}-train.examples",
|
||||
"dev,#{baseDir}random-split-#{x}-dev.examples",
|
||||
nil),
|
||||
let(:unseen, 1),
|
||||
nil)
|
||||
end
|
||||
# That's it!
|
||||
l(
|
||||
o('splitDevFromTrain', false),
|
||||
o('TableKnowledgeGraph.baseCSVDir', csvDir),
|
||||
sel(:data, datasets),
|
||||
nil)
|
||||
|
|
@ -813,23 +888,25 @@ end
|
|||
|
||||
def tablesPruningStrategies
|
||||
[
|
||||
# Formula
|
||||
"atomic",
|
||||
"multipleSuperlatives",
|
||||
"sameMerge",
|
||||
"forwardBackward",
|
||||
"doubleNext",
|
||||
# Denotation
|
||||
### Critical strategies
|
||||
"emptyDenotation",
|
||||
"nonLambdaError",
|
||||
### Strategies that do not depend on the children's actual formulas
|
||||
"singleton",
|
||||
"tooManyValues",
|
||||
"badSuperlativeHead",
|
||||
"badSummarizerHead",
|
||||
"mistypedMerge",
|
||||
### Strategies that depend on the children's formulas
|
||||
"doubleNext",
|
||||
"doubleSummarizers",
|
||||
"sameMerge",
|
||||
"unsortedMerge",
|
||||
"typeRowMerge",
|
||||
nil].compact
|
||||
end
|
||||
|
||||
|
||||
|
||||
############################################################
|
||||
# {2015-01-18} Generate utterances [Percy]
|
||||
addMode('genovernight', 'Generate utterances for overnight semantic parsing', lambda { |e| l(
|
||||
'fig/bin/qcreate',
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
../../lib/data/tables/compiled/
|
||||
|
|
@ -1 +0,0 @@
|
|||
../../lib/data/tables/csv/
|
||||
|
|
@ -1 +0,0 @@
|
|||
../../state/execs/
|
||||
|
|
@ -1,14 +1,15 @@
|
|||
// Codalab schema for SEMPRE runs
|
||||
% schema sempre
|
||||
% add uuid uuid [0:8]
|
||||
% add name
|
||||
% add state state
|
||||
% add date created date
|
||||
% add host remote s/\..*/
|
||||
% add host remote s/[.].*/
|
||||
% add disk data_size size
|
||||
% add mem memory size
|
||||
% add time time duration
|
||||
% add |
|
||||
% add data /options.map:Dataset.inPaths "s/WikiTableQuestions\/compiled\// | s/\.examples/ | s/unnormalized/U | s/normalized/N | s/unseen-(train|dev)-seed/u"
|
||||
% add data /options.map:Dataset.inPaths "s/WikiTableQuestions\/compiled\// | s/\.examples/ | s/unnormalized/U | s/normalized/N | s/unseen-(train|dev)-seed/u | s/random-split/u"
|
||||
% add maxex /options.map:Dataset.maxExamples
|
||||
% add parser /options.map:Builder.parser
|
||||
% add beam /options.map:Parser.beamSize %d
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Convert LispTree formulas to Postfix.
|
||||
|
||||
If the file is tab-separated, only process the first column.
|
||||
"""
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
import fileinput
|
||||
import lisptree
|
||||
|
||||
NUMBER_PREFIX = 'N'
|
||||
DATE_PREFIX = 'D'
|
||||
TYPE_ROW = 'type-row'
|
||||
AGGREGATE = ['count', 'min', 'max', 'sum', 'avg']
|
||||
MERGE_ARITH = ['and', 'or', '-']
|
||||
SUPERLATIVE = ['argmin', 'argmax']
|
||||
|
||||
def convert(tree, args):
|
||||
answer = []
|
||||
u_stack = []
|
||||
def recurse(subtree):
|
||||
if isinstance(subtree, basestring):
|
||||
answer.append(subtree)
|
||||
return
|
||||
if isinstance(subtree[0], basestring):
|
||||
opr = subtree[0]
|
||||
if opr in AGGREGATE:
|
||||
assert len(subtree) == 2, str(subtree)
|
||||
recurse(subtree[1])
|
||||
answer.append(opr)
|
||||
elif opr in MERGE_ARITH:
|
||||
assert len(subtree) == 3, str(subtree)
|
||||
recurse(subtree[1])
|
||||
recurse(subtree[2])
|
||||
answer.append(opr)
|
||||
elif opr in SUPERLATIVE:
|
||||
assert len(subtree) in (3, 5), str(subtree)
|
||||
if len(subtree) == 3:
|
||||
u, b = subtree[1], subtree[2]
|
||||
else:
|
||||
u, b = subtree[3], subtree[4]
|
||||
if args.implicit_superlative_lambda:
|
||||
assert b[0] == 'reverse'
|
||||
assert b[1][0] == 'lambda'
|
||||
u_stack.append(convert(u, args))
|
||||
recurse(b[1][2])
|
||||
answer.append(opr)
|
||||
u_stack.pop()
|
||||
else:
|
||||
recurse(u)
|
||||
recurse(b)
|
||||
answer.append(opr)
|
||||
elif opr == 'lambda':
|
||||
assert len(subtree) == 3, str(subtree)
|
||||
recurse(subtree[2])
|
||||
answer.append(opr)
|
||||
elif opr == 'reverse':
|
||||
assert len(subtree) == 2, str(subtree)
|
||||
recurse(subtree[1])
|
||||
answer.append(opr)
|
||||
elif opr == 'var':
|
||||
assert len(subtree) == 2, str(subtree)
|
||||
if args.implicit_superlative_lambda:
|
||||
answer.extend(u_stack[-1])
|
||||
answer.append(subtree[1])
|
||||
elif opr == 'number':
|
||||
assert len(subtree) == 2, str(subtree)
|
||||
answer.append(NUMBER_PREFIX + subtree[1])
|
||||
elif opr == 'date':
|
||||
assert len(subtree) == 4, str(subtree)
|
||||
answer.append(DATE_PREFIX + '-'.join(
|
||||
'XX' if x == '-1' else x for x in subtree[1:4]))
|
||||
else: # Join with a name
|
||||
assert len(subtree) == 2, str(subtree)
|
||||
if (args.collapse_type_row and
|
||||
'fb:type.object.type' == opr and
|
||||
'fb:type.row' == subtree[1]):
|
||||
answer.append(TYPE_ROW)
|
||||
else:
|
||||
recurse(subtree[1])
|
||||
answer.append(opr)
|
||||
if not args.implicit_join:
|
||||
answer.append('.')
|
||||
else: # Join with a complex construct
|
||||
assert len(subtree) == 2, str(subtree)
|
||||
if args.implicit_join:
|
||||
# Only allows ((reverse ...) ...)
|
||||
assert subtree[0][0] == 'reverse', str(subtree)
|
||||
assert len(subtree[0]) == 2, str(subtree)
|
||||
recurse(subtree[1])
|
||||
answer.append('!' + subtree[0][1])
|
||||
else:
|
||||
recurse(subtree[1])
|
||||
recurse(subtree[0])
|
||||
answer.append('.')
|
||||
recurse(tree)
|
||||
return answer
|
||||
|
||||
def process(line, args):
|
||||
line = line[:-1].split('\t')
|
||||
print '\t'.join([' '.join(convert(lisptree.parse(line[0]), args))] + line[1:])
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-j', '--implicit-join', action='store_true',
|
||||
help='Do not output "." for joins')
|
||||
parser.add_argument('-s', '--implicit-superlative-lambda', action='store_true',
|
||||
help='Do not output "lambda reverse" for superlatives')
|
||||
parser.add_argument('-t', '--collapse-type-row', action='store_true',
|
||||
help='Collapse "(type row)" into a single token')
|
||||
parser.add_argument('infile')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.infile != '-':
|
||||
opener = gzip.open if args.infile.endswith('.gz') else open
|
||||
with opener(args.infile, 'r', 'utf8') as fin:
|
||||
for line in fin:
|
||||
process(line, args)
|
||||
else:
|
||||
for line in sys.stdin:
|
||||
process(line, args)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Count the size of each example in the file.
|
||||
"""
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
import glob, gzip
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-m', '--max-size', default=8)
|
||||
parser.add_argument('indirs', nargs='+')
|
||||
args = parser.parse_args()
|
||||
|
||||
for i, indir in enumerate(sorted(args.indirs)):
|
||||
print >> sys.stderr, 'INDIR {:2}: {}'.format(i, indir)
|
||||
|
||||
filenames = []
|
||||
for indir in args.indirs:
|
||||
filenames.extend(glob.glob(os.path.join(indir, '*.gz')))
|
||||
print >> sys.stderr, len(filenames), 'files.'
|
||||
|
||||
count = 0
|
||||
for filename in sorted(filenames):
|
||||
print >> sys.stderr, 'Processing', filename
|
||||
with gzip.open(filename) as fin:
|
||||
for line in fin:
|
||||
if line.startswith('#'):
|
||||
tokens = line.strip().split()
|
||||
assert tokens[1] == 'Example'
|
||||
ex_id = tokens[2]
|
||||
sizes = defaultdict(int)
|
||||
count += 1
|
||||
elif line.startswith(')'):
|
||||
print '\t'.join([ex_id] + [str(sizes[i]) for i in xrange(args.max_size + 1)])
|
||||
else:
|
||||
match = re.search(r'\$ROOT:(\d+)', line)
|
||||
if match:
|
||||
sizes[int(match.group(1))] += 1
|
||||
print >> sys.stderr, 'Processed', count, 'examples'
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Convert derivation LispTree to TSV.
|
||||
|
||||
Each input line has the following format:
|
||||
(derivation
|
||||
(formula ...)
|
||||
(value ...)
|
||||
(type ...)
|
||||
(canonicalUtterance ...)
|
||||
)
|
||||
|
||||
Currently prints out 'formula <tab> size'
|
||||
"""
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
import gzip, fileinput
|
||||
import lisptree
|
||||
|
||||
def process(line):
|
||||
line = line.strip()
|
||||
if line.startswith('Processing'):
|
||||
print '#' * 20, line[:-1], '#' * 20
|
||||
if not line.startswith('(derivation '):
|
||||
return
|
||||
# Just find matching parentheses
|
||||
# Wow this is an awful hack
|
||||
line = re.sub(r'^\(derivation \(formula ', '', line)
|
||||
num_open = 0
|
||||
stuff = []
|
||||
for char in line:
|
||||
stuff.append(char)
|
||||
if char == '(':
|
||||
num_open += 1
|
||||
elif char == ')':
|
||||
num_open -= 1
|
||||
if not num_open:
|
||||
break
|
||||
formula = ''.join(stuff)
|
||||
size = int(re.search(r'\$ROOT:(\d+)', line).group(1))
|
||||
print '\t'.join(str(x) for x in [formula, size])
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('infile')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.infile != '-':
|
||||
opener = gzip.open if args.infile.endswith('.gz') else open
|
||||
with opener(args.infile, 'r', 'utf8') as fin:
|
||||
for line in fin:
|
||||
process(line)
|
||||
else:
|
||||
for line in sys.stdin:
|
||||
process(line)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Convert dump gzip file to training and test files.
|
||||
- Training file does not contain the contexts
|
||||
- Test file has only the first 200 examples.
|
||||
|
||||
The parser is super specific to the dump produced by
|
||||
./run @mode=tables @class=dump
|
||||
"""
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
import gzip
|
||||
|
||||
def process(fin, fout_train, fout_test):
|
||||
# First line
|
||||
meta = fin.readline()
|
||||
fout_train.write(meta)
|
||||
fout_test.write("(metadata (group test) (size 200))\n")
|
||||
# The rest
|
||||
count = 0
|
||||
for line in fin:
|
||||
if line[0] == '#':
|
||||
count += 1
|
||||
if count % 500 == 0:
|
||||
print >> sys.stderr, 'Processing', count, '...'
|
||||
if not line.startswith(' (context'):
|
||||
fout_train.write(line)
|
||||
if count <= 200:
|
||||
fout_test.write(line)
|
||||
print >> sys.stderr, 'Done processing', count, 'examples.'
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('infile')
|
||||
args = parser.parse_args()
|
||||
|
||||
infile = args.infile
|
||||
assert infile.endswith('.gz')
|
||||
out_train = re.sub(r'\.gz$', '-train.gz', infile)
|
||||
out_test = re.sub(r'\.gz$', '-test.gz', infile)
|
||||
|
||||
with gzip.open(args.infile) as fin:
|
||||
with gzip.open(out_train, 'w') as fout_train:
|
||||
with gzip.open(out_test, 'w') as fout_test:
|
||||
process(fin, fout_train, fout_test)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""LispTree utilities."""
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
def parse(x):
|
||||
"""Parse a LispTree string into a (nested) Python list."""
|
||||
stack = [[]]
|
||||
buff = []
|
||||
in_quote = False
|
||||
is_escape = False
|
||||
for c in x:
|
||||
if in_quote:
|
||||
if is_escape:
|
||||
is_escape = False
|
||||
buff.append(c)
|
||||
else:
|
||||
if c == '"':
|
||||
in_quote = False
|
||||
stack[-1].append(''.join(buff))
|
||||
buff = None
|
||||
elif c == '\\':
|
||||
is_escape = True
|
||||
else:
|
||||
buff.append(c)
|
||||
else:
|
||||
if c == '(':
|
||||
assert buff == []
|
||||
stack.append([])
|
||||
elif c == '"':
|
||||
assert buff == []
|
||||
in_quote = True
|
||||
elif c == ')':
|
||||
if buff:
|
||||
stack[-1].append(''.join(buff))
|
||||
buff = []
|
||||
last = stack.pop()
|
||||
stack[-1].append(last)
|
||||
elif c in ' \n\t':
|
||||
if buff:
|
||||
stack[-1].append(''.join(buff))
|
||||
buff = []
|
||||
else:
|
||||
assert buff is not None
|
||||
buff.append(c)
|
||||
assert len(stack) == 1 and len(stack[0]) == 1
|
||||
return stack[0][0]
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Tests
|
||||
a = '(derivation (formula (max ((reverse fb:cell.cell.date) ((reverse fb:row.row.year) (and (fb:row.row.league fb:cell.usl_a_league) (fb:row.row.playoffs (!= fb:cell.usl_a_league))))))) (value (list (date 2004 -1 -1))) (type (union fb:type.number fb:type.datetime)) (canonicalUtterance $ROOT:8))'
|
||||
print a
|
||||
print parse(a)
|
||||
a = r'(derivation (formula ((reverse fb:row.row.venue) (argmax (number 1) (number 1) (fb:row.row.venue ((reverse fb:row.row.venue) (fb:row.row.position fb:cell.1st))) (reverse (lambda x ((reverse fb:row.row.index) (var x))))))) (value (list (name fb:cell.bangkok_thailand "Bangkok, \"Thai\"land"))) (type fb:type.cell) (canonicalUtterance $ROOT:8))'
|
||||
print a
|
||||
print parse(a)
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Take in lines of derivations
|
||||
and print the abbreviated formulas."""
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
import gzip
|
||||
|
||||
def process(line):
|
||||
line = line.strip()
|
||||
if line.startswith('Processing'):
|
||||
print '#' * 20, line[:-1], '#' * 20
|
||||
if not line.startswith('(derivation '):
|
||||
return
|
||||
# Extract only formula
|
||||
line = re.sub(r'^\(derivation \(formula ', '', line)
|
||||
num_open = 0
|
||||
stuff = []
|
||||
for char in line:
|
||||
stuff.append(char)
|
||||
if char == '(':
|
||||
num_open += 1
|
||||
elif char == ')':
|
||||
num_open -= 1
|
||||
if not num_open:
|
||||
break
|
||||
line = ''.join(stuff)
|
||||
# Abbreviate stuff
|
||||
line = re.sub(r'\(reverse ([^()]*)\)', r'!\1', line)
|
||||
line = re.sub(r'\(number ([^()]*)\)', r'\1', line)
|
||||
line = line.replace('fb:row.row.next', '@next')
|
||||
line = line.replace('fb:row.row.index', '@index')
|
||||
line = line.replace('fb:type.object.type', '@type')
|
||||
line = line.replace('fb:type.row', '@row')
|
||||
line = line.replace('fb:row.row.', 'r.')
|
||||
line = line.replace('fb:cell.cell.', '@p.')
|
||||
line = line.replace('@p.number', '@p.num')
|
||||
line = line.replace('fb:cell.', 'c.')
|
||||
print line
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('infile')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.infile != '-':
|
||||
opener = gzip.open if args.infile.endswith('.gz') else open
|
||||
with opener(args.infile, 'r', 'utf8') as fin:
|
||||
for line in fin:
|
||||
process(line)
|
||||
else:
|
||||
for line in sys.stdin:
|
||||
process(line)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Split gzip files generated from SerializedDumper
|
||||
into multiple files, each with only a single example.
|
||||
|
||||
The filenames are the example IDs.
|
||||
"""
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
import glob, gzip
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('destdir')
|
||||
parser.add_argument('indirs', nargs='+')
|
||||
args = parser.parse_args()
|
||||
|
||||
for i, indir in enumerate(sorted(args.indirs)):
|
||||
print >> sys.stderr, 'INDIR {:2}: {}'.format(i, indir)
|
||||
|
||||
if os.path.exists(args.destdir):
|
||||
if raw_input('Delete {} directory? (y/N): '.format(args.destdir))[0:].lower() != 'y':
|
||||
return
|
||||
shutil.rmtree(args.destdir)
|
||||
os.makedirs(args.destdir)
|
||||
|
||||
filenames = []
|
||||
for indir in args.indirs:
|
||||
filenames.extend(glob.glob(os.path.join(indir, '*.gz')))
|
||||
print >> sys.stderr, len(filenames), 'files.'
|
||||
|
||||
count = 0
|
||||
for filename in sorted(filenames):
|
||||
print >> sys.stderr, 'Processing', filename
|
||||
with gzip.open(filename) as fin:
|
||||
fout = None
|
||||
metadata = fin.readline()
|
||||
group = re.search(r'\(group ([^)]*)\)', metadata).group(1)
|
||||
for line in fin:
|
||||
if line.startswith('#'):
|
||||
tokens = line.strip().split()
|
||||
assert tokens[1] == 'Example'
|
||||
ex_id = tokens[2]
|
||||
if fout:
|
||||
fout.close()
|
||||
new_filename = 'dumped-single_{}-{:06d}-{}.gz'.format(group, count, ex_id)
|
||||
fout = gzip.open(os.path.join(args.destdir, new_filename), 'wb')
|
||||
print >> fout, '(metadata (group {}) (offset {}) (size 1))'.format(group, count)
|
||||
count += 1
|
||||
if fout:
|
||||
fout.write(line)
|
||||
if fout:
|
||||
fout.close()
|
||||
print >> sys.stderr, 'Written', count, 'files'
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Write a simple sequence-to-sequence dataset
|
||||
in the default format of nn-semparse.
|
||||
|
||||
- Only the shortest formula is retained.
|
||||
- Optionally, example ID is given as the first column
|
||||
"""
|
||||
|
||||
import sys, os, shutil, re, argparse, json, gzip, random
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
def get_formula(fpost):
|
||||
formulas = []
|
||||
for line in fpost:
|
||||
formula, size = line[:-1].split('\t')
|
||||
formulas.append([formula, int(size)])
|
||||
if not formulas:
|
||||
return None
|
||||
# Choose a formula
|
||||
best_size = min(x[1] for x in formulas)
|
||||
best_formulas = [x[0] for x in formulas if x[1] == best_size]
|
||||
return random.choice(best_formulas)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('annotated_path',
|
||||
help='paths to annotated TSV')
|
||||
parser.add_argument('dataset_path',
|
||||
help='path to dataset TSV')
|
||||
parser.add_argument('postfix_dir',
|
||||
help='path to postfix directory')
|
||||
parser.add_argument('-e', '--ex-id', action='store_true',
|
||||
help='add example ID as the first column')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Read annotated data
|
||||
annotated = {}
|
||||
with open(args.annotated_path, 'r', 'utf8') as fin:
|
||||
header = fin.readline()[:-1].split('\t')
|
||||
for line in fin:
|
||||
record = dict(zip(header, line[:-1].split('\t')))
|
||||
annotated[record['id']] = record
|
||||
|
||||
# Read dataset in order
|
||||
count = 0
|
||||
with open(args.dataset_path, 'r', 'utf8') as fin:
|
||||
header = fin.readline()[:-1].split('\t')
|
||||
for line in fin:
|
||||
record = dict(zip(header, line[:-1].split('\t')))
|
||||
record = annotated[record['id']]
|
||||
ex_id = record['id']
|
||||
question = record['tokens'].split('|')
|
||||
with gzip.open(os.path.join(args.postfix_dir, ex_id + '.gz')) as fpost:
|
||||
random.seed(ex_id)
|
||||
formula = get_formula(fpost)
|
||||
if formula:
|
||||
fields = [' '.join(question), formula]
|
||||
if args.ex_id:
|
||||
fields.insert(0, ex_id)
|
||||
print u'\t'.join(fields)
|
||||
count += 1
|
||||
print >> sys.stderr, 'Printed', count, 'examples.'
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
# Grammar for Printing out Base Predicates
|
||||
|
||||
################################################################
|
||||
# Macros
|
||||
|
||||
(def @R reverse)
|
||||
(def @type fb:type.object.type)
|
||||
(def @row fb:type.row)
|
||||
(def @numdate (union fb:type.number fb:type.datetime))
|
||||
|
||||
(def @next fb:row.row.next)
|
||||
(def @index fb:row.row.index)
|
||||
(def @p.num fb:cell.cell.number)
|
||||
(def @p.date fb:cell.cell.date)
|
||||
(def @p.num2 fb:cell.cell.num2)
|
||||
(def @p.part fb:cell.cell.part)
|
||||
|
||||
################################################################
|
||||
# Base Rules for Sets
|
||||
|
||||
################################
|
||||
# Anchored Rules
|
||||
|
||||
# Fuzzy match the utterance phrase to a table cell (or part of cell)
|
||||
(rule $Set ($PHRASE) (FuzzyMatchFn entity) (anchored 1))
|
||||
|
||||
# Extract number or date value from the phrase
|
||||
(rule $Set ($PHRASE) (NumberFn) (anchored 1))
|
||||
(rule $Set ($PHRASE) (DateFn) (anchored 1))
|
||||
(when yearrange
|
||||
(rule $Set ($PHRASE) (YearRangeFn) (anchored 1))
|
||||
)
|
||||
|
||||
################################
|
||||
# Floating Rules
|
||||
|
||||
# Set of all rows
|
||||
(rule $Set (nothing) (ConstantFn (@type @row)))
|
||||
|
||||
(when closedclass
|
||||
# Generate things like "null"
|
||||
(rule $Set (nothing) (tables.match.ClosedClassFn generic))
|
||||
# If a column contains only a few unique things, generate them
|
||||
(rule $Set (nothing) (tables.match.ClosedClassFn column))
|
||||
)
|
||||
|
||||
################################################################
|
||||
# Binaries
|
||||
|
||||
# Table column relations (row <-> cell)
|
||||
(rule $Binary (nothing) (FuzzyMatchFn any binary))
|
||||
|
||||
# Normalization relations (cell <-> primitive)
|
||||
(when (not prenormalized)
|
||||
(for @property (@next @index @p.num @p.date @p.num2)
|
||||
(rule $Binary (nothing) (ConstantFn @property))
|
||||
)
|
||||
(when part
|
||||
(rule $Binary (nothing) (ConstantFn @p.part))
|
||||
)
|
||||
)
|
||||
(when prenormalized
|
||||
(for @property (@next @index)
|
||||
(rule $Binary (nothing) (ConstantFn @property))
|
||||
)
|
||||
)
|
||||
|
||||
# Comparison
|
||||
(when comparison
|
||||
(for @comparison (< > <= >=)
|
||||
(rule $Comparison (nothing) (ConstantFn @comparison))
|
||||
)
|
||||
)
|
||||
(when neq
|
||||
(rule $Neq (nothing) (ConstantFn !=))
|
||||
)
|
||||
|
||||
################################################################
|
||||
# ROOT
|
||||
|
||||
(rule $ROOT ($Set) (IdentityFn))
|
||||
(rule $ROOT ($Binary) (IdentityFn))
|
||||
(rule $ROOT ($Comparison) (IdentityFn))
|
||||
(rule $ROOT ($Neq) (IdentityFn))
|
||||
|
|
@ -17,8 +17,8 @@
|
|||
(def @!p.num !fb:cell.cell.number)
|
||||
(def @p.date fb:cell.cell.date)
|
||||
(def @!p.date !fb:cell.cell.date)
|
||||
(def @p.second fb:cell.cell.second)
|
||||
(def @!p.second !fb:cell.cell.second)
|
||||
(def @p.num2 fb:cell.cell.num2)
|
||||
(def @!p.num2 !fb:cell.cell.num2)
|
||||
|
||||
################################################################
|
||||
# Lexicon
|
||||
|
|
@ -32,6 +32,7 @@
|
|||
|
||||
################################
|
||||
# Create binary from thin air
|
||||
#(rule $Entity (nothing) (ConstantFn fb:cell.null))
|
||||
(rule $Binary (nothing) (FuzzyMatchFn any binary))
|
||||
(rule $Unary (nothing) (FuzzyMatchFn any unary))
|
||||
|
||||
|
|
@ -42,7 +43,7 @@
|
|||
(rule $Property (nothing) (ConstantFn @property))
|
||||
)
|
||||
(when second
|
||||
(rule $Property (nothing) (ConstantFn @p.second))
|
||||
(rule $Property (nothing) (ConstantFn @p.num2))
|
||||
)
|
||||
|
||||
################################################################
|
||||
|
|
@ -97,8 +98,7 @@
|
|||
(when movement
|
||||
# (@next (...[rows]...)) --> ...[rows]...
|
||||
(for @movement (@next @!next)
|
||||
# HACK: Added 'nothing' to prevent unary cycle
|
||||
(rule $RowSet (nothing $RowSet) (lambda r (@movement (var r))))
|
||||
(rule $RowSet ($RowSet) (lambda r (@movement (var r))))
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -228,7 +228,7 @@
|
|||
(when superlative
|
||||
(for @argm (argmax argmin)
|
||||
# "first", "last"
|
||||
(rule $RowSet (nothing $RowSet) (lambda r (@argm 1 1 (var r) @index)))
|
||||
(rule $RowSet ($RowSet) (lambda r (@argm 1 1 (var r) @index)))
|
||||
# Generic argmax / argmin on rows or values
|
||||
(rule $RowSet ($RowSet $FnOnRow) (lambda r (lambda f (@argm 1 1 (var r) (@R (var f))))))
|
||||
(when u-superlative
|
||||
|
|
|
|||
|
|
@ -1,174 +0,0 @@
|
|||
# Generic Grammar
|
||||
# Use more generic compositional patterns.
|
||||
|
||||
################################################################
|
||||
# Macros
|
||||
|
||||
(def @R reverse)
|
||||
(def @type fb:type.object.type)
|
||||
(def @row fb:type.row)
|
||||
|
||||
(def @next fb:row.row.next)
|
||||
(def @!next !fb:row.row.next)
|
||||
(def @index fb:row.row.index)
|
||||
(def @!index !fb:row.row.index)
|
||||
|
||||
(def @p.num fb:cell.cell.number)
|
||||
(def @!p.num !fb:cell.cell.number)
|
||||
(def @p.date fb:cell.cell.date)
|
||||
(def @!p.date !fb:cell.cell.date)
|
||||
|
||||
################################################################
|
||||
# Lexicon
|
||||
|
||||
################################
|
||||
# Anchored Rules: Entity, Unary, Binary
|
||||
(rule $Entity ($PHRASE) (FuzzyMatchFn entity) (anchored 1))
|
||||
(rule $Binary ($PHRASE) (FuzzyMatchFn binary) (anchored 1))
|
||||
(rule $Entity ($PHRASE) (NumberFn) (anchored 1))
|
||||
(rule $Entity ($PHRASE) (DateFn) (anchored 1))
|
||||
|
||||
################################
|
||||
# Create binary from thin air
|
||||
(rule $Binary (nothing) (FuzzyMatchFn any binary))
|
||||
|
||||
################################
|
||||
# Property
|
||||
# (@p.num (number 42)) / (@p.date (date 2012 12 21))
|
||||
(for @property (@p.num @p.date)
|
||||
(rule $Property (nothing) (ConstantFn @property))
|
||||
)
|
||||
|
||||
################################################################
|
||||
# Composition
|
||||
|
||||
# $RowSet = list of rows (type = fb:row.row)
|
||||
# $ValueSet = list of values (NameValue or other primitives)
|
||||
# $SingleValue = number representing count (may not make sense to compose it with other things)
|
||||
|
||||
################################
|
||||
# Generic Rows
|
||||
# (@type @row)
|
||||
(rule $RowSet (nothing) (ConstantFn (@type @row)))
|
||||
|
||||
################################
|
||||
# Join
|
||||
# c_name.barack_obama
|
||||
(rule $ValueSet ($Entity) (IdentityFn))
|
||||
# (or c_name.barack_obama c_name.bill_clinton)
|
||||
(rule $ValueSet ($Entity $Entity) (lambda e1 (lambda e2 (or (var e1) (var e2)))))
|
||||
# (c.name c_name.barack_obama)
|
||||
(rule $RowSet ($Binary $ValueSet) (lambda b (lambda v ((var b) (var v)))))
|
||||
# (!c.name (...[rows]...)) --> c_name.barack_obama
|
||||
(rule $ValueSet ($Binary $RowSet) (lambda b (lambda r ((@R (var b)) (var r)))))
|
||||
# (c.height (@p.num (number 180)))
|
||||
(rule $RowSet ($Binary $Property $ValueSet)
|
||||
(lambda b (lambda p (lambda v ((var b) ((var p) (var v))))))
|
||||
)
|
||||
# (@!p.num (!c.height (...[rows]...))) --> (number 180)
|
||||
(rule $ValueSet ($Binary $Property $RowSet)
|
||||
(lambda b (lambda p (lambda r ((@R (var p)) ((@R (var b)) (var r))))))
|
||||
)
|
||||
# (@next (...[rows]...)) --> ...[rows]...
|
||||
(for @movement (@next @!next)
|
||||
# HACK: Added 'nothing' to prevent unary cycle
|
||||
(rule $RowSet (nothing $RowSet) (lambda r (@movement (var r))))
|
||||
)
|
||||
|
||||
(when comparison
|
||||
# (c.name (!= c_name.barack_obama))
|
||||
(rule $RowSet ($Binary $Entity)
|
||||
(lambda b (lambda e ((var b) (!= (var e)))))
|
||||
)
|
||||
# (c.height (@p.num (>= (number 180))))
|
||||
(for @comparison (!= < > <= >=)
|
||||
(rule $RowSet ($Binary $Property $Entity)
|
||||
(lambda b (lambda p (lambda e ((var b) ((var p) (@comparison (var e)))))))
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
################################
|
||||
# Aggregate
|
||||
|
||||
# (count (...[rows]...)) --> (number 4)
|
||||
(rule $SingleValue ($RowSet) (lambda r (count (var r))))
|
||||
(when aggregate
|
||||
(rule $SingleValue ($ValueSet) (lambda r (min (var r))))
|
||||
(rule $SingleValue ($ValueSet) (lambda r (max (var r))))
|
||||
(rule $SingleValue ($ValueSet) (lambda r (sum (var r))))
|
||||
(rule $SingleValue ($ValueSet) (lambda r (avg (var r))))
|
||||
)
|
||||
|
||||
################################
|
||||
# Superlative
|
||||
|
||||
# Function on row set -- used in:
|
||||
# (argmax 1 1 (@type @row) (reverse ___))
|
||||
# ___ = (lambda r (@!p.num (!c.height (var r))))
|
||||
# "highest"
|
||||
(rule $FnOnRow ($Binary $Property)
|
||||
(lambda b (lambda p (lambda r ((@R (var p)) ((@R (var b)) (var r))))))
|
||||
)
|
||||
|
||||
# Function on value set -- used in:
|
||||
# (argmax 1 1 (!c.name (@type @row)) (reverse ___))
|
||||
# ___ = (lambda v (count (c.name (var v))))
|
||||
# "which name occurs the most often"
|
||||
(rule $FnOnValue ($Binary)
|
||||
(lambda b (lambda v (count ((var b) (var v)))))
|
||||
)
|
||||
# (argmax 1 1 (@!p.num (!c.height (@type @row))) (reverse ___))
|
||||
# ___ = (lambda v (count (c.height (@p.num (var v)))))
|
||||
# "which height occurs the most often"
|
||||
(rule $FnOnValue ($Binary $Property)
|
||||
(lambda b (lambda p (lambda v (count ((var b) ((var p) (var v)))))))
|
||||
)
|
||||
# (argmax 1 1 (!c.name (@type @row)) (reverse ___))
|
||||
# ___ = (lambda v (@!p.num (!c.height (c.name (var v)))))
|
||||
# "which person is the highest"
|
||||
(rule $FnOnValue ($Binary $Binary $Property)
|
||||
(lambda b1 (lambda b2 (lambda p (lambda v ((@R (var p)) ((@R (var b2)) ((var b1) (var v))))))))
|
||||
)
|
||||
|
||||
# Actual declaration for argmin and argmax
|
||||
(for @argm (argmax argmin)
|
||||
# "first", "last"
|
||||
(rule $RowSet (nothing $RowSet) (lambda r (@argm 1 1 (var r) @index)))
|
||||
# Generic argmax / argmin on rows or values
|
||||
(rule $RowSet ($RowSet $FnOnRow) (lambda r (lambda f (@argm 1 1 (var r) (reverse (var f))))))
|
||||
(rule $ValueSet ($ValueSet $FnOnValue) (lambda v (lambda f (@argm 1 1 (var v) (reverse (var f))))))
|
||||
)
|
||||
|
||||
################################
|
||||
# Arithmetic
|
||||
|
||||
# (- (... (c_name.barack_obama)) (... (c_name.bill_clinton)))
|
||||
# "How much taller is Barack Obama than Bill Clinton"
|
||||
(when arithmetic
|
||||
(rule $SingleValue ($FnOnValue $Entity $Entity)
|
||||
(lambda f (lambda e1 (lambda e2 (- ((var f) (var e1)) ((var f) (var e2))))))
|
||||
)
|
||||
)
|
||||
|
||||
################################
|
||||
# Merge
|
||||
# Warning: Without pruning, this can blow up the search space!
|
||||
(when merge
|
||||
(rule $RowSet ($RowSet $RowSet) (lambda r1 (lambda r2 (and (var r1) (var r2)))))
|
||||
)
|
||||
|
||||
################################
|
||||
# ROOT
|
||||
(rule $ROOT ($ValueSet) (IdentityFn))
|
||||
(rule $ROOT ($SingleValue) (IdentityFn))
|
||||
|
||||
################################################################
|
||||
# For debugging:
|
||||
|
||||
(when debug
|
||||
(rule $Any ($ValueSet) (IdentityFn))
|
||||
(rule $Any ($RowSet) (IdentityFn))
|
||||
(rule $Any ($SingleValue) (IdentityFn))
|
||||
(rule $ROOT ($Any) (IdentityFn))
|
||||
)
|
||||
|
|
@ -0,0 +1,204 @@
|
|||
# Grow Grammar
|
||||
#
|
||||
# Grow formulas from solid entities. No floating relations are formed.
|
||||
# This is more similar to the search method in Jonathan Berant's ACL2014 paper.
|
||||
#
|
||||
# Main Categories:
|
||||
# - Set: a set of things
|
||||
# - Map[Set s]: a mapping whose domain is s
|
||||
|
||||
################################################################
|
||||
# Macros
|
||||
|
||||
(def @R reverse)
|
||||
(def @type fb:type.object.type)
|
||||
(def @row fb:type.row)
|
||||
(def @numdate (union fb:type.number fb:type.datetime))
|
||||
|
||||
(def @next fb:row.row.next)
|
||||
(def @index fb:row.row.index)
|
||||
(def @p.num fb:cell.cell.number)
|
||||
(def @p.date fb:cell.cell.date)
|
||||
(def @p.num2 fb:cell.cell.num2)
|
||||
(def @p.part fb:cell.cell.part)
|
||||
|
||||
################################################################
|
||||
# Base Rules for Sets
|
||||
|
||||
################################
|
||||
# Anchored Rules
|
||||
|
||||
# Fuzzy match the utterance phrase to a table cell (or part of cell)
|
||||
(rule $Set ($PHRASE) (FuzzyMatchFn entity) (anchored 1))
|
||||
|
||||
# Extract number or date value from the phrase
|
||||
(rule $Set ($PHRASE) (NumberFn) (anchored 1))
|
||||
(rule $Set ($PHRASE) (DateFn) (anchored 1))
|
||||
(when yearrange
|
||||
(rule $Set ($PHRASE) (DateRangeFn) (anchored 1))
|
||||
)
|
||||
|
||||
# Allow |or| on two entities. (Allowing |or| on any Sets leads to massive explosion)
|
||||
(when alternative
|
||||
(rule $Entity ($PHRASE) (FuzzyMatchFn entity) (anchored 1))
|
||||
(rule $Set ($Entity $Entity)
|
||||
(tables.grow.ApplyFn (lambda e1 (lambda e2 (or (var e1) (var e2)))))
|
||||
)
|
||||
)
|
||||
|
||||
################################
|
||||
# Floating Rules
|
||||
|
||||
# Set of all rows
|
||||
(rule $Set (nothing) (ConstantFn (@type @row)))
|
||||
|
||||
(when closedclass
|
||||
# Generate things like "null"
|
||||
(rule $Set (nothing) (tables.match.ClosedClassFn generic))
|
||||
# If a column contains only a few unique things, generate them
|
||||
(rule $Set (nothing) (tables.match.ClosedClassFn column))
|
||||
)
|
||||
|
||||
################################################################
|
||||
# Binaries
|
||||
|
||||
# Table column relations (row <-> cell)
|
||||
(rule $Binary (nothing) (FuzzyMatchFn any binary))
|
||||
|
||||
# Normalization relations (cell <-> primitive)
|
||||
(when (not prenormalized)
|
||||
(for @property (@next @index @p.num @p.date @p.num2)
|
||||
(rule $Binary (nothing) (ConstantFn @property))
|
||||
)
|
||||
(when part
|
||||
(rule $Binary (nothing) (ConstantFn @p.part))
|
||||
)
|
||||
)
|
||||
(when prenormalized
|
||||
(for @property (@next @index)
|
||||
(rule $Binary (nothing) (ConstantFn @property))
|
||||
)
|
||||
)
|
||||
|
||||
# Comparison
|
||||
(when comparison
|
||||
(for @comparison (< > <= >=)
|
||||
(rule $Comparison (nothing) (ConstantFn @comparison))
|
||||
)
|
||||
)
|
||||
(when neq
|
||||
(rule $Neq (nothing) (ConstantFn !=))
|
||||
)
|
||||
|
||||
################################################################
|
||||
# Composition on Sets
|
||||
|
||||
################################
|
||||
# Set = g(Set)
|
||||
|
||||
# Joins (apply Binary on Set)
|
||||
(rule $Set ($Set $Binary) (tables.grow.ApplyFn (lambda s (lambda b ((var b) (var s))))))
|
||||
(rule $Set ($Set $Binary) (tables.grow.ApplyFn (lambda s (lambda b ((@R (var b)) (var s))))))
|
||||
(rule $Set ($Set $Comparison) (tables.grow.ApplyFn (lambda s (lambda c ((var c) (var s))))))
|
||||
(rule $Set ($Set $Neq) (tables.grow.ApplyFn (lambda s (lambda c ((var c) (var s))))))
|
||||
|
||||
# Aggregates
|
||||
(for @aggregate (count min max sum avg)
|
||||
(rule $Set ($Set) (tables.grow.ApplyFn (lambda s (@aggregate (var s)))))
|
||||
)
|
||||
|
||||
################################
|
||||
# Set = g(Set, Set)
|
||||
|
||||
# Merge
|
||||
(when (or merge merge-and)
|
||||
(rule $Set ($Set $Set)
|
||||
(tables.grow.ApplyFn (lambda s1 (lambda s2 (and (var s1) (var s2))))
|
||||
same-type))
|
||||
)
|
||||
(when (or merge merge-or)
|
||||
(rule $Set ($Set $Set)
|
||||
(tables.grow.ApplyFn (lambda s1 (lambda s2 (or (var s1) (var s2))))
|
||||
same-type))
|
||||
)
|
||||
|
||||
# Arithmetic
|
||||
(when arithmetic
|
||||
(rule $Set ($Set $Set)
|
||||
(tables.grow.ApplyFn (lambda s1 (lambda s2 (- (var s1) (var s2))))
|
||||
(arg1-type fb:type.number) (arg2-type fb:type.number)))
|
||||
)
|
||||
|
||||
################################################################
|
||||
# ScopedFormula
|
||||
|
||||
(when scoped
|
||||
|
||||
################################
|
||||
# Map = Grow[start](Set)
|
||||
|
||||
# Set s -> Identity map with domain = s
|
||||
(rule $Map ($Set) (tables.grow.BeginGrowFn))
|
||||
|
||||
################################
|
||||
# Map = g(Map)
|
||||
|
||||
# Joins
|
||||
(rule $Map ($Map $Binary) (tables.grow.ApplyFn (lambda m (lambda b (lambda x ((var b) ((var m) (var x))))))))
|
||||
(rule $Map ($Map $Binary) (tables.grow.ApplyFn (lambda m (lambda b (lambda x ((@R (var b)) ((var m) (var x))))))))
|
||||
(rule $Map ($Map $Comparison) (tables.grow.ApplyFn (lambda m (lambda c (lambda x ((var c) ((var m) (var x))))))))
|
||||
|
||||
# Aggregates
|
||||
(for @aggregate (count min max sum avg)
|
||||
(rule $Map ($Map) (tables.grow.ApplyFn (lambda m (lambda x (@aggregate ((var m) (var x)))))))
|
||||
)
|
||||
|
||||
################################
|
||||
# Map = g(Set|Map, Set|Map) with at least 1 Map
|
||||
|
||||
(when (or scoped-2args-merge scoped-2args-merge-and)
|
||||
# Need both rules since the children in ingredients are not commutable
|
||||
(rule $Map ($Map $Map) (tables.grow.ApplyFn (lambda m1 (lambda m2 (lambda x (and ((var m1) (var x)) ((var m2) (var x))))))
|
||||
same-type))
|
||||
(rule $Map ($Map $Set) (tables.grow.ApplyFn (lambda m1 (lambda s2 (lambda x (and ((var m1) (var x)) (var s2)))))
|
||||
same-type))
|
||||
(rule $Map ($Set $Map) (tables.grow.ApplyFn (lambda s1 (lambda m2 (lambda x (and (var s1) ((var m2) (var x))))))
|
||||
same-type))
|
||||
)
|
||||
# This is super productive
|
||||
(when (or scoped-2args-merge scoped-2args-merge-or)
|
||||
(rule $Map ($Map $Map) (tables.grow.ApplyFn (lambda m1 (lambda m2 (lambda x (or ((var m1) (var x)) ((var m2) (var x))))))
|
||||
same-type))
|
||||
(rule $Map ($Map $Set) (tables.grow.ApplyFn (lambda m1 (lambda s2 (lambda x (or ((var m1) (var x)) (var s2)))))
|
||||
same-type))
|
||||
(rule $Map ($Set $Map) (tables.grow.ApplyFn (lambda s1 (lambda m2 (lambda x (or (var s1) ((var m2) (var x))))))
|
||||
same-type))
|
||||
)
|
||||
# This is super productive
|
||||
(when scoped-2args-arithmetic
|
||||
(rule $Map ($Map $Map) (tables.grow.ApplyFn (lambda m1 (lambda m2 (lambda x (- ((var m1) (var x)) ((var m2) (var x))))))
|
||||
(arg1-type fb:type.number) (arg2-type fb:type.number)))
|
||||
(rule $Map ($Map $Set) (tables.grow.ApplyFn (lambda m1 (lambda s2 (lambda x (- ((var m1) (var x)) (var s2)))))
|
||||
(arg1-type fb:type.number) (arg2-type fb:type.number)))
|
||||
(rule $Map ($Set $Map) (tables.grow.ApplyFn (lambda s1 (lambda m2 (lambda x (- (var s1) ((var m2) (var x))))))
|
||||
(arg1-type fb:type.number) (arg2-type fb:type.number)))
|
||||
)
|
||||
|
||||
################################
|
||||
# Set = Grow[end, f](Map)
|
||||
|
||||
# Superlatives
|
||||
(rule $Set ($Map) (tables.grow.EndGrowFn (lambda s (lambda r (argmax 1 1 (var s) (reverse (var r)))))))
|
||||
(rule $Set ($Map) (tables.grow.EndGrowFn (lambda s (lambda r (argmin 1 1 (var s) (reverse (var r)))))))
|
||||
|
||||
# Set (x in s such that r(x) is nonempty)
|
||||
(when set
|
||||
(rule $Set ($Map) (tables.grow.EndGrowFn (lambda s (lambda r (and (var s) (mark x (: ((var r) (var x)))))))))
|
||||
)
|
||||
|
||||
)
|
||||
|
||||
################################################################
|
||||
# ROOT
|
||||
|
||||
(rule $ROOT ($Set) (IdentityFn))
|
||||
|
|
@ -1,106 +1,131 @@
|
|||
# Simple Grammar
|
||||
# Use only joins, next, and maybe superlatives
|
||||
#
|
||||
# Main Categories:
|
||||
# - Set
|
||||
# - Relation (function from Set to Set)
|
||||
|
||||
################################################################
|
||||
# Macros
|
||||
# Macros
|
||||
|
||||
(def @R reverse)
|
||||
(def @type fb:type.object.type)
|
||||
(def @row fb:type.row)
|
||||
|
||||
(def @next fb:row.row.next)
|
||||
(def @!next !fb:row.row.next)
|
||||
(def @index fb:row.row.index)
|
||||
(def @!index !fb:row.row.index)
|
||||
|
||||
(def @p.num fb:cell.cell.number)
|
||||
(def @!p.num !fb:cell.cell.number)
|
||||
(def @p.date fb:cell.cell.date)
|
||||
(def @!p.date !fb:cell.cell.date)
|
||||
(def @p.num2 fb:cell.cell.num2)
|
||||
(def @p.str1 fb:cell.cell.str1)
|
||||
(def @p.str2 fb:cell.cell.str2)
|
||||
|
||||
################################################################
|
||||
# Lexicon
|
||||
|
||||
################################
|
||||
# Anchored Rules: Entity, Unary, Binary
|
||||
# Anchored Rules
|
||||
(rule $Entity ($PHRASE) (FuzzyMatchFn entity) (anchored 1))
|
||||
(rule $Binary ($PHRASE) (FuzzyMatchFn binary) (anchored 1))
|
||||
(rule $Entity ($PHRASE) (NumberFn) (anchored 1))
|
||||
(rule $Entity ($PHRASE) (DateFn) (anchored 1))
|
||||
|
||||
################################
|
||||
# Create binary from thin air
|
||||
(rule $Binary (nothing) (FuzzyMatchFn any binary))
|
||||
(rule $Set ($Entity) (IdentityFn))
|
||||
|
||||
(when alternative
|
||||
(rule $Set ($Entity $Entity) (lambda e1 (lambda e2 (or (var e1) (var e2)))))
|
||||
)
|
||||
|
||||
################################
|
||||
# Property
|
||||
# (@p.num (number 42)) / (@p.date (date 2012 12 21))
|
||||
(for @property (@p.num @p.date)
|
||||
(rule $Property (nothing) (ConstantFn @property))
|
||||
# Floating Rules
|
||||
(rule $Set (nothing) (ConstantFn (@type @row)))
|
||||
|
||||
################################################################
|
||||
# Binaries
|
||||
|
||||
(rule $Binary (nothing) (FuzzyMatchFn any binary))
|
||||
(for @property (@p.num @p.date @p.num2 @next)
|
||||
(rule $Binary (nothing) (ConstantFn @property))
|
||||
)
|
||||
# Treat comparisons specially as reversed comparisons are redundant.
|
||||
(for @comparison (< > <= >= !=)
|
||||
(rule $Comparison (nothing) (ConstantFn @comparison))
|
||||
)
|
||||
|
||||
################################################################
|
||||
# Composition
|
||||
|
||||
# $RowSet = list of rows (type = fb:row.row)
|
||||
# $ValueSet = list of values (NameValue or other primitives)
|
||||
|
||||
################################
|
||||
# Generic Rows
|
||||
# (@type @row)
|
||||
(rule $RowSet (nothing) (ConstantFn (@type @row)))
|
||||
# Composition
|
||||
|
||||
# Option 1: Create any Relation with any domain
|
||||
(when (not scoped)
|
||||
(rule $Relation ($Binary) (IdentityFn))
|
||||
(rule $Relation ($Binary) (lambda b (lambda x ((@R (var b)) (var x)))))
|
||||
(rule $Relation ($Comparison) (IdentityFn))
|
||||
)
|
||||
# Option 2: Relations must be scoped with a Set
|
||||
(when scoped
|
||||
(rule $Relation ($Set $Binary)
|
||||
(lambda s (lambda b (lambda x ((var b) (and (var s) (var x)))))))
|
||||
(rule $Relation ($Set $Binary)
|
||||
(lambda s (lambda b (lambda x ((@R (var b)) (and (var s) (var x)))))))
|
||||
(rule $Relation ($Set $Comparison)
|
||||
(lambda s (lambda c (lambda x ((var c) (and (var s) (var x)))))))
|
||||
)
|
||||
|
||||
(rule $Relation ($Binary $Relation)
|
||||
(lambda b (lambda r (lambda x ((var b) ((var r) (var x)))))))
|
||||
(rule $Relation ($Binary $Relation)
|
||||
(lambda b (lambda r (lambda x ((@R (var b)) ((var r) (var x)))))))
|
||||
(rule $Relation ($Comparison $Relation)
|
||||
(lambda b (lambda r (lambda x ((var b) ((var r) (var x)))))))
|
||||
|
||||
################################
|
||||
# Join
|
||||
# c_name.barack_obama
|
||||
(rule $ValueSet ($Entity) (IdentityFn))
|
||||
# (c.name c_name.barack_obama)
|
||||
(rule $RowSet ($Binary $ValueSet) (lambda b (lambda v ((var b) (var v)))))
|
||||
# (!c.name (...[rows]...)) --> c_name.barack_obama
|
||||
(rule $ValueSet ($Binary $RowSet) (lambda b (lambda r ((@R (var b)) (var r)))))
|
||||
# (c.height (@p.num (number 180)))
|
||||
(rule $RowSet ($Binary $Property $ValueSet)
|
||||
(lambda b (lambda p (lambda v ((var b) ((var p) (var v))))))
|
||||
|
||||
# Option 1: Repeatedly apply binaries to Set
|
||||
(when layerjoin
|
||||
(rule $Set ($Binary $Set) (lambda b (lambda s ((var b) (var s)))))
|
||||
(rule $Set ($Binary $Set) (lambda b (lambda s ((@R (var b)) (var s)))))
|
||||
(rule $Set ($Comparison $Set) (lambda c (lambda s ((var c) (var s)))))
|
||||
)
|
||||
# (@!p.num (!c.height (...[rows]...))) --> (number 180)
|
||||
(rule $ValueSet ($Binary $Property $RowSet)
|
||||
(lambda b (lambda p (lambda r ((@R (var p)) ((@R (var b)) (var r))))))
|
||||
# Option 2: Apply composed relations to Set
|
||||
# This is cleaner and more general, but it produces redundant formulas.
|
||||
(when (not layerjoin)
|
||||
(rule $Set ($Relation $Set) (lambda r (lambda s ((var r) (var s)))))
|
||||
)
|
||||
|
||||
(when movement
|
||||
# (@next (...[rows]...)) --> ...[rows]...
|
||||
(for @movement (@next @!next)
|
||||
# HACK: Added 'nothing' to prevent unary cycle
|
||||
(rule $RowSet (nothing $RowSet) (lambda r (@movement (var r))))
|
||||
)
|
||||
################################
|
||||
# Aggregate
|
||||
(for @aggregate (count min max sum avg)
|
||||
# Technically can be combined with the rule below,
|
||||
# but the resulting formula will be different from the annotation
|
||||
(rule $Set (nothing $Set) (lambda s (@aggregate (var s))))
|
||||
(rule $Relation (nothing $Relation) (lambda r (lambda x (@aggregate ((var r) (var x))))))
|
||||
)
|
||||
|
||||
################################
|
||||
# Superlative
|
||||
(for @argm (argmax argmin)
|
||||
(rule $Set (nothing $Set) (lambda s (@argm 1 1 (var s) @index)))
|
||||
(rule $Set ($Set $Relation) (lambda s (lambda r (@argm 1 1 (var s) (@R (var r))))))
|
||||
)
|
||||
|
||||
(when superlative
|
||||
(for @argm (argmax argmin)
|
||||
# "first", "last"
|
||||
(rule $RowSet (nothing $RowSet) (lambda r (@argm 1 1 (var r) @index)))
|
||||
# Generic argmax / argmin on rows or values
|
||||
# (argmax 1 1 (@type @row) (reverse ___))
|
||||
# ___ = (lambda r (@!p.num (!c.height (var r))))
|
||||
# "highest"
|
||||
(rule $RowSet ($RowSet $Binary $Property)
|
||||
(lambda r (lambda b (lambda p (@argm 1 1 (var r) (reverse (lambda x ((@R (var p)) ((@R (var b)) (var x) ))))))))
|
||||
)
|
||||
################################
|
||||
# Arithmetic
|
||||
(when arithmetic
|
||||
(rule $Set ($Set $Set) (lambda s1 (lambda s2 (- (var s1) (var s2)))))
|
||||
(when r-arithmetic
|
||||
(rule $Set ($Relation $Set $Set)
|
||||
(lambda r (lambda s1 (lambda s2 (- ((var r) (var s1)) ((var r) (var s2)))))))
|
||||
)
|
||||
)
|
||||
|
||||
################################
|
||||
# ROOT
|
||||
(rule $ROOT ($ValueSet) (IdentityFn))
|
||||
|
||||
################################################################
|
||||
# For debugging:
|
||||
|
||||
(when debug
|
||||
(rule $Any ($ValueSet) (IdentityFn))
|
||||
(rule $Any ($RowSet) (IdentityFn))
|
||||
(rule $ROOT ($Any) (IdentityFn))
|
||||
# Merge
|
||||
(when merge
|
||||
(rule $Set ($Set $Set) (lambda s1 (lambda s2 (and (var s1) (var s2)))))
|
||||
)
|
||||
|
||||
################################
|
||||
# ROOT
|
||||
(rule $ROOT ($Set) (IdentityFn))
|
||||
|
|
|
|||
|
|
@ -1,107 +0,0 @@
|
|||
# Tiny grammar for semantic parsing on tables.
|
||||
# Use emnlp2013.grammar as the base grammar.
|
||||
|
||||
################################################################
|
||||
# Macros
|
||||
|
||||
(def @R reverse)
|
||||
(def @type fb:type.object.type)
|
||||
(def @row fb:type.row)
|
||||
(def @cell fb:type.cell)
|
||||
(def @any fb:type.any)
|
||||
|
||||
################################################################
|
||||
# Lexicon
|
||||
|
||||
################################
|
||||
# Padding: can skip these sequences
|
||||
(rule $Padding ($PHRASE) (IdentityFn))
|
||||
|
||||
################################
|
||||
# TokenSpan
|
||||
(rule $TokenSpan ($PHRASE) (FilterSpanLengthFn 1))
|
||||
|
||||
################################
|
||||
# Entity, Unary, Binary (using FuzzyMatchFn)
|
||||
(rule $Entity ($TokenSpan) (FuzzyMatchFn entity))
|
||||
(rule $Entity ($PHRASE) (NumberFn))
|
||||
(rule $Entity ($PHRASE) (DateFn))
|
||||
(rule $Binary ($TokenSpan) (FuzzyMatchFn binary))
|
||||
|
||||
################################
|
||||
# Hard coded rules
|
||||
(rule $Operator (how many) (ConstantFn (lambda x (count (var x)))))
|
||||
(rule $Binary (after) (ConstantFn fb:row.row.next))
|
||||
(rule $Binary (before) (ConstantFn fb:row.row.next))
|
||||
|
||||
# (argmax 1 1 (!r.nation *) (@R (lambda c (!r.total (r.nation (var c))))))
|
||||
# (argmax 1 1 (!x *) (@R (lambda c (!y (x (var c)))))
|
||||
|
||||
(rule $DoubleOperator (most) (ConstantFn
|
||||
(lambda x (lambda y
|
||||
(argmax 1 1
|
||||
((@R (var x)) *)
|
||||
(@R (lambda c ((@R (var y)) ((var x) (var c)))))
|
||||
)
|
||||
))
|
||||
(-> (-> @cell @row) (-> (-> @cell @row) @cell))
|
||||
))
|
||||
(rule $DoubleOperator (most) (ConstantFn
|
||||
(lambda x (lambda y
|
||||
(argmax 1 1
|
||||
((@R (var x)) *)
|
||||
(@R (lambda c (count ((@R (var y)) ((var x) (var c))))))
|
||||
)
|
||||
))
|
||||
(-> (-> @cell @row) (-> (-> @cell @row) @cell))
|
||||
))
|
||||
#(rule $Operator (most length) (ConstantFn
|
||||
# (lambda x
|
||||
# (argmax 1 1
|
||||
# ((@R (var x)) *)
|
||||
# (@R (lambda c (!fb:row.row.length ((var x) (var c)))))
|
||||
# )
|
||||
# )
|
||||
# (-> (-> @cell @row) @cell)
|
||||
#))
|
||||
#(rule $Operator (title with the most) (ConstantFn
|
||||
# (lambda y
|
||||
# (argmax 1 1
|
||||
# (!fb:row.row.title *)
|
||||
# (@R (lambda c ((@R (var y)) (fb:row.row.title (var c)))))
|
||||
# )
|
||||
# )
|
||||
# (-> (-> @cell @row) @cell)
|
||||
#))
|
||||
|
||||
################################################################
|
||||
# Composition
|
||||
|
||||
################################
|
||||
# Join
|
||||
(rule $Set ($Entity) (IdentityFn))
|
||||
(rule $Set ($Binary ($Padding optional) $Set) (JoinFn binary,unary unaryCanBeArg0 unaryCanBeArg1))
|
||||
(rule $Set ($Set ($Padding optional) $Binary) (JoinFn unary,binary unaryCanBeArg0 unaryCanBeArg1))
|
||||
(rule $Set ($Set ($Padding optional) $Set) (MergeFn and))
|
||||
|
||||
################################
|
||||
# Aggregation / Superlative
|
||||
(rule $Set ($Operator ($Padding optional) $Set) (JoinFn binary,unary unaryCanBeArg1 betaReduce))
|
||||
(rule $Set ($Set ($Padding optional) $Operator) (JoinFn unary,binary unaryCanBeArg1 betaReduce))
|
||||
(rule $Set ($Operator ($Padding optional) $Binary) (JoinFn binary,unary unaryCanBeArg1 betaReduce))
|
||||
(rule $Set ($Binary ($Padding optional) $Operator) (JoinFn unary,binary unaryCanBeArg1 betaReduce))
|
||||
(rule $Operator ($Binary ($Padding optional) $DoubleOperator) (JoinFn unary,binary unaryCanBeArg1 betaReduce))
|
||||
(rule $Operator ($DoubleOperator ($Padding optional) $Binary) (JoinFn binary,unary unaryCanBeArg1 betaReduce))
|
||||
|
||||
################################
|
||||
# ROOT
|
||||
(rule $ROOT (($Padding optional) $Set ($Padding optional)) (IdentityFn))
|
||||
|
||||
################################################################
|
||||
# For debugging:
|
||||
|
||||
(rule $Any ($Set) (IdentityFn))
|
||||
(rule $Any ($Binary) (IdentityFn))
|
||||
(rule $Any ($Operator) (IdentityFn))
|
||||
(rule $Any ($DoubleOperator) (IdentityFn))
|
||||
#(rule $ROOT ($Any) (IdentityFn))
|
||||
|
|
@ -1 +0,0 @@
|
|||
../../lib/data/tables/compiled
|
||||
|
|
@ -1 +0,0 @@
|
|||
../../lib/data/tables/csv
|
||||
|
|
@ -1 +0,0 @@
|
|||
../../state/execs
|
||||
|
|
@ -0,0 +1,228 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Dump all possible context predicates of each example.
|
||||
|
||||
Output:
|
||||
- Example ID
|
||||
- Tokens <tab> Context <tab> Canonicalized Target
|
||||
- Lines of records with the following fields (tab-separated):
|
||||
- flag (see below)
|
||||
- predicate
|
||||
- words in the original string of the predicate
|
||||
If the predicate is from an utterance, also add these fields:
|
||||
- inclusive begin index of the utterance span
|
||||
- exclusive end index of the utterance span
|
||||
- source utterance span
|
||||
|
||||
Flags:
|
||||
1st bit = Whether it is from the utterance
|
||||
2nd bit = Whether it is from the table
|
||||
3rd bit = Whether it is a unary (0), binary (1), or operation (2; not used)
|
||||
The next few bits indicate the type
|
||||
- fb:cell. (or a union thereof)
|
||||
- fb:part. (or a union thereof)
|
||||
- fb:row.row.
|
||||
- !fb:row.row.
|
||||
- number or number range
|
||||
- date
|
||||
|
||||
Interpretation of the first two flag bits:
|
||||
00 = predefined predicates (not used here)
|
||||
01 = floating stuff (relations, closed class cells)
|
||||
10 = matched values (pure numbers, pure dates, date ranges)
|
||||
11 = fuzzy matched (cells, parts, unions of cells or parts)
|
||||
"""
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict, Counter
|
||||
|
||||
MONTH = [None, 'january', 'february', 'march', 'april',
|
||||
'may', 'june', 'july', 'august',
|
||||
'september', 'october', 'november', 'december']
|
||||
|
||||
UNSEEN = Counter()
|
||||
|
||||
class Record(object):
|
||||
def __init__(self):
|
||||
self.predicates = []
|
||||
|
||||
def add_info(self, line):
|
||||
m = re.match(r'iter=(\S*): example (\d+)/\d+: (\S+) {', line)
|
||||
if m:
|
||||
self.group, self.num, self.ex_id = m.groups()
|
||||
return
|
||||
m = re.match(r'Tokens: \[(.*)\]$', line)
|
||||
if m:
|
||||
self.question, = m.groups()
|
||||
self.question = self.question.split(', ')
|
||||
return
|
||||
m = re.match(r'(\$[^(]*)\(([-0-9]*),([-0-9]*)\): ([^{]*) {.*', line)
|
||||
if m:
|
||||
cat, start, end, formula = m.groups()
|
||||
if cat != '$ROOT':
|
||||
start, end = int(start), int(end)
|
||||
formulas = self.encode(formula)
|
||||
if not formulas:
|
||||
return
|
||||
if isinstance(formulas, (str, unicode)):
|
||||
formulas = [formulas]
|
||||
for formula in formulas:
|
||||
# Get flags
|
||||
flag = self.get_flag(cat, start, end, formula)
|
||||
if flag.startswith('00'):
|
||||
continue
|
||||
if flag[0] == '0':
|
||||
self.predicates.append((flag, formula, self.get_words(formula)))
|
||||
else:
|
||||
self.predicates.append((flag, formula, self.get_words(formula),
|
||||
start, end, ' '.join(self.question[start:end])))
|
||||
return
|
||||
if line.startswith('$'):
|
||||
raise Exception('Unknown line format: %s' % line)
|
||||
|
||||
def encode(self, formula):
|
||||
if formula == '(date -1 -1 -1)':
|
||||
return None
|
||||
# Postfix encoding for specific cases
|
||||
if formula == '(fb:type.object.type fb:type.row)':
|
||||
return 'type-row'
|
||||
if formula.startswith('(number'):
|
||||
tokens = formula[1:-1].split()
|
||||
assert len(tokens) == 2, '|'.join(tokens)
|
||||
return 'N' + tokens[1]
|
||||
if formula.startswith('(date'):
|
||||
tokens = formula[1:-1].split()
|
||||
assert len(tokens) == 4, '|'.join(tokens)
|
||||
return 'D' + '-'.join(
|
||||
'XX' if x == '-1' else x for x in tokens[1:4])
|
||||
if formula.startswith('(or'):
|
||||
tokens = formula.replace('(or ', '').replace(')', '').split()
|
||||
# Technically we should output each predicate ...
|
||||
return ' '.join([tokens[0]] + [x + ' or' for x in tokens[1:]])
|
||||
if formula.startswith('(and'):
|
||||
tokens = formula.replace('(', '').replace(')', '').split()
|
||||
assert len(tokens) == 7
|
||||
assert tokens[1] == '<'
|
||||
assert tokens[4] == '>='
|
||||
assert tokens[2] == tokens[5] == 'number'
|
||||
return ' '.join(['N' + tokens[3], tokens[1],
|
||||
'N' + tokens[6], tokens[4], tokens[0]])
|
||||
assert not formula.startswith('(')
|
||||
if formula.startswith('fb:row.row.'):
|
||||
return [formula, '!' + formula]
|
||||
return formula
|
||||
|
||||
def get_flag(self, cat, start, end, formula):
|
||||
stuff = formula.split()
|
||||
# First bit: from the utterance?
|
||||
anchored = (start != -1)
|
||||
flag = ('1' if anchored else '0')
|
||||
# Second bit: from the table?
|
||||
if anchored:
|
||||
flag += ('1' if any(x.startswith('fb:') for x in stuff) else '0')
|
||||
else:
|
||||
flag += ('0' if any(x.startswith('fb:cell.cell.') for x in stuff)
|
||||
or formula.replace('!', '') in ('<', '<=', '>', '>=', '=',
|
||||
'type-row', 'fb:row.row.next', 'fb:row.row.index') else '1')
|
||||
# Third bit: relation?
|
||||
flag += ('1' if formula in ('<', '<=', '>', '>=', '!=')
|
||||
or formula.replace('!', '').startswith('fb:cell.cell.')
|
||||
or formula.replace('!', '').startswith('fb:row.row.') else '0')
|
||||
# Next 6 bits: types
|
||||
flag += ('1' if not formula.startswith('fb:cell.cell.') and
|
||||
any(x.startswith('fb:cell.') for x in stuff) else '0')
|
||||
flag += ('1' if any(x.startswith('fb:part.') for x in stuff) else '0')
|
||||
flag += ('1' if formula.startswith('fb:row.row.') else '0')
|
||||
flag += ('1' if formula.startswith('!fb:row.row.') else '0')
|
||||
flag += ('1' if any(x.startswith('N') for x in stuff) else '0')
|
||||
flag += ('1' if any(x.startswith('D') for x in stuff) else '0')
|
||||
return flag
|
||||
|
||||
def get_words(self, formula):
|
||||
words = []
|
||||
for x in formula.split():
|
||||
for prefix in ['fb:cell.cell.', 'fb:row.row.', '!fb:row.row.',
|
||||
'fb:cell.', 'fb:part.']:
|
||||
if not x.startswith(prefix):
|
||||
continue
|
||||
x = x[len(prefix):]
|
||||
# TODO: Use original string instead!
|
||||
words.extend(x.split('_'))
|
||||
break
|
||||
else:
|
||||
if x[0] == 'N':
|
||||
words.append(x[1:])
|
||||
elif x[0] == 'D':
|
||||
if x[1] == '-':
|
||||
# negative year?
|
||||
x = 'D' + x[2:]
|
||||
year, month, day = x[1:].split('-')
|
||||
if year != 'XX':
|
||||
words.append(year)
|
||||
if month != 'XX':
|
||||
words.append(MONTH[int(month)])
|
||||
if day != 'XX':
|
||||
words.append(day)
|
||||
else:
|
||||
UNSEEN[x] += 1
|
||||
return ' '.join(words)
|
||||
|
||||
def dump(self, args, annotated_data):
|
||||
filename = os.path.join(args.outdir, self.ex_id)
|
||||
assert not os.path.exists(filename)
|
||||
with open(filename, 'w', 'utf8') as fout:
|
||||
print >> fout, self.ex_id
|
||||
print >> fout, '\t'.join([u' '.join(self.question),
|
||||
annotated_data['context'], annotated_data['targetCanon']])
|
||||
for stuff in sorted(set(self.predicates)):
|
||||
print >> fout, u'\t'.join(unicode(x) for x in stuff)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('infile',
|
||||
help='Input log file')
|
||||
parser.add_argument('outdir',
|
||||
help='Output directory to create (must NOT exist)')
|
||||
parser.add_argument('-a', '--annotated-dir', default='t/annotated/',
|
||||
help='Path to annotated directory')
|
||||
args = parser.parse_args()
|
||||
|
||||
assert not os.path.exists(args.outdir)
|
||||
os.makedirs(args.outdir)
|
||||
|
||||
# Read annotated data
|
||||
annotated_data = {}
|
||||
dirname = os.path.join(args.annotated_dir, 'data')
|
||||
for filename in os.listdir(dirname):
|
||||
print >> sys.stderr, 'Reading', filename
|
||||
with open(os.path.join(dirname, filename), 'r', 'utf8') as fin:
|
||||
header = fin.readline().rstrip('\n').split('\t')
|
||||
for line in fin:
|
||||
record = dict(zip(header, line.rstrip('\n').split('\t')))
|
||||
annotated_data[record['id']] = record
|
||||
print >> sys.stderr, 'Read %d annotated records' % len(annotated_data)
|
||||
|
||||
# Process the log
|
||||
current_record = None
|
||||
num_records = 0
|
||||
with open(args.infile, 'r', 'utf8') as fin:
|
||||
for line in fin:
|
||||
line = line.strip()
|
||||
if line.startswith('iter='):
|
||||
if current_record:
|
||||
current_record.dump(args, annotated_data[current_record.ex_id])
|
||||
if num_records % 1000 == 0:
|
||||
print >> sys.stderr, 'Processed %d ...' % num_records
|
||||
num_records += 1
|
||||
current_record = Record()
|
||||
if current_record:
|
||||
current_record.add_info(line)
|
||||
current_record.dump(args, annotated_data[current_record.ex_id])
|
||||
print >> sys.stderr, 'Finished reading %d records' % num_records
|
||||
print UNSEEN
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -6,28 +6,54 @@ from codecs import open
|
|||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
def process(line):
|
||||
# Extract only formula
|
||||
line = re.sub(r'^ *True@\d+: \(derivation \(formula ', '', line)
|
||||
num_open = 0
|
||||
stuff = []
|
||||
for char in line:
|
||||
stuff.append(char)
|
||||
if char == '(':
|
||||
num_open += 1
|
||||
elif char == ')':
|
||||
num_open -= 1
|
||||
if not num_open:
|
||||
break
|
||||
line = ''.join(stuff)
|
||||
# Abbreviate stuff
|
||||
line = re.sub(r'\(reverse ([^()]*)\)', r'!\1', line)
|
||||
line = re.sub(r'\(number ([^()]*)\)', r'\1', line)
|
||||
line = line.replace('fb:row.row.next', '@next')
|
||||
line = line.replace('fb:row.row.index', '@index')
|
||||
line = line.replace('fb:type.object.type', '@type')
|
||||
line = line.replace('fb:type.row', '@row')
|
||||
line = line.replace('fb:row.row.', 'r.')
|
||||
line = line.replace('fb:cell.cell.', '@p.')
|
||||
line = line.replace('@p.number', '@p.num')
|
||||
line = line.replace('fb:cell.', 'c.')
|
||||
return line
|
||||
|
||||
def dump(formulas):
|
||||
formulas.sort()
|
||||
for x in formulas:
|
||||
print x
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('infile')
|
||||
args = parser.parse_args()
|
||||
|
||||
state = None
|
||||
formulas = []
|
||||
|
||||
with open(args.infile, 'r', 'utf8') as fin:
|
||||
for line in fin:
|
||||
line = line.rstrip()
|
||||
if line.strip().startswith('Example: '):
|
||||
state = 'started'
|
||||
print
|
||||
print '#' * 200
|
||||
print
|
||||
print line
|
||||
elif state == 'started':
|
||||
print line
|
||||
if line.strip() == '}':
|
||||
state = None
|
||||
if formulas:
|
||||
dump(formulas)
|
||||
print '#', re.sub('^Example: ', '', line.strip())
|
||||
elif line.strip().startswith('True@'):
|
||||
print line
|
||||
formulas.append(process(line))
|
||||
if formulas:
|
||||
dump(formulas)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,88 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-f', '--field', type=int, default=4)
|
||||
parser.add_argument('infile')
|
||||
args = parser.parse_args()
|
||||
|
||||
firsts = []
|
||||
seconds = []
|
||||
|
||||
with open(args.infile, 'r', 'utf8') as fin:
|
||||
for line in fin:
|
||||
line = line.strip()
|
||||
if line.startswith('Reading Example'):
|
||||
firsts.append([])
|
||||
seconds.append([])
|
||||
elif line.startswith('(FIRST)') or line.startswith('(SECOND)'):
|
||||
line = line.split()
|
||||
if line[1].isdigit():
|
||||
amount = int(line[args.field])
|
||||
if line[0] == '(FIRST)':
|
||||
firsts[-1].append(amount)
|
||||
elif line[0] == '(SECOND)':
|
||||
seconds[-1].append(amount)
|
||||
else:
|
||||
raise Exception('Huh?')
|
||||
print >> sys.stderr, 'Read', len(firsts), 'examples'
|
||||
|
||||
# Pad
|
||||
def pad(stuff):
|
||||
n = max(len(x) for x in stuff)
|
||||
for y in stuff:
|
||||
while len(y) < n:
|
||||
y.append(y[-1])
|
||||
pad(firsts)
|
||||
pad(seconds)
|
||||
|
||||
# Find statistics
|
||||
def stat(stuff):
|
||||
if not stuff:
|
||||
return
|
||||
answer = []
|
||||
for i in xrange(len(stuff[0])):
|
||||
data = [x[i] for x in stuff]
|
||||
data.sort()
|
||||
n = len(data)
|
||||
mean = sum(float(x) for x in data) / n
|
||||
var = sum(float(x) * float(x) for x in data) / n - mean * mean
|
||||
print '{}: N={} mean={:.2f} sd={:.2f} min={:d} q1={:d} q2={:d} q3={:d} max={:d}'.format(
|
||||
i, n, mean, var**.5, data[0], data[n/4], data[n/2], data[3*n/4], data[-1])
|
||||
answer.append([data[n/4], data[n/2], data[3*n/4]])
|
||||
return zip(*answer)
|
||||
print 'FIRST:'
|
||||
answer_first = stat(firsts)
|
||||
print 'SECOND:'
|
||||
answer_second = stat(seconds)
|
||||
|
||||
# Plot
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
fig = plt.figure()
|
||||
ax1 = fig.add_subplot(211)
|
||||
if answer_first:
|
||||
for series in answer_first:
|
||||
print series
|
||||
ax1.plot(series)
|
||||
#for stuff in firsts:
|
||||
# ax1.plot(stuff)
|
||||
ax2 = fig.add_subplot(212)
|
||||
if answer_second:
|
||||
for series in answer_second:
|
||||
print series
|
||||
ax2.plot(series)
|
||||
#for stuff in seconds:
|
||||
# ax2.plot(stuff)
|
||||
plt.show()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
/*\
|
||||
|*|
|
||||
|*| :: cookies.js ::
|
||||
|*|
|
||||
|*| A complete cookies reader/writer framework with full unicode support.
|
||||
|*|
|
||||
|*| Revision #1 - September 4, 2014
|
||||
|*|
|
||||
|*| https://developer.mozilla.org/en-US/docs/Web/API/document.cookie
|
||||
|*| https://developer.mozilla.org/User:fusionchess
|
||||
|*|
|
||||
|*| This framework is released under the GNU Public License, version 3 or later.
|
||||
|*| http://www.gnu.org/licenses/gpl-3.0-standalone.html
|
||||
|*|
|
||||
|*| Syntaxes:
|
||||
|*|
|
||||
|*| * docCookies.setItem(name, value[, end[, path[, domain[, secure]]]])
|
||||
|*| * docCookies.getItem(name)
|
||||
|*| * docCookies.removeItem(name[, path[, domain]])
|
||||
|*| * docCookies.hasItem(name)
|
||||
|*| * docCookies.keys()
|
||||
|*|
|
||||
\*/
|
||||
|
||||
var docCookies = {
|
||||
getItem: function (sKey) {
|
||||
if (!sKey) { return null; }
|
||||
return decodeURIComponent(document.cookie.replace(new RegExp("(?:(?:^|.*;)\\s*" + encodeURIComponent(sKey).replace(/[\-\.\+\*]/g, "\\$&") + "\\s*\\=\\s*([^;]*).*$)|^.*$"), "$1")) || null;
|
||||
},
|
||||
setItem: function (sKey, sValue, vEnd, sPath, sDomain, bSecure) {
|
||||
if (!sKey || /^(?:expires|max\-age|path|domain|secure)$/i.test(sKey)) { return false; }
|
||||
var sExpires = "";
|
||||
if (vEnd) {
|
||||
switch (vEnd.constructor) {
|
||||
case Number:
|
||||
sExpires = vEnd === Infinity ? "; expires=Fri, 31 Dec 9999 23:59:59 GMT" : "; max-age=" + vEnd;
|
||||
break;
|
||||
case String:
|
||||
sExpires = "; expires=" + vEnd;
|
||||
break;
|
||||
case Date:
|
||||
sExpires = "; expires=" + vEnd.toUTCString();
|
||||
break;
|
||||
}
|
||||
}
|
||||
document.cookie = encodeURIComponent(sKey) + "=" + encodeURIComponent(sValue) + sExpires + (sDomain ? "; domain=" + sDomain : "") + (sPath ? "; path=" + sPath : "") + (bSecure ? "; secure" : "");
|
||||
return true;
|
||||
},
|
||||
removeItem: function (sKey, sPath, sDomain) {
|
||||
if (!this.hasItem(sKey)) { return false; }
|
||||
document.cookie = encodeURIComponent(sKey) + "=; expires=Thu, 01 Jan 1970 00:00:00 GMT" + (sDomain ? "; domain=" + sDomain : "") + (sPath ? "; path=" + sPath : "");
|
||||
return true;
|
||||
},
|
||||
hasItem: function (sKey) {
|
||||
if (!sKey) { return false; }
|
||||
return (new RegExp("(?:^|;\\s*)" + encodeURIComponent(sKey).replace(/[\-\.\+\*]/g, "\\$&") + "\\s*\\=")).test(document.cookie);
|
||||
},
|
||||
keys: function () {
|
||||
var aKeys = document.cookie.replace(/((?:^|\s*;)[^\=]+)(?=;|$)|^\s*|\s*(?:\=[^;]*)?(?:\1|$)/g, "").split(/\s*(?:\=[^;]*)?;\s*/);
|
||||
for (var nLen = aKeys.length, nIdx = 0; nIdx < nLen; nIdx++) { aKeys[nIdx] = decodeURIComponent(aKeys[nIdx]); }
|
||||
return aKeys;
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
/*\
|
||||
|*|
|
||||
|*| :: cookies.js ::
|
||||
|*|
|
||||
|*| A complete cookies reader/writer framework with full unicode support.
|
||||
|*|
|
||||
|*| Revision #1 - September 4, 2014
|
||||
|*|
|
||||
|*| https://developer.mozilla.org/en-US/docs/Web/API/document.cookie
|
||||
|*| https://developer.mozilla.org/User:fusionchess
|
||||
|*|
|
||||
|*| This framework is released under the GNU Public License, version 3 or later.
|
||||
|*| http://www.gnu.org/licenses/gpl-3.0-standalone.html
|
||||
|*|
|
||||
|*| Syntaxes:
|
||||
|*|
|
||||
|*| * docCookies.setItem(name, value[, end[, path[, domain[, secure]]]])
|
||||
|*| * docCookies.getItem(name)
|
||||
|*| * docCookies.removeItem(name[, path[, domain]])
|
||||
|*| * docCookies.hasItem(name)
|
||||
|*| * docCookies.keys()
|
||||
|*|
|
||||
\*/
|
||||
var docCookies={getItem:function(e){return e?decodeURIComponent(document.cookie.replace(new RegExp("(?:(?:^|.*;)\\s*"+encodeURIComponent(e).replace(/[\-\.\+\*]/g,"\\$&")+"\\s*\\=\\s*([^;]*).*$)|^.*$"),"$1"))||null:null},setItem:function(e,n,o,t,c,r){if(!e||/^(?:expires|max\-age|path|domain|secure)$/i.test(e))return!1
|
||||
var s=""
|
||||
if(o)switch(o.constructor){case Number:s=o===1/0?"; expires=Fri, 31 Dec 9999 23:59:59 GMT":"; max-age="+o
|
||||
break
|
||||
case String:s="; expires="+o
|
||||
break
|
||||
case Date:s="; expires="+o.toUTCString()}return document.cookie=encodeURIComponent(e)+"="+encodeURIComponent(n)+s+(c?"; domain="+c:"")+(t?"; path="+t:"")+(r?"; secure":""),!0},removeItem:function(e,n,o){return this.hasItem(e)?(document.cookie=encodeURIComponent(e)+"=; expires=Thu, 01 Jan 1970 00:00:00 GMT"+(o?"; domain="+o:"")+(n?"; path="+n:""),!0):!1},hasItem:function(e){return e?new RegExp("(?:^|;\\s*)"+encodeURIComponent(e).replace(/[\-\.\+\*]/g,"\\$&")+"\\s*\\=").test(document.cookie):!1},keys:function(){for(var e=document.cookie.replace(/((?:^|\s*;)[^\=]+)(?=;|$)|^\s*|\s*(?:\=[^;]*)?(?:\1|$)/g,"").split(/\s*(?:\=[^;]*)?;\s*/),n=e.length,o=0;n>o;o++)e[o]=decodeURIComponent(e[o])
|
||||
return e}}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
var QUESTIONS=[];
|
||||
QUESTIONS.push({"retained":{"score":0.989,"otherAlterIndices":[0,4,10,16,17,19]},"sprite":["10,727","11th","1st","1st Round","1st, Western","2","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2nd","2nd Round","2nd, Pacific","3rd Round","3rd, Pacific","3rd, USL (3rd)","4th Round","4th, Western","5,575","5,628","5,871","5th","6,028","6,260","6,851","7,169","8,567","9,734","Avg. Attendance","Did not qualify","Division","League","Open Cup","Playoffs","Quarterfinals","Regular Season","Semifinals","USL A-League","USL First Division","USSF D-2 Pro League","Year"],"question":"what was the last year where this team was a part of the usl a-league?","alterIndex":0,"table":[[46,36,37,41,39,38,34],[6,5,43,23,40,35,31],[7,5,43,18,3,35,29],[8,5,43,20,35,35,26],[9,5,43,4,40,22,25],[10,5,44,27,40,22,28],[11,5,44,1,35,19,24],[12,5,44,16,42,17,30],[13,5,44,1,35,3,32],[14,5,44,2,42,19,33],[15,5,45,21,40,19,0]],"id":"nt-0","metadata":{"url":"http://en.wikipedia.org/wiki?action=render&curid=14240688&oldid=599565109","tableIndex":0,"title":"Portland Timbers (2001–10)"}});
|
||||
QUESTIONS.push({"retained":{"score":0.989,"otherAlterIndices":[0,4,10,16,17,19]},"sprite":["1st Round","2","2001","2002","2003","2004","2005","2006","2nd, Pacific","3rd, Pacific","3rd, USL (3rd)","4th Round","4th, Western","5,575","5,628","5,871","5th","6,028","6,260","7,169","Avg. Attendance","Did not qualify","Division","League","Open Cup","Playoffs","Quarterfinals","Regular Season","USL A-League","USL First Division","USSF D-2 Pro League","Year"],"question":"what was the last year where this team was a part of the usl a-league?","alterIndex":4,"table":[[31,22,23,27,25,24,20],[2,1,29,12,26,11,14],[3,1,28,8,0,11,13],[4,1,28,9,21,11,15],[5,1,30,8,26,11,18],[6,1,28,16,26,11,19],[7,1,28,10,26,11,17]],"id":"nt-0","metadata":{"url":"http://en.wikipedia.org/wiki?action=render&curid=14240688&oldid=599565109","tableIndex":0,"title":"Portland Timbers (2001–10)"}});
|
||||
QUESTIONS.push({"retained":{"score":0.989,"otherAlterIndices":[0,4,10,16,17,19]},"sprite":["10,727","11th","1st Round","2","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2nd, Pacific","3rd Round","3rd, Pacific","3rd, USL (3rd)","4th Round","4th, Western","5,575","5,628","5,871","6,028","6,260","6,851","7,169","8,567","9,734","Avg. Attendance","Did not qualify","Division","League","Open Cup","Playoffs","Quarterfinals","Regular Season","Semifinals","USL A-League","USL First Division","USSF D-2 Pro League","Year"],"question":"what was the last year where this team was a part of the usl a-league?","alterIndex":10,"table":[[41,31,32,36,34,33,29],[4,3,40,19,30,15,22],[5,3,40,1,30,2,26],[6,3,39,1,30,2,24],[7,3,39,17,35,18,23],[8,3,39,16,35,15,27],[9,3,38,14,35,15,20],[10,3,39,1,37,15,25],[11,3,39,1,37,18,28],[12,3,38,17,37,15,21],[13,3,39,17,35,18,0]],"id":"nt-0","metadata":{"url":"http://en.wikipedia.org/wiki?action=render&curid=14240688&oldid=599565109","tableIndex":0,"title":"Portland Timbers (2001–10)"}});
|
||||
QUESTIONS.push({"retained":{"score":0.989,"otherAlterIndices":[0,4,10,16,17,19]},"sprite":["11th","1st Round","1st, Western","2","2001","2002","2003","2004","2005","2006","2007","2008","2nd","3rd Round","3rd, USL (3rd)","4th Round","4th, Western","5,575","5,628","5,871","5th","6,028","6,260","6,851","7,169","8,567","Avg. Attendance","Did not qualify","Division","League","Open Cup","Playoffs","Quarterfinals","Regular Season","Semifinals","USL A-League","USL First Division","USSF D-2 Pro League","Year"],"question":"what was the last year where this team was a part of the usl a-league?","alterIndex":16,"table":[[38,28,29,33,31,30,26],[4,3,36,12,32,27,24],[5,3,36,0,27,27,17],[6,3,36,16,27,27,18],[7,3,35,14,32,1,22],[8,3,36,20,34,15,19],[9,3,37,2,27,27,21],[10,3,35,20,32,13,23],[11,3,35,2,27,27,25]],"id":"nt-0","metadata":{"url":"http://en.wikipedia.org/wiki?action=render&curid=14240688&oldid=599565109","tableIndex":0,"title":"Portland Timbers (2001–10)"}});
|
||||
QUESTIONS.push({"retained":{"score":0.989,"otherAlterIndices":[0,4,10,16,17,19]},"sprite":["11th","1st Round","1st, Western","2","2001","2002","2003","2004","2005","2006","2007","2008","2009","2nd Round","2nd, Pacific","3rd, USL (3rd)","4th Round","5,575","5,628","5,871","5th","6,028","6,260","6,851","7,169","8,567","9,734","Avg. Attendance","Did not qualify","Division","League","Open Cup","Playoffs","Quarterfinals","Regular Season","Semifinals","USL A-League","USL First Division","USSF D-2 Pro League","Year"],"question":"what was the last year where this team was a part of the usl a-league?","alterIndex":17,"table":[[39,29,30,34,32,31,27],[4,3,37,20,33,13,21],[5,3,37,2,28,28,25],[6,3,38,0,33,28,23],[7,3,36,0,35,1,26],[8,3,36,0,28,16,17],[9,3,37,15,35,13,24],[10,3,36,14,1,28,18],[11,3,36,20,33,16,19],[12,3,36,0,33,28,22]],"id":"nt-0","metadata":{"url":"http://en.wikipedia.org/wiki?action=render&curid=14240688&oldid=599565109","tableIndex":0,"title":"Portland Timbers (2001–10)"}});
|
||||
QUESTIONS.push({"retained":{"score":0.989,"otherAlterIndices":[0,4,10,16,17,19]},"sprite":["10,727","11th","1st","1st Round","1st, Western","2","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2nd","2nd, Pacific","3rd Round","3rd, Pacific","3rd, USL (3rd)","4th Round","4th, Western","5,575","5,628","5,871","5th","6,028","6,260","6,851","7,169","8,567","9,734","Avg. Attendance","Did not qualify","Division","League","Open Cup","Playoffs","Quarterfinals","Regular Season","Semifinals","USL A-League","USL First Division","USSF D-2 Pro League","Year"],"question":"what was the last year where this team was a part of the usl a-league?","alterIndex":19,"table":[[45,35,36,40,38,37,33],[6,5,44,16,41,21,25],[7,5,43,26,3,21,27],[8,5,43,1,41,18,23],[9,5,43,1,3,18,28],[10,5,42,22,41,21,24],[11,5,44,17,41,18,32],[12,5,42,19,34,34,0],[13,5,43,20,34,21,30],[14,5,43,4,39,3,29],[15,5,42,2,39,34,31]],"id":"nt-0","metadata":{"url":"http://en.wikipedia.org/wiki?action=render&curid=14240688&oldid=599565109","tableIndex":0,"title":"Portland Timbers (2001–10)"}});
|
||||
QUESTIONS.push({"retained":{"score":0.96,"otherAlterIndices":[0,3,7,24,27,30]},"sprite":["11th (sf)","1:50.46","1st","2001","2003","2005","2006","2007","2008","2009","2nd","2nd (h)","3:00.32","3:01.73","3:02.05","3:02.57","3:04.41","3:05.69","3:06.10","3:06.12","3:08.14","3:08.62","3:08.76","3rd","400 m","46.62","46.69","46.85","46.89","47.12","4th","4x400 m relay","7th","Bangkok, Thailand","Beijing, China","Belgrade, Serbia","Birmingham, United Kingdom","Competition","Debrecen, Hungary","Erfurt, Germany","European Championships","European Indoor Championships","European Junior Championships","European U23 Championships","Event","Gothenburg, Sweden","Grosseto, Italy","Izmir, Turkey","Medley relay","Moscow, Russia","Notes","Olympic Games","Position","Tampere, Finland","Universiade","Valencia, Spain","Venue","World Indoor Championships","World Youth Championships","Year"],"question":"in what city did piotr's last 1st place finish occur?","alterIndex":0,"table":[[59,37,56,52,44,50],[3,58,38,10,24,29],[3,58,38,2,48,1],[3,42,46,2,31,19],[4,42,53,23,24,26],[4,42,53,10,31,21],[5,43,39,0,24,25],[5,43,39,2,31,16],[5,54,47,32,24,28],[5,54,47,2,31,15],[6,57,49,11,31,18],[6,40,45,23,31,13],[7,41,36,23,31,20],[7,54,33,32,24,27],[7,54,33,2,31,14],[8,57,55,30,31,22],[8,51,34,32,31,12],[9,54,35,10,31,17]],"id":"nt-1","metadata":{"url":"http://en.wikipedia.org/wiki?action=render&curid=35891489&oldid=589337922","tableIndex":0,"title":"Piotr Kędzia"}});
|
||||
QUESTIONS.push({"retained":{"score":0.96,"otherAlterIndices":[0,3,7,24,27,30]},"sprite":["11th (sf)","1:50.46","1st","2001","2003","2005","2007","2008","2nd","2nd (h)","3:00.32","3:01.73","3:02.05","3:02.57","3:04.41","3:06.10","3:06.12","3:08.14","3:08.62","3:08.76","3rd","400 m","46.62","46.69","46.85","46.89","47.12","4th","4x400 m relay","7th","Beijing, China","Birmingham, United Kingdom","Competition","Debrecen, Hungary","Erfurt, Germany","European Indoor Championships","European Junior Championships","European U23 Championships","Event","Grosseto, Italy","Izmir, Turkey","Medley relay","Moscow, Russia","Notes","Position","Tampere, Finland","Universiade","Venue","World Indoor Championships","World Youth Championships","Year"],"question":"in what city did piotr's last 1st place finish occur?","alterIndex":3,"table":[[50,32,47,44,38,43],[3,48,30,9,41,11],[3,35,42,27,28,17],[3,36,39,0,28,10],[3,49,31,2,28,24],[4,36,31,20,28,14],[4,46,40,29,28,23],[4,46,33,9,28,12],[5,46,33,29,28,19],[5,46,34,8,41,16],[5,37,45,2,21,13],[6,46,34,0,28,18],[7,48,39,8,21,15],[7,48,42,2,28,26],[7,48,33,8,28,25],[7,36,45,2,28,1],[7,46,34,2,28,22]],"id":"nt-1","metadata":{"url":"http://en.wikipedia.org/wiki?action=render&curid=35891489&oldid=589337922","tableIndex":0,"title":"Piotr Kędzia"}});
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<link rel="stylesheet" type="text/css" href="style.css">
|
||||
<script type="text/javascript" src="jquery.min.js"></script>
|
||||
<script type="text/javascript" src="cookies.min.js"></script>
|
||||
<script type="text/javascript" src="main.js"></script>
|
||||
<script type="text/javascript" src="data/501-alter/0.js"></script>
|
||||
<title>MTurk</title>
|
||||
</head>
|
||||
<body style="margin:0;padding:0;height:100%;width:100%;">
|
||||
<form>
|
||||
<input value="dASSIGNMENT_ID_NOT_AVAILABLE" name="assignmentId" id="assignmentId" type="hidden">
|
||||
<div id=content></div>
|
||||
</form>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<link rel="stylesheet" type="text/css" href="style.min.css">
|
||||
<script type="text/javascript" src="jquery.min.js"></script>
|
||||
<script type="text/javascript" src="cookies.min.js"></script>
|
||||
<script type="text/javascript" src="main.min.js"></script>
|
||||
<script type="text/javascript" src="data/501-alter/0.js"></script>
|
||||
<title>MTurk</title>
|
||||
</head>
|
||||
<body style="margin:0;padding:0;height:100%;width:100%;">
|
||||
<form>
|
||||
<input value="dASSIGNMENT_ID_NOT_AVAILABLE" name="assignmentId" id="assignmentId" type="hidden">
|
||||
<div id=content></div>
|
||||
</form>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,363 @@
|
|||
$(function() {
|
||||
var VERSION = "5.2";
|
||||
|
||||
$('#content').html(['<div id=webpage class=column> </div>',
|
||||
'<div id=wrapper class=column>',
|
||||
'<div id=instruction>',
|
||||
'<div id=hidingInstruction class=hidden><button type=button class=floatButton id=showInstructionButton>Show Instructions</button></div>',
|
||||
'<div id=showingInstruction>',
|
||||
'<h2>Task: Answer the question using the information from the table.</h2>',
|
||||
'<button type=button class=floatButton id=hideInstructionButton disabled>Loading ...</button>',
|
||||
'<ul>',
|
||||
'<li>The tables were generated by shuffling the cells in some original table. Assume that each table presents data from a fictional world.</li>',
|
||||
'<li>As such, some questions may sound a little strange (e.g., asking for <em>the city</em> but many cities fit the criteria). Please answer according to the intent of the question.</li>',
|
||||
'<li>If the question really cannot be answered (e.g., when the main entity in the question is missing from the table, or when the table does not make sense), briefly describe the reason in the <strong>second box</strong> and leave the answer box blank.</li>',
|
||||
'<li>We <strong>manually</strong> review the answers and check some of the answers with the gold standard. We use answers from other workers to detect suspicious submissions, but will not reject your work as long as it correctly follows the instructions.</li>',
|
||||
'</ul>',
|
||||
'<h2>Answer Format</h2>',
|
||||
'<ul>',
|
||||
'<li>If the answer appears in the table, please <strong>copy and paste</strong> to avoid typos.</li>',
|
||||
'<li><strong>Do not</strong> add punctuation marks or elaboration to the answers: use <span class=whitebox>Paris</span> instead of <span class=redbox>Paris.</span> or <span class=redbox>Paris because it is in France</span>.</li>',
|
||||
'<li>If the answer is a <strong>number</strong>, please enter the number directly (e.g., <span class=whitebox>13</span>). <strong>Do not</strong> spell out the number (e.g., <span class=redbox>thirteen</span>), except when it is a <a target=_blank href="https://en.wikipedia.org/wiki/Thirteen_%282003_film%29">name</a>.</li>',
|
||||
'<li>If the answer is a <strong>list</strong> or has <strong>multiple parts</strong>, use multiple lines with <strong>one answer per line</strong> (e.g., <span class=whitebox>Paris</span> <kbd>Enter</kbd> <span class=whitebox>Madrid</span>, not <span class=redbox>Paris and Madrid</span>).</li>',
|
||||
'</ul>',
|
||||
'<h2>Tools</h2>',
|
||||
'<ul>',
|
||||
'<li><strong><kbd>Ctrl</kbd> (or <kbd>⌘</kbd>) + click on a table cell:</strong> copy cell content to the answer box.</li>',
|
||||
'<li><strong>Hold <kbd>Shift</kbd> + paint on table cells:</strong> highlight cells and display statistics in the box above.</li>',
|
||||
'<li><strong>Click on table header:</strong> sort the table by that column. (Note: does not work well on dates.)</li>',
|
||||
'</ul>',
|
||||
'<p><em>If you have any problems or suggestions, please leave a comment or <a href="mailto:percyliangmturk@gmail.com?subject=%5BMTurk%5D%20table-alter" id=email>contact us</a>.</em></p>',
|
||||
'</div></div>',
|
||||
'<div id=warning>Preview Mode<br>Please accept the HIT to start working.</div>',
|
||||
'<div id=answerForm><div id=questionWrapper> </div>',
|
||||
'<div id=navbar class=centerize>',
|
||||
'<input type=hidden id=assignmentId name=assignmentId>',
|
||||
'<button id=prevButton type=button>Previous</button> <span id=qNum> </span> <button id=nextButton type=button>Next</button> ',
|
||||
'<button id=submitButton type=button class=hidden disabled><strong>Submit</strong></button>',
|
||||
'</div>',
|
||||
'<div id=usedReasons class=hidden><p>Used reasons (<kbd>Ctrl</kbd> + click to copy):</p><ul></ul></div>',
|
||||
'<p id=incompleteWarning class="red hidden"><strong>Error:</strong> Please answer all questions (Missing question <span id=missingQuestions> </span>)</p>',
|
||||
'</div></div>'].join(''));
|
||||
|
||||
var questionDivs = [], tableDivs = [], formElements = [], currentQuestion = 0;
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Form submission
|
||||
|
||||
function clean(text, cleanNewlines) {
|
||||
var answer = [];
|
||||
text.split(/\n/).forEach(function (x) {
|
||||
x = x.replace(/\s+/g, ' ').replace(/^ | $/g, '');
|
||||
if (x.length) answer.push(x);
|
||||
});
|
||||
return answer.join(cleanNewlines ? " " : "\n");
|
||||
}
|
||||
|
||||
var iWillSubmit = false;
|
||||
|
||||
$('#submitButton').click(function () {
|
||||
iWillSubmit = true;
|
||||
$('#submitButton').closest('form').submit();
|
||||
});
|
||||
|
||||
$('#submitButton').keyup(function(e) {
|
||||
var keyCode = e.keyCode || e.which;
|
||||
if (keyCode === 13 || keyCode === 32) {
|
||||
iWillSubmit = true;
|
||||
$('#submitButton').closest('form').submit();
|
||||
}
|
||||
});
|
||||
|
||||
$('#submitButton').closest('form').submit(function() {
|
||||
if (!iWillSubmit) return false;
|
||||
// Check if all text fields are filled.
|
||||
var missingQuestions = [];
|
||||
for (var i = 0; i < questionDivs.length; i++) {
|
||||
var a = formElements[i].a, b = formElements[i].b;
|
||||
a.val(clean(a.val()));
|
||||
b.val(clean(b.val(), true));
|
||||
if ((!a.prop('disabled') && !a.val().length) ||
|
||||
(!b.prop('disabled') && !b.val().length)) {
|
||||
missingQuestions.push(i + 1);
|
||||
}
|
||||
}
|
||||
if (!missingQuestions.length) {
|
||||
$('#incompleteWarning').hide();
|
||||
$('#submitButton').prop('disabled', true);
|
||||
return true;
|
||||
} else {
|
||||
$('#missingQuestions').text(missingQuestions.join(', '));
|
||||
$('#incompleteWarning').show();
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
function gatherUsedReasons() {
|
||||
$('#usedReasons').hide();
|
||||
$('#usedReasons ul').empty();
|
||||
for (var i = 0; i < questionDivs.length; i++) {
|
||||
var b = formElements[i].b;
|
||||
if (!b.prop('disabled')) {
|
||||
b.val(clean(b.val(), true));
|
||||
var reason = b.val(), used = false;
|
||||
if (!reason.length) continue;
|
||||
$('#usedReasons ul li').each(function (_, x) {
|
||||
if ($(x).text() === reason) { used = true; return false; }
|
||||
});
|
||||
if (!used) {
|
||||
$('#usedReasons').show();
|
||||
$('<li>').text(reason).appendTo('#usedReasons ul')
|
||||
.click(function (event) {
|
||||
if (!event.ctrlKey) return;
|
||||
formElements[currentQuestion].cb.click();
|
||||
var reasonBox = formElements[currentQuestion].b;
|
||||
reasonBox.val(clean(reasonBox.val() + ' ' + $(this).text()));
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Instructions
|
||||
|
||||
function toggleInstructions(state) {
|
||||
$("#showingInstruction").toggle(!!state);
|
||||
$("#hidingInstruction").toggle(!state);
|
||||
}
|
||||
|
||||
$("#showInstructionButton").click(function () {
|
||||
toggleInstructions(true);
|
||||
});
|
||||
|
||||
$("#hideInstructionButton").click(function () {
|
||||
toggleInstructions(false);
|
||||
});
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Navigation
|
||||
|
||||
function goToQuestion(questionNum) {
|
||||
$('.question, .table').hide();
|
||||
questionDivs[questionNum].show();
|
||||
tableDivs[questionNum].show();
|
||||
$('html, body').scrollTop(0);
|
||||
$('#qNum').text('' + (questionNum + 1) + ' / ' + questionDivs.length);
|
||||
$('#prevButton').prop('disabled', questionNum === 0);
|
||||
$('#nextButton').toggle(questionNum !== questionDivs.length - 1);
|
||||
$('#submitButton').toggle(questionNum === questionDivs.length - 1);
|
||||
currentQuestion = questionNum;
|
||||
$('#qNum').trigger('questionChanged');
|
||||
var a = formElements[currentQuestion].a, b = formElements[currentQuestion].b;
|
||||
if (!a.prop('disabled')) a.focus();
|
||||
if (!b.prop('disabled')) b.focus();
|
||||
gatherUsedReasons();
|
||||
}
|
||||
|
||||
$('#prevButton').click(function () {
|
||||
goToQuestion(currentQuestion - 1);
|
||||
});
|
||||
|
||||
$('#nextButton').click(function () {
|
||||
goToQuestion(currentQuestion + 1);
|
||||
});
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Load data
|
||||
|
||||
function getNumber(x) {
|
||||
var result = /^[0-9,.]*/.exec(x);
|
||||
if (result === null) return null;
|
||||
result = result[0].replace(/,/g, '');
|
||||
if (result === '') return null;
|
||||
return +result;
|
||||
}
|
||||
|
||||
function constructTable(tableData, sprite, tableInfoBox, answerRadio, answerBox, actionVal) {
|
||||
var table = $('<table>'), header, originalRowOrder = [], currentRow, i, j;
|
||||
// First row is a header
|
||||
for (i = 0; i < tableData.length; i++) {
|
||||
currentRow = $('<tr>');
|
||||
for (j = 0; j < tableData[0].length; j++) {
|
||||
$(i === 0 ? '<th>' : '<td>')
|
||||
.html(sprite[tableData[i][j]])
|
||||
.appendTo(currentRow);
|
||||
}
|
||||
if (i == 0)
|
||||
header = currentRow;
|
||||
originalRowOrder.push(currentRow);
|
||||
}
|
||||
// Restore order
|
||||
function restoreOrder() {
|
||||
table.empty().append(header);
|
||||
originalRowOrder.forEach(function (x) {
|
||||
table.append(x);
|
||||
});
|
||||
header.children().removeClass('selected');
|
||||
restoreRowOrderButton.prop('disabled', true);
|
||||
sortInfoSpan.text('(original order)');
|
||||
actionVal.val(actionVal.val() + 'O;');
|
||||
}
|
||||
var restoreRowOrderButton = $('<button>').text('Restore row order')
|
||||
.appendTo(tableInfoBox).click(restoreOrder);
|
||||
tableInfoBox.append(' ');
|
||||
var sortInfoSpan = $('<span>').appendTo(tableInfoBox);
|
||||
restoreOrder();
|
||||
// Sort
|
||||
table.on("click", "th", function (event) {
|
||||
// Stable Sort
|
||||
j = $(this).index();
|
||||
var sortedIndices = [], rows = table.find('tr').get();
|
||||
for (i = 1; i < rows.length; i++) sortedIndices.push(i);
|
||||
sortedIndices.sort(function (a, b) {
|
||||
var textA = table.find('tr:eq(' + a + ') td:eq(' + j + ')').text();
|
||||
var textB = table.find('tr:eq(' + b + ') td:eq(' + j + ')').text();
|
||||
var numA = getNumber(textA), numB = getNumber(textB);
|
||||
// Put numbers first
|
||||
if (numA !== null) {
|
||||
if (numB === null) return -1;
|
||||
if (numA != numB) return numA - numB;
|
||||
} else {
|
||||
if (numB !== null) return 1;
|
||||
}
|
||||
return textA > textB ? 1 : textA < textB ? -1 : a - b;
|
||||
});
|
||||
table.empty().append(header);
|
||||
sortedIndices.forEach(function (a) {
|
||||
table.append(rows[a]);
|
||||
});
|
||||
header.children().removeClass('selected');
|
||||
$(this).addClass('selected');
|
||||
restoreRowOrderButton.prop('disabled', false);
|
||||
sortInfoSpan.text('(ordered by column ' + (j+1) + ')');
|
||||
actionVal.val(actionVal.val() + 'S' + j + ';');
|
||||
});
|
||||
// Selection
|
||||
tableInfoBox.append(' ');
|
||||
var computeInfoText = $('<span>').appendTo(tableInfoBox);
|
||||
function computeSelected() {
|
||||
var count = 0; sum = 0, numbers = [];
|
||||
table.find('td.selected').get().forEach(function (x) {
|
||||
count++;
|
||||
var num = getNumber($(x).text().replace(/^[^0-9]*/, ''));
|
||||
if (num !== null) {
|
||||
numbers.push(num);
|
||||
sum += num;
|
||||
}
|
||||
});
|
||||
if (count == 0)
|
||||
computeInfoText.text('').attr('title', null);
|
||||
else {
|
||||
computeInfoText.text('count = ' + count + ' | sum = ' + sum + ' (hover for details)');
|
||||
computeInfoText.attr('title', 'sum = ' + (numbers.length ? numbers.join(' + ') : '(none)'));
|
||||
}
|
||||
}
|
||||
var shiftSelecting = false;
|
||||
table.on("mousedown", "td", function (event) {
|
||||
if (event.ctrlKey) {
|
||||
answerRadio.click();
|
||||
var originalAnswer = answerBox.val();
|
||||
if (originalAnswer.length && originalAnswer[originalAnswer.length - 1] !== '\n')
|
||||
originalAnswer += '\n';
|
||||
answerBox.val(originalAnswer + clean($(this).text(), true) + '\n');
|
||||
actionVal.val(actionVal.val() + 'C' + $(this).parent().index() + '-' + $(this).index() + ';');
|
||||
event.preventDefault();
|
||||
} else if (event.shiftKey) {
|
||||
$(this).toggleClass('selected');
|
||||
shiftSelecting = $(this).hasClass('selected');
|
||||
computeSelected();
|
||||
actionVal.val(actionVal.val() + 'X' + $(this).parent().index() + '-' + $(this).index() + ';');
|
||||
event.preventDefault();
|
||||
}
|
||||
});
|
||||
table.on("mouseenter", "td", function (event) {
|
||||
if (event.shiftKey && event.buttons > 0) {
|
||||
$(this).toggleClass('selected', shiftSelecting);
|
||||
computeSelected();
|
||||
actionVal.val(actionVal.val() + 'X' + $(this).parent().index() + '-' + $(this).index() + ';');
|
||||
event.preventDefault();
|
||||
}
|
||||
});
|
||||
return table;
|
||||
}
|
||||
|
||||
function newHidden(name, index) {
|
||||
return $('<input type=hidden>').attr('name', name + index);
|
||||
}
|
||||
|
||||
function populateDatum(datum, index) {
|
||||
var answerRadio, answerBox, reasonRadio, reasonBox, actionVal;
|
||||
questionDivs.push(
|
||||
$('<div class=question>').appendTo('#questionWrapper').hide()
|
||||
.append($('<fieldset>')
|
||||
.append(newHidden('id', index).val(datum.id))
|
||||
.append(newHidden('title', index).val(datum.metadata.title))
|
||||
.append(newHidden('url', index).val(datum.metadata.url))
|
||||
.append(newHidden('tableid', index).val(datum.metadata.tableIndex))
|
||||
.append(newHidden('question', index).val(datum.question))
|
||||
.append(newHidden('alterIndex', index).val(datum.alterIndex))
|
||||
.append(newHidden('otherAlterIndices', index).val(datum.retained.otherAlterIndices))
|
||||
.append(newHidden('score', index).val(datum.retained.score))
|
||||
.append(actionVal = newHidden('actions', index).val(''))
|
||||
.append($('<h2>Question</h2>'))
|
||||
.append($('<div class=questionText>').text(datum.question).addClass('q' + index))
|
||||
.append($('<p>')
|
||||
.append($('<label>').attr('for', 'c' + index + 'a')
|
||||
.append(answerRadio = $('<input type=radio value=a checked=checked disabled>')
|
||||
.attr('name', 'c' + index).attr('id', 'c' + index + 'a')
|
||||
.change(function () {
|
||||
reasonBox.prop('disabled', true);
|
||||
answerBox.prop('disabled', false).focus();
|
||||
}))
|
||||
.append('The answer is:')))
|
||||
.append(answerBox = $('<textarea class=answerBox disabled rows=4>').attr('name', 'a' + index).attr('id', 'a' + index))
|
||||
.append($('<p>')
|
||||
.append($('<label>').attr('for', 'c' + index + 'b')
|
||||
.append(reasonRadio = $('<input type=radio value=b disabled>')
|
||||
.attr('name', 'c' + index).attr('id', 'c' + index + 'b')
|
||||
.change(function () {
|
||||
answerBox.prop('disabled', true);
|
||||
reasonBox.prop('disabled', false).focus();
|
||||
}))
|
||||
.append('Cannot be answered because:')))
|
||||
.append(reasonBox = $('<textarea class=reasonBox disabled rows=2>').attr('name', 'b' + index).attr('id', 'b' + index))
|
||||
));
|
||||
var tableInfoBox = $('<div class=table-info>');
|
||||
var table = constructTable(datum.table, datum.sprite,
|
||||
tableInfoBox, answerRadio, answerBox, actionVal).addClass('q' + index);
|
||||
tableDivs.push(
|
||||
$('<div class=table>').hide()
|
||||
.append(tableInfoBox)
|
||||
.append($('<div class=table-lower>')
|
||||
.append($('<h2>').append('Page Title: ')
|
||||
.append($('<a target=_blank>').attr('href', datum.metadata.url)
|
||||
.text(datum.metadata.title)))
|
||||
.append(table)).appendTo('#webpage'));
|
||||
formElements.push(
|
||||
{'ca': answerRadio, 'a': answerBox, 'cb': reasonRadio, 'b': reasonBox});
|
||||
}
|
||||
|
||||
function populateData(data) {
|
||||
$('#webpage, #questionWrapper').empty();
|
||||
data.forEach(populateDatum);
|
||||
// Show / Hide instructions
|
||||
$("#hideInstructionButton").text("Hide").prop('disabled', false);
|
||||
toggleInstructions(false);
|
||||
if (docCookies.getItem("percyTableSeenInstruction") != VERSION) {
|
||||
toggleInstructions(true);
|
||||
docCookies.setItem("percyTableSeenInstruction", VERSION);
|
||||
}
|
||||
if ($('#assignmentId').val() === 'ASSIGNMENT_ID_NOT_AVAILABLE') {
|
||||
toggleInstructions(true);
|
||||
} else {
|
||||
$('.question textarea.answerBox, .question input, #submitButton').prop('disabled', false);
|
||||
$('#warning').hide();
|
||||
}
|
||||
goToQuestion(0);
|
||||
}
|
||||
|
||||
populateData(QUESTIONS);
|
||||
|
||||
});
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
$(function(){function e(e,t){var n=[]
|
||||
return e.split(/\n/).forEach(function(e){e=e.replace(/\s+/g," ").replace(/^ | $/g,""),e.length&&n.push(e)}),n.join(t?" ":"\n")}function t(){$("#usedReasons").hide(),$("#usedReasons ul").empty()
|
||||
for(var t=0;t<u.length;t++){var n=c[t].b
|
||||
if(!n.prop("disabled")){n.val(e(n.val(),!0))
|
||||
var a=n.val(),s=!1
|
||||
if(!a.length)continue
|
||||
$("#usedReasons ul li").each(function(e,t){return $(t).text()===a?(s=!0,!1):void 0}),s||($("#usedReasons").show(),$("<li>").text(a).appendTo("#usedReasons ul").click(function(t){if(t.ctrlKey){c[h].cb.click()
|
||||
var n=c[h].b
|
||||
n.val(e(n.val()+" "+$(this).text()))}}))}}}function n(e){$("#showingInstruction").toggle(!!e),$("#hidingInstruction").toggle(!e)}function a(e){$(".question, .table").hide(),u[e].show(),p[e].show(),$("html, body").scrollTop(0),$("#qNum").text(""+(e+1)+" / "+u.length),$("#prevButton").prop("disabled",0===e),$("#nextButton").toggle(e!==u.length-1),$("#submitButton").toggle(e===u.length-1),h=e,$("#qNum").trigger("questionChanged")
|
||||
var n=c[h].a,a=c[h].b
|
||||
n.prop("disabled")||n.focus(),a.prop("disabled")||a.focus(),t()}function s(e){var t=/^[0-9,.]*/.exec(e)
|
||||
return null===t?null:(t=t[0].replace(/,/g,""),""===t?null:+t)}function i(t,n,a,i,o,r){function l(){b.empty().append(u),g.forEach(function(e){b.append(e)}),u.children().removeClass("selected"),m.prop("disabled",!0),v.text("(original order)"),r.val(r.val()+"O;")}function d(){var e=0
|
||||
sum=0,numbers=[],b.find("td.selected").get().forEach(function(t){e++
|
||||
var n=s($(t).text().replace(/^[^0-9]*/,""))
|
||||
null!==n&&(numbers.push(n),sum+=n)}),0==e?f.text("").attr("title",null):(f.text("count = "+e+" | sum = "+sum+" (hover for details)"),f.attr("title","sum = "+(numbers.length?numbers.join(" + "):"(none)")))}var u,p,c,h,b=$("<table>"),g=[]
|
||||
for(c=0;c<t.length;c++){for(p=$("<tr>"),h=0;h<t[0].length;h++)$(0===c?"<th>":"<td>").html(n[t[c][h]]).appendTo(p)
|
||||
0==c&&(u=p),g.push(p)}var m=$("<button>").text("Restore row order").appendTo(a).click(l)
|
||||
a.append(" ")
|
||||
var v=$("<span>").appendTo(a)
|
||||
l(),b.on("click","th",function(e){h=$(this).index()
|
||||
var t=[],n=b.find("tr").get()
|
||||
for(c=1;c<n.length;c++)t.push(c)
|
||||
t.sort(function(e,t){var n=b.find("tr:eq("+e+") td:eq("+h+")").text(),a=b.find("tr:eq("+t+") td:eq("+h+")").text(),i=s(n),o=s(a)
|
||||
if(null!==i){if(null===o)return-1
|
||||
if(i!=o)return i-o}else if(null!==o)return 1
|
||||
return n>a?1:a>n?-1:e-t}),b.empty().append(u),t.forEach(function(e){b.append(n[e])}),u.children().removeClass("selected"),$(this).addClass("selected"),m.prop("disabled",!1),v.text("(ordered by column "+(h+1)+")"),r.val(r.val()+"S"+h+";")}),a.append(" ")
|
||||
var f=$("<span>").appendTo(a),w=!1
|
||||
return b.on("mousedown","td",function(t){if(t.ctrlKey){i.click()
|
||||
var n=o.val()
|
||||
n.length&&"\n"!==n[n.length-1]&&(n+="\n"),o.val(n+e($(this).text(),!0)+"\n"),r.val(r.val()+"C"+$(this).parent().index()+"-"+$(this).index()+";"),t.preventDefault()}else t.shiftKey&&($(this).toggleClass("selected"),w=$(this).hasClass("selected"),d(),r.val(r.val()+"X"+$(this).parent().index()+"-"+$(this).index()+";"),t.preventDefault())}),b.on("mouseenter","td",function(e){e.shiftKey&&e.buttons>0&&($(this).toggleClass("selected",w),d(),r.val(r.val()+"X"+$(this).parent().index()+"-"+$(this).index()+";"),e.preventDefault())}),b}function o(e,t){return $("<input type=hidden>").attr("name",e+t)}function r(e,t){var n,a,s,r,l
|
||||
u.push($("<div class=question>").appendTo("#questionWrapper").hide().append($("<fieldset>").append(o("id",t).val(e.id)).append(o("title",t).val(e.metadata.title)).append(o("url",t).val(e.metadata.url)).append(o("tableid",t).val(e.metadata.tableIndex)).append(o("question",t).val(e.question)).append(o("alterIndex",t).val(e.alterIndex)).append(o("otherAlterIndices",t).val(e.retained.otherAlterIndices)).append(o("score",t).val(e.retained.score)).append(l=o("actions",t).val("")).append($("<h2>Question</h2>")).append($("<div class=questionText>").text(e.question).addClass("q"+t)).append($("<p>").append($("<label>").attr("for","c"+t+"a").append(n=$("<input type=radio value=a checked=checked disabled>").attr("name","c"+t).attr("id","c"+t+"a").change(function(){r.prop("disabled",!0),a.prop("disabled",!1).focus()})).append("The answer is:"))).append(a=$("<textarea class=answerBox disabled rows=4>").attr("name","a"+t).attr("id","a"+t)).append($("<p>").append($("<label>").attr("for","c"+t+"b").append(s=$("<input type=radio value=b disabled>").attr("name","c"+t).attr("id","c"+t+"b").change(function(){a.prop("disabled",!0),r.prop("disabled",!1).focus()})).append("Cannot be answered because:"))).append(r=$("<textarea class=reasonBox disabled rows=2>").attr("name","b"+t).attr("id","b"+t))))
|
||||
var d=$("<div class=table-info>"),h=i(e.table,e.sprite,d,n,a,l).addClass("q"+t)
|
||||
p.push($("<div class=table>").hide().append(d).append($("<div class=table-lower>").append($("<h2>").append("Page Title: ").append($("<a target=_blank>").attr("href",e.metadata.url).text(e.metadata.title))).append(h)).appendTo("#webpage")),c.push({ca:n,a:a,cb:s,b:r})}function l(e){$("#webpage, #questionWrapper").empty(),e.forEach(r),$("#hideInstructionButton").text("Hide").prop("disabled",!1),n(!1),docCookies.getItem("percyTableSeenInstruction")!=d&&(n(!0),docCookies.setItem("percyTableSeenInstruction",d)),"ASSIGNMENT_ID_NOT_AVAILABLE"===$("#assignmentId").val()?n(!0):($(".question textarea.answerBox, .question input, #submitButton").prop("disabled",!1),$("#warning").hide()),a(0)}var d="5.2"
|
||||
$("#content").html(["<div id=webpage class=column> </div>","<div id=wrapper class=column>","<div id=instruction>","<div id=hidingInstruction class=hidden><button type=button class=floatButton id=showInstructionButton>Show Instructions</button></div>","<div id=showingInstruction>","<h2>Task: Answer the question using the information from the table.</h2>","<button type=button class=floatButton id=hideInstructionButton disabled>Loading ...</button>","<ul>","<li>The tables were generated by shuffling the cells in some original table. Assume that each table presents data from a fictional world.</li>","<li>As such, some questions may sound a little strange (e.g., asking for <em>the city</em> but many cities fit the criteria). Please answer according to the intent of the question.</li>","<li>If the question really cannot be answered (e.g., when the main entity in the question is missing from the table, or when the table does not make sense), briefly describe the reason in the <strong>second box</strong> and leave the answer box blank.</li>","<li>We <strong>manually</strong> review the answers and check some of the answers with the gold standard. We use answers from other workers to detect suspicious submissions, but will not reject your work as long as it correctly follows the instructions.</li>","</ul>","<h2>Answer Format</h2>","<ul>","<li>If the answer appears in the table, please <strong>copy and paste</strong> to avoid typos.</li>","<li><strong>Do not</strong> add punctuation marks or elaboration to the answers: use <span class=whitebox>Paris</span> instead of <span class=redbox>Paris.</span> or <span class=redbox>Paris because it is in France</span>.</li>",'<li>If the answer is a <strong>number</strong>, please enter the number directly (e.g., <span class=whitebox>13</span>). <strong>Do not</strong> spell out the number (e.g., <span class=redbox>thirteen</span>), except when it is a <a target=_blank href="https://en.wikipedia.org/wiki/Thirteen_%282003_film%29">name</a>.</li>',"<li>If the answer is a <strong>list</strong> or has <strong>multiple parts</strong>, use multiple lines with <strong>one answer per line</strong> (e.g., <span class=whitebox>Paris</span> <kbd>Enter</kbd> <span class=whitebox>Madrid</span>, not <span class=redbox>Paris and Madrid</span>).</li>","</ul>","<h2>Tools</h2>","<ul>","<li><strong><kbd>Ctrl</kbd> (or <kbd>⌘</kbd>) + click on a table cell:</strong> copy cell content to the answer box.</li>","<li><strong>Hold <kbd>Shift</kbd> + paint on table cells:</strong> highlight cells and display statistics in the box above.</li>","<li><strong>Click on table header:</strong> sort the table by that column. (Note: does not work well on dates.)</li>","</ul>",'<p><em>If you have any problems or suggestions, please leave a comment or <a href="mailto:percyliangmturk@gmail.com?subject=%5BMTurk%5D%20table-alter" id=email>contact us</a>.</em></p>',"</div></div>","<div id=warning>Preview Mode<br>Please accept the HIT to start working.</div>","<div id=answerForm><div id=questionWrapper> </div>","<div id=navbar class=centerize>","<input type=hidden id=assignmentId name=assignmentId>","<button id=prevButton type=button>Previous</button> <span id=qNum> </span> <button id=nextButton type=button>Next</button> ","<button id=submitButton type=button class=hidden disabled><strong>Submit</strong></button>","</div>","<div id=usedReasons class=hidden><p>Used reasons (<kbd>Ctrl</kbd> + click to copy):</p><ul></ul></div>",'<p id=incompleteWarning class="red hidden"><strong>Error:</strong> Please answer all questions (Missing question <span id=missingQuestions> </span>)</p>',"</div></div>"].join(""))
|
||||
var u=[],p=[],c=[],h=0,b=!1
|
||||
$("#submitButton").click(function(){b=!0,$("#submitButton").closest("form").submit()}),$("#submitButton").keyup(function(e){var t=e.keyCode||e.which;(13===t||32===t)&&(b=!0,$("#submitButton").closest("form").submit())}),$("#submitButton").closest("form").submit(function(){if(!b)return!1
|
||||
for(var t=[],n=0;n<u.length;n++){var a=c[n].a,s=c[n].b
|
||||
a.val(e(a.val())),s.val(e(s.val(),!0)),(!a.prop("disabled")&&!a.val().length||!s.prop("disabled")&&!s.val().length)&&t.push(n+1)}return t.length?($("#missingQuestions").text(t.join(", ")),$("#incompleteWarning").show(),!1):($("#incompleteWarning").hide(),$("#submitButton").prop("disabled",!0),!0)}),$("#showInstructionButton").click(function(){n(!0)}),$("#hideInstructionButton").click(function(){n(!1)}),$("#prevButton").click(function(){a(h-1)}),$("#nextButton").click(function(){a(h+1)}),l(QUESTIONS)})
|
||||
|
|
@ -0,0 +1,254 @@
|
|||
.centerize { text-align: center; margin: 0 auto; }
|
||||
.hidden { display: none; }
|
||||
.red { color: red; }
|
||||
.btn-primary { max-width: 7em; }
|
||||
|
||||
.floatButton {
|
||||
position: absolute;
|
||||
top: 1em;
|
||||
right: 1em;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||||
}
|
||||
|
||||
.column {
|
||||
left: 0;
|
||||
top: 0;
|
||||
}
|
||||
|
||||
kbd {
|
||||
border: 1px solid #aaa;
|
||||
border-radius: 0.2em;
|
||||
box-shadow: 0.1em 0.2em 0.2em #ddd;
|
||||
background-color: #f9f9f9;
|
||||
padding: 0 0.5em;
|
||||
font-family: inherit;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
/*
|
||||
button {
|
||||
box-shadow: inset 0px 1px 0px 0px #5AF;
|
||||
background-color: #06A;
|
||||
border-radius: 3px;
|
||||
border: 1px solid #147;
|
||||
cursor: pointer;
|
||||
color: #FFF;
|
||||
padding: 3px 10px;
|
||||
}
|
||||
button:hover {
|
||||
background-color: #059;
|
||||
}
|
||||
*/
|
||||
|
||||
/*****************************************************************/
|
||||
|
||||
#wrapper {
|
||||
position: fixed;
|
||||
width: 350px;
|
||||
background-color: #F3F3FF;
|
||||
border-bottom-right-radius: 1em;
|
||||
}
|
||||
|
||||
#wrapper h1 {
|
||||
font-size: 120%;
|
||||
margin: 0.5em auto;
|
||||
}
|
||||
|
||||
#wrapper h2 {
|
||||
font-size: 100%;
|
||||
margin: 0.8em auto 0.5em;
|
||||
}
|
||||
|
||||
#instruction, #warning, #answerForm, #footer, #navbar {
|
||||
margin: 0.8em;
|
||||
}
|
||||
|
||||
#warning {
|
||||
margin-top: 3em;
|
||||
margin-bottom: -1em;
|
||||
padding: 0.8em;
|
||||
text-align: center;
|
||||
color: #D22;
|
||||
background-color: #EDD;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
#showingInstruction {
|
||||
font-size: 85%;
|
||||
padding: 0.2em 1em;
|
||||
width: 600px;
|
||||
position: absolute;
|
||||
margin-left: 100px;
|
||||
border-radius: 1em;
|
||||
background-color: #ded;
|
||||
border: 2px solid #bcb;
|
||||
}
|
||||
|
||||
#showingInstruction h2 {
|
||||
margin: 0.5em 0 0.3em;
|
||||
font-size: 108%;
|
||||
}
|
||||
|
||||
#showingInstruction p {
|
||||
margin: 0.3em 0;
|
||||
}
|
||||
|
||||
#showingInstruction ul {
|
||||
margin-top: 0.3em;
|
||||
margin-bottom: 0.3em;
|
||||
padding-left: 1em;
|
||||
}
|
||||
|
||||
#showingInstruction ul li {
|
||||
margin-bottom: 0.3em;
|
||||
line-height: 1.4em;
|
||||
}
|
||||
|
||||
#showingInstruction .button {
|
||||
border: 1px solid gray;
|
||||
border-radius: 0.2em;
|
||||
border-style: outset;
|
||||
background-color: #DDD;
|
||||
padding: 0 0.5em;
|
||||
white-space: nowrap;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
#showingInstruction .whitebox {
|
||||
background-color: #F9F9FF;
|
||||
border: 1px solid gray;
|
||||
padding: 0 3px;
|
||||
}
|
||||
|
||||
#showingInstruction .redbox {
|
||||
background-color: #F9DDDD;
|
||||
border: 1px solid gray;
|
||||
padding: 0 3px;
|
||||
}
|
||||
|
||||
#qNum {
|
||||
margin: 0 0.5em;
|
||||
}
|
||||
|
||||
#navbar button {
|
||||
width: 7em;
|
||||
height: 2.5em;
|
||||
}
|
||||
|
||||
#navbar button:disabled {
|
||||
color: #aaa;
|
||||
}
|
||||
|
||||
.questionText {
|
||||
padding: 0.5em;
|
||||
text-align: center;
|
||||
border: 1px solid #CCC;
|
||||
background-color: #F9F9FF;
|
||||
}
|
||||
|
||||
#answerForm fieldset {
|
||||
border: none;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
#answerForm textarea {
|
||||
width: 96%;
|
||||
margin-top: 0.5em;
|
||||
}
|
||||
|
||||
#answerForm button.s {
|
||||
padding: 2px 0;
|
||||
}
|
||||
|
||||
#answerForm p {
|
||||
margin: 0.5em 0 0.3em;
|
||||
}
|
||||
|
||||
#answerForm input[type=radio] {
|
||||
margin-right: 0.5em;
|
||||
margin-top: -2px;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
#answerForm label {
|
||||
cursor: pointer;
|
||||
display: inline-block;
|
||||
width: 98%;
|
||||
}
|
||||
|
||||
#answerForm ul {
|
||||
margin: 0.5em 0;
|
||||
padding-left: 1.5em;
|
||||
}
|
||||
|
||||
/*****************************************************************/
|
||||
|
||||
#webpage {
|
||||
position: absolute;
|
||||
margin-left: 350px;
|
||||
padding: 1em;
|
||||
}
|
||||
|
||||
#webpage .table {
|
||||
position: relative;
|
||||
}
|
||||
|
||||
#webpage .table-info {
|
||||
position: fixed;
|
||||
top: 1em;
|
||||
border: 1px solid #BCB;
|
||||
background-color: #EFE;
|
||||
padding: 0.4em 0.8em;
|
||||
}
|
||||
#webpage .table-info * {
|
||||
vertical-align: text-bottom;
|
||||
}
|
||||
|
||||
#webpage .table-lower {
|
||||
margin-top: 3em;
|
||||
}
|
||||
|
||||
#webpage h2 {
|
||||
font-size: 108%;
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
#webpage a {
|
||||
text-decoration: none;
|
||||
color: #0645AD;
|
||||
}
|
||||
#webpage a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
#webpage table {
|
||||
border-collapse: collapse;
|
||||
}
|
||||
|
||||
#webpage table, #webpage th, #webpage td {
|
||||
border: 1px solid #DDD;
|
||||
padding: 2px 8px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
#webpage th {
|
||||
background-color: #DDD;
|
||||
}
|
||||
#webpage th.selected {
|
||||
background-color: #BBB;
|
||||
}
|
||||
#webpage td {
|
||||
background-color: #FFF;
|
||||
}
|
||||
#webpage td.selected {
|
||||
background-color: #F6B900;
|
||||
}
|
||||
#webpage table td:hover {
|
||||
background-color: #CCD;
|
||||
}
|
||||
#webpage table td.selected:hover {
|
||||
background-color: #E87D00;
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
.centerize{text-align:center;margin:0 auto}.hidden{display:none}.red{color:red}.btn-primary{max-width:7em}.floatButton{position:absolute;top:1em;right:1em}body{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif}.column{left:0;top:0}kbd{border:1px solid #aaa;border-radius:0.2em;box-shadow:0.1em 0.2em 0.2em #ddd;background-color:#f9f9f9;padding:0 0.5em;font-family:inherit;font-size:0.9em}#wrapper{position:fixed;width:350px;background-color:#F3F3FF;border-bottom-right-radius:1em}#wrapper h1{font-size:120%;margin:0.5em auto}#wrapper h2{font-size:100%;margin:0.8em auto 0.5em}#instruction,#warning,#answerForm,#footer,#navbar{margin:0.8em}#warning{margin-top:3em;margin-bottom:-1em;padding:0.8em;text-align:center;color:#D22;background-color:#EDD;font-weight:bold}#showingInstruction{font-size:85%;padding:0.2em 1em;width:600px;position:absolute;margin-left:100px;border-radius:1em;background-color:#ded;border:2px solid #bcb}#showingInstruction h2{margin:0.5em 0 0.3em;font-size:108%}#showingInstruction p{margin:0.3em 0}#showingInstruction ul{margin-top:0.3em;margin-bottom:0.3em;padding-left:1em}#showingInstruction ul li{margin-bottom:0.3em;line-height:1.4em}#showingInstruction .button{border:1px solid gray;border-radius:0.2em;border-style:outset;background-color:#DDD;padding:0 0.5em;white-space:nowrap;font-size:0.9em}#showingInstruction .whitebox{background-color:#F9F9FF;border:1px solid gray;padding:0 3px}#showingInstruction .redbox{background-color:#F9DDDD;border:1px solid gray;padding:0 3px}#qNum{margin:0 0.5em}#navbar button{width:7em;height:2.5em}#navbar button:disabled{color:#aaa}.questionText{padding:0.5em;text-align:center;border:1px solid #CCC;background-color:#F9F9FF}#answerForm fieldset{border:none;padding:0}#answerForm textarea{width:96%;margin-top:0.5em}#answerForm button.s{padding:2px 0}#answerForm p{margin:0.5em 0 0.3em}#answerForm input[type=radio]{margin-right:0.5em;margin-top:-2px;vertical-align:middle}#answerForm label{cursor:pointer;display:inline-block;width:98%}#answerForm ul{margin:0.5em 0;padding-left:1.5em}#webpage{position:absolute;margin-left:350px;padding:1em}#webpage .table{position:relative}#webpage .table-info{position:fixed;top:1em;border:1px solid #BCB;background-color:#EFE;padding:0.4em 0.8em}#webpage .table-info *{vertical-align:text-bottom}#webpage .table-lower{margin-top:3em}#webpage h2{font-size:108%;margin-top:0}#webpage a{text-decoration:none;color:#0645AD}#webpage a:hover{text-decoration:underline}#webpage table{border-collapse:collapse}#webpage table,#webpage th,#webpage td{border:1px solid #DDD;padding:2px 8px;cursor:pointer}#webpage th{background-color:#DDD}#webpage th.selected{background-color:#BBB}#webpage td{background-color:#FFF}#webpage td.selected{background-color:#F6B900}#webpage table td:hover{background-color:#CCD}#webpage table td.selected:hover{background-color:#E87D00}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
minify main.js
|
||||
minify style.css
|
||||
rsync -avzuL data main.min.js cookies.min.js jquery.min.js style.min.css jamie:~/www/mturk/tables-alter-v5/ $@
|
||||
rsync -avzuL index.min.html jamie:~/www/mturk/tables-alter-v5/index.html $@
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
- create a zip file
|
||||
- update `README.md`
|
||||
- add `README.md`, `t/data/` and `t/csv/`
|
||||
- rsync the html data from the cluster and then add to zip file
|
||||
- run the following:
|
||||
|
||||
./run @mode=tables @class=ann-data @data=none -Dataset.inPaths train,t/data/training.examples dev,t/data/pristine-seen-tables.examples test,t/data/pristine-unseen-tables.examples -splitdevfromtrain false
|
||||
./run @mode=tables @class=ann-table
|
||||
|
||||
and then put the results in `annotated`
|
||||
|
|
@ -0,0 +1,169 @@
|
|||
WikiTableQuestions Dataset
|
||||
==========================
|
||||
Version 0.3 (January 14, 2016)
|
||||
|
||||
Introduction
|
||||
------------
|
||||
|
||||
The WikiTableQuestions dataset is for the task of question answering on
|
||||
semi-structured HTML tables as presented in the paper:
|
||||
|
||||
> Panupong Pasupat, Percy Liang.
|
||||
> Compositional Semantic Parsing on Semi-Structured Tables
|
||||
> Association for Computational Linguistics (ACL), 2015.
|
||||
|
||||
About TSV Format
|
||||
----------------
|
||||
|
||||
Many files in this dataset are stored as tab-separated values (TSV).
|
||||
They use the following special constructs:
|
||||
|
||||
- List items are separated by `|` (e.g., `when|was|taylor|swift|born|?`).
|
||||
|
||||
- The following characters are escaped:
|
||||
newline (=> `\n`), backslash (`\` => `\\`), and pipe (`|` => `\p`)
|
||||
Note that pipes become `\p` so that doing `x.split('|')` will work.
|
||||
|
||||
- Consecutive whitespaces (except newlines) are collapsed into a single space.
|
||||
|
||||
|
||||
Questions and Answers
|
||||
---------------------
|
||||
|
||||
The `data/` directory contains the questions, answers, and the ID of the tables
|
||||
that the questions are asking about.
|
||||
|
||||
Each portion of the dataset is stored as a TSV file where each line contains
|
||||
one example.
|
||||
|
||||
**Field descriptions:**
|
||||
- id: unique ID of the example
|
||||
- utterance: the question in its original format
|
||||
- context: the table used to answer the question
|
||||
- targetValue: the answer, possibly a `|`-separated list
|
||||
|
||||
**Dataset Splits:** We split 22033 examples into multiple sets:
|
||||
|
||||
- `training`:
|
||||
Training data (14152 examples)
|
||||
|
||||
- `pristine-unseen-tables`:
|
||||
Test data -- the tables are *not seen* in training data (4344 examples)
|
||||
|
||||
- `pristine-seen-tables`:
|
||||
Additional data where the tables are *seen* in training data. (3537 examples)
|
||||
(Initially intended to be used as development data, this portion of the
|
||||
dataset has not been used in any experiment in the paper.)
|
||||
|
||||
- `random-split-*`:
|
||||
For development, we split `training.tsv` into random 80-20 splits.
|
||||
Within each split, tables in the training data (`random-split-seed-*-train`)
|
||||
and the test data (`random-split-seed-*-test`) are disjoint.
|
||||
|
||||
For our ACL 2015 paper:
|
||||
|
||||
- In development set experiments:
|
||||
we trained on `random-split-seed-{1,2,3}-train`
|
||||
and tested on `random-split-seed-{1,2,3}-test`, respectively.
|
||||
|
||||
- In test set experiments:
|
||||
we trained on `training` and tested on `pristine-unseen-tables`.
|
||||
|
||||
**Supplementary Files:**
|
||||
|
||||
- `*.examples` files:
|
||||
The LispTree format of the dataset is used internally in our
|
||||
[SEMPRE](http://nlp.stanford.edu/software/sempre/) code base.
|
||||
The `*.examples` files contain the same information as the TSV files.
|
||||
|
||||
Tables
|
||||
------
|
||||
|
||||
The `csv/` directory contains the extracted tables, while the `page/` directory
|
||||
contains the raw HTML data of the whole web page.
|
||||
|
||||
**Table Formats:**
|
||||
|
||||
- `csv/xxx-csv/yyy.csv`:
|
||||
Comma-separated table (The first row is treated as the column header)
|
||||
The escaped characters include:
|
||||
double quote (`"` => `\"`) and backslash (`\` => `\\`).
|
||||
Newlines are represented as quoted line breaks.
|
||||
|
||||
- `csv/xxx-csv/yyy.tsv`:
|
||||
Tab-separated table. The TSV escapes explained at the beginning are used.
|
||||
|
||||
- `csv/xxx-csv/yyy.table`:
|
||||
Human-readable column-aligned table. Some information was loss during
|
||||
data conversion, so this format should not be used as an input.
|
||||
|
||||
- `csv/xxx-csv/yyy.html`:
|
||||
Formatted HTML of just the table
|
||||
|
||||
- `page/xxx-page/yyy.html`:
|
||||
Raw HTML of the whole web page
|
||||
|
||||
- `page/xxx-page/yyy.json`:
|
||||
Metadata including the URL, the page title, and the index of the chosen table.
|
||||
(Only tables with the `wikitable` class are considered.)
|
||||
|
||||
The conversion from HTML to CSV and TSV was done using `table-to-tsv.py`.
|
||||
Its dependency is in the `weblib/` directory.
|
||||
|
||||
CoreNLP Annotated Files
|
||||
-----------------------
|
||||
Questions and tables are annotated using CoreNLP 3.5.2.
|
||||
The annotation is not perfect (e.g., it cannot detect the date "13-12-1989"),
|
||||
but it is usually good enough.
|
||||
|
||||
- `annotated/data/*.annotated`:
|
||||
Annotated questions. Each line contains one example.
|
||||
|
||||
Field descriptions:
|
||||
- id: unique ID of the example
|
||||
- utterance: the question in its original format
|
||||
- context: the table used to answer the question
|
||||
- targetValue: the answer, possibly a `|`-separated list
|
||||
- tokens: the question, tokenized
|
||||
- lemmaTokens: the question, tokenized and lemmatized
|
||||
- posTags: the part of speech tag of each token
|
||||
- nerTags: the name entity tag of each token
|
||||
- nerValues: if the NER tag is numerical or temporal, the value of that
|
||||
NER span will be listed here
|
||||
- targetCanon: canonical form of the answers where numbers and dates
|
||||
are converted into normalized values
|
||||
|
||||
- `annotated/xxx-annotated/yyy.annotated`:
|
||||
Tab-separated file containing the CoreNLP annotation of each table cell.
|
||||
Each line represents one table cell.
|
||||
|
||||
Mandatory fields:
|
||||
- row: row index (-1 is the header row)
|
||||
- col: column index
|
||||
- id: unique ID of the cell.
|
||||
- Each header cell gets a unique ID even when the contents are identical
|
||||
- Non-header cells get the same ID if they have exactly the same content
|
||||
- content: the cell text (images and hidden spans are removed)
|
||||
- tokens: the cell text, tokenized
|
||||
- lemmaTokens: the cell text, tokenized and lemmatized
|
||||
- posTags: the part of speech tag of each token
|
||||
- nerTags: the name entity tag of each token
|
||||
- nerValues: if the NER tag is numerical or temporal, the value of that
|
||||
NER span will be listed here
|
||||
|
||||
The following fields are optional:
|
||||
- number: interpretation as a number (for multiple numbers, the first
|
||||
number is extracted)
|
||||
- date: interpretation as a date
|
||||
- num2: the second number in the cell (useful for scores like `1-2`)
|
||||
- list: interpretation as a list of items
|
||||
|
||||
Header cells do not have these optional fields.
|
||||
|
||||
Version History
|
||||
---------------
|
||||
|
||||
0.3 - Repaired table headers / Added raw HTML tables and CoreNLP annotated data
|
||||
0.2 - Initial release
|
||||
|
||||
For questions and comments, please contact Ice Pasupat <ppasupat@cs.stanford.edu>
|
||||
|
|
@ -1 +0,0 @@
|
|||
../../lib/data/tables/csv/
|
||||
|
|
@ -0,0 +1,589 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Perform experiments on the dumped denotations."""
|
||||
|
||||
import sys, os, shutil, re, argparse, json, gzip
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict, Counter
|
||||
|
||||
def clean(x):
|
||||
# Remove actual string from names
|
||||
x = x.decode('utf8').replace(ur'\"', '')
|
||||
x = re.sub(ur'\(name fb:(?:cell|part)\.([^ )]*)\)', r'(name \1)', x)
|
||||
x = re.sub(ur'\(name fb:(?:cell|part)\.([^ )]*) [^")]*\)', r'(name \1)', x)
|
||||
x = re.sub(ur'\(name fb:(?:cell|part)\.([^ )]*) "[^"]*"\)', r'(name \1)', x)
|
||||
return x
|
||||
|
||||
def read_denotation_file(ex_id, fin):
|
||||
"""Return n, k, d, denotations, grid"""
|
||||
n, k, d = [int(x) for x in fin.readline().split()]
|
||||
denotations = []
|
||||
for i in xrange(d):
|
||||
denotations.append(clean(fin.readline()[:-1]))
|
||||
if not denotations:
|
||||
print >> sys.stderr, 'Warning: no denotations - ', ex_id
|
||||
elif denotations[0] == '(list)':
|
||||
print >> sys.stderr, 'Warning: denotations[0] == (list) - ', ex_id
|
||||
denotations.append(None)
|
||||
grid = []
|
||||
for i in xrange(n):
|
||||
row = [denotations[int(x)] for x in fin.readline()[:-1].split()]
|
||||
grid.append(row)
|
||||
denotations.pop()
|
||||
return n, k, d, denotations, grid
|
||||
|
||||
################################################
|
||||
|
||||
# Count number of equivalence classes
|
||||
def process1(ex_id, fin):
|
||||
n, k, d, denotations, grid = read_denotation_file(ex_id, fin)
|
||||
equiv_classes = {0: set(), 5: set(), 10: set(),
|
||||
30: set(), 100: set(), 300: set()}
|
||||
for row in grid:
|
||||
row = tuple(row)
|
||||
for num_tables, value in equiv_classes.items():
|
||||
value.add(row[:num_tables+1])
|
||||
|
||||
fields = [ex_id, n, k, d]
|
||||
for key, value in sorted(equiv_classes.items()):
|
||||
fields.append(len(value))
|
||||
print '\t'.join(str(x) for x in fields)
|
||||
sys.stdout.flush()
|
||||
|
||||
# Find out how many are split from main class when going from 30 --> 300
|
||||
def process2(ex_id, fin):
|
||||
n, k, d, denotations, grid = read_denotation_file(ex_id, fin)
|
||||
thirties = defaultdict(Counter)
|
||||
for row in grid:
|
||||
row = tuple(row)
|
||||
thirties[row[:31]][row[:301]] += 1
|
||||
#thirties[row[:6]][row[:31]] += 1
|
||||
|
||||
cores = splits = 0
|
||||
for key, value in thirties.items():
|
||||
core = value.most_common(1)[0][1]
|
||||
cores += core
|
||||
splits += (sum(value.values()) - core)
|
||||
|
||||
fields = [ex_id, n, k, d, len(thirties), cores, splits]
|
||||
print '\t'.join(str(x) for x in fields)
|
||||
sys.stdout.flush()
|
||||
|
||||
# Find out the splitters from 30 --> 300
|
||||
def process3(ex_id, fin):
|
||||
n, k, d, denotations, grid = read_denotation_file(ex_id, fin)
|
||||
thirties = defaultdict(lambda: defaultdict(list))
|
||||
counts = Counter()
|
||||
for i, row in enumerate(grid):
|
||||
row = tuple(row)
|
||||
thirties[row[:31]][row].append(i)
|
||||
counts[row[:31]] += 1
|
||||
|
||||
cores = []
|
||||
splits = []
|
||||
if counts:
|
||||
first_thirty_one, count = counts.most_common(1)[0]
|
||||
full = thirties[first_thirty_one]
|
||||
# full = map from full denotation tuple to deriv indices
|
||||
core = max(len(x) for x in full.values())
|
||||
for indices in full.values():
|
||||
if len(indices) == core:
|
||||
cores.extend(indices)
|
||||
else:
|
||||
splits.extend(indices)
|
||||
if not splits:
|
||||
cores = []
|
||||
else:
|
||||
splits = sorted(splits)
|
||||
|
||||
fields = [ex_id, n, k, d, len(thirties), len(cores), len(splits),
|
||||
' '.join(str(x) for x in cores[:30]),
|
||||
' '.join(str(x) for x in splits[:30])]
|
||||
print '\t'.join(str(x) for x in fields)
|
||||
sys.stdout.flush()
|
||||
|
||||
# Find out more stuff about annotation
|
||||
def process5(ex_id, fin, ann_fin, ret_data, size_path, formula_path):
|
||||
n, k, d, denotations, grid = read_denotation_file(ex_id, fin)
|
||||
grid = [row[:31] for row in grid]
|
||||
|
||||
# Equiv class to list of indices
|
||||
equiv_classes = defaultdict(list)
|
||||
for i, row in enumerate(grid):
|
||||
equiv_classes[tuple(row)].append(i)
|
||||
|
||||
# Read annotation
|
||||
annotations = []
|
||||
for line in ann_fin:
|
||||
annotations.append(clean(line[:-1]))
|
||||
annotations = tuple(annotations[:31])
|
||||
|
||||
# Read size
|
||||
if size_path:
|
||||
with open(size_path) as fin:
|
||||
sizes = [int(x) for x in fin]
|
||||
else:
|
||||
sizes = [0] * n
|
||||
sizes_counter = Counter(sizes)
|
||||
|
||||
# Read formulas
|
||||
if formula_path:
|
||||
with (open if formula_path.endswith('.gz') else gzip.open)(formula_path) as fin:
|
||||
formulas = fin.readlines()
|
||||
else:
|
||||
formulas = [''] * n
|
||||
|
||||
score, chosen_entropy = ret_data
|
||||
chosen_random = range(len(chosen_entropy))
|
||||
|
||||
fields = [ex_id, n, k, d, len(equiv_classes),
|
||||
'{:.3}'.format(score),
|
||||
' '.join(str(x) for x in chosen_entropy),
|
||||
' '.join('{}:{}'.format(u, sizes_counter[u])
|
||||
for u in xrange(1, 9))]
|
||||
if annotations[0] == 'null':
|
||||
fields.append('no')
|
||||
else:
|
||||
fields.append('yes')
|
||||
# Count the number of matches
|
||||
matched = len(equiv_classes[annotations])
|
||||
fields.append(matched)
|
||||
# Chosen equiv classes based on chosen tables
|
||||
def count_on_chosen(chosen):
|
||||
chosen_annotations = tuple(annotations[x] for x in chosen)
|
||||
num_classes_matched = 0
|
||||
num_derivs_matched = []
|
||||
derivs_matched = []
|
||||
for key, value in equiv_classes.items():
|
||||
chosen_key = tuple(key[x] for x in chosen)
|
||||
if chosen_key == chosen_annotations:
|
||||
num_classes_matched += 1
|
||||
num_derivs_matched.append(len(value))
|
||||
derivs_matched.extend(value)
|
||||
fields.extend([num_classes_matched, sum(num_derivs_matched)])
|
||||
fields.append(list(reversed(sorted(num_derivs_matched))))
|
||||
derivs_matched.sort()
|
||||
matched_sizes = [sizes[x] for x in derivs_matched]
|
||||
matched_sizes_counter = Counter(matched_sizes)
|
||||
fields.append(' '.join('{}:{}'.format(u, matched_sizes_counter[u])
|
||||
for u in xrange(1, 9)))
|
||||
# Is the formula with the lowest size the annotated one?
|
||||
if matched_sizes:
|
||||
min_size, min_index = min((u,i) for (u,i)
|
||||
in zip(matched_sizes, derivs_matched))
|
||||
fields.append([min_size, min_index, formulas[min_index]])
|
||||
else:
|
||||
fields.append('')
|
||||
count_on_chosen(chosen_entropy)
|
||||
count_on_chosen(chosen_random)
|
||||
|
||||
print '\t'.join(str(x) for x in fields)
|
||||
sys.stdout.flush()
|
||||
|
||||
# Find out stuff about annotation; turked version
|
||||
def process6(ex_id, fin, ann_fin, size_path, formula_path, exec_dir):
|
||||
# These are only for computing equivalence classes
|
||||
n, k, d, denotations, grid = read_denotation_file(ex_id, fin)
|
||||
grid = [row[:31] for row in grid]
|
||||
|
||||
# Equiv class to list of indices
|
||||
equiv_classes = defaultdict(list)
|
||||
for i, row in enumerate(grid):
|
||||
equiv_classes[tuple(row)].append(i)
|
||||
deriv_to_rep = {}
|
||||
for equiv_class in equiv_classes.values():
|
||||
rep = min(equiv_class)
|
||||
for x in equiv_class:
|
||||
deriv_to_rep[x] = rep
|
||||
|
||||
# Read annotation
|
||||
annotations = []
|
||||
for line in ann_fin:
|
||||
annotations.append(clean(line[:-1]))
|
||||
annotations = tuple(annotations[:31])
|
||||
|
||||
# Read size
|
||||
if size_path:
|
||||
with open(size_path) as fin:
|
||||
sizes = [int(x) for x in fin]
|
||||
else:
|
||||
sizes = [0] * n
|
||||
sizes_counter = Counter(sizes)
|
||||
|
||||
# Read formulas
|
||||
if formula_path:
|
||||
with (open if formula_path.endswith('.gz') else gzip.open)(formula_path) as fin:
|
||||
formulas = fin.readlines()
|
||||
else:
|
||||
formulas = [''] * n
|
||||
|
||||
fields = [ex_id, n, k, d, len(equiv_classes),
|
||||
' '.join('{}:{}'.format(u, sizes_counter[u])
|
||||
for u in xrange(1, 9))]
|
||||
correct_class = None
|
||||
if annotations[0] == 'null':
|
||||
fields.append('no')
|
||||
fields.append('')
|
||||
else:
|
||||
fields.append('yes')
|
||||
# Count the number of matches
|
||||
matched = len(equiv_classes[annotations])
|
||||
fields.append(matched)
|
||||
if matched > 0:
|
||||
correct_class = min(equiv_classes[annotations])
|
||||
fields.append(correct_class)
|
||||
|
||||
def get_flag(x):
|
||||
if x == '(number 42)' or x == '(name Ao)':
|
||||
return 'G'
|
||||
elif x == '(number 13)' or x == '(name Ax)':
|
||||
return 'F'
|
||||
elif x == '(number 666)':
|
||||
return 'B'
|
||||
elif x == '(number 777)' or x == '(name Bo)':
|
||||
return 'g'
|
||||
elif x == '(number 999)' or x == '(name Bx)':
|
||||
return 'f'
|
||||
elif x == '(number 88)' or x == '(name X)':
|
||||
return 'X'
|
||||
else:
|
||||
return None
|
||||
|
||||
# Read Turk data
|
||||
if os.path.exists(os.path.join(exec_dir, 'denotations')):
|
||||
prefix = ''
|
||||
else:
|
||||
prefix = 'actual-'
|
||||
with gzip.open(os.path.join(exec_dir, prefix + 'denotations', 'nt-{}.gz'.format(ex_id))) as fin:
|
||||
_, _, _, _, turk_grid = read_denotation_file(ex_id, fin)
|
||||
with gzip.open(os.path.join(exec_dir, 'check-denotations', 'nt-{}.gz'.format(ex_id))) as fin:
|
||||
_, _, _, _, check_grid = read_denotation_file(ex_id, fin)
|
||||
check_grid = [[get_flag(x) for x in y] for y in check_grid]
|
||||
with gzip.open(os.path.join(exec_dir, prefix + 'annotated-denotations', 'nt-{}.gz'.format(ex_id))) as fin:
|
||||
anno_grid = [x[:-1] for x in fin]
|
||||
with gzip.open(os.path.join(exec_dir, 'check-annotated-denotations', 'nt-{}.gz'.format(ex_id))) as fin:
|
||||
check_anno_grid = [x[:-1] for x in fin]
|
||||
check_anno_grid = [get_flag(x) for x in check_anno_grid]
|
||||
|
||||
# Used indices
|
||||
if check_grid:
|
||||
used_indices = [i for (i,x) in enumerate(check_grid[0]) if x is not None]
|
||||
flags = [check_grid[0][i] for i in used_indices]
|
||||
else:
|
||||
used_indices = [i for (i,x) in enumerate(anno_grid) if x is not None]
|
||||
flags = [check_anno_grid[i] for i in used_indices]
|
||||
fields.append(used_indices)
|
||||
fields.append(''.join([{'G': 'A', 'F': 'A', 'B': 'B',
|
||||
'g': 'B', 'f': 'B', 'X': 'X', None: '?'}[x] for x in flags]))
|
||||
fields.append(check_anno_grid[0])
|
||||
|
||||
# Scheme is a function that takes in a check vector (only for used indices)
|
||||
# and return True (matched) or False (not matched).
|
||||
#
|
||||
# For each possible scheme, compute
|
||||
# - number of classes matched
|
||||
# - number of formulas matched
|
||||
# - class sizes
|
||||
# - whether the annotated formula matches turked data
|
||||
# - whether the "correct" class is among the matched classes
|
||||
def check_scheme(scheme):
|
||||
hello = []
|
||||
for check_vector in check_grid:
|
||||
hello.append(scheme([check_vector[x] for x in used_indices]))
|
||||
hello_anno = scheme([check_anno_grid[x] for x in used_indices])
|
||||
# num classes matched
|
||||
matched_derivs = [i for (i,x) in enumerate(hello) if x]
|
||||
matched_classes = set(deriv_to_rep[x] for x in matched_derivs)
|
||||
fields.append(len(matched_classes))
|
||||
fields.append(len(matched_derivs))
|
||||
class_sizes = [(len(equiv_classes[tuple(grid[x])]), x) for x in matched_classes]
|
||||
#class_sizes.sort(key=lambda x: -x[0])
|
||||
#fields.append([x[1] for x in class_sizes])
|
||||
#fields.append([x[0] for x in class_sizes])
|
||||
# manually annotated formula
|
||||
fields.append(hello_anno)
|
||||
# correct is among?
|
||||
fields.append(correct_class in matched_classes)
|
||||
|
||||
def no_big_f(vector):
|
||||
return all(x != 'F' for x in vector)
|
||||
check_scheme(no_big_f)
|
||||
def no_f_ever(vector):
|
||||
return all((x != 'f' and x != 'F') for x in vector)
|
||||
check_scheme(no_f_ever)
|
||||
def at_most_one_big_f(vector):
|
||||
return sum([x == 'F' for x in vector], 0) <= 1
|
||||
check_scheme(at_most_one_big_f)
|
||||
|
||||
print '\t'.join(str(x) for x in fields)
|
||||
sys.stdout.flush()
|
||||
|
||||
# Dump equivalence classes to a file
|
||||
def process7(ex_id, fin, ann_fin, size_path, formula_path, exec_dir, outdir):
|
||||
n, k, d, denotations, grid = read_denotation_file(ex_id, fin)
|
||||
denotations_set = set(denotations)
|
||||
#grid = [row[:31] for row in grid]
|
||||
|
||||
# Read annotation
|
||||
annotations = []
|
||||
for line in ann_fin:
|
||||
annotations.append(clean(line[:-1]))
|
||||
for x in set(annotations):
|
||||
if x not in denotations_set and n > 0:
|
||||
print >> sys.stderr, 'WTF:', ex_id, n, k, d, x
|
||||
annotations = tuple(annotations)
|
||||
print annotations
|
||||
#annotations = annotations[:31]
|
||||
|
||||
# Read size
|
||||
if size_path:
|
||||
with open(size_path) as fin:
|
||||
sizes = [int(x) for x in fin]
|
||||
else:
|
||||
sizes = [0] * n
|
||||
sizes_counter = Counter(sizes)
|
||||
|
||||
# Read formulas
|
||||
if formula_path:
|
||||
with (open if formula_path.endswith('.gz') else gzip.open)(formula_path) as fin:
|
||||
formulas = [x.strip() for x in fin.readlines()]
|
||||
else:
|
||||
formulas = [''] * n
|
||||
|
||||
fields = [ex_id, n, k, d]
|
||||
if annotations[0] == 'null':
|
||||
fields.append('no')
|
||||
else:
|
||||
fields.append('yes')
|
||||
|
||||
def get_flag(x):
|
||||
if x == '(name Ao)':
|
||||
return 'G'
|
||||
elif x == '(name Ax)':
|
||||
return 'F'
|
||||
elif x == '(name Bo)':
|
||||
return 'g'
|
||||
elif x == '(name Bx)':
|
||||
return 'f'
|
||||
elif x == '(name X)':
|
||||
return 'X'
|
||||
else:
|
||||
assert x is None or x == 'null'
|
||||
return None
|
||||
|
||||
# Read Turk data
|
||||
with gzip.open(os.path.join(exec_dir, 'actual-denotations', 'nt-{}.gz'.format(ex_id))) as fin:
|
||||
_, _, _, _, turk_grid = read_denotation_file(ex_id, fin)
|
||||
with gzip.open(os.path.join(exec_dir, 'check-denotations', 'nt-{}.gz'.format(ex_id))) as fin:
|
||||
_, _, _, _, check_grid = read_denotation_file(ex_id, fin)
|
||||
check_grid = [[get_flag(x) for x in y] for y in check_grid]
|
||||
with gzip.open(os.path.join(exec_dir, 'actual-annotated-denotations', 'nt-{}.gz'.format(ex_id))) as fin:
|
||||
anno_grid = [x[:-1] for x in fin]
|
||||
with gzip.open(os.path.join(exec_dir, 'check-annotated-denotations', 'nt-{}.gz'.format(ex_id))) as fin:
|
||||
check_anno_grid = [x[:-1] for x in fin]
|
||||
check_anno_grid = [get_flag(x) for x in check_anno_grid]
|
||||
|
||||
# Dump equivalence classes
|
||||
# Also dump whether the class
|
||||
# - Matches the annotated formula
|
||||
# - Matches the turked data
|
||||
deno_tuple_to_indices = defaultdict(list)
|
||||
for i, row in enumerate(grid):
|
||||
row = tuple(row)
|
||||
deno_tuple_to_indices[row].append(i)
|
||||
classes = []
|
||||
flag_counts_classes = [0, 0, 0, 0]
|
||||
flag_counts_formulas = [0, 0, 0, 0]
|
||||
for deno_tuple, indices in deno_tuple_to_indices.iteritems():
|
||||
real = all(x != 'F' for x in check_grid[indices[0]])
|
||||
ideal = (deno_tuple == annotations)
|
||||
classes.append([real, ideal, indices])
|
||||
flag_counts_classes[real * 2 + ideal] += 1
|
||||
flag_counts_formulas[real * 2 + ideal] += len(indices)
|
||||
classes.sort(key=lambda x: (-x[0]-x[1], -x[0], -len(x[2])))
|
||||
fields.append(len(classes))
|
||||
fields.extend(flag_counts_classes)
|
||||
fields.extend(flag_counts_formulas)
|
||||
with open(os.path.join(outdir, 'nt-' + str(ex_id)), 'w') as fout:
|
||||
print >> fout, '{} classes ({} formulas) {{'.format(len(classes), n)
|
||||
for i, (real, ideal, indices) in enumerate(classes):
|
||||
equiv_class = [(sizes[i], formulas[i]) for i in indices]
|
||||
equiv_class.sort(key=lambda x: x[1])
|
||||
print >> fout, ' CLASS {} ({} formulas){}{} {{'.format(
|
||||
i, len(equiv_class), '[IDEAL]' if ideal else '',
|
||||
'[TURK]' if real else '')
|
||||
for s, f in equiv_class:
|
||||
print >> fout, ' ', s, f
|
||||
print >> fout, ' }'
|
||||
print >> fout, '}'
|
||||
|
||||
print '\t'.join(str(x) for x in fields)
|
||||
sys.stdout.flush()
|
||||
|
||||
# Dump matching formulas to a file
|
||||
def process8(ex_id, fin, formula_path, exec_dir, outdir):
|
||||
# Actually fin is not needed ...
|
||||
n, k, d = [int(x) for x in fin.readline().split()]
|
||||
fields = [ex_id, n, k, d]
|
||||
|
||||
def get_flag(x):
|
||||
if x == '(name Ao)':
|
||||
return 'G'
|
||||
elif x == '(name Ax)':
|
||||
return 'F'
|
||||
elif x == '(name Bo)':
|
||||
return 'g'
|
||||
elif x == '(name Bx)':
|
||||
return 'f'
|
||||
elif x == '(name X)':
|
||||
return 'X'
|
||||
else:
|
||||
assert x is None or x == 'null'
|
||||
return None
|
||||
|
||||
# Read Turk data
|
||||
with gzip.open(os.path.join(exec_dir, 'check-denotations', 'nt-{}.gz'.format(ex_id))) as fin:
|
||||
_, _, _, _, check_grid = read_denotation_file(ex_id, fin)
|
||||
check_grid = [[get_flag(x) for x in y] for y in check_grid]
|
||||
if n > 0:
|
||||
used_indices = [i for (i,x) in enumerate(check_grid[0]) if x is not None]
|
||||
flags = [check_grid[0][i] for i in used_indices]
|
||||
else:
|
||||
used_indices = []
|
||||
flags = []
|
||||
fields.append(''.join([{'G': 'A', 'F': 'A', 'B': 'B',
|
||||
'g': 'B', 'f': 'B', 'X': 'X', None: '?'}[x] for x in flags]))
|
||||
|
||||
# Get the "correct" derivations
|
||||
corrects = []
|
||||
cur_i = 0
|
||||
with gzip.open(formula_path) as fin:
|
||||
for line in fin:
|
||||
line = line.strip()
|
||||
if line.startswith('(derivation (formula'):
|
||||
if all(check_grid[cur_i][i] != 'F' for i in used_indices):
|
||||
corrects.append(line)
|
||||
cur_i += 1
|
||||
fields.append(len(corrects))
|
||||
|
||||
# Dump matching formulas
|
||||
with gzip.open(os.path.join(outdir, 'nt-{}.gz'.format(ex_id)), 'w') as fout:
|
||||
for correct in corrects:
|
||||
print >> fout, correct
|
||||
|
||||
print '\t'.join(str(x) for x in fields)
|
||||
sys.stdout.flush()
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-s', '--step', type=int, default=1)
|
||||
parser.add_argument('-a', '--annotation-dir',
|
||||
help='directory to search for dumped annotated formula denotations')
|
||||
parser.add_argument('-r', '--retained-tables',
|
||||
help='path to retained-tables file')
|
||||
parser.add_argument('-z', '--sizes-dir',
|
||||
help='path to sizes directory')
|
||||
parser.add_argument('-f', '--formulas-dir',
|
||||
help='path to formulas directory')
|
||||
parser.add_argument('-e', '--exec-dir',
|
||||
help='path to exec dir with denotations, annotated-denotations, check-*')
|
||||
parser.add_argument('-R', '--range', nargs=2, type=int,
|
||||
help='range of example ids')
|
||||
parser.add_argument('denotation_dir',
|
||||
help='directory to search for dumped denotations')
|
||||
parser.add_argument('-o', '--outdir',
|
||||
help='output directory')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Read denotation directories
|
||||
ex_id_to_path = {}
|
||||
directory = args.denotation_dir
|
||||
for filename in os.listdir(directory):
|
||||
match = re.match(r'nt-(\d+)\.gz', filename)
|
||||
assert match, filename
|
||||
ex_id_to_path[int(match.group(1))] = os.path.join(directory, filename)
|
||||
print >> sys.stderr, 'There are', len(ex_id_to_path), 'denotation files'
|
||||
|
||||
ex_id_to_ann_path = {}
|
||||
if args.annotation_dir:
|
||||
directory = args.annotation_dir
|
||||
for filename in os.listdir(directory):
|
||||
match = re.match(r'nt-(\d+)\.gz', filename)
|
||||
assert match, filename
|
||||
ex_id_to_ann_path[int(match.group(1))] = os.path.join(directory, filename)
|
||||
print >> sys.stderr, 'There are', len(ex_id_to_ann_path), 'annotated denotation files'
|
||||
|
||||
ex_id_to_size_path = {}
|
||||
if args.sizes_dir:
|
||||
directory = args.sizes_dir
|
||||
for filename in os.listdir(directory):
|
||||
match = re.match(r'nt-(\d+)', filename)
|
||||
assert match, filename
|
||||
ex_id_to_size_path[int(match.group(1))] = os.path.join(directory, filename)
|
||||
print >> sys.stderr, 'There are', len(ex_id_to_size_path), 'annotated sizes files'
|
||||
|
||||
ex_id_to_formula_path = {}
|
||||
if args.formulas_dir:
|
||||
directory = args.formulas_dir
|
||||
for filename in os.listdir(directory):
|
||||
match = re.match(r'nt-(\d+)(\.gz)?', filename)
|
||||
assert match, filename
|
||||
ex_id_to_formula_path[int(match.group(1))] = os.path.join(directory, filename)
|
||||
print >> sys.stderr, 'There are', len(ex_id_to_formula_path), 'annotated formulas files'
|
||||
|
||||
if args.retained_tables:
|
||||
ret_data = {}
|
||||
with open(args.retained_tables) as fin:
|
||||
for line in fin:
|
||||
line = line[:-1].split('\t')
|
||||
ex_id = int(line[0][3:])
|
||||
score = float(line[1])
|
||||
chosen = [int(x) for x in line[2].split()]
|
||||
ret_data[ex_id] = (score, chosen)
|
||||
|
||||
print >> sys.stderr, 'STEP:', args.step
|
||||
for i, ex_id in enumerate(sorted(ex_id_to_path)):
|
||||
if i % 20 == 0:
|
||||
print >> sys.stderr, 'Processing', i
|
||||
if args.range and not (args.range[0] <= i <= args.range[1]):
|
||||
continue
|
||||
with gzip.open(ex_id_to_path[ex_id]) as fin:
|
||||
if args.step == 1:
|
||||
process1(i, fin)
|
||||
elif args.step == 2:
|
||||
process2(i, fin)
|
||||
elif args.step == 3:
|
||||
process3(i, fin)
|
||||
elif args.step == 5:
|
||||
with gzip.open(ex_id_to_ann_path[ex_id]) as ann_fin:
|
||||
process5(i, fin, ann_fin, ret_data[ex_id],
|
||||
ex_id_to_size_path.get(i),
|
||||
ex_id_to_formula_path.get(i))
|
||||
elif args.step == 6:
|
||||
with gzip.open(ex_id_to_ann_path[ex_id]) as ann_fin:
|
||||
process6(i, fin, ann_fin,
|
||||
ex_id_to_size_path.get(i),
|
||||
ex_id_to_formula_path.get(i),
|
||||
args.exec_dir)
|
||||
elif args.step == 7:
|
||||
assert args.outdir
|
||||
if not os.path.exists(args.outdir):
|
||||
os.makedirs(args.outdir)
|
||||
with gzip.open(ex_id_to_ann_path[ex_id]) as ann_fin:
|
||||
process7(i, fin, ann_fin,
|
||||
ex_id_to_size_path.get(i),
|
||||
ex_id_to_formula_path.get(i),
|
||||
args.exec_dir, args.outdir)
|
||||
elif args.step == 8:
|
||||
assert args.outdir
|
||||
if not os.path.exists(args.outdir):
|
||||
os.makedirs(args.outdir)
|
||||
process8(i, fin,
|
||||
ex_id_to_formula_path.get(i),
|
||||
args.exec_dir, args.outdir)
|
||||
print >> sys.stderr, 'DONE!'
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('id_range', type=int, nargs=2)
|
||||
parser.add_argument('-d', '--basedir', default='altered')
|
||||
parser.add_argument('-o', '--outfile')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.outfile and os.path.exists(args.outfile):
|
||||
if raw_input('Delete {}? (y/N): '.format(args.outfile))[0:].lower() != 'y':
|
||||
return
|
||||
os.remove(args.outfile)
|
||||
|
||||
if args.outfile:
|
||||
fout = open(args.outfile, 'w', 'utf8')
|
||||
else:
|
||||
fout = sys.stdout
|
||||
|
||||
num_ex = num_files = 0
|
||||
for i in xrange(args.id_range[0], args.id_range[1] + 1):
|
||||
print >> sys.stderr, 'Processing', i
|
||||
basedir = os.path.join(args.basedir, 'nt-' + str(i))
|
||||
j = 1
|
||||
while True:
|
||||
infile = os.path.join(basedir, str(j) + '.tsv')
|
||||
if not os.path.exists(infile):
|
||||
break
|
||||
with open(infile, 'r', 'utf8') as fin:
|
||||
data = fin.readlines()
|
||||
print >> fout, '{}\t{}\t{}'.format('nt-' + str(i), j, len(data))
|
||||
for line in data:
|
||||
fout.write(line)
|
||||
j += 1
|
||||
num_files += 1
|
||||
num_ex += 1
|
||||
|
||||
if args.outfile:
|
||||
fout.close()
|
||||
print >> sys.stderr, 'Written {} files ({} examples)'.format(num_files, num_ex)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Fix up the turk-info.tsv file.
|
||||
- Add numClasses field
|
||||
- Swap numClassesMatched and numDerivsMatched
|
||||
"""
|
||||
|
||||
import sys, os, shutil, re, argparse, json, gzip
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('turk_info_file',
|
||||
help='path to turk info file')
|
||||
parser.add_argument('denotation_dirs', nargs='+',
|
||||
help='directories to search for dumped denotations')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Read denotation directories
|
||||
ex_id_to_path = {}
|
||||
for directory in args.denotation_dirs:
|
||||
for filename in os.listdir(directory):
|
||||
match = re.match(r'(nt-\d+)\.gz', filename)
|
||||
assert match, filename
|
||||
ex_id_to_path[match.group(1)] = os.path.join(directory, filename)
|
||||
print >> sys.stderr, 'There are', len(ex_id_to_path), 'denotation files'
|
||||
|
||||
def compute_num_classes(ex_id):
|
||||
with gzip.open(ex_id_to_path[ex_id]) as fin:
|
||||
n, k, d = [int(x) for x in fin.readline().split()]
|
||||
for i in xrange(d):
|
||||
fin.readline()
|
||||
unique_denotation_tuples = set()
|
||||
for j in xrange(n):
|
||||
unique_denotation_tuples.add(fin.readline())
|
||||
return len(unique_denotation_tuples)
|
||||
|
||||
data = []
|
||||
with open(args.turk_info_file, 'r', 'utf8') as fin:
|
||||
header = fin.readline().rstrip('\n').split('\t')
|
||||
header.insert(1, 'numClasses')
|
||||
header[8], header[9] = header[9], header[8]
|
||||
print u'\t'.join(header)
|
||||
for i, line in enumerate(fin):
|
||||
if i % 500 == 0:
|
||||
print >> sys.stderr, 'Processing', i
|
||||
line = line.rstrip('\n').split('\t')
|
||||
line.insert(1, str(compute_num_classes(line[0])))
|
||||
line[8], line[9] = line[9], line[8]
|
||||
print u'\t'.join(line)
|
||||
print >> sys.stderr, 'DONE!'
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Compute the sizes of equivalence classes.
|
||||
"""
|
||||
|
||||
import sys, os, shutil, re, argparse, json, gzip
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict, Counter
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('denotation_dirs', nargs='+',
|
||||
help='directories to search for dumped denotations')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Read denotation directories
|
||||
ex_id_to_path = {}
|
||||
for directory in args.denotation_dirs:
|
||||
for filename in os.listdir(directory):
|
||||
match = re.match(r'nt-(\d+)\.gz', filename)
|
||||
assert match, filename
|
||||
ex_id_to_path[int(match.group(1))] = os.path.join(directory, filename)
|
||||
print >> sys.stderr, 'There are', len(ex_id_to_path), 'denotation files'
|
||||
|
||||
def compute_num_classes(ex_id):
|
||||
with gzip.open(ex_id_to_path[ex_id]) as fin:
|
||||
n, k, d = [int(x) for x in fin.readline().split()]
|
||||
for i in xrange(d):
|
||||
fin.readline()
|
||||
counter = Counter()
|
||||
for j in xrange(n):
|
||||
counter[fin.readline()] += 1
|
||||
return counter
|
||||
|
||||
for i, ex_id in enumerate(sorted(ex_id_to_path)):
|
||||
if i % 500 == 0:
|
||||
print >> sys.stderr, 'Processing', i
|
||||
sizes = compute_num_classes(ex_id).values()
|
||||
if sizes:
|
||||
counter = Counter(sizes)
|
||||
print ' '.join(('{}x{}'.format(x, y) if y > 1 else str(x))
|
||||
for (x, y) in reversed(sorted(counter.items())))
|
||||
else:
|
||||
print 0
|
||||
print >> sys.stderr, 'DONE!'
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys, os, shutil, re, argparse, json, shutil
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
def read_dataset(filename):
|
||||
with open(filename, 'r', 'utf8') as fin:
|
||||
header = fin.readline().rstrip('\n').split('\t')
|
||||
dataset = {}
|
||||
for line in fin:
|
||||
example = dict(zip(header, line.rstrip('\n').split('\t')))
|
||||
example['targetValue'] = example['targetValue'].split('|')
|
||||
dataset[example['id']] = example
|
||||
print >> sys.stderr, 'Read {} examples'.format(len(dataset))
|
||||
return dataset
|
||||
|
||||
class RetainedTablesData(object):
|
||||
def __init__(self, ex_id, alter_id, score, other_alter_ids):
|
||||
self.ex_id = ex_id
|
||||
self.alter_id = int(alter_id)
|
||||
self.score = round(float(score), 3)
|
||||
self.other_alter_ids = [int(x) for x in other_alter_ids]
|
||||
|
||||
def read_retained(filename):
|
||||
"""Format: ID <tab> score <tab> altered table ids"""
|
||||
retained = []
|
||||
with open(filename) as fin:
|
||||
for line in fin:
|
||||
ex_id, score, alter_ids = line.rstrip().split('\t')
|
||||
alter_ids = alter_ids.split()
|
||||
for alter_id in alter_ids:
|
||||
retained.append(RetainedTablesData(ex_id, alter_id, score, alter_ids))
|
||||
print >> sys.stderr, 'Read {} retained tables'.format(len(retained))
|
||||
return retained
|
||||
|
||||
def escaped_TSV_to_HTML(x):
|
||||
# '\n' -> newline, '\p' -> pipe, '\\' -> backslash
|
||||
x = x.replace(r'\n', '\n').replace(r'\p', '|').replace('\\\\', '\\')
|
||||
# & -> '&', '<' -> '<', '>' -> '>'
|
||||
x = x.replace('&', '&').replace('<', '<').replace('>', '>')
|
||||
# \n -> <br>
|
||||
x = x.replace('\n', '<br>')
|
||||
return x
|
||||
|
||||
def read_table(filename):
|
||||
with open(filename, 'r', 'utf8') as fin:
|
||||
lines = []
|
||||
for line in fin:
|
||||
cells = line.rstrip('\n').split('\t')
|
||||
cells = [escaped_TSV_to_HTML(x) for x in cells]
|
||||
lines.append(cells)
|
||||
return lines
|
||||
|
||||
def generate_json(example, metadata, table, data):
|
||||
sprite = set()
|
||||
for row in table:
|
||||
sprite.update(row)
|
||||
sprite = sorted(sprite)
|
||||
table_sprited = []
|
||||
for row in table:
|
||||
table_sprited.append([sprite.index(cell) for cell in row])
|
||||
stuff = {
|
||||
'id': example['id'],
|
||||
'question': example['utterance'],
|
||||
'metadata': {
|
||||
'title': metadata['title'],
|
||||
'url': metadata['url'],
|
||||
'tableIndex': metadata['tableIndex'],
|
||||
},
|
||||
'sprite': sprite,
|
||||
'table': table_sprited,
|
||||
'alterIndex': data.alter_id,
|
||||
'retained': {
|
||||
'score': data.score,
|
||||
'otherAlterIndices': data.other_alter_ids,
|
||||
},
|
||||
}
|
||||
return stuff
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-d', '--dataset_filename', default='t/data/training.tsv')
|
||||
parser.add_argument('-j', '--json_dir', default='t/json/')
|
||||
parser.add_argument('-o', '--out_dir', default='out/')
|
||||
parser.add_argument('-a', '--altered_dir', default='altered')
|
||||
parser.add_argument('-q', '--questions', default=8, type=int)
|
||||
parser.add_argument('retained_filename')
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.out_dir):
|
||||
if raw_input('Delete {}? (y/N): '.format(args.out_dir))[0:].lower() != 'y':
|
||||
return
|
||||
shutil.rmtree(args.out_dir)
|
||||
os.makedirs(args.out_dir)
|
||||
|
||||
# Read examples
|
||||
dataset = read_dataset(args.dataset_filename)
|
||||
|
||||
# Read retained tables
|
||||
retained = read_retained(args.retained_filename)
|
||||
|
||||
print >> sys.stderr, args.questions, 'per file'
|
||||
print >> sys.stderr, '->', len(retained) / args.questions, ' * ', args.questions,
|
||||
print >> sys.stderr, ' + ', len(retained) % args.questions
|
||||
fout = None
|
||||
for i, data in enumerate(retained):
|
||||
if i % 500 == 0:
|
||||
print >> sys.stderr, 'Processing {} / {} ...'.format(i, len(retained))
|
||||
if i % args.questions == 0:
|
||||
if fout is not None:
|
||||
fout.close()
|
||||
fout = open(os.path.join(args.out_dir, '{}.js'.format(i / args.questions)), 'w', 'utf8')
|
||||
fout.write('var QUESTIONS=[];\n')
|
||||
# Read table
|
||||
table = read_table(os.path.join(args.altered_dir, data.ex_id, str(data.alter_id) + '.tsv'))
|
||||
# Read example
|
||||
example = dataset[data.ex_id]
|
||||
xxx, yyy = re.match(r'csv/(\d+)-csv/(\d+).csv', example['context']).groups()
|
||||
# Read metadata
|
||||
with open(os.path.join(args.json_dir, xxx + '-json', yyy + '.json'), 'r', 'utf8') as fin:
|
||||
metadata = json.load(fin)
|
||||
# Generate JSON
|
||||
fout.write('QUESTIONS.push(')
|
||||
json.dump(generate_json(example, metadata, table, data), fout,
|
||||
ensure_ascii=False, separators=(',', ':'))
|
||||
fout.write(');\n')
|
||||
if fout is not None:
|
||||
fout.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('infile')
|
||||
parser.add_argument('-p', '--plot', action='store_true',
|
||||
help='Plot number of classes vs number of denotations')
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument('-s', '--summarize', action='store_true',
|
||||
help='Summarize the number of classes')
|
||||
group.add_argument('-d', '--dump', action='store_true',
|
||||
help='Dump the list of examples with at least one agreed classes')
|
||||
group.add_argument('-D', '--dataset-file',
|
||||
help='Take a dataset file and print a filtered list of only examples'
|
||||
' with at least one agreed classes')
|
||||
args = parser.parse_args()
|
||||
|
||||
data = []
|
||||
with open(args.infile) as fin:
|
||||
print >> sys.stderr, 'Reading from', args.infile
|
||||
header = fin.readline().rstrip('\n').split('\t')
|
||||
for line in fin:
|
||||
data.append(dict(zip(header, line.rstrip('\n').split('\t'))))
|
||||
print >> sys.stderr, 'Read', len(data), 'records.'
|
||||
# ['id', 'numDerivs', 'allTurkedTables', 'agreedTurkedTables',
|
||||
# 'origTableTarget', 'origTableTurkedTarget', 'origTableFlag',
|
||||
# 'numClassesMatched', 'numDerivsMatched']
|
||||
|
||||
# Classify examples
|
||||
no_derivs = []
|
||||
orig_table_mismatch = []
|
||||
no_classes_matched = []
|
||||
classes_matched = defaultdict(list)
|
||||
|
||||
plt_num_classes, plt_num_derivs = [], []
|
||||
|
||||
for record in data:
|
||||
if record['numDerivs'] == '0':
|
||||
no_derivs.append(record)
|
||||
assert record['numDerivsMatched'] == '0'
|
||||
continue
|
||||
if record['origTableFlag'] == 'mismatched':
|
||||
orig_table_mismatch.append(record)
|
||||
#assert record['numDerivsMatched'] == '0'
|
||||
continue
|
||||
if record['numClassesMatched'] == '0':
|
||||
no_classes_matched.append(record)
|
||||
assert record['numDerivsMatched'] == '0'
|
||||
continue
|
||||
assert record['numDerivsMatched'] != '0'
|
||||
num_classes = int(record['numClassesMatched'])
|
||||
plt_num_classes.append(num_classes)
|
||||
plt_num_derivs.append(int(record['numDerivsMatched']))
|
||||
if num_classes < 10:
|
||||
classes_matched[num_classes].append(record)
|
||||
else:
|
||||
classes_matched['> 10'].append(record)
|
||||
|
||||
if args.summarize:
|
||||
print 'No derivs:', len(no_derivs)
|
||||
print 'Original table mismatched:', len(orig_table_mismatch)
|
||||
print 'No classes matched:', len(no_classes_matched)
|
||||
print 'Classes matched:'
|
||||
total = 0
|
||||
for key in sorted(classes_matched):
|
||||
num_matches = len(classes_matched[key])
|
||||
total += num_matches
|
||||
print ' {}: {} (cum = {})'.format(key, num_matches, total)
|
||||
|
||||
if args.plot:
|
||||
import matplotlib.pyplot as plt
|
||||
plt.scatter(plt_num_classes, plt_num_derivs)
|
||||
plt.show()
|
||||
|
||||
if args.dump:
|
||||
for key in sorted(classes_matched):
|
||||
for x in classes_matched[key]:
|
||||
print x['id']
|
||||
|
||||
if args.dataset_file:
|
||||
indices = set(int(x['id'].replace('nt-', ''))
|
||||
for y in classes_matched.values() for x in y)
|
||||
count_all, count_filtered = 0, 0
|
||||
with open(args.dataset_file, 'r', 'utf8') as fin:
|
||||
for i, line in enumerate(fin):
|
||||
count_all += 1
|
||||
if i in indices:
|
||||
print line.rstrip('\n')
|
||||
count_filtered += 1
|
||||
print >> sys.stderr, 'Printed {} / {} lines'.format(count_filtered, count_all)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys, os, shutil, re, argparse, json
|
||||
from codecs import open
|
||||
from itertools import izip
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-z', '--remove-zeros', action='store_true',
|
||||
help='remove lines with zero score (= no candidates)')
|
||||
parser.add_argument('retained')
|
||||
parser.add_argument('aggregated')
|
||||
args = parser.parse_args()
|
||||
|
||||
agreed = defaultdict(set)
|
||||
|
||||
# Read Turk aggregated data
|
||||
with open(args.aggregated, 'r', 'utf8') as fin:
|
||||
for line in fin:
|
||||
tokens = line.rstrip().split('\t')
|
||||
ex_id = tokens[1]
|
||||
table_id = int(tokens[2])
|
||||
flag = tokens[3]
|
||||
if flag != 'X':
|
||||
agreed[ex_id].add(table_id)
|
||||
|
||||
# Read Retained
|
||||
with open(args.retained, 'r', 'utf8') as fin:
|
||||
for line in fin:
|
||||
ex_id, score, alter_table_ids = line.rstrip().split('\t')
|
||||
if args.remove_zeros and float(score) == 0:
|
||||
continue
|
||||
alter_table_ids = [int(x) for x in alter_table_ids.split()]
|
||||
non_agreed = [x for x in alter_table_ids if x not in agreed[ex_id]]
|
||||
if non_agreed:
|
||||
print '\t'.join([ex_id, score, ' '.join(str(x) for x in sorted(non_agreed))])
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
53
tables/x
53
tables/x
|
|
@ -1,21 +1,52 @@
|
|||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
# Usage:
|
||||
# ./x up
|
||||
# Upload the compiled code and the run file into a bundle called `stuff`
|
||||
# ./x {walk|run|jog} [-n] [-- additional_dependencies --] run_arguments
|
||||
# Call cl run on the ./run script. The run name is automatically generated.
|
||||
# Use -n to print the command and exit.
|
||||
|
||||
operation=$1
|
||||
shift
|
||||
queue=
|
||||
if [ "$operation" == "up" ]; then
|
||||
cl upload program libsempre module-classes.txt tables/grammars run -L -n stuff
|
||||
elif [ "$operation" == "jog" ]; then
|
||||
queue="nlp"
|
||||
cl upload libsempre module-classes.txt tables/grammars run -L -n stuff
|
||||
exit 0
|
||||
elif [ "$operation" == "walk" ]; then
|
||||
queue="--request-memory 11g --request-cpus 2 --request-queue nlp"
|
||||
elif [ "$operation" == "run" ]; then
|
||||
queue="john"
|
||||
queue="--request-memory 11g --request-cpus 2 --request-queue john"
|
||||
elif [ "$operation" == "jog" ]; then
|
||||
queue="--request-memory 11g --request-cpus 2 --request-queue jag"
|
||||
else
|
||||
echo "Available operation: up jog run"
|
||||
echo "Available operation: up walk run jog"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$queue" ]; then
|
||||
name=$(YAY=$(cat .cnt.info || echo 0) && echo $(($YAY + 1)) > .cnt.info && echo -n `hostname`-$YAY.cl)
|
||||
cl run --request_cpus 3 --request_memory 10g --request_queue "$queue" -n "$name" \
|
||||
:fig :lib libsempre:stuff/libsempre module-classes.txt:stuff/module-classes.txt run:stuff/run grammars:stuff/grammars :WikiTableQuestions \
|
||||
"./run @cldir=1 $*"
|
||||
dryrun=
|
||||
if [ "$1" == "-n" ]; then
|
||||
shift
|
||||
dryrun=1;
|
||||
fi
|
||||
|
||||
dependencies=":fig :lib libsempre:stuff/libsempre module-classes.txt:stuff/module-classes.txt run:stuff/run grammars:stuff/grammars :WikiTableQuestions"
|
||||
if [ "$1" == "--" ]; then
|
||||
shift
|
||||
while [ "$1" != "--" ]; do
|
||||
dependencies="$dependencies $1"
|
||||
shift
|
||||
done
|
||||
shift
|
||||
fi
|
||||
|
||||
prefix=`hostname`
|
||||
if [[ "$prefix" == j* ]]; then
|
||||
prefix="nlp"
|
||||
elif [ "$prefix" != "nazrin" -a "$prefix" != "fandango" ]; then
|
||||
prefix="what"
|
||||
fi
|
||||
name=$(YAY=$(cat .cnt.info || echo 0) && echo $(($YAY + 1)) > .cnt.info && echo -n $prefix-$YAY.cl)
|
||||
cl_run_args="cl run $queue -n $name $dependencies"
|
||||
cl_run_command="LC_ALL=C.UTF-8 ./run @cldir=1 $*"
|
||||
echo "$cl_run_args \"$cl_run_command\""
|
||||
[ $dryrun ] || $cl_run_args "$cl_run_command"
|
||||
|
|
|
|||
Loading…
Reference in New Issue