natas_hw
/
sempre
mirror of https://github.com/percyliang/sempre
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updated the Wikipedia scraping scripts

This commit is contained in:
Panupong Pasupat 2017-08-27 13:50:55 -07:00
parent af0957b445
commit f203345ffd
5 changed files with 175 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tables/wikipedia-scripts/download-wikipedia-pages.py
View File

@ -28,7 +28,7 @@ def download(data, i, outdir):
hashcode = data['hashcode'] = CACHE.get_hashcode(url)
data['url'] = url
result = CACHE.get_page(url)
if result is not None:
if result is not None and not os.path.exists(os.path.join(outdir, str(data['id']) + '.html')):
os.symlink(os.path.join('..', 'web.cache', hashcode),
os.path.join(outdir, str(data['id']) + '.html'))
with open(os.path.join(outdir, str(data['id']) + '.json'), 'w') as fout:

95
tables/wikipedia-scripts/find-good-tables.py
View File

@ -49,10 +49,10 @@ class TableStat(object):
self.num_min = min(table.num_rows, table.num_cols)
self.num_max = max(table.num_rows, table.num_cols)
self.num_cells = table.num_cells
self.num_empty_cells = len([x for x in table.cells if not x])
self.num_long = len([x for x in table.cells if len(x) >= 40])
self.num_empty_cells = len([x for x in table.cells if not x[1]])
self.num_long = len([x for x in table.cells if len(x[1]) >= 40])
self.num_short_headers = len([x for x in table.rows[0] if len(x) <= 3])
self.num_numeric_cells = len([x for x in table.cells if re.search('[0-9]', x)])
self.num_numeric_cells = len([x for x in table.cells if re.search('[0-9]', x[1])])
self.num_repetitive_cols = sum([self.is_repetitive(col) for col in table.cols], 0)
self.num_similar_colpairs = sum([self.are_similar(col1, col2)
for (col1, col2) in combinations(table.cols, 2)], 0)
@ -61,11 +61,11 @@ class TableStat(object):
self.get_scores()
def is_repetitive(self, col):
return len(set(get_repeatable_token(x) for x in col)) < 0.5 * len(col)
return len(set(get_repeatable_token(x[1]) for x in col)) < 0.5 * len(col)
def are_similar(self, col1, col2):
col1 = set(get_alpha(x) for x in col1) - set([''])
col2 = set(get_alpha(x) for x in col2) - set([''])
col1 = set(get_alpha(x[1]) for x in col1) - set([''])
col2 = set(get_alpha(x[1]) for x in col2) - set([''])
return len(col1 & col2) >= 3
def __str__(self):
@ -119,6 +119,39 @@ def check_table(table, criterion):
and stat.num_short_headers < stat.num_cols * 0.3):
return stat
def dump(i, page_id, table_id, stat, args):
with open(os.path.join(args.source_dir, '%d.json' % page_id), 'r', 'utf8') as fin:
meta = json.load(fin)
meta['tableIndex'] = table_id
if args.custom_filenames:
while True:
try:
i = int(raw_input('filename for "%s": ' % meta['title']))
break
except:
pass
with open(os.path.join(args.json_dir, '%d.json' % i), 'w', 'utf8') as fout:
json.dump(meta, fout)
if args.copy:
shutil.copy(os.path.join(args.source_dir, '%d.html' % page_id),
os.path.join(args.page_dir, '%d.html' % i))
else:
os.symlink(os.path.relpath(os.path.join(args.source_dir, '%d.html' % page_id),
args.page_dir),
os.path.join(args.page_dir, '%d.html' % i))
print '>' * 30, 'Written', i, page_id, table_id, '<' * 30
print stat
def read_used_pages(json_files, args):
used_pages = set()
for filename in json_files:
with open(os.path.join(args.json_dir, filename)) as fin:
data = json.load(fin)
used_pages.add(data['id'])
print 'Found {} used pages'.format(len(used_pages))
return used_pages
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--amount', type=int, default=20,
@ -130,26 +163,41 @@ def main():
help="criterion for finding tables (higher = more strict)")
parser.add_argument('-d', '--dry-run', action='store_true',
help="do not copy files")
parser.add_argument('-C', '--copy', action='store_true',
help='copy the html file instead of making a symlink')
parser.add_argument('-i', '--fixed-id', type=int,
help="use this fixed page id")
parser.add_argument('--custom-filenames', action='store_true',
help="set custom filename for each file")
parser.add_argument('-q', '--quick', action='store_true',
help='dump the good tables immediately after they are found')
parser.add_argument('-r', '--resume-quick', action='store_true',
help='continue the quick mode')
parser.add_argument('outprefix')
args = parser.parse_args()
json_dir = os.path.join(args.outprefix + '-json')
page_dir = os.path.join(args.outprefix + '-page')
if os.path.exists(json_dir) or os.path.exists(page_dir):
quick_index, quick_used_pages = 0, set()
args.json_dir = os.path.join(args.outprefix + '-json')
args.page_dir = os.path.join(args.outprefix + '-page')
if args.resume_quick:
# Read the highest number in json_dir
json_files = os.listdir(args.json_dir)
print 'Found {} JSON files'.format(len(json_files))
quick_index = max(int(x.replace('.json', '')) for x in json_files) + 1
quick_used_pages = read_used_pages(json_files, args)
elif os.path.exists(args.json_dir) or os.path.exists(args.page_dir):
if raw_input('Path exists. Clobber? ')[0:].lower() != 'y':
exit(1)
elif not args.dry_run:
os.makedirs(json_dir)
os.makedirs(page_dir)
os.makedirs(args.json_dir)
os.makedirs(args.page_dir)
if args.fixed_id is not None:
filenames = [args.fixed_id]
else:
filenames = get_filenames(args.source_dir)
if args.quick:
filenames = sorted(set(filenames) - set(quick_used_pages))
print >> sys.stderr, 'Got %d candidates' % len(filenames)
random.shuffle(filenames)
filenames = filenames[:args.limit]
@ -171,6 +219,14 @@ def main():
print '%2d' % sum(stat.scores),
print 'SCORE %9d (ID %9d) table %2d' % (page_id, meta['id'], table_id),
print stat, stat.scores
if args.quick:
for table_id, stat in good_table_ids:
dump(quick_index, page_id, table_id, stat, args)
quick_index += 1
if args.quick:
return
random.shuffle(good_tables)
good_tables = good_tables[:args.amount]
if args.dry_run:
@ -178,22 +234,7 @@ def main():
for i, (page_id, good_table_ids) in enumerate(good_tables):
table_id, stat = random.choice(good_table_ids)
with open(os.path.join(args.source_dir, '%d.json' % page_id), 'r', 'utf8') as fin:
meta = json.load(fin)
meta['tableIndex'] = table_id
if args.custom_filenames:
while True:
try:
i = int(raw_input('filename for "%s": ' % meta['title']))
break
except:
pass
with open(os.path.join(json_dir, '%d.json' % i), 'w', 'utf8') as fout:
json.dump(meta, fout)
shutil.copy(os.path.join(args.source_dir, '%d.html' % page_id),
os.path.join(page_dir, '%d.html' % i))
print '>' * 30, 'Written', i, page_id, table_id, '<' * 30
print stat
dump(i, page_id, table_id, stat, args)
if __name__ == '__main__':
main()

2
tables/wikipedia-scripts/get-wikipedia-pages.py
View File

@ -12,7 +12,7 @@ html_parser = HTMLParser.HTMLParser()
def is_sane(content, args):
if content.startswith('#REDIRECT'):
return False
if args.table and '{|' not in content:
if args.table and ('{|' not in content or 'wikitable' not in content):
return False
return True

243
tables/wikipedia-scripts/table-to-csv.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Convert HTML table into CSV / TSV / pretty-printed table."""
import sys, os, re, argparse, json
from codecs import open
@ -10,24 +11,40 @@ from itertools import izip_longest
################ Dump CSV
def simple_normalize_text(text):
return text.replace('\\', '\\\\').replace('"', r'\"').replace('\n', r'\n').replace(u'\xa0', ' ').strip()
return text.replace('\\', '\\\\').replace('"', r'\"').replace('\n', r'\\n').replace(u'\xa0', ' ').strip()
def dump_csv(rows, fout):
for row in rows:
fout.write(','.join('"%s"' % simple_normalize_text(x) for x in row) + '\n')
fout.write(','.join('"%s"' % simple_normalize_text(x[1]) for x in row) + '\n')
def tab_normalize_text(text):
return re.sub(r'\s+', ' ', text.replace('\n', r'\n'), re.U).strip()
return re.sub(r'\s+', ' ', text.replace('\\', '\\\\').replace('|', r'\p').replace('\n', r'\n'), re.U).strip()
def dump_tsv(rows, fout):
for row in rows:
fout.write('\t'.join('%s' % tab_normalize_text(x) for x in row) + '\n')
fout.write('\t'.join('%s' % tab_normalize_text(x[1]) for x in row) + '\n')
def table_normalize_text(text):
return re.sub(r'\s+', ' ', text, re.U).strip()
def dump_table(rows, fout):
widths = defaultdict(int)
for row in rows:
for i, cell in enumerate(row):
widths[i] = max(widths[i], len(table_normalize_text(cell[1])) + 1)
for row in rows:
fout.write('|')
for i, cell in enumerate(row):
# wow this is so hacky
fout.write((' %-' + str(widths[i]) + 's') % table_normalize_text(cell[1]))
fout.write('|')
fout.write('\n')
################ More table normalization
def debug_print(stuff):
for x in stuff:
print >> sys.stderr, [simple_normalize_text(y) for y in x]
print >> sys.stderr, [simple_normalize_text(y[1]) for y in x]
def transpose(rows):
cols = []
@ -38,7 +55,7 @@ def transpose(rows):
try:
col.append(row[i])
except LookupError:
col.append(None)
col.append(('', ''))
cols.append(col)
return cols
@ -53,7 +70,7 @@ def anti_transpose(cols):
if col[i] is not None:
row.append(col[i])
else:
row.append('')
row.append(('', ''))
rows.append(row)
return rows
@ -65,7 +82,7 @@ def remove_empty_columns(orig_cols):
"""Remove columns with <= 1 non-empty cells."""
cols = []
for col in orig_cols:
non_empty = sum((bool(cell) for cell in col), 0)
non_empty = sum((bool(cell[1]) for cell in col), 0)
if non_empty >= 2:
cols.append(col)
return cols
@ -77,9 +94,9 @@ def are_mergeable(col1, col2):
merged = []
for i in xrange(len(col1)):
c1, c2 = col1[i], col2[i]
if not c1:
if not c1[1]:
merged.append(c2)
elif not c2 or c1 == c2:
elif not c2[1] or c1 == c2:
merged.append(c1)
else:
return None
@ -96,161 +113,93 @@ def merge_similar_columns(orig_cols):
i += 1
return orig_cols
#### Split columns
#### Merge header rows
REGEX_NEWLINE = re.compile(ur'^([^\n]*)\n([^\n]*)$', re.U | re.DOTALL)
REGEX_NEWLINE_PAREN = re.compile(ur'^(.*)\n(\(.*\))$', re.U | re.DOTALL)
REGEX_SPLIT_PAREN = re.compile(ur'^(.*) +(\(.*\))$', re.U | re.DOTALL)
def split_multiline_columns(orig_cols):
"""Split columns with newline in each cell."""
i = 0
while i < len(orig_cols):
for regex, threshold in ((REGEX_NEWLINE, 0.5 * len(orig_cols[i])),
(REGEX_NEWLINE_PAREN, 1),
(REGEX_SPLIT_PAREN, 2)):
matches = [regex.match(cell or '') for cell in orig_cols[i]]
num_matches = sum((bool(match) for match in matches[1:]), 0)
if num_matches >= threshold:
splitted = [((None, None) if cell is None
else (cell, '') if not match
else (match.group(1), match.group(2)))
for (cell, match) in zip(orig_cols[i], matches)]
orig_cols[i:i+1] = [[(None if x[0] is None else x[0].strip()) for x in splitted],
[(None if x[1] is None else x[1].strip()) for x in splitted]]
if not orig_cols[i+1][0]:
orig_cols[i+1][0] = (orig_cols[i][0] or '') + '#n'
break
i += 1
return orig_cols
REGEX_FROM_TO = re.compile(ur'^(.*)[-–—‒―](.*)$', re.U)
def split_from_to_columns(orig_cols):
"""Split columns with pattern '... - ...'."""
i = 0
while i < len(orig_cols):
matches = [REGEX_FROM_TO.match(cell or '') for cell in orig_cols[i]]
num_matches = sum((bool(match) for match in matches), 0)
if num_matches > 0.5 * len(orig_cols[i]):
splitted = [((None, None) if cell is None
else (cell, '') if not match
else (match.group(1), match.group(2)))
for (cell, match) in zip(orig_cols[i], matches)]
orig_cols[i:i+1] = [[(None if x[0] is None else x[0].strip()) for x in splitted],
[(None if x[1] is None else x[1].strip()) for x in splitted]]
if not orig_cols[i+1][0]:
orig_cols[i+1][0] = orig_cols[i][0]
orig_cols[i][0] += '#f'
orig_cols[i+1][0] += '#t'
i += 1
return orig_cols
#### Normalize by column
REGEX_NUMBERING = re.compile(ur'^([0-9]+)[.)]$', re.U)
REGEX_REFERENCE = re.compile(ur'^(.*)[‡^†*]+$', re.U)
REGEX_PARENS = re.compile(ur'^\((.*)\)$', re.U)
REGEX_QUOTES = re.compile(ur'^"(.*)"$', re.U)
def normalize_common_punctuations(orig_cols):
"""Normalize some punctuations if they appear a lot in the same row."""
cols = []
for col in orig_cols:
for regex, threshold in ((REGEX_NUMBERING, 0.8 * len(col)),
(REGEX_REFERENCE, 0.2 * len(col)),
(REGEX_PARENS, 0.5 * len(col)),
(REGEX_QUOTES, 0.5 * len(col))):
matches = [regex.match(cell or '') for cell in col]
num_matches = sum((not cell or bool(match)
for (cell, match) in zip(col, matches)), 0)
if num_matches >= threshold:
col = [(cell if not match else match.group(1))
for (cell, match) in zip(col, matches)]
cols.append(col)
return cols
#### Normalize by cell
REGEX_COMMA = re.compile(ur'([0-9]),([0-9][0-9][0-9])', re.U)
def remove_commas_from_single_number(x):
if not x:
return x
while REGEX_COMMA.search(x):
x = REGEX_COMMA.sub(r'\1\2', x)
return x
def remove_commas_from_numbers(orig_stuff):
stuff = []
for slab in orig_stuff:
stuff.append([remove_commas_from_single_number(x) for x in slab])
return stuff
def merge_header_rows(orig_rows):
"""Merge all header rows together."""
header_rows, body_rows = [], []
still_header = True
for row in orig_rows:
if not still_header or any(cell[0] == 'td' for cell in row):
still_header = False
body_rows.append(row)
else:
header_rows.append(row)
if len(header_rows) < 2 or not body_rows:
return orig_rows
# Merge header rows with '\n'
header_cols = transpose(header_rows)
header_row = []
for col in header_cols:
texts = [None]
for cell in col:
if cell[1] != texts[-1]:
texts.append(cell[1])
header_row.append(('th', '\n'.join(texts[1:])))
return [header_row] + body_rows
################ Main function
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-s', '--source-dir', default='wikidump.cache/output',
help="source directory")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-j', '--json',
help="json metadata file specifying page and table id")
group.add_argument('-J', '--turk-json',
parser.add_argument('-j', '--turk-json',
help="json metadata file from MTurk task")
group.add_argument('-p', '--page-id', type=int,
help="page index")
parser.add_argument('-t', '--table-id', type=int, default=0,
help="table index (only used for -p / --page-id)")
parser.add_argument('-o', '--outfile',
help="output filename (default = stdout)")
parser.add_argument('-n', '--normalize', action='count',
help='degree of normalization')
parser.add_argument('--tsv', action='store_true',
help='output TSV instead of CSV')
help='also print out tsv')
parser.add_argument('--human', action='store_true',
help='also print out human-readable table')
parser.add_argument('--html', action='store_true',
help='also print out cleaned html for the table')
parser.add_argument('--keep-hidden', action='store_true',
help='keep hidden texts as is')
args = parser.parse_args()
assert not args.tsv or args.outfile.endswith('.csv')
if args.json:
with open(args.json) as fin:
metadata = json.load(fin)
args.page_id = metadata['id']
args.table_id = metadata['tableIndex']
inhtml = os.path.join(args.source_dir, '%d.html' % args.page_id)
elif args.turk_json:
with open(args.turk_json) as fin:
metadata = json.load(fin)
args.page_id = metadata['id']
args.table_id = metadata['tableIndex']
# The following replacement is pretty hacky:
inhtml = args.turk_json.replace('-json', '-page').replace('.json', '.html')
else:
inhtml = os.path.join(args.source_dir, '%d.html' % args.page_id)
with open(args.turk_json) as fin:
metadata = json.load(fin)
# Get the path to the HTML file
# This is kind of hacky
match = re.match(r'^(?:json|page)/(\d+)-(?:json|page)/(\d+).json$', args.turk_json)
batch_id, data_id = match.groups()
inhtml = 'page/{}-page/{}.html'.format(batch_id, data_id)
with open(inhtml, 'r', 'utf8') as fin:
table = Table.get_wikitable(fin.read(), args.table_id, normalization=Table.NORM_DUPLICATE)
raw = fin.read()
table = Table.get_wikitable(raw, metadata['tableIndex'],
normalization=Table.NORM_DUPLICATE,
remove_hidden=(not args.keep_hidden))
if args.html:
raw_table = Table.get_wikitable(raw, metadata['tableIndex'],
remove_hidden=False).table
rows = table.rows
if args.normalize >= 1:
# Remove redundant rows and columns
rows = remove_full_rowspans(rows)
cols = transpose(rows)
cols = remove_empty_columns(cols)
cols = merge_similar_columns(cols)
rows = anti_transpose(cols)
if args.normalize >= 2:
# Split cells / Normalize texts
cols = transpose(rows)
cols = split_multiline_columns(cols)
cols = normalize_common_punctuations(cols)
cols = split_from_to_columns(cols)
cols = remove_commas_from_numbers(cols)
rows = anti_transpose(cols)
#debug_print(transpose(rows))
outputter = dump_tsv if args.tsv else dump_csv
# rows = list of columns; column = list of cells; cell = (tag, text)
# Remove redundant rows and columns
rows = remove_full_rowspans(rows)
cols = transpose(rows)
cols = remove_empty_columns(cols)
cols = merge_similar_columns(cols)
rows = anti_transpose(cols)
rows = merge_header_rows(rows)
# Dump
if not args.outfile:
outputter(rows, sys.stdout)
dump_csv(rows, sys.stdout)
else:
stem = re.sub('\.csv$', '', args.outfile)
with open(args.outfile, 'w', 'utf8') as fout:
outputter(rows, fout)
dump_csv(rows, fout)
if args.tsv:
with open(stem + '.tsv', 'w', 'utf8') as fout:
dump_tsv(rows, fout)
if args.human:
with open(stem + '.table', 'w', 'utf8') as fout:
dump_table(rows, fout)
if args.html:
with open(stem + '.html', 'w', 'utf8') as fout:
print >> fout, unicode(raw_table)
if __name__ == '__main__':
main()

18
tables/wikipedia-scripts/weblib/table.py
View File

@ -7,7 +7,9 @@ Get statistics about a table and convert it to CSV.
import sys, os, re, json
from codecs import open
from collections import defaultdict
from bs4 import BeautifulSoup
from bs4 import BeautifulSoup as BeautifulSoupOriginal
def BeautifulSoup(markup=""):
return BeautifulSoupOriginal(markup, 'html.parser')
class Table(object):
NORM_NONE = 0
@ -21,14 +23,11 @@ class Table(object):
self.table = table
if remove_hidden:
self.remove_hidden()
if normalization == Table.NORM_NONE:
self.get_cells()
elif normalization == Table.NORM_CORNER:
if normalization == Table.NORM_CORNER:
self.normalize_table()
self.get_cells()
elif normalization == Table.NORM_DUPLICATE:
self.normalize_table(deep=True)
self.get_cells()
self.get_cells()
@staticmethod
def get_wikitable(raw_html, index=None, **kwargs):
@ -53,11 +52,12 @@ class Table(object):
tag.extract()
def get_cells(self):
"""Each cell is (tag, text)"""
self.rows, self.cells = [], []
for x in self.table.find_all('tr', recursive=False):
row = []
for y in x.find_all(['th', 'td'], recursive=False):
row.append(y.text.strip())
row.append((y.name, y.text.strip()))
self.rows.append(row)
self.cells.extend(row)
self.num_rows = len(self.rows)
@ -82,7 +82,7 @@ class Table(object):
def get_cloned_cell(self, cell, rowspan=1, deep=False):
if deep:
# Hacky but works
return BeautifulSoup(unicode(cell)).body.contents[0]
return BeautifulSoup(unicode(cell)).contents[0]
tag = Table.SOUP.new_tag(cell.name)
if rowspan > 1:
tag['rowspan'] = rowspan
@ -152,4 +152,4 @@ def test():
print table.table
if __name__ == '__main__':
test()
test_wiki()