61 lines
1.6 KiB
Python
61 lines
1.6 KiB
Python
'''
|
|
Created on 2016/1/26
|
|
|
|
@author: Fisher Yu
|
|
'''
|
|
|
|
from pynlpl.formats.sonar import Corpus
|
|
from pynlpl.lm.lm import SimpleLanguageModel
|
|
|
|
sentence = 'this is a foo bar sentences and i want to ngramize it.'
|
|
t = 'this is a foo bar sentences and i want to ngramize it.'
|
|
lm = SimpleLanguageModel()
|
|
file = "C:\\Users\\Administrator\\Desktop\\kenlm-tools\\bible.en.txt"
|
|
#lm.load("C:\\Users\\Administrator\\Desktop\\kenlm-tools\\bible.en.txt")
|
|
lm.append(sentence)
|
|
sc = lm.scoresentence(sentence)
|
|
print sc
|
|
|
|
|
|
'''
|
|
import nltk
|
|
from nltk.util import ngrams
|
|
sentence = 'this is a foo bar sentences and i want to ngramize it'
|
|
ts = 'foo i'
|
|
n = 3
|
|
tokenize = nltk.word_tokenize(sentence)
|
|
n_model = ngrams(tokenize, n)
|
|
for grams in n_model:
|
|
print grams
|
|
'''
|
|
|
|
|
|
'''
|
|
import nltk
|
|
|
|
print "... build"
|
|
brown = nltk.corpus.brown
|
|
corpus = [word.lower() for word in brown.words()]
|
|
|
|
# Train on 95% f the corpus and test on the rest
|
|
spl = 95*len(corpus)/100
|
|
train = corpus[:100]
|
|
test = corpus[100:120]
|
|
|
|
# Remove rare words from the corpus
|
|
fdist = nltk.FreqDist(w for w in train)
|
|
vocabulary = set(map(lambda x: x[0], filter(lambda x: x[1] >= 5, fdist.iteritems())))
|
|
|
|
train = map(lambda x: x if x in vocabulary else "*unknown*", train)
|
|
test = map(lambda x: x if x in vocabulary else "*unknown*", test)
|
|
|
|
print "... train"
|
|
from nltk.model.ngram import NgramModel
|
|
from nltk.probability import LidstoneProbDist
|
|
|
|
estimator = lambda fdist, bins: LidstoneProbDist(fdist, 0.2)
|
|
lm = NgramModel(3, train, estimator=estimator)
|
|
|
|
print "len(corpus) = %s, len(vocabulary) = %s, len(train) = %s, len(test) = %s" % ( len(corpus), len(vocabulary), len(train), len(test) )
|
|
print "perplexity(test) =", lm.perplexity(test)
|
|
''' |