Counting words¶

from __future__ import print_function

import re
non_word = re.compile(r'[\W\d]+', re.UNICODE)

def normalize_word(word):
    """normalize a word
    
    simply strips non-word characters and case
    """
    word = word.lower()
    word = non_word.sub('', word)
    return word

common_words = {
'the','of','and','in','to','a','is','it','that','which','as','on','by',
'be','this','with','are','from','will','at','you','not','for','no','have',
'i','or','if','his','its','they','but','their','one','all','he','when',
'than','so','these','them','may','see','other','was','has','an','there',
'more','we','footnote', 'who', 'had', 'been',  'she', 'do', 'what',
'her', 'him', 'my', 'me', 'would', 'could', 'said', 'am', 'were', 'very',
'your', 'did', 'not',
}

def yield_words(filename):
    """returns a generator of words in a file"""
    import io
    with io.open(filename, encoding='latin-1') as f:
        for line in f:
            for word in line.split():
                word = normalize_word(word)
                if word:
                    yield word

def ngrams(filename, n=1):
    """compute ngram counts for the contents of a file"""
    word_iterator = yield_words(filename)
    counts = {}
    def _count_gram(gram):
        common = sum(word in common_words for word in gram)
        if common > n / 2.0:
            # don't count ngrams that are >= 50% common words
            return
        sgram = ' '.join(gram)
        counts.setdefault(sgram, 0)
        counts[sgram] += 1
    
    gram = []
    
    # get the first word
    while len(gram) < n:
        try:
            word = next(word_iterator)
            if not word:
                continue
        except StopIteration:
            return counts
        else:
            gram.append(word)
    
    _count_gram(gram)

    while True:
        try:
            word = next(word_iterator)
        except StopIteration:
            break
        else:
            gram.append(word)
            gram.pop(0)
            _count_gram(gram)
    return counts
            

try: 
    from urllib.request import urlretrieve # py3
except ImportError:
    from urllib import urlretrieve # py2

davinci_url = "http://www.gutenberg.org/cache/epub/5000/pg5000.txt"

if not os.path.exists('davinci.txt'):
    # download from project gutenberg
    print("Downloading Da Vinci's notebooks from Project Gutenberg")
    urlretrieve(davinci_url, 'davinci.txt')

    

import sys

def print_common(freqs, n=10):
    """Print the n most common keys by count."""
    words, counts = freqs.keys(), freqs.values()
    items = zip(counts, words)
    items.sort(reverse=True)
    justify = 0
    for (count, word) in items[:n]:
        justify = max(justify, len(word))
    
    for (count, word) in items[:n]:
        print(word.rjust(justify), count)
    sys.stdout.flush()

# split the davinci.txt into one file per engine:
text = open('davinci.txt').read()
lines = text.splitlines()
nlines = len(lines)
n = 10

block = nlines//n
for i in range(n):
    chunk = lines[i*block:(i+1)*(block)]
    with open('davinci%i.txt' % i, 'w') as f:
        f.write('\n'.join(chunk))

import os
cwd = os.path.abspath(os.getcwd())
fnames = [ os.path.join(cwd, 'davinci%i.txt' % i) for i in range(n)]

from IPython import parallel
rc = parallel.Client()

def ngrams_parallel(view, fnames, n=1):
    """Compute ngrams in parallel
    
    view - An IPython View
    fnames - The filenames containing the split data.
    """
    pass

%load ../soln/ngrams.py

gutenberg_samples = 'http://nltk.github.com/nltk_data/packages/corpora/gutenberg.zip'
if not os.path.isdir('gutenberg'):
    if not os.path.exists('gutenberg.zip'):
        urlretrieve(gutenberg_samples, 'gutenberg.zip')
    !unzip gutenberg.zip

import glob
gutenberg_files = glob.glob(os.path.abspath(os.path.join('gutenberg', '*.txt')))
# remove the bible, because it's too big relative to the rest
gutenberg_files.remove(os.path.abspath(os.path.join('gutenberg', 'bible-kjv.txt')))