text_plain_output_1.png
from collections import Counter

from nltk import pos_tag, RegexpParser

from nltk.tokenize import PunktSentenceTokenizer, word_tokenize

import os
import nltk
from nltk import pos_tag, RegexpParser
from nltk.tokenize import PunktSentenceTokenizer, word_tokenize
from collections import Counter
def word_sentence_tokenize(text):
    sentence_tokenizer = PunktSentenceTokenizer(text)
    sentence_tokenized = sentence_tokenizer.tokenize(text)
    word_tokenized = list()
    for tokenized_sentence in sentence_tokenized:
        word_tokenized.append(word_tokenize(tokenized_sentence))
    return word_tokenized
def np_chunk_counter(chunked_sentences):
    chunks = list()
    for chunked_sentence in chunked_sentences:
        for subtree in chunked_sentence.subtrees(filter=lambda t: t.label() == 'NP'):
            chunks.append(tuple(subtree))
    chunk_counter = Counter()
    for chunk in chunks:
        chunk_counter[chunk] += 1
    return chunk_counter.most_common(30)
def vp_chunk_counter(chunked_sentences):
    chunks = list()
    for chunked_sentence in chunked_sentences:
        for subtree in chunked_sentence.subtrees(filter=lambda t: t.label() == 'VP'):
            chunks.append(tuple(subtree))
    chunk_counter = Counter()
    for chunk in chunks:
        chunk_counter[chunk] += 1
    return chunk_counter.most_common(30)

text = open('../input/the_wizard_of_oz.txt', encoding='utf-8').read().lower()

sentence_tokenizer = PunktSentenceTokenizer(text)
sentence_tokenized = sentence_tokenizer.tokenize(text)

word_tokenized = list()
for sentence in sentence_tokenized:
    word_tokenized.append(word_tokenize(sentence))

pos_tagged_text = list()
for sentence in word_tokenized:
    pos_tagged_text.append(pos_tag(sentence))

chunk_grammar = 'NP: {<DT>?<JJ>*<NN>}'
vp_chunk_grammar = 'VP: {<VB.*><DT>?<JJ>*<NN><RB.?>?}'
chunk_parser = RegexpParser(chunk_grammar)
vp_chunk_parser = RegexpParser(vp_chunk_grammar)
chunked_sentence = chunk_parser.parse(pos_tagged_text[10])
print(chunked_sentence)
vp_chunked_sentence = vp_chunk_parser.parse(pos_tagged_text[10])
print(vp_chunked_sentence)