text_plain_output_1.png
from hmmlearn import hmm

import numpy as np

import pandas as pd

import warnings

test_corpus = np.load('../input/preprocessing/test_corp.npy')
train_corpus = np.load('../input/preprocessing/train_corp.npy')
glove_table = pd.read_csv('../input/preprocessing/filled_glove_table.csv', index_col=0)

glove_table.loc[['man', 'woman', 'man']].as_matrix().shape

train_sequence = []
train_sequence_lengths = []
for sentence_chunk in train_corpus:
    words = [word for word, tag in sentence_chunk]
    train_sequence.append(glove_table.loc[words].as_matrix())
    train_sequence_lengths.append(len(words))

from hmmlearn import hmm
def hmm_train_model(train_sequence, train_sequence_lengths):
    num_hidden_states = 20
    num_dims = train_sequence[0].shape[1]
    model = hmm.GaussianHMM(n_components=num_hidden_states, covariance_type='spherical', n_iter=10)
    start_prob = np.random.rand(num_hidden_states)
    start_prob = start_prob / np.sum(start_prob)
    model.startprob_ = start_prob
    transmat = np.random.rand(num_hidden_states, num_hidden_states)
    transmat = transmat / np.sum(transmat, axis=1)
    model.transmat_ = transmat
    model.means_ = np.random.rand(num_hidden_states, num_dims)
    model.covars_ = np.tile(np.identity(num_dims), (num_hidden_states, 1, 1))
    return model.fit(np.concatenate(train_sequence), train_sequence_lengths)

import warnings
with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    model = hmm_train_model(train_sequence[0:10], train_sequence_lengths[0:10])

model