text_plain_output_1.png
from sklearn import linear_model

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.metrics.pairwise import cosine_similarity

from sklearn.preprocessing import LabelEncoder

import matplotlib.mlab as mlab

import matplotlib.pyplot as plt

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

data = pd.read_csv('../input/winemag-data_first150k.csv')

import matplotlib.mlab as mlab

num_bins = 20
n, bins, patches = plt.hist(data['points'], num_bins, normed=1, facecolor='blue', alpha=0.5)
plt.title('Distribution of Wine Scores')
plt.xlabel('Score out of 100')
plt.ylabel('Frequency')

mu = 88 # mean of distribution
sigma = 3 # standard deviation of distribution

y = mlab.normpdf(bins, mu, sigma) # create the y line

plt.plot(bins, y, 'r--')

df = data.dropna(subset=['description'])  # drop all NaNs

df_sorted = df.sort_values(by='points', ascending=True)  # sort by points

num_of_wines = df_sorted.shape[0]  # number of wines
worst = df_sorted.head(int(0.25*num_of_wines))  # 25 % of worst wines listed
best = df_sorted.tail(int(0.25*num_of_wines))  # 25 % of best wines listed

from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(stop_words='english', analyzer='word')

X1 = vectorizer.fit_transform(best['description'])
idf = vectorizer.idf_
goodlist = vectorizer.vocabulary_

X = vectorizer.fit_transform(worst['description'])
idf = vectorizer.idf_
not_so_good_list = vectorizer.vocabulary_

from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
y = le.fit_transform(best['variety'])

reg = linear_model.Ridge(alpha=0.5, solver='sag')

y = data['points']
x = vectorizer.fit_transform(data['description'])
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=32)
reg.fit(x_train, y_train)

import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import nltk
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.model_selection import cross_val_score
from sklearn.metrics.pairwise import euclidean_distances
pd.set_option('display.max_colwidth', 1500)
vectorizer = TfidfVectorizer(stop_words='english', binary=False, max_df=0.95, min_df=0.15, ngram_range=(1, 2), use_idf=False, norm=None)
doc_vectors = vectorizer.fit_transform(data['description'])

def comp_description(query, results_number=20):
    results = []
    q_vector = vectorizer.transform([query])
    results.append(cosine_similarity(q_vector, doc_vectors.toarray()))
    f = 0
    elem_list = []
    for i in results[:10]:
        for elem in i[0]:
            elem_list.append(elem)
            f += 1

comp_description('This wine highlights how the power of Lake County’s Red Hills seamlessly compliments the elegance and aromatic freshness of the High Valley. Aromas of plum, allspice and clove develop into flavors of fresh dark cherry and cedar on the palate. The Red Hills’ fine tannins provide a smoothly textured palate sensation from start to finish. Fresh acidity from the High Valley culminates in a bright finish of cherry with a gentle note of French oak.')