text_html_output_1.png text_plain_output_1.png
from mlxtend.frequent_patterns import fpgrowth, association_rules  # MBA

import matplotlib.pyplot as plt

import numpy as np

import os

import pandas as pd

def preDot(text):
    return text.rsplit('.', 1)[0]
np.random.seed(73)
pd.options.mode.chained_assignment = None
dataDict = {}
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        with ZipFile(os.path.join(dirname, filename), 'r') as zipf:
            unzipped_fn = preDot(filename)
            with zipf.open(unzipped_fn) as f:
                dataDict[preDot(unzipped_fn)] = pd.read_csv(f)
train_orders = dataDict['orders'][dataDict['orders']['eval_set'] == 'train'].drop('eval_set', axis=1)
prior_orders = dataDict['orders'][dataDict['orders']['eval_set'] == 'prior'].drop('eval_set', axis=1)
test_orders = dataDict['orders'][dataDict['orders']['eval_set'] == 'test'].drop('eval_set', axis=1)

small_train = dataDict['order_products__train'][['order_id', 'product_id']]
small_train_split = (small_train[:461543], small_train[461543:461543 * 2 - 1], small_train[461543 * 2 - 1:])

pivots = []
for df in small_train_split:
    pvt = ~df.pivot(index='order_id', columns='product_id', values='product_id').isna()
    pivots.append(pvt.astype(pd.SparseDtype(bool)))
del pvt
product_cols = sorted(small_train.product_id.unique())

for i in range(len(pivots)):
    pivots[i] = pivots[i].reindex(columns=product_cols, fill_value=False).astype(pd.SparseDtype(bool))
    pivots[i] = sparse.csr_matrix(pivots[i])
pivots = sparse.vstack(pivots)

truth_table = pd.DataFrame(pivots.todense(), index=small_train.order_id.unique(), columns=product_cols)

frequent_itemsets = fpgrowth(truth_table, min_support=5 / len(truth_table), use_colnames=True)

rules = association_rules(frequent_itemsets, metric='confidence', min_threshold=0.8)

rules_ante_cons = rules[['antecedents', 'consequents']]

baskets = small_train.groupby('order_id')['product_id'].apply(frozenset)
baskets.name = 'basket'

recommendations = train_orders.join(baskets, on='order_id')
recommendations['recommendations'] = [frozenset() for _ in range(len(recommendations))]

for idx, antecedent in enumerate(rules_ante_cons['antecedents']):
    lookup = (antecedent <= recommendations.basket, 'recommendations')
    recommendations.loc[lookup] = recommendations.loc[lookup].apply(frozenset.union, args=(rules_ante_cons.loc[idx, 'consequents'],))
recommendations.loc[:, 'recommendations'] = recommendations.recommendations - recommendations.basket

non_empty_recs = recommendations[recommendations.recommendations.apply(bool)]
def map_products(codes):
    if isinstance(codes, pd.Series):
        return codes.apply(map_products)
    return frozenset(map(products.get, codes))
products = dataDict['products']
products = products.set_index('product_id')['product_name'].to_dict()
non_empty_recs.loc[:, ['basket', 'recommendations']] = non_empty_recs[['basket', 'recommendations']].apply(map_products)

def mba_diagram(sample_basket, sample_recommendation):
    import matplotlib.pyplot as plt

    def get_text_box_coords(txt):
        we = plt.Text.get_window_extent(txt, renderer=fig.canvas.get_renderer())
        return ax.transAxes.inverted().transform(we)
    def get_rightmost_vmid(box):
        return box[1][0], (box[0][1] + box[1][1]) / 2

    fig, ax = plt.subplots(figsize=(20,10))
    title = ax.set_title("An illustration of a recommendation system for a sample customer basket\n(basket ← suggestion)", fontsize=18)
    ax.axis('off')
    basket_txt = ax.text(.05, .95, sample_basket, ha='left', va='top', wrap=True,size=12,
                  bbox=dict(boxstyle='round,pad=1', fc='w', ec='lightblue'))

    basket_rightmost, basket_vmid = get_rightmost_vmid(get_text_box_coords(basket_txt))

    arrow_txt = ax.text(
        basket_rightmost*1.4, basket_vmid, "Add", ha="center", va="center", size=35,
        bbox=dict(boxstyle="larrow,pad=0.6", fc="lightgreen", ec="g", lw=2))
    arrow_rightmost, arrow_vmid = get_rightmost_vmid(get_text_box_coords(arrow_txt))

    recommendation_txt = ax.text(arrow_rightmost * 1.14, arrow_vmid, sample_recommendation, ha='left', va='top', wrap=True, fontsize=25,
                  bbox=dict(boxstyle='round,pad=1', fc='w', ec='r'))
    recommendation_txt_pos = recommendation_txt.get_position()
    recommendation_txt.set_position((
        recommendation_txt_pos[0],
        recommendation_txt_pos[1] + (get_text_box_coords(recommendation_txt)[1][1]-get_text_box_coords(recommendation_txt)[0][1]) / 2
    ))

sample_index = np.random.randint(len(non_empty_recs))
sample_basket = '\n'.join(non_empty_recs.iloc[sample_index].loc['basket'])
sample_recommendation = '\n'.join(non_empty_recs.iloc[sample_index].loc['recommendations'])
mba_diagram(sample_basket, sample_recommendation)