text_html_output_1.png
from mlxtend.frequent_patterns import fpgrowth, association_rules  # MBA

import numpy as np

import os

import pandas as pd

def preDot(text):
    return text.rsplit('.', 1)[0]
np.random.seed(73)
pd.options.mode.chained_assignment = None
dataDict = {}
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        with ZipFile(os.path.join(dirname, filename), 'r') as zipf:
            unzipped_fn = preDot(filename)
            with zipf.open(unzipped_fn) as f:
                dataDict[preDot(unzipped_fn)] = pd.read_csv(f)
train_orders = dataDict['orders'][dataDict['orders']['eval_set'] == 'train'].drop('eval_set', axis=1)
prior_orders = dataDict['orders'][dataDict['orders']['eval_set'] == 'prior'].drop('eval_set', axis=1)
test_orders = dataDict['orders'][dataDict['orders']['eval_set'] == 'test'].drop('eval_set', axis=1)

small_train = dataDict['order_products__train'][['order_id', 'product_id']]
small_train_split = (small_train[:461543], small_train[461543:461543 * 2 - 1], small_train[461543 * 2 - 1:])

pivots = []
for df in small_train_split:
    pvt = ~df.pivot(index='order_id', columns='product_id', values='product_id').isna()
    pivots.append(pvt.astype(pd.SparseDtype(bool)))
del pvt
product_cols = sorted(small_train.product_id.unique())

for i in range(len(pivots)):
    pivots[i] = pivots[i].reindex(columns=product_cols, fill_value=False).astype(pd.SparseDtype(bool))
    pivots[i] = sparse.csr_matrix(pivots[i])
pivots = sparse.vstack(pivots)

truth_table = pd.DataFrame(pivots.todense(), index=small_train.order_id.unique(), columns=product_cols)

frequent_itemsets = fpgrowth(truth_table, min_support=5 / len(truth_table), use_colnames=True)

rules = association_rules(frequent_itemsets, metric='confidence', min_threshold=0.8)

print('μ number of consequents:', rules['consequents'].apply(len).mean())
rules