text_plain_output_1.png
import gc

import numpy as np

import pandas as pd

import pickle

def correlation_score(y_true, y_pred):
    """Scores the predictions according to the competition rules. 
    It is assumed that the predictions are not constant.
    Returns the average of each sample's Pearson correlation coefficient"""
    if type(y_true) == pd.DataFrame:
        y_true = y_true.values
    if type(y_pred) == pd.DataFrame:
        y_pred = y_pred.values
    if y_true.shape != y_pred.shape:
        raise ValueError('Shapes are different.')
    corrsum = 0
    for i in range(len(y_true)):
        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]
    return corrsum / len(y_true)

def load(fname):
    with open(fname, 'rb') as f:
        return pickle.load(f)

pca = load('../input/fork-of-msci-multiome-randomsampling-sp-6b182b/pca.pkl')
pca2 = load('../input/fork-of-msci-multiome-randomsampling-sp-6b182b/pca2.pkl')

n = 1
test_len = multi_test_x.shape[0]
d = test_len // n
x = []
for i in range(n):
    x.append(multi_test_x[i * d:i * d + d])
del multi_test_x
gc.collect()

index = 2
from catboost import CatBoostRegressor

preds = np.zeros((test_len, 23418), dtype='float16')
for i, xx in enumerate(x):
    for ind in range(index):
        with open(f'../input/fork-of-msci-multiome-randomsampling-sp-6b182b/model{ind:02}.pkl', 'rb') as file:
            model = pickle.load(file)
        preds[i * d:i * d + d, :] += model.predict(xx) @ pca2.components_ / index
        gc.collect()
    del xx
gc.collect()

del x
gc.collect()

del eval_ids_cell_num, eval_ids_gene_num, valid_multi_rows, eval_ids, test_index, y_columns
gc.collect()