text_html_output_2.png text_html_output_1.png
from google.colab import drive

from transformers import AutoTokenizer, AutoConfig, AutoModel

import numpy as np

import pandas as pd

import torch

class Config(object):
    competition_name = 'LECR'
    seed = 2022
    seeds = [1, 11, 111, 1111, 11111, 2, 22, 222, 2222]
    env = 'kaggle'
    ver = 'v17g'
    if env == 'colab':
        from google.colab import drive
        drive.mount('/content/drive')
    mode = 'infer'
    use_tqdm = True
    use_log = False
    debug = False
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    if env in ['colab', 'vastai']:
        backbone = 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2'
    elif env == 'kaggle':
        backbone = '/kaggle/input/paraphrasemultilingualmpnetbasev2'
    tokenizer = AutoTokenizer.from_pretrained(backbone)
    config = AutoConfig.from_pretrained(backbone)
    sep_token = '[LECR]'
    sep_token_id = tokenizer.vocab_size + 1
    special_tokens_dict = {'additional_special_tokens': [sep_token]}
    tokenizer.add_special_tokens(special_tokens_dict)
    embedding_model = 'v17a'
    nfolds = 5
    languages_map = {'ar': 0, 'as': 1, 'bg': 2, 'bn': 3, 'en': 4, 'es': 5, 'fil': 6, 'fr': 7, 'gu': 8, 'hi': 9, 'it': 10, 'km': 11, 'kn': 12, 'mr': 13, 'mul': 14, 'my': 15, 'or': 16, 'pl': 17, 'pnb': 18, 'pt': 19, 'ru': 20, 'sw': 21, 'swa': 22, 'ta': 23, 'te': 24, 'tr': 25, 'ur': 26, 'zh': 27}
    max_len = 128
    batch_size = 128 if not debug else 4
    num_workers = os.cpu_count()
    apex = True
    thres = {'cosine': None, 'num_k': 2000}
    if env == 'colab':
        comp_data_dir = f'/content/drive/My Drive/Kaggle competitions/{competition_name}/comp_data'
        ext_data_dir = f'/content/drive/My Drive/Kaggle competitions/{competition_name}/ext_data'
        model_dir = f'/content/drive/My Drive/Kaggle competitions/{competition_name}/model'
        embedding_model_dir = f'{model_dir}/{embedding_model[:-1]}/{embedding_model[-1]}'
        os.makedirs(os.path.join(model_dir, ver[:-1], ver[-1]), exist_ok=True)
    elif env == 'kaggle':
        comp_data_dir = '/kaggle/input/learning-equality-curriculum-recommendations'
        ext_data_dir = '/kaggle/input/lecrext-data'
        embedding_model_dir = f'/kaggle/input/lecr{embedding_model}'
        model_dir = f'/kaggle/input/{competition_name.lower()}{ver.lower()}'
    elif env == 'vastai':
        comp_data_dir = 'data'
        ext_data_dir = 'ext_data'
        model_dir = f'model'
        embedding_model_dir = f'{model_dir}/{embedding_model[:-1]}/{embedding_model[-1]}'
        os.makedirs(os.path.join(model_dir, ver[:-1], ver[-1]), exist_ok=True)
cfg = Config()

def set_random_seed(seed, use_cuda=True):
    np.random.seed(seed)
    torch.manual_seed(seed)
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    if use_cuda:
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
set_random_seed(cfg.seed)

def print_log(cfg, message):
    pass

content_df = pd.read_csv(os.path.join(cfg.comp_data_dir, 'content.csv'))
topic_df = pd.read_csv(os.path.join(cfg.comp_data_dir, 'topics.csv'))

def process_data(cfg, df, is_content=False):
    df['title'] = df['title'].fillna(' ')
    df['description'] = df['description'].fillna(' ')
    if is_content:
        df['text'] = df['text'].fillna(' ')
    df['encoded_language'] = df['language'].map(cfg.languages_map)
    return df
topic_df = process_data(cfg, topic_df)
content_df = process_data(cfg, content_df, is_content=True)
display(topic_df)
display(content_df)