text_plain_output_5.png application_vnd.jupyter.stderr_output_4.png text_plain_output_6.png application_vnd.jupyter.stderr_output_3.png text_plain_output_2.png text_plain_output_1.png
import gc

import glob

import multiprocessing

import numpy as np

import os

import random

import torch

from tqdm.auto import tqdm
import os
import sys
import random
import numpy as np
import pandas as pd
import glob
import gc
gc.enable()
from joblib import Parallel, delayed
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from pytorch_lightning import LightningModule, LightningDataModule
from pytorch_lightning import Trainer
import multiprocessing
from transformers import AutoTokenizer, AutoModel, AutoConfig, AutoModelForTokenClassification
from transformers.models.deberta_v2.tokenization_deberta_v2_fast import DebertaV2TokenizerFast
def seed_everything(seed=42):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = True
    os.environ['PYTHONHASHSEED'] = str(seed)
target_id_map = {'O': 0, 'B-Lead': 1, 'I-Lead': 2, 'B-Position': 3, 'I-Position': 4, 'B-Claim': 5, 'I-Claim': 6, 'B-Counterclaim': 7, 'I-Counterclaim': 8, 'B-Rebuttal': 9, 'I-Rebuttal': 10, 'B-Evidence': 11, 'I-Evidence': 12, 'B-Concluding Statement': 13, 'I-Concluding Statement': 14, 'PAD': -100}
'\ntarget_id_map2 = {\n    "B-Lead": 0,\n    "I-Lead": 1,\n    "B-Position": 2,\n    "I-Position": 3,\n    "B-Evidence": 4,\n    "I-Evidence": 5,\n    "B-Claim": 6,\n    "I-Claim": 7,\n    "B-Concluding Statement": 8,\n    "I-Concluding Statement": 9,\n    "B-Counterclaim": 10,\n    "I-Counterclaim": 11,\n    "B-Rebuttal": 12,\n    "I-Rebuttal": 13,\n    "O": 14,\n    "PAD": -100,\n}\n'
length_threshold = {'Lead': 9, 'Position': 5, 'Claim': 3, 'Counterclaim': 6, 'Rebuttal': 4, 'Evidence': 14, 'Concluding Statement': 11}
probability_threshold = {'Lead': 0.6, 'Position': 0.6, 'Claim': 0.6, 'Counterclaim': 0.6, 'Rebuttal': 0.6, 'Evidence': 0.6, 'Concluding Statement': 0.6}
id_target_map = {v: k for k, v in target_id_map.items()}
seed_everything(2022)
os.environ['TOKENIZERS_PARALLELISM'] = 'false'
def process(func):
    def worker(func, q):
        q.put(func())
    out = None
    q = multiprocessing.Queue()
    p = multiprocessing.Process(target=worker, args=(func, q))
    p.start()
    out = q.get()
    p.join()
    return out

DEBUG = False
if DEBUG:
    text_dir = '../input/feedback-prize-2021/train'
    valid_id = [f.split('/')[-1][:-4] for f in glob.glob(text_dir + '/*.txt')]
    valid_id = sorted(valid_id)[0:10000]
    num_valid = len(valid_id)
    print('len(valid_id)', len(valid_id))
else:
    text_dir = '../input/feedback-prize-2021/test'
    valid_id = [f.split('/')[-1][:-4] for f in glob.glob(text_dir + '/*.txt')]
    valid_id = sorted(valid_id)
    num_valid = len(valid_id)
    print('len(valid_id)', len(valid_id))
size = [os.path.getsize(text_dir + '/%s.txt' % id) for id in valid_id]
valid_id = [id for id, s in sorted(zip(valid_id, size), key=lambda pair: -pair[1])]
del size
gc.collect()
print('len(valid_id)', len(valid_id))