image_output_1.png
from tqdm import tqdm

from vncorenlp import VnCoreNLP

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

import string

import string
class Config:
    max_sequence_length = -1
    word_vec = -1
    punctuations = f'[{string.punctuation}\\d\n]'
    seed = 44

data_path = '../input/vietnamese-text-classification-dataset/train.csv'
df = pd.read_csv(data_path, header=None, names=['label', 'text'])

with open('../input/vietnamese-stopwords/vietnamese-stopwords.txt', 'r') as f:
    stop_words = f.read().split('\n')

from vncorenlp import VnCoreNLP
vncorenlp = VnCoreNLP('../input/vncorenlp/VnCoreNLP-1.1.1.jar', annotators='wseg,pos,ner,parse', max_heap_size='-Xmx2g')

vncorenlp_example = vncorenlp.tokenize('nắng chiếu lung linh trên hoa vàng')

import fasttext.util
cbow_pretrain = fasttext.load_model('../input/cbow-model/cc.vi.300.bin')

import re
def format_sentence(sentence):
    sentence = re.sub(f'[{string.punctuation}\\d\n]', '', sentence)
    tokens = vncorenlp.tokenize(sentence.lower())[0]
    sentence = ' '.join(tokens)
    embedding_sentence = cbow_pretrain.get_sentence_vector(sentence)
    return embedding_sentence

"""
lấy phần text trong dataframe:
df[tên cột]: pandas.core.series.Series
df[tên cột].values: numpy.ndarray
"""
corpus = df['text'].values.tolist()

corpus = list(map(format_sentence, corpus))

X = np.array(corpus)
Config.word_vec = X.shape[1]

max_len_sent = max([len(sent) for sent in corpus])
Config.max_sequence_length = max_len_sent

labels = df['label'].values

class GenData:
    def __init__(self, stop_words, num_sentence, list_sentence):
        self.stop_words = stop_words
        self.num_sentence = num_sentence
        self.list_sentence = list_sentence
    def __gen_new_item__(self):
        num_sentece_gen = len(self.list_sentence) - self.num_sentence
        np.random.seed(Config.seed)
        np.random.shuffle(self.stop_words)
        stop_words_gen = self.stop_words[:num_sentence_gen]
        np.random.shuffle(self.list_sentence)
        sentence_gen = self.list_sentence[:num_sentence_gen]
        new_sentences = [[stop_words_gen[idx]] + sentence_gen[idx] for idx in range(num_sentencce_gen)]
        return new_sentences

Y = []
X = []
for idx in tqdm(range(len(labels))):
    if labels[idx] == 0 or labels[idx] == 2:
        Y.append(labels[idx])
        X.append(corpus[idx])

X = np.array(X)
Y = np.array(Y)
(X.shape, Y.shape)