text_plain_output_1.png
from collections import Counter

from gensim.models import keyedvectors

import io

import numpy as np

import os

import pandas as pd

import torch

import torch.nn as nn

import torch.nn.functional as F

import torch.optim as optim

import torch.utils.data as Data

root_path = '/kaggle/input/test-train'
train_path = '/kaggle/input/test-train/train_clean.txt'
import pandas as pd
import io
with open('/kaggle/input/test-train/train_clean.txt', 'r') as f:
    train_text = f.read()
train_data = pd.read_csv(io.StringIO(train_text), sep='\t', names=['label', 'comment'])
train_data['comment'] = train_data['comment'].astype(str)

comments_len = train_data.iloc[:, 1].apply(lambda x: len(str(x).split()))
comments_len

comments_len = train_data.iloc[:, 1].apply(lambda x: len(str(x)))
comments_len
train_data['comments_len'] = comments_len

from collections import Counter
words = []
for i in range(len(train_data)):
    com = train_data['comment'][i].split()
    words = words + com
len(words)

Freq = 30
import os
with open(os.path.join('/kaggle/working/', 'word_freq.txt'), 'w', encoding='utf-8') as fout:
    for word, freq in Counter(words).most_common():
        if freq > Freq:
            fout.write(word + '\n')

with open(os.path.join('/kaggle/working/', 'word_freq.txt'), encoding='utf-8') as fin:
    vocab = [i.strip() for i in fin]
vocab = set(vocab)
word2idx = {i: index for index, i in enumerate(vocab)}
idx2word = {index: i for index, i in enumerate(vocab)}
vocab_size = len(vocab)
len(vocab)

pad_id = 923

sequence_length = 62
def tokenizer():
    inputs = []
    sentence_char = [str(i).split() for i in train_data['comment']]
    for index, i in enumerate(sentence_char):
        temp = [word2idx.get(j, pad_id) for j in i]
        if len(i) < sequence_length:
            for _ in range(sequence_length - len(i)):
                temp.append(pad_id)
        else:
            temp = temp[:sequence_length]
        inputs.append(temp)
    return inputs
data_input = tokenizer()

import torch
import torch.nn as nn
import torch.utils.data as Data
import torch.optim as optim
import torch.nn.functional as F
import numpy as np
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
Embedding_size = 100
Batch_Size = 32
Kernel = 3
Filter_num = 20
Epoch = 100
Dropout = 0.5
Learning_rate = 0.001
class TextCNNDataSet(Data.Dataset):
    def __init__(self, data_inputs, data_targets):
        self.inputs = torch.LongTensor(data_inputs)
        self.label = torch.LongTensor(data_targets)
    def __getitem__(self, index):
        return (self.inputs[index], self.label[index])
    def __len__(self):
        return len(self.inputs)
TextCNNDataSet = TextCNNDataSet(data_input, list(train_data['label']))
train_size = int(len(data_input) * 0.8)
test_size = int(len(data_input) * 0.15)
val_size = len(data_input) - train_size - test_size
train_dataset, val_dataset, test_dataset = torch.utils.data.random_split(TextCNNDataSet, [train_size, val_size, test_size])
TrainDataLoader = Data.DataLoader(train_dataset, batch_size=Batch_Size, shuffle=True)
TestDataLoader = Data.DataLoader(test_dataset, batch_size=Batch_Size, shuffle=True)

import gensim
from gensim.models import keyedvectors
w2v = keyedvectors.load_word2vec_format('/kaggle/input/tencent/tencent-ailab-embedding-zh-d100-v0.2.0-s/tencent-ailab-embedding-zh-d100-v0.2.0-s.txt', binary=False)

def word2vec(x):
    x2v = np.ones((len(x), x.shape[1], Embedding_size))
    for i in range(len(x)):
        try:
            x2v[i] = w2v[[idx2word[j.item()] for j in x[i]]]
        except Exception as e:
            x2v[i] = np.random.randn(62, 100)
    return torch.tensor(x2v, dtype=torch.float32)
num_classs = 2
class TextCNN(nn.Module):
    def __init__(self):
        super(TextCNN, self).__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_size)
        out_channel = Filter_num
        self.conv = nn.Sequential(nn.Conv2d(1, out_channel, (2, Embedding_size)), nn.ReLU(), nn.MaxPool2d((sequence_length - 1, 1)))
        self.dropout = nn.Dropout(Dropout)
        self.fc = nn.Linear(out_channel, num_classs)
    def forward(self, X):
        batch_size = X.shape[0]
        X = self.embedding(X)
        X = X.unsqueeze(1)
        conved = self.conv(X)
        conved = self.dropout(conved)
        flatten = conved.view(batch_size, -1)
        output = self.fc(flatten)
        return F.log_softmax(output)

vocab_size = 2000000
embedding_size = 100
model = TextCNN().to(device)
optimizer = optim.Adam(model.parameters(), lr=Learning_rate)
def binary_acc(pred, y):
    """
    计算模型的准确率
    :param pred: 预测值
    :param y: 实际真实值
    :return: 返回准确率
    """
    correct = torch.eq(pred, y).float()
    acc = correct.sum() / len(correct)
    return acc.item()
def train():
    avg_acc = []
    model.train()
    for index, (batch_x, batch_y) in enumerate(TrainDataLoader):
        batch_x, batch_y = (batch_x.to(device), batch_y.to(device))
        batch_x = batch_x.long()
        pred = model(batch_x)
        loss = F.nll_loss(pred, batch_y)
        acc = binary_acc(torch.max(pred, dim=1)[1], batch_y)
        avg_acc.append(acc)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    avg_acc = np.array(avg_acc).mean()
    return avg_acc
model_train_acc, model_test_acc = ([], [])
for epoch in range(Epoch):
    train_acc = train()
    print('epoch = {}, 训练准确率={}'.format(epoch + 1, train_acc))
    model_train_acc.append(train_acc)